POI-Karte: Offline-Import aus OSM statt Live-Overpass-Scan (Build 4)
- osm.py: Live-Scanner deaktiviert — /pois liest nur noch aus DB, /analyze ist No-Op. Behebt wiederholte OSM-Banns (Tile-Load + Scanning). - tools/osm-extract: Extraktion (pyosmium) + Loader (schützt user_edited) + Docker-Refresh-Job mit osmium-tags-filter-Vorstufe (RAM-schonend). - docker-compose.osm.yml: Refresh-Service (mem_limit 4g), monatlich via DSM-Aufgabenplaner.
This commit is contained in:
parent
214543559c
commit
4bc7454258
9 changed files with 457 additions and 26 deletions
71
tools/osm-extract/load_into_prod.py
Normal file
71
tools/osm-extract/load_into_prod.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Lädt die offline extrahierten POIs (dach.sqlite) in die Produktiv-DB.
|
||||
|
||||
Semantik des Monats-Refresh:
|
||||
* Alle nicht-editierten OSM-POIs (user_edited=0) werden ersetzt → POIs, die
|
||||
aus OSM verschwunden sind, fallen sauber raus.
|
||||
* Von Nutzern korrigierte POIs (user_edited=1, via Moderation) bleiben
|
||||
UNANGETASTET — INSERT OR IGNORE überspringt sie bei Kollision.
|
||||
* Community-Marker (Tabelle user_map_pois) sind separat und werden nie
|
||||
berührt.
|
||||
|
||||
Läuft in EINER Transaktion. Bei Fehler bleibt die alte DB unverändert.
|
||||
|
||||
Aufruf:
|
||||
python3 load_into_prod.py <extract.sqlite> <ziel/banyaro.db>
|
||||
"""
|
||||
import sys
|
||||
import sqlite3
|
||||
|
||||
COLS = "osm_id, type, lat, lon, name, opening_hours, phone, website, user_edited, cached_at"
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print(__doc__)
|
||||
sys.exit(1)
|
||||
extract_path, prod_path = sys.argv[1], sys.argv[2]
|
||||
|
||||
# timeout/busy_timeout: die App schreibt evtl. parallel — auf Lock warten,
|
||||
# statt sofort zu scheitern. Der Load läuft in EINER Transaktion.
|
||||
conn = sqlite3.connect(prod_path, timeout=120)
|
||||
conn.execute("PRAGMA busy_timeout=120000")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute(f"ATTACH DATABASE ? AS ext", (extract_path,))
|
||||
|
||||
before = conn.execute("SELECT COUNT(*) FROM osm_pois").fetchone()[0]
|
||||
edited = conn.execute("SELECT COUNT(*) FROM osm_pois WHERE user_edited=1").fetchone()[0]
|
||||
incoming = conn.execute("SELECT COUNT(*) FROM ext.osm_pois").fetchone()[0]
|
||||
|
||||
try:
|
||||
conn.execute("BEGIN")
|
||||
# 1) nicht-editierte OSM-POIs verwerfen (editierte bleiben stehen)
|
||||
conn.execute("DELETE FROM osm_pois WHERE user_edited=0")
|
||||
# 2) frische Extraktion einspielen; editierte Survivor nicht überschreiben
|
||||
conn.execute(
|
||||
f"INSERT OR IGNORE INTO osm_pois ({COLS}) "
|
||||
f"SELECT {COLS} FROM ext.osm_pois"
|
||||
)
|
||||
conn.execute("COMMIT")
|
||||
except Exception:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
after = conn.execute("SELECT COUNT(*) FROM osm_pois").fetchone()[0]
|
||||
print(f"Vorher: {before:>10,}")
|
||||
print(f"davon user_edited:{edited:>10,} (geschützt)")
|
||||
print(f"Eingespielt: {incoming:>10,}")
|
||||
print(f"Nachher: {after:>10,}")
|
||||
print("\nPro Typ (nachher):")
|
||||
for ty, cnt in conn.execute(
|
||||
"SELECT type, COUNT(*) FROM osm_pois GROUP BY type ORDER BY 2 DESC"
|
||||
):
|
||||
print(f" {ty:16s} {cnt:>9,}")
|
||||
|
||||
conn.execute("DETACH DATABASE ext")
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue