POI-Karte: Offline-Import aus OSM statt Live-Overpass-Scan (Build 4)
- osm.py: Live-Scanner deaktiviert — /pois liest nur noch aus DB, /analyze ist No-Op. Behebt wiederholte OSM-Banns (Tile-Load + Scanning). - tools/osm-extract: Extraktion (pyosmium) + Loader (schützt user_edited) + Docker-Refresh-Job mit osmium-tags-filter-Vorstufe (RAM-schonend). - docker-compose.osm.yml: Refresh-Service (mem_limit 4g), monatlich via DSM-Aufgabenplaner.
This commit is contained in:
parent
214543559c
commit
4bc7454258
9 changed files with 457 additions and 26 deletions
|
|
@ -1,6 +1,11 @@
|
||||||
"""
|
"""
|
||||||
BAN YARO — OSM/Overpass POI-Cache + Community-Pins
|
BAN YARO — OSM POI-Daten + Community-Pins
|
||||||
Cacht OSM-Daten lokal, erlaubt Nutzern eigene Marker und Meldungen.
|
Liest OSM-POIs aus der lokalen Tabelle osm_pois (monatlicher Offline-Import,
|
||||||
|
tools/osm-extract/), erlaubt Nutzern eigene Marker und Meldungen.
|
||||||
|
|
||||||
|
Build 4: Live-Scannen gegen overpass-api.de ist DEAKTIVIERT (war Bann-Quelle).
|
||||||
|
Die Overpass-Hilfsfunktionen unten sind ungenutzt und können später entfernt
|
||||||
|
werden. /geocode nutzt weiterhin Nominatim für die Adresssuche (geringe Last).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import math
|
import math
|
||||||
|
|
@ -191,17 +196,9 @@ async def get_pois(
|
||||||
fetched_fresh = False
|
fetched_fresh = False
|
||||||
|
|
||||||
if type in OSM_QUERIES:
|
if type in OSM_QUERIES:
|
||||||
tiles = _covering_tiles(south, west, north, east, CACHE_ZOOM)
|
# Scanner deaktiviert (Build 4): keine Live-Overpass-Abfragen mehr.
|
||||||
stale = _stale_tiles(type, tiles)
|
# POIs stammen aus dem monatlichen Offline-Import in die Tabelle
|
||||||
|
# osm_pois (tools/osm-extract/). Hier wird nur noch daraus gelesen.
|
||||||
if stale and not fast:
|
|
||||||
async def _bg_fetch(poi_type, stale_tiles):
|
|
||||||
for (x, y) in stale_tiles:
|
|
||||||
await _fetch_and_store_tile(poi_type, x, y)
|
|
||||||
task = asyncio.create_task(_bg_fetch(type, stale))
|
|
||||||
_bg_tasks.add(task)
|
|
||||||
task.add_done_callback(_bg_tasks.discard)
|
|
||||||
|
|
||||||
with db() as conn:
|
with db() as conn:
|
||||||
reported = {
|
reported = {
|
||||||
row[0] for row in conn.execute(
|
row[0] for row in conn.execute(
|
||||||
|
|
@ -364,24 +361,17 @@ async def report_poi(body: ReportIn, user = Depends(get_current_user)):
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@router.post('/analyze')
|
@router.post('/analyze')
|
||||||
async def analyze_region(
|
async def analyze_region(
|
||||||
background_tasks: BackgroundTasks,
|
|
||||||
south: float = Query(...),
|
south: float = Query(...),
|
||||||
west: float = Query(...),
|
west: float = Query(...),
|
||||||
north: float = Query(...),
|
north: float = Query(...),
|
||||||
east: float = Query(...),
|
east: float = Query(...),
|
||||||
):
|
):
|
||||||
tiles = _covering_tiles(south, west, north, east, CACHE_ZOOM)
|
# Scanner deaktiviert (Build 4): kein Live-Overpass-Warmup mehr. POIs
|
||||||
|
# kommen aus dem monatlichen Offline-Import (tools/osm-extract/). Endpoint
|
||||||
async def _warmup():
|
# bleibt als No-Op erhalten, damit bestehende Frontends nicht 404 laufen.
|
||||||
tasks = [
|
return {'status': 'offline-import',
|
||||||
_fetch_and_store_tile(pt, x, y)
|
'message': 'POIs werden monatlich offline importiert — kein Live-Scan nötig.',
|
||||||
for pt in OSM_QUERIES
|
'types': list(OSM_QUERIES.keys())}
|
||||||
for (x, y) in _stale_tiles(pt, tiles)
|
|
||||||
]
|
|
||||||
await asyncio.gather(*tasks)
|
|
||||||
|
|
||||||
background_tasks.add_task(_warmup)
|
|
||||||
return {'status': 'gestartet', 'tiles': len(tiles), 'types': list(OSM_QUERIES.keys())}
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
|
||||||
16
docker-compose.osm.yml
Normal file
16
docker-compose.osm.yml
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Monatlicher OSM-POI-Refresh (Build 4) — NICHT Teil des Default-Stacks.
|
||||||
|
# Wird manuell oder vom DSM-Aufgabenplaner getriggert:
|
||||||
|
# docker compose -f docker-compose.osm.yml run --rm osm-refresh
|
||||||
|
# Schreibt in dieselbe SQLite-DB wie der App-Container (./data:/data).
|
||||||
|
services:
|
||||||
|
osm-refresh:
|
||||||
|
build: ./tools/osm-extract
|
||||||
|
image: banyaro-osm-refresh
|
||||||
|
container_name: banyaro-osm-refresh
|
||||||
|
mem_limit: 4g # Schutzschranke gegen die anderen Container
|
||||||
|
volumes:
|
||||||
|
- ./data:/data # gleiche DB wie die App (/data/banyaro.db)
|
||||||
|
environment:
|
||||||
|
- DB_PATH=/data/banyaro.db
|
||||||
|
# - COUNTRIES=switzerland austria germany # bei Bedarf überschreiben
|
||||||
|
restart: "no"
|
||||||
5
tools/osm-extract/.gitignore
vendored
Normal file
5
tools/osm-extract/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
# Große/temporäre Datendateien — nie committen
|
||||||
|
*.osm.pbf
|
||||||
|
*.sqlite
|
||||||
|
*.db
|
||||||
|
*.log
|
||||||
14
tools/osm-extract/Dockerfile
Normal file
14
tools/osm-extract/Dockerfile
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
FROM python:3.12-slim-bookworm
|
||||||
|
|
||||||
|
# osmium-tool = RAM-schonende tags-filter-Vorstufe (C++, streaming),
|
||||||
|
# pyosmium = Extraktion + Schwerpunktberechnung.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
osmium-tool curl ca-certificates \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
RUN pip install --no-cache-dir osmium
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY extract_osm_pois.py load_into_prod.py refresh.sh /app/
|
||||||
|
RUN chmod +x /app/refresh.sh
|
||||||
|
|
||||||
|
ENTRYPOINT ["/app/refresh.sh"]
|
||||||
74
tools/osm-extract/INSTALL.md
Normal file
74
tools/osm-extract/INSTALL.md
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
# Build 4 — POI-Offline-Umstellung auf der DiskStation (DSM-Upload)
|
||||||
|
|
||||||
|
Ziel: Live-Overpass-Scanner abschalten + die 1,45 Mio DACH-POIs in die
|
||||||
|
Produktiv-DB migrieren. Ohne 5,7-GB-Download (die fertige `dach.sqlite` wird
|
||||||
|
mit hochgeladen).
|
||||||
|
|
||||||
|
App-Verzeichnis auf der DS: **`/volume1/docker/banyaro/`** (im File Station:
|
||||||
|
`docker` → `banyaro`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Schritt 1 — Code hochladen (File Station)
|
||||||
|
|
||||||
|
1. `build4-osm-code.zip` in den Ordner **`docker/banyaro`** hochladen.
|
||||||
|
2. Rechtsklick → **Entpacken → Hierher entpacken**. Das überschreibt
|
||||||
|
`backend/routes/osm.py` (Scanner aus) und legt `tools/osm-extract/` +
|
||||||
|
`docker-compose.osm.yml` an. Andere Dateien bleiben unberührt.
|
||||||
|
|
||||||
|
## Schritt 2 — Vorbereiteten POI-Extrakt hochladen
|
||||||
|
|
||||||
|
3. `dach.sqlite` (181 MB) in den Ordner **`docker/banyaro/data`** hochladen.
|
||||||
|
(Liegt dann im Container als `/data/dach.sqlite`.)
|
||||||
|
|
||||||
|
## Schritt 3 — Migration + Deploy (SSH-Terminal: `ssh ds`)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd /volume1/docker/banyaro
|
||||||
|
|
||||||
|
# Refresh-Image bauen (einmalig)
|
||||||
|
docker compose -f docker-compose.osm.yml build
|
||||||
|
|
||||||
|
# MIGRATION: lädt dach.sqlite in die Produktiv-DB (Backup wird vorher angelegt,
|
||||||
|
# user_edited-POIs + Community-Marker bleiben geschützt)
|
||||||
|
docker compose -f docker-compose.osm.yml run --rm \
|
||||||
|
-e PREBUILT_SQLITE=/data/dach.sqlite osm-refresh
|
||||||
|
|
||||||
|
# DEPLOY: baut die App neu → scanner-lose osm.py geht live
|
||||||
|
docker compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
> Reihenfolge bewusst: erst Daten laden, dann App neu bauen → kein Fenster mit
|
||||||
|
> leerer Karte.
|
||||||
|
|
||||||
|
## Schritt 4 — Prüfen
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose -f docker-compose.osm.yml run --rm --entrypoint python3 \
|
||||||
|
osm-refresh -c "import sqlite3; c=sqlite3.connect('/data/banyaro.db'); \
|
||||||
|
print('osm_pois:', c.execute('select count(*) from osm_pois').fetchone()[0]); \
|
||||||
|
print(c.execute('select type,count(*) from osm_pois group by type order by 2 desc').fetchall())"
|
||||||
|
```
|
||||||
|
Erwartung: ~1.452.675 POIs (bank ~1,0 Mio, restaurant ~60k …). In der App die
|
||||||
|
Karte öffnen → Marker laden ohne Overpass.
|
||||||
|
|
||||||
|
## Aufräumen (optional)
|
||||||
|
|
||||||
|
`dach.sqlite` aus `docker/banyaro/data` kann nach erfolgreicher Migration weg.
|
||||||
|
|
||||||
|
## Monatlicher Auto-Refresh (DSM-Aufgabenplaner)
|
||||||
|
|
||||||
|
Systemsteuerung → Aufgabenplaner → Erstellen → Geplante Aufgabe →
|
||||||
|
Benutzerdefiniertes Skript. Benutzer `root`, monatlich z. B. 1. um 04:00:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd /volume1/docker/banyaro && \
|
||||||
|
docker compose -f docker-compose.osm.yml run --rm osm-refresh \
|
||||||
|
>> /volume1/docker/banyaro/data/osm-refresh.log 2>&1
|
||||||
|
```
|
||||||
|
(ohne `PREBUILT_SQLITE` → holt frisch von Geofabrik, ~1–2 GB RAM dank tags-filter)
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
Vor jedem Lauf entsteht `data/banyaro.pre-osm-JJJJMMTT.db`. Im Notfall:
|
||||||
|
App stoppen, diese Datei auf `banyaro.db` zurückkopieren, App starten.
|
||||||
55
tools/osm-extract/README.md
Normal file
55
tools/osm-extract/README.md
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
# OSM-POI Offline-Refresh (Build 4)
|
||||||
|
|
||||||
|
Ersetzt das Live-Scannen gegen `overpass-api.de` (war wiederholt OSM-Bann-Quelle)
|
||||||
|
durch einen **monatlichen Offline-Batch**: POIs werden aus den Geofabrik-OSM-Daten
|
||||||
|
extrahiert und in die Produktiv-DB geladen. Danach keine OSM-Live-Last mehr.
|
||||||
|
|
||||||
|
## Bestandteile
|
||||||
|
|
||||||
|
| Datei | Zweck |
|
||||||
|
|---|---|
|
||||||
|
| `extract_osm_pois.py` | pbf → `osm_pois`-Schema (pyosmium). Kanonisches Kategorie→Tag-Mapping. |
|
||||||
|
| `load_into_prod.py` | Extrakt → Produktiv-DB. Schützt `user_edited=1`, ersetzt den Rest. |
|
||||||
|
| `refresh.sh` | Orchestrierung im Container: download → tags-filter → extract → load. |
|
||||||
|
| `Dockerfile` | Image mit osmium-tool + pyosmium. |
|
||||||
|
| `../../docker-compose.osm.yml` | Eigener Service, **nicht** im Default-Stack. |
|
||||||
|
|
||||||
|
## Was es tut
|
||||||
|
|
||||||
|
- Lädt CH/AT/DE von Geofabrik (~5,7 GB), dampft sie mit `osmium tags-filter`
|
||||||
|
(streaming, <500 MB RAM) auf die relevanten Objekte ein, extrahiert die 9
|
||||||
|
Ban-Yaro-Kategorien und lädt sie in `/data/banyaro.db`.
|
||||||
|
- **Sicherheitskopie** der DB vor jedem Lauf (letzte 3 bleiben).
|
||||||
|
- `user_edited=1`-POIs und `user_map_pois` (Community-Marker) bleiben unberührt.
|
||||||
|
- Peak-RAM ~1–2 GB, hartes `mem_limit: 4g` als Schutzschranke.
|
||||||
|
- Aktuelle Größenordnung: DACH ~1,45 Mio POIs, ~180 MB.
|
||||||
|
|
||||||
|
## Manuell ausführen (Test)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd /pfad/zu/banyaro # dort, wo docker-compose.yml liegt
|
||||||
|
docker compose -f docker-compose.osm.yml build
|
||||||
|
docker compose -f docker-compose.osm.yml run --rm osm-refresh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Monatlich per DSM-Aufgabenplaner
|
||||||
|
|
||||||
|
1. **Systemsteuerung → Aufgabenplaner → Erstellen → Geplante Aufgabe → Benutzerdefiniertes Skript**
|
||||||
|
2. Benutzer: `root` (für Docker-Zugriff)
|
||||||
|
3. Zeitplan: **monatlich**, z. B. am 1. um 04:00 (lastarm)
|
||||||
|
4. Aufgabeneinstellungen → Benutzerdefiniertes Skript:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd /volume1/docker/banyaro && \
|
||||||
|
/usr/local/bin/docker compose -f docker-compose.osm.yml run --rm osm-refresh \
|
||||||
|
>> /volume1/docker/banyaro/data/osm-refresh.log 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
> Pfad `/volume1/docker/banyaro` an euer Compose-Verzeichnis anpassen.
|
||||||
|
> Docker-Binary auf DSM ist meist `/usr/local/bin/docker` (`docker compose`),
|
||||||
|
> bei älteren DSM ggf. `docker-compose`.
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
Vor jedem Lauf wird `banyaro.pre-osm-JJJJMMTT.db` neben der DB abgelegt. Im
|
||||||
|
Notfall App stoppen, diese Datei auf `banyaro.db` zurückkopieren, App starten.
|
||||||
140
tools/osm-extract/extract_osm_pois.py
Normal file
140
tools/osm-extract/extract_osm_pois.py
Normal file
|
|
@ -0,0 +1,140 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Offline-Extraktion der Ban-Yaro-POIs aus einer OSM-.pbf-Datei.
|
||||||
|
|
||||||
|
Ersetzt das Live-Scannen gegen overpass-api.de (backend/routes/osm.py) durch
|
||||||
|
einen einmaligen/periodischen Batch-Lauf — keine OSM-Live-Last, kein Bann mehr.
|
||||||
|
|
||||||
|
Das Kategorie→Tag-Mapping ist 1:1 aus OSM_QUERIES in backend/routes/osm.py
|
||||||
|
übernommen. Schreibt ins selbe Schema wie die Produktiv-Tabelle `osm_pois`
|
||||||
|
(osm_id, type, lat, lon, name, opening_hours, phone, website, cached_at).
|
||||||
|
|
||||||
|
Aufruf:
|
||||||
|
python3 extract_osm_pois.py <input.osm.pbf> <output.sqlite>
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import sqlite3
|
||||||
|
import osmium
|
||||||
|
|
||||||
|
|
||||||
|
# --- Kategorie-Klassifikation (1:1 aus OSM_QUERIES, backend/routes/osm.py) ---
|
||||||
|
def classify(t) -> list[str]:
|
||||||
|
"""Gibt alle Ban-Yaro-Typen zurück, die auf dieses OSM-Objekt passen."""
|
||||||
|
types: list[str] = []
|
||||||
|
a = t.get("amenity")
|
||||||
|
shop = t.get("shop")
|
||||||
|
craft = t.get("craft")
|
||||||
|
leisure = t.get("leisure")
|
||||||
|
tourism = t.get("tourism")
|
||||||
|
dog = t.get("dog")
|
||||||
|
outdoor = t.get("outdoor_seating")
|
||||||
|
# "hundefreundlich, breiter gefasst": explizit erlaubt ODER Terrasse
|
||||||
|
dog_ok = dog in ("yes", "allowed", "leashed")
|
||||||
|
|
||||||
|
if a == "waste_basket":
|
||||||
|
types.append("waste_basket")
|
||||||
|
if a == "drinking_water":
|
||||||
|
types.append("drinking_water")
|
||||||
|
if a == "veterinary":
|
||||||
|
types.append("tierarzt")
|
||||||
|
if a == "bench":
|
||||||
|
types.append("bank")
|
||||||
|
if leisure == "dog_park" or (leisure == "park" and dog == "yes"):
|
||||||
|
types.append("dog_park")
|
||||||
|
if shop == "pet":
|
||||||
|
types.append("shop")
|
||||||
|
if shop == "pet_grooming" or craft == "pet_grooming":
|
||||||
|
types.append("hundesalon")
|
||||||
|
if (a in ("restaurant", "cafe") and (dog_ok or outdoor == "yes")) or a == "biergarten":
|
||||||
|
types.append("restaurant")
|
||||||
|
if tourism in ("hotel", "guest_house", "hostel") and dog_ok:
|
||||||
|
types.append("hotel")
|
||||||
|
return types
|
||||||
|
|
||||||
|
|
||||||
|
SCHEMA = """
|
||||||
|
CREATE TABLE IF NOT EXISTS osm_pois (
|
||||||
|
osm_id INTEGER NOT NULL,
|
||||||
|
type TEXT NOT NULL,
|
||||||
|
lat REAL NOT NULL,
|
||||||
|
lon REAL NOT NULL,
|
||||||
|
name TEXT,
|
||||||
|
opening_hours TEXT,
|
||||||
|
phone TEXT,
|
||||||
|
website TEXT,
|
||||||
|
user_edited INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cached_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
PRIMARY KEY (osm_id, type)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_osm_pois_loc ON osm_pois(type, lat, lon);
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class PoiHandler(osmium.SimpleHandler):
|
||||||
|
def __init__(self, conn):
|
||||||
|
super().__init__()
|
||||||
|
self.conn = conn
|
||||||
|
self.rows = 0
|
||||||
|
self.objs = 0
|
||||||
|
|
||||||
|
def _save(self, osm_id, tags, lat, lon):
|
||||||
|
types = classify(tags)
|
||||||
|
if not types:
|
||||||
|
return
|
||||||
|
self.objs += 1
|
||||||
|
name = tags.get("name")
|
||||||
|
oh = tags.get("opening_hours")
|
||||||
|
phone = tags.get("phone") or tags.get("contact:phone")
|
||||||
|
web = tags.get("website") or tags.get("contact:website")
|
||||||
|
for ty in types:
|
||||||
|
self.conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO osm_pois "
|
||||||
|
"(osm_id, type, lat, lon, name, opening_hours, phone, website, cached_at) "
|
||||||
|
"VALUES (?,?,?,?,?,?,?,?, datetime('now'))",
|
||||||
|
(osm_id, ty, lat, lon, name, oh, phone, web),
|
||||||
|
)
|
||||||
|
self.rows += 1
|
||||||
|
|
||||||
|
def node(self, n):
|
||||||
|
if n.tags:
|
||||||
|
self._save(n.id, n.tags, n.location.lat, n.location.lon)
|
||||||
|
|
||||||
|
def way(self, w):
|
||||||
|
# Wege (z. B. Tierarzt im Gebäude) → Schwerpunkt aus Knoten ("out center")
|
||||||
|
if not w.tags:
|
||||||
|
return
|
||||||
|
lats, lons = [], []
|
||||||
|
for nd in w.nodes:
|
||||||
|
if nd.location.valid():
|
||||||
|
lats.append(nd.location.lat)
|
||||||
|
lons.append(nd.location.lon)
|
||||||
|
if lats:
|
||||||
|
self._save(w.id, w.tags, sum(lats) / len(lats), sum(lons) / len(lons))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) != 3:
|
||||||
|
print(__doc__)
|
||||||
|
sys.exit(1)
|
||||||
|
src, dst = sys.argv[1], sys.argv[2]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(dst)
|
||||||
|
conn.executescript(SCHEMA)
|
||||||
|
|
||||||
|
h = PoiHandler(conn)
|
||||||
|
# locations=True: Knoten-Koordinaten im Speicher halten, damit Wege einen
|
||||||
|
# Schwerpunkt bekommen. flex_mem skaliert bis Länder-Extrakte.
|
||||||
|
h.apply_file(src, locations=True, idx="flex_mem")
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
print(f"\nObjekte mit Treffer: {h.objs:,} eingefügte Zeilen: {h.rows:,}")
|
||||||
|
print("\nPro Typ:")
|
||||||
|
for ty, cnt in conn.execute(
|
||||||
|
"SELECT type, COUNT(*) FROM osm_pois GROUP BY type ORDER BY 2 DESC"
|
||||||
|
):
|
||||||
|
print(f" {ty:16s} {cnt:>8,}")
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
71
tools/osm-extract/load_into_prod.py
Normal file
71
tools/osm-extract/load_into_prod.py
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Lädt die offline extrahierten POIs (dach.sqlite) in die Produktiv-DB.
|
||||||
|
|
||||||
|
Semantik des Monats-Refresh:
|
||||||
|
* Alle nicht-editierten OSM-POIs (user_edited=0) werden ersetzt → POIs, die
|
||||||
|
aus OSM verschwunden sind, fallen sauber raus.
|
||||||
|
* Von Nutzern korrigierte POIs (user_edited=1, via Moderation) bleiben
|
||||||
|
UNANGETASTET — INSERT OR IGNORE überspringt sie bei Kollision.
|
||||||
|
* Community-Marker (Tabelle user_map_pois) sind separat und werden nie
|
||||||
|
berührt.
|
||||||
|
|
||||||
|
Läuft in EINER Transaktion. Bei Fehler bleibt die alte DB unverändert.
|
||||||
|
|
||||||
|
Aufruf:
|
||||||
|
python3 load_into_prod.py <extract.sqlite> <ziel/banyaro.db>
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
COLS = "osm_id, type, lat, lon, name, opening_hours, phone, website, user_edited, cached_at"
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) != 3:
|
||||||
|
print(__doc__)
|
||||||
|
sys.exit(1)
|
||||||
|
extract_path, prod_path = sys.argv[1], sys.argv[2]
|
||||||
|
|
||||||
|
# timeout/busy_timeout: die App schreibt evtl. parallel — auf Lock warten,
|
||||||
|
# statt sofort zu scheitern. Der Load läuft in EINER Transaktion.
|
||||||
|
conn = sqlite3.connect(prod_path, timeout=120)
|
||||||
|
conn.execute("PRAGMA busy_timeout=120000")
|
||||||
|
conn.execute("PRAGMA foreign_keys=ON")
|
||||||
|
conn.execute(f"ATTACH DATABASE ? AS ext", (extract_path,))
|
||||||
|
|
||||||
|
before = conn.execute("SELECT COUNT(*) FROM osm_pois").fetchone()[0]
|
||||||
|
edited = conn.execute("SELECT COUNT(*) FROM osm_pois WHERE user_edited=1").fetchone()[0]
|
||||||
|
incoming = conn.execute("SELECT COUNT(*) FROM ext.osm_pois").fetchone()[0]
|
||||||
|
|
||||||
|
try:
|
||||||
|
conn.execute("BEGIN")
|
||||||
|
# 1) nicht-editierte OSM-POIs verwerfen (editierte bleiben stehen)
|
||||||
|
conn.execute("DELETE FROM osm_pois WHERE user_edited=0")
|
||||||
|
# 2) frische Extraktion einspielen; editierte Survivor nicht überschreiben
|
||||||
|
conn.execute(
|
||||||
|
f"INSERT OR IGNORE INTO osm_pois ({COLS}) "
|
||||||
|
f"SELECT {COLS} FROM ext.osm_pois"
|
||||||
|
)
|
||||||
|
conn.execute("COMMIT")
|
||||||
|
except Exception:
|
||||||
|
conn.execute("ROLLBACK")
|
||||||
|
raise
|
||||||
|
|
||||||
|
after = conn.execute("SELECT COUNT(*) FROM osm_pois").fetchone()[0]
|
||||||
|
print(f"Vorher: {before:>10,}")
|
||||||
|
print(f"davon user_edited:{edited:>10,} (geschützt)")
|
||||||
|
print(f"Eingespielt: {incoming:>10,}")
|
||||||
|
print(f"Nachher: {after:>10,}")
|
||||||
|
print("\nPro Typ (nachher):")
|
||||||
|
for ty, cnt in conn.execute(
|
||||||
|
"SELECT type, COUNT(*) FROM osm_pois GROUP BY type ORDER BY 2 DESC"
|
||||||
|
):
|
||||||
|
print(f" {ty:16s} {cnt:>9,}")
|
||||||
|
|
||||||
|
conn.execute("DETACH DATABASE ext")
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
66
tools/osm-extract/refresh.sh
Normal file
66
tools/osm-extract/refresh.sh
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Monatlicher POI-Refresh (Build 4) — läuft im Docker-Container auf der Synology.
|
||||||
|
# download → tags-filter (RAM-schonend) → extract → load in die Produktiv-DB.
|
||||||
|
# Ersetzt das Live-Overpass-Scannen (war Bann-Quelle).
|
||||||
|
#
|
||||||
|
# Erst-Migration ohne 5,7-GB-Download: vorab gebaute dach.sqlite mitliefern und
|
||||||
|
# PREBUILT_SQLITE=/data/dach.sqlite setzen → überspringt download/extract.
|
||||||
|
#
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
DB="${DB_PATH:-/data/banyaro.db}"
|
||||||
|
WORK="${WORK_DIR:-/work}"
|
||||||
|
COUNTRIES="${COUNTRIES:-switzerland austria germany}"
|
||||||
|
GEOFABRIK="${GEOFABRIK_BASE:-https://download.geofabrik.de/europe}"
|
||||||
|
KEEP_BACKUPS="${KEEP_BACKUPS:-3}"
|
||||||
|
PREBUILT_SQLITE="${PREBUILT_SQLITE:-}"
|
||||||
|
|
||||||
|
# OSM-Tags für die 9 Ban-Yaro-Kategorien (Superset; finale Klassifikation macht
|
||||||
|
# extract_osm_pois.py). nw/ = node+way, referenzierte Knoten bleiben für die
|
||||||
|
# Weg-Geometrie automatisch erhalten.
|
||||||
|
FILTER=(
|
||||||
|
"nw/amenity=waste_basket,drinking_water,veterinary,bench,biergarten,restaurant,cafe"
|
||||||
|
"nw/leisure=dog_park,park"
|
||||||
|
"nw/shop=pet,pet_grooming"
|
||||||
|
"nw/craft=pet_grooming"
|
||||||
|
"nw/tourism=hotel,guest_house,hostel"
|
||||||
|
)
|
||||||
|
|
||||||
|
mkdir -p "$WORK"; cd "$WORK"
|
||||||
|
echo "[$(date -u)] POI-Refresh start → $DB"
|
||||||
|
|
||||||
|
# 1) Sicherheitskopie der Produktiv-DB (nur die letzten N behalten)
|
||||||
|
if [ -f "$DB" ]; then
|
||||||
|
bak="${DB%.db}.pre-osm-$(date -u +%Y%m%d).db"
|
||||||
|
cp -p "$DB" "$bak"
|
||||||
|
echo "Backup: $bak"
|
||||||
|
ls -1t "${DB%.db}".pre-osm-*.db 2>/dev/null | tail -n +$((KEEP_BACKUPS + 1)) | xargs -r rm -f
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2a) Schnellweg: vorab gebauten Extrakt direkt laden (kein Download/Extract)
|
||||||
|
if [ -n "$PREBUILT_SQLITE" ]; then
|
||||||
|
[ -f "$PREBUILT_SQLITE" ] || { echo "FEHLER: $PREBUILT_SQLITE nicht gefunden"; exit 1; }
|
||||||
|
echo "[$(date -u)] PREBUILT_SQLITE=$PREBUILT_SQLITE → überspringe download/extract"
|
||||||
|
python3 /app/load_into_prod.py "$PREBUILT_SQLITE" "$DB"
|
||||||
|
echo "[$(date -u)] POI-Refresh (prebuilt) fertig."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2b) Regulärer Monatslauf: frisch holen + extrahieren
|
||||||
|
rm -f dach.sqlite
|
||||||
|
for c in $COUNTRIES; do
|
||||||
|
echo "[$(date -u)] $c: download"
|
||||||
|
curl -fSL --retry 3 -o "$c.osm.pbf" "$GEOFABRIK/$c-latest.osm.pbf"
|
||||||
|
echo "[$(date -u)] $c: tags-filter"
|
||||||
|
osmium tags-filter --overwrite -o "$c.f.osm.pbf" "$c.osm.pbf" "${FILTER[@]}"
|
||||||
|
rm -f "$c.osm.pbf"
|
||||||
|
echo "[$(date -u)] $c: extract"
|
||||||
|
python3 /app/extract_osm_pois.py "$c.f.osm.pbf" dach.sqlite
|
||||||
|
rm -f "$c.f.osm.pbf"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "[$(date -u)] load → Produktiv-DB"
|
||||||
|
python3 /app/load_into_prod.py dach.sqlite "$DB"
|
||||||
|
rm -f dach.sqlite
|
||||||
|
echo "[$(date -u)] POI-Refresh fertig."
|
||||||
Loading…
Add table
Add a link
Reference in a new issue