Scheduler aufräumen: Prewarm-Job entfernt, Rassen-Seed monatlich, Admin-Panel Wiki-Label — SW by-v433, APP_VER 412
This commit is contained in:
parent
c3d14d2342
commit
85836e4e6e
4 changed files with 17 additions and 199 deletions
|
|
@ -68,21 +68,21 @@ def start():
|
|||
id="import_events_startup",
|
||||
replace_existing=True,
|
||||
)
|
||||
# Einmalig beim Start (nach 15s Verzögerung) — Rassen aus TheDogAPI befüllen
|
||||
# Alle 4 Wochen Di 03:00 — Rassen aus TheDogAPI aktualisieren
|
||||
_scheduler.add_job(
|
||||
_job_seed_breeds,
|
||||
'date',
|
||||
run_date=datetime.now(tz=_TZ) + timedelta(seconds=15),
|
||||
id="seed_breeds_startup",
|
||||
CronTrigger(day=1, hour=3, minute=0), # 1. jedes Monats
|
||||
id="seed_breeds",
|
||||
replace_existing=True,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
# Einmalig beim Start (nach 45s Verzögerung) — fehlende Rassen aus Wikidata ergänzen
|
||||
# Alle 4 Wochen Di 04:00 — fehlende Rassen aus Wikidata ergänzen
|
||||
_scheduler.add_job(
|
||||
_job_seed_wikidata_breeds,
|
||||
'date',
|
||||
run_date=datetime.now(tz=_TZ) + timedelta(seconds=45),
|
||||
id="seed_wikidata_startup",
|
||||
CronTrigger(day=1, hour=4, minute=0), # 1. jedes Monats
|
||||
id="seed_wikidata",
|
||||
replace_existing=True,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
# Jeden Montag 09:00 — Wöchentlicher Fortschritts-Lober
|
||||
_scheduler.add_job(
|
||||
|
|
@ -109,7 +109,7 @@ def start():
|
|||
misfire_grace_time=3600,
|
||||
)
|
||||
_scheduler.start()
|
||||
logger.info("Scheduler gestartet — Health-Reminder 08:00, Giftköder-Archiv 03:00, Wetter-Alert 07:30, Meilenstein-Check 00:05, Event-Import So 02:00, Rassen-Seed beim Start. OSM-Cache: on-demand (kein Prewarm).")
|
||||
logger.info("Scheduler gestartet — Health-Reminder 08:00, Giftköder-Archiv 03:00, Wetter-Alert 07:30, Meilenstein-Check 00:05, Event-Import So 02:00, Rassen-Seed monatlich 1. des Monats. OSM-Cache: on-demand (kein Prewarm).")
|
||||
|
||||
|
||||
def stop():
|
||||
|
|
@ -417,192 +417,10 @@ async def _job_seed_wikidata_breeds():
|
|||
from scraper.wikipedia_photos import fetch_wikipedia_photos
|
||||
wp_count = await fetch_wikipedia_photos()
|
||||
logger.info(f"Wikipedia photo fetch done: {wp_count} Fotos")
|
||||
_log_job("seed_wikidata_startup", "ok", f"{count} Rassen, {mirrored}+{wp_count} Fotos")
|
||||
_log_job("seed_wikidata", "ok", f"{count} Rassen, {mirrored}+{wp_count} Fotos")
|
||||
except Exception as e:
|
||||
logger.error(f"Wikidata-Seed: Fehler: {e}")
|
||||
_log_job("seed_wikidata_startup", "error", str(e))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# JOB: OSM-Tiles für deutsche Großstädte vorwärmen
|
||||
# Läuft einmalig 90s nach Start + wöchentlich So 01:00 Uhr.
|
||||
# Nur stale Tiles werden abgerufen (bereits gecachte werden übersprungen).
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
# Deutsche Städte mit >50.000 Einwohnern (lat, lon, name)
|
||||
_CITIES_DE = [
|
||||
(52.5200, 13.4050, "Berlin"),
|
||||
(53.5753, 10.0153, "Hamburg"),
|
||||
(48.1372, 11.5755, "München"),
|
||||
(51.2217, 6.7762, "Düsseldorf"),
|
||||
(50.9333, 6.9500, "Köln"),
|
||||
(50.1109, 8.6821, "Frankfurt"),
|
||||
(48.7775, 9.1800, "Stuttgart"),
|
||||
(51.4566, 7.0116, "Dortmund"),
|
||||
(51.5136, 7.4653, "Dortmund-Ost"),
|
||||
(51.4508, 7.0131, "Essen"),
|
||||
(51.3388, 12.3799, "Leipzig"),
|
||||
(51.2254, 6.7762, "Düsseldorf"),
|
||||
(51.0534, 13.7373, "Dresden"),
|
||||
(52.3759, 9.7320, "Hannover"),
|
||||
(51.4818, 7.2162, "Bochum"),
|
||||
(51.9607, 7.6261, "Münster"),
|
||||
(51.3670, 7.4595, "Hagen"),
|
||||
(50.7753, 6.0839, "Aachen"),
|
||||
(51.2563, 7.1500, "Wuppertal"),
|
||||
(49.4521, 11.0767, "Nürnberg"),
|
||||
(53.0758, 8.8072, "Bremen"),
|
||||
(50.7323, 7.0955, "Bonn"),
|
||||
(49.0069, 8.4037, "Karlsruhe"),
|
||||
(51.9607, 7.6261, "Münster"),
|
||||
(51.4344, 6.7623, "Duisburg"),
|
||||
(51.6667, 6.1667, "Moers"),
|
||||
(48.3705, 10.8978, "Augsburg"),
|
||||
(52.2689, 10.5268, "Braunschweig"),
|
||||
(50.9287, 11.5861, "Jena"),
|
||||
(53.8655, 10.6866, "Lübeck"),
|
||||
(54.3233, 10.1394, "Kiel"),
|
||||
(53.1435, 8.2146, "Oldenburg"),
|
||||
(52.0302, 8.5325, "Bielefeld"),
|
||||
(51.3167, 9.5000, "Kassel"),
|
||||
(50.0000, 8.2731, "Mainz"),
|
||||
(49.8728, 8.6512, "Darmstadt"),
|
||||
(49.0047, 12.0949, "Regensburg"),
|
||||
(48.9960, 8.4025, "Pforzheim"),
|
||||
(53.4706, 9.9817, "Hamburg-Süd"),
|
||||
(50.8283, 12.9209, "Chemnitz"),
|
||||
(51.7227, 8.7559, "Paderborn"),
|
||||
(52.1205, 11.6276, "Magdeburg"),
|
||||
(52.6367, 11.8683, "Magdeburg-Ost"),
|
||||
(50.3569, 7.5890, "Koblenz"),
|
||||
(48.4010, 9.9876, "Ulm"),
|
||||
(51.0504, 13.7373, "Dresden-Mitte"),
|
||||
(49.4875, 8.4660, "Mannheim"),
|
||||
(49.2354, 7.0038, "Kaiserslautern"),
|
||||
(50.1155, 8.6782, "Frankfurt-Mitte"),
|
||||
(50.0782, 8.2398, "Wiesbaden"),
|
||||
(52.4227, 10.7865, "Wolfsburg"),
|
||||
(51.9607, 8.8693, "Gütersloh"),
|
||||
(53.5753, 9.8500, "Hamburg-West"),
|
||||
(48.5216, 9.0576, "Reutlingen"),
|
||||
(48.9522, 9.4358, "Heilbronn"),
|
||||
(49.4478, 7.7691, "Kaiserslautern-W"),
|
||||
(53.6333, 9.9833, "Hamburg-Nord"),
|
||||
(52.3905, 13.0645, "Potsdam"),
|
||||
(54.0924, 12.1407, "Rostock"),
|
||||
(53.4339, 14.5508, "Szczecin-grenze"),
|
||||
(51.7563, 14.3329, "Cottbus"),
|
||||
(50.4782, 12.3598, "Zwickau"),
|
||||
(53.5507, 9.9967, "Hamburg-Mitte"),
|
||||
(51.8127, 10.3354, "Goslar"),
|
||||
(48.6843, 9.0061, "Böblingen"),
|
||||
(48.7761, 9.1775, "Stuttgart-Mitte"),
|
||||
(49.4521, 8.4660, "Heidelberg"),
|
||||
(50.8088, 8.7667, "Marburg"),
|
||||
(51.9607, 7.6261, "Münster-Mitte"),
|
||||
(52.2763, 8.0479, "Osnabrück"),
|
||||
(53.8755, 10.7000, "Lübeck-Ost"),
|
||||
(51.9333, 6.8667, "Borken"),
|
||||
# München Umland
|
||||
(48.0734, 11.9661, "Ebersberg"),
|
||||
(47.9947, 11.6612, "Holzkirchen"),
|
||||
(48.0628, 11.6574, "Ottobrunn"),
|
||||
(48.2456, 11.3712, "Dachau"),
|
||||
(48.1667, 11.7833, "Vaterstetten"),
|
||||
(48.2667, 11.6667, "Garching"),
|
||||
(48.0667, 11.4667, "Gauting"),
|
||||
(47.9833, 11.3000, "Starnberg"),
|
||||
]
|
||||
|
||||
async def _job_prewarm_cities():
|
||||
import os, asyncio, time
|
||||
from routes.osm import _covering_tiles, _stale_tiles, _fetch_and_store_tile, OSM_QUERIES, CACHE_ZOOM
|
||||
from mailer import send_email
|
||||
|
||||
ADMIN = os.getenv("ADMIN_EMAIL", "")
|
||||
REPORT_INTERVAL = 5 * 3600 # alle 5 Stunden
|
||||
|
||||
logger.info("City-Prewarm Job startet…")
|
||||
sem = asyncio.Semaphore(1)
|
||||
total_fetched = 0
|
||||
cities_done = 0
|
||||
start_time = time.monotonic()
|
||||
last_report = start_time
|
||||
|
||||
async def _fetch(poi_type, x, y):
|
||||
nonlocal total_fetched
|
||||
async with sem:
|
||||
await _fetch_and_store_tile(poi_type, x, y)
|
||||
total_fetched += 1
|
||||
await asyncio.sleep(5)
|
||||
|
||||
async def _send_progress(subject_prefix, cities_done, total_cities, eta_str=""):
|
||||
if not ADMIN:
|
||||
return
|
||||
elapsed = int(time.monotonic() - start_time)
|
||||
h, m = divmod(elapsed // 60, 60)
|
||||
elapsed_str = f"{h}h {m:02d}min" if h else f"{m}min"
|
||||
pct = round(cities_done / total_cities * 100)
|
||||
body_plain = (
|
||||
f"City-Prewarm Fortschritt\n\n"
|
||||
f"Städte: {cities_done}/{total_cities} ({pct}%)\n"
|
||||
f"Tiles geladen: {total_fetched}\n"
|
||||
f"Laufzeit: {elapsed_str}\n"
|
||||
f"{('Verbleibend (ca.): ' + eta_str) if eta_str else ''}"
|
||||
)
|
||||
body_html = f"""\
|
||||
<div style="font-family:sans-serif;max-width:480px;margin:0 auto;padding:24px">
|
||||
<h2 style="color:#C4843A;margin:0 0 16px">🗺️ City-Prewarm {subject_prefix}</h2>
|
||||
<table style="border-collapse:collapse;width:100%">
|
||||
<tr><td style="padding:6px 0;color:#666">Städte:</td>
|
||||
<td style="padding:6px 0;font-weight:600">{cities_done} / {total_cities} ({pct}%)</td></tr>
|
||||
<tr><td style="padding:6px 0;color:#666">Tiles geladen:</td>
|
||||
<td style="padding:6px 0;font-weight:600">{total_fetched}</td></tr>
|
||||
<tr><td style="padding:6px 0;color:#666">Laufzeit:</td>
|
||||
<td style="padding:6px 0;font-weight:600">{elapsed_str}</td></tr>
|
||||
{f'<tr><td style="padding:6px 0;color:#666">Verbleibend (ca.):</td><td style="padding:6px 0;font-weight:600">{eta_str}</td></tr>' if eta_str else ''}
|
||||
</table>
|
||||
</div>"""
|
||||
try:
|
||||
await send_email(ADMIN, f"Ban Yaro — City-Prewarm {subject_prefix}", body_html, body_plain)
|
||||
except Exception as e:
|
||||
logger.warning(f"City-Prewarm Mail fehlgeschlagen: {e}")
|
||||
|
||||
total_cities = len(_CITIES_DE)
|
||||
for lat, lon, city in _CITIES_DE:
|
||||
dlat = 0.18
|
||||
dlon = 0.25
|
||||
south, west, north, east = lat - dlat, lon - dlon, lat + dlat, lon + dlon
|
||||
tiles = _covering_tiles(south, west, north, east, CACHE_ZOOM)
|
||||
|
||||
tasks = []
|
||||
for poi_type in OSM_QUERIES:
|
||||
stale = _stale_tiles(poi_type, tiles)
|
||||
for (x, y) in stale:
|
||||
tasks.append(_fetch(poi_type, x, y))
|
||||
|
||||
if tasks:
|
||||
logger.info(f"City-Prewarm: {city} — {len(tasks)} Tiles zu laden")
|
||||
await asyncio.gather(*tasks)
|
||||
else:
|
||||
logger.debug(f"City-Prewarm: {city} — alle Tiles frisch")
|
||||
|
||||
cities_done += 1
|
||||
|
||||
# Fortschritts-Mail alle 5 Stunden
|
||||
now = time.monotonic()
|
||||
if ADMIN and (now - last_report) >= REPORT_INTERVAL:
|
||||
elapsed = now - start_time
|
||||
rate = cities_done / elapsed if elapsed > 0 else 0
|
||||
remaining = int((total_cities - cities_done) / rate) if rate > 0 else 0
|
||||
rh, rm = divmod(remaining // 60, 60)
|
||||
eta_str = f"{rh}h {rm:02d}min" if rh else f"{rm}min"
|
||||
await _send_progress("Fortschritt", cities_done, total_cities, eta_str)
|
||||
last_report = now
|
||||
|
||||
logger.info(f"City-Prewarm Job fertig — {total_fetched} Tiles geladen.")
|
||||
await _send_progress("abgeschlossen ✓", cities_done, total_cities)
|
||||
|
||||
_log_job("seed_wikidata", "error", str(e))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Hilfsfunktion: Job-Protokoll aktualisieren
|
||||
|
|
@ -876,8 +694,8 @@ async def _job_status_report():
|
|||
"weather_alert": "Wetter-Alert",
|
||||
"milestone_check": "Meilenstein-Check",
|
||||
"import_events": "Event-Import (VDH)",
|
||||
"seed_breeds_startup": "Rassen-Seed (TheDogAPI)",
|
||||
"seed_wikidata_startup":"Rassen-Seed (Wikidata)",
|
||||
"seed_breeds": "Rassen-Seed (TheDogAPI, monatlich)",
|
||||
"seed_wikidata": "Rassen-Seed (Wikidata, monatlich)",
|
||||
"weekly_praise": "Wöchentlicher Lober (Mo 09:00)",
|
||||
"ki_health_report": "KI-Gesundheitsberichte",
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue