From 32d630d5a1850641cefc8599bafe306cde60fb89 Mon Sep 17 00:00:00 2001 From: rene Date: Wed, 15 Apr 2026 22:01:58 +0200 Subject: [PATCH] Sprint 11b: Wiki-Foto-Einreichungen + Wikipedia-Foto-Scraper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - User können Fotos für Rassen vorschlagen (Upload-Modal in Rassen-Detail) - Mod/Admin-Review-Tab im Wiki mit Freischalten/Ablehnen + Push-Notification - wikipedia_photos.py: holt Fotos über Wikidata-QID → Wikipedia-API - Foto-Status: 578 lokal, 186 extern, 238 ohne Foto - DB: wiki_foto_submissions Tabelle - SW by-v90 --- backend/database.py | 16 +++ backend/routes/wiki.py | 183 ++++++++++++++++++++++++- backend/scheduler.py | 4 + backend/scraper/wikipedia_photos.py | 198 ++++++++++++++++++++++++++++ backend/static/js/pages/wiki.js | 198 +++++++++++++++++++++++++++- backend/static/sw.js | 2 +- 6 files changed, 598 insertions(+), 3 deletions(-) create mode 100644 backend/scraper/wikipedia_photos.py diff --git a/backend/database.py b/backend/database.py index 83eea09..7572c35 100644 --- a/backend/database.py +++ b/backend/database.py @@ -525,6 +525,22 @@ def _migrate(conn_factory): ON events(external_id) WHERE external_id IS NOT NULL; """) + # Wiki: User-Foto-Einreichungen + conn.executescript(""" + CREATE TABLE IF NOT EXISTS wiki_foto_submissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + rasse_id INTEGER NOT NULL REFERENCES wiki_rassen(id) ON DELETE CASCADE, + foto_url TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + reviewed_by INTEGER REFERENCES users(id), + reviewed_at TEXT, + reject_reason TEXT + ); + CREATE INDEX IF NOT EXISTS idx_wfs_status ON wiki_foto_submissions(status, created_at DESC); + """) + # Freundschaften + Direktnachrichten conn.executescript(""" CREATE TABLE IF NOT EXISTS friendships ( diff --git a/backend/routes/wiki.py b/backend/routes/wiki.py index 917d92f..edf98e5 100644 --- a/backend/routes/wiki.py +++ b/backend/routes/wiki.py @@ -1,10 +1,19 @@ """BAN YARO — Hunde-Wiki Routes""" -from fastapi import APIRouter, Depends, HTTPException, Query +import os +import shutil +import time +import logging +from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile, File from pydantic import BaseModel from database import db from auth import get_current_user +logger = logging.getLogger(__name__) +MEDIA_DIR = os.getenv("MEDIA_DIR", "/data/media") +BREEDS_DIR = os.path.join(MEDIA_DIR, "breeds") +SUBMIT_DIR = os.path.join(BREEDS_DIR, "submissions") + router = APIRouter() @@ -256,3 +265,175 @@ async def quiz_result( ] return {"results": top3} + + +# ------------------------------------------------------------------ +# POST /api/wiki/rassen/{slug}/foto — User reicht Foto ein +# ------------------------------------------------------------------ +@router.post("/rassen/{slug}/foto", status_code=201) +async def submit_foto( + slug: str, + file: UploadFile = File(...), + user = Depends(get_current_user), +): + with db() as conn: + rasse = conn.execute( + "SELECT id, name, external_id FROM wiki_rassen WHERE slug=?", (slug,) + ).fetchone() + if not rasse: + raise HTTPException(404, "Rasse nicht gefunden.") + + # Dateiformat prüfen + ct = file.content_type or "" + if not ct.startswith("image/"): + raise HTTPException(400, "Nur Bilddateien erlaubt.") + + os.makedirs(SUBMIT_DIR, exist_ok=True) + ts = int(time.time()) + filename = f"{slug}_{user['id']}_{ts}.jpg" + path = os.path.join(SUBMIT_DIR, filename) + + content = await file.read() + if len(content) > 8 * 1024 * 1024: + raise HTTPException(400, "Datei zu groß (max. 8 MB).") + with open(path, "wb") as f: + f.write(content) + + local_url = f"/media/breeds/submissions/{filename}" + + with db() as conn: + # Bestehende pending-Einreichung des Users für diese Rasse ersetzen + old = conn.execute( + "SELECT foto_url FROM wiki_foto_submissions WHERE rasse_id=? AND user_id=? AND status='pending'", + (rasse["id"], user["id"]) + ).fetchone() + if old: + try: + old_path = old["foto_url"].replace("/media/", MEDIA_DIR + "/", 1) + if os.path.exists(old_path): + os.remove(old_path) + except Exception: + pass + conn.execute( + "DELETE FROM wiki_foto_submissions WHERE rasse_id=? AND user_id=? AND status='pending'", + (rasse["id"], user["id"]) + ) + + conn.execute(""" + INSERT INTO wiki_foto_submissions (user_id, rasse_id, foto_url) + VALUES (?,?,?) + """, (user["id"], rasse["id"], local_url)) + + logger.info(f"Foto-Einreichung: {rasse['name']} von User {user['id']}") + return {"ok": True, "foto_url": local_url} + + +# ------------------------------------------------------------------ +# GET /api/wiki/foto-submissions — offene Einreichungen (Mod/Admin) +# ------------------------------------------------------------------ +@router.get("/foto-submissions") +async def list_submissions(user=Depends(get_current_user)): + if not (user.get("is_moderator") or user.get("rolle") == "admin"): + raise HTTPException(403, "Nur Moderatoren.") + + with db() as conn: + rows = conn.execute(""" + SELECT s.id, s.foto_url, s.status, s.created_at, + u.name AS user_name, + r.name AS rasse_name, r.slug AS rasse_slug, + r.foto_url AS aktuell_foto + FROM wiki_foto_submissions s + JOIN users u ON u.id = s.user_id + JOIN wiki_rassen r ON r.id = s.rasse_id + WHERE s.status = 'pending' + ORDER BY s.created_at ASC + """).fetchall() + return [dict(r) for r in rows] + + +# ------------------------------------------------------------------ +# PATCH /api/wiki/foto-submissions/{id} — genehmigen oder ablehnen +# ------------------------------------------------------------------ +class ReviewModel(BaseModel): + action: str # "approve" | "reject" + reject_reason: str = "" + + +@router.patch("/foto-submissions/{sub_id}") +async def review_submission(sub_id: int, data: ReviewModel, user=Depends(get_current_user)): + if not (user.get("is_moderator") or user.get("rolle") == "admin"): + raise HTTPException(403, "Nur Moderatoren.") + if data.action not in ("approve", "reject"): + raise HTTPException(400, "action muss 'approve' oder 'reject' sein.") + + with db() as conn: + sub = conn.execute( + "SELECT * FROM wiki_foto_submissions WHERE id=? AND status='pending'", + (sub_id,) + ).fetchone() + if not sub: + raise HTTPException(404, "Einreichung nicht gefunden.") + + rasse = conn.execute( + "SELECT id, external_id, slug FROM wiki_rassen WHERE id=?", + (sub["rasse_id"],) + ).fetchone() + + if data.action == "approve": + # Ziel-Dateiname aus external_id ableiten + ext_id = rasse["external_id"] if rasse else None + if ext_id and str(ext_id).startswith("wd_"): + qid = str(ext_id).replace("wd_", "") + dest_name = f"{qid}.jpg" + elif ext_id: + dest_name = f"{ext_id}.jpg" + else: + dest_name = f"{rasse['slug']}.jpg" + + os.makedirs(BREEDS_DIR, exist_ok=True) + src = sub["foto_url"].replace("/media/", MEDIA_DIR + "/", 1) + dest = os.path.join(BREEDS_DIR, dest_name) + try: + shutil.copy2(src, dest) + except Exception as e: + raise HTTPException(500, f"Datei konnte nicht kopiert werden: {e}") + + new_url = f"/media/breeds/{dest_name}" + conn.execute( + "UPDATE wiki_rassen SET foto_url=? WHERE id=?", + (new_url, rasse["id"]) + ) + conn.execute(""" + UPDATE wiki_foto_submissions + SET status='approved', reviewed_by=?, reviewed_at=datetime('now') + WHERE id=? + """, (user["id"], sub_id)) + + # Push-Notification an Einreicher + try: + from routes.push import send_push_to_user + send_push_to_user(sub["user_id"], { + "title": "Foto freigeschalten!", + "body": f"Dein Foto wurde im Wiki veröffentlicht.", + "type": "wiki_foto_approved", + "data": {"page": "wiki"}, + }) + except Exception: + pass + + else: # reject + conn.execute(""" + UPDATE wiki_foto_submissions + SET status='rejected', reviewed_by=?, reviewed_at=datetime('now'), + reject_reason=? + WHERE id=? + """, (user["id"], data.reject_reason or "Nicht geeignet.", sub_id)) + # Temp-Datei löschen + try: + path = sub["foto_url"].replace("/media/", MEDIA_DIR + "/", 1) + if os.path.exists(path): + os.remove(path) + except Exception: + pass + + return {"ok": True} diff --git a/backend/scheduler.py b/backend/scheduler.py index b08b8ee..b8aa803 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -376,6 +376,10 @@ async def _job_seed_wikidata_breeds(): logger.info(f"Wikidata breed seed done: {count} neue Rassen") mirrored = await mirror_wikidata_photos() logger.info(f"Wikidata photo mirror done: {mirrored} Fotos") + # Wikipedia-Fotos für Rassen die noch kein Bild haben + from scraper.wikipedia_photos import fetch_wikipedia_photos + wp_count = await fetch_wikipedia_photos() + logger.info(f"Wikipedia photo fetch done: {wp_count} Fotos") except Exception as e: logger.error(f"Wikidata-Seed: Fehler: {e}") diff --git a/backend/scraper/wikipedia_photos.py b/backend/scraper/wikipedia_photos.py new file mode 100644 index 0000000..3a5a808 --- /dev/null +++ b/backend/scraper/wikipedia_photos.py @@ -0,0 +1,198 @@ +""" +Holt Fotos für Wikidata-Rassen ohne Bild über die Wikipedia-API. + +Strategie: +1. Wikidata-API: QID → Wikipedia-Artikel-Titel (DE bevorzugt, Fallback EN) +2. Wikipedia pageimages-API: Artikel-Titel → Bild-URL +3. Wikimedia Commons: Bild herunterladen und lokal speichern +""" + +import asyncio +import logging +import os +import re +import httpx + +from database import db + +logger = logging.getLogger(__name__) +MEDIA_DIR = os.getenv("MEDIA_DIR", "/data/media") +BREEDS_DIR = os.path.join(MEDIA_DIR, "breeds") + +WIKIDATA_API = "https://www.wikidata.org/w/api.php" +WP_DE_API = "https://de.wikipedia.org/w/api.php" +WP_EN_API = "https://en.wikipedia.org/w/api.php" +HEADERS = {"User-Agent": "BanYaro/1.0 (https://banyaro.app; contact@banyaro.app)"} +BATCH_SIZE = 50 # Wikidata API erlaubt max 50 IDs pro Request +SLEEP_MS = 0.35 # 350ms zwischen Downloads + + +def _qid_from_ext(ext_id: str) -> str | None: + """Extrahiert QID aus external_id wie 'wd_Q12345' → 'Q12345'.""" + m = re.match(r"wd_(Q\d+)$", ext_id) + return m.group(1) if m else None + + +async def _fetch_sitelinks(qids: list[str], client: httpx.AsyncClient) -> dict[str, dict]: + """ + Gibt {qid: {'de': 'Titel_DE', 'en': 'Titel_EN'}} zurück + für alle QIDs, die mindestens einen Wikipedia-Sitelink haben. + """ + if not qids: + return {} + try: + r = await client.get(WIKIDATA_API, params={ + "action": "wbgetentities", + "ids": "|".join(qids), + "props": "sitelinks", + "sitefilter": "dewiki|enwiki", + "format": "json", + }) + r.raise_for_status() + data = r.json() + except Exception as e: + logger.warning(f"Wikidata sitelinks Fehler: {e}") + return {} + + result = {} + for qid, entity in data.get("entities", {}).items(): + sitelinks = entity.get("sitelinks", {}) + titles = {} + if "dewiki" in sitelinks: + titles["de"] = sitelinks["dewiki"]["title"] + if "enwiki" in sitelinks: + titles["en"] = sitelinks["enwiki"]["title"] + if titles: + result[qid] = titles + return result + + +async def _fetch_wp_image(title: str, lang: str, client: httpx.AsyncClient) -> str | None: + """ + Gibt die Thumbnail-URL eines Wikipedia-Artikels zurück (600px-Version). + """ + api = WP_DE_API if lang == "de" else WP_EN_API + try: + r = await client.get(api, params={ + "action": "query", + "titles": title, + "prop": "pageimages", + "pithumbsize": 600, + "format": "json", + }) + r.raise_for_status() + pages = r.json().get("query", {}).get("pages", {}) + for page in pages.values(): + thumb = page.get("thumbnail", {}).get("source") + if thumb: + return thumb + except Exception as e: + logger.debug(f"WP pageimage Fehler ({lang}/{title}): {e}") + return None + + +async def _download_image(url: str, path: str, client: httpx.AsyncClient) -> bool: + """Lädt Bild herunter, speichert unter path. True bei Erfolg.""" + for attempt in range(2): + try: + await asyncio.sleep(SLEEP_MS) + r = await client.get(url) + if r.status_code == 200 and r.headers.get("content-type", "").startswith("image"): + with open(path, "wb") as f: + f.write(r.content) + return True + if r.status_code == 429: + await asyncio.sleep(15 * (attempt + 1)) + except Exception as e: + logger.debug(f"Download Fehler {url}: {e}") + return False + + +async def fetch_wikipedia_photos() -> int: + """ + Haupt-Funktion: Holt Wikipedia-Fotos für alle Rassen ohne foto_url. + Gibt Anzahl erfolgreich gespeicherter Fotos zurück. + """ + os.makedirs(BREEDS_DIR, exist_ok=True) + + with db() as conn: + rows = conn.execute(""" + SELECT id, external_id, name + FROM wiki_rassen + WHERE (foto_url IS NULL OR foto_url = '') + AND external_id LIKE 'wd_%' + """).fetchall() + + if not rows: + logger.info("Wikipedia-Fotos: nichts zu tun") + return 0 + + logger.info(f"Wikipedia-Fotos: {len(rows)} Rassen ohne Foto") + + # QID → DB-Row mappen + qid_map = {} # { 'Q12345': {'id': 1, 'external_id': 'wd_Q12345', 'name': '...'} } + for row in rows: + qid = _qid_from_ext(row["external_id"]) + if qid: + qid_map[qid] = dict(row) + + qids = list(qid_map.keys()) + saved = 0 + + async with httpx.AsyncClient( + timeout=30, + follow_redirects=True, + headers=HEADERS + ) as client: + + # Sitelinks in Batches holen + sitelinks: dict[str, dict] = {} + for i in range(0, len(qids), BATCH_SIZE): + batch = qids[i:i + BATCH_SIZE] + chunk = await _fetch_sitelinks(batch, client) + sitelinks.update(chunk) + await asyncio.sleep(0.5) + logger.info(f"Sitelinks: {i + len(batch)}/{len(qids)} abgefragt, {len(sitelinks)} mit WP-Link") + + logger.info(f"Wikipedia-Links gefunden: {len(sitelinks)}/{len(qids)}") + + # Für jeden mit Sitelink → Bild holen + herunterladen + for idx, (qid, titles) in enumerate(sitelinks.items()): + row = qid_map[qid] + row_id = row["id"] + lang = "de" if "de" in titles else "en" + title = titles[lang] + + img_url = await _fetch_wp_image(title, lang, client) + if not img_url: + # Zweiter Versuch mit EN wenn DE kein Bild hat + if lang == "de" and "en" in titles: + img_url = await _fetch_wp_image(titles["en"], "en", client) + + if not img_url: + logger.debug(f"Kein WP-Bild für {row['name']} ({qid})") + continue + + local_path = os.path.join(BREEDS_DIR, f"{qid}.jpg") + local_url = f"/media/breeds/{qid}.jpg" + + if os.path.exists(local_path): + # Datei existiert bereits → nur DB updaten + with db() as conn: + conn.execute("UPDATE wiki_rassen SET foto_url=? WHERE id=?", (local_url, row_id)) + saved += 1 + continue + + ok = await _download_image(img_url, local_path, client) + if ok: + with db() as conn: + conn.execute("UPDATE wiki_rassen SET foto_url=? WHERE id=?", (local_url, row_id)) + saved += 1 + else: + logger.debug(f"Download fehlgeschlagen: {row['name']}") + + if idx % 50 == 0 and idx > 0: + logger.info(f"Wikipedia-Fotos: {saved}/{idx + 1} bisher") + + logger.info(f"Wikipedia-Fotos gespeichert: {saved}/{len(sitelinks)} (mit WP-Link)") + return saved diff --git a/backend/static/js/pages/wiki.js b/backend/static/js/pages/wiki.js index c195850..fe31466 100644 --- a/backend/static/js/pages/wiki.js +++ b/backend/static/js/pages/wiki.js @@ -94,12 +94,15 @@ window.Page_wiki = (() => { // RENDER // ---------------------------------------------------------- async function _render() { + const isMod = _appState.user && (_appState.user.is_moderator || _appState.user.rolle === 'admin'); + _container.innerHTML = `
+ ${isMod ? `` : ''}
`; @@ -122,6 +125,97 @@ window.Page_wiki = (() => { else if (_tab === 'gesundheit') _renderGesundheit(content); else if (_tab === 'recht') _renderRecht(content); else if (_tab === 'quiz') _renderQuiz(content); + else if (_tab === 'fotos') await _renderFotoSubmissions(content); + } + + // ---------------------------------------------------------- + // TAB: Foto-Einreichungen (Mod/Admin) + // ---------------------------------------------------------- + async function _renderFotoSubmissions(el) { + el.innerHTML = `
${UI.skeleton(3)}
`; + let subs; + try { + subs = await _apiFetch('/api/wiki/foto-submissions'); + } catch (e) { + el.innerHTML = `

${_esc(e.message)}

`; + return; + } + + // Badge updaten + const badge = document.getElementById('wiki-fotos-badge'); + if (badge) { badge.textContent = subs.length; badge.style.display = subs.length ? '' : 'none'; } + + if (!subs.length) { + el.innerHTML = ` +
+ ${UI.icon('check')} +

Keine ausstehenden Foto-Einreichungen.

+
`; + return; + } + + el.innerHTML = ` +
+

+ Ausstehende Fotos (${subs.length}) +

+
+ ${subs.map(s => ` +
+
+ +
+
${_esc(s.rasse_name)}
+
+ von ${_esc(s.user_name)} · ${_formatDate(s.created_at)} +
+ ${s.aktuell_foto + ? `
+ Aktuelles Foto: +
` + : `
Kein Foto vorhanden
` + } +
+
+
+ + +
+
+ `).join('')} +
+
+ `; + } + + async function _approveSubmission(id) { + try { + await _apiPatch(`/api/wiki/foto-submissions/${id}`, { action: 'approve' }); + document.getElementById(`wiki-sub-${id}`)?.remove(); + UI.toast('Foto freigeschaltet!', 'success'); + const badge = document.getElementById('wiki-fotos-badge'); + if (badge) { + const n = Math.max(0, parseInt(badge.textContent || '0') - 1); + badge.textContent = n; badge.style.display = n ? '' : 'none'; + } + } catch (e) { UI.toast(e.message, 'danger'); } + } + + async function _rejectSubmission(id) { + const reason = prompt('Ablehnungsgrund (optional):') ?? null; + if (reason === null) return; // Abbrechen + try { + await _apiPatch(`/api/wiki/foto-submissions/${id}`, { action: 'reject', reject_reason: reason }); + document.getElementById(`wiki-sub-${id}`)?.remove(); + UI.toast('Einreichung abgelehnt.', 'info'); + } catch (e) { UI.toast(e.message, 'danger'); } } // ---------------------------------------------------------- @@ -366,6 +460,12 @@ window.Page_wiki = (() => { Anmelden, um einen Bericht zu schreiben.

` } + ${_appState.user ? ` +
+ +
` : ''} `; UI.modal.open({ title: _esc(rasse.name), body }); @@ -374,6 +474,87 @@ window.Page_wiki = (() => { UI.modal.close(); setTimeout(() => _showBerichtForm(slug, rasse.name), 350); }); + + document.getElementById('wiki-foto-submit-btn')?.addEventListener('click', () => { + UI.modal.close(); + setTimeout(() => _showFotoSubmitForm(slug, rasse.name), 350); + }); + } + + // ---------------------------------------------------------- + // Foto vorschlagen + // ---------------------------------------------------------- + function _showFotoSubmitForm(slug, rasseName) { + const body = ` +

+ Dein Foto wird nach einer kurzen Prüfung freigeschaltet und als Hauptbild im Wiki verwendet. +

+
+
+ + +
+ JPG, PNG oder WebP · max. 8 MB · möglichst hochauflösend +
+
+ +
+ `; + const footer = ` + + + `; + + UI.modal.open({ title: 'Foto vorschlagen', body, footer }); + document.getElementById('wiki-foto-cancel')?.addEventListener('click', UI.modal.close); + + document.getElementById('wiki-foto-input')?.addEventListener('change', e => { + const file = e.target.files?.[0]; + if (!file) return; + const preview = document.getElementById('wiki-foto-preview'); + const img = document.getElementById('wiki-foto-preview-img'); + const url = URL.createObjectURL(file); + img.src = url; + preview.style.display = ''; + }); + + document.getElementById('wiki-foto-form')?.addEventListener('submit', async e => { + e.preventDefault(); + const input = document.getElementById('wiki-foto-input'); + const file = input?.files?.[0]; + if (!file) return; + + const btn = document.getElementById('wiki-foto-submit'); + btn.disabled = true; + btn.textContent = 'Wird hochgeladen…'; + + try { + const fd = new FormData(); + fd.append('file', file); + const token = localStorage.getItem('by_token'); + const resp = await fetch(`/api/wiki/rassen/${encodeURIComponent(slug)}/foto`, { + method: 'POST', + credentials: 'include', + headers: token ? { 'Authorization': `Bearer ${token}` } : {}, + body: fd, + }); + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + throw new Error(err.detail || 'Upload fehlgeschlagen'); + } + UI.modal.close(); + UI.toast('Danke! Dein Foto wird geprüft und dann veröffentlicht.', 'success'); + } catch (err) { + UI.toast(err.message, 'danger'); + btn.disabled = false; + btn.innerHTML = `${UI.icon('paper-plane-tilt')} Einreichen`; + } + }); } function _renderBerichteHtml(berichte, slug) { @@ -637,6 +818,21 @@ window.Page_wiki = (() => { return resp.json(); } + async function _apiPatch(url, body) { + const token = localStorage.getItem('by_token'); + const resp = await fetch(url, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json', ...(token ? { 'Authorization': `Bearer ${token}` } : {}) }, + credentials: 'include', + body: JSON.stringify(body), + }); + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + throw new Error(err.detail || `HTTP ${resp.status}`); + } + return resp.json(); + } + async function _apiPost(url, body) { const resp = await fetch(url, { method: 'POST', @@ -682,6 +878,6 @@ window.Page_wiki = (() => { // ---------------------------------------------------------- // PUBLIC // ---------------------------------------------------------- - return { init, refresh }; + return { init, refresh, _approveSubmission, _rejectSubmission }; })(); diff --git a/backend/static/sw.js b/backend/static/sw.js index cac16d4..de75268 100644 --- a/backend/static/sw.js +++ b/backend/static/sw.js @@ -3,7 +3,7 @@ Offline-Cache + Push Notifications + Tile-Cache ============================================================ */ -const CACHE_VERSION = 'by-v89'; +const CACHE_VERSION = 'by-v90'; const CACHE_STATIC = `${CACHE_VERSION}-static`; const CACHE_TILES = 'ban-yaro-tiles-v1'; // bleibt über SW-Updates erhalten