Scraper: fetch_wiki_images mit Commons-Dateisuche (File-Namespace)
This commit is contained in:
parent
c3d33547c7
commit
93ea8a69fd
1 changed files with 177 additions and 101 deletions
|
|
@ -1,18 +1,20 @@
|
|||
"""
|
||||
BAN YARO — Fehlende Rassen-Fotos von Wikipedia/Wikimedia holen
|
||||
|
||||
Strategie:
|
||||
1. Alle Rassen ohne foto_url aus wiki_rassen holen
|
||||
2. Pro Rasse: Wikipedia pageimages API (de → en Fallback)
|
||||
3. Letzter Fallback: Wikimedia Commons pageimages API
|
||||
4. Sinnlose Bilder filtern (SVG, Flaggen-Icons, Karten, Logos)
|
||||
5. URL direkt in wiki_rassen.foto_url speichern
|
||||
Strategie (in Reihenfolge):
|
||||
1. Wikipedia pageimages DE (exakter Artikel-Treffer)
|
||||
2. Wikipedia pageimages EN
|
||||
3. Wikimedia Commons pageimages (exakter Artikel-Treffer)
|
||||
4. Wikimedia Commons Datei-Suche (action=query&list=search im File-Namespace)
|
||||
→ Sucht nach Bilddateien die den Rassenamen enthalten
|
||||
5. Gleiche Suche mit name_de (falls vorhanden)
|
||||
|
||||
Alle Bilder werden als externe URLs gespeichert (Wikimedia CDN).
|
||||
Lizenz: CC-BY-SA (Wikimedia Commons) — Attribution in Wiki-Seite anzeigen.
|
||||
|
||||
CLI-Optionen:
|
||||
--limit N Nur N Rassen bearbeiten (Default: 100)
|
||||
--dry-run Nur anzeigen, nicht speichern
|
||||
--model NAME Claude-Modell für ggf. zukünftige Text-Tasks
|
||||
(Default: claude-sonnet-4-6)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
|
@ -20,6 +22,7 @@ import asyncio
|
|||
import logging
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -48,11 +51,19 @@ _SKIP_PATTERNS = (
|
|||
"commons-logo",
|
||||
"question_mark",
|
||||
"noimage",
|
||||
"placeholder",
|
||||
"silhouette",
|
||||
"icon_",
|
||||
"_icon",
|
||||
"logo_",
|
||||
"_logo",
|
||||
)
|
||||
|
||||
# Suffixe die beim Normalisieren abgeschnitten werden
|
||||
_BREED_SUFFIXES = (" dog", " hound", " terrier", " spaniel", " shepherd")
|
||||
|
||||
|
||||
def _is_usable(url: str) -> bool:
|
||||
"""Gibt True zurück wenn die Bild-URL brauchbar erscheint."""
|
||||
low = url.lower()
|
||||
if low.endswith(".svg"):
|
||||
return False
|
||||
|
|
@ -62,11 +73,40 @@ def _is_usable(url: str) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
async def _fetch_wp_image(name: str, lang: str, client: httpx.AsyncClient) -> str | None:
|
||||
"""
|
||||
Fragt Wikipedia pageimages API für `name` in `lang` ab.
|
||||
Gibt Thumbnail-URL zurück oder None.
|
||||
"""
|
||||
def _name_variants(name: str, name_de: str | None) -> list[str]:
|
||||
"""Gibt Suchbegriff-Varianten zurück (dedupliziert, Reihenfolge bleibt)."""
|
||||
seen = set()
|
||||
result = []
|
||||
|
||||
def _add(n: str):
|
||||
n = n.strip()
|
||||
if n and n not in seen:
|
||||
seen.add(n)
|
||||
result.append(n)
|
||||
|
||||
_add(name)
|
||||
if name_de:
|
||||
_add(name_de)
|
||||
|
||||
# Ohne Klammern-Zusatz: "Foo (Bar)" → "Foo"
|
||||
if "(" in name:
|
||||
_add(name.split("(")[0].strip())
|
||||
|
||||
# Bindestrich → Leerzeichen
|
||||
_add(name.replace("-", " "))
|
||||
|
||||
# Suffix abschneiden
|
||||
low = name.lower()
|
||||
for suf in _BREED_SUFFIXES:
|
||||
if low.endswith(suf):
|
||||
_add(name[: -len(suf)].strip())
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def _wp_pageimages(name: str, lang: str, client: httpx.AsyncClient) -> str | None:
|
||||
"""Wikipedia pageimages API — gibt Thumbnail-URL oder None zurück."""
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"https://{lang}.wikipedia.org/w/api.php",
|
||||
|
|
@ -80,23 +120,19 @@ async def _fetch_wp_image(name: str, lang: str, client: httpx.AsyncClient) -> st
|
|||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
pages = resp.json().get("query", {}).get("pages", {})
|
||||
for page in pages.values():
|
||||
for page in resp.json().get("query", {}).get("pages", {}).values():
|
||||
if page.get("pageid", -1) == -1:
|
||||
continue
|
||||
thumb = page.get("thumbnail", {}).get("source", "")
|
||||
if thumb and _is_usable(thumb):
|
||||
return thumb
|
||||
except Exception as exc:
|
||||
logger.debug("WP pageimages (%s/%s) Fehler: %s", lang, name, exc)
|
||||
logger.debug("WP pageimages (%s/%s): %s", lang, name, exc)
|
||||
return None
|
||||
|
||||
|
||||
async def _fetch_commons_image(name: str, client: httpx.AsyncClient) -> str | None:
|
||||
"""
|
||||
Fragt Wikimedia Commons pageimages API für `name` ab.
|
||||
Wird als letzter Fallback genutzt.
|
||||
"""
|
||||
async def _commons_pageimages(name: str, client: httpx.AsyncClient) -> str | None:
|
||||
"""Wikimedia Commons pageimages API (exakter Artikel-Treffer)."""
|
||||
try:
|
||||
resp = await client.get(
|
||||
"https://commons.wikimedia.org/w/api.php",
|
||||
|
|
@ -109,23 +145,84 @@ async def _fetch_commons_image(name: str, client: httpx.AsyncClient) -> str | No
|
|||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
pages = resp.json().get("query", {}).get("pages", {})
|
||||
for page in pages.values():
|
||||
for page in resp.json().get("query", {}).get("pages", {}).values():
|
||||
if page.get("pageid", -1) == -1:
|
||||
continue
|
||||
thumb = page.get("thumbnail", {}).get("source", "")
|
||||
if thumb and _is_usable(thumb):
|
||||
return thumb
|
||||
except Exception as exc:
|
||||
logger.debug("Commons pageimages (%s) Fehler: %s", name, exc)
|
||||
logger.debug("Commons pageimages (%s): %s", name, exc)
|
||||
return None
|
||||
|
||||
|
||||
async def _commons_search(query: str, client: httpx.AsyncClient) -> str | None:
|
||||
"""
|
||||
Wikimedia Commons Datei-Suche im File-Namespace (6).
|
||||
Gibt Thumbnail-URL des ersten brauchbaren Treffers zurück.
|
||||
"""
|
||||
try:
|
||||
# Schritt 1: Dateinamen suchen
|
||||
resp = await client.get(
|
||||
"https://commons.wikimedia.org/w/api.php",
|
||||
params={
|
||||
"action": "query",
|
||||
"list": "search",
|
||||
"srsearch": query,
|
||||
"srnamespace": "6", # File-Namespace
|
||||
"srlimit": "5",
|
||||
"format": "json",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
hits = resp.json().get("query", {}).get("search", [])
|
||||
if not hits:
|
||||
return None
|
||||
|
||||
# Schritt 2: Für jeden Treffer imageinfo holen
|
||||
titles = "|".join(h["title"] for h in hits[:5])
|
||||
resp2 = await client.get(
|
||||
"https://commons.wikimedia.org/w/api.php",
|
||||
params={
|
||||
"action": "query",
|
||||
"titles": titles,
|
||||
"prop": "imageinfo",
|
||||
"iiprop": "url",
|
||||
"iiurlwidth": _THUMB_SIZE,
|
||||
"format": "json",
|
||||
},
|
||||
)
|
||||
resp2.raise_for_status()
|
||||
pages = resp2.json().get("query", {}).get("pages", {})
|
||||
|
||||
# Trefferqualität: bevorzuge Bilder die den Suchbegriff im Dateinamen haben
|
||||
query_lower = query.lower().replace(" ", "_")
|
||||
best: str | None = None
|
||||
|
||||
for page in pages.values():
|
||||
if page.get("pageid", -1) == -1:
|
||||
continue
|
||||
for ii in page.get("imageinfo", []):
|
||||
thumb = ii.get("thumburl") or ii.get("url", "")
|
||||
if not thumb or not _is_usable(thumb):
|
||||
continue
|
||||
fname = urllib.parse.unquote(thumb).lower()
|
||||
if query_lower in fname and best is None:
|
||||
best = thumb
|
||||
elif best is None:
|
||||
best = thumb # Fallback: erster brauchbarer Treffer
|
||||
|
||||
return best
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug("Commons search (%s): %s", query, exc)
|
||||
return None
|
||||
|
||||
|
||||
async def fetch_wiki_images(limit: int = 100, dry_run: bool = False) -> dict:
|
||||
"""
|
||||
Holt Wikipedia-Fotos für alle Rassen ohne foto_url.
|
||||
|
||||
Returns: {'found': int, 'saved': int, 'missing': int}
|
||||
Holt Fotos für alle Rassen ohne foto_url.
|
||||
Versucht mehrere Quellen und Namensvarianten.
|
||||
"""
|
||||
with db() as conn:
|
||||
rows = conn.execute(
|
||||
|
|
@ -142,54 +239,52 @@ async def fetch_wiki_images(limit: int = 100, dry_run: bool = False) -> dict:
|
|||
logger.info("Alle Rassen haben bereits ein Foto — nichts zu tun.")
|
||||
return {"found": 0, "saved": 0, "missing": 0}
|
||||
|
||||
logger.info("%d Rassen ohne Foto werden verarbeitet (limit=%d).", total, limit)
|
||||
|
||||
found = 0
|
||||
saved = 0
|
||||
logger.info("%d Rassen ohne Foto (limit=%d).", total, limit)
|
||||
found = saved = 0
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=12,
|
||||
follow_redirects=True,
|
||||
headers=_WP_HEADERS,
|
||||
timeout=15, follow_redirects=True, headers=_WP_HEADERS
|
||||
) as client:
|
||||
|
||||
for idx, row in enumerate(rows, start=1):
|
||||
row = dict(row)
|
||||
name = row["name"]
|
||||
name_de = row["name_de"] or ""
|
||||
slug = row["slug"] or name
|
||||
|
||||
# Suchreihenfolge: DE-Name → EN-Name → Commons mit EN-Name
|
||||
candidates: list[tuple[str, str]] = []
|
||||
|
||||
if name_de:
|
||||
candidates.append((name_de, "de"))
|
||||
candidates.append((name, "en"))
|
||||
if name_de:
|
||||
candidates.append((name_de, "en"))
|
||||
name_de = row.get("name_de") or ""
|
||||
variants = _name_variants(name, name_de or None)
|
||||
|
||||
foto_url: str | None = None
|
||||
source: str = ""
|
||||
|
||||
for search_name, lang in candidates:
|
||||
foto_url = await _fetch_wp_image(search_name, lang, client)
|
||||
# ── Stufe 1+2: Wikipedia pageimages DE / EN ──────────────────
|
||||
for lang in ("de", "en"):
|
||||
for variant in variants:
|
||||
foto_url = await _wp_pageimages(variant, lang, client)
|
||||
if foto_url:
|
||||
source = f"WP-{lang.upper()} ({variant})"
|
||||
break
|
||||
if foto_url:
|
||||
logger.info(
|
||||
"[%d/%d] ✓ %s → WP %s (%s)",
|
||||
idx, total, name, lang.upper(), search_name,
|
||||
)
|
||||
break
|
||||
|
||||
# Letzter Fallback: Wikimedia Commons
|
||||
# ── Stufe 3: Commons pageimages (exakter Treffer) ─────────────
|
||||
if not foto_url:
|
||||
foto_url = await _fetch_commons_image(name, client)
|
||||
for variant in variants:
|
||||
foto_url = await _commons_pageimages(variant, client)
|
||||
if foto_url:
|
||||
logger.info(
|
||||
"[%d/%d] ✓ %s → Commons", idx, total, name
|
||||
)
|
||||
source = f"Commons-exact ({variant})"
|
||||
break
|
||||
|
||||
# ── Stufe 4+5: Commons Datei-Suche ───────────────────────────
|
||||
if not foto_url:
|
||||
for variant in variants:
|
||||
foto_url = await _commons_search(variant, client)
|
||||
if foto_url:
|
||||
source = f"Commons-search ({variant})"
|
||||
break
|
||||
|
||||
if foto_url:
|
||||
found += 1
|
||||
if dry_run:
|
||||
logger.info(" [dry-run] würde setzen: %s", foto_url)
|
||||
else:
|
||||
logger.info("[%d/%d] ✓ %s → %s", idx, total, name, source)
|
||||
if not dry_run:
|
||||
try:
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
|
|
@ -198,19 +293,19 @@ async def fetch_wiki_images(limit: int = 100, dry_run: bool = False) -> dict:
|
|||
)
|
||||
saved += 1
|
||||
except Exception as exc:
|
||||
logger.error("DB-Update fehlgeschlagen für %s: %s", name, exc)
|
||||
logger.error("DB-Update %s: %s", name, exc)
|
||||
else:
|
||||
logger.info("[%d/%d] ✗ %s — kein Foto gefunden", idx, total, name)
|
||||
logger.info(" [dry-run] %s", foto_url)
|
||||
else:
|
||||
logger.info("[%d/%d] ✗ %s", idx, total, name)
|
||||
|
||||
# Rate-Limit: 1 Sekunde zwischen Anfragen
|
||||
await asyncio.sleep(1.0)
|
||||
await asyncio.sleep(0.8)
|
||||
|
||||
missing = total - found
|
||||
logger.info(
|
||||
"Fertig: %d/%d Fotos gefunden, %d gespeichert, %d ohne Treffer.",
|
||||
found, total, saved, missing,
|
||||
"Fertig: %d/%d gefunden, %d gespeichert, %d ohne Treffer.",
|
||||
found, total, saved, total - found,
|
||||
)
|
||||
return {"found": found, "saved": saved, "missing": missing}
|
||||
return {"found": found, "saved": saved, "missing": total - found}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -219,32 +314,13 @@ if __name__ == "__main__":
|
|||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Fehlende Rassen-Fotos von Wikipedia/Wikimedia holen"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=100,
|
||||
metavar="N",
|
||||
help="Maximale Anzahl Rassen (Default: 100)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Nur anzeigen, nicht in DB speichern",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="claude-sonnet-4-6",
|
||||
metavar="MODEL",
|
||||
help="Claude-Modell für Text-Tasks (Default: claude-sonnet-4-6)",
|
||||
)
|
||||
parser = argparse.ArgumentParser(description="Rassen-Fotos von Wikimedia holen")
|
||||
parser.add_argument("--limit", type=int, default=100, metavar="N")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.dry_run:
|
||||
logger.info("DRY-RUN Modus — keine DB-Änderungen.")
|
||||
logger.info("DRY-RUN — keine DB-Änderungen.")
|
||||
|
||||
result = asyncio.run(fetch_wiki_images(limit=args.limit, dry_run=args.dry_run))
|
||||
print(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue