Feature: Tagebuch-Import (NoteStation .nsx + CSV) + Transponder in Gesundheitsdaten
- Import-Endpoint für Synology NoteStation (.nsx): HTML→Text, GPS, Bilder, Unix-Timestamp→Datum - Import-Endpoint für CSV (Komma/Semikolon, BOM-safe, DE-Datumsformat) - Import-Modal im Tagebuch mit Format-Auswahl-Karten und Ergebnis-Anzeige - Transpondernummer in Gesundheitsdaten: Anzeige + Inline-Edit via Modal - SW-Cache by-v142
This commit is contained in:
parent
6fcf841594
commit
94e0ed3daa
7 changed files with 505 additions and 5 deletions
270
backend/routes/import_data.py
Normal file
270
backend/routes/import_data.py
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
"""BAN YARO — Import: Synology NoteStation (.nsx) + CSV"""
|
||||
|
||||
import os, io, uuid, json, zipfile, csv, re, datetime
|
||||
from html.parser import HTMLParser
|
||||
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form
|
||||
from database import db
|
||||
from auth import get_current_user
|
||||
|
||||
router = APIRouter()
|
||||
MEDIA_DIR = os.getenv("MEDIA_DIR", "/data/media")
|
||||
|
||||
MAX_NSX_MB = 200
|
||||
MAX_CSV_MB = 10
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# HTML → Plaintext
|
||||
# ------------------------------------------------------------------
|
||||
class _HTMLStripper(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.parts = []
|
||||
|
||||
def handle_data(self, data):
|
||||
self.parts.append(data)
|
||||
|
||||
def handle_entityref(self, name):
|
||||
if name == "nbsp":
|
||||
self.parts.append(" ")
|
||||
|
||||
def handle_charref(self, name):
|
||||
if name in ("160", "xa0"):
|
||||
self.parts.append(" ")
|
||||
|
||||
|
||||
def _html_to_text(html: str) -> str:
|
||||
if not html:
|
||||
return ""
|
||||
# Replace block-level tags with newlines before stripping
|
||||
html = re.sub(r"</?(div|p|br|h[1-6]|li)[^>]*>", "\n", html, flags=re.IGNORECASE)
|
||||
s = _HTMLStripper()
|
||||
s.feed(html)
|
||||
text = "".join(s.parts)
|
||||
# Collapse excessive blank lines
|
||||
text = re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helfer: Bild aus ZIP in Media-Ordner speichern
|
||||
# ------------------------------------------------------------------
|
||||
def _save_image_from_zip(zf: zipfile.ZipFile, md5: str, mime: str) -> str | None:
|
||||
zip_name = f"file_{md5}"
|
||||
if zip_name not in zf.namelist():
|
||||
return None
|
||||
ext = mime.split("/")[-1] if mime else "jpg"
|
||||
ext = ext.replace("jpeg", "jpg")
|
||||
dest_dir = os.path.join(MEDIA_DIR, "diary")
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
filename = f"import_{uuid.uuid4().hex}.{ext}"
|
||||
dest = os.path.join(dest_dir, filename)
|
||||
with zf.open(zip_name) as src, open(dest, "wb") as dst:
|
||||
dst.write(src.read())
|
||||
return f"/media/diary/{filename}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# POST /api/import/notestation
|
||||
# ------------------------------------------------------------------
|
||||
@router.post("/notestation")
|
||||
async def import_notestation(
|
||||
dog_id: int = Form(...),
|
||||
file: UploadFile = File(...),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
if not file.filename.lower().endswith(".nsx"):
|
||||
raise HTTPException(400, "Bitte eine .nsx-Datei hochladen.")
|
||||
|
||||
raw = await file.read()
|
||||
if len(raw) > MAX_NSX_MB * 1024 * 1024:
|
||||
raise HTTPException(413, f"Datei zu groß (max {MAX_NSX_MB} MB).")
|
||||
|
||||
with db() as conn:
|
||||
dog = conn.execute(
|
||||
"SELECT id FROM dogs WHERE id=? AND user_id=?", (dog_id, user["id"])
|
||||
).fetchone()
|
||||
if not dog:
|
||||
raise HTTPException(404, "Hund nicht gefunden.")
|
||||
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(raw))
|
||||
except zipfile.BadZipFile:
|
||||
raise HTTPException(400, "Ungültige .nsx-Datei (kein ZIP).")
|
||||
|
||||
config = json.loads(zf.read("config.json"))
|
||||
note_ids = config.get("note", [])
|
||||
|
||||
imported = 0
|
||||
skipped = 0
|
||||
errors = []
|
||||
|
||||
with db() as conn:
|
||||
for nid in note_ids:
|
||||
try:
|
||||
note = json.loads(zf.read(nid))
|
||||
except Exception as e:
|
||||
errors.append(f"{nid}: Lesefehler – {e}")
|
||||
continue
|
||||
|
||||
if note.get("category") == "notebook":
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Datum aus ctime (Unix-Timestamp)
|
||||
ctime = note.get("ctime") or note.get("mtime")
|
||||
if ctime:
|
||||
datum = datetime.datetime.fromtimestamp(ctime).strftime("%Y-%m-%d")
|
||||
else:
|
||||
datum = datetime.date.today().isoformat()
|
||||
|
||||
titel = (note.get("title") or "").strip() or None
|
||||
text = _html_to_text(note.get("content", "")) or None
|
||||
lat = note.get("latitude") or None
|
||||
lon = note.get("longitude") or None
|
||||
# Koordinate (0,0) ist kein echter Ort
|
||||
if lat == 0.0:
|
||||
lat = None
|
||||
if lon == 0.0:
|
||||
lon = None
|
||||
tags = note.get("tag") or []
|
||||
|
||||
# Eintrag anlegen
|
||||
conn.execute(
|
||||
"""INSERT INTO diary (dog_id, datum, typ, titel, text, tags, gps_lat, gps_lon, is_milestone)
|
||||
VALUES (?, ?, 'eintrag', ?, ?, ?, ?, ?, 0)""",
|
||||
(dog_id, datum, titel, text, json.dumps(tags), lat, lon),
|
||||
)
|
||||
entry_id = conn.execute(
|
||||
"SELECT last_insert_rowid() AS id"
|
||||
).fetchone()["id"]
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO diary_dogs (diary_id, dog_id) VALUES (?,?)",
|
||||
(entry_id, dog_id),
|
||||
)
|
||||
|
||||
# Erstes Bild speichern
|
||||
attachments = note.get("attachment") or {}
|
||||
media_url = None
|
||||
for att in attachments.values():
|
||||
md5 = att.get("md5", "")
|
||||
mime = att.get("type", "image/jpeg")
|
||||
if not mime.startswith("image/"):
|
||||
continue
|
||||
media_url = _save_image_from_zip(zf, md5, mime)
|
||||
if media_url:
|
||||
break
|
||||
|
||||
if media_url:
|
||||
conn.execute(
|
||||
"UPDATE diary SET media_url=? WHERE id=?",
|
||||
(media_url, entry_id),
|
||||
)
|
||||
|
||||
imported += 1
|
||||
|
||||
return {"imported": imported, "skipped": skipped, "errors": errors}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# POST /api/import/csv
|
||||
# Spalten (Header-Zeile): datum, titel, text, tags, gps_lat, gps_lon, is_milestone
|
||||
# Trenner: Komma oder Semikolon | Encoding: UTF-8 (mit oder ohne BOM)
|
||||
# ------------------------------------------------------------------
|
||||
@router.post("/csv")
|
||||
async def import_csv(
|
||||
dog_id: int = Form(...),
|
||||
file: UploadFile = File(...),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
if not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(400, "Bitte eine .csv-Datei hochladen.")
|
||||
|
||||
raw = await file.read()
|
||||
if len(raw) > MAX_CSV_MB * 1024 * 1024:
|
||||
raise HTTPException(413, f"Datei zu groß (max {MAX_CSV_MB} MB).")
|
||||
|
||||
with db() as conn:
|
||||
dog = conn.execute(
|
||||
"SELECT id FROM dogs WHERE id=? AND user_id=?", (dog_id, user["id"])
|
||||
).fetchone()
|
||||
if not dog:
|
||||
raise HTTPException(404, "Hund nicht gefunden.")
|
||||
|
||||
text = raw.decode("utf-8-sig") # utf-8-sig entfernt BOM falls vorhanden
|
||||
# Semikolon oder Komma erkennen
|
||||
first_line = text.split("\n")[0]
|
||||
delimiter = ";" if first_line.count(";") >= first_line.count(",") else ","
|
||||
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter=delimiter)
|
||||
# Normalisierte Header (lowercase, strip)
|
||||
reader.fieldnames = [h.strip().lower() for h in (reader.fieldnames or [])]
|
||||
|
||||
REQUIRED = {"datum", "titel", "text"}
|
||||
if not REQUIRED.issubset(set(reader.fieldnames)):
|
||||
missing = REQUIRED - set(reader.fieldnames)
|
||||
raise HTTPException(
|
||||
400,
|
||||
f"CSV fehlen Pflicht-Spalten: {', '.join(sorted(missing))}. "
|
||||
"Erwartet: datum, titel, text (+ optional: tags, gps_lat, gps_lon, is_milestone)"
|
||||
)
|
||||
|
||||
imported = 0
|
||||
skipped = 0
|
||||
errors = []
|
||||
|
||||
with db() as conn:
|
||||
for i, row in enumerate(reader, start=2):
|
||||
try:
|
||||
datum = (row.get("datum") or "").strip()
|
||||
if not datum:
|
||||
skipped += 1
|
||||
continue
|
||||
# Datum normalisieren: DD.MM.YYYY → YYYY-MM-DD
|
||||
if re.match(r"^\d{1,2}\.\d{1,2}\.\d{4}$", datum):
|
||||
d, m, y = datum.split(".")
|
||||
datum = f"{y}-{int(m):02d}-{int(d):02d}"
|
||||
|
||||
titel = (row.get("titel") or "").strip() or None
|
||||
text = (row.get("text") or "").strip() or None
|
||||
|
||||
raw_tags = (row.get("tags") or "").strip()
|
||||
if raw_tags.startswith("["):
|
||||
tags = json.loads(raw_tags)
|
||||
else:
|
||||
tags = [t.strip() for t in re.split(r"[;,]", raw_tags) if t.strip()]
|
||||
|
||||
lat = None
|
||||
lon = None
|
||||
try:
|
||||
lat_s = (row.get("gps_lat") or "").strip().replace(",", ".")
|
||||
lon_s = (row.get("gps_lon") or "").strip().replace(",", ".")
|
||||
if lat_s:
|
||||
lat = float(lat_s)
|
||||
if lon_s:
|
||||
lon = float(lon_s)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
is_milestone = (row.get("is_milestone") or "").strip().lower() in (
|
||||
"1", "true", "ja", "yes"
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"""INSERT INTO diary (dog_id, datum, typ, titel, text, tags, gps_lat, gps_lon, is_milestone)
|
||||
VALUES (?, ?, 'eintrag', ?, ?, ?, ?, ?, ?)""",
|
||||
(dog_id, datum, titel, text, json.dumps(tags), lat, lon, int(is_milestone)),
|
||||
)
|
||||
entry_id = conn.execute(
|
||||
"SELECT last_insert_rowid() AS id"
|
||||
).fetchone()["id"]
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO diary_dogs (diary_id, dog_id) VALUES (?,?)",
|
||||
(entry_id, dog_id),
|
||||
)
|
||||
imported += 1
|
||||
|
||||
except Exception as e:
|
||||
errors.append(f"Zeile {i}: {e}")
|
||||
|
||||
return {"imported": imported, "skipped": skipped, "errors": errors}
|
||||
Loading…
Add table
Add a link
Reference in a new issue