alpha01 backoffice: crossword engine, lexicon curation and JSON contract

enrich_review_from_wiktionary.py

@@ -0,0 +1,678 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import time
+import urllib.parse
+import urllib.request
+import urllib.error
+from copy import deepcopy
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Sequence, Tuple
+
+from refine_lexicon_topics import REFINED_LEXICON_OUTPUT_PATH
+
+
+REVIEW_INPUT_PATH = Path(__file__).with_name("to_be_review.json")
+WIKTIONARY_CACHE_PATH = Path(__file__).with_name(".wiktionary_cache.json")
+WIKTIONARY_OUTPUT_PATH = Path(__file__).with_name("lexicon_it_refined_plus_wiktionary.json")
+WIKTIONARY_API_URL = "https://it.wiktionary.org/w/api.php"
+
+DEFAULT_REVIEW_REASONS = {"no_viable_definition", "only_general_topics", "babelnet_ambiguous"}
+
+POS_ALIASES = {
+    "sostantivo": "NOUN",
+    "nome": "NOUN",
+    "sost": "NOUN",
+    "aggettivo": "ADJ",
+    "agg": "ADJ",
+    "verbo": "VERB",
+    "verb": "VERB",
+    "verb form": "VERB_FORM",
+    "avverbio": "ADV",
+    "avv": "ADV",
+    "preposizione": "PREP",
+    "prep": "PREP",
+    "congiunzione": "CONJ",
+    "cong": "CONJ",
+    "pronome": "PRON",
+    "pron": "PRON",
+    "articolo": "ART",
+    "interiezione": "INTJ",
+    "inter": "INTJ",
+    "locuzione": "PHRASE",
+    "loc": "PHRASE",
+}
+
+TOPIC_KEYWORDS = {
+    "religion": ("religione", "cattolic", "sacro", "sacra", "devozion", "scapolare", "abbazia", "monastero"),
+    "clothing": ("abito", "vestito", "vestit", "abbigliamento", "indumento", "stoffa"),
+    "grammar": ("diminutivo", "voce verbale", "congiuntivo", "plurale", "singolare", "grammatica", "verbo"),
+    "geography": ("comune", "paese", "regione", "provincia", "citta", "localita", "frazione"),
+    "transport": ("veicolo", "motore", "treno", "aereo", "trasporto", "nave", "imbarcazione"),
+    "health": ("medicina", "ospedale", "malattia", "cura", "feriti", "ammalati", "sanitario"),
+}
+
+GRAMMAR_KEYWORDS = (
+    "diminutivo",
+    "accrescitivo",
+    "peggiorativo",
+    "alterato",
+    "voce verbale",
+    "congiuntivo",
+    "participio",
+    "plurale",
+    "singolare",
+    "maschile",
+    "femminile",
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Arricchisce le voci problematiche del lessico refined con definizioni e metadati "
+            "estratti da it.wiktionary.org."
+        )
+    )
+    parser.add_argument(
+        "--input",
+        type=Path,
+        default=REFINED_LEXICON_OUTPUT_PATH,
+        help="Lessico refined di partenza.",
+    )
+    parser.add_argument(
+        "--review",
+        type=Path,
+        default=REVIEW_INPUT_PATH,
+        help="File to_be_review.json da usare per selezionare le voci prioritarie.",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=WIKTIONARY_OUTPUT_PATH,
+        help="Nuovo lessico con blocco wiktionary aggiunto.",
+    )
+    parser.add_argument(
+        "--cache",
+        type=Path,
+        default=WIKTIONARY_CACHE_PATH,
+        help="Cache locale delle risposte Wiktionary.",
+    )
+    parser.add_argument(
+        "--word-limit",
+        type=int,
+        default=0,
+        help="Limite massimo di parole da elaborare. 0 = tutte le candidate.",
+    )
+    parser.add_argument(
+        "--sleep",
+        type=float,
+        default=1.0,
+        help="Pausa tra le richieste HTTP a Wiktionary.",
+    )
+    parser.add_argument(
+        "--save-every",
+        type=int,
+        default=25,
+        help="Salva cache e output ogni N parole elaborate per non perdere progresso.",
+    )
+    parser.add_argument(
+        "--retry-429",
+        type=int,
+        default=3,
+        help="Numero massimo di tentativi aggiuntivi se Wiktionary risponde HTTP 429.",
+    )
+    parser.add_argument(
+        "--backoff-429",
+        type=float,
+        default=30.0,
+        help="Secondi di attesa iniziali dopo un HTTP 429; raddoppiano a ogni nuovo tentativo.",
+    )
+    parser.add_argument(
+        "--stop-on-429",
+        action="store_true",
+        help="Se attivo, al primo HTTP 429 salva lo stato e interrompe il batch senza altri tentativi.",
+    )
+    parser.add_argument(
+        "--words",
+        default="",
+        help="Lista separata da virgole di lemmi specifici da arricchire.",
+    )
+    parser.add_argument(
+        "--review-reasons",
+        default=",".join(sorted(DEFAULT_REVIEW_REASONS)),
+        help="Motivi del file review da trattare con priorita, separati da virgole.",
+    )
+    parser.add_argument(
+        "--api-url",
+        default=WIKTIONARY_API_URL,
+        help="Endpoint MediaWiki Action API di Wiktionary.",
+    )
+    parser.add_argument(
+        "--skip-existing",
+        action="store_true",
+        help="Salta le voci che nel lessico di input hanno già un blocco wiktionary con stato utile.",
+    )
+    return parser.parse_args()
+
+
+def load_json(path: Path, default: object) -> object:
+    if not path.exists():
+        return default
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def write_json(path: Path, payload: object) -> None:
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def parse_csv_set(value: str) -> set[str]:
+    return {item.strip().lower() for item in str(value or "").split(",") if item.strip()}
+
+
+def entry_key(entry: Dict[str, object]) -> Tuple[str, str]:
+    form = str(entry.get("normalized_form") or entry.get("form") or "").strip().lower()
+    pos = str(entry.get("pos") or "").strip().upper()
+    return form, pos
+
+
+def fetch_wikitext(title: str, api_url: str) -> Dict[str, object]:
+    params = {
+        "action": "query",
+        "prop": "revisions",
+        "titles": title,
+        "rvprop": "content",
+        "rvslots": "main",
+        "formatversion": "2",
+        "format": "json",
+    }
+    url = f"{api_url}?{urllib.parse.urlencode(params)}"
+    request = urllib.request.Request(
+        url,
+        headers={
+            "User-Agent": "cruciverba-alpha/0.1 (local lexical enrichment)",
+            "Accept": "application/json",
+        },
+    )
+    with urllib.request.urlopen(request, timeout=30) as response:
+        payload = json.loads(response.read().decode("utf-8"))
+    pages = ((payload.get("query") or {}).get("pages") or [])
+    if not pages:
+        return {"status": "missing"}
+    page = pages[0]
+    if page.get("missing"):
+        return {"status": "missing", "title": page.get("title", title)}
+    revisions = page.get("revisions") or []
+    content = ""
+    if revisions:
+        slots = revisions[0].get("slots") or {}
+        main_slot = slots.get("main") or {}
+        content = str(main_slot.get("content") or "")
+    return {
+        "status": "ok" if content else "empty",
+        "title": page.get("title", title),
+        "pageid": page.get("pageid"),
+        "wikitext": content,
+    }
+
+
+def fetch_wikitext_with_retry(title: str, args: argparse.Namespace) -> Dict[str, object]:
+    attempts = 0
+    delay = max(1.0, float(args.backoff_429))
+    while True:
+        try:
+            return fetch_wikitext(title, args.api_url)
+        except urllib.error.HTTPError as exc:
+            if exc.code != 429:
+                raise
+            if args.stop_on_429:
+                raise
+            if attempts >= max(0, int(args.retry_429)):
+                raise
+            attempts += 1
+            print(f"[429] {title}: attendo {delay:.1f}s prima del tentativo {attempts}/{args.retry_429}")
+            time.sleep(delay)
+            delay *= 2
+
+
+def normalize_heading(text: str) -> str:
+    raw = str(text or "").strip().lower().replace(" ", "")
+    if raw == "{{-it-}}":
+        return "{{-it-}}"
+    cleaned = strip_wikicode(text).strip().lower()
+    return cleaned
+
+
+def extract_italian_section(wikitext: str) -> str:
+    section_pattern = re.compile(r"^==\s*(.*?)\s*==\s*$", re.MULTILINE)
+    matches = list(section_pattern.finditer(wikitext))
+    for index, match in enumerate(matches):
+        raw_heading = str(match.group(1) or "").strip().lower().replace(" ", "")
+        heading = normalize_heading(match.group(1))
+        if raw_heading == "{{-it-}}" or heading in {"italiano", "it"}:
+            start = match.end()
+            end = matches[index + 1].start() if index + 1 < len(matches) else len(wikitext)
+            return wikitext[start:end]
+    return ""
+
+
+def strip_templates(text: str) -> str:
+    previous = None
+    current = text
+    while previous != current:
+        previous = current
+        current = re.sub(r"\{\{([^{}|]+)\|([^{}]+?)\}\}", r"\2", current)
+        current = re.sub(r"\{\{[^{}]+\}\}", "", current)
+    return current
+
+
+def strip_wikicode(text: str) -> str:
+    value = str(text or "")
+    value = re.sub(r"<!--.*?-->", " ", value, flags=re.DOTALL)
+    value = re.sub(r"<ref[^>]*>.*?</ref>", " ", value, flags=re.DOTALL)
+    value = re.sub(r"<[^>]+>", " ", value)
+    value = strip_templates(value)
+    value = re.sub(r"\[\[([^|\]]+)\|([^\]]+)\]\]", r"\2", value)
+    value = re.sub(r"\[\[([^\]]+)\]\]", r"\1", value)
+    value = value.replace("'''", "").replace("''", "")
+    value = value.replace("&nbsp;", " ")
+    value = re.sub(r"\s+", " ", value)
+    return value.strip(" .;:-")
+
+
+def infer_topics(definitions: Sequence[str], categories: Sequence[str]) -> List[str]:
+    text = " ".join(definitions + list(categories)).lower()
+    topics = []
+    for topic, keywords in TOPIC_KEYWORDS.items():
+        if any(keyword in text for keyword in keywords):
+            topics.append(topic)
+    return sorted(set(topics))
+
+
+def infer_grammar_hints(definitions: Sequence[str], raw_section: str) -> List[str]:
+    text = f"{' '.join(definitions)} {raw_section}".lower()
+    hints = []
+    for keyword in GRAMMAR_KEYWORDS:
+        if keyword in text:
+            hints.append(keyword)
+    return sorted(set(hints))
+
+
+def detect_pos_from_heading(heading: str) -> Optional[str]:
+    normalized = normalize_heading(heading)
+    if not normalized:
+        return None
+    for label, pos in sorted(POS_ALIASES.items(), key=lambda item: len(item[0]), reverse=True):
+        if label in normalized:
+            return pos
+    return None
+
+
+def parse_template_marker(line: str) -> Tuple[Optional[str], Optional[str]]:
+    stripped = line.strip()
+    match = re.match(r"^\{\{-([^{}|]+?)-?(?:\|.*)?\}\}$", stripped, flags=re.IGNORECASE)
+    if not match:
+        return None, None
+    marker = match.group(1).strip().lower()
+    if marker == "it":
+        return "language", "it"
+    for label, pos in sorted(POS_ALIASES.items(), key=lambda item: len(item[0]), reverse=True):
+        if marker.startswith(label):
+            return "pos", pos
+    if marker.startswith("sinon"):
+        return "subsection", "sinonimi"
+    if marker.startswith(("etim", "trad", "sill", "pron", "var", "note")):
+        return "subsection", marker
+    return "subsection", marker
+
+
+def parse_wiktionary_section(section_text: str) -> Dict[str, object]:
+    lines = section_text.splitlines()
+    entries: List[Dict[str, object]] = []
+    categories: List[str] = []
+    current: Optional[Dict[str, object]] = None
+    current_subsection = ""
+
+    heading_pattern = re.compile(r"^(={3,4})\s*(.*?)\s*\1\s*$")
+
+    for raw_line in lines:
+        line = raw_line.rstrip()
+        if not line:
+            continue
+
+        for category_match in re.findall(r"\[\[Categoria:([^\]]+)\]\]", line):
+            categories.append(strip_wikicode(category_match))
+
+        marker_kind, marker_value = parse_template_marker(line)
+        if marker_kind == "pos":
+            current = {
+                "pos": marker_value,
+                "heading": marker_value,
+                "definitions": [],
+                "examples": [],
+                "synonyms": [],
+            }
+            entries.append(current)
+            current_subsection = ""
+            continue
+        if marker_kind == "subsection":
+            current_subsection = str(marker_value or "")
+            continue
+
+        heading_match = heading_pattern.match(line)
+        if heading_match:
+            level = len(heading_match.group(1))
+            heading = heading_match.group(2)
+            if level == 3:
+                pos = detect_pos_from_heading(heading)
+                if pos:
+                    current = {
+                        "pos": pos,
+                        "heading": strip_wikicode(heading),
+                        "definitions": [],
+                        "examples": [],
+                        "synonyms": [],
+                    }
+                    entries.append(current)
+                    current_subsection = ""
+                    continue
+            current_subsection = normalize_heading(heading)
+            continue
+
+        if current is None:
+            continue
+
+        stripped = line.lstrip()
+        if stripped.startswith("#") and not stripped.startswith(("#:", "#*", "#;")):
+            definition = strip_wikicode(stripped.lstrip("#").strip())
+            if definition:
+                current["definitions"].append(definition)
+            continue
+
+        if stripped.startswith("#:") or stripped.startswith("#*"):
+            example = strip_wikicode(stripped[2:].strip())
+            if example:
+                current["examples"].append(example)
+            continue
+
+        if current_subsection.startswith("sinonim") and stripped.startswith("*"):
+            synonym = strip_wikicode(stripped.lstrip("*").strip())
+            if synonym:
+                current["synonyms"].append(synonym)
+
+    flat_definitions = [definition for entry in entries for definition in entry["definitions"]]
+    topic_hints = infer_topics(flat_definitions, categories)
+    grammar_hints = infer_grammar_hints(flat_definitions, section_text)
+
+    return {
+        "entries": entries,
+        "categories": sorted(set(filter(None, categories))),
+        "definitions": flat_definitions,
+        "topic_hints": topic_hints,
+        "grammar_hints": grammar_hints,
+    }
+
+
+def wiktionary_payload_for_entry(entry: Dict[str, object], api_response: Dict[str, object]) -> Dict[str, object]:
+    status = str(api_response.get("status", "missing"))
+    if status != "ok":
+        return {
+            "status": status,
+            "matched": False,
+            "page_title": api_response.get("title") or entry.get("form"),
+            "source_url": f"https://it.wiktionary.org/wiki/{urllib.parse.quote(str(entry.get('form', '')))}",
+            "definitions": [],
+            "entries": [],
+            "topic_hints": [],
+            "grammar_hints": [],
+            "categories": [],
+        }
+
+    italian_section = extract_italian_section(str(api_response.get("wikitext") or ""))
+    if not italian_section:
+        return {
+            "status": "no_italian_section",
+            "matched": False,
+            "page_title": api_response.get("title") or entry.get("form"),
+            "source_url": f"https://it.wiktionary.org/wiki/{urllib.parse.quote(str(api_response.get('title') or entry.get('form', '')))}",
+            "definitions": [],
+            "entries": [],
+            "topic_hints": [],
+            "grammar_hints": [],
+            "categories": [],
+        }
+
+    parsed = parse_wiktionary_section(italian_section)
+    matched = bool(parsed["definitions"])
+
+    return {
+        "status": "enriched" if matched else "section_without_definitions",
+        "matched": matched,
+        "page_title": api_response.get("title") or entry.get("form"),
+        "pageid": api_response.get("pageid"),
+        "source_url": f"https://it.wiktionary.org/wiki/{urllib.parse.quote(str(api_response.get('title') or entry.get('form', '')))}",
+        "definitions": parsed["definitions"],
+        "entries": parsed["entries"],
+        "topic_hints": parsed["topic_hints"],
+        "grammar_hints": parsed["grammar_hints"],
+        "categories": parsed["categories"],
+        "raw_excerpt": italian_section[:4000],
+    }
+
+
+def select_targets(
+    refined_payload: Dict[str, object],
+    review_payload: Dict[str, object],
+    review_reasons: set[str],
+    explicit_words: set[str],
+    word_limit: int,
+    skip_existing: bool,
+) -> Tuple[List[Dict[str, object]], int]:
+    refined_entries = [entry for entry in refined_payload.get("entries", []) or [] if isinstance(entry, dict)]
+    refined_by_word = {str(entry.get("form", "")).lower(): entry for entry in refined_entries if entry.get("form")}
+
+    if explicit_words:
+        selected = []
+        skipped_existing_count = 0
+        for word in explicit_words:
+            entry = refined_by_word.get(word)
+            if entry is None:
+                continue
+            if skip_existing and wiktionary_already_useful(entry):
+                skipped_existing_count += 1
+                continue
+            selected.append(entry)
+        selected = selected[:word_limit] if word_limit > 0 else selected
+        return selected, skipped_existing_count
+
+    review_entries = [entry for entry in review_payload.get("entries", []) or [] if isinstance(entry, dict)]
+    selected_words: List[str] = []
+    seen = set()
+    skipped_existing_count = 0
+
+    for review_entry in review_entries:
+        word = str(review_entry.get("form", "")).strip().lower()
+        if not word or word in seen:
+            continue
+        reasons = {str(item).lower() for item in review_entry.get("review_reasons", []) or []}
+        refined = refined_by_word.get(word)
+        if refined is None:
+            continue
+        if skip_existing and wiktionary_already_useful(refined):
+            skipped_existing_count += 1
+            continue
+        babelnet_status = str((refined.get("babelnet") or {}).get("status", "")).lower()
+        if reasons.intersection(review_reasons) or babelnet_status == "no_match":
+            selected_words.append(word)
+            seen.add(word)
+            if word_limit > 0 and len(selected_words) >= word_limit:
+                break
+
+    return [refined_by_word[word] for word in selected_words if word in refined_by_word], skipped_existing_count
+
+
+def wiktionary_already_useful(entry: Dict[str, object]) -> bool:
+    wiktionary = entry.get("wiktionary", {})
+    if not isinstance(wiktionary, dict):
+        return False
+    status = str(wiktionary.get("status", "")).lower()
+    if status == "enriched" and (wiktionary.get("definitions") or wiktionary.get("entries")):
+        return True
+    if status in {"missing", "no_italian_section", "section_without_definitions", "empty"}:
+        return True
+    return False
+
+
+def enrich_from_wiktionary(args: argparse.Namespace) -> Dict[str, object]:
+    refined_payload = load_json(args.input, {"entries": []})
+    if not isinstance(refined_payload, dict) or "entries" not in refined_payload:
+        raise ValueError(f"Lessico refined non valido: {args.input}")
+
+    review_payload = load_json(args.review, {"entries": []})
+    if not isinstance(review_payload, dict):
+        review_payload = {"entries": []}
+
+    cache = load_json(args.cache, {})
+    if not isinstance(cache, dict):
+        cache = {}
+
+    targets, skipped_existing_count = select_targets(
+        refined_payload,
+        review_payload,
+        parse_csv_set(args.review_reasons),
+        parse_csv_set(args.words),
+        args.word_limit,
+        args.skip_existing,
+    )
+
+    enriched_entries = []
+    cache_hits = 0
+    network_calls = 0
+    network_attempts = 0
+    processed_count = 0
+    stopped_reason = None
+    stop_word = None
+
+    print(
+        f"Target selezionati: {len(targets)}"
+        + (f" | già saltati per wiktionary esistente: {skipped_existing_count}" if args.skip_existing else "")
+    )
+
+    def persist_progress() -> None:
+        refined_index = {
+            entry_key(entry): entry
+            for entry in refined_payload.get("entries", []) or []
+            if isinstance(entry, dict)
+        }
+        for item in enriched_entries:
+            refined_index[entry_key(item)] = item
+
+        merged_entries = list(refined_index.values())
+        merged_entries.sort(key=lambda item: (str(item.get("normalized_form", "")), str(item.get("pos", ""))))
+
+        merged_payload = {
+            "meta": {
+                **(refined_payload.get("meta", {}) if isinstance(refined_payload.get("meta"), dict) else {}),
+                "wiktionary_source": args.api_url,
+                "wiktionary_generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
+                "wiktionary_target_count": len(targets),
+                "wiktionary_processed_count": processed_count,
+                "wiktionary_skipped_existing_count": skipped_existing_count,
+                "wiktionary_cache_hits": cache_hits,
+                "wiktionary_network_calls": network_calls,
+                "wiktionary_network_attempts": network_attempts,
+                "wiktionary_stopped_reason": stopped_reason,
+                "wiktionary_stop_word": stop_word,
+            },
+            "entries": merged_entries,
+        }
+
+        write_json(args.cache, cache)
+        write_json(args.output, merged_payload)
+
+    for index, entry in enumerate(targets, start=1):
+        updated = deepcopy(entry)
+        word = str(entry.get("form", "")).strip()
+        cache_key = word.lower()
+
+        if cache_key in cache:
+            api_response = cache[cache_key]
+            cache_hits += 1
+        else:
+            try:
+                network_attempts += 1
+                api_response = fetch_wikitext_with_retry(word, args)
+            except urllib.error.HTTPError as exc:
+                if exc.code == 429:
+                    stop_word = word
+                    stopped_reason = f"http_429_after_{processed_count}_words"
+                    print(f"[STOP] Wiktionary ha risposto 429 su '{word}'. Salvo il progresso e interrompo il batch.")
+                    persist_progress()
+                    return {
+                        "target_count": len(targets),
+                        "processed_count": processed_count,
+                        "skipped_existing_count": skipped_existing_count,
+                        "cache_hits": cache_hits,
+                        "network_calls": network_calls,
+                        "network_attempts": network_attempts,
+                        "output": str(args.output),
+                        "stopped_reason": stopped_reason,
+                        "stop_word": stop_word,
+                    }
+                raise
+            cache[cache_key] = api_response
+            network_calls += 1
+            if args.sleep > 0:
+                time.sleep(args.sleep)
+
+        updated["wiktionary"] = wiktionary_payload_for_entry(updated, api_response)
+        updated["wiktionary_generated_at"] = datetime.now().astimezone().isoformat(timespec="seconds")
+        enriched_entries.append(updated)
+        processed_count += 1
+        print(
+            f"[{index}/{len(targets)}] {word}: "
+            f"status={updated['wiktionary'].get('status')} "
+            f"def={len(updated['wiktionary'].get('definitions', []))} "
+            f"topics={len(updated['wiktionary'].get('topic_hints', []))}"
+        )
+        if args.save_every > 0 and processed_count % int(args.save_every) == 0:
+            persist_progress()
+            print(f"[save] progresso salvato dopo {processed_count} parole")
+
+    persist_progress()
+
+    return {
+        "target_count": len(targets),
+        "processed_count": processed_count,
+        "skipped_existing_count": skipped_existing_count,
+        "cache_hits": cache_hits,
+        "network_calls": network_calls,
+        "network_attempts": network_attempts,
+        "output": str(args.output),
+        "stopped_reason": stopped_reason,
+        "stop_word": stop_word,
+    }
+
+
+def main() -> None:
+    args = parse_args()
+    result = enrich_from_wiktionary(args)
+    print(f"Lessico con Wiktionary generato: {result['output']}")
+    print(f"Voci trattate: {result.get('processed_count', result['target_count'])}/{result['target_count']}")
+    if "skipped_existing_count" in result:
+        print(f"Voci già saltate: {result['skipped_existing_count']}")
+    print(f"Cache hit: {result['cache_hits']}")
+    print(f"Chiamate rete: {result['network_calls']}")
+    if "network_attempts" in result:
+        print(f"Tentativi di rete: {result['network_attempts']}")
+    if result.get("stopped_reason"):
+        print(f"Batch interrotto: {result['stopped_reason']}")
+    if result.get("stop_word"):
+        print(f"Ultima parola bloccante: {result['stop_word']}")
+
+
+if __name__ == "__main__":
+    main()