Original commit record

Full diff

@@ -1,29 +1,57 @@\n-"""Deterministic topic-photo fallbacks for stories the strict matcher cannot cover.\n+"""Fallback photos for stories the strict matcher cannot cover.\n \n-This stage runs after images.py. It only fills articles whose ``image`` is\n-still empty, and only for explicitly mapped aviation topics with a vetted,\n-freely licensed Wikimedia Commons photograph. This preserves images.py's\n-strict event/airframe matching while preventing visually relevant policy or\n-regulatory stories from being left without a real illustration.\n+This stage runs after images.py and only fills articles whose ``image`` is\n+still empty. It has two responsibilities:\n \n-The site templates still provide the final SKYTICAL brand-image fallback, so\n-published pages never render an empty image slot even when no topic rule\n-matches.\n+1. deterministic, manually vetted topic photos for a small set of clearly\n+   identifiable aviation topics;\n+2. a conservative global-airport fallback driven by the article's verified\n+   ``entities.airports`` values. The Commons result must be freely licensed,\n+   bitmap-based, visually usable, and contain a meaningful airport-name token\n+   in the file title.\n+\n+The site templates remain the final brand-image fallback, so a transient\n+network failure can never leave a published article page with an empty visual\n+slot.\n """\n from __future__ import annotations\n \n import re\n import sys\n from datetime import timedelta\n+from html import unescape\n from pathlib import Path\n \n+import requests\n+\n sys.path.insert(0, str(Path(__file__).resolve().parent))\n-from common import ARTICLES_DIR, load_json, now_utc, parse_iso, save_json\n+from common import (\n+    ARTICLES_DIR,\n+    USER_AGENT,\n+    load_json,\n+    now_utc,\n+    parse_iso,\n+    save_json,\n+)\n \n MAX_ARTICLE_AGE_DAYS = 14\n-\n-# Every entry must be manually vetted for relevance and reusable licensing.\n-# Keep these generic: they are file photos, never represented as event photos.\n+TIMEOUT = (10, 30)\n+HEADERS = {"User-Agent": USER_AGENT}\n+COMMONS_API = "https://commons.wikimedia.org/w/api.php"\n+\n+_FREE_LICENSE_RE = re.compile(r"\b(cc[ -]|cc0|public domain|pd-)", re.I)\n+_BAD_TITLE_RE = re.compile(\n+    r"map|logo|diagram|seal|icon|flag|emblem|coat of arms|chart|plan\b",\n+    re.I,\n+)\n+_TAG_RE = re.compile(r"<[^>]+>")\n+_GENERIC_AIRPORT_WORDS = {\n+    "airport", "international", "intl", "aeroporto", "aeropuerto",\n+    "aéroport", "flughafen", "terminal", "airfield",\n+}\n+\n+# Every deterministic entry must be manually vetted for relevance and reuse.\n+# They are file photos, never represented as photos from the current event.\n _TOPIC_IMAGES = (\n     {\n         "name": "drone",\n@@ -56,12 +84,108 @@ def _article_text(article: dict) -> str:\n     return " ".join(parts)\n \n \n+def _strip_html(value: str) -> str:\n+    return re.sub(r"\s+", " ", _TAG_RE.sub(" ", unescape(value))).strip()\n+\n+\n+def _airport_entity(article: dict) -> str | None:\n+    entities = article.get("entities") or {}\n+    if not isinstance(entities, dict):\n+        return None\n+    airports = entities.get("airports") or []\n+    if not isinstance(airports, list):\n+        return None\n+    for value in airports:\n+        name = str(value or "").strip()\n+        if len(name) >= 3:\n+            return name\n+    return None\n+\n+\n+def _airport_title_tokens(name: str) -> list[str]:\n+    """Meaningful tokens that must identify the airport in a Commons title."""\n+    tokens = re.findall(r"[A-Za-zÀ-ÖØ-öø-ÿ0-9]+", name)\n+    meaningful = [\n+        token for token in tokens\n+        if len(token) >= 4 and token.casefold() not in _GENERIC_AIRPORT_WORDS\n+    ]\n+    if meaningful:\n+        return meaningful[:3]\n+    # CJK or an unusual short airport name: use the literal name as a strict\n+    # token rather than silently accepting an unrelated result.\n+    return [name]\n+\n+\n+def lookup_commons_airport(airport: str) -> dict | None:\n+    """Find a freely licensed file photo that visibly identifies ``airport``."""\n+    tokens = _airport_title_tokens(airport)\n+    resp = requests.get(\n+        COMMONS_API,\n+        headers=HEADERS,\n+        timeout=TIMEOUT,\n+        params={\n+            "action": "query",\n+            "format": "json",\n+            "generator": "search",\n+            "gsrnamespace": 6,\n+            "gsrlimit": 20,\n+            "gsrsearch": f"filetype:bitmap {airport}",\n+            "prop": "imageinfo",\n+            "iiprop": "url|extmetadata",\n+            "iiurlwidth": 1280,\n+        },\n+    )\n+    if resp.status_code != 200:\n+        return None\n+\n+    pages = ((resp.json() or {}).get("query") or {}).get("pages") or {}\n+    for page in sorted(pages.values(), key=lambda p: p.get("index", 99)):\n+        title = str(page.get("title") or "")\n+        title_low = title.casefold()\n+        if _BAD_TITLE_RE.search(title):\n+            continue\n+        if not any(token.casefold() in title_low for token in tokens):\n+            continue\n+        for info in page.get("imageinfo") or []:\n+            meta = info.get("extmetadata") or {}\n+            license_name = _strip_html(\n+                str((meta.get("LicenseShortName") or {}).get("value") or "")\n+            )\n+            if not _FREE_LICENSE_RE.search(license_name):\n+                continue\n+            thumb = info.get("thumburl") or info.get("url")\n+            if not thumb:\n+                continue\n+            artist = _strip_html(\n+                str((meta.get("Artist") or {}).get("value") or "")\n+            )[:80]\n+            return {\n+                "url": str(thumb),\n+                "link": str(\n+                    info.get("descriptionshorturl")\n+                    or info.get("descriptionurl")\n+                    or thumb\n+                ),\n+                "credit": artist or None,\n+                "license": license_name or None,\n+                "provider": "Wikimedia Commons",\n+                "kind": "file_photo",\n+                "matched": f"airport:{airport}",\n+                "subject": f"{airport} 資料照片",\n+            }\n+    return None\n+\n+\n def topic_image(article: dict) -> dict | None:\n-    """Return a vetted generic file photo for a clearly matched topic."""\n+    """Return a vetted topic image, then a strict verified-airport image."""\n     text = _article_text(article)\n     for rule in _TOPIC_IMAGES:\n         if any(pattern.search(text) for pattern in rule["patterns"]):\n             return dict(rule["image"])\n+\n+    airport = _airport_entity(article)\n+    if airport:\n+        return lookup_commons_airport(airport)\n     return None\n \n \n@@ -85,7 +209,17 @@ def main() -> int:\n                     continue\n             except (TypeError, ValueError):\n                 continue\n-            image = topic_image(article)\n+            try:\n+                image = topic_image(article)\n+            except Exception as exc:\n+                # Enhancement only: article publishing and the final brand\n+                # fallback must continue even when Commons is unavailable.\n+                print(\n+                    "image_fallbacks: lookup failed for "\n+                    f"{article.get('id') or path.name}: "\n+                    f"{type(exc).__name__}: {exc}"\n+                )\n+                continue\n             if not image:\n                 continue\n             article["image"] = image\n@@ -98,7 +232,7 @@ def main() -> int:\n         if changed:\n             save_json(path, batch)\n \n-    print(f"image_fallbacks: {attached} article(s) got a topic photo")\n+    print(f"image_fallbacks: {attached} article(s) got a fallback photo")\n     return 0\n \n \n@@ -1,7 +1,8 @@\n-"""Offline checks for pipeline/image_fallbacks.py."""\n+"""Offline regression checks for pipeline/image_fallbacks.py."""\n from __future__ import annotations\n \n import sys\n+import types\n from pathlib import Path\n \n REPO = Path(__file__).resolve().parent.parent\n@@ -10,14 +11,36 @@\n import image_fallbacks  # noqa: E402\n \n \n-def article(title: str, body: str = "") -> dict:\n+class FakeResp:\n+    def __init__(self, status=200, data=None):\n+        self.status_code = status\n+        self._data = data or {}\n+\n+    def json(self):\n+        return self._data\n+\n+\n+class FakeHttp:\n+    def __init__(self, response):\n+        self.response = response\n+        self.calls = []\n+\n+    def get(self, url, **kwargs):\n+        self.calls.append((url, kwargs))\n+        return self.response\n+\n+\n+def article(title: str, body: str = "", airport: str | None = None) -> dict:\n+    entities = {"airports": [airport]} if airport else {}\n     return {\n         "zh": {"title": title, "summary": body, "body": [body]},\n         "en": {"title": "", "summary": "", "body": []},\n+        "entities": entities,\n     }\n \n \n def main() -> int:\n+    # Existing deterministic topic fallback must remain unchanged.\n     drone = image_fallbacks.topic_image(\n         article("美國對無人機開徵232條款關稅", "一般商業無人機適用新稅率")\n     )\n@@ -34,12 +57,85 @@ def main() -> int:\n     assert english is not None\n     assert english["matched"] == "topic:drone"\n \n+    # Regression: the Etna story already exposes Catania airport as a verified\n+    # entity, so a non-Taiwan airport must now become a Commons lookup target.\n+    catania = article(\n+        "埃特納火山噴發影響 義大利卡塔尼亞機場關閉延長至週六",\n+        "埃特納火山噴發持續影響機場營運。",\n+        airport="Catania airport",\n+    )\n+    assert image_fallbacks._airport_entity(catania) == "Catania airport"\n+    assert image_fallbacks._airport_title_tokens("Catania airport") == ["Catania"]\n+\n+    good = FakeResp(200, {"query": {"pages": {\n+        "1": {\n+            "index": 0,\n+            "title": "File:Catania airport.jpg",\n+            "imageinfo": [{\n+                "thumburl": "https://upload.wikimedia.org/catania-airport.jpg",\n+                "descriptionshorturl": "https://commons.wikimedia.org/wiki/File:Catania_airport.jpg",\n+                "extmetadata": {\n+                    "LicenseShortName": {"value": "CC BY-SA 4.0"},\n+                    "Artist": {"value": "DornaXenia"},\n+                },\n+            }],\n+        },\n+    }}})\n+    fake = FakeHttp(good)\n+    original_requests = image_fallbacks.requests\n+    image_fallbacks.requests = types.SimpleNamespace(get=fake.get)\n+    try:\n+        matched = image_fallbacks.topic_image(catania)\n+    finally:\n+        image_fallbacks.requests = original_requests\n+    assert matched is not None\n+    assert matched["url"] == "https://upload.wikimedia.org/catania-airport.jpg"\n+    assert matched["matched"] == "airport:Catania airport"\n+    assert matched["license"] == "CC BY-SA 4.0"\n+    assert matched["kind"] == "file_photo"\n+    assert fake.calls[0][1]["params"]["gsrsearch"] == "filetype:bitmap Catania airport"\n+\n+    # Safety regression: reject branding, unrelated airports and non-free files.\n+    bad = FakeResp(200, {"query": {"pages": {\n+        "1": {\n+            "index": 0,\n+            "title": "File:Catania Airport logo.png",\n+            "imageinfo": [{\n+                "thumburl": "https://upload.wikimedia.org/logo.png",\n+                "extmetadata": {"LicenseShortName": {"value": "CC BY-SA 4.0"}},\n+            }],\n+        },\n+        "2": {\n+            "index": 1,\n+            "title": "File:Catania airport terminal.jpg",\n+            "imageinfo": [{\n+                "thumburl": "https://upload.wikimedia.org/nonfree.jpg",\n+                "extmetadata": {"LicenseShortName": {"value": "All rights reserved"}},\n+            }],\n+        },\n+        "3": {\n+            "index": 2,\n+            "title": "File:Palermo airport.jpg",\n+            "imageinfo": [{\n+                "thumburl": "https://upload.wikimedia.org/wrong-airport.jpg",\n+                "extmetadata": {"LicenseShortName": {"value": "CC BY 4.0"}},\n+            }],\n+        },\n+    }}})\n+    fake = FakeHttp(bad)\n+    image_fallbacks.requests = types.SimpleNamespace(get=fake.get)\n+    try:\n+        assert image_fallbacks.lookup_commons_airport("Catania airport") is None\n+    finally:\n+        image_fallbacks.requests = original_requests\n+\n     unrelated = image_fallbacks.topic_image(\n         article("航空公司公布季度財報", "營收與載客率同步成長")\n     )\n     assert unrelated is None\n \n-    # Returned mappings must be copies so callers cannot mutate the registry.\n+    # Returned deterministic mappings must be copies so callers cannot mutate\n+    # the registry for later articles.\n     drone["subject"] = "changed"\n     fresh = image_fallbacks.topic_image(article("無人機政策更新"))\n     assert fresh is not None