Original commit record

Full diff

@@ -13,6 +13,8 @@ on:\n       - "pipeline/flightnews.py"\n       - "pipeline/write.py"\n       - "pipeline/provider_health.py"\n+      - "pipeline/article_schema.py"\n+      - "pipeline/build.py"\n       - "pipeline/common.py"\n       - "pipeline/dedupe.py"\n       - "pipeline/fulltext.py"\n@@ -58,6 +60,7 @@ jobs:\n           python tests/test_dedupe.py\n           python tests/test_write_offline.py\n           python tests/test_provider_health.py\n+          python tests/test_build_resilience.py\n           python tests/test_news_freshness.py\n \n   update:\n@@ -14,6 +14,7 @@ on:\n       - "pipeline/common.py"\n       - "pipeline/radar_snapshot.py"\n       - "pipeline/build.py"\n+      - "pipeline/article_schema.py"\n       - "static/radar.js"\n       - "templates/radar.html"\n       - "tests/test_radar.py"\n@@ -26,6 +27,7 @@ on:\n       - "pipeline/common.py"\n       - "pipeline/radar_snapshot.py"\n       - "pipeline/build.py"\n+      - "pipeline/article_schema.py"\n       - "static/radar.js"\n       - "templates/radar.html"\n \n@@ -48,6 +50,7 @@ jobs:\n       - name: Run radar regression tests\n         run: |\n           python tests/test_radar_snapshot.py\n+          python tests/test_build_resilience.py\n           python tests/test_radar.py\n \n   build:\n@@ -0,0 +1,81 @@\n+"""Read-boundary normalization for persisted SKYTICAL article records.\n+\n+Historical JSON is append-only and spans multiple schema generations.  This\n+module turns any dictionary-shaped article row into the current read contract\n+without rewriting the archive on disk. Required publication identity and\n+source fields are deliberately not invented; build.py still rejects rows that\n+lack those essentials.\n+"""\n+from __future__ import annotations\n+\n+from copy import deepcopy\n+\n+ARTICLE_SCHEMA_VERSION = 1\n+\n+_OPTIONAL_LIST_FIELDS = (\n+    "availableLanguages",\n+    "writerModels",\n+    "sourceImageCandidates",\n+    "facts",\n+    "riskFlags",\n+    "entityEvidence",\n+)\n+\n+\n+def _language_block(value) -> dict:\n+    block = deepcopy(value) if isinstance(value, dict) else {}\n+    body = block.get("body")\n+    if isinstance(body, str):\n+        body = [body]\n+    elif not isinstance(body, list):\n+        body = []\n+    return {\n+        **block,\n+        "title": str(block.get("title") or ""),\n+        "summary": str(block.get("summary") or ""),\n+        "body": [str(item) for item in body if str(item).strip()],\n+    }\n+\n+\n+def normalize_article_record(raw) -> dict | None:\n+    """Return a current-schema copy of one persisted article row.\n+\n+    Optional fields always exist after this boundary. The function never\n+    mutates the caller's object and never fabricates required identity,\n+    timestamp, or source evidence.\n+    """\n+    if not isinstance(raw, dict):\n+        return None\n+\n+    article = deepcopy(raw)\n+    article["schemaVersion"] = ARTICLE_SCHEMA_VERSION\n+    article["archived"] = article.get("archived") is True\n+    article["primarySource"] = str(article.get("primarySource") or "")\n+    article["image"] = article.get("image")\n+    article["zh"] = _language_block(article.get("zh"))\n+    article["en"] = _language_block(article.get("en"))\n+\n+    sources = article.get("sources")\n+    article["sources"] = list(sources) if isinstance(sources, list) else []\n+\n+    for key in _OPTIONAL_LIST_FIELDS:\n+        value = article.get(key)\n+        article[key] = list(value) if isinstance(value, list) else []\n+\n+    entities = article.get("entities")\n+    article["entities"] = deepcopy(entities) if isinstance(entities, dict) else {}\n+    airlines = article["entities"].get("airlines")\n+    article["entities"]["airlines"] = (\n+        list(airlines) if isinstance(airlines, list) else []\n+    )\n+\n+    article["writer"] = (\n+        str(article.get("writer")) if isinstance(article.get("writer"), str)\n+        else None\n+    )\n+    article["articleFormat"] = (\n+        article.get("articleFormat")\n+        if article.get("articleFormat") in ("full", "brief", "roundup")\n+        else "full"\n+    )\n+    return article\n@@ -5,8 +5,9 @@\n CONTRACTS.md: zh at /, en under /en/, one page per article ever published,\n incidents / sources / methodology / changelog pages, copied assets, 404.html\n and .nojekyll.\n-A single bad input never crashes the build: bad entries are skipped with a\n-one-line log to stdout and the script exits 0.\n+Isolated bad inputs degrade gracefully and are skipped with a one-line log.\n+A bounded render-failure budget prevents a systemic schema/template regression\n+from publishing a substantially incomplete site.\n """\n from __future__ import annotations\n \n@@ -56,6 +57,7 @@\n     MODEL_ORDER,\n )\n from image_captions import clean_description\n+from article_schema import normalize_article_record\n \n _caption_cache = load_json(DATA_DIR / "image-captions.json", {})\n IMAGE_CAPTIONS = (_caption_cache.get("images", {})\n@@ -67,6 +69,22 @@\n PUBLIC_DIR = PROJECT_DIR / "public"\n CONFIG_DIR = PROJECT_DIR / "config"\n \n+# Render failures are tolerated only while they remain genuinely isolated.\n+# Exceeding either bound is treated as systemic and blocks artifact upload.\n+MAX_TOLERATED_RENDER_FAILURES = 5\n+MAX_TOLERATED_RENDER_FAILURE_RATIO = 0.01\n+\n+\n+def render_failures_are_catastrophic(failed: int, expected: int) -> bool:\n+    """Return True when partial rendering is too broad to publish safely."""\n+    failed = max(0, int(failed or 0))\n+    expected = max(0, int(expected or 0))\n+    if failed == 0:\n+        return False\n+    if failed > MAX_TOLERATED_RENDER_FAILURES:\n+        return True\n+    return expected > 0 and failed / expected > MAX_TOLERATED_RENDER_FAILURE_RATIO\n+\n \n def static_asset_version() -> str:\n     digest = hashlib.sha256()\n@@ -1113,7 +1131,8 @@ def writer_model(writer, writer_models=None):\n \n \n def prep_article(raw):\n-    if not isinstance(raw, dict):\n+    raw = normalize_article_record(raw)\n+    if raw is None:\n         return None\n     if raw.get("archived") is True:\n         return None\n@@ -1271,14 +1290,15 @@ def collect_articles():\n                 print(f"build: {path.name}: no articles[] — skipped")\n                 continue\n             for raw in rows:\n-                art = prep_article(raw)\n+                normalized = normalize_article_record(raw)\n+                art = prep_article(normalized)\n                 if art is None:\n                     reason = ("archived article skipped"\n-                              if isinstance(raw, dict)\n-                              and raw.get("archived") is True\n+                              if isinstance(normalized, dict)\n+                              and normalized.get("archived") is True\n                               else "bad article entry skipped")\n                     print(f"build: {path.name}: {reason}")\n-                elif not is_transport_headline(raw):\n+                elif not is_transport_headline(normalized):\n                     print(f"build: {path.name}: non-transport headline skipped")\n                 else:\n                     arts.append(art)\n@@ -3680,6 +3700,10 @@ def main() -> int:\n         DATA_DIR / "briefings" / "index.json", {}).get("briefings") or [])\n         if isinstance(r, dict)]\n     latest_row = latest_briefing(brief_rows)\n+    expected_guarded_pages = (\n+        sum(len(article["available_languages"]) for article in articles)\n+        + (2 * len(briefings))\n+    )\n     changelog_raw = load_json(\n         Path(__file__).resolve().parent.parent\n         / "config" / "changelog.json", {})\n@@ -4089,9 +4113,20 @@ def main() -> int:\n \n     write_search_discovery_files(articles, briefings, now)\n \n+    catastrophic = render_failures_are_catastrophic(\n+        failed, expected_guarded_pages)\n     print(f"build: {len(articles)} articles, {len(flashes)} flashes, "\n           f"{len(incidents)} incidents, {pages} pages, {failed} failed "\n           f"-> {SITE_DIR} (base='{BASE_PATH}') in {time.time() - t0:.2f}s")\n+    if catastrophic:\n+        ratio = failed / expected_guarded_pages if expected_guarded_pages else 1.0\n+        print(\n+            "::error::Build blocked: render failures exceeded the safety "\n+            f"budget ({failed}/{expected_guarded_pages}, {ratio:.2%}; "\n+            f"limits: {MAX_TOLERATED_RENDER_FAILURES} pages and "\n+            f"{MAX_TOLERATED_RENDER_FAILURE_RATIO:.0%})"\n+        )\n+        return 1\n     return 0\n \n \n@@ -0,0 +1,112 @@\n+"""Regression checks for article schema evolution and build fail-closed guards."""\n+from __future__ import annotations\n+\n+import copy\n+import json\n+import sys\n+from pathlib import Path\n+\n+ROOT = Path(__file__).resolve().parent.parent\n+sys.path.insert(0, str(ROOT / "pipeline"))\n+\n+import article_schema  # noqa: E402\n+import build  # noqa: E402\n+\n+\n+def main() -> None:\n+    legacy = {\n+        "id": "a-20260701-1200-legacy-schema",\n+        "publishedUtc": "2026-07-01T12:00Z",\n+        "cat": "ops",\n+        "zh": {\n+            "title": "舊格式文章",\n+            "summary": "驗證歷史文章缺少新欄位時仍能正常讀取。",\n+            "body": "舊格式曾允許單一字串正文。",\n+        },\n+        "sources": [{\n+            "name": "Example Aviation",\n+            "url": "https://example.com/aviation/legacy",\n+        }],\n+    }\n+    untouched = copy.deepcopy(legacy)\n+    normalized = article_schema.normalize_article_record(legacy)\n+    assert legacy == untouched\n+    assert normalized is not legacy\n+    assert normalized["schemaVersion"] == article_schema.ARTICLE_SCHEMA_VERSION\n+    assert normalized["archived"] is False\n+    assert normalized["primarySource"] == ""\n+    assert normalized["image"] is None\n+    assert normalized["zh"]["body"] == ["舊格式曾允許單一字串正文。"]\n+    assert normalized["en"] == {"title": "", "summary": "", "body": []}\n+    assert normalized["availableLanguages"] == []\n+    assert normalized["writerModels"] == []\n+    assert normalized["sourceImageCandidates"] == []\n+    assert normalized["facts"] == []\n+    assert normalized["riskFlags"] == []\n+    assert normalized["entityEvidence"] == []\n+    assert normalized["entities"]["airlines"] == []\n+    assert normalized["writer"] is None\n+    assert normalized["articleFormat"] == "full"\n+\n+    prepared = build.prep_article(legacy)\n+    assert prepared is not None\n+    assert prepared["late_ingest"] is False\n+    assert prepared["article_format"] == "full"\n+    assert prepared["writer_model"] is None\n+    assert prepared["image"] is None\n+    assert prepared["airline_entities"] == []\n+    assert prepared["available_languages"] == ["zh", "en"]\n+    assert prepared["en"]["title"] == prepared["zh"]["title"]\n+\n+    # Every currently persisted article generation must cross the same\n+    # normalization boundary and expose a complete prepared-view contract.\n+    required_view_keys = {\n+        "id", "dt", "news_dt", "published_dt", "modified_dt",\n+        "published_iso", "modified_iso", "published_meta_ts",\n+        "modified_meta_ts", "late_ingest", "cat", "tag_class", "image",\n+        "source", "time", "meta_ts", "zh", "en", "sources",\n+        "airline_entities", "writer_model", "available_languages",\n+        "article_format",\n+    }\n+    accepted = 0\n+    for path in sorted((ROOT / "data" / "articles").glob("*.json")):\n+        payload = json.loads(path.read_text(encoding="utf-8"))\n+        rows = payload.get("articles") if isinstance(payload, dict) else None\n+        if not isinstance(rows, list):\n+            continue\n+        for raw in rows:\n+            if not isinstance(raw, dict):\n+                continue\n+            normalized = article_schema.normalize_article_record(raw)\n+            assert normalized is not None\n+            assert normalized["schemaVersion"] == article_schema.ARTICLE_SCHEMA_VERSION\n+            prepared = build.prep_article(raw)\n+            if prepared is None:\n+                continue\n+            accepted += 1\n+            assert required_view_keys <= set(prepared)\n+            assert prepared["article_format"] in {"full", "brief", "roundup"}\n+            assert isinstance(prepared["late_ingest"], bool)\n+            assert all(lang in {"zh", "en"} for lang in prepared["available_languages"])\n+    assert accepted > 900\n+\n+    # Isolated failures remain degradable, but either safety bound blocks a\n+    # partial site from being published.\n+    assert build.render_failures_are_catastrophic(0, 2000) is False\n+    assert build.render_failures_are_catastrophic(5, 2000) is False\n+    assert build.render_failures_are_catastrophic(6, 2000) is True\n+    assert build.render_failures_are_catastrophic(2, 200) is False\n+    assert build.render_failures_are_catastrophic(3, 200) is True\n+    assert build.render_failures_are_catastrophic(1, 50) is True\n+\n+    source = (ROOT / "pipeline" / "build.py").read_text(encoding="utf-8")\n+    assert "if catastrophic:" in source\n+    assert "::error::Build blocked: render failures exceeded the safety " in source\n+    assert "budget (" in source\n+    assert "return 1" in source\n+\n+    print(f"test_build_resilience: OK ({accepted} historical article rows accepted)")\n+\n+\n+if __name__ == "__main__":\n+    main()