{ "sha": "6135a203e055017f95affec2ee40c3b9dd41ed8b", "message": "Add P1 article schema compatibility and build safety guard\n\nNormalize historical article records at the build read boundary and block Pages publication when render failures exceed the bounded safety budget.", "author": { "login": "nnneeeooo-nnneeeooo", "name": "", "email": "" }, "committer": { "login": "web-flow", "name": "", "email": "" }, "created_at": "2026-10-01T15:07:49Z", "files": [ { "filename": ".github/workflows/hourly.yml", "previous_filename": "", "status": "modified", "additions": 3, "deletions": 0, "changes": 3 }, { "filename": ".github/workflows/radar-snapshot.yml", "previous_filename": "", "status": "modified", "additions": 3, "deletions": 0, "changes": 3 }, { "filename": "pipeline/article_schema.py", "previous_filename": "", "status": "added", "additions": 81, "deletions": 0, "changes": 81 }, { "filename": "pipeline/build.py", "previous_filename": "", "status": "modified", "additions": 42, "deletions": 7, "changes": 49 }, { "filename": "tests/test_build_resilience.py", "previous_filename": "", "status": "added", "additions": 112, "deletions": 0, "changes": 112 } ], "diff": "@@ -13,6 +13,8 @@ on:\\n - \"pipeline/flightnews.py\"\\n - \"pipeline/write.py\"\\n - \"pipeline/provider_health.py\"\\n+ - \"pipeline/article_schema.py\"\\n+ - \"pipeline/build.py\"\\n - \"pipeline/common.py\"\\n - \"pipeline/dedupe.py\"\\n - \"pipeline/fulltext.py\"\\n@@ -58,6 +60,7 @@ jobs:\\n python tests/test_dedupe.py\\n python tests/test_write_offline.py\\n python tests/test_provider_health.py\\n+ python tests/test_build_resilience.py\\n python tests/test_news_freshness.py\\n \\n update:\\n@@ -14,6 +14,7 @@ on:\\n - \"pipeline/common.py\"\\n - \"pipeline/radar_snapshot.py\"\\n - \"pipeline/build.py\"\\n+ - \"pipeline/article_schema.py\"\\n - \"static/radar.js\"\\n - \"templates/radar.html\"\\n - \"tests/test_radar.py\"\\n@@ -26,6 +27,7 @@ on:\\n - \"pipeline/common.py\"\\n - \"pipeline/radar_snapshot.py\"\\n - \"pipeline/build.py\"\\n+ - \"pipeline/article_schema.py\"\\n - \"static/radar.js\"\\n - \"templates/radar.html\"\\n \\n@@ -48,6 +50,7 @@ jobs:\\n - name: Run radar regression tests\\n run: |\\n python tests/test_radar_snapshot.py\\n+ python tests/test_build_resilience.py\\n python tests/test_radar.py\\n \\n build:\\n@@ -0,0 +1,81 @@\\n+\"\"\"Read-boundary normalization for persisted SKYTICAL article records.\\n+\\n+Historical JSON is append-only and spans multiple schema generations. This\\n+module turns any dictionary-shaped article row into the current read contract\\n+without rewriting the archive on disk. Required publication identity and\\n+source fields are deliberately not invented; build.py still rejects rows that\\n+lack those essentials.\\n+\"\"\"\\n+from __future__ import annotations\\n+\\n+from copy import deepcopy\\n+\\n+ARTICLE_SCHEMA_VERSION = 1\\n+\\n+_OPTIONAL_LIST_FIELDS = (\\n+ \"availableLanguages\",\\n+ \"writerModels\",\\n+ \"sourceImageCandidates\",\\n+ \"facts\",\\n+ \"riskFlags\",\\n+ \"entityEvidence\",\\n+)\\n+\\n+\\n+def _language_block(value) -> dict:\\n+ block = deepcopy(value) if isinstance(value, dict) else {}\\n+ body = block.get(\"body\")\\n+ if isinstance(body, str):\\n+ body = [body]\\n+ elif not isinstance(body, list):\\n+ body = []\\n+ return {\\n+ **block,\\n+ \"title\": str(block.get(\"title\") or \"\"),\\n+ \"summary\": str(block.get(\"summary\") or \"\"),\\n+ \"body\": [str(item) for item in body if str(item).strip()],\\n+ }\\n+\\n+\\n+def normalize_article_record(raw) -> dict | None:\\n+ \"\"\"Return a current-schema copy of one persisted article row.\\n+\\n+ Optional fields always exist after this boundary. The function never\\n+ mutates the caller's object and never fabricates required identity,\\n+ timestamp, or source evidence.\\n+ \"\"\"\\n+ if not isinstance(raw, dict):\\n+ return None\\n+\\n+ article = deepcopy(raw)\\n+ article[\"schemaVersion\"] = ARTICLE_SCHEMA_VERSION\\n+ article[\"archived\"] = article.get(\"archived\") is True\\n+ article[\"primarySource\"] = str(article.get(\"primarySource\") or \"\")\\n+ article[\"image\"] = article.get(\"image\")\\n+ article[\"zh\"] = _language_block(article.get(\"zh\"))\\n+ article[\"en\"] = _language_block(article.get(\"en\"))\\n+\\n+ sources = article.get(\"sources\")\\n+ article[\"sources\"] = list(sources) if isinstance(sources, list) else []\\n+\\n+ for key in _OPTIONAL_LIST_FIELDS:\\n+ value = article.get(key)\\n+ article[key] = list(value) if isinstance(value, list) else []\\n+\\n+ entities = article.get(\"entities\")\\n+ article[\"entities\"] = deepcopy(entities) if isinstance(entities, dict) else {}\\n+ airlines = article[\"entities\"].get(\"airlines\")\\n+ article[\"entities\"][\"airlines\"] = (\\n+ list(airlines) if isinstance(airlines, list) else []\\n+ )\\n+\\n+ article[\"writer\"] = (\\n+ str(article.get(\"writer\")) if isinstance(article.get(\"writer\"), str)\\n+ else None\\n+ )\\n+ article[\"articleFormat\"] = (\\n+ article.get(\"articleFormat\")\\n+ if article.get(\"articleFormat\") in (\"full\", \"brief\", \"roundup\")\\n+ else \"full\"\\n+ )\\n+ return article\\n@@ -5,8 +5,9 @@\\n CONTRACTS.md: zh at /, en under /en/, one page per article ever published,\\n incidents / sources / methodology / changelog pages, copied assets, 404.html\\n and .nojekyll.\\n-A single bad input never crashes the build: bad entries are skipped with a\\n-one-line log to stdout and the script exits 0.\\n+Isolated bad inputs degrade gracefully and are skipped with a one-line log.\\n+A bounded render-failure budget prevents a systemic schema/template regression\\n+from publishing a substantially incomplete site.\\n \"\"\"\\n from __future__ import annotations\\n \\n@@ -56,6 +57,7 @@\\n MODEL_ORDER,\\n )\\n from image_captions import clean_description\\n+from article_schema import normalize_article_record\\n \\n _caption_cache = load_json(DATA_DIR / \"image-captions.json\", {})\\n IMAGE_CAPTIONS = (_caption_cache.get(\"images\", {})\\n@@ -67,6 +69,22 @@\\n PUBLIC_DIR = PROJECT_DIR / \"public\"\\n CONFIG_DIR = PROJECT_DIR / \"config\"\\n \\n+# Render failures are tolerated only while they remain genuinely isolated.\\n+# Exceeding either bound is treated as systemic and blocks artifact upload.\\n+MAX_TOLERATED_RENDER_FAILURES = 5\\n+MAX_TOLERATED_RENDER_FAILURE_RATIO = 0.01\\n+\\n+\\n+def render_failures_are_catastrophic(failed: int, expected: int) -> bool:\\n+ \"\"\"Return True when partial rendering is too broad to publish safely.\"\"\"\\n+ failed = max(0, int(failed or 0))\\n+ expected = max(0, int(expected or 0))\\n+ if failed == 0:\\n+ return False\\n+ if failed > MAX_TOLERATED_RENDER_FAILURES:\\n+ return True\\n+ return expected > 0 and failed / expected > MAX_TOLERATED_RENDER_FAILURE_RATIO\\n+\\n \\n def static_asset_version() -> str:\\n digest = hashlib.sha256()\\n@@ -1113,7 +1131,8 @@ def writer_model(writer, writer_models=None):\\n \\n \\n def prep_article(raw):\\n- if not isinstance(raw, dict):\\n+ raw = normalize_article_record(raw)\\n+ if raw is None:\\n return None\\n if raw.get(\"archived\") is True:\\n return None\\n@@ -1271,14 +1290,15 @@ def collect_articles():\\n print(f\"build: {path.name}: no articles[] — skipped\")\\n continue\\n for raw in rows:\\n- art = prep_article(raw)\\n+ normalized = normalize_article_record(raw)\\n+ art = prep_article(normalized)\\n if art is None:\\n reason = (\"archived article skipped\"\\n- if isinstance(raw, dict)\\n- and raw.get(\"archived\") is True\\n+ if isinstance(normalized, dict)\\n+ and normalized.get(\"archived\") is True\\n else \"bad article entry skipped\")\\n print(f\"build: {path.name}: {reason}\")\\n- elif not is_transport_headline(raw):\\n+ elif not is_transport_headline(normalized):\\n print(f\"build: {path.name}: non-transport headline skipped\")\\n else:\\n arts.append(art)\\n@@ -3680,6 +3700,10 @@ def main() -> int:\\n DATA_DIR / \"briefings\" / \"index.json\", {}).get(\"briefings\") or [])\\n if isinstance(r, dict)]\\n latest_row = latest_briefing(brief_rows)\\n+ expected_guarded_pages = (\\n+ sum(len(article[\"available_languages\"]) for article in articles)\\n+ + (2 * len(briefings))\\n+ )\\n changelog_raw = load_json(\\n Path(__file__).resolve().parent.parent\\n / \"config\" / \"changelog.json\", {})\\n@@ -4089,9 +4113,20 @@ def main() -> int:\\n \\n write_search_discovery_files(articles, briefings, now)\\n \\n+ catastrophic = render_failures_are_catastrophic(\\n+ failed, expected_guarded_pages)\\n print(f\"build: {len(articles)} articles, {len(flashes)} flashes, \"\\n f\"{len(incidents)} incidents, {pages} pages, {failed} failed \"\\n f\"-> {SITE_DIR} (base='{BASE_PATH}') in {time.time() - t0:.2f}s\")\\n+ if catastrophic:\\n+ ratio = failed / expected_guarded_pages if expected_guarded_pages else 1.0\\n+ print(\\n+ \"::error::Build blocked: render failures exceeded the safety \"\\n+ f\"budget ({failed}/{expected_guarded_pages}, {ratio:.2%}; \"\\n+ f\"limits: {MAX_TOLERATED_RENDER_FAILURES} pages and \"\\n+ f\"{MAX_TOLERATED_RENDER_FAILURE_RATIO:.0%})\"\\n+ )\\n+ return 1\\n return 0\\n \\n \\n@@ -0,0 +1,112 @@\\n+\"\"\"Regression checks for article schema evolution and build fail-closed guards.\"\"\"\\n+from __future__ import annotations\\n+\\n+import copy\\n+import json\\n+import sys\\n+from pathlib import Path\\n+\\n+ROOT = Path(__file__).resolve().parent.parent\\n+sys.path.insert(0, str(ROOT / \"pipeline\"))\\n+\\n+import article_schema # noqa: E402\\n+import build # noqa: E402\\n+\\n+\\n+def main() -> None:\\n+ legacy = {\\n+ \"id\": \"a-20260701-1200-legacy-schema\",\\n+ \"publishedUtc\": \"2026-07-01T12:00Z\",\\n+ \"cat\": \"ops\",\\n+ \"zh\": {\\n+ \"title\": \"舊格式文章\",\\n+ \"summary\": \"驗證歷史文章缺少新欄位時仍能正常讀取。\",\\n+ \"body\": \"舊格式曾允許單一字串正文。\",\\n+ },\\n+ \"sources\": [{\\n+ \"name\": \"Example Aviation\",\\n+ \"url\": \"https://example.com/aviation/legacy\",\\n+ }],\\n+ }\\n+ untouched = copy.deepcopy(legacy)\\n+ normalized = article_schema.normalize_article_record(legacy)\\n+ assert legacy == untouched\\n+ assert normalized is not legacy\\n+ assert normalized[\"schemaVersion\"] == article_schema.ARTICLE_SCHEMA_VERSION\\n+ assert normalized[\"archived\"] is False\\n+ assert normalized[\"primarySource\"] == \"\"\\n+ assert normalized[\"image\"] is None\\n+ assert normalized[\"zh\"][\"body\"] == [\"舊格式曾允許單一字串正文。\"]\\n+ assert normalized[\"en\"] == {\"title\": \"\", \"summary\": \"\", \"body\": []}\\n+ assert normalized[\"availableLanguages\"] == []\\n+ assert normalized[\"writerModels\"] == []\\n+ assert normalized[\"sourceImageCandidates\"] == []\\n+ assert normalized[\"facts\"] == []\\n+ assert normalized[\"riskFlags\"] == []\\n+ assert normalized[\"entityEvidence\"] == []\\n+ assert normalized[\"entities\"][\"airlines\"] == []\\n+ assert normalized[\"writer\"] is None\\n+ assert normalized[\"articleFormat\"] == \"full\"\\n+\\n+ prepared = build.prep_article(legacy)\\n+ assert prepared is not None\\n+ assert prepared[\"late_ingest\"] is False\\n+ assert prepared[\"article_format\"] == \"full\"\\n+ assert prepared[\"writer_model\"] is None\\n+ assert prepared[\"image\"] is None\\n+ assert prepared[\"airline_entities\"] == []\\n+ assert prepared[\"available_languages\"] == [\"zh\", \"en\"]\\n+ assert prepared[\"en\"][\"title\"] == prepared[\"zh\"][\"title\"]\\n+\\n+ # Every currently persisted article generation must cross the same\\n+ # normalization boundary and expose a complete prepared-view contract.\\n+ required_view_keys = {\\n+ \"id\", \"dt\", \"news_dt\", \"published_dt\", \"modified_dt\",\\n+ \"published_iso\", \"modified_iso\", \"published_meta_ts\",\\n+ \"modified_meta_ts\", \"late_ingest\", \"cat\", \"tag_class\", \"image\",\\n+ \"source\", \"time\", \"meta_ts\", \"zh\", \"en\", \"sources\",\\n+ \"airline_entities\", \"writer_model\", \"available_languages\",\\n+ \"article_format\",\\n+ }\\n+ accepted = 0\\n+ for path in sorted((ROOT / \"data\" / \"articles\").glob(\"*.json\")):\\n+ payload = json.loads(path.read_text(encoding=\"utf-8\"))\\n+ rows = payload.get(\"articles\") if isinstance(payload, dict) else None\\n+ if not isinstance(rows, list):\\n+ continue\\n+ for raw in rows:\\n+ if not isinstance(raw, dict):\\n+ continue\\n+ normalized = article_schema.normalize_article_record(raw)\\n+ assert normalized is not None\\n+ assert normalized[\"schemaVersion\"] == article_schema.ARTICLE_SCHEMA_VERSION\\n+ prepared = build.prep_article(raw)\\n+ if prepared is None:\\n+ continue\\n+ accepted += 1\\n+ assert required_view_keys <= set(prepared)\\n+ assert prepared[\"article_format\"] in {\"full\", \"brief\", \"roundup\"}\\n+ assert isinstance(prepared[\"late_ingest\"], bool)\\n+ assert all(lang in {\"zh\", \"en\"} for lang in prepared[\"available_languages\"])\\n+ assert accepted > 900\\n+\\n+ # Isolated failures remain degradable, but either safety bound blocks a\\n+ # partial site from being published.\\n+ assert build.render_failures_are_catastrophic(0, 2000) is False\\n+ assert build.render_failures_are_catastrophic(5, 2000) is False\\n+ assert build.render_failures_are_catastrophic(6, 2000) is True\\n+ assert build.render_failures_are_catastrophic(2, 200) is False\\n+ assert build.render_failures_are_catastrophic(3, 200) is True\\n+ assert build.render_failures_are_catastrophic(1, 50) is True\\n+\\n+ source = (ROOT / \"pipeline\" / \"build.py\").read_text(encoding=\"utf-8\")\\n+ assert \"if catastrophic:\" in source\\n+ assert \"::error::Build blocked: render failures exceeded the safety \" in source\\n+ assert \"budget (\" in source\\n+ assert \"return 1\" in source\\n+\\n+ print(f\"test_build_resilience: OK ({accepted} historical article rows accepted)\")\\n+\\n+\\n+if __name__ == \"__main__\":\\n+ main()" }