{ "sha": "a0fd69b602e23054be269549559affc6f136b686", "message": "補回星宇聯營及越航期中報告來源，縮短故障模型等待 (#231)\n\n* Backfill missed codeshare and safety-report discovery with bounded expiry\n\n* Align offline fixtures with per-quote and entity evidence contracts\n\n* Keep FAA-only test evidence free of unrelated IATA metadata", "author": { "login": "nnneeeooo-nnneeeooo", "name": "", "email": "" }, "committer": { "login": "web-flow", "name": "", "email": "" }, "created_at": "2026-09-30T11:16:38Z", "files": [ { "filename": ".github/workflows/hourly.yml", "previous_filename": "", "status": "modified", "additions": 5, "deletions": 1, "changes": 6 }, { "filename": "data/news-recovery.json", "previous_filename": "", "status": "added", "additions": 31, "deletions": 0, "changes": 31 }, { "filename": "pipeline/dedupe.py", "previous_filename": "", "status": "modified", "additions": 26, "deletions": 2, "changes": 28 }, { "filename": "pipeline/fulltext.py", "previous_filename": "", "status": "modified", "additions": 2, "deletions": 0, "changes": 2 }, { "filename": "tests/test_dedupe.py", "previous_filename": "", "status": "modified", "additions": 14, "deletions": 13, "changes": 27 }, { "filename": "tests/test_draft_recovery.py", "previous_filename": "", "status": "modified", "additions": 26, "deletions": 0, "changes": 26 }, { "filename": "tests/test_write_offline.py", "previous_filename": "", "status": "modified", "additions": 28, "deletions": 5, "changes": 33 } ], "diff": "@@ -13,6 +13,8 @@ on:\\n - \"pipeline/flightnews.py\"\\n - \"pipeline/write.py\"\\n - \"pipeline/common.py\"\\n+ - \"pipeline/dedupe.py\"\\n+ - \"pipeline/fulltext.py\"\\n - \"pipeline/draft_recovery.py\"\\n - \"pipeline/manual_draft.py\"\\n - \"pipeline/airport_codes.py\"\\n@@ -58,6 +60,8 @@ jobs:\\n python tests/test_manufacturer_sources.py\\n python tests/test_source_links.py\\n python tests/test_draft_recovery.py\\n+ python tests/test_dedupe.py\\n+ python tests/test_write_offline.py\\n \\n update:\\n if: github.event_name != 'pull_request'\\n@@ -120,7 +124,7 @@ jobs:\\n AVWIRE_GEMINI_MODEL: ${{ vars.AVWIRE_GEMINI_MODEL }}\\n AVWIRE_NVIDIA_MODEL: ${{ vars.AVWIRE_NVIDIA_MODEL }}\\n AVWIRE_WECHAT_MODEL: ${{ vars.AVWIRE_WECHAT_MODEL }}\\n- AVWIRE_PROVIDER_ORDER: gemini:gemini-3.6-flash,gemini:gemini-3.5-flash,nvidia:nvidia/nemotron-3-ultra-550b-a55b,wechat:Deepseek-v4-flash,openrouter:nvidia/nemotron-3-ultra-550b-a55b:free,nvidia:nvidia/nemotron-3-super-120b-a12b\\n+ AVWIRE_PROVIDER_ORDER: gemini:gemini-3.6-flash,gemini:gemini-3.5-flash,wechat:Deepseek-v4-flash,nvidia:nvidia/nemotron-3-ultra-550b-a55b,openrouter:nvidia/nemotron-3-ultra-550b-a55b:free,nvidia:nvidia/nemotron-3-super-120b-a12b\\n # freshness window (hours); code default 120, validated 24-336\\n NEWS_MAX_AGE_HOURS: ${{ vars.NEWS_MAX_AGE_HOURS }}\\n # private usage-dashboard URL token (page skipped when unset)\\n@@ -0,0 +1,31 @@\\n+{\\n+ \"_comment\": \"User-requested recovery of missed news, verified against publisher pages on 2026-09-30. Original source dates are retained; this temporary discovery input expires and normal seen/evidence gates remain authoritative.\",\\n+ \"expiresUtc\": \"2026-10-07T00:00Z\",\\n+ \"items\": [\\n+ {\\n+ \"title\": \"STARLUX Airlines and American Airlines launch codeshare partnership\",\\n+ \"url\": \"https://latestnews.starlux-airlines.com/en-TW/about-us/travel-advisories/advisories/latest-news/aa-codeshare\",\\n+ \"publishedUtc\": \"2026-09-23T00:00Z\",\\n+ \"summary\": \"with tickets available starting September 23, 2026, and flights officially launching on September 30, 2026! 20 popular North American cities\",\\n+ \"source\": \"STARLUX Airlines\",\\n+ \"sourceKey\": \"starlux-official-recovery\"\\n+ },\\n+ {\\n+ \"title\": \"American Airlines and STARLUX Airlines launch codeshare partnership\",\\n+ \"url\": \"https://news.aa.com/news/news-details/2026/American-Airlines-and-STARLUX-Airlines-launch-codeshare-partnership-to-expand-trans-Pacific-flight-options-NET-ALP-09/default.aspx\",\\n+ \"publishedUtc\": \"2026-09-23T00:00Z\",\\n+ \"dateInferred\": true,\\n+ \"summary\": \"American Airlines and STARLUX Airlines have launched a new codeshare partnership connecting North America with Taipei (TPE).\",\\n+ \"source\": \"American Airlines\",\\n+ \"sourceKey\": \"american-official-recovery\"\\n+ },\\n+ {\\n+ \"title\": \"BFU interim findings on Vietnam Airlines Munich runway accident\",\\n+ \"url\": \"https://aerospaceglobalnews.com/news/vietnam-airlines-munich-runway-incident-investigation/\",\\n+ \"publishedUtc\": \"2026-09-29T00:00Z\",\\n+ \"summary\": \"The BFU stresses that it has not yet reached a final conclusion about the cause of the accident.\",\\n+ \"source\": \"Aerospace Global News\",\\n+ \"sourceKey\": \"aerospaceglobalnews\"\\n+ }\\n+ ]\\n+}\\n@@ -189,6 +189,24 @@ def _load_raw_items() -> list[dict]:\\n item[\"source\"] = _source_label(source, item)\\n item[\"sourceKey\"] = key\\n items.append(item)\\n+ # Explicit editorial recovery inputs are bounded by an expiry and retain\\n+ # the source's original date. They are still subject to unseen, source,\\n+ # prompt-injection and evidence gates; no article is published here.\\n+ recovery = load_json(DATA_DIR / \"news-recovery.json\", {})\\n+ if isinstance(recovery, dict):\\n+ try:\\n+ until = parse_iso(str(recovery.get(\"expiresUtc\") or \"\"))\\n+ except (TypeError, ValueError):\\n+ until = None\\n+ if until is not None and now_utc() < until:\\n+ for raw in recovery.get(\"items\") or []:\\n+ if (not isinstance(raw, dict)\\n+ or not str(raw.get(\"url\") or \"\").startswith(\"https://\")\\n+ or not raw.get(\"source\") or not raw.get(\"sourceKey\")):\\n+ continue\\n+ item = dict(raw)\\n+ item[\"_recoveryUntilUtc\"] = iso_minute(until)\\n+ items.append(item)\\n return items\\n \\n \\n@@ -220,8 +238,13 @@ def _is_fresh(item: dict, now: datetime, stats: dict) -> bool:\\n published = _published(item)\\n if published is not None:\\n if now - published > timedelta(hours=MAX_ITEM_AGE_HOURS):\\n- stats[\"skipped_too_old\"] += 1\\n- return False\\n+ try:\\n+ recovery_until = parse_iso(str(item.get(\"_recoveryUntilUtc\") or \"\"))\\n+ except (TypeError, ValueError):\\n+ recovery_until = None\\n+ if recovery_until is None or now >= recovery_until:\\n+ stats[\"skipped_too_old\"] += 1\\n+ return False\\n if published - now > timedelta(hours=FUTURE_SKEW_HOURS):\\n # Beyond timezone/parse skew: a source clock error, not news.\\n stats[\"skipped_future_date\"] += 1\\n@@ -448,3 +471,4 @@ def main() -> None:\\n \\n if __name__ == \"__main__\":\\n main()\\n+\\n@@ -90,6 +90,7 @@\\n \"english.comac.cc\", \"www.comac.cc\", \"comac.cc\",\\n # Editorial sources with direct article URLs. robots.txt is still checked\\n # on every host and any unreachable/denied policy fails closed.\\n+ \"latestnews.starlux-airlines.com\", \"news.aa.com\",\\n \"www.aerotime.aero\", \"aerotime.aero\",\\n \"www.aerospaceglobalnews.com\", \"aerospaceglobalnews.com\",\\n \"www.flightglobal.com\", \"flightglobal.com\",\\n@@ -242,3 +243,4 @@ def enrich_pending(groups: list) -> int:\\n if fetches or enriched:\\n print(f\"fulltext: {fetches} fetch(es), {enriched} item(s) enriched\")\\n return enriched\\n+\\n@@ -142,9 +142,10 @@ def test_filtering_grouping_and_ranking() -> None:\\n # ok=false snapshots are carried forward by fetch.py — items survive.\\n assert \"EASA carried-forward item survives source flap\" in all_titles\\n \\n- # Group 1: the multi-source Denver story ranks first; Reuters has\\n- # the longer summary so it is primary and its item comes first.\\n- denver = groups[0]\\n+ # Safety investigation updates may rank first. Within the Denver\\n+ # event, Reuters has the longer summary and remains primary.\\n+ denver = next(g for g in groups if {i[\"sourceKey\"] for i in g[\"items\"]}\\n+ == {\"faa\", \"reuters\"})\\n assert {i[\"sourceKey\"] for i in denver[\"items\"]} == {\"faa\", \"reuters\"}\\n assert denver[\"primarySource\"] == \"Reuters\"\\n assert denver[\"items\"][0][\"sourceKey\"] == \"reuters\"\\n@@ -155,16 +156,15 @@ def test_filtering_grouping_and_ranking() -> None:\\n }\\n assert denver[\"items\"][0][\"image\"] == \"https://www.reuters.com/img/denver.jpg\"\\n \\n- # Remaining single-source groups ranked by newest publishedUtc. The\\n- # Boeing URL is new, so its seen-title match becomes an article update.\\n- assert groups[1][\"items\"][0][\"sourceKey\"] == \"easa\"\\n- assert groups[2][\"items\"][0][\"title\"] == \\\\n- \"Boeing delivers 100th 787 to Emirates!\"\\n- assert groups[2][\"updateCandidate\"] is True\\n- assert groups[3][\"items\"][0][\"title\"] == (\\n- \"Airbus opens new A320 production line in Toulouse\"\\n- ) # seen-title match is 30 days old -> outside the 21-day window\\n- assert groups[4][\"items\"][0][\"title\"].startswith(\"NTSB opens investigation\")\\n+ # Investigation updates get priority, then material/multi-source\\n+ # events; the remaining ordinary groups retain recency ordering.\\n+ assert groups[0][\"editorialPriority\"] == \"major\"\\n+ assert groups[0][\"items\"][0][\"title\"].startswith(\"NTSB opens investigation\")\\n+ assert groups[1] is denver\\n+ assert groups[2][\"items\"][0][\"sourceKey\"] == \"easa\"\\n+ assert groups[3][\"items\"][0][\"title\"] == \"Boeing delivers 100th 787 to Emirates!\"\\n+ assert groups[3][\"updateCandidate\"] is True\\n+ assert groups[4][\"items\"][0][\"title\"] == \"Airbus opens new A320 production line in Toulouse\"\\n \\n # Balloon group fits the six-item current-source cap.\\n balloons = groups[5]\\n@@ -498,3 +498,4 @@ def main() -> None:\\n \\n if __name__ == \"__main__\":\\n main()\\n+\\n@@ -1,11 +1,16 @@\\n \"\"\"Offline regression tests: presentation recovery never rewrites evidence.\"\"\"\\n import sys\\n import unittest\\n+import json\\n+import tempfile\\n+from datetime import datetime, timedelta, timezone\\n+from unittest.mock import patch\\n from pathlib import Path\\n \\n sys.path.insert(0, str(Path(__file__).resolve().parents[1] / \"pipeline\"))\\n from draft_recovery import normalize_reader_copy, repair_prompt\\n import common\\n+import dedupe\\n \\n \\n class Converter:\\n@@ -42,6 +47,27 @@ def test_retry_keeps_sources_and_feedback_without_padding_permission(self):\\n self.assertIn(\"preserve verbatim source quotes\", prompt)\\n self.assertIn(\"Do not add facts\", prompt)\\n \\n+ def test_recovery_expiry_original_dates_and_seen_gate(self):\\n+ now = datetime(2026, 9, 30, 12, tzinfo=timezone.utc)\\n+ old = {\"title\": \"STARLUX launches American Airlines codeshare\", \"url\": \"https://example.com/news\",\\n+ \"publishedUtc\": \"2026-09-23T00:00Z\", \"source\": \"Publisher\", \"sourceKey\": \"publisher\"}\\n+ with tempfile.TemporaryDirectory() as tmp:\\n+ root = Path(tmp)\\n+ path = root / \"news-recovery.json\"\\n+ path.write_text(json.dumps({\"expiresUtc\": \"2026-10-07T00:00Z\", \"items\": [old]}))\\n+ with patch.object(dedupe, \"DATA_DIR\", root), patch.object(dedupe, \"RAW_DIR\", root / \"raw\"), patch.object(dedupe, \"now_utc\", lambda: now):\\n+ rows = dedupe._load_raw_items()\\n+ self.assertEqual(len(rows), 1)\\n+ self.assertEqual(rows[0][\"publishedUtc\"], old[\"publishedUtc\"])\\n+ stats = {key: 0 for key in (\"skipped_untitled\", \"skipped_too_old\", \"skipped_future_date\")}\\n+ self.assertTrue(dedupe._is_fresh(rows[0], now, stats))\\n+ self.assertFalse(dedupe._is_unseen(rows[0], {common.norm_url(old[\"url\"])}, [])[0])\\n+ future = {**rows[0], \"publishedUtc\": \"2026-11-01T00:00Z\"}\\n+ self.assertFalse(dedupe._is_fresh(future, now, stats))\\n+ path.write_text(json.dumps({\"expiresUtc\": \"2026-09-29T00:00Z\", \"items\": [old]}))\\n+ self.assertEqual(dedupe._load_raw_items(), [])\\n+ self.assertFalse(dedupe._is_fresh(rows[0], now + timedelta(days=8), stats))\\n+\\n def test_safety_findings_and_codeshare_are_retained_before_enrichment(self):\\n for title in (\\n \"Vietnam Airlines Munich runway incident: Investigation reveals brakes activated during takeoff roll\",\\n@@ -70,7 +70,7 @@\\n \"facts\": [\\n {\"factId\": \"F1\",\\n \"claim\": \"NTSB 已對丹佛 737 衝出跑道事件展開調查\",\\n- \"sourceQuote\": \"NTSB said it is investigating a runway excursion\"},\\n+ \"sourceQuote\": \"The NTSB said it is investigating a runway excursion involving a United Airlines Boeing 737-8 at Denver International Airport during landing on July 25.\"},\\n {\"factId\": \"F2\",\\n \"claim\": \"無人受傷\",\\n \"sourceQuote\": \"No injuries were reported.\"},\\n@@ -122,7 +122,7 @@\\n \"sourceQuote\": \"rose 8.2% in June 2026\"},\\n {\"factId\": \"F2\",\\n \"claim\": \"需求以貨運噸公里計算並與 2025 年六月比較\",\\n- \"sourceQuote\": \"measured in cargo tonne-kilometers\"},\\n+ \"sourceQuote\": \"measured in cargo tonne-kilometers, rose 8.2% in June 2026 compared with June 2025.\"},\\n ],\\n \"headlineSupportedBy\": [\"F1\"],\\n \"summarySupportedBy\": [\"F1\", \"F2\"],\\n@@ -150,6 +150,22 @@\\n \"incident\": None,\\n }\\n \\n+# Fixtures must satisfy the current per-entity evidence contract. Keep the\\n+# source quote intact while normalizing displayed Chinese as production does.\\n+DRAFT_SAFETY[\"entityEvidence\"] = [\\n+ {\"entityType\": key, \"value\": value,\\n+ \"sourceQuote\": DRAFT_SAFETY[\"facts\"][0][\"sourceQuote\"],\\n+ \"sourceUrl\": \"https://www.ntsb.gov/news/press-releases/denver-737\"}\\n+ for key, values in DRAFT_SAFETY[\"entities\"].items() for value in values\\n+]\\n+DRAFT_BIZ[\"entityEvidence\"] = [{\\n+ \"entityType\": \"organizations\", \"value\": \"International Air Transport Association\",\\n+ \"sourceQuote\": \"The International Air Transport Association reported\",\\n+ \"sourceUrl\": \"https://www.iata.org/en/pressroom/2026-releases/2026-07-24-01/\",\\n+}]\\n+DRAFT_SAFETY = write.normalize_reader_copy(DRAFT_SAFETY, write._S2T)\\n+DRAFT_BIZ = write.normalize_reader_copy(DRAFT_BIZ, write._S2T)\\n+\\n _pass = 0\\n _fail = 0\\n \\n@@ -429,6 +445,8 @@ def test_group_cap_and_unique_ids():\\n \"sourceQuote\": \"operational details for certificate holders\"},\\n ]\\n cap_draft[\"summarySupportedBy\"] = [\"F1\", \"F2\"]\\n+ cap_draft[\"entities\"] = dict(_EMPTY_ENTITIES)\\n+ cap_draft[\"entityEvidence\"] = []\\n original = write.draft_group\\n write.draft_group = lambda client, group: cap_draft\\n try:\\n@@ -589,7 +607,8 @@ def test_extract_json_and_validate_draft():\\n retrying_provider, cargo_group, tries=2)\\n check(retrying_provider.calls == 2,\\n \"short body automatically triggers the allowed retry\")\\n- check(candidate == DRAFT_BIZ and facts,\\n+ check(candidate[\"zh\"] == DRAFT_BIZ[\"zh\"]\\n+ and write.validate_draft(candidate) is None and facts,\\n \"length-compliant retry survives validation and quote checks\")\\n check(write.validate_draft(\"nope\") is not None, \"non-dict rejected\")\\n broken = json.loads(json.dumps(DRAFT_BIZ))\\n@@ -701,8 +720,8 @@ def test_provider_failover():\\n \"fallback publication does not create review entries\")\\n check(primary.calls == 1,\\n f\"dead primary is never called again, got {primary.calls} calls\")\\n- check(middle.calls == 3 and backup.calls == 3,\\n- \"surviving providers tried once per group\")\\n+ check(middle.calls == 6 and backup.calls == 3,\\n+ \"first responsive fallback gets one bounded repair per group\")\\n pending = load(DATA / \"pending.json\")\\n ids = [g[\"id\"] for g in pending[\"groups\"]]\\n check(ids == [\"g-20260726-0503-3\"],\\n@@ -848,6 +867,7 @@ def test_editorial_gate():\\n fabricated = json.loads(json.dumps(DRAFT_BIZ))\\n fabricated[\"facts\"] = [{\"factId\": \"F1\", \"claim\": \"完全捏造的主張\",\\n \"sourceQuote\": \"this text appears nowhere\"}]\\n+ fabricated[\"summarySupportedBy\"] = [\"F1\"]\\n partial = json.loads(json.dumps(DRAFT_BIZ))\\n partial[\"facts\"] = [\\n {\"factId\": \"F1\", \"claim\": \"FAA 發布鬧事乘客執法聲明\",\\n@@ -859,6 +879,8 @@ def test_editorial_gate():\\n ]\\n partial[\"headlineSupportedBy\"] = [\"F1\"]\\n partial[\"summarySupportedBy\"] = [\"F1\", \"F2\"]\\n+ partial[\"entities\"] = dict(_EMPTY_ENTITIES)\\n+ partial[\"entityEvidence\"] = []\\n # g1 -> editorial reject; g2 -> all quotes fabricated; g3 -> one good.\\n solo = FakeProvider(\"gemini\", [reject_draft, fabricated, partial])\\n \\n@@ -1042,3 +1064,4 @@ def main():\\n \\n if __name__ == \"__main__\":\\n main()\\n+" }