提交原始紀錄

完整差異

@@ -13,6 +13,8 @@ on:\n       - "pipeline/flightnews.py"\n       - "pipeline/write.py"\n       - "pipeline/common.py"\n+      - "pipeline/dedupe.py"\n+      - "pipeline/fulltext.py"\n       - "pipeline/draft_recovery.py"\n       - "pipeline/manual_draft.py"\n       - "pipeline/airport_codes.py"\n@@ -58,6 +60,8 @@ jobs:\n           python tests/test_manufacturer_sources.py\n           python tests/test_source_links.py\n           python tests/test_draft_recovery.py\n+          python tests/test_dedupe.py\n+          python tests/test_write_offline.py\n \n   update:\n     if: github.event_name != 'pull_request'\n@@ -120,7 +124,7 @@ jobs:\n           AVWIRE_GEMINI_MODEL: ${{ vars.AVWIRE_GEMINI_MODEL }}\n           AVWIRE_NVIDIA_MODEL: ${{ vars.AVWIRE_NVIDIA_MODEL }}\n           AVWIRE_WECHAT_MODEL: ${{ vars.AVWIRE_WECHAT_MODEL }}\n-          AVWIRE_PROVIDER_ORDER: gemini:gemini-3.6-flash,gemini:gemini-3.5-flash,nvidia:nvidia/nemotron-3-ultra-550b-a55b,wechat:Deepseek-v4-flash,openrouter:nvidia/nemotron-3-ultra-550b-a55b:free,nvidia:nvidia/nemotron-3-super-120b-a12b\n+          AVWIRE_PROVIDER_ORDER: gemini:gemini-3.6-flash,gemini:gemini-3.5-flash,wechat:Deepseek-v4-flash,nvidia:nvidia/nemotron-3-ultra-550b-a55b,openrouter:nvidia/nemotron-3-ultra-550b-a55b:free,nvidia:nvidia/nemotron-3-super-120b-a12b\n           # freshness window (hours); code default 120, validated 24-336\n           NEWS_MAX_AGE_HOURS: ${{ vars.NEWS_MAX_AGE_HOURS }}\n           # private usage-dashboard URL token (page skipped when unset)\n@@ -0,0 +1,31 @@\n+{\n+  "_comment": "User-requested recovery of missed news, verified against publisher pages on 2026-09-30. Original source dates are retained; this temporary discovery input expires and normal seen/evidence gates remain authoritative.",\n+  "expiresUtc": "2026-10-07T00:00Z",\n+  "items": [\n+    {\n+      "title": "STARLUX Airlines and American Airlines launch codeshare partnership",\n+      "url": "https://latestnews.starlux-airlines.com/en-TW/about-us/travel-advisories/advisories/latest-news/aa-codeshare",\n+      "publishedUtc": "2026-09-23T00:00Z",\n+      "summary": "with tickets available starting September 23, 2026, and flights officially launching on September 30, 2026! 20 popular North American cities",\n+      "source": "STARLUX Airlines",\n+      "sourceKey": "starlux-official-recovery"\n+    },\n+    {\n+      "title": "American Airlines and STARLUX Airlines launch codeshare partnership",\n+      "url": "https://news.aa.com/news/news-details/2026/American-Airlines-and-STARLUX-Airlines-launch-codeshare-partnership-to-expand-trans-Pacific-flight-options-NET-ALP-09/default.aspx",\n+      "publishedUtc": "2026-09-23T00:00Z",\n+      "dateInferred": true,\n+      "summary": "American Airlines and STARLUX Airlines have launched a new codeshare partnership connecting North America with Taipei (TPE).",\n+      "source": "American Airlines",\n+      "sourceKey": "american-official-recovery"\n+    },\n+    {\n+      "title": "BFU interim findings on Vietnam Airlines Munich runway accident",\n+      "url": "https://aerospaceglobalnews.com/news/vietnam-airlines-munich-runway-incident-investigation/",\n+      "publishedUtc": "2026-09-29T00:00Z",\n+      "summary": "The BFU stresses that it has not yet reached a final conclusion about the cause of the accident.",\n+      "source": "Aerospace Global News",\n+      "sourceKey": "aerospaceglobalnews"\n+    }\n+  ]\n+}\n@@ -189,6 +189,24 @@ def _load_raw_items() -> list[dict]:\n             item["source"] = _source_label(source, item)\n             item["sourceKey"] = key\n             items.append(item)\n+    # Explicit editorial recovery inputs are bounded by an expiry and retain\n+    # the source's original date. They are still subject to unseen, source,\n+    # prompt-injection and evidence gates; no article is published here.\n+    recovery = load_json(DATA_DIR / "news-recovery.json", {})\n+    if isinstance(recovery, dict):\n+        try:\n+            until = parse_iso(str(recovery.get("expiresUtc") or ""))\n+        except (TypeError, ValueError):\n+            until = None\n+        if until is not None and now_utc() < until:\n+            for raw in recovery.get("items") or []:\n+                if (not isinstance(raw, dict)\n+                        or not str(raw.get("url") or "").startswith("https://")\n+                        or not raw.get("source") or not raw.get("sourceKey")):\n+                    continue\n+                item = dict(raw)\n+                item["_recoveryUntilUtc"] = iso_minute(until)\n+                items.append(item)\n     return items\n \n \n@@ -220,8 +238,13 @@ def _is_fresh(item: dict, now: datetime, stats: dict) -> bool:\n     published = _published(item)\n     if published is not None:\n         if now - published > timedelta(hours=MAX_ITEM_AGE_HOURS):\n-            stats["skipped_too_old"] += 1\n-            return False\n+            try:\n+                recovery_until = parse_iso(str(item.get("_recoveryUntilUtc") or ""))\n+            except (TypeError, ValueError):\n+                recovery_until = None\n+            if recovery_until is None or now >= recovery_until:\n+                stats["skipped_too_old"] += 1\n+                return False\n         if published - now > timedelta(hours=FUTURE_SKEW_HOURS):\n             # Beyond timezone/parse skew: a source clock error, not news.\n             stats["skipped_future_date"] += 1\n@@ -448,3 +471,4 @@ def main() -> None:\n \n if __name__ == "__main__":\n     main()\n+\n@@ -90,6 +90,7 @@\n     "english.comac.cc", "www.comac.cc", "comac.cc",\n     # Editorial sources with direct article URLs. robots.txt is still checked\n     # on every host and any unreachable/denied policy fails closed.\n+    "latestnews.starlux-airlines.com", "news.aa.com",\n     "www.aerotime.aero", "aerotime.aero",\n     "www.aerospaceglobalnews.com", "aerospaceglobalnews.com",\n     "www.flightglobal.com", "flightglobal.com",\n@@ -242,3 +243,4 @@ def enrich_pending(groups: list) -> int:\n     if fetches or enriched:\n         print(f"fulltext: {fetches} fetch(es), {enriched} item(s) enriched")\n     return enriched\n+\n@@ -142,9 +142,10 @@ def test_filtering_grouping_and_ranking() -> None:\n         # ok=false snapshots are carried forward by fetch.py — items survive.\n         assert "EASA carried-forward item survives source flap" in all_titles\n \n-        # Group 1: the multi-source Denver story ranks first; Reuters has\n-        # the longer summary so it is primary and its item comes first.\n-        denver = groups[0]\n+        # Safety investigation updates may rank first. Within the Denver\n+        # event, Reuters has the longer summary and remains primary.\n+        denver = next(g for g in groups if {i["sourceKey"] for i in g["items"]}\n+                      == {"faa", "reuters"})\n         assert {i["sourceKey"] for i in denver["items"]} == {"faa", "reuters"}\n         assert denver["primarySource"] == "Reuters"\n         assert denver["items"][0]["sourceKey"] == "reuters"\n@@ -155,16 +156,15 @@ def test_filtering_grouping_and_ranking() -> None:\n         }\n         assert denver["items"][0]["image"] == "https://www.reuters.com/img/denver.jpg"\n \n-        # Remaining single-source groups ranked by newest publishedUtc. The\n-        # Boeing URL is new, so its seen-title match becomes an article update.\n-        assert groups[1]["items"][0]["sourceKey"] == "easa"\n-        assert groups[2]["items"][0]["title"] == \\n-            "Boeing delivers 100th 787 to Emirates!"\n-        assert groups[2]["updateCandidate"] is True\n-        assert groups[3]["items"][0]["title"] == (\n-            "Airbus opens new A320 production line in Toulouse"\n-        )  # seen-title match is 30 days old -> outside the 21-day window\n-        assert groups[4]["items"][0]["title"].startswith("NTSB opens investigation")\n+        # Investigation updates get priority, then material/multi-source\n+        # events; the remaining ordinary groups retain recency ordering.\n+        assert groups[0]["editorialPriority"] == "major"\n+        assert groups[0]["items"][0]["title"].startswith("NTSB opens investigation")\n+        assert groups[1] is denver\n+        assert groups[2]["items"][0]["sourceKey"] == "easa"\n+        assert groups[3]["items"][0]["title"] == "Boeing delivers 100th 787 to Emirates!"\n+        assert groups[3]["updateCandidate"] is True\n+        assert groups[4]["items"][0]["title"] == "Airbus opens new A320 production line in Toulouse"\n \n         # Balloon group fits the six-item current-source cap.\n         balloons = groups[5]\n@@ -498,3 +498,4 @@ def main() -> None:\n \n if __name__ == "__main__":\n     main()\n+\n@@ -1,11 +1,16 @@\n """Offline regression tests: presentation recovery never rewrites evidence."""\n import sys\n import unittest\n+import json\n+import tempfile\n+from datetime import datetime, timedelta, timezone\n+from unittest.mock import patch\n from pathlib import Path\n \n sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "pipeline"))\n from draft_recovery import normalize_reader_copy, repair_prompt\n import common\n+import dedupe\n \n \n class Converter:\n@@ -42,6 +47,27 @@ def test_retry_keeps_sources_and_feedback_without_padding_permission(self):\n         self.assertIn("preserve verbatim source quotes", prompt)\n         self.assertIn("Do not add facts", prompt)\n \n+    def test_recovery_expiry_original_dates_and_seen_gate(self):\n+        now = datetime(2026, 9, 30, 12, tzinfo=timezone.utc)\n+        old = {"title": "STARLUX launches American Airlines codeshare", "url": "https://example.com/news",\n+               "publishedUtc": "2026-09-23T00:00Z", "source": "Publisher", "sourceKey": "publisher"}\n+        with tempfile.TemporaryDirectory() as tmp:\n+            root = Path(tmp)\n+            path = root / "news-recovery.json"\n+            path.write_text(json.dumps({"expiresUtc": "2026-10-07T00:00Z", "items": [old]}))\n+            with patch.object(dedupe, "DATA_DIR", root), patch.object(dedupe, "RAW_DIR", root / "raw"), patch.object(dedupe, "now_utc", lambda: now):\n+                rows = dedupe._load_raw_items()\n+                self.assertEqual(len(rows), 1)\n+                self.assertEqual(rows[0]["publishedUtc"], old["publishedUtc"])\n+                stats = {key: 0 for key in ("skipped_untitled", "skipped_too_old", "skipped_future_date")}\n+                self.assertTrue(dedupe._is_fresh(rows[0], now, stats))\n+                self.assertFalse(dedupe._is_unseen(rows[0], {common.norm_url(old["url"])}, [])[0])\n+                future = {**rows[0], "publishedUtc": "2026-11-01T00:00Z"}\n+                self.assertFalse(dedupe._is_fresh(future, now, stats))\n+                path.write_text(json.dumps({"expiresUtc": "2026-09-29T00:00Z", "items": [old]}))\n+                self.assertEqual(dedupe._load_raw_items(), [])\n+                self.assertFalse(dedupe._is_fresh(rows[0], now + timedelta(days=8), stats))\n+\n     def test_safety_findings_and_codeshare_are_retained_before_enrichment(self):\n         for title in (\n             "Vietnam Airlines Munich runway incident: Investigation reveals brakes activated during takeoff roll",\n@@ -70,7 +70,7 @@\n     "facts": [\n         {"factId": "F1",\n          "claim": "NTSB 已對丹佛 737 衝出跑道事件展開調查",\n-         "sourceQuote": "NTSB said it is investigating a runway excursion"},\n+         "sourceQuote": "The NTSB said it is investigating a runway excursion involving a United Airlines Boeing 737-8 at Denver International Airport during landing on July 25."},\n         {"factId": "F2",\n          "claim": "無人受傷",\n          "sourceQuote": "No injuries were reported."},\n@@ -122,7 +122,7 @@\n          "sourceQuote": "rose 8.2% in June 2026"},\n         {"factId": "F2",\n          "claim": "需求以貨運噸公里計算並與 2025 年六月比較",\n-         "sourceQuote": "measured in cargo tonne-kilometers"},\n+         "sourceQuote": "measured in cargo tonne-kilometers, rose 8.2% in June 2026 compared with June 2025."},\n     ],\n     "headlineSupportedBy": ["F1"],\n     "summarySupportedBy": ["F1", "F2"],\n@@ -150,6 +150,22 @@\n     "incident": None,\n }\n \n+# Fixtures must satisfy the current per-entity evidence contract. Keep the\n+# source quote intact while normalizing displayed Chinese as production does.\n+DRAFT_SAFETY["entityEvidence"] = [\n+    {"entityType": key, "value": value,\n+     "sourceQuote": DRAFT_SAFETY["facts"][0]["sourceQuote"],\n+     "sourceUrl": "https://www.ntsb.gov/news/press-releases/denver-737"}\n+    for key, values in DRAFT_SAFETY["entities"].items() for value in values\n+]\n+DRAFT_BIZ["entityEvidence"] = [{\n+    "entityType": "organizations", "value": "International Air Transport Association",\n+    "sourceQuote": "The International Air Transport Association reported",\n+    "sourceUrl": "https://www.iata.org/en/pressroom/2026-releases/2026-07-24-01/",\n+}]\n+DRAFT_SAFETY = write.normalize_reader_copy(DRAFT_SAFETY, write._S2T)\n+DRAFT_BIZ = write.normalize_reader_copy(DRAFT_BIZ, write._S2T)\n+\n _pass = 0\n _fail = 0\n \n@@ -429,6 +445,8 @@ def test_group_cap_and_unique_ids():\n          "sourceQuote": "operational details for certificate holders"},\n     ]\n     cap_draft["summarySupportedBy"] = ["F1", "F2"]\n+    cap_draft["entities"] = dict(_EMPTY_ENTITIES)\n+    cap_draft["entityEvidence"] = []\n     original = write.draft_group\n     write.draft_group = lambda client, group: cap_draft\n     try:\n@@ -589,7 +607,8 @@ def test_extract_json_and_validate_draft():\n         retrying_provider, cargo_group, tries=2)\n     check(retrying_provider.calls == 2,\n           "short body automatically triggers the allowed retry")\n-    check(candidate == DRAFT_BIZ and facts,\n+    check(candidate["zh"] == DRAFT_BIZ["zh"]\n+          and write.validate_draft(candidate) is None and facts,\n           "length-compliant retry survives validation and quote checks")\n     check(write.validate_draft("nope") is not None, "non-dict rejected")\n     broken = json.loads(json.dumps(DRAFT_BIZ))\n@@ -701,8 +720,8 @@ def test_provider_failover():\n           "fallback publication does not create review entries")\n     check(primary.calls == 1,\n           f"dead primary is never called again, got {primary.calls} calls")\n-    check(middle.calls == 3 and backup.calls == 3,\n-          "surviving providers tried once per group")\n+    check(middle.calls == 6 and backup.calls == 3,\n+          "first responsive fallback gets one bounded repair per group")\n     pending = load(DATA / "pending.json")\n     ids = [g["id"] for g in pending["groups"]]\n     check(ids == ["g-20260726-0503-3"],\n@@ -848,6 +867,7 @@ def test_editorial_gate():\n     fabricated = json.loads(json.dumps(DRAFT_BIZ))\n     fabricated["facts"] = [{"factId": "F1", "claim": "完全捏造的主張",\n                             "sourceQuote": "this text appears nowhere"}]\n+    fabricated["summarySupportedBy"] = ["F1"]\n     partial = json.loads(json.dumps(DRAFT_BIZ))\n     partial["facts"] = [\n         {"factId": "F1", "claim": "FAA 發布鬧事乘客執法聲明",\n@@ -859,6 +879,8 @@ def test_editorial_gate():\n     ]\n     partial["headlineSupportedBy"] = ["F1"]\n     partial["summarySupportedBy"] = ["F1", "F2"]\n+    partial["entities"] = dict(_EMPTY_ENTITIES)\n+    partial["entityEvidence"] = []\n     # g1 -> editorial reject; g2 -> all quotes fabricated; g3 -> one good.\n     solo = FakeProvider("gemini", [reject_draft, fabricated, partial])\n \n@@ -1042,3 +1064,4 @@ def main():\n \n if __name__ == "__main__":\n     main()\n+