提交原始紀錄

完整差異

@@ -12,6 +12,7 @@ on:\n       - "pipeline/flightwatch.py"\n       - "pipeline/flightnews.py"\n       - "pipeline/write.py"\n+      - "pipeline/provider_health.py"\n       - "pipeline/common.py"\n       - "pipeline/dedupe.py"\n       - "pipeline/fulltext.py"\n@@ -56,6 +57,7 @@ jobs:\n           python tests/test_draft_recovery.py\n           python tests/test_dedupe.py\n           python tests/test_write_offline.py\n+          python tests/test_provider_health.py\n           python tests/test_news_freshness.py\n \n   update:\n@@ -88,6 +90,9 @@ jobs:\n       - name: Test news freshness invariants\n         run: python tests/test_news_freshness.py\n \n+      - name: Test provider cooldown persistence\n+        run: python tests/test_provider_health.py\n+\n       - name: Test summary boundary guard\n         run: python tests/test_summary_guard.py\n \n@@ -8,21 +8,15 @@ on:\n       - ".github/deploy-trigger"\n       - "data/articles/scheduled-*.json"\n \n-concurrency:\n-  group: skytical-pages-publish\n-  cancel-in-progress: false\n-  queue: max\n-\n jobs:\n-  deploy:\n+  build:\n     runs-on: ubuntu-latest\n+    concurrency:\n+      group: skytical-manual-article-build\n+      cancel-in-progress: true\n     permissions:\n       contents: read\n       pages: write\n-      id-token: write\n-    environment:\n-      name: github-pages\n-      url: ${{ steps.deployment.outputs.page_url }}\n     steps:\n       - uses: actions/checkout@v5\n \n@@ -96,8 +90,20 @@ jobs:\n       - uses: actions/upload-pages-artifact@v5\n         with:\n           path: site\n-          # Preserve v3 packaging, including .nojekyll and other site dotfiles.\n           include-hidden-files: true\n \n+  deploy:\n+    needs: build\n+    runs-on: ubuntu-latest\n+    concurrency:\n+      group: skytical-pages-publish\n+      cancel-in-progress: false\n+    permissions:\n+      pages: write\n+      id-token: write\n+    environment:\n+      name: github-pages\n+      url: ${{ steps.deployment.outputs.page_url }}\n+    steps:\n       - uses: actions/deploy-pages@v5\n         id: deployment\n@@ -29,15 +29,13 @@ on:\n       - "static/radar.js"\n       - "templates/radar.html"\n \n-concurrency:\n-  group: ${{ github.event_name == 'pull_request' && format('radar-snapshot-pr-{0}', github.event.pull_request.number) || 'skytical-pages-publish' }}\n-  cancel-in-progress: false\n-  queue: max\n-\n jobs:\n   test:\n     if: github.event_name == 'pull_request'\n     runs-on: ubuntu-latest\n+    concurrency:\n+      group: radar-snapshot-pr-${{ github.event.pull_request.number }}\n+      cancel-in-progress: true\n     permissions:\n       contents: read\n     steps:\n@@ -52,16 +50,17 @@ jobs:\n           python tests/test_radar_snapshot.py\n           python tests/test_radar.py\n \n-  deploy:\n+  build:\n     if: github.event_name != 'pull_request'\n     runs-on: ubuntu-latest\n+    concurrency:\n+      group: skytical-radar-build\n+      cancel-in-progress: true\n     permissions:\n       contents: read\n       pages: write\n-      id-token: write\n-    environment:\n-      name: github-pages\n-      url: ${{ steps.deployment.outputs.page_url }}\n+    outputs:\n+      should_deploy: ${{ steps.snapshot.outputs.should_deploy }}\n     steps:\n       - uses: actions/checkout@v5\n \n@@ -78,6 +77,7 @@ jobs:\n       - run: pip install -r requirements.txt\n \n       - name: Test and build current radar snapshot\n+        id: snapshot\n         env:\n           AVWIRE_USAGE_TOKEN: ${{ secrets.AVWIRE_USAGE_TOKEN }}\n           AVWIRE_MANUAL_TOKEN: ${{ secrets.AVWIRE_MANUAL_TOKEN }}\n@@ -87,9 +87,16 @@ jobs:\n         run: |\n           python tests/test_radar_snapshot.py\n           python pipeline/build.py\n-          python pipeline/radar_snapshot.py\n+          python pipeline/radar_snapshot.py --optional\n+          if [ -f site/assets/radar.json ]; then\n+            echo "should_deploy=true" >> "$GITHUB_OUTPUT"\n+          else\n+            echo "::warning::ADSB.lol is temporarily unavailable; keeping the last deployed radar snapshot"\n+            echo "should_deploy=false" >> "$GITHUB_OUTPUT"\n+          fi\n \n       - name: Scan publishable output\n+        if: steps.snapshot.outputs.should_deploy == 'true'\n         run: |\n           scan() {\n             set +e\n@@ -109,9 +116,24 @@ jobs:\n           echo "scan: site/ and data/ clean"\n \n       - uses: actions/upload-pages-artifact@v5\n+        if: steps.snapshot.outputs.should_deploy == 'true'\n         with:\n           path: site\n           include-hidden-files: true\n \n+  deploy:\n+    if: needs.build.outputs.should_deploy == 'true'\n+    needs: build\n+    runs-on: ubuntu-latest\n+    concurrency:\n+      group: skytical-pages-publish\n+      cancel-in-progress: false\n+    permissions:\n+      pages: write\n+      id-token: write\n+    environment:\n+      name: github-pages\n+      url: ${{ steps.deployment.outputs.page_url }}\n+    steps:\n       - uses: actions/deploy-pages@v5\n         id: deployment\n@@ -0,0 +1,93 @@\n+"""Persistent provider-platform cooldowns for SKYTICAL drafting.\n+\n+The writer already has a run-local circuit breaker. This module persists only\n+platform-level infrastructure failures that are unlikely to recover within the\n+next hourly run, so a known-bad free provider is not hammered every hour.\n+\n+No prompts, source text, API keys or raw exception messages are stored here.\n+"""\n+from __future__ import annotations\n+\n+from datetime import timedelta\n+\n+from common import DATA_DIR, iso_minute, load_json, now_utc, parse_iso, save_json\n+\n+HEALTH_PATH = DATA_DIR / "provider-health.json"\n+SCHEMA_VERSION = 1\n+COOLDOWNS = {\n+    "transient": timedelta(hours=1),\n+    "quota": timedelta(hours=6),\n+    "auth": timedelta(hours=12),\n+}\n+\n+\n+def _load(now=None) -> dict:\n+    now = now or now_utc()\n+    payload = load_json(HEALTH_PATH, {})\n+    rows = payload.get("platforms") if isinstance(payload, dict) else None\n+    if not isinstance(rows, dict):\n+        rows = {}\n+    active = {}\n+    for name, row in rows.items():\n+        if not isinstance(row, dict):\n+            continue\n+        try:\n+            until = parse_iso(str(row.get("cooldownUntilUtc") or ""))\n+        except (TypeError, ValueError):\n+            continue\n+        if until <= now:\n+            continue\n+        failure_class = str(row.get("failureClass") or "")\n+        if failure_class not in COOLDOWNS:\n+            continue\n+        active[str(name)] = {\n+            "failureClass": failure_class,\n+            "cooldownUntilUtc": iso_minute(until),\n+            "updatedUtc": str(row.get("updatedUtc") or ""),\n+        }\n+    return {"schemaVersion": SCHEMA_VERSION, "platforms": active}\n+\n+\n+def _save(payload: dict) -> None:\n+    save_json(HEALTH_PATH, payload)\n+\n+\n+def active_platforms(now=None) -> set[str]:\n+    """Return provider platform names still inside a persisted cooldown."""\n+    return set(_load(now)["platforms"])\n+\n+\n+def cooldown_rows(now=None) -> dict:\n+    """Return sanitized active cooldown metadata for diagnostics."""\n+    return dict(_load(now)["platforms"])\n+\n+\n+def mark_failure(platform: str, failure_class: str, now=None):\n+    """Persist a platform cooldown and return its expiry timestamp."""\n+    platform = str(platform or "").strip()\n+    if not platform:\n+        return None\n+    kind = failure_class if failure_class in COOLDOWNS else "transient"\n+    now = now or now_utc()\n+    until = now + COOLDOWNS[kind]\n+    payload = _load(now)\n+    payload["platforms"][platform] = {\n+        "failureClass": kind,\n+        "cooldownUntilUtc": iso_minute(until),\n+        "updatedUtc": iso_minute(now),\n+    }\n+    _save(payload)\n+    return until\n+\n+\n+def mark_success(platform: str, now=None) -> bool:\n+    """Clear a platform cooldown after a completed provider response."""\n+    platform = str(platform or "").strip()\n+    if not platform:\n+        return False\n+    payload = _load(now)\n+    if platform not in payload["platforms"]:\n+        return False\n+    del payload["platforms"][platform]\n+    _save(payload)\n+    return True\n@@ -66,6 +66,7 @@\n )\n import flightnews\n import archive_context\n+import provider_health\n from fulltext import FULLTEXT_PROMPT_CHARS, enrich_pending\n from providers import (\n     ProviderAuthError,\n@@ -1396,14 +1397,16 @@ def _record_transient_platform_failure(provider, exc, alive: list,\n     alive[:] = [candidate for candidate in alive\n                 if candidate.name != name]\n     dead_platforms.add(name)\n+    provider_health.mark_failure(name, "transient")\n     print(f"write: '{name}' hit {count} consecutive transient failures; "\n-          "disabling the platform for this run")\n+          "disabling the platform for this run and cooling it down for 1 hour")\n     return True\n \n \n def _record_provider_response(provider, failure_counts: dict) -> None:\n     """A completed response proves the platform circuit should recover."""\n     failure_counts.pop(provider.name, None)\n+    provider_health.mark_success(provider.name)\n \n \n def _normalize_reject_reason(candidate: dict) -> None:\n@@ -2460,6 +2463,7 @@ def _process_flight_events(queue, providers, dead_platforms, dead_auth,\n                       f"disabling all '{provider.name}' providers")\n                 dead_platforms.add(provider.name)\n                 dead_auth.add(provider.name)\n+                provider_health.mark_failure(provider.name, "auth")\n                 continue\n             except ProviderQuotaError as exc:\n                 run_trace.add_duration("drafting", drafting_started)\n@@ -2470,6 +2474,7 @@ def _process_flight_events(queue, providers, dead_platforms, dead_auth,\n                 print(f"write: {provider.label} quota exhausted ({exc}); "\n                       f"disabling all '{provider.name}' providers")\n                 dead_platforms.add(provider.name)\n+                provider_health.mark_failure(provider.name, "quota")\n                 continue\n             except ProviderError as exc:\n                 run_trace.add_duration("drafting", drafting_started)\n@@ -2683,7 +2688,16 @@ def main() -> None:\n         save_json(REVIEW_PATH, [])\n \n     # --- 2. draft the pending groups through the provider chain ----------\n-    providers = build_providers()\n+    configured_providers = build_providers()\n+    persisted_cooldowns = provider_health.active_platforms(now)\n+    providers = [p for p in configured_providers\n+                 if p.name not in persisted_cooldowns]\n+    if persisted_cooldowns:\n+        rows = provider_health.cooldown_rows(now)\n+        details = ", ".join(\n+            f"{name} until {rows[name]['cooldownUntilUtc']}"\n+            for name in sorted(persisted_cooldowns) if name in rows)\n+        print(f"write: persisted provider cooldowns active: {details}")\n     major_deferred = 0\n     if not providers:\n         print("write: no LLM API key set (OPENCODE_API_KEY / "\n@@ -2833,6 +2847,7 @@ def main() -> None:\n                     dead_auth.add(provider.name)\n                     dead_platforms.add(provider.name)\n                     transient_failure_counts.pop(provider.name, None)\n+                    provider_health.mark_failure(provider.name, "auth")\n                     run_trace.disable_last_provider()\n                     continue\n                 except ProviderQuotaError as exc:\n@@ -2843,6 +2858,7 @@ def main() -> None:\n                     alive[:] = [p for p in alive if p.name != provider.name]\n                     dead_platforms.add(provider.name)\n                     transient_failure_counts.pop(provider.name, None)\n+                    provider_health.mark_failure(provider.name, "quota")\n                     run_trace.disable_last_provider()\n                     continue\n                 except (ProviderError, ProviderCallTimeout) as exc:\n@@ -3002,7 +3018,9 @@ def main() -> None:\n         f"{p.label}={p.http_calls}" for p in providers\n         if getattr(p, "http_calls", 0)) or "none"\n     print(f"write stats: ai_calls {calls_note} | http_calls {http_note}")\n-    if providers and dead_auth and dead_auth == {p.name for p in providers}:\n+    if configured_providers and dead_auth and dead_auth == {\n+            p.name for p in configured_providers\n+            if p.name not in persisted_cooldowns}:\n         # Every configured platform failed authentication: turn the Actions\n         # run red instead of silently green.\n         raise SystemExit(1)\n@@ -0,0 +1,59 @@\n+"""Offline checks for persisted provider cooldowns."""\n+from __future__ import annotations\n+\n+import json\n+import sys\n+import tempfile\n+from datetime import datetime, timedelta, timezone\n+from pathlib import Path\n+\n+ROOT = Path(__file__).resolve().parent.parent\n+sys.path.insert(0, str(ROOT / "pipeline"))\n+\n+import provider_health  # noqa: E402\n+\n+\n+def main() -> None:\n+    base = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc)\n+    with tempfile.TemporaryDirectory() as tmp:\n+        provider_health.HEALTH_PATH = Path(tmp) / "provider-health.json"\n+\n+        until = provider_health.mark_failure("gemini", "transient", base)\n+        assert until == base + timedelta(hours=1)\n+        assert provider_health.active_platforms(base + timedelta(minutes=59)) == {\n+            "gemini"\n+        }\n+        assert provider_health.active_platforms(base + timedelta(hours=1, minutes=1)) == set()\n+\n+        provider_health.mark_failure("nvidia", "quota", base)\n+        rows = provider_health.cooldown_rows(base + timedelta(hours=5))\n+        assert rows["nvidia"]["failureClass"] == "quota"\n+        assert "nvidia" not in provider_health.active_platforms(\n+            base + timedelta(hours=6, minutes=1)\n+        )\n+\n+        provider_health.mark_failure("anthropic", "auth", base)\n+        assert "anthropic" in provider_health.active_platforms(\n+            base + timedelta(hours=11, minutes=59)\n+        )\n+        assert provider_health.mark_success(\n+            "anthropic", base + timedelta(hours=1)\n+        ) is True\n+        assert "anthropic" not in provider_health.active_platforms(\n+            base + timedelta(hours=1)\n+        )\n+\n+        payload = json.loads(\n+            provider_health.HEALTH_PATH.read_text(encoding="utf-8")\n+        )\n+        assert payload["schemaVersion"] == 1\n+        assert all(\n+            set(row) <= {"failureClass", "cooldownUntilUtc", "updatedUtc"}\n+            for row in payload["platforms"].values()\n+        )\n+\n+    print("test_provider_health: OK")\n+\n+\n+if __name__ == "__main__":\n+    main()\n@@ -231,24 +231,34 @@ def main() -> None:\n         ROOT / ".github" / "workflows" / "manual-article-deploy.yml",\n         ROOT / ".github" / "workflows" / "radar-snapshot.yml",\n     ]\n-    for workflow in workflows:\n-        source = workflow.read_text(encoding="utf-8")\n+    workflow_sources = {\n+        workflow.name: workflow.read_text(encoding="utf-8")\n+        for workflow in workflows\n+    }\n+    for source in workflow_sources.values():\n         assert "python pipeline/radar_snapshot.py" in source\n         assert source.index("python pipeline/radar_snapshot.py") < source.index(\n             "actions/upload-pages-artifact@v5")\n+        # Only the final deploy job may occupy the shared Pages queue.\n+        assert source.count("group: skytical-pages-publish") == 1\n+        assert "cancel-in-progress: false" in source\n \n-        workflow_header = source.split("\njobs:\n", 1)[0]\n-        if workflow.name in {"hourly.yml", "briefing.yml"}:\n-            # Data generation is intentionally independent of the Pages queue;\n-            # only the deploy job remains serialized.\n-            assert "group: skytical-pages-publish" in source\n-            assert "cancel-in-progress: false" in source\n-        else:\n-            assert "skytical-pages-publish" in workflow_header\n-            assert re.search(r"(?m)^  queue: max$", workflow_header)\n-            assert re.search(r"(?m)^  cancel-in-progress: false$", workflow_header)\n-        if workflow.name == "radar-snapshot.yml":\n-            assert "github.event.pull_request.number" in workflow_header\n+    manual_workflow = workflow_sources["manual-article-deploy.yml"]\n+    assert "group: skytical-manual-article-build" in manual_workflow\n+    assert manual_workflow.index("  build:") < manual_workflow.index("  deploy:")\n+    assert manual_workflow.index("actions/upload-pages-artifact@v5") < (\n+        manual_workflow.index("actions/deploy-pages@v5")\n+    )\n+\n+    radar_workflow = workflow_sources["radar-snapshot.yml"]\n+    assert "group: radar-snapshot-pr-${{ github.event.pull_request.number }}" in (\n+        radar_workflow\n+    )\n+    assert "group: skytical-radar-build" in radar_workflow\n+    assert "python pipeline/radar_snapshot.py --optional" in radar_workflow\n+    assert "should_deploy=false" in radar_workflow\n+    assert "keeping the last deployed radar snapshot" in radar_workflow\n+    assert "if: needs.build.outputs.should_deploy == 'true'" in radar_workflow\n \n     assert any(\n         "本文由自動化系統彙整生成，內容以原始來源為準 • "\n@@ -828,6 +828,9 @@ def test_model_chain_and_routing_policy():\n     ]\n     check(len(disabled_attempts) == 1,\n           "circuit-breaking failure is marked disabled in the run ledger")\n+    health = load(DATA / "provider-health.json")\n+    check(health["platforms"]["gemini"]["failureClass"] == "transient",\n+          "transient platform circuit persists a cross-run cooldown")\n     check(write._is_transient_platform_failure(\n               providers.ProviderError("finishReason MAX_TOKENS")) is False,\n           "model truncation does not trip the platform circuit")\n@@ -1021,6 +1024,11 @@ def test_all_providers_auth_dead():\n         write.build_providers = original\n \n     check(exited == 1, f"SystemExit(1) when all providers fail auth, got {exited}")\n+    health = load(DATA / "provider-health.json")\n+    check(set(health["platforms"]) == {"gemini", "nvidia"}\n+          and all(row["failureClass"] == "auth"\n+                  for row in health["platforms"].values()),\n+          "auth-dead platforms persist a cross-run cooldown")\n     check(not list((DATA / "articles").glob("*.json")),\n           "no articles when every provider is auth-dead")\n     check((DATA / "pending.json").read_bytes() == pending_before,