{ "sha": "87d95e47c1a10e28a7bd72f05be3215781462b48", "message": "Improve P0 pipeline resilience and deploy isolation\n\nDecouple radar/manual builds from the shared Pages deploy queue, degrade gracefully on ADSB.lol outages, and persist provider cooldowns across hourly runs.", "author": { "login": "nnneeeooo-nnneeeooo", "name": "", "email": "" }, "committer": { "login": "web-flow", "name": "", "email": "" }, "created_at": "2026-10-01T14:53:35Z", "files": [ { "filename": ".github/workflows/hourly.yml", "previous_filename": "", "status": "modified", "additions": 5, "deletions": 0, "changes": 5 }, { "filename": ".github/workflows/manual-article-deploy.yml", "previous_filename": "", "status": "modified", "additions": 17, "deletions": 11, "changes": 28 }, { "filename": ".github/workflows/radar-snapshot.yml", "previous_filename": "", "status": "modified", "additions": 33, "deletions": 11, "changes": 44 }, { "filename": "pipeline/provider_health.py", "previous_filename": "", "status": "added", "additions": 93, "deletions": 0, "changes": 93 }, { "filename": "pipeline/write.py", "previous_filename": "", "status": "modified", "additions": 21, "deletions": 3, "changes": 24 }, { "filename": "tests/test_provider_health.py", "previous_filename": "", "status": "added", "additions": 59, "deletions": 0, "changes": 59 }, { "filename": "tests/test_radar.py", "previous_filename": "", "status": "modified", "additions": 24, "deletions": 14, "changes": 38 }, { "filename": "tests/test_write_offline.py", "previous_filename": "", "status": "modified", "additions": 8, "deletions": 0, "changes": 8 } ], "diff": "@@ -12,6 +12,7 @@ on:\\n - \"pipeline/flightwatch.py\"\\n - \"pipeline/flightnews.py\"\\n - \"pipeline/write.py\"\\n+ - \"pipeline/provider_health.py\"\\n - \"pipeline/common.py\"\\n - \"pipeline/dedupe.py\"\\n - \"pipeline/fulltext.py\"\\n@@ -56,6 +57,7 @@ jobs:\\n python tests/test_draft_recovery.py\\n python tests/test_dedupe.py\\n python tests/test_write_offline.py\\n+ python tests/test_provider_health.py\\n python tests/test_news_freshness.py\\n \\n update:\\n@@ -88,6 +90,9 @@ jobs:\\n - name: Test news freshness invariants\\n run: python tests/test_news_freshness.py\\n \\n+ - name: Test provider cooldown persistence\\n+ run: python tests/test_provider_health.py\\n+\\n - name: Test summary boundary guard\\n run: python tests/test_summary_guard.py\\n \\n@@ -8,21 +8,15 @@ on:\\n - \".github/deploy-trigger\"\\n - \"data/articles/scheduled-*.json\"\\n \\n-concurrency:\\n- group: skytical-pages-publish\\n- cancel-in-progress: false\\n- queue: max\\n-\\n jobs:\\n- deploy:\\n+ build:\\n runs-on: ubuntu-latest\\n+ concurrency:\\n+ group: skytical-manual-article-build\\n+ cancel-in-progress: true\\n permissions:\\n contents: read\\n pages: write\\n- id-token: write\\n- environment:\\n- name: github-pages\\n- url: ${{ steps.deployment.outputs.page_url }}\\n steps:\\n - uses: actions/checkout@v5\\n \\n@@ -96,8 +90,20 @@ jobs:\\n - uses: actions/upload-pages-artifact@v5\\n with:\\n path: site\\n- # Preserve v3 packaging, including .nojekyll and other site dotfiles.\\n include-hidden-files: true\\n \\n+ deploy:\\n+ needs: build\\n+ runs-on: ubuntu-latest\\n+ concurrency:\\n+ group: skytical-pages-publish\\n+ cancel-in-progress: false\\n+ permissions:\\n+ pages: write\\n+ id-token: write\\n+ environment:\\n+ name: github-pages\\n+ url: ${{ steps.deployment.outputs.page_url }}\\n+ steps:\\n - uses: actions/deploy-pages@v5\\n id: deployment\\n@@ -29,15 +29,13 @@ on:\\n - \"static/radar.js\"\\n - \"templates/radar.html\"\\n \\n-concurrency:\\n- group: ${{ github.event_name == 'pull_request' && format('radar-snapshot-pr-{0}', github.event.pull_request.number) || 'skytical-pages-publish' }}\\n- cancel-in-progress: false\\n- queue: max\\n-\\n jobs:\\n test:\\n if: github.event_name == 'pull_request'\\n runs-on: ubuntu-latest\\n+ concurrency:\\n+ group: radar-snapshot-pr-${{ github.event.pull_request.number }}\\n+ cancel-in-progress: true\\n permissions:\\n contents: read\\n steps:\\n@@ -52,16 +50,17 @@ jobs:\\n python tests/test_radar_snapshot.py\\n python tests/test_radar.py\\n \\n- deploy:\\n+ build:\\n if: github.event_name != 'pull_request'\\n runs-on: ubuntu-latest\\n+ concurrency:\\n+ group: skytical-radar-build\\n+ cancel-in-progress: true\\n permissions:\\n contents: read\\n pages: write\\n- id-token: write\\n- environment:\\n- name: github-pages\\n- url: ${{ steps.deployment.outputs.page_url }}\\n+ outputs:\\n+ should_deploy: ${{ steps.snapshot.outputs.should_deploy }}\\n steps:\\n - uses: actions/checkout@v5\\n \\n@@ -78,6 +77,7 @@ jobs:\\n - run: pip install -r requirements.txt\\n \\n - name: Test and build current radar snapshot\\n+ id: snapshot\\n env:\\n AVWIRE_USAGE_TOKEN: ${{ secrets.AVWIRE_USAGE_TOKEN }}\\n AVWIRE_MANUAL_TOKEN: ${{ secrets.AVWIRE_MANUAL_TOKEN }}\\n@@ -87,9 +87,16 @@ jobs:\\n run: |\\n python tests/test_radar_snapshot.py\\n python pipeline/build.py\\n- python pipeline/radar_snapshot.py\\n+ python pipeline/radar_snapshot.py --optional\\n+ if [ -f site/assets/radar.json ]; then\\n+ echo \"should_deploy=true\" >> \"$GITHUB_OUTPUT\"\\n+ else\\n+ echo \"::warning::ADSB.lol is temporarily unavailable; keeping the last deployed radar snapshot\"\\n+ echo \"should_deploy=false\" >> \"$GITHUB_OUTPUT\"\\n+ fi\\n \\n - name: Scan publishable output\\n+ if: steps.snapshot.outputs.should_deploy == 'true'\\n run: |\\n scan() {\\n set +e\\n@@ -109,9 +116,24 @@ jobs:\\n echo \"scan: site/ and data/ clean\"\\n \\n - uses: actions/upload-pages-artifact@v5\\n+ if: steps.snapshot.outputs.should_deploy == 'true'\\n with:\\n path: site\\n include-hidden-files: true\\n \\n+ deploy:\\n+ if: needs.build.outputs.should_deploy == 'true'\\n+ needs: build\\n+ runs-on: ubuntu-latest\\n+ concurrency:\\n+ group: skytical-pages-publish\\n+ cancel-in-progress: false\\n+ permissions:\\n+ pages: write\\n+ id-token: write\\n+ environment:\\n+ name: github-pages\\n+ url: ${{ steps.deployment.outputs.page_url }}\\n+ steps:\\n - uses: actions/deploy-pages@v5\\n id: deployment\\n@@ -0,0 +1,93 @@\\n+\"\"\"Persistent provider-platform cooldowns for SKYTICAL drafting.\\n+\\n+The writer already has a run-local circuit breaker. This module persists only\\n+platform-level infrastructure failures that are unlikely to recover within the\\n+next hourly run, so a known-bad free provider is not hammered every hour.\\n+\\n+No prompts, source text, API keys or raw exception messages are stored here.\\n+\"\"\"\\n+from __future__ import annotations\\n+\\n+from datetime import timedelta\\n+\\n+from common import DATA_DIR, iso_minute, load_json, now_utc, parse_iso, save_json\\n+\\n+HEALTH_PATH = DATA_DIR / \"provider-health.json\"\\n+SCHEMA_VERSION = 1\\n+COOLDOWNS = {\\n+ \"transient\": timedelta(hours=1),\\n+ \"quota\": timedelta(hours=6),\\n+ \"auth\": timedelta(hours=12),\\n+}\\n+\\n+\\n+def _load(now=None) -> dict:\\n+ now = now or now_utc()\\n+ payload = load_json(HEALTH_PATH, {})\\n+ rows = payload.get(\"platforms\") if isinstance(payload, dict) else None\\n+ if not isinstance(rows, dict):\\n+ rows = {}\\n+ active = {}\\n+ for name, row in rows.items():\\n+ if not isinstance(row, dict):\\n+ continue\\n+ try:\\n+ until = parse_iso(str(row.get(\"cooldownUntilUtc\") or \"\"))\\n+ except (TypeError, ValueError):\\n+ continue\\n+ if until <= now:\\n+ continue\\n+ failure_class = str(row.get(\"failureClass\") or \"\")\\n+ if failure_class not in COOLDOWNS:\\n+ continue\\n+ active[str(name)] = {\\n+ \"failureClass\": failure_class,\\n+ \"cooldownUntilUtc\": iso_minute(until),\\n+ \"updatedUtc\": str(row.get(\"updatedUtc\") or \"\"),\\n+ }\\n+ return {\"schemaVersion\": SCHEMA_VERSION, \"platforms\": active}\\n+\\n+\\n+def _save(payload: dict) -> None:\\n+ save_json(HEALTH_PATH, payload)\\n+\\n+\\n+def active_platforms(now=None) -> set[str]:\\n+ \"\"\"Return provider platform names still inside a persisted cooldown.\"\"\"\\n+ return set(_load(now)[\"platforms\"])\\n+\\n+\\n+def cooldown_rows(now=None) -> dict:\\n+ \"\"\"Return sanitized active cooldown metadata for diagnostics.\"\"\"\\n+ return dict(_load(now)[\"platforms\"])\\n+\\n+\\n+def mark_failure(platform: str, failure_class: str, now=None):\\n+ \"\"\"Persist a platform cooldown and return its expiry timestamp.\"\"\"\\n+ platform = str(platform or \"\").strip()\\n+ if not platform:\\n+ return None\\n+ kind = failure_class if failure_class in COOLDOWNS else \"transient\"\\n+ now = now or now_utc()\\n+ until = now + COOLDOWNS[kind]\\n+ payload = _load(now)\\n+ payload[\"platforms\"][platform] = {\\n+ \"failureClass\": kind,\\n+ \"cooldownUntilUtc\": iso_minute(until),\\n+ \"updatedUtc\": iso_minute(now),\\n+ }\\n+ _save(payload)\\n+ return until\\n+\\n+\\n+def mark_success(platform: str, now=None) -> bool:\\n+ \"\"\"Clear a platform cooldown after a completed provider response.\"\"\"\\n+ platform = str(platform or \"\").strip()\\n+ if not platform:\\n+ return False\\n+ payload = _load(now)\\n+ if platform not in payload[\"platforms\"]:\\n+ return False\\n+ del payload[\"platforms\"][platform]\\n+ _save(payload)\\n+ return True\\n@@ -66,6 +66,7 @@\\n )\\n import flightnews\\n import archive_context\\n+import provider_health\\n from fulltext import FULLTEXT_PROMPT_CHARS, enrich_pending\\n from providers import (\\n ProviderAuthError,\\n@@ -1396,14 +1397,16 @@ def _record_transient_platform_failure(provider, exc, alive: list,\\n alive[:] = [candidate for candidate in alive\\n if candidate.name != name]\\n dead_platforms.add(name)\\n+ provider_health.mark_failure(name, \"transient\")\\n print(f\"write: '{name}' hit {count} consecutive transient failures; \"\\n- \"disabling the platform for this run\")\\n+ \"disabling the platform for this run and cooling it down for 1 hour\")\\n return True\\n \\n \\n def _record_provider_response(provider, failure_counts: dict) -> None:\\n \"\"\"A completed response proves the platform circuit should recover.\"\"\"\\n failure_counts.pop(provider.name, None)\\n+ provider_health.mark_success(provider.name)\\n \\n \\n def _normalize_reject_reason(candidate: dict) -> None:\\n@@ -2460,6 +2463,7 @@ def _process_flight_events(queue, providers, dead_platforms, dead_auth,\\n f\"disabling all '{provider.name}' providers\")\\n dead_platforms.add(provider.name)\\n dead_auth.add(provider.name)\\n+ provider_health.mark_failure(provider.name, \"auth\")\\n continue\\n except ProviderQuotaError as exc:\\n run_trace.add_duration(\"drafting\", drafting_started)\\n@@ -2470,6 +2474,7 @@ def _process_flight_events(queue, providers, dead_platforms, dead_auth,\\n print(f\"write: {provider.label} quota exhausted ({exc}); \"\\n f\"disabling all '{provider.name}' providers\")\\n dead_platforms.add(provider.name)\\n+ provider_health.mark_failure(provider.name, \"quota\")\\n continue\\n except ProviderError as exc:\\n run_trace.add_duration(\"drafting\", drafting_started)\\n@@ -2683,7 +2688,16 @@ def main() -> None:\\n save_json(REVIEW_PATH, [])\\n \\n # --- 2. draft the pending groups through the provider chain ----------\\n- providers = build_providers()\\n+ configured_providers = build_providers()\\n+ persisted_cooldowns = provider_health.active_platforms(now)\\n+ providers = [p for p in configured_providers\\n+ if p.name not in persisted_cooldowns]\\n+ if persisted_cooldowns:\\n+ rows = provider_health.cooldown_rows(now)\\n+ details = \", \".join(\\n+ f\"{name} until {rows[name]['cooldownUntilUtc']}\"\\n+ for name in sorted(persisted_cooldowns) if name in rows)\\n+ print(f\"write: persisted provider cooldowns active: {details}\")\\n major_deferred = 0\\n if not providers:\\n print(\"write: no LLM API key set (OPENCODE_API_KEY / \"\\n@@ -2833,6 +2847,7 @@ def main() -> None:\\n dead_auth.add(provider.name)\\n dead_platforms.add(provider.name)\\n transient_failure_counts.pop(provider.name, None)\\n+ provider_health.mark_failure(provider.name, \"auth\")\\n run_trace.disable_last_provider()\\n continue\\n except ProviderQuotaError as exc:\\n@@ -2843,6 +2858,7 @@ def main() -> None:\\n alive[:] = [p for p in alive if p.name != provider.name]\\n dead_platforms.add(provider.name)\\n transient_failure_counts.pop(provider.name, None)\\n+ provider_health.mark_failure(provider.name, \"quota\")\\n run_trace.disable_last_provider()\\n continue\\n except (ProviderError, ProviderCallTimeout) as exc:\\n@@ -3002,7 +3018,9 @@ def main() -> None:\\n f\"{p.label}={p.http_calls}\" for p in providers\\n if getattr(p, \"http_calls\", 0)) or \"none\"\\n print(f\"write stats: ai_calls {calls_note} | http_calls {http_note}\")\\n- if providers and dead_auth and dead_auth == {p.name for p in providers}:\\n+ if configured_providers and dead_auth and dead_auth == {\\n+ p.name for p in configured_providers\\n+ if p.name not in persisted_cooldowns}:\\n # Every configured platform failed authentication: turn the Actions\\n # run red instead of silently green.\\n raise SystemExit(1)\\n@@ -0,0 +1,59 @@\\n+\"\"\"Offline checks for persisted provider cooldowns.\"\"\"\\n+from __future__ import annotations\\n+\\n+import json\\n+import sys\\n+import tempfile\\n+from datetime import datetime, timedelta, timezone\\n+from pathlib import Path\\n+\\n+ROOT = Path(__file__).resolve().parent.parent\\n+sys.path.insert(0, str(ROOT / \"pipeline\"))\\n+\\n+import provider_health # noqa: E402\\n+\\n+\\n+def main() -> None:\\n+ base = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc)\\n+ with tempfile.TemporaryDirectory() as tmp:\\n+ provider_health.HEALTH_PATH = Path(tmp) / \"provider-health.json\"\\n+\\n+ until = provider_health.mark_failure(\"gemini\", \"transient\", base)\\n+ assert until == base + timedelta(hours=1)\\n+ assert provider_health.active_platforms(base + timedelta(minutes=59)) == {\\n+ \"gemini\"\\n+ }\\n+ assert provider_health.active_platforms(base + timedelta(hours=1, minutes=1)) == set()\\n+\\n+ provider_health.mark_failure(\"nvidia\", \"quota\", base)\\n+ rows = provider_health.cooldown_rows(base + timedelta(hours=5))\\n+ assert rows[\"nvidia\"][\"failureClass\"] == \"quota\"\\n+ assert \"nvidia\" not in provider_health.active_platforms(\\n+ base + timedelta(hours=6, minutes=1)\\n+ )\\n+\\n+ provider_health.mark_failure(\"anthropic\", \"auth\", base)\\n+ assert \"anthropic\" in provider_health.active_platforms(\\n+ base + timedelta(hours=11, minutes=59)\\n+ )\\n+ assert provider_health.mark_success(\\n+ \"anthropic\", base + timedelta(hours=1)\\n+ ) is True\\n+ assert \"anthropic\" not in provider_health.active_platforms(\\n+ base + timedelta(hours=1)\\n+ )\\n+\\n+ payload = json.loads(\\n+ provider_health.HEALTH_PATH.read_text(encoding=\"utf-8\")\\n+ )\\n+ assert payload[\"schemaVersion\"] == 1\\n+ assert all(\\n+ set(row) <= {\"failureClass\", \"cooldownUntilUtc\", \"updatedUtc\"}\\n+ for row in payload[\"platforms\"].values()\\n+ )\\n+\\n+ print(\"test_provider_health: OK\")\\n+\\n+\\n+if __name__ == \"__main__\":\\n+ main()\\n@@ -231,24 +231,34 @@ def main() -> None:\\n ROOT / \".github\" / \"workflows\" / \"manual-article-deploy.yml\",\\n ROOT / \".github\" / \"workflows\" / \"radar-snapshot.yml\",\\n ]\\n- for workflow in workflows:\\n- source = workflow.read_text(encoding=\"utf-8\")\\n+ workflow_sources = {\\n+ workflow.name: workflow.read_text(encoding=\"utf-8\")\\n+ for workflow in workflows\\n+ }\\n+ for source in workflow_sources.values():\\n assert \"python pipeline/radar_snapshot.py\" in source\\n assert source.index(\"python pipeline/radar_snapshot.py\") < source.index(\\n \"actions/upload-pages-artifact@v5\")\\n+ # Only the final deploy job may occupy the shared Pages queue.\\n+ assert source.count(\"group: skytical-pages-publish\") == 1\\n+ assert \"cancel-in-progress: false\" in source\\n \\n- workflow_header = source.split(\"\\njobs:\\n\", 1)[0]\\n- if workflow.name in {\"hourly.yml\", \"briefing.yml\"}:\\n- # Data generation is intentionally independent of the Pages queue;\\n- # only the deploy job remains serialized.\\n- assert \"group: skytical-pages-publish\" in source\\n- assert \"cancel-in-progress: false\" in source\\n- else:\\n- assert \"skytical-pages-publish\" in workflow_header\\n- assert re.search(r\"(?m)^ queue: max$\", workflow_header)\\n- assert re.search(r\"(?m)^ cancel-in-progress: false$\", workflow_header)\\n- if workflow.name == \"radar-snapshot.yml\":\\n- assert \"github.event.pull_request.number\" in workflow_header\\n+ manual_workflow = workflow_sources[\"manual-article-deploy.yml\"]\\n+ assert \"group: skytical-manual-article-build\" in manual_workflow\\n+ assert manual_workflow.index(\" build:\") < manual_workflow.index(\" deploy:\")\\n+ assert manual_workflow.index(\"actions/upload-pages-artifact@v5\") < (\\n+ manual_workflow.index(\"actions/deploy-pages@v5\")\\n+ )\\n+\\n+ radar_workflow = workflow_sources[\"radar-snapshot.yml\"]\\n+ assert \"group: radar-snapshot-pr-${{ github.event.pull_request.number }}\" in (\\n+ radar_workflow\\n+ )\\n+ assert \"group: skytical-radar-build\" in radar_workflow\\n+ assert \"python pipeline/radar_snapshot.py --optional\" in radar_workflow\\n+ assert \"should_deploy=false\" in radar_workflow\\n+ assert \"keeping the last deployed radar snapshot\" in radar_workflow\\n+ assert \"if: needs.build.outputs.should_deploy == 'true'\" in radar_workflow\\n \\n assert any(\\n \"本文由自動化系統彙整生成，內容以原始來源為準 • \"\\n@@ -828,6 +828,9 @@ def test_model_chain_and_routing_policy():\\n ]\\n check(len(disabled_attempts) == 1,\\n \"circuit-breaking failure is marked disabled in the run ledger\")\\n+ health = load(DATA / \"provider-health.json\")\\n+ check(health[\"platforms\"][\"gemini\"][\"failureClass\"] == \"transient\",\\n+ \"transient platform circuit persists a cross-run cooldown\")\\n check(write._is_transient_platform_failure(\\n providers.ProviderError(\"finishReason MAX_TOKENS\")) is False,\\n \"model truncation does not trip the platform circuit\")\\n@@ -1021,6 +1024,11 @@ def test_all_providers_auth_dead():\\n write.build_providers = original\\n \\n check(exited == 1, f\"SystemExit(1) when all providers fail auth, got {exited}\")\\n+ health = load(DATA / \"provider-health.json\")\\n+ check(set(health[\"platforms\"]) == {\"gemini\", \"nvidia\"}\\n+ and all(row[\"failureClass\"] == \"auth\"\\n+ for row in health[\"platforms\"].values()),\\n+ \"auth-dead platforms persist a cross-run cooldown\")\\n check(not list((DATA / \"articles\").glob(\"*.json\")),\\n \"no articles when every provider is auth-dead\")\\n check((DATA / \"pending.json\").read_bytes() == pending_before," }