Full diff
@@ -110,6 +110,16 @@ def model_matches(model, text):\n return False\n \n \n+def verified_cargo_model(article):\n+ """True only when structured entities name a freighter subtype."""\n+ entities = article.get("entities") or {}\n+ models = (entities.get("aircraft_models")\n+ if isinstance(entities, dict) else []) or []\n+ return any(re.search(\n+ r"(?:P2F|BCF|BDSF|ERF|freighter|cargo)|(?:[/ -]|\d)C?F\)?$",\n+ str(model), re.I) for model in models)\n+\n+\n def rejection_reason(article, raw, captions=None):\n from image_policy import (article_context_text, article_headline_text,\n article_is_airport_operations, article_is_cabin_story,\n@@ -135,7 +145,10 @@ def rejection_reason(article, raw, captions=None):\n return "registration-mismatch"\n if article_is_airport_operations(article) and not article_is_incident(article):\n return "airframe-not-airport-facility"\n- if re.search(r"彩繪|藝術機|塗裝|\blivery\b|客艙|座椅|貨機|freighter", head, re.I):\n+ if re.search(r"彩繪|藝術機|塗裝|\blivery\b|客艙|座椅", head, re.I):\n+ return "airframe-role-unverified"\n+ if (re.search(r"貨機|貨運|freighter|cargo", head, re.I)\n+ and not verified_cargo_model(article)):\n return "airframe-role-unverified"\n carrier = images.find_airline(article)\n if carrier and not any(phrase(alias, ev) for alias in images._airline_aliases(carrier)):\n@@ -160,7 +173,10 @@ def rejection_reason(article, raw, captions=None):\n matched = str(im.get("matched") or "")\n # Exact event caption can describe seats/ceremony without repeating model.\n if source_bound and im.get("kind") == "event_photo":\n- if airline and not any(phrase(a, ev) for a in images._airline_aliases(airline)):\n+ event_evidence = f"{ev} {im.get('subject') or ''}"\n+ if airline and not any(\n+ phrase(a, event_evidence)\n+ for a in images._airline_aliases(airline)):\n return "source-airline-unverified"\n return None\n headline_model = images.find_aircraft_type({"en": {"title": head},\n@@ -73,9 +73,11 @@\n TIMEOUT = (10, 30)\n HEADERS = {"User-Agent": USER_AGENT}\n \n-# Politeness caps per run; negative matches back off but keep retrying while\n-# the article remains in the user-facing seven-day window.\n-MAX_LOOKUPS_PER_RUN = 6\n+# Politeness caps per run; scale within a bounded range when due no-image\n+# articles accumulate, then back off negative matches while they remain fresh.\n+BASE_LOOKUPS_PER_RUN = 6\n+MAX_LOOKUPS_PER_RUN = 18\n+LOOKUP_BACKLOG_DIVISOR = 3\n FAST_RETRY_ATTEMPTS = 3\n RETRY_AFTER_HOURS = 6\n LONG_RETRY_AFTER_HOURS = 24\n@@ -121,7 +123,10 @@ def _retry_after_hours(attempts: int) -> int:\n # letter suffix or 4-5 digits, JA####, and common hyphenated prefixes.\n _REG_RE = re.compile(\n r"\b(B-\d{4,5}|N\d{2,4}[A-Z]{1,2}|N\d{4,5}|JA\d{3,4}[A-Z]?"\n- r"|(?:HL|VH|HS|9V|9M|D|F|G|PH|EI|OE|VT|A6|A7|TC)-[A-Z]{3,4})\b")\n+ r"|(?:HL|VH|HS|9S|9V|9M|D|F|G|PH|EI|OE|VT|A6|A7|TC)-[A-Z]{3,4})\b")\n+_ENTITY_REG_RE = re.compile(\n+ r"^(?:B-\d{4,5}|N\d{2,5}[A-Z]{0,2}|JA\d{3,4}[A-Z]?|"\n+ r"[A-Z0-9]{1,2}-[A-Z0-9]{3,5})$", re.I)\n \n # Location wording -> Taiwan airport photo query. Articles about island\n # routes say 澎湖/金門/馬祖, never the airport's registry name, so the\n@@ -200,7 +205,18 @@ def _article_text(article: dict) -> str:\n \n \n def find_registration(article: dict) -> str | None:\n- match = _REG_RE.search(article_lead_text(article))\n+ lead = article_lead_text(article)\n+ entities = article.get("entities") or {}\n+ registrations = (entities.get("registration_numbers")\n+ if isinstance(entities, dict) else []) or []\n+ for value in registrations:\n+ candidate = str(value or "").strip().upper()\n+ if (_ENTITY_REG_RE.fullmatch(candidate)\n+ and re.search(\n+ rf"(?<![A-Z0-9]){re.escape(candidate)}(?![A-Z0-9])",\n+ lead, re.I)):\n+ return candidate\n+ match = _REG_RE.search(lead)\n return match.group(1) if match else None\n \n \n@@ -513,6 +529,9 @@ def lookup_planespotters(reg: str, *, article=None) -> dict | None:\n link = photo.get("link")\n if not thumb or not link:\n continue\n+ description = " ".join(str(photo.get(key) or "").strip()\n+ for key in ("airline", "aircraft", "reg"))\n+ description = " ".join(description.split())\n candidate = {\n "url": str(thumb),\n "link": str(link),\n@@ -521,7 +540,8 @@ def lookup_planespotters(reg: str, *, article=None) -> dict | None:\n "provider": "Planespotters.net",\n "kind": "airframe_photo",\n "matched": reg,\n- "subject": f"註冊號 {reg}",\n+ "subject": description or f"註冊號 {reg}",\n+ "description": description,\n }\n if article is None or existing_image_matches(article, candidate):\n return candidate\n@@ -607,26 +627,40 @@ def lookup_commons(query: str, require_tokens: list[str],\n return max(candidates, key=lambda row: row[:3])[3] if candidates else None\n \n \n-def resolve_image(article: dict, *, usage=None) -> dict | None:\n+def resolve_image(article: dict, *, usage=None, diagnostics=None) -> dict | None:\n """Try independently failing providers; never relax subject constraints."""\n if article.get("articleFormat") == "roundup":\n return None\n def attempt(fn, *args, **kwargs):\n+ label = getattr(fn, "__name__", "provider")\n try:\n photo = fn(*args, **kwargs)\n- if photo and existing_image_matches(article, photo):\n+ if not photo:\n+ if diagnostics is not None:\n+ diagnostics.append(f"{label}:no-result")\n+ return None\n+ reason = rejection_reason(article, photo)\n+ if not reason:\n if photo.get("kind") == "file_photo" and (usage or {}).get(image_key(photo), 0) >= MAX_STOCK_REUSE:\n+ if diagnostics is not None:\n+ diagnostics.append(f"{label}:stock-reuse-limit")\n return None\n return photo\n+ if diagnostics is not None:\n+ diagnostics.append(f"{label}:{reason}")\n except (RequestException, OSError, ValueError, TypeError, KeyError) as exc:\n- print(f"images: {getattr(fn, "__name__", "provider")}: {type(exc).__name__}")\n+ if diagnostics is not None:\n+ diagnostics.append(\n+ f"{label}:lookup-error:{type(exc).__name__}")\n+ print(f"images: {label}: {type(exc).__name__}")\n return None\n photo = attempt(lookup_official_source_photo, article)\n if photo:\n return photo\n reg = find_registration(article)\n if reg:\n- photo = attempt(lookup_planespotters, reg, article=article)\n+ # Validate once in ``attempt`` so the precise reason is retained.\n+ photo = attempt(lookup_planespotters, reg)\n if photo:\n return photo\n airline, actype, airport = find_airline(article), find_aircraft_type(article), find_airport(article)\n@@ -682,12 +716,32 @@ def _recent_batches():\n yield path, batch, articles\n \n \n+def _lookup_budget(articles, entries, now) -> int:\n+ """Scale lookups with due no-image backlog without becoming unbounded."""\n+ due = 0\n+ for article in articles:\n+ if article.get("articleFormat") == "roundup" or article.get("image"):\n+ continue\n+ entry = entries.get(str(article.get("id") or "")) or {}\n+ try:\n+ if (entry.get("status") == "none" and entry.get("next_retry_utc")\n+ and parse_iso(entry["next_retry_utc"]) > now):\n+ continue\n+ except (TypeError, ValueError):\n+ pass\n+ due += 1\n+ base = min(BASE_LOOKUPS_PER_RUN, MAX_LOOKUPS_PER_RUN)\n+ demand = (due + LOOKUP_BACKLOG_DIVISOR - 1) // LOOKUP_BACKLOG_DIVISOR\n+ return min(MAX_LOOKUPS_PER_RUN, max(base, demand))\n+\n+\n def main() -> int:\n if not ARTICLES_DIR.is_dir():\n print("images: no articles yet")\n return 0\n enforce_recent()\n- usage = stock_usage([a for _, _, aa in _recent_batches() for a in aa])\n+ recent_batches = list(_recent_batches())\n+ usage = stock_usage([a for _, _, aa in recent_batches for a in aa])\n cache = load_json(CACHE_PATH, {})\n entries = cache.get("articles")\n if not isinstance(entries, dict):\n@@ -696,9 +750,11 @@ def main() -> int:\n cache = {"articles": entries, "sources": source_entries}\n now = now_utc()\n cutoff = now - timedelta(days=MAX_ARTICLE_AGE_DAYS)\n+ lookup_budget = _lookup_budget(\n+ [a for _, _, aa in recent_batches for a in aa], entries, now)\n lookups = attached = source_lookups = 0\n \n- for path, batch, articles in _recent_batches():\n+ for path, batch, articles in recent_batches:\n changed = False\n for article in articles:\n art_id = str(article.get("id") or "")\n@@ -714,12 +770,16 @@ def main() -> int:\n changed = True\n entries.pop(art_id, None)\n continue\n+ rejection_reasons = []\n # Exact-source caption recovery outranks Commons, but a caption\n # still has to identify this article's primary visual subject.\n for candidate in (article.get("sourceImageCandidates") or []) if can_upgrade(article) else []:\n bound = dict(article, sources=candidate.get("sources") or [])\n source_image = prepare_image(bound, candidate.get("url"))\n- if (source_image and existing_image_matches(article, source_image)\n+ reason = rejection_reason(bound, candidate.get("url"))\n+ if reason:\n+ rejection_reasons.append(f"source-candidate:{reason}")\n+ if (source_image and not reason\n and usage[image_key(source_image)] < MAX_STOCK_REUSE):\n if source_image != article.get("image"):\n article["image"] = source_image\n@@ -740,16 +800,25 @@ def main() -> int:\n retry_due = parse_iso(source_entry["next_retry_utc"]) <= now\n except (KeyError, TypeError, ValueError):\n pass\n- if not source_image and retry_due and source_lookups < MAX_LOOKUPS_PER_RUN:\n+ if not source_image and retry_due and source_lookups < lookup_budget:\n source_lookups += 1\n+ source_reason = "no-event-photo"\n try:\n source_image = lookup_source_photo(article)\n except Exception as exc:\n+ source_reason = f"lookup-error:{type(exc).__name__}"\n print(f"images: source lookup failed for {art_id}: {type(exc).__name__}")\n+ if source_image:\n+ source_reason = rejection_reason(article, source_image)\n+ if source_reason:\n+ source_image = None\n source_entries[source_url] = {\n "image": source_image,\n+ "rejection_reason": source_reason,\n "next_retry_utc": (now + timedelta(hours=6)).isoformat(),\n }\n+ if source_reason:\n+ rejection_reasons.append(f"source-photo:{source_reason}")\n if source_image and existing_image_matches(article, source_image):\n article["image"] = source_image\n entries[art_id] = {"status": "matched", "image": source_image,\n@@ -784,11 +853,12 @@ def main() -> int:\n continue\n entries.pop(art_id, None)\n entry = {}\n- if lookups >= MAX_LOOKUPS_PER_RUN:\n+ if lookups >= lookup_budget:\n continue\n lookups += 1\n try:\n- image = resolve_image(article, usage=usage)\n+ image = resolve_image(article, usage=usage,\n+ diagnostics=rejection_reasons)\n except Exception as exc: # enhancement only: never fail the run\n print(f"images: lookup failed for {art_id}: "\n f"{type(exc).__name__}: {exc}")\n@@ -806,6 +876,8 @@ def main() -> int:\n attempts = entry.get("attempts", 0) + 1\n entries[art_id] = {\n "status": "none", "attempts": attempts,\n+ "rejection_reasons": list(dict.fromkeys(\n+ rejection_reasons or ["no-suitable-image"])),\n "checked_utc": now.isoformat(),\n "next_retry_utc":\n (now + timedelta(hours=_retry_after_hours(attempts)))\n@@ -816,7 +888,8 @@ def main() -> int:\n \n if cache != load_json(CACHE_PATH, {}):\n save_json(CACHE_PATH, cache)\n- print(f"images: {lookups} stock lookup(s), {source_lookups} source lookup(s), "\n+ print(f"images: budget {lookup_budget}; {lookups} stock lookup(s), "\n+ f"{source_lookups} source lookup(s), "\n f"{attached} article(s) got a photo")\n return 0\n \n@@ -22,6 +22,9 @@ def supported_source(article):\n {"www.cna.com.tw", "cna.com.tw"}\n and re.fullmatch(r"/news/[a-z]+/\d+\.aspx", parts.path)):\n return url\n+ if (parts.scheme == "https" and parts.hostname == "www.aerotime.aero"\n+ and re.fullmatch(r"/articles/[a-z0-9-]+/?", parts.path)):\n+ return url\n return None\n \n \n@@ -78,6 +81,58 @@ def parse_cna_photo(html, source_url):\n return None\n \n \n+def _aerotime_image_key(url):\n+ parts = urlsplit(url)\n+ if (parts.scheme != "https" or parts.hostname != "www.aerotime.aero"\n+ or not re.fullmatch(\n+ r"/images/\d{4}/\d{2}/[^/]+\.(?:jpe?g|png|webp)",\n+ parts.path, re.I)):\n+ return None\n+ path = re.sub(r"-\d{2,4}x\d{2,4}(?=\.[^.]+$)", "", parts.path)\n+ return re.sub(r"\.webp$", "", path, flags=re.I)\n+\n+\n+def _aerotime_subject(article):\n+ """Describe an exact article-bound event photo with verified entities."""\n+ entities = article.get("entities") or {}\n+ values = []\n+ for key in ("airlines", "aircraft_models", "registration_numbers"):\n+ group = entities.get(key) if isinstance(entities, dict) else []\n+ if group:\n+ value = clean_description(str(group[0]))\n+ if value and value not in values:\n+ values.append(value)\n+ return " ".join(values)\n+\n+\n+def parse_aerotime_photo(html, source_url, article):\n+ """Return AeroTime's credited lead event image, never an unbound OG ad."""\n+ soup = BeautifulSoup(html, "html.parser")\n+ lead = soup.find("meta", property="og:image")\n+ url = lead.get("content", "") if lead else ""\n+ key = _aerotime_image_key(url)\n+ if not key:\n+ return None\n+ for figure in soup.select("figure.cs-entry__post-media"):\n+ img = figure.find("img")\n+ caption = figure.find("figcaption")\n+ if (not img or not caption\n+ or _aerotime_image_key(img.get("src", "")) != key):\n+ continue\n+ credit = clean_description(" ".join(caption.stripped_strings))\n+ subject = _aerotime_subject(article)\n+ if not credit or not subject:\n+ continue\n+ return {\n+ "url": url, "link": source_url,\n+ "subject": subject, "sourceCaption": credit,\n+ "credit": credit, "license": None,\n+ "provider": "AeroTime", "kind": "event_photo",\n+ "matched": "source:aerotime",\n+ }\n+ return None\n+\n+\n def lookup_source_photo(article):\n url = supported_source(article)\n if not url:\n@@ -88,4 +143,9 @@ def lookup_source_photo(article):\n # Do not accept a redirect to a different article or publisher.\n if response.url.rstrip("/") != url.rstrip("/"):\n return None\n- return parse_cna_photo(response.text, url)\n+ host = urlsplit(url).hostname\n+ if host in {"www.cna.com.tw", "cna.com.tw"}:\n+ return parse_cna_photo(response.text, url)\n+ if host == "www.aerotime.aero":\n+ return parse_aerotime_photo(response.text, url, article)\n+ return None\n@@ -59,6 +59,12 @@ def test_background_registration_is_not_primary(self):\n a["en"]["body"] = ["Earnings increased.", "A historical flight used N512DN."]\n self.assertIsNone(images.find_registration(a))\n \n+ def test_verified_entity_registration_supports_9s_prefix(self):\n+ a = article(\n+ "Trans Air Cargo Service DC-8-73CF 9S-AJO runway excursion",\n+ entities={"registration_numbers": ["9S-AJO", "OB-2231P"]})\n+ self.assertEqual(images.find_registration(a), "9S-AJO")\n+\n def test_country_is_not_airline(self):\n self.assertIsNone(images.find_airline(article("美國公布無人機新規則")))\n \n@@ -136,6 +142,40 @@ def test_airframe_registration_does_not_attest_new_operator(self):\n "kind": "airframe_photo", "matched": "N512DN", "link": "https://www.planespotters.net/photo/1"}\n self.assertEqual(policy.rejection_reason(a, im), "airframe-operator-unverified")\n \n+ def test_exact_airframe_and_verified_freighter_subtype_are_allowed(self):\n+ a = article(\n+ "Trans Air Cargo Service DC-8 freighter 9S-AJO runway excursion",\n+ entities={\n+ "airlines": ["Trans Air Cargo Service"],\n+ "aircraft_models": ["DC-8-73CF", "DC-8"],\n+ "registration_numbers": ["9S-AJO"],\n+ })\n+ im = {\n+ "url": "https://cdn.planespotters.net/9s-ajo.jpg",\n+ "provider": "Planespotters.net", "kind": "airframe_photo",\n+ "matched": "9S-AJO",\n+ "description": "Trans Air Cargo Service DC-8-73CF 9S-AJO",\n+ "link": "https://www.planespotters.net/photo/9s-ajo-trans-air-cargo-service",\n+ }\n+ self.assertIsNone(policy.rejection_reason(a, im))\n+ a["entities"]["aircraft_models"] = ["DC-8"]\n+ self.assertEqual(\n+ policy.rejection_reason(a, im), "airframe-role-unverified")\n+\n+ def test_bound_aerotime_event_uses_verified_subject(self):\n+ url = "https://www.aerotime.aero/articles/dc-8-runway-excursion"\n+ a = article(\n+ "Trans Air Cargo Service DC-8 freighter 9S-AJO runway excursion",\n+ sources=[{"url": url}],\n+ entities={"airlines": ["Trans Air Cargo Service"]})\n+ im = {\n+ "url": "https://www.aerotime.aero/images/2026/09/dc8.jpeg",\n+ "link": url, "provider": "AeroTime", "kind": "event_photo",\n+ "matched": "source:aerotime", "sourceCaption": "Bystander video",\n+ "subject": "Trans Air Cargo Service DC-8-73CF 9S-AJO",\n+ }\n+ self.assertIsNone(policy.rejection_reason(a, im))\n+\n def test_url_identity_handles_resize_but_not_attachment_id(self):\n original = "https://upload.wikimedia.org/wikipedia/commons/a/ab/Test.jpg"\n thumb = "https://thumb.wikimedia.org/wikipedia/commons/thumb/a/ab/Test.jpg/1280px-Test.jpg?utm_source=x"\n@@ -142,6 +142,11 @@ def reset():\n check("registration found in article text",\n images.find_registration(make_article(\n "a", "Delta A350 registration N512DN arrived")) == "N512DN")\n+congo_dc8 = make_article(\n+ "a", "Trans Air Cargo Service DC-8-73CF 9S-AJO runway excursion")\n+congo_dc8["entities"]["registration_numbers"] = ["9S-AJO", "OB-2231P"]\n+check("verified entity registration supports 9S prefix",\n+ images.find_registration(congo_dc8) == "9S-AJO")\n check("plain N95 is never treated as a registration",\n images.find_registration(make_article(\n "a", "Airport staff wore N95 masks")) is None)\n@@ -560,6 +565,19 @@ def reset():\n images.main()\n check("negative result is not retried before the retry window",\n fake.calls == [] and read_batch(p)[0]["image"] is None)\n+negative_entry = json.loads(\n+ (TMP / "images.json").read_text(encoding="utf-8"))["articles"]["a-none"]\n+check("negative cache records provider rejection reasons",\n+ bool(negative_entry.get("rejection_reasons"))\n+ and any("no-result" in reason\n+ for reason in negative_entry["rejection_reasons"]))\n+\n+due_articles = [make_article(str(i), "Delta Air Lines Airbus A350")\n+ for i in range(40)]\n+check("lookup budget scales with backlog and remains bounded",\n+ images._lookup_budget(due_articles, {}, now_utc()) == 14\n+ and images._lookup_budget(due_articles * 10, {}, now_utc())\n+ == images.MAX_LOOKUPS_PER_RUN)\n \n # A negative cache entry must never become permanent. After the initial\n # six-hour retries, a newly available image is still attached on a later\n@@ -13,6 +13,20 @@\n <figure class="floatImg center"><img src="{PHOTO.replace('/1024/', '/800/')}">\n <figcaption>華航名古屋航線限定頭墊紙。（華航提供）中央社記者傳真 115年9月7日</figcaption>\n </figure>'''\n+AEROTIME_URL = "https://www.aerotime.aero/articles/dc-8-runway-excursion"\n+AEROTIME_PHOTO = "https://www.aerotime.aero/images/2026/09/dc8congo.jpeg"\n+AEROTIME_HTML = f'''<meta property="og:image" content="{AEROTIME_PHOTO}">\n+<figure class="cs-entry__post-media post-media">\n+<img src="{AEROTIME_PHOTO.replace('.jpeg', '-800x500.jpeg')}">\n+<figcaption class="cs-entry__caption-text">Bystander video</figcaption>\n+</figure>'''\n+AEROTIME_ARTICLE = {\n+ "entities": {\n+ "airlines": ["Trans Air Cargo Service"],\n+ "aircraft_models": ["DC-8-73CF"],\n+ "registration_numbers": ["9S-AJO"],\n+ }\n+}\n \n \n def test_captioned_body_photo_and_resize_variants():\n@@ -30,6 +44,28 @@ def test_reject_unrelated_og_and_uncredited_photo():\n assert source.parse_cna_photo(f'<meta property="og:image" content="{PHOTO}">', URL) is None\n \n \n+def test_aerotime_exact_lead_event_photo_with_credit_and_entities():\n+ photo = source.parse_aerotime_photo(\n+ AEROTIME_HTML, AEROTIME_URL, AEROTIME_ARTICLE)\n+ assert photo["url"] == AEROTIME_PHOTO\n+ assert photo["subject"] == \\n+ "Trans Air Cargo Service DC-8-73CF 9S-AJO"\n+ assert photo["credit"] == "Bystander video"\n+ assert photo["license"] is None\n+ assert photo["kind"] == "event_photo"\n+ assert source.supported_source({\n+ "sources": [{"url": AEROTIME_URL}]}) == AEROTIME_URL\n+\n+\n+def test_aerotime_rejects_unbound_or_uncredited_lead_image():\n+ assert source.parse_aerotime_photo(\n+ AEROTIME_HTML.replace("dc8congo-800x500", "different-800x500"),\n+ AEROTIME_URL, AEROTIME_ARTICLE) is None\n+ assert source.parse_aerotime_photo(\n+ AEROTIME_HTML.replace("Bystander video", "Image"),\n+ AEROTIME_URL, AEROTIME_ARTICLE) is None\n+\n+\n def test_protect_manual_and_exact_airframe_images():\n for image in ["https://example.com/manual.jpg", {"provider": "manual"},\n {"provider": "Planespotters", "kind": "airframe_photo"}]:
