import live www.sillylaird.ca webrootHEADmain

1 files changed, 135 insertions, 0 deletions

diff --git a/tools/link_rot.py b/tools/link_rot.py
new file mode 100755
index 0000000..b3a7613
--- /dev/null
+++ b/tools/link_rot.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""External link rot checker.
+
+Collects every external http(s) link from the site's *.php / *.html files,
+dedupes them, and checks each one over the network. Reports links that are
+dead (connection error, timeout) or return a 4xx/5xx status.
+
+Network-bound and slow-ish, so it is NOT part of the default audit run. Wire
+it in with CHECK_LINKS=1 ./tools/run_audits.sh, or run it directly.
+
+Skips mailto/xmpp/onion/i2p and other non-web schemes. Treats 401/403/405 and
+429 as "alive but gated" by default (many sites block bots) — pass --strict to
+flag those too.
+
+Usage:
+  python tools/link_rot.py [--timeout 10] [--workers 16] [--strict]
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError, URLError
+
+ROOT = Path(__file__).resolve().parents[1]
+
+SKIP_DIRS = {".git", ".agents", ".claude", "tools", "docs", "locales", "partials"}
+
+RE_URL = re.compile(r"\b(?:href|src)=(['\"])(https?://.*?)\1", re.I)
+RE_DYNAMIC = re.compile(r"<\?.*?\?>", re.S)
+
+# Hosts that are part of this site / known-internal; no point rot-checking.
+SKIP_HOST_SUBSTR = (".onion", ".i2p", "localhost", "127.0.0.1")
+
+# Statuses that mean "reachable but bot-gated" — not rot unless --strict.
+SOFT_STATUSES = {401, 403, 405, 429}
+
+UA = "Mozilla/5.0 (compatible; sillylaird-linkrot/1.0)"
+
+
+def collect() -> dict[str, set[str]]:
+    """Return {url: {source files}} for every static external link."""
+    links: dict[str, set[str]] = {}
+    files: list[Path] = []
+    for ext in ("*.html", "*.php"):
+        files.extend(ROOT.rglob(ext))
+    for f in files:
+        if any(part in SKIP_DIRS for part in f.parts):
+            continue
+        text = f.read_text(encoding="utf-8", errors="ignore")
+        for m in RE_URL.finditer(text):
+            url = m.group(2).strip()
+            if RE_DYNAMIC.search(url):
+                continue
+            if any(s in url.lower() for s in SKIP_HOST_SUBSTR):
+                continue
+            links.setdefault(url, set()).add(f.relative_to(ROOT).as_posix())
+    return links
+
+
+def check(url: str, timeout: float) -> tuple[str, int | None, str]:
+    """Return (url, status_or_None, note). status None => connection failure."""
+    # Try HEAD first, fall back to GET (many servers reject HEAD).
+    for method in ("HEAD", "GET"):
+        try:
+            req = Request(url, method=method, headers={"User-Agent": UA})
+            with urlopen(req, timeout=timeout) as resp:
+                return url, resp.status, ""
+        except HTTPError as e:
+            if method == "HEAD" and e.code in (403, 405, 501):
+                continue  # retry with GET
+            return url, e.code, ""
+        except (URLError, TimeoutError) as e:
+            reason = getattr(e, "reason", e)
+            if method == "HEAD":
+                continue  # retry with GET before giving up
+            return url, None, str(reason)
+        except Exception as e:  # noqa: BLE001 - report anything odd as dead
+            return url, None, str(e)
+    return url, None, "unreachable"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="External link rot checker")
+    ap.add_argument("--timeout", type=float, default=10.0)
+    ap.add_argument("--workers", type=int, default=16)
+    ap.add_argument("--strict", action="store_true",
+                    help="also flag 401/403/405/429 (bot-gated) responses")
+    args = ap.parse_args()
+
+    links = collect()
+    if not links:
+        print("OK: no external links found")
+        return 0
+
+    print(f"Checking {len(links)} external links…", file=sys.stderr)
+    dead: list[tuple[str, int | None, str]] = []
+    soft: list[tuple[str, int | None, str]] = []
+
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        futs = {ex.submit(check, u, args.timeout): u for u in links}
+        for fut in as_completed(futs):
+            url, status, note = fut.result()
+            if status is None:
+                dead.append((url, status, note))
+            elif status >= 400:
+                (soft if status in SOFT_STATUSES else dead).append((url, status, note))
+
+    def show(rows):
+        for url, status, note in sorted(rows, key=lambda r: r[0]):
+            tag = status if status is not None else "DEAD"
+            extra = f" ({note})" if note else ""
+            srcs = ", ".join(sorted(links[url]))
+            print(f"- [{tag}] {url}{extra}\n    in: {srcs}")
+
+    if not args.strict and soft:
+        print(f"\nbot-gated (not counted; use --strict): {len(soft)}")
+        show(soft)
+
+    flagged = dead + (soft if args.strict else [])
+    if flagged:
+        print(f"\nBroken external links ({len(flagged)}):")
+        show(flagged)
+        return 1
+
+    print(f"OK: all {len(links)} external links reachable")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())