1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
#!/usr/bin/env python3
"""External link rot checker.
Collects every external http(s) link from the site's *.php / *.html files,
dedupes them, and checks each one over the network. Reports links that are
dead (connection error, timeout) or return a 4xx/5xx status.
Network-bound and slow-ish, so it is NOT part of the default audit run. Wire
it in with CHECK_LINKS=1 ./tools/run_audits.sh, or run it directly.
Skips mailto/xmpp/onion/i2p and other non-web schemes. Treats 401/403/405 and
429 as "alive but gated" by default (many sites block bots) — pass --strict to
flag those too.
Usage:
python tools/link_rot.py [--timeout 10] [--workers 16] [--strict]
"""
from __future__ import annotations
import argparse
import re
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
ROOT = Path(__file__).resolve().parents[1]
SKIP_DIRS = {".git", ".agents", ".claude", "tools", "docs", "locales", "partials"}
RE_URL = re.compile(r"\b(?:href|src)=(['\"])(https?://.*?)\1", re.I)
RE_DYNAMIC = re.compile(r"<\?.*?\?>", re.S)
# Hosts that are part of this site / known-internal; no point rot-checking.
SKIP_HOST_SUBSTR = (".onion", ".i2p", "localhost", "127.0.0.1")
# Statuses that mean "reachable but bot-gated" — not rot unless --strict.
SOFT_STATUSES = {401, 403, 405, 429}
UA = "Mozilla/5.0 (compatible; sillylaird-linkrot/1.0)"
def collect() -> dict[str, set[str]]:
"""Return {url: {source files}} for every static external link."""
links: dict[str, set[str]] = {}
files: list[Path] = []
for ext in ("*.html", "*.php"):
files.extend(ROOT.rglob(ext))
for f in files:
if any(part in SKIP_DIRS for part in f.parts):
continue
text = f.read_text(encoding="utf-8", errors="ignore")
for m in RE_URL.finditer(text):
url = m.group(2).strip()
if RE_DYNAMIC.search(url):
continue
if any(s in url.lower() for s in SKIP_HOST_SUBSTR):
continue
links.setdefault(url, set()).add(f.relative_to(ROOT).as_posix())
return links
def check(url: str, timeout: float) -> tuple[str, int | None, str]:
"""Return (url, status_or_None, note). status None => connection failure."""
# Try HEAD first, fall back to GET (many servers reject HEAD).
for method in ("HEAD", "GET"):
try:
req = Request(url, method=method, headers={"User-Agent": UA})
with urlopen(req, timeout=timeout) as resp:
return url, resp.status, ""
except HTTPError as e:
if method == "HEAD" and e.code in (403, 405, 501):
continue # retry with GET
return url, e.code, ""
except (URLError, TimeoutError) as e:
reason = getattr(e, "reason", e)
if method == "HEAD":
continue # retry with GET before giving up
return url, None, str(reason)
except Exception as e: # noqa: BLE001 - report anything odd as dead
return url, None, str(e)
return url, None, "unreachable"
def main() -> int:
ap = argparse.ArgumentParser(description="External link rot checker")
ap.add_argument("--timeout", type=float, default=10.0)
ap.add_argument("--workers", type=int, default=16)
ap.add_argument("--strict", action="store_true",
help="also flag 401/403/405/429 (bot-gated) responses")
args = ap.parse_args()
links = collect()
if not links:
print("OK: no external links found")
return 0
print(f"Checking {len(links)} external links…", file=sys.stderr)
dead: list[tuple[str, int | None, str]] = []
soft: list[tuple[str, int | None, str]] = []
with ThreadPoolExecutor(max_workers=args.workers) as ex:
futs = {ex.submit(check, u, args.timeout): u for u in links}
for fut in as_completed(futs):
url, status, note = fut.result()
if status is None:
dead.append((url, status, note))
elif status >= 400:
(soft if status in SOFT_STATUSES else dead).append((url, status, note))
def show(rows):
for url, status, note in sorted(rows, key=lambda r: r[0]):
tag = status if status is not None else "DEAD"
extra = f" ({note})" if note else ""
srcs = ", ".join(sorted(links[url]))
print(f"- [{tag}] {url}{extra}\n in: {srcs}")
if not args.strict and soft:
print(f"\nbot-gated (not counted; use --strict): {len(soft)}")
show(soft)
flagged = dead + (soft if args.strict else [])
if flagged:
print(f"\nBroken external links ({len(flagged)}):")
show(flagged)
return 1
print(f"OK: all {len(links)} external links reachable")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|
