#!/usr/bin/env python3 """Comment out EXTERNAL links that are truly dead — i.e. the host/domain no longer exists (DNS does not resolve / host unreachable / connection failed). This is the external analogue of fix_dead_links.py (which does the same for internal page links). It is deliberately conservative: * ONLY links whose lychee reason matches the "domain-gone" class are auto-commented: - "Connection failed" (lychee's umbrella for "couldn't even connect" — the exact reason a dead domain like jabfreejobs.in produces) - "Could not resolve host" (explicit DNS failure) - "HostUnreachable" (host unreachable) * Everything else is LEFT ALONE and written to a manual-review file: - HTTP status rejections (404 / 429 / 500 / redirects) -> the domain is alive and the server responded; a 404 is "page gone", not "domain gone". - SSL / TLS / certificate / HTTP2 errors -> the site is up. - Connection refused / reset / closed -> weaker, possibly transient; not the "domain doesn't exist" class. - "Error (cached)" -> a stale lychee cache entry, not a fresh signal. Commenting (NOT deleting) wraps the markdown link in an HTML comment: [Old](/gone/) -> so it disappears from the rendered page and the Pagefind index, and is no longer detected by lychee on the next run (idempotent). Usage: python3 scripts/fix_dead_external_links.py --report linkcheck/external.json # DRY RUN (default) python3 scripts/fix_dead_external_links.py --report linkcheck/external.json --apply # actually edit """ import argparse import json import re import time from pathlib import Path from urllib.parse import urlsplit # Case-insensitive substrings that mark a link as "the domain/host no longer exists". DOMAIN_GONE_PATTERNS = [ "connection failed", "could not resolve host", "hostunreachable", ] def is_domain_gone(status_text: str) -> bool: s = status_text.lower() return any(p in s for p in DOMAIN_GONE_PATTERNS) def is_onion(url: str) -> bool: """True for .onion hosts — only reachable through Tor. This checker runs off-Tor, so every .onion link 'fails' here by construction; they must never be auto-commented (manual: verify via Tor).""" host = urlsplit(url).hostname or "" return host.endswith(".onion") def main(): ap = argparse.ArgumentParser() ap.add_argument("--report", required=True, help="lychee JSON report (linkcheck/external.json)") ap.add_argument("--apply", action="store_true", help="actually edit the files (default is a dry-run preview)") ap.add_argument("--repo", default=".", help="repo root the error_map file paths are relative to") args = ap.parse_args() repo = Path(args.repo).resolve() data = json.loads(Path(args.report).read_text()) err_map = data.get("error_map", {}) to_comment = [] # (rel_md_file, line, url) manual = [] # (rel_md_file, line, url, status_text) for md_file, items in err_map.items(): rel = _rel(md_file, repo) for it in items: url = it.get("url", "") status = it.get("status", {}).get("text", "") line = it.get("span", {}).get("line") if not url.startswith(("http://", "https://")): continue if is_onion(url): manual.append((rel, line, url, "(.onion — only reachable via Tor; not routable from " "this network. Verify in Tor; do NOT auto-comment.)")) continue if is_domain_gone(status): to_comment.append((rel, line, url)) else: manual.append((rel, line, url, status)) # Group edits by file, apply bottom-up so line numbers stay valid. by_file = {} for rel, line, url in to_comment: by_file.setdefault(rel, []).append((line, url)) for rel in by_file: by_file[rel].sort(reverse=True) plan = 0 for rel in sorted(by_file): p = repo / rel if not p.exists(): print(f" ! missing source, skipping: {rel}") continue lines = p.read_text().splitlines(keepends=True) changed = False for line_no, url in by_file[rel]: if not line_no: continue idx = line_no - 1 if idx < 0 or idx >= len(lines): continue cur = lines[idx] # Find the markdown link on this line whose href matches the dead URL # (allow a trailing-slash mismatch). link = _find_link(cur, url) if not link: print(f" ? could not locate link on {rel}:{line_no} for {url}") continue link_txt = link.group(0) # Skip if it is already inside an HTML comment (idempotency guard). if _already_commented(cur, link.start()): continue lines[idx] = cur[:link.start()] + "!-- " + link_txt + " --" + cur[link.end():] changed = True plan += 1 print(f" {'APPLY' if args.apply else 'DRY '} {rel}:{line_no} {link_txt}") if changed and args.apply: p.write_text("".join(lines)) # Manual-review report for everything we deliberately did NOT touch. outdir = repo / "linkcheck" outdir.mkdir(parents=True, exist_ok=True) mr = outdir / "manual_review_external.txt" with mr.open("w") as f: f.write("External links NOT auto-commented (domain still alive / ambiguous).\n") f.write("Auto-comment only covers the domain-gone class (DNS failed / host\n") f.write("unreachable). Review these by hand.\n\n") for rel, line, url, status in sorted(manual): f.write(f"{rel}:{line}\t{url}\t[{status}]\n") print(f"\nAuto-comment {'applied' if args.apply else 'planned'}: {plan}") print(f"Manual review entries: {len(manual)} -> {mr}") if not args.apply: print("\nDRY RUN — nothing was changed. Re-run with --apply to comment these out.") def _rel(path: str, repo: Path) -> str: try: return str(Path(path).resolve().relative_to(repo)) except ValueError: return path def _find_link(line: str, url: str): """Return a match for the markdown link [text](url) whose href equals url (ignoring a trailing-slash difference).""" # Escape the url for regex, then allow an optional trailing slash on either side. esc = re.escape(url) patterns = [ r"\[[^\]]*\]\(" + esc + r"\s*\)", r"\[[^\]]*\]\(" + esc.rstrip("/") + r"/?\s*\)", ] for pat in patterns: m = re.search(pat, line) if m: return m return None def _already_commented(line: str, link_start: int) -> bool: """True if an opening between them (i.e. the link sits inside an existing HTML comment).""" pre = line[:link_start] opens = pre.count("") return opens > closes if __name__ == "__main__": main()