diff --git a/scripts/check_external_links.py b/scripts/check_external_links.py index df0f4c5..d26d4a2 100755 --- a/scripts/check_external_links.py +++ b/scripts/check_external_links.py @@ -89,6 +89,10 @@ def main(): for src, line, url, status in sorted(dead): f.write(f"{src}:{line}\t{url}\t[{status}]\n") + # Save the full lychee JSON so `make external-fix` can act on this run's data + # without re-pinging the web (see scripts/fix_dead_external_links.py). + (outdir / "external.json").write_text(json.dumps(data, indent=2)) + print(f"External link check: {total_checked} links checked, {ok} ok, {len(dead)} dead.") if dead: print("\nDead external links:") diff --git a/scripts/fix_dead_external_links.py b/scripts/fix_dead_external_links.py new file mode 100644 index 0000000..d97d9db --- /dev/null +++ b/scripts/fix_dead_external_links.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Comment out EXTERNAL links that are truly dead — i.e. the host/domain no longer +exists (DNS does not resolve / host unreachable / connection failed). + +This is the external analogue of fix_dead_links.py (which does the same for internal +page links). It is deliberately conservative: + + * ONLY links whose lychee reason matches the "domain-gone" class are auto-commented: + - "Connection failed" (lychee's umbrella for "couldn't even connect" — + the exact reason a dead domain like jabfreejobs.in + produces) + - "Could not resolve host" (explicit DNS failure) + - "HostUnreachable" (host unreachable) + * Everything else is LEFT ALONE and written to a manual-review file: + - HTTP status rejections (404 / 429 / 500 / redirects) -> the domain is alive + and the server responded; a 404 is "page gone", not "domain gone". + - SSL / TLS / certificate / HTTP2 errors -> the site is up. + - Connection refused / reset / closed -> weaker, possibly + transient; not the "domain doesn't exist" class. + - "Error (cached)" -> a stale lychee cache + entry, not a fresh signal. + +Commenting (NOT deleting) wraps the markdown link in an HTML comment: + [Old](/gone/) -> +so it disappears from the rendered page and the Pagefind index, and is no longer +detected by lychee on the next run (idempotent). + +Usage: + python3 scripts/fix_dead_external_links.py --report linkcheck/external.json # DRY RUN (default) + python3 scripts/fix_dead_external_links.py --report linkcheck/external.json --apply # actually edit +""" +import argparse +import json +import re +import time +from pathlib import Path +from urllib.parse import urlsplit + +# Case-insensitive substrings that mark a link as "the domain/host no longer exists". +DOMAIN_GONE_PATTERNS = [ + "connection failed", + "could not resolve host", + "hostunreachable", +] + + +def is_domain_gone(status_text: str) -> bool: + s = status_text.lower() + return any(p in s for p in DOMAIN_GONE_PATTERNS) + + +def is_onion(url: str) -> bool: + """True for .onion hosts — only reachable through Tor. This checker runs + off-Tor, so every .onion link 'fails' here by construction; they must + never be auto-commented (manual: verify via Tor).""" + host = urlsplit(url).hostname or "" + return host.endswith(".onion") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--report", required=True, + help="lychee JSON report (linkcheck/external.json)") + ap.add_argument("--apply", action="store_true", + help="actually edit the files (default is a dry-run preview)") + ap.add_argument("--repo", default=".", + help="repo root the error_map file paths are relative to") + args = ap.parse_args() + + repo = Path(args.repo).resolve() + data = json.loads(Path(args.report).read_text()) + err_map = data.get("error_map", {}) + + to_comment = [] # (rel_md_file, line, url) + manual = [] # (rel_md_file, line, url, status_text) + for md_file, items in err_map.items(): + rel = _rel(md_file, repo) + for it in items: + url = it.get("url", "") + status = it.get("status", {}).get("text", "") + line = it.get("span", {}).get("line") + if not url.startswith(("http://", "https://")): + continue + if is_onion(url): + manual.append((rel, line, url, + "(.onion — only reachable via Tor; not routable from " + "this network. Verify in Tor; do NOT auto-comment.)")) + continue + if is_domain_gone(status): + to_comment.append((rel, line, url)) + else: + manual.append((rel, line, url, status)) + + # Group edits by file, apply bottom-up so line numbers stay valid. + by_file = {} + for rel, line, url in to_comment: + by_file.setdefault(rel, []).append((line, url)) + for rel in by_file: + by_file[rel].sort(reverse=True) + + plan = 0 + for rel in sorted(by_file): + p = repo / rel + if not p.exists(): + print(f" ! missing source, skipping: {rel}") + continue + lines = p.read_text().splitlines(keepends=True) + changed = False + for line_no, url in by_file[rel]: + if not line_no: + continue + idx = line_no - 1 + if idx < 0 or idx >= len(lines): + continue + cur = lines[idx] + # Find the markdown link on this line whose href matches the dead URL + # (allow a trailing-slash mismatch). + link = _find_link(cur, url) + if not link: + print(f" ? could not locate link on {rel}:{line_no} for {url}") + continue + link_txt = link.group(0) + # Skip if it is already inside an HTML comment (idempotency guard). + if _already_commented(cur, link.start()): + continue + lines[idx] = cur[:link.start()] + "!-- " + link_txt + " --" + cur[link.end():] + changed = True + plan += 1 + print(f" {'APPLY' if args.apply else 'DRY '} {rel}:{line_no} {link_txt}") + if changed and args.apply: + p.write_text("".join(lines)) + + # Manual-review report for everything we deliberately did NOT touch. + outdir = repo / "linkcheck" + outdir.mkdir(parents=True, exist_ok=True) + mr = outdir / "manual_review_external.txt" + with mr.open("w") as f: + f.write("External links NOT auto-commented (domain still alive / ambiguous).\n") + f.write("Auto-comment only covers the domain-gone class (DNS failed / host\n") + f.write("unreachable). Review these by hand.\n\n") + for rel, line, url, status in sorted(manual): + f.write(f"{rel}:{line}\t{url}\t[{status}]\n") + + print(f"\nAuto-comment {'applied' if args.apply else 'planned'}: {plan}") + print(f"Manual review entries: {len(manual)} -> {mr}") + if not args.apply: + print("\nDRY RUN — nothing was changed. Re-run with --apply to comment these out.") + + +def _rel(path: str, repo: Path) -> str: + try: + return str(Path(path).resolve().relative_to(repo)) + except ValueError: + return path + + +def _find_link(line: str, url: str): + """Return a match for the markdown link [text](url) whose href equals url + (ignoring a trailing-slash difference).""" + # Escape the url for regex, then allow an optional trailing slash on either side. + esc = re.escape(url) + patterns = [ + r"\[[^\]]*\]\(" + esc + r"\s*\)", + r"\[[^\]]*\]\(" + esc.rstrip("/") + r"/?\s*\)", + ] + for pat in patterns: + m = re.search(pat, line) + if m: + return m + return None + + +def _already_commented(line: str, link_start: int) -> bool: + """True if an opening between + them (i.e. the link sits inside an existing HTML comment).""" + pre = line[:link_start] + opens = pre.count("") + return opens > closes + + +if __name__ == "__main__": + main()