fix: ship external-fix script + lychee JSON save so make external-fix works fresh
scripts/fix_dead_external_links.py was referenced by the Makefile but never committed, and check_external_links.py did not yet save linkcheck/external.json (both were working-tree only), so external-fix/external-fix-apply failed on any fresh clone. The fixer is deliberately conservative: only DNS-failed / connection-failed / host-unreachable links are auto-commented; .onion links and ambiguous failures (404/5xx/SSL/refused) go to manual review.
This commit is contained in:
noether
@@ -89,6 +89,10 @@ def main():
|
||||
for src, line, url, status in sorted(dead):
|
||||
f.write(f"{src}:{line}\t{url}\t[{status}]\n")
|
||||
|
||||
# Save the full lychee JSON so `make external-fix` can act on this run's data
|
||||
# without re-pinging the web (see scripts/fix_dead_external_links.py).
|
||||
(outdir / "external.json").write_text(json.dumps(data, indent=2))
|
||||
|
||||
print(f"External link check: {total_checked} links checked, {ok} ok, {len(dead)} dead.")
|
||||
if dead:
|
||||
print("\nDead external links:")
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Comment out EXTERNAL links that are truly dead — i.e. the host/domain no longer
|
||||
exists (DNS does not resolve / host unreachable / connection failed).
|
||||
|
||||
This is the external analogue of fix_dead_links.py (which does the same for internal
|
||||
page links). It is deliberately conservative:
|
||||
|
||||
* ONLY links whose lychee reason matches the "domain-gone" class are auto-commented:
|
||||
- "Connection failed" (lychee's umbrella for "couldn't even connect" —
|
||||
the exact reason a dead domain like jabfreejobs.in
|
||||
produces)
|
||||
- "Could not resolve host" (explicit DNS failure)
|
||||
- "HostUnreachable" (host unreachable)
|
||||
* Everything else is LEFT ALONE and written to a manual-review file:
|
||||
- HTTP status rejections (404 / 429 / 500 / redirects) -> the domain is alive
|
||||
and the server responded; a 404 is "page gone", not "domain gone".
|
||||
- SSL / TLS / certificate / HTTP2 errors -> the site is up.
|
||||
- Connection refused / reset / closed -> weaker, possibly
|
||||
transient; not the "domain doesn't exist" class.
|
||||
- "Error (cached)" -> a stale lychee cache
|
||||
entry, not a fresh signal.
|
||||
|
||||
Commenting (NOT deleting) wraps the markdown link in an HTML comment:
|
||||
[Old](/gone/) -> <!-- [Old](/gone/) -->
|
||||
so it disappears from the rendered page and the Pagefind index, and is no longer
|
||||
detected by lychee on the next run (idempotent).
|
||||
|
||||
Usage:
|
||||
python3 scripts/fix_dead_external_links.py --report linkcheck/external.json # DRY RUN (default)
|
||||
python3 scripts/fix_dead_external_links.py --report linkcheck/external.json --apply # actually edit
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
# Case-insensitive substrings that mark a link as "the domain/host no longer exists".
|
||||
DOMAIN_GONE_PATTERNS = [
|
||||
"connection failed",
|
||||
"could not resolve host",
|
||||
"hostunreachable",
|
||||
]
|
||||
|
||||
|
||||
def is_domain_gone(status_text: str) -> bool:
|
||||
s = status_text.lower()
|
||||
return any(p in s for p in DOMAIN_GONE_PATTERNS)
|
||||
|
||||
|
||||
def is_onion(url: str) -> bool:
|
||||
"""True for .onion hosts — only reachable through Tor. This checker runs
|
||||
off-Tor, so every .onion link 'fails' here by construction; they must
|
||||
never be auto-commented (manual: verify via Tor)."""
|
||||
host = urlsplit(url).hostname or ""
|
||||
return host.endswith(".onion")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--report", required=True,
|
||||
help="lychee JSON report (linkcheck/external.json)")
|
||||
ap.add_argument("--apply", action="store_true",
|
||||
help="actually edit the files (default is a dry-run preview)")
|
||||
ap.add_argument("--repo", default=".",
|
||||
help="repo root the error_map file paths are relative to")
|
||||
args = ap.parse_args()
|
||||
|
||||
repo = Path(args.repo).resolve()
|
||||
data = json.loads(Path(args.report).read_text())
|
||||
err_map = data.get("error_map", {})
|
||||
|
||||
to_comment = [] # (rel_md_file, line, url)
|
||||
manual = [] # (rel_md_file, line, url, status_text)
|
||||
for md_file, items in err_map.items():
|
||||
rel = _rel(md_file, repo)
|
||||
for it in items:
|
||||
url = it.get("url", "")
|
||||
status = it.get("status", {}).get("text", "")
|
||||
line = it.get("span", {}).get("line")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
continue
|
||||
if is_onion(url):
|
||||
manual.append((rel, line, url,
|
||||
"(.onion — only reachable via Tor; not routable from "
|
||||
"this network. Verify in Tor; do NOT auto-comment.)"))
|
||||
continue
|
||||
if is_domain_gone(status):
|
||||
to_comment.append((rel, line, url))
|
||||
else:
|
||||
manual.append((rel, line, url, status))
|
||||
|
||||
# Group edits by file, apply bottom-up so line numbers stay valid.
|
||||
by_file = {}
|
||||
for rel, line, url in to_comment:
|
||||
by_file.setdefault(rel, []).append((line, url))
|
||||
for rel in by_file:
|
||||
by_file[rel].sort(reverse=True)
|
||||
|
||||
plan = 0
|
||||
for rel in sorted(by_file):
|
||||
p = repo / rel
|
||||
if not p.exists():
|
||||
print(f" ! missing source, skipping: {rel}")
|
||||
continue
|
||||
lines = p.read_text().splitlines(keepends=True)
|
||||
changed = False
|
||||
for line_no, url in by_file[rel]:
|
||||
if not line_no:
|
||||
continue
|
||||
idx = line_no - 1
|
||||
if idx < 0 or idx >= len(lines):
|
||||
continue
|
||||
cur = lines[idx]
|
||||
# Find the markdown link on this line whose href matches the dead URL
|
||||
# (allow a trailing-slash mismatch).
|
||||
link = _find_link(cur, url)
|
||||
if not link:
|
||||
print(f" ? could not locate link on {rel}:{line_no} for {url}")
|
||||
continue
|
||||
link_txt = link.group(0)
|
||||
# Skip if it is already inside an HTML comment (idempotency guard).
|
||||
if _already_commented(cur, link.start()):
|
||||
continue
|
||||
lines[idx] = cur[:link.start()] + "!-- " + link_txt + " --" + cur[link.end():]
|
||||
changed = True
|
||||
plan += 1
|
||||
print(f" {'APPLY' if args.apply else 'DRY '} {rel}:{line_no} {link_txt}")
|
||||
if changed and args.apply:
|
||||
p.write_text("".join(lines))
|
||||
|
||||
# Manual-review report for everything we deliberately did NOT touch.
|
||||
outdir = repo / "linkcheck"
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
mr = outdir / "manual_review_external.txt"
|
||||
with mr.open("w") as f:
|
||||
f.write("External links NOT auto-commented (domain still alive / ambiguous).\n")
|
||||
f.write("Auto-comment only covers the domain-gone class (DNS failed / host\n")
|
||||
f.write("unreachable). Review these by hand.\n\n")
|
||||
for rel, line, url, status in sorted(manual):
|
||||
f.write(f"{rel}:{line}\t{url}\t[{status}]\n")
|
||||
|
||||
print(f"\nAuto-comment {'applied' if args.apply else 'planned'}: {plan}")
|
||||
print(f"Manual review entries: {len(manual)} -> {mr}")
|
||||
if not args.apply:
|
||||
print("\nDRY RUN — nothing was changed. Re-run with --apply to comment these out.")
|
||||
|
||||
|
||||
def _rel(path: str, repo: Path) -> str:
|
||||
try:
|
||||
return str(Path(path).resolve().relative_to(repo))
|
||||
except ValueError:
|
||||
return path
|
||||
|
||||
|
||||
def _find_link(line: str, url: str):
|
||||
"""Return a match for the markdown link [text](url) whose href equals url
|
||||
(ignoring a trailing-slash difference)."""
|
||||
# Escape the url for regex, then allow an optional trailing slash on either side.
|
||||
esc = re.escape(url)
|
||||
patterns = [
|
||||
r"\[[^\]]*\]\(" + esc + r"\s*\)",
|
||||
r"\[[^\]]*\]\(" + esc.rstrip("/") + r"/?\s*\)",
|
||||
]
|
||||
for pat in patterns:
|
||||
m = re.search(pat, line)
|
||||
if m:
|
||||
return m
|
||||
return None
|
||||
|
||||
|
||||
def _already_commented(line: str, link_start: int) -> bool:
|
||||
"""True if an opening <!-- appears before the link with no matching --> between
|
||||
them (i.e. the link sits inside an existing HTML comment)."""
|
||||
pre = line[:link_start]
|
||||
opens = pre.count("<!--")
|
||||
closes = pre.count("-->")
|
||||
return opens > closes
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user