libertytoolsio/scripts/fix_dead_external_links.py

#!/usr/bin/env python3
"""Comment out EXTERNAL links that are truly dead — i.e. the host/domain no longer
exists (DNS does not resolve / host unreachable / connection failed).

This is the external analogue of fix_dead_links.py (which does the same for internal
page links). It is deliberately conservative:

  * ONLY links whose lychee reason matches the "domain-gone" class are auto-commented:
      - "Connection failed"        (lychee's umbrella for "couldn't even connect" —
                                    the exact reason a dead domain like jabfreejobs.in
                                    produces)
      - "Could not resolve host"   (explicit DNS failure)
      - "HostUnreachable"          (host unreachable)
  * Everything else is LEFT ALONE and written to a manual-review file:
      - HTTP status rejections (404 / 429 / 500 / redirects)  -> the domain is alive
        and the server responded; a 404 is "page gone", not "domain gone".
      - SSL / TLS / certificate / HTTP2 errors                -> the site is up.
      - Connection refused / reset / closed                   -> weaker, possibly
        transient; not the "domain doesn't exist" class.
      - "Error (cached)"                                       -> a stale lychee cache
        entry, not a fresh signal.

Commenting (NOT deleting) wraps the markdown link in an HTML comment:
    [Old](/gone/)  ->  <!-- [Old](/gone/) -->
so it disappears from the rendered page and the Pagefind index, and is no longer
detected by lychee on the next run (idempotent).

Usage:
  python3 scripts/fix_dead_external_links.py --report linkcheck/external.json            # DRY RUN (default)
  python3 scripts/fix_dead_external_links.py --report linkcheck/external.json --apply    # actually edit
"""
import argparse
import json
import re
import time
from pathlib import Path
from urllib.parse import urlsplit

# Case-insensitive substrings that mark a link as "the domain/host no longer exists".
DOMAIN_GONE_PATTERNS = [
    "connection failed",
    "could not resolve host",
    "hostunreachable",
]


def is_domain_gone(status_text: str) -> bool:
    s = status_text.lower()
    return any(p in s for p in DOMAIN_GONE_PATTERNS)


def is_onion(url: str) -> bool:
    """True for .onion hosts — only reachable through Tor. This checker runs
    off-Tor, so every .onion link 'fails' here by construction; they must
    never be auto-commented (manual: verify via Tor)."""
    host = urlsplit(url).hostname or ""
    return host.endswith(".onion")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--report", required=True,
                    help="lychee JSON report (linkcheck/external.json)")
    ap.add_argument("--apply", action="store_true",
                    help="actually edit the files (default is a dry-run preview)")
    ap.add_argument("--repo", default=".",
                    help="repo root the error_map file paths are relative to")
    args = ap.parse_args()

    repo = Path(args.repo).resolve()
    data = json.loads(Path(args.report).read_text())
    err_map = data.get("error_map", {})

    to_comment = []   # (rel_md_file, line, url)
    manual = []       # (rel_md_file, line, url, status_text)
    for md_file, items in err_map.items():
        rel = _rel(md_file, repo)
        for it in items:
            url = it.get("url", "")
            status = it.get("status", {}).get("text", "")
            line = it.get("span", {}).get("line")
            if not url.startswith(("http://", "https://")):
                continue
            if is_onion(url):
                manual.append((rel, line, url,
                               "(.onion — only reachable via Tor; not routable from "
                               "this network. Verify in Tor; do NOT auto-comment.)"))
                continue
            if is_domain_gone(status):
                to_comment.append((rel, line, url))
            else:
                manual.append((rel, line, url, status))

    # Group edits by file, apply bottom-up so line numbers stay valid.
    by_file = {}
    for rel, line, url in to_comment:
        by_file.setdefault(rel, []).append((line, url))
    for rel in by_file:
        by_file[rel].sort(reverse=True)

    plan = 0
    for rel in sorted(by_file):
        p = repo / rel
        if not p.exists():
            print(f"  ! missing source, skipping: {rel}")
            continue
        lines = p.read_text().splitlines(keepends=True)
        changed = False
        for line_no, url in by_file[rel]:
            if not line_no:
                continue
            idx = line_no - 1
            if idx < 0 or idx >= len(lines):
                continue
            cur = lines[idx]
            # Find the markdown link on this line whose href matches the dead URL
            # (allow a trailing-slash mismatch).
            link = _find_link(cur, url)
            if not link:
                print(f"  ? could not locate link on {rel}:{line_no} for {url}")
                continue
            link_txt = link.group(0)
            # Skip if it is already inside an HTML comment (idempotency guard).
            if _already_commented(cur, link.start()):
                continue
            lines[idx] = cur[:link.start()] + "!-- " + link_txt + " --" + cur[link.end():]
            changed = True
            plan += 1
            print(f"  {'APPLY' if args.apply else 'DRY  '}  {rel}:{line_no}  {link_txt}")
        if changed and args.apply:
            p.write_text("".join(lines))

    # Manual-review report for everything we deliberately did NOT touch.
    outdir = repo / "linkcheck"
    outdir.mkdir(parents=True, exist_ok=True)
    mr = outdir / "manual_review_external.txt"
    with mr.open("w") as f:
        f.write("External links NOT auto-commented (domain still alive / ambiguous).\n")
        f.write("Auto-comment only covers the domain-gone class (DNS failed / host\n")
        f.write("unreachable). Review these by hand.\n\n")
        for rel, line, url, status in sorted(manual):
            f.write(f"{rel}:{line}\t{url}\t[{status}]\n")

    print(f"\nAuto-comment {'applied' if args.apply else 'planned'}: {plan}")
    print(f"Manual review entries: {len(manual)} -> {mr}")
    if not args.apply:
        print("\nDRY RUN — nothing was changed. Re-run with --apply to comment these out.")


def _rel(path: str, repo: Path) -> str:
    try:
        return str(Path(path).resolve().relative_to(repo))
    except ValueError:
        return path


def _find_link(line: str, url: str):
    """Return a match for the markdown link [text](url) whose href equals url
    (ignoring a trailing-slash difference)."""
    # Escape the url for regex, then allow an optional trailing slash on either side.
    esc = re.escape(url)
    patterns = [
        r"\[[^\]]*\]\(" + esc + r"\s*\)",
        r"\[[^\]]*\]\(" + esc.rstrip("/") + r"/?\s*\)",
    ]
    for pat in patterns:
        m = re.search(pat, line)
        if m:
            return m
    return None


def _already_commented(line: str, link_start: int) -> bool:
    """True if an opening <!-- appears before the link with no matching --> between
    them (i.e. the link sits inside an existing HTML comment)."""
    pre = line[:link_start]
    opens = pre.count("<!--")
    closes = pre.count("-->")
    return opens > closes


if __name__ == "__main__":
    main()