#!/usr/bin/env python3 """Daily external-link liveness check. Runs lychee over every http/https link in the site's content/*.md and prints a concise report of the ones that failed (dead / 4xx / 5xx). Designed for a scheduled job: read-only (HTTP GET only), always exits 0 so the job never "fails" — the report is the output. Defaults are absolute to the repo, so it runs from ANY working directory (including a cron scheduler). Override with flags if the repo lives elsewhere. Usage: python3 scripts/check_external_links.py python3 scripts/check_external_links.py --timeout 20 --retries 1 Report goes to stdout and to linkcheck/external_dead.txt (one dead URL per line with its source file:line). """ import argparse import json import shutil import subprocess from pathlib import Path DEFAULT_REPO = Path("/home/noether/libertytoolsio") LYCHEE_CANDIDATES = ["lychee", str(Path.home() / ".local/bin/lychee")] def find_lychee() -> str: for c in LYCHEE_CANDIDATES: p = shutil.which(c) if "/" not in c else (c if Path(c).exists() else None) if p: return p raise SystemExit("lychee not found. Install it first (plan Task 7a).") def main(): ap = argparse.ArgumentParser() ap.add_argument("--repo", default=str(DEFAULT_REPO)) ap.add_argument("--timeout", type=int, default=20) ap.add_argument("--retries", type=int, default=1) ap.add_argument("--threads", type=int, default="8") args = ap.parse_args() repo = Path(args.repo).resolve() content = repo / "content" md_files = sorted(str(p) for p in content.rglob("*.md")) if not md_files: print(f"No markdown files under {content}/") return lychee = find_lychee() cmd = [ lychee, "--format", "json", "--scheme", "http", "--scheme", "https", # web links only "--exclude", "mailto:", "--exclude", "tel:", "--timeout", str(args.timeout), "--max-retries", str(args.retries), "--threads", str(args.threads), ] + md_files proc = subprocess.run(cmd, capture_output=True, text=True) # lychee exits non-zero when it finds dead links; we still parse the JSON. try: data = json.loads(proc.stdout) except json.JSONDecodeError: print("Could not parse lychee output. stderr:\n", (proc.stderr or "")[-2000:]) return dead = [] for src, items in data.get("error_map", {}).items(): for it in items: url = it.get("url", "") if url.startswith(("http://", "https://")): status = it.get("status", {}).get("text", "") span = it.get("span", {}) # report the source path relative to the repo for readability try: rel = Path(src).resolve().relative_to(repo) except ValueError: rel = src dead.append((str(rel), span.get("line"), url, status)) total_checked = data.get("total", 0) ok = data.get("successful", 0) outdir = repo / "linkcheck" outdir.mkdir(exist_ok=True) with (outdir / "external_dead.txt").open("w") as f: for src, line, url, status in sorted(dead): f.write(f"{src}:{line}\t{url}\t[{status}]\n") print(f"External link check: {total_checked} links checked, {ok} ok, {len(dead)} dead.") if dead: print("\nDead external links:") for src, line, url, status in sorted(dead): print(f" {src}:{line} {url} [{status}]") print(f"\nFull list written to {outdir/'external_dead.txt'}") else: print("No dead external links found.") if __name__ == "__main__": main()