From 2d8f79b7fd6454cec2261f33838aeb7d23567568 Mon Sep 17 00:00:00 2001 From: noether Date: Sun, 11 Oct 2026 08:58:50 -0600 Subject: [PATCH] build: replace ad-hoc run with a full make workflow + helper scripts The old Makefile was a single 'run' target (hugo server + pagefind --serve). Replace it with documented targets: build hugo --gc -> public/ index build + pagefind --site public -> public/pagefind/ serve / run index + pagefind's own server on 127.0.0.1:1414 (local) serve-lan static-serve public/ on 0.0.0.0:8090 for other machines link-check offline internal-link scan (lychee --scheme file) -> JSON link-fix comment out confirmed-dead internal links, then rebuild external-check network scan of external links (report-only) clean / all Add scripts/: serve_lan.py stdlib static server rooted at public/ (prints LAN IP) fix_dead_links.py idempotent internal dead-link comment-out check_external_links.py external link report (drives the daily cron job) .gitignore: add linkcheck/, resources/_gen/ (regenerable Hugo asset cache), __pycache__/, and .hermes/. --- .gitignore | 4 ++ Makefile | 55 ++++++++++++++++- scripts/check_external_links.py | 103 ++++++++++++++++++++++++++++++++ scripts/fix_dead_links.py | 100 +++++++++++++++++++++++++++++++ scripts/serve_lan.py | 45 ++++++++++++++ 5 files changed, 304 insertions(+), 3 deletions(-) create mode 100755 scripts/check_external_links.py create mode 100755 scripts/fix_dead_links.py create mode 100755 scripts/serve_lan.py diff --git a/.gitignore b/.gitignore index 8d285df..432e139 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ **/public/ +linkcheck/ +resources/_gen/ +__pycache__/ +.hermes/ diff --git a/Makefile b/Makefile index 73822b8..e992c9f 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,52 @@ -run: - hugo server & - npx -y pagefind --site public --serve +HUGO := hugo +PAGEFIND := npx -y pagefind +LYCHEE := lychee +OUT := public + +.PHONY: all build index serve run link-check link-fix external-check serve-lan clean + +all: index + +# Build Hugo content -> public/ +build: + $(HUGO) --gc + +# (Re)build the Pagefind search index from public/ -> public/pagefind/ +index: build + $(PAGEFIND) --site $(OUT) + +# Build content + index, then serve the built site WITH search on :1414 (dev preview). +# Uses pagefind's own server. Local only (binds 127.0.0.1) -- for LAN use `make serve-lan`. +serve: index + $(PAGEFIND) --site $(OUT) --serve + +run: serve + +# DEAD LINKS -- internal (offline, fast): scan the SOURCE markdown; internal links +# resolve to file:// under public/. External links are skipped by --scheme file. +# Output: linkcheck/internal.json (exit != 0 when dead links exist; that is fine). +link-check: build + mkdir -p linkcheck + $(LYCHEE) --root-dir $(OUT) --default-extension html --scheme file \ + --format json --timeout 10 \ + $(wildcard content/*.md) $(wildcard content/*/*.md) > linkcheck/internal.json + +# Comment out confirmed dead INTERNAL page links, then rebuild content + index so they +# leave the live page AND the search index. (External failures -> manual_review.txt.) +link-fix: + python3 scripts/fix_dead_links.py --report linkcheck/internal.json --apply + $(HUGO) --gc + $(PAGEFIND) --site $(OUT) + +# DEAD LINKS -- external (network): scan all http/https links in content. Report-only. +# Always exits 0; the report is the point. +external-check: + python3 scripts/check_external_links.py + +# Serve the BUILT site on all interfaces so another LAN machine can open it: +# http://:8090/ (IP printed at startup; also `hostname -I | head -1`) +serve-lan: + python3 scripts/serve_lan.py + +clean: + rm -rf $(OUT) linkcheck diff --git a/scripts/check_external_links.py b/scripts/check_external_links.py new file mode 100755 index 0000000..df0f4c5 --- /dev/null +++ b/scripts/check_external_links.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Daily external-link liveness check. + +Runs lychee over every http/https link in the site's content/*.md and prints a +concise report of the ones that failed (dead / 4xx / 5xx). Designed for a +scheduled job: read-only (HTTP GET only), always exits 0 so the job never +"fails" — the report is the output. + +Defaults are absolute to the repo, so it runs from ANY working directory +(including a cron scheduler). Override with flags if the repo lives elsewhere. + +Usage: + python3 scripts/check_external_links.py + python3 scripts/check_external_links.py --timeout 20 --retries 1 + +Report goes to stdout and to linkcheck/external_dead.txt (one dead URL per line +with its source file:line). +""" +import argparse +import json +import shutil +import subprocess +from pathlib import Path + +DEFAULT_REPO = Path("/home/noether/libertytoolsio") +LYCHEE_CANDIDATES = ["lychee", str(Path.home() / ".local/bin/lychee")] + + +def find_lychee() -> str: + for c in LYCHEE_CANDIDATES: + p = shutil.which(c) if "/" not in c else (c if Path(c).exists() else None) + if p: + return p + raise SystemExit("lychee not found. Install it first (plan Task 7a).") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--repo", default=str(DEFAULT_REPO)) + ap.add_argument("--timeout", type=int, default=20) + ap.add_argument("--retries", type=int, default=1) + ap.add_argument("--threads", type=int, default="8") + args = ap.parse_args() + + repo = Path(args.repo).resolve() + content = repo / "content" + md_files = sorted(str(p) for p in content.rglob("*.md")) + if not md_files: + print(f"No markdown files under {content}/") + return + + lychee = find_lychee() + cmd = [ + lychee, "--format", "json", + "--scheme", "http", "--scheme", "https", # web links only + "--exclude", "mailto:", "--exclude", "tel:", + "--timeout", str(args.timeout), "--max-retries", str(args.retries), + "--threads", str(args.threads), + ] + md_files + + proc = subprocess.run(cmd, capture_output=True, text=True) + # lychee exits non-zero when it finds dead links; we still parse the JSON. + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + print("Could not parse lychee output. stderr:\n", (proc.stderr or "")[-2000:]) + return + + dead = [] + for src, items in data.get("error_map", {}).items(): + for it in items: + url = it.get("url", "") + if url.startswith(("http://", "https://")): + status = it.get("status", {}).get("text", "") + span = it.get("span", {}) + # report the source path relative to the repo for readability + try: + rel = Path(src).resolve().relative_to(repo) + except ValueError: + rel = src + dead.append((str(rel), span.get("line"), url, status)) + + total_checked = data.get("total", 0) + ok = data.get("successful", 0) + + outdir = repo / "linkcheck" + outdir.mkdir(exist_ok=True) + with (outdir / "external_dead.txt").open("w") as f: + for src, line, url, status in sorted(dead): + f.write(f"{src}:{line}\t{url}\t[{status}]\n") + + print(f"External link check: {total_checked} links checked, {ok} ok, {len(dead)} dead.") + if dead: + print("\nDead external links:") + for src, line, url, status in sorted(dead): + print(f" {src}:{line} {url} [{status}]") + print(f"\nFull list written to {outdir/'external_dead.txt'}") + else: + print("No dead external links found.") + + +if __name__ == "__main__": + main() diff --git a/scripts/fix_dead_links.py b/scripts/fix_dead_links.py new file mode 100755 index 0000000..f4626a5 --- /dev/null +++ b/scripts/fix_dead_links.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Comment out confirmed dead INTERNAL page links, report external/bare-domain failures. + +Reads lychee's JSON report (linkcheck/internal.json). A link is auto-commented ONLY +if lychee flagged its file:// path as a missing file under the built site (public/). +External https:// failures and bare-domain failures are written to a manual-review +report and left untouched. Idempotent: already-commented links are not detected by +lychee, so re-runs are safe. + +IMPORTANT: run lychee with the SOURCE markdown as input (e.g. `lychee --root-dir public +--default-extension html content/*.md`) so the error_map keys are the markdown files +we actually edit. They are relative to the repo root. + +Usage: + python3 scripts/fix_dead_links.py --report linkcheck/internal.json # dry-run + python3 scripts/fix_dead_links.py --report linkcheck/internal.json --apply # edit +""" +import argparse +import json +import re +from pathlib import Path + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--report", required=True) + ap.add_argument("--apply", action="store_true") + ap.add_argument("--public-dir", default="public", + help="built site dir that lychee's --root-dir pointed at") + args = ap.parse_args() + + data = json.loads(Path(args.report).read_text()) + err_map = data.get("error_map", {}) + + to_comment = [] # (md_file_relative, line, dest) + manual = [] # (md_file_relative, line, url, status_text) + for md_file, items in err_map.items(): + for it in items: + url = it.get("url", "") + status = it.get("status", {}).get("text", "") + span = it.get("span", {}) + if url.startswith("file://"): + after = url[len("file://"):] + m = re.search(r"/" + re.escape(args.public_dir) + r"/(.*)$", after) + if m: + to_comment.append((md_file, span.get("line"), "/" + m.group(1))) + continue + manual.append((md_file, span.get("line"), url, status)) + + # Group by source file (error_map keys are relative to repo root). + by_file = {} + for md_file, line, dest in to_comment: + by_file.setdefault(md_file, []).append((line, dest)) + for md_file in by_file: + by_file[md_file].sort(reverse=True) # edit bottom-up; keep line count stable + + plan = 0 + for md_file, edits in sorted(by_file.items()): + p = Path(md_file) + if not p.exists(): + print(f" ! missing source, skipping: {p}") + continue + lines = p.read_text().splitlines(keepends=True) + changed = False + for line_no, dest in edits: + if not line_no: + continue + idx = line_no - 1 + if idx < 0 or idx >= len(lines): + continue + cur = lines[idx] + # lychee reports the resolved file path (trailing slash stripped); the + # markdown link may keep a trailing slash, so allow an optional "/". + m = re.search(r"\[([^\]]*)\]\(" + re.escape(dest) + r"/?\s*\)", cur) + if not m: + print(f" ? could not locate link on {md_file}:{line_no} for {dest}") + continue + link_txt = m.group(0) + if link_txt.count("")): + continue # already commented + lines[idx] = cur[:m.start()] + "" + cur[m.end():] + changed = True + plan += 1 + print(f" {'APPLY ' if args.apply else 'DRY '} {md_file}:{line_no} {link_txt}") + if changed and args.apply: + p.write_text("".join(lines)) + + outdir = Path("linkcheck") + outdir.mkdir(parents=True, exist_ok=True) + with (outdir / "manual_review.txt").open("w") as f: + f.write("Manual review (NOT auto-commented). External/bare-domain failures.\n\n") + for md_file, line, url, status in sorted(manual): + f.write(f"{md_file}:{line}\t{url}\t[{status}]\n") + + print(f"\nAuto-comment planned: {plan} (apply={args.apply})") + print(f"Manual review entries: {len(manual)} -> {outdir/'manual_review.txt'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/serve_lan.py b/scripts/serve_lan.py new file mode 100755 index 0000000..d9f6f60 --- /dev/null +++ b/scripts/serve_lan.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Serve the built site (public/) on all interfaces so other machines on the +same LAN can reach it. Usage: python3 serve_lan.py [port] (default 8090) + +The site must be built first: make build index +Then start this: python3 serve_lan.py +Address to open from ANOTHER machine on the same network: + http://:8090/ +where the LAN IP is printed at startup (and is also hostname -I | head -1). +""" +import functools +import http.server +import socket +import subprocess +import sys + +PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8090 + +# Serve the built site (public/), NOT the repo root. +handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory="public") +server = http.server.ThreadingHTTPServer(("0.0.0.0", PORT), handler) + + +def lan_ip() -> str: + try: + return subprocess.check_output(["hostname", "-I"], text=True).split()[0] + except Exception: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("8.8.8.8", 80)) + return s.getsockname()[0] + finally: + s.close() + + +ip = lan_ip() +print(f"Serving public/ -> http://0.0.0.0:{PORT}/", flush=True) +print(f"LAN address -> http://{ip}:{PORT}/ (open this from another machine)", flush=True) +print("Press Ctrl+C to stop.", flush=True) +try: + server.serve_forever() +except KeyboardInterrupt: + pass +finally: + server.server_close()