noether
2d8f79b7fd
The old Makefile was a single 'run' target (hugo server + pagefind --serve). Replace it with documented targets: build hugo --gc -> public/ index build + pagefind --site public -> public/pagefind/ serve / run index + pagefind's own server on 127.0.0.1:1414 (local) serve-lan static-serve public/ on 0.0.0.0:8090 for other machines link-check offline internal-link scan (lychee --scheme file) -> JSON link-fix comment out confirmed-dead internal links, then rebuild external-check network scan of external links (report-only) clean / all Add scripts/: serve_lan.py stdlib static server rooted at public/ (prints LAN IP) fix_dead_links.py idempotent internal dead-link comment-out check_external_links.py external link report (drives the daily cron job) .gitignore: add linkcheck/, resources/_gen/ (regenerable Hugo asset cache), __pycache__/, and .hermes/.
104 lines
3.6 KiB
Python
Executable File
104 lines
3.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Daily external-link liveness check.
|
|
|
|
Runs lychee over every http/https link in the site's content/*.md and prints a
|
|
concise report of the ones that failed (dead / 4xx / 5xx). Designed for a
|
|
scheduled job: read-only (HTTP GET only), always exits 0 so the job never
|
|
"fails" — the report is the output.
|
|
|
|
Defaults are absolute to the repo, so it runs from ANY working directory
|
|
(including a cron scheduler). Override with flags if the repo lives elsewhere.
|
|
|
|
Usage:
|
|
python3 scripts/check_external_links.py
|
|
python3 scripts/check_external_links.py --timeout 20 --retries 1
|
|
|
|
Report goes to stdout and to linkcheck/external_dead.txt (one dead URL per line
|
|
with its source file:line).
|
|
"""
|
|
import argparse
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
DEFAULT_REPO = Path("/home/noether/libertytoolsio")
|
|
LYCHEE_CANDIDATES = ["lychee", str(Path.home() / ".local/bin/lychee")]
|
|
|
|
|
|
def find_lychee() -> str:
|
|
for c in LYCHEE_CANDIDATES:
|
|
p = shutil.which(c) if "/" not in c else (c if Path(c).exists() else None)
|
|
if p:
|
|
return p
|
|
raise SystemExit("lychee not found. Install it first (plan Task 7a).")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--repo", default=str(DEFAULT_REPO))
|
|
ap.add_argument("--timeout", type=int, default=20)
|
|
ap.add_argument("--retries", type=int, default=1)
|
|
ap.add_argument("--threads", type=int, default="8")
|
|
args = ap.parse_args()
|
|
|
|
repo = Path(args.repo).resolve()
|
|
content = repo / "content"
|
|
md_files = sorted(str(p) for p in content.rglob("*.md"))
|
|
if not md_files:
|
|
print(f"No markdown files under {content}/")
|
|
return
|
|
|
|
lychee = find_lychee()
|
|
cmd = [
|
|
lychee, "--format", "json",
|
|
"--scheme", "http", "--scheme", "https", # web links only
|
|
"--exclude", "mailto:", "--exclude", "tel:",
|
|
"--timeout", str(args.timeout), "--max-retries", str(args.retries),
|
|
"--threads", str(args.threads),
|
|
] + md_files
|
|
|
|
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
# lychee exits non-zero when it finds dead links; we still parse the JSON.
|
|
try:
|
|
data = json.loads(proc.stdout)
|
|
except json.JSONDecodeError:
|
|
print("Could not parse lychee output. stderr:\n", (proc.stderr or "")[-2000:])
|
|
return
|
|
|
|
dead = []
|
|
for src, items in data.get("error_map", {}).items():
|
|
for it in items:
|
|
url = it.get("url", "")
|
|
if url.startswith(("http://", "https://")):
|
|
status = it.get("status", {}).get("text", "")
|
|
span = it.get("span", {})
|
|
# report the source path relative to the repo for readability
|
|
try:
|
|
rel = Path(src).resolve().relative_to(repo)
|
|
except ValueError:
|
|
rel = src
|
|
dead.append((str(rel), span.get("line"), url, status))
|
|
|
|
total_checked = data.get("total", 0)
|
|
ok = data.get("successful", 0)
|
|
|
|
outdir = repo / "linkcheck"
|
|
outdir.mkdir(exist_ok=True)
|
|
with (outdir / "external_dead.txt").open("w") as f:
|
|
for src, line, url, status in sorted(dead):
|
|
f.write(f"{src}:{line}\t{url}\t[{status}]\n")
|
|
|
|
print(f"External link check: {total_checked} links checked, {ok} ok, {len(dead)} dead.")
|
|
if dead:
|
|
print("\nDead external links:")
|
|
for src, line, url, status in sorted(dead):
|
|
print(f" {src}:{line} {url} [{status}]")
|
|
print(f"\nFull list written to {outdir/'external_dead.txt'}")
|
|
else:
|
|
print("No dead external links found.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|