build: replace ad-hoc run with a full make workflow + helper scripts
The old Makefile was a single 'run' target (hugo server + pagefind --serve). Replace it with documented targets: build hugo --gc -> public/ index build + pagefind --site public -> public/pagefind/ serve / run index + pagefind's own server on 127.0.0.1:1414 (local) serve-lan static-serve public/ on 0.0.0.0:8090 for other machines link-check offline internal-link scan (lychee --scheme file) -> JSON link-fix comment out confirmed-dead internal links, then rebuild external-check network scan of external links (report-only) clean / all Add scripts/: serve_lan.py stdlib static server rooted at public/ (prints LAN IP) fix_dead_links.py idempotent internal dead-link comment-out check_external_links.py external link report (drives the daily cron job) .gitignore: add linkcheck/, resources/_gen/ (regenerable Hugo asset cache), __pycache__/, and .hermes/.
This commit is contained in:
noether
@@ -1 +1,5 @@
|
||||
**/public/
|
||||
linkcheck/
|
||||
resources/_gen/
|
||||
__pycache__/
|
||||
.hermes/
|
||||
|
||||
@@ -1,3 +1,52 @@
|
||||
run:
|
||||
hugo server &
|
||||
npx -y pagefind --site public --serve
|
||||
HUGO := hugo
|
||||
PAGEFIND := npx -y pagefind
|
||||
LYCHEE := lychee
|
||||
OUT := public
|
||||
|
||||
.PHONY: all build index serve run link-check link-fix external-check serve-lan clean
|
||||
|
||||
all: index
|
||||
|
||||
# Build Hugo content -> public/
|
||||
build:
|
||||
$(HUGO) --gc
|
||||
|
||||
# (Re)build the Pagefind search index from public/ -> public/pagefind/
|
||||
index: build
|
||||
$(PAGEFIND) --site $(OUT)
|
||||
|
||||
# Build content + index, then serve the built site WITH search on :1414 (dev preview).
|
||||
# Uses pagefind's own server. Local only (binds 127.0.0.1) -- for LAN use `make serve-lan`.
|
||||
serve: index
|
||||
$(PAGEFIND) --site $(OUT) --serve
|
||||
|
||||
run: serve
|
||||
|
||||
# DEAD LINKS -- internal (offline, fast): scan the SOURCE markdown; internal links
|
||||
# resolve to file:// under public/. External links are skipped by --scheme file.
|
||||
# Output: linkcheck/internal.json (exit != 0 when dead links exist; that is fine).
|
||||
link-check: build
|
||||
mkdir -p linkcheck
|
||||
$(LYCHEE) --root-dir $(OUT) --default-extension html --scheme file \
|
||||
--format json --timeout 10 \
|
||||
$(wildcard content/*.md) $(wildcard content/*/*.md) > linkcheck/internal.json
|
||||
|
||||
# Comment out confirmed dead INTERNAL page links, then rebuild content + index so they
|
||||
# leave the live page AND the search index. (External failures -> manual_review.txt.)
|
||||
link-fix:
|
||||
python3 scripts/fix_dead_links.py --report linkcheck/internal.json --apply
|
||||
$(HUGO) --gc
|
||||
$(PAGEFIND) --site $(OUT)
|
||||
|
||||
# DEAD LINKS -- external (network): scan all http/https links in content. Report-only.
|
||||
# Always exits 0; the report is the point.
|
||||
external-check:
|
||||
python3 scripts/check_external_links.py
|
||||
|
||||
# Serve the BUILT site on all interfaces so another LAN machine can open it:
|
||||
# http://<this-machine-LAN-IP>:8090/ (IP printed at startup; also `hostname -I | head -1`)
|
||||
serve-lan:
|
||||
python3 scripts/serve_lan.py
|
||||
|
||||
clean:
|
||||
rm -rf $(OUT) linkcheck
|
||||
|
||||
Executable
+103
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Daily external-link liveness check.
|
||||
|
||||
Runs lychee over every http/https link in the site's content/*.md and prints a
|
||||
concise report of the ones that failed (dead / 4xx / 5xx). Designed for a
|
||||
scheduled job: read-only (HTTP GET only), always exits 0 so the job never
|
||||
"fails" — the report is the output.
|
||||
|
||||
Defaults are absolute to the repo, so it runs from ANY working directory
|
||||
(including a cron scheduler). Override with flags if the repo lives elsewhere.
|
||||
|
||||
Usage:
|
||||
python3 scripts/check_external_links.py
|
||||
python3 scripts/check_external_links.py --timeout 20 --retries 1
|
||||
|
||||
Report goes to stdout and to linkcheck/external_dead.txt (one dead URL per line
|
||||
with its source file:line).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_REPO = Path("/home/noether/libertytoolsio")
|
||||
LYCHEE_CANDIDATES = ["lychee", str(Path.home() / ".local/bin/lychee")]
|
||||
|
||||
|
||||
def find_lychee() -> str:
|
||||
for c in LYCHEE_CANDIDATES:
|
||||
p = shutil.which(c) if "/" not in c else (c if Path(c).exists() else None)
|
||||
if p:
|
||||
return p
|
||||
raise SystemExit("lychee not found. Install it first (plan Task 7a).")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--repo", default=str(DEFAULT_REPO))
|
||||
ap.add_argument("--timeout", type=int, default=20)
|
||||
ap.add_argument("--retries", type=int, default=1)
|
||||
ap.add_argument("--threads", type=int, default="8")
|
||||
args = ap.parse_args()
|
||||
|
||||
repo = Path(args.repo).resolve()
|
||||
content = repo / "content"
|
||||
md_files = sorted(str(p) for p in content.rglob("*.md"))
|
||||
if not md_files:
|
||||
print(f"No markdown files under {content}/")
|
||||
return
|
||||
|
||||
lychee = find_lychee()
|
||||
cmd = [
|
||||
lychee, "--format", "json",
|
||||
"--scheme", "http", "--scheme", "https", # web links only
|
||||
"--exclude", "mailto:", "--exclude", "tel:",
|
||||
"--timeout", str(args.timeout), "--max-retries", str(args.retries),
|
||||
"--threads", str(args.threads),
|
||||
] + md_files
|
||||
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True)
|
||||
# lychee exits non-zero when it finds dead links; we still parse the JSON.
|
||||
try:
|
||||
data = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
print("Could not parse lychee output. stderr:\n", (proc.stderr or "")[-2000:])
|
||||
return
|
||||
|
||||
dead = []
|
||||
for src, items in data.get("error_map", {}).items():
|
||||
for it in items:
|
||||
url = it.get("url", "")
|
||||
if url.startswith(("http://", "https://")):
|
||||
status = it.get("status", {}).get("text", "")
|
||||
span = it.get("span", {})
|
||||
# report the source path relative to the repo for readability
|
||||
try:
|
||||
rel = Path(src).resolve().relative_to(repo)
|
||||
except ValueError:
|
||||
rel = src
|
||||
dead.append((str(rel), span.get("line"), url, status))
|
||||
|
||||
total_checked = data.get("total", 0)
|
||||
ok = data.get("successful", 0)
|
||||
|
||||
outdir = repo / "linkcheck"
|
||||
outdir.mkdir(exist_ok=True)
|
||||
with (outdir / "external_dead.txt").open("w") as f:
|
||||
for src, line, url, status in sorted(dead):
|
||||
f.write(f"{src}:{line}\t{url}\t[{status}]\n")
|
||||
|
||||
print(f"External link check: {total_checked} links checked, {ok} ok, {len(dead)} dead.")
|
||||
if dead:
|
||||
print("\nDead external links:")
|
||||
for src, line, url, status in sorted(dead):
|
||||
print(f" {src}:{line} {url} [{status}]")
|
||||
print(f"\nFull list written to {outdir/'external_dead.txt'}")
|
||||
else:
|
||||
print("No dead external links found.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Comment out confirmed dead INTERNAL page links, report external/bare-domain failures.
|
||||
|
||||
Reads lychee's JSON report (linkcheck/internal.json). A link is auto-commented ONLY
|
||||
if lychee flagged its file:// path as a missing file under the built site (public/).
|
||||
External https:// failures and bare-domain failures are written to a manual-review
|
||||
report and left untouched. Idempotent: already-commented links are not detected by
|
||||
lychee, so re-runs are safe.
|
||||
|
||||
IMPORTANT: run lychee with the SOURCE markdown as input (e.g. `lychee --root-dir public
|
||||
--default-extension html content/*.md`) so the error_map keys are the markdown files
|
||||
we actually edit. They are relative to the repo root.
|
||||
|
||||
Usage:
|
||||
python3 scripts/fix_dead_links.py --report linkcheck/internal.json # dry-run
|
||||
python3 scripts/fix_dead_links.py --report linkcheck/internal.json --apply # edit
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--report", required=True)
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--public-dir", default="public",
|
||||
help="built site dir that lychee's --root-dir pointed at")
|
||||
args = ap.parse_args()
|
||||
|
||||
data = json.loads(Path(args.report).read_text())
|
||||
err_map = data.get("error_map", {})
|
||||
|
||||
to_comment = [] # (md_file_relative, line, dest)
|
||||
manual = [] # (md_file_relative, line, url, status_text)
|
||||
for md_file, items in err_map.items():
|
||||
for it in items:
|
||||
url = it.get("url", "")
|
||||
status = it.get("status", {}).get("text", "")
|
||||
span = it.get("span", {})
|
||||
if url.startswith("file://"):
|
||||
after = url[len("file://"):]
|
||||
m = re.search(r"/" + re.escape(args.public_dir) + r"/(.*)$", after)
|
||||
if m:
|
||||
to_comment.append((md_file, span.get("line"), "/" + m.group(1)))
|
||||
continue
|
||||
manual.append((md_file, span.get("line"), url, status))
|
||||
|
||||
# Group by source file (error_map keys are relative to repo root).
|
||||
by_file = {}
|
||||
for md_file, line, dest in to_comment:
|
||||
by_file.setdefault(md_file, []).append((line, dest))
|
||||
for md_file in by_file:
|
||||
by_file[md_file].sort(reverse=True) # edit bottom-up; keep line count stable
|
||||
|
||||
plan = 0
|
||||
for md_file, edits in sorted(by_file.items()):
|
||||
p = Path(md_file)
|
||||
if not p.exists():
|
||||
print(f" ! missing source, skipping: {p}")
|
||||
continue
|
||||
lines = p.read_text().splitlines(keepends=True)
|
||||
changed = False
|
||||
for line_no, dest in edits:
|
||||
if not line_no:
|
||||
continue
|
||||
idx = line_no - 1
|
||||
if idx < 0 or idx >= len(lines):
|
||||
continue
|
||||
cur = lines[idx]
|
||||
# lychee reports the resolved file path (trailing slash stripped); the
|
||||
# markdown link may keep a trailing slash, so allow an optional "/".
|
||||
m = re.search(r"\[([^\]]*)\]\(" + re.escape(dest) + r"/?\s*\)", cur)
|
||||
if not m:
|
||||
print(f" ? could not locate link on {md_file}:{line_no} for {dest}")
|
||||
continue
|
||||
link_txt = m.group(0)
|
||||
if link_txt.count("<!--") or (cur.count("<!--") > cur.count("-->")):
|
||||
continue # already commented
|
||||
lines[idx] = cur[:m.start()] + "<!-- " + link_txt + " -->" + cur[m.end():]
|
||||
changed = True
|
||||
plan += 1
|
||||
print(f" {'APPLY ' if args.apply else 'DRY '} {md_file}:{line_no} {link_txt}")
|
||||
if changed and args.apply:
|
||||
p.write_text("".join(lines))
|
||||
|
||||
outdir = Path("linkcheck")
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
with (outdir / "manual_review.txt").open("w") as f:
|
||||
f.write("Manual review (NOT auto-commented). External/bare-domain failures.\n\n")
|
||||
for md_file, line, url, status in sorted(manual):
|
||||
f.write(f"{md_file}:{line}\t{url}\t[{status}]\n")
|
||||
|
||||
print(f"\nAuto-comment planned: {plan} (apply={args.apply})")
|
||||
print(f"Manual review entries: {len(manual)} -> {outdir/'manual_review.txt'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Serve the built site (public/) on all interfaces so other machines on the
|
||||
same LAN can reach it. Usage: python3 serve_lan.py [port] (default 8090)
|
||||
|
||||
The site must be built first: make build index
|
||||
Then start this: python3 serve_lan.py
|
||||
Address to open from ANOTHER machine on the same network:
|
||||
http://<THIS-MACHINE-LAN-IP>:8090/
|
||||
where the LAN IP is printed at startup (and is also hostname -I | head -1).
|
||||
"""
|
||||
import functools
|
||||
import http.server
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8090
|
||||
|
||||
# Serve the built site (public/), NOT the repo root.
|
||||
handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory="public")
|
||||
server = http.server.ThreadingHTTPServer(("0.0.0.0", PORT), handler)
|
||||
|
||||
|
||||
def lan_ip() -> str:
|
||||
try:
|
||||
return subprocess.check_output(["hostname", "-I"], text=True).split()[0]
|
||||
except Exception:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
try:
|
||||
s.connect(("8.8.8.8", 80))
|
||||
return s.getsockname()[0]
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
|
||||
ip = lan_ip()
|
||||
print(f"Serving public/ -> http://0.0.0.0:{PORT}/", flush=True)
|
||||
print(f"LAN address -> http://{ip}:{PORT}/ (open this from another machine)", flush=True)
|
||||
print("Press Ctrl+C to stop.", flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
Reference in New Issue
Block a user