#!/usr/bin/env python3 """Run every example, and hold its output to a recorded answer key. This is the spine of the library. A lesson page never hand-types what a program prints; it marks the spot and this tool fills it from a real run: Inside the markers is generated, outside is yours. There is a second kind, `source:`, which pastes the program itself — for the pages where the code *is* the lesson and a hand-copied fence could quietly drift from the file CI runs. Three kinds of example, told apart by extension ----------------------------------------------- examples/.py stdlib-only Python, run as `python3 -I .py` examples/.rs compiled with `rustc --edition 2024`, no cargo, no crates examples/.sh run as `bash .sh` — xxd, od, hexdump, iconv, printf examples/.c compiled with `cc -std=c11 -Wall -Wextra` (clang or gcc), no libraries Stems are unique repo-wide *across* extensions, because a Markdown block names a bare stem with no path and no extension. The convention is a language suffix: `hex_is_a_shorthand_py`, `hex_is_a_shorthand_rs`, `hex_is_a_shorthand_sh`. Every example runs under one fixed environment — `LC_ALL=C`, `LANG=C`, `PYTHONUTF8=1` — so the answer key does not depend on whoever ran it. A lesson whose subject is the locale sets its own inside the script, on purpose and in view. Output is captured as bytes and decoded as UTF-8; a byte that is not valid UTF-8 is written into the key as `\\xNN`, which is deterministic, and is usually the moment a shell example should have piped through `xxd` instead. Four modes ---------- python3 tools/run_examples.py verify + refill the .md blocks python3 tools/run_examples.py --update accept current output as the key python3 tools/run_examples.py --check write nothing; fail on any drift (CI) python3 tools/run_examples.py --only X touch example X and nothing else ``--only`` narrows both the running and the refilling to the stems you name. A full ``--update`` re-records *every* answer key and refills *every* page, which in a checkout open in two sessions means adopting whatever a colleague's half-finished example happens to print. It is never right in CI: a partial run cannot see repo-wide drift, which is the whole job there. """ from __future__ import annotations import argparse import difflib import os import re import subprocess import sys import tempfile from pathlib import Path REPO = Path(__file__).resolve().parent.parent EDITION = "2024" # extension -> (how the page labels the source fence, how the tool runs it) LANGS = {".py": "python", ".rs": "rust", ".sh": "bash", ".c": "c"} # ...generated... # ...generated... BLOCK = re.compile( r"(?P)" r"(?P.*?)" r"(?P)", re.DOTALL, ) SKIP_DIRS = {".git", "site", ".venv", "__pycache__", ".github"} # A fenced code block, opened or closed. The pages that DOCUMENT this mechanism # (README.md, CONTRIBUTING.md) show the markers inside a fence — those are # documentation, not blocks to fill. FENCE = re.compile(r"^[ \t]*(?P`{3,}|~{3,})", re.MULTILINE) def fenced_spans(text: str) -> list[tuple[int, int]]: """Character ranges covered by fenced code blocks.""" spans: list[tuple[int, int]] = [] open_at: int | None = None open_fence = "" for m in FENCE.finditer(text): fence = m.group("f") if open_at is None: open_at, open_fence = m.start(), fence elif fence[0] == open_fence[0] and len(fence) >= len(open_fence): spans.append((open_at, m.end())) open_at = None if open_at is not None: spans.append((open_at, len(text))) return spans def walk(root: Path): for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] for name in filenames: yield Path(dirpath) / name def find_examples() -> dict[str, Path]: """Map stem -> path for every example under an examples/ folder. Stems are unique.""" found: dict[str, Path] = {} for path in sorted(walk(REPO)): if path.suffix not in LANGS or path.parent.name != "examples": continue if path.stem in found: sys.exit( f"ERROR: duplicate example stem {path.stem!r}\n" f" {found[path.stem].relative_to(REPO)}\n {path.relative_to(REPO)}\n" "Stems are named bare in Markdown blocks, so they must be unique " "across languages too — add a _py / _rs / _sh / _c suffix." ) found[path.stem] = path return found def fixed_env() -> dict[str, str]: """One environment for every run, so the key is the program's and not the machine's.""" env = dict(os.environ) for k in list(env): if k.startswith("LC_") or k in {"LANG", "LANGUAGE"}: del env[k] env.update({"LC_ALL": "C", "LANG": "C", "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8"}) return env def _decode(raw: bytes) -> str: return raw.decode("utf-8", errors="backslashreplace") def run_example(src: Path, workdir: Path) -> str: """Run (or compile and run) one example; return its stdout. Exits on failure. Run from the example's own folder so a page can tell the reader to `cd` there and type the same command. Python gets -I so a stray module beside it or on PYTHONPATH cannot change what the page claims. """ env = fixed_env() if src.suffix in (".rs", ".c"): binary = workdir / src.stem if src.suffix == ".rs": build_cmd = ["rustc", "--edition", EDITION, str(src), "-o", str(binary)] else: build_cmd = ["cc", "-std=c11", "-Wall", "-Wextra", str(src), "-o", str(binary)] build = subprocess.run(build_cmd, capture_output=True, text=True) if build.returncode != 0: sys.exit(f"ERROR: {src.relative_to(REPO)} failed to compile\n{build.stderr}") if build.stderr.strip(): print(f" note: {src.relative_to(REPO)} compiled with warnings:\n{build.stderr}") cmd = [str(binary)] elif src.suffix == ".py": cmd = [sys.executable, "-I", src.name] else: cmd = ["bash", src.name] proc = subprocess.run(cmd, cwd=src.parent, capture_output=True, env=env, timeout=60) if proc.returncode != 0: sys.exit( f"ERROR: {src.relative_to(REPO)} exited {proc.returncode}\n{_decode(proc.stderr)}" ) if proc.stderr.strip(): print(f" note: {src.relative_to(REPO)} wrote to stderr:\n{_decode(proc.stderr)}") return _decode(proc.stdout) def rendered_block(kind: str, src: Path, output: str, page: Path) -> str: """The generated body that goes between the markers on `page`.""" href = os.path.relpath(src, page.parent) if kind == "source": body = src.read_text(encoding="utf-8").strip("\n") return ( f"\n*[`{src.name}`]({href}) in full — pasted here by " f"`tools/run_examples.py` from the file CI runs.*\n\n" f"```{LANGS[src.suffix]}\n{body}\n```\n" ) return ( f"\n*Verified output of [`{src.name}`]({href}) — regenerated by " f"`tools/run_examples.py`, never hand-typed.*\n\n" f"```text\n{output.strip(chr(10))}\n```\n" ) def fill_pages( outputs: dict[str, str], sources: dict[str, Path], write: bool, problems: list[str], only: set[str] | None = None, ) -> list[str]: """Refill every generated block on every Markdown page. Returns drift. A block naming a stem that no longer exists is recorded in `problems` and left untouched rather than exiting on the spot — dying on the first one would leave every other page unfilled. With `only` set, a block naming any other stem is left exactly as it is. The selection is checked before the kind is, so a run scoped to one stem cannot rewrite either half of somebody else's page. """ drift: list[str] = [] for page in sorted(walk(REPO)): if page.suffix != ".md": continue text = page.read_text(encoding="utf-8") if "