From 66181b84891d6e177ae62ff74fd8ee8d3290fd89 Mon Sep 17 00:00:00 2001 From: kathleen Date: Thu, 19 Nov 2026 02:49:25 +0000 Subject: [PATCH] Send a changed route PDF to the bot by pull request The daily check keeps the community route PDF only in its private raw store (never published here). When that copy differs from "Threshing Options.pdf" in the bot's private repo, `python -m collection send-pdf` opens (or updates) a pull request there with the new PDF and the lines its text gained and lost, so Ken can update the hand-written routes in src/pdfRoutes.js (the bot does not read the PDF itself). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ --- .gitea/workflows/update-sources.yml | 6 ++ collection/etl/send_pdf.py | 116 ++++++++++++++++++++++++++++ collection/run.py | 4 + 3 files changed, 126 insertions(+) create mode 100644 collection/etl/send_pdf.py diff --git a/.gitea/workflows/update-sources.yml b/.gitea/workflows/update-sources.yml index 676ab63..8f0000b 100644 --- a/.gitea/workflows/update-sources.yml +++ b/.gitea/workflows/update-sources.yml @@ -86,6 +86,12 @@ jobs: with: path: ${{ runner.temp }}/raw-archive key: raw-archive-${{ hashFiles('data/manifest.json') }} + - name: Send a changed route PDF to the bot (its private repo; the PDF is never published here) + if: always() && steps.check.outcome == 'success' + continue-on-error: true + env: + ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} + run: python -m collection send-pdf - name: Open a pull request with the new data (it merges itself once CI passes) if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build') env: diff --git a/collection/etl/send_pdf.py b/collection/etl/send_pdf.py new file mode 100644 index 0000000..cd6f901 --- /dev/null +++ b/collection/etl/send_pdf.py @@ -0,0 +1,116 @@ +"""Send the community route PDF to the bot's private repo when it changed (python -m collection send-pdf). + +The PDF is never published by the Atlas: the daily check keeps it in the private raw store only +(data/raw/reddit_pdf/current.pdf). The bot's repo (private, same host) keeps its own copy, "Threshing Options.pdf", +which its hand-written routes (src/pdfRoutes.js) were transcribed from. When the two differ, this opens (or updates) +one pull request there, on branch atlas/route-pdf, with the new file and the lines its text gained and lost. + +The token is ATLAS_BOT_TOKEN (write access to the bot's repo); git gets it through GIT_CONFIG_* variables, never argv. +""" +import hashlib +import io +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +from . import forge +from .common import path + +BOT_REPO = "DragonKind/DragonKind" +BOT_PDF = "Threshing Options.pdf" +BRANCH = "atlas/route-pdf" +RAW_PDF = "data/raw/reddit_pdf/current.pdf" +MAX_LINES = 60 + + +def pdf_lines(data): + """The PDF's text, one entry per non-empty line, whitespace collapsed.""" + from pypdf import PdfReader + out = [] + for page in PdfReader(io.BytesIO(data)).pages: + for line in (page.extract_text() or "").splitlines(): + line = " ".join(line.split()) + if line: + out.append(line) + return out + + +def text_changes(old, new): + """(lines only in the new PDF, lines only in the old one), each in the order they appear.""" + o, n = pdf_lines(old), pdf_lines(new) + so, sn = set(o), set(n) + gained = list(dict.fromkeys(l for l in n if l not in so)) + lost = list(dict.fromkeys(l for l in o if l not in sn)) + return gained, lost + + +def _bullets(lines): + if not lines: + return "- (none)" + shown = [f"- {l}" for l in lines[:MAX_LINES]] + if len(lines) > MAX_LINES: + shown.append(f"- ... and {len(lines) - MAX_LINES} more") + return "\n".join(shown) + + +def body(new, old, gained, lost): + h = lambda b: hashlib.sha256(b).hexdigest()[:12] + return ( + "The community route PDF (u/ClumsyBadger's Threshing Options, the Google Drive file linked from the " + "megathreads) changed. This replaces `Threshing Options.pdf` with the current version.\n\n" + f"| | bytes | sha256 |\n|---|---|---|\n| this repo's copy | {len(old):,} | `{h(old)}` |\n" + f"| current PDF | {len(new):,} | `{h(new)}` |\n\n" + "**The bot does not read the PDF itself**: its routes are written out by hand in `src/pdfRoutes.js`. " + "Check them against the changes below.\n\n" + f"### Text that is new in this version\n{_bullets(gained)}\n\n" + f"### Text no longer in it\n{_bullets(lost)}\n\n" + "(Text extracted line by line from a flowchart, so some lines are fragments of boxes.) " + "Opened automatically by the Threshing Atlas's daily source check; it is updated in place when the PDF " + "changes again.\n" + ) + + +def _git(args, cwd=None, env=None): + subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True) + + +def send(env=None): + env = dict(os.environ if env is None else env) + src = path(RAW_PDF) + if not src.exists(): + print("no route PDF in the raw store: nothing to send") + return 0 + tok = forge.token(env) + if not tok: + print("ATLAS_BOT_TOKEN is not set: the PDF is not sent") + return 0 + f = forge.config() + host = f["url"].rstrip("/") + genv = {**env, "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": f"http.{host}/.extraheader", "GIT_CONFIG_VALUE_0": f"AUTHORIZATION: token {tok}"} + new = src.read_bytes() + tmp = Path(tempfile.mkdtemp(prefix="dragonkind-")) + try: + repo = tmp / "repo" + _git(["clone", "--quiet", "--depth", "1", f"{host}/{BOT_REPO}.git", str(repo)], env=genv) + dst = repo / BOT_PDF + old = dst.read_bytes() if dst.exists() else b"" + if old == new: + print("the bot already has the current route PDF") + return 0 + gained, lost = text_changes(old, new) if old else (pdf_lines(new), []) + dst.write_bytes(new) + _git(["config", "user.name", "Atlas data bot"], cwd=repo) + _git(["config", "user.email", "atlas-bot@noreply.localhost"], cwd=repo) + _git(["checkout", "-q", "-b", BRANCH], cwd=repo) + _git(["add", BOT_PDF], cwd=repo) + _git(["commit", "-q", "-m", f"Community route PDF: current version ({len(new):,} bytes)"], cwd=repo) + _git(["push", "-q", "--force", "origin", BRANCH], cwd=repo, env=genv) + n = forge.open_pr(BRANCH, "Community route PDF changed", body(new, old, gained, lost), env=env, + repo=BOT_REPO, base="main") + print(f"route PDF sent: pull request #{n} on {BOT_REPO} ({len(gained)} lines new, {len(lost)} gone)") + return 0 + finally: + shutil.rmtree(tmp, ignore_errors=True) diff --git a/collection/run.py b/collection/run.py index e766b0e..b092bfb 100644 --- a/collection/run.py +++ b/collection/run.py @@ -131,6 +131,7 @@ def main(argv=None): k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot") k.add_argument("--out", default="exports") sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build") + sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)") r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)") r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)") c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)") @@ -154,6 +155,9 @@ def main(argv=None): if a.cmd == "rebuild": from .etl.handoff import rebuild return rebuild() + if a.cmd == "send-pdf": + from .etl.send_pdf import send + return send() if a.cmd == "refresh": from .etl.handoff import refresh return refresh(main, a.bot) -- 2.54.0