diff --git a/.gitea/workflows/update-sources.yml b/.gitea/workflows/update-sources.yml index 676ab63..8f0000b 100644 --- a/.gitea/workflows/update-sources.yml +++ b/.gitea/workflows/update-sources.yml @@ -86,6 +86,12 @@ jobs: with: path: ${{ runner.temp }}/raw-archive key: raw-archive-${{ hashFiles('data/manifest.json') }} + - name: Send a changed route PDF to the bot (its private repo; the PDF is never published here) + if: always() && steps.check.outcome == 'success' + continue-on-error: true + env: + ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} + run: python -m collection send-pdf - name: Open a pull request with the new data (it merges itself once CI passes) if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build') env: diff --git a/collection/etl/send_pdf.py b/collection/etl/send_pdf.py new file mode 100644 index 0000000..cd6f901 --- /dev/null +++ b/collection/etl/send_pdf.py @@ -0,0 +1,116 @@ +"""Send the community route PDF to the bot's private repo when it changed (python -m collection send-pdf). + +The PDF is never published by the Atlas: the daily check keeps it in the private raw store only +(data/raw/reddit_pdf/current.pdf). The bot's repo (private, same host) keeps its own copy, "Threshing Options.pdf", +which its hand-written routes (src/pdfRoutes.js) were transcribed from. When the two differ, this opens (or updates) +one pull request there, on branch atlas/route-pdf, with the new file and the lines its text gained and lost. + +The token is ATLAS_BOT_TOKEN (write access to the bot's repo); git gets it through GIT_CONFIG_* variables, never argv. +""" +import hashlib +import io +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +from . import forge +from .common import path + +BOT_REPO = "DragonKind/DragonKind" +BOT_PDF = "Threshing Options.pdf" +BRANCH = "atlas/route-pdf" +RAW_PDF = "data/raw/reddit_pdf/current.pdf" +MAX_LINES = 60 + + +def pdf_lines(data): + """The PDF's text, one entry per non-empty line, whitespace collapsed.""" + from pypdf import PdfReader + out = [] + for page in PdfReader(io.BytesIO(data)).pages: + for line in (page.extract_text() or "").splitlines(): + line = " ".join(line.split()) + if line: + out.append(line) + return out + + +def text_changes(old, new): + """(lines only in the new PDF, lines only in the old one), each in the order they appear.""" + o, n = pdf_lines(old), pdf_lines(new) + so, sn = set(o), set(n) + gained = list(dict.fromkeys(l for l in n if l not in so)) + lost = list(dict.fromkeys(l for l in o if l not in sn)) + return gained, lost + + +def _bullets(lines): + if not lines: + return "- (none)" + shown = [f"- {l}" for l in lines[:MAX_LINES]] + if len(lines) > MAX_LINES: + shown.append(f"- ... and {len(lines) - MAX_LINES} more") + return "\n".join(shown) + + +def body(new, old, gained, lost): + h = lambda b: hashlib.sha256(b).hexdigest()[:12] + return ( + "The community route PDF (u/ClumsyBadger's Threshing Options, the Google Drive file linked from the " + "megathreads) changed. This replaces `Threshing Options.pdf` with the current version.\n\n" + f"| | bytes | sha256 |\n|---|---|---|\n| this repo's copy | {len(old):,} | `{h(old)}` |\n" + f"| current PDF | {len(new):,} | `{h(new)}` |\n\n" + "**The bot does not read the PDF itself**: its routes are written out by hand in `src/pdfRoutes.js`. " + "Check them against the changes below.\n\n" + f"### Text that is new in this version\n{_bullets(gained)}\n\n" + f"### Text no longer in it\n{_bullets(lost)}\n\n" + "(Text extracted line by line from a flowchart, so some lines are fragments of boxes.) " + "Opened automatically by the Threshing Atlas's daily source check; it is updated in place when the PDF " + "changes again.\n" + ) + + +def _git(args, cwd=None, env=None): + subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True) + + +def send(env=None): + env = dict(os.environ if env is None else env) + src = path(RAW_PDF) + if not src.exists(): + print("no route PDF in the raw store: nothing to send") + return 0 + tok = forge.token(env) + if not tok: + print("ATLAS_BOT_TOKEN is not set: the PDF is not sent") + return 0 + f = forge.config() + host = f["url"].rstrip("/") + genv = {**env, "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": f"http.{host}/.extraheader", "GIT_CONFIG_VALUE_0": f"AUTHORIZATION: token {tok}"} + new = src.read_bytes() + tmp = Path(tempfile.mkdtemp(prefix="dragonkind-")) + try: + repo = tmp / "repo" + _git(["clone", "--quiet", "--depth", "1", f"{host}/{BOT_REPO}.git", str(repo)], env=genv) + dst = repo / BOT_PDF + old = dst.read_bytes() if dst.exists() else b"" + if old == new: + print("the bot already has the current route PDF") + return 0 + gained, lost = text_changes(old, new) if old else (pdf_lines(new), []) + dst.write_bytes(new) + _git(["config", "user.name", "Atlas data bot"], cwd=repo) + _git(["config", "user.email", "atlas-bot@noreply.localhost"], cwd=repo) + _git(["checkout", "-q", "-b", BRANCH], cwd=repo) + _git(["add", BOT_PDF], cwd=repo) + _git(["commit", "-q", "-m", f"Community route PDF: current version ({len(new):,} bytes)"], cwd=repo) + _git(["push", "-q", "--force", "origin", BRANCH], cwd=repo, env=genv) + n = forge.open_pr(BRANCH, "Community route PDF changed", body(new, old, gained, lost), env=env, + repo=BOT_REPO, base="main") + print(f"route PDF sent: pull request #{n} on {BOT_REPO} ({len(gained)} lines new, {len(lost)} gone)") + return 0 + finally: + shutil.rmtree(tmp, ignore_errors=True) diff --git a/collection/run.py b/collection/run.py index e766b0e..b092bfb 100644 --- a/collection/run.py +++ b/collection/run.py @@ -131,6 +131,7 @@ def main(argv=None): k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot") k.add_argument("--out", default="exports") sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build") + sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)") r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)") r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)") c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)") @@ -154,6 +155,9 @@ def main(argv=None): if a.cmd == "rebuild": from .etl.handoff import rebuild return rebuild() + if a.cmd == "send-pdf": + from .etl.send_pdf import send + return send() if a.cmd == "refresh": from .etl.handoff import refresh return refresh(main, a.bot)