Send a changed route PDF to the bot by pull request #16

Merged
kathleen merged 1 commits from send-pdf-to-bot into main 2026-11-19 02:50:31 +00:00
3 changed files with 126 additions and 0 deletions
+6
View File
@@ -86,6 +86,12 @@ jobs:
with: with:
path: ${{ runner.temp }}/raw-archive path: ${{ runner.temp }}/raw-archive
key: raw-archive-${{ hashFiles('data/manifest.json') }} key: raw-archive-${{ hashFiles('data/manifest.json') }}
- name: Send a changed route PDF to the bot (its private repo; the PDF is never published here)
if: always() && steps.check.outcome == 'success'
continue-on-error: true
env:
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
run: python -m collection send-pdf
- name: Open a pull request with the new data (it merges itself once CI passes) - name: Open a pull request with the new data (it merges itself once CI passes)
if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build') if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build')
env: env:
+116
View File
@@ -0,0 +1,116 @@
"""Send the community route PDF to the bot's private repo when it changed (python -m collection send-pdf).
The PDF is never published by the Atlas: the daily check keeps it in the private raw store only
(data/raw/reddit_pdf/current.pdf). The bot's repo (private, same host) keeps its own copy, "Threshing Options.pdf",
which its hand-written routes (src/pdfRoutes.js) were transcribed from. When the two differ, this opens (or updates)
one pull request there, on branch atlas/route-pdf, with the new file and the lines its text gained and lost.
The token is ATLAS_BOT_TOKEN (write access to the bot's repo); git gets it through GIT_CONFIG_* variables, never argv.
"""
import hashlib
import io
import os
import shutil
import subprocess
import tempfile
from pathlib import Path
from . import forge
from .common import path
BOT_REPO = "DragonKind/DragonKind"
BOT_PDF = "Threshing Options.pdf"
BRANCH = "atlas/route-pdf"
RAW_PDF = "data/raw/reddit_pdf/current.pdf"
MAX_LINES = 60
def pdf_lines(data):
"""The PDF's text, one entry per non-empty line, whitespace collapsed."""
from pypdf import PdfReader
out = []
for page in PdfReader(io.BytesIO(data)).pages:
for line in (page.extract_text() or "").splitlines():
line = " ".join(line.split())
if line:
out.append(line)
return out
def text_changes(old, new):
"""(lines only in the new PDF, lines only in the old one), each in the order they appear."""
o, n = pdf_lines(old), pdf_lines(new)
so, sn = set(o), set(n)
gained = list(dict.fromkeys(l for l in n if l not in so))
lost = list(dict.fromkeys(l for l in o if l not in sn))
return gained, lost
def _bullets(lines):
if not lines:
return "- (none)"
shown = [f"- {l}" for l in lines[:MAX_LINES]]
if len(lines) > MAX_LINES:
shown.append(f"- ... and {len(lines) - MAX_LINES} more")
return "\n".join(shown)
def body(new, old, gained, lost):
h = lambda b: hashlib.sha256(b).hexdigest()[:12]
return (
"The community route PDF (u/ClumsyBadger's Threshing Options, the Google Drive file linked from the "
"megathreads) changed. This replaces `Threshing Options.pdf` with the current version.\n\n"
f"| | bytes | sha256 |\n|---|---|---|\n| this repo's copy | {len(old):,} | `{h(old)}` |\n"
f"| current PDF | {len(new):,} | `{h(new)}` |\n\n"
"**The bot does not read the PDF itself**: its routes are written out by hand in `src/pdfRoutes.js`. "
"Check them against the changes below.\n\n"
f"### Text that is new in this version\n{_bullets(gained)}\n\n"
f"### Text no longer in it\n{_bullets(lost)}\n\n"
"(Text extracted line by line from a flowchart, so some lines are fragments of boxes.) "
"Opened automatically by the Threshing Atlas's daily source check; it is updated in place when the PDF "
"changes again.\n"
)
def _git(args, cwd=None, env=None):
subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True)
def send(env=None):
env = dict(os.environ if env is None else env)
src = path(RAW_PDF)
if not src.exists():
print("no route PDF in the raw store: nothing to send")
return 0
tok = forge.token(env)
if not tok:
print("ATLAS_BOT_TOKEN is not set: the PDF is not sent")
return 0
f = forge.config()
host = f["url"].rstrip("/")
genv = {**env, "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": f"http.{host}/.extraheader", "GIT_CONFIG_VALUE_0": f"AUTHORIZATION: token {tok}"}
new = src.read_bytes()
tmp = Path(tempfile.mkdtemp(prefix="dragonkind-"))
try:
repo = tmp / "repo"
_git(["clone", "--quiet", "--depth", "1", f"{host}/{BOT_REPO}.git", str(repo)], env=genv)
dst = repo / BOT_PDF
old = dst.read_bytes() if dst.exists() else b""
if old == new:
print("the bot already has the current route PDF")
return 0
gained, lost = text_changes(old, new) if old else (pdf_lines(new), [])
dst.write_bytes(new)
_git(["config", "user.name", "Atlas data bot"], cwd=repo)
_git(["config", "user.email", "atlas-bot@noreply.localhost"], cwd=repo)
_git(["checkout", "-q", "-b", BRANCH], cwd=repo)
_git(["add", BOT_PDF], cwd=repo)
_git(["commit", "-q", "-m", f"Community route PDF: current version ({len(new):,} bytes)"], cwd=repo)
_git(["push", "-q", "--force", "origin", BRANCH], cwd=repo, env=genv)
n = forge.open_pr(BRANCH, "Community route PDF changed", body(new, old, gained, lost), env=env,
repo=BOT_REPO, base="main")
print(f"route PDF sent: pull request #{n} on {BOT_REPO} ({len(gained)} lines new, {len(lost)} gone)")
return 0
finally:
shutil.rmtree(tmp, ignore_errors=True)
+4
View File
@@ -131,6 +131,7 @@ def main(argv=None):
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot") k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
k.add_argument("--out", default="exports") k.add_argument("--out", default="exports")
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build") sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)") r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)") r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)") c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
@@ -154,6 +155,9 @@ def main(argv=None):
if a.cmd == "rebuild": if a.cmd == "rebuild":
from .etl.handoff import rebuild from .etl.handoff import rebuild
return rebuild() return rebuild()
if a.cmd == "send-pdf":
from .etl.send_pdf import send
return send()
if a.cmd == "refresh": if a.cmd == "refresh":
from .etl.handoff import refresh from .etl.handoff import refresh
return refresh(main, a.bot) return refresh(main, a.bot)