Merge pull request 'Send a changed route PDF to the bot by pull request' (#16) from send-pdf-to-bot into main
CI / test (push) Successful in 40s
CI / test (push) Successful in 40s
This commit was merged in pull request #16.
This commit is contained in:
@@ -86,6 +86,12 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
path: ${{ runner.temp }}/raw-archive
|
path: ${{ runner.temp }}/raw-archive
|
||||||
key: raw-archive-${{ hashFiles('data/manifest.json') }}
|
key: raw-archive-${{ hashFiles('data/manifest.json') }}
|
||||||
|
- name: Send a changed route PDF to the bot (its private repo; the PDF is never published here)
|
||||||
|
if: always() && steps.check.outcome == 'success'
|
||||||
|
continue-on-error: true
|
||||||
|
env:
|
||||||
|
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
run: python -m collection send-pdf
|
||||||
- name: Open a pull request with the new data (it merges itself once CI passes)
|
- name: Open a pull request with the new data (it merges itself once CI passes)
|
||||||
if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build')
|
if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build')
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""Send the community route PDF to the bot's private repo when it changed (python -m collection send-pdf).
|
||||||
|
|
||||||
|
The PDF is never published by the Atlas: the daily check keeps it in the private raw store only
|
||||||
|
(data/raw/reddit_pdf/current.pdf). The bot's repo (private, same host) keeps its own copy, "Threshing Options.pdf",
|
||||||
|
which its hand-written routes (src/pdfRoutes.js) were transcribed from. When the two differ, this opens (or updates)
|
||||||
|
one pull request there, on branch atlas/route-pdf, with the new file and the lines its text gained and lost.
|
||||||
|
|
||||||
|
The token is ATLAS_BOT_TOKEN (write access to the bot's repo); git gets it through GIT_CONFIG_* variables, never argv.
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from . import forge
|
||||||
|
from .common import path
|
||||||
|
|
||||||
|
BOT_REPO = "DragonKind/DragonKind"
|
||||||
|
BOT_PDF = "Threshing Options.pdf"
|
||||||
|
BRANCH = "atlas/route-pdf"
|
||||||
|
RAW_PDF = "data/raw/reddit_pdf/current.pdf"
|
||||||
|
MAX_LINES = 60
|
||||||
|
|
||||||
|
|
||||||
|
def pdf_lines(data):
|
||||||
|
"""The PDF's text, one entry per non-empty line, whitespace collapsed."""
|
||||||
|
from pypdf import PdfReader
|
||||||
|
out = []
|
||||||
|
for page in PdfReader(io.BytesIO(data)).pages:
|
||||||
|
for line in (page.extract_text() or "").splitlines():
|
||||||
|
line = " ".join(line.split())
|
||||||
|
if line:
|
||||||
|
out.append(line)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def text_changes(old, new):
|
||||||
|
"""(lines only in the new PDF, lines only in the old one), each in the order they appear."""
|
||||||
|
o, n = pdf_lines(old), pdf_lines(new)
|
||||||
|
so, sn = set(o), set(n)
|
||||||
|
gained = list(dict.fromkeys(l for l in n if l not in so))
|
||||||
|
lost = list(dict.fromkeys(l for l in o if l not in sn))
|
||||||
|
return gained, lost
|
||||||
|
|
||||||
|
|
||||||
|
def _bullets(lines):
|
||||||
|
if not lines:
|
||||||
|
return "- (none)"
|
||||||
|
shown = [f"- {l}" for l in lines[:MAX_LINES]]
|
||||||
|
if len(lines) > MAX_LINES:
|
||||||
|
shown.append(f"- ... and {len(lines) - MAX_LINES} more")
|
||||||
|
return "\n".join(shown)
|
||||||
|
|
||||||
|
|
||||||
|
def body(new, old, gained, lost):
|
||||||
|
h = lambda b: hashlib.sha256(b).hexdigest()[:12]
|
||||||
|
return (
|
||||||
|
"The community route PDF (u/ClumsyBadger's Threshing Options, the Google Drive file linked from the "
|
||||||
|
"megathreads) changed. This replaces `Threshing Options.pdf` with the current version.\n\n"
|
||||||
|
f"| | bytes | sha256 |\n|---|---|---|\n| this repo's copy | {len(old):,} | `{h(old)}` |\n"
|
||||||
|
f"| current PDF | {len(new):,} | `{h(new)}` |\n\n"
|
||||||
|
"**The bot does not read the PDF itself**: its routes are written out by hand in `src/pdfRoutes.js`. "
|
||||||
|
"Check them against the changes below.\n\n"
|
||||||
|
f"### Text that is new in this version\n{_bullets(gained)}\n\n"
|
||||||
|
f"### Text no longer in it\n{_bullets(lost)}\n\n"
|
||||||
|
"(Text extracted line by line from a flowchart, so some lines are fragments of boxes.) "
|
||||||
|
"Opened automatically by the Threshing Atlas's daily source check; it is updated in place when the PDF "
|
||||||
|
"changes again.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _git(args, cwd=None, env=None):
|
||||||
|
subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True)
|
||||||
|
|
||||||
|
|
||||||
|
def send(env=None):
|
||||||
|
env = dict(os.environ if env is None else env)
|
||||||
|
src = path(RAW_PDF)
|
||||||
|
if not src.exists():
|
||||||
|
print("no route PDF in the raw store: nothing to send")
|
||||||
|
return 0
|
||||||
|
tok = forge.token(env)
|
||||||
|
if not tok:
|
||||||
|
print("ATLAS_BOT_TOKEN is not set: the PDF is not sent")
|
||||||
|
return 0
|
||||||
|
f = forge.config()
|
||||||
|
host = f["url"].rstrip("/")
|
||||||
|
genv = {**env, "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_COUNT": "1",
|
||||||
|
"GIT_CONFIG_KEY_0": f"http.{host}/.extraheader", "GIT_CONFIG_VALUE_0": f"AUTHORIZATION: token {tok}"}
|
||||||
|
new = src.read_bytes()
|
||||||
|
tmp = Path(tempfile.mkdtemp(prefix="dragonkind-"))
|
||||||
|
try:
|
||||||
|
repo = tmp / "repo"
|
||||||
|
_git(["clone", "--quiet", "--depth", "1", f"{host}/{BOT_REPO}.git", str(repo)], env=genv)
|
||||||
|
dst = repo / BOT_PDF
|
||||||
|
old = dst.read_bytes() if dst.exists() else b""
|
||||||
|
if old == new:
|
||||||
|
print("the bot already has the current route PDF")
|
||||||
|
return 0
|
||||||
|
gained, lost = text_changes(old, new) if old else (pdf_lines(new), [])
|
||||||
|
dst.write_bytes(new)
|
||||||
|
_git(["config", "user.name", "Atlas data bot"], cwd=repo)
|
||||||
|
_git(["config", "user.email", "atlas-bot@noreply.localhost"], cwd=repo)
|
||||||
|
_git(["checkout", "-q", "-b", BRANCH], cwd=repo)
|
||||||
|
_git(["add", BOT_PDF], cwd=repo)
|
||||||
|
_git(["commit", "-q", "-m", f"Community route PDF: current version ({len(new):,} bytes)"], cwd=repo)
|
||||||
|
_git(["push", "-q", "--force", "origin", BRANCH], cwd=repo, env=genv)
|
||||||
|
n = forge.open_pr(BRANCH, "Community route PDF changed", body(new, old, gained, lost), env=env,
|
||||||
|
repo=BOT_REPO, base="main")
|
||||||
|
print(f"route PDF sent: pull request #{n} on {BOT_REPO} ({len(gained)} lines new, {len(lost)} gone)")
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(tmp, ignore_errors=True)
|
||||||
@@ -131,6 +131,7 @@ def main(argv=None):
|
|||||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
||||||
k.add_argument("--out", default="exports")
|
k.add_argument("--out", default="exports")
|
||||||
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
||||||
|
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
|
||||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||||
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
||||||
@@ -154,6 +155,9 @@ def main(argv=None):
|
|||||||
if a.cmd == "rebuild":
|
if a.cmd == "rebuild":
|
||||||
from .etl.handoff import rebuild
|
from .etl.handoff import rebuild
|
||||||
return rebuild()
|
return rebuild()
|
||||||
|
if a.cmd == "send-pdf":
|
||||||
|
from .etl.send_pdf import send
|
||||||
|
return send()
|
||||||
if a.cmd == "refresh":
|
if a.cmd == "refresh":
|
||||||
from .etl.handoff import refresh
|
from .etl.handoff import refresh
|
||||||
return refresh(main, a.bot)
|
return refresh(main, a.bot)
|
||||||
|
|||||||
Reference in New Issue
Block a user