Compare commits
2
Commits
df0e576a0a
...
e61134ecc8
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e61134ecc8 | ||
|
|
66181b8489 |
@@ -86,6 +86,12 @@ jobs:
|
||||
with:
|
||||
path: ${{ runner.temp }}/raw-archive
|
||||
key: raw-archive-${{ hashFiles('data/manifest.json') }}
|
||||
- name: Send a changed route PDF to the bot (its private repo; the PDF is never published here)
|
||||
if: always() && steps.check.outcome == 'success'
|
||||
continue-on-error: true
|
||||
env:
|
||||
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||
run: python -m collection send-pdf
|
||||
- name: Open a pull request with the new data (it merges itself once CI passes)
|
||||
if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build')
|
||||
env:
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Send the community route PDF to the bot's private repo when it changed (python -m collection send-pdf).
|
||||
|
||||
The PDF is never published by the Atlas: the daily check keeps it in the private raw store only
|
||||
(data/raw/reddit_pdf/current.pdf). The bot's repo (private, same host) keeps its own copy, "Threshing Options.pdf",
|
||||
which its hand-written routes (src/pdfRoutes.js) were transcribed from. When the two differ, this opens (or updates)
|
||||
one pull request there, on branch atlas/route-pdf, with the new file and the lines its text gained and lost.
|
||||
|
||||
The token is ATLAS_BOT_TOKEN (write access to the bot's repo); git gets it through GIT_CONFIG_* variables, never argv.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from . import forge
|
||||
from .common import path
|
||||
|
||||
BOT_REPO = "DragonKind/DragonKind"
|
||||
BOT_PDF = "Threshing Options.pdf"
|
||||
BRANCH = "atlas/route-pdf"
|
||||
RAW_PDF = "data/raw/reddit_pdf/current.pdf"
|
||||
MAX_LINES = 60
|
||||
|
||||
|
||||
def pdf_lines(data):
|
||||
"""The PDF's text, one entry per non-empty line, whitespace collapsed."""
|
||||
from pypdf import PdfReader
|
||||
out = []
|
||||
for page in PdfReader(io.BytesIO(data)).pages:
|
||||
for line in (page.extract_text() or "").splitlines():
|
||||
line = " ".join(line.split())
|
||||
if line:
|
||||
out.append(line)
|
||||
return out
|
||||
|
||||
|
||||
def text_changes(old, new):
|
||||
"""(lines only in the new PDF, lines only in the old one), each in the order they appear."""
|
||||
o, n = pdf_lines(old), pdf_lines(new)
|
||||
so, sn = set(o), set(n)
|
||||
gained = list(dict.fromkeys(l for l in n if l not in so))
|
||||
lost = list(dict.fromkeys(l for l in o if l not in sn))
|
||||
return gained, lost
|
||||
|
||||
|
||||
def _bullets(lines):
|
||||
if not lines:
|
||||
return "- (none)"
|
||||
shown = [f"- {l}" for l in lines[:MAX_LINES]]
|
||||
if len(lines) > MAX_LINES:
|
||||
shown.append(f"- ... and {len(lines) - MAX_LINES} more")
|
||||
return "\n".join(shown)
|
||||
|
||||
|
||||
def body(new, old, gained, lost):
|
||||
h = lambda b: hashlib.sha256(b).hexdigest()[:12]
|
||||
return (
|
||||
"The community route PDF (u/ClumsyBadger's Threshing Options, the Google Drive file linked from the "
|
||||
"megathreads) changed. This replaces `Threshing Options.pdf` with the current version.\n\n"
|
||||
f"| | bytes | sha256 |\n|---|---|---|\n| this repo's copy | {len(old):,} | `{h(old)}` |\n"
|
||||
f"| current PDF | {len(new):,} | `{h(new)}` |\n\n"
|
||||
"**The bot does not read the PDF itself**: its routes are written out by hand in `src/pdfRoutes.js`. "
|
||||
"Check them against the changes below.\n\n"
|
||||
f"### Text that is new in this version\n{_bullets(gained)}\n\n"
|
||||
f"### Text no longer in it\n{_bullets(lost)}\n\n"
|
||||
"(Text extracted line by line from a flowchart, so some lines are fragments of boxes.) "
|
||||
"Opened automatically by the Threshing Atlas's daily source check; it is updated in place when the PDF "
|
||||
"changes again.\n"
|
||||
)
|
||||
|
||||
|
||||
def _git(args, cwd=None, env=None):
|
||||
subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True)
|
||||
|
||||
|
||||
def send(env=None):
|
||||
env = dict(os.environ if env is None else env)
|
||||
src = path(RAW_PDF)
|
||||
if not src.exists():
|
||||
print("no route PDF in the raw store: nothing to send")
|
||||
return 0
|
||||
tok = forge.token(env)
|
||||
if not tok:
|
||||
print("ATLAS_BOT_TOKEN is not set: the PDF is not sent")
|
||||
return 0
|
||||
f = forge.config()
|
||||
host = f["url"].rstrip("/")
|
||||
genv = {**env, "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_COUNT": "1",
|
||||
"GIT_CONFIG_KEY_0": f"http.{host}/.extraheader", "GIT_CONFIG_VALUE_0": f"AUTHORIZATION: token {tok}"}
|
||||
new = src.read_bytes()
|
||||
tmp = Path(tempfile.mkdtemp(prefix="dragonkind-"))
|
||||
try:
|
||||
repo = tmp / "repo"
|
||||
_git(["clone", "--quiet", "--depth", "1", f"{host}/{BOT_REPO}.git", str(repo)], env=genv)
|
||||
dst = repo / BOT_PDF
|
||||
old = dst.read_bytes() if dst.exists() else b""
|
||||
if old == new:
|
||||
print("the bot already has the current route PDF")
|
||||
return 0
|
||||
gained, lost = text_changes(old, new) if old else (pdf_lines(new), [])
|
||||
dst.write_bytes(new)
|
||||
_git(["config", "user.name", "Atlas data bot"], cwd=repo)
|
||||
_git(["config", "user.email", "atlas-bot@noreply.localhost"], cwd=repo)
|
||||
_git(["checkout", "-q", "-b", BRANCH], cwd=repo)
|
||||
_git(["add", BOT_PDF], cwd=repo)
|
||||
_git(["commit", "-q", "-m", f"Community route PDF: current version ({len(new):,} bytes)"], cwd=repo)
|
||||
_git(["push", "-q", "--force", "origin", BRANCH], cwd=repo, env=genv)
|
||||
n = forge.open_pr(BRANCH, "Community route PDF changed", body(new, old, gained, lost), env=env,
|
||||
repo=BOT_REPO, base="main")
|
||||
print(f"route PDF sent: pull request #{n} on {BOT_REPO} ({len(gained)} lines new, {len(lost)} gone)")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
@@ -131,6 +131,7 @@ def main(argv=None):
|
||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
||||
k.add_argument("--out", default="exports")
|
||||
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
||||
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
|
||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
||||
@@ -154,6 +155,9 @@ def main(argv=None):
|
||||
if a.cmd == "rebuild":
|
||||
from .etl.handoff import rebuild
|
||||
return rebuild()
|
||||
if a.cmd == "send-pdf":
|
||||
from .etl.send_pdf import send
|
||||
return send()
|
||||
if a.cmd == "refresh":
|
||||
from .etl.handoff import refresh
|
||||
return refresh(main, a.bot)
|
||||
|
||||
Reference in New Issue
Block a user