|
|
|
@@ -0,0 +1,116 @@
|
|
|
|
|
"""Send the community route PDF to the bot's private repo when it changed (python -m collection send-pdf).
|
|
|
|
|
|
|
|
|
|
The PDF is never published by the Atlas: the daily check keeps it in the private raw store only
|
|
|
|
|
(data/raw/reddit_pdf/current.pdf). The bot's repo (private, same host) keeps its own copy, "Threshing Options.pdf",
|
|
|
|
|
which its hand-written routes (src/pdfRoutes.js) were transcribed from. When the two differ, this opens (or updates)
|
|
|
|
|
one pull request there, on branch atlas/route-pdf, with the new file and the lines its text gained and lost.
|
|
|
|
|
|
|
|
|
|
The token is ATLAS_BOT_TOKEN (write access to the bot's repo); git gets it through GIT_CONFIG_* variables, never argv.
|
|
|
|
|
"""
|
|
|
|
|
import hashlib
|
|
|
|
|
import io
|
|
|
|
|
import os
|
|
|
|
|
import shutil
|
|
|
|
|
import subprocess
|
|
|
|
|
import tempfile
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
from . import forge
|
|
|
|
|
from .common import path
|
|
|
|
|
|
|
|
|
|
BOT_REPO = "DragonKind/DragonKind"
|
|
|
|
|
BOT_PDF = "Threshing Options.pdf"
|
|
|
|
|
BRANCH = "atlas/route-pdf"
|
|
|
|
|
RAW_PDF = "data/raw/reddit_pdf/current.pdf"
|
|
|
|
|
MAX_LINES = 60
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def pdf_lines(data):
|
|
|
|
|
"""The PDF's text, one entry per non-empty line, whitespace collapsed."""
|
|
|
|
|
from pypdf import PdfReader
|
|
|
|
|
out = []
|
|
|
|
|
for page in PdfReader(io.BytesIO(data)).pages:
|
|
|
|
|
for line in (page.extract_text() or "").splitlines():
|
|
|
|
|
line = " ".join(line.split())
|
|
|
|
|
if line:
|
|
|
|
|
out.append(line)
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def text_changes(old, new):
|
|
|
|
|
"""(lines only in the new PDF, lines only in the old one), each in the order they appear."""
|
|
|
|
|
o, n = pdf_lines(old), pdf_lines(new)
|
|
|
|
|
so, sn = set(o), set(n)
|
|
|
|
|
gained = list(dict.fromkeys(l for l in n if l not in so))
|
|
|
|
|
lost = list(dict.fromkeys(l for l in o if l not in sn))
|
|
|
|
|
return gained, lost
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _bullets(lines):
|
|
|
|
|
if not lines:
|
|
|
|
|
return "- (none)"
|
|
|
|
|
shown = [f"- {l}" for l in lines[:MAX_LINES]]
|
|
|
|
|
if len(lines) > MAX_LINES:
|
|
|
|
|
shown.append(f"- ... and {len(lines) - MAX_LINES} more")
|
|
|
|
|
return "\n".join(shown)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def body(new, old, gained, lost):
|
|
|
|
|
h = lambda b: hashlib.sha256(b).hexdigest()[:12]
|
|
|
|
|
return (
|
|
|
|
|
"The community route PDF (u/ClumsyBadger's Threshing Options, the Google Drive file linked from the "
|
|
|
|
|
"megathreads) changed. This replaces `Threshing Options.pdf` with the current version.\n\n"
|
|
|
|
|
f"| | bytes | sha256 |\n|---|---|---|\n| this repo's copy | {len(old):,} | `{h(old)}` |\n"
|
|
|
|
|
f"| current PDF | {len(new):,} | `{h(new)}` |\n\n"
|
|
|
|
|
"**The bot does not read the PDF itself**: its routes are written out by hand in `src/pdfRoutes.js`. "
|
|
|
|
|
"Check them against the changes below.\n\n"
|
|
|
|
|
f"### Text that is new in this version\n{_bullets(gained)}\n\n"
|
|
|
|
|
f"### Text no longer in it\n{_bullets(lost)}\n\n"
|
|
|
|
|
"(Text extracted line by line from a flowchart, so some lines are fragments of boxes.) "
|
|
|
|
|
"Opened automatically by the Threshing Atlas's daily source check; it is updated in place when the PDF "
|
|
|
|
|
"changes again.\n"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _git(args, cwd=None, env=None):
|
|
|
|
|
subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def send(env=None):
|
|
|
|
|
env = dict(os.environ if env is None else env)
|
|
|
|
|
src = path(RAW_PDF)
|
|
|
|
|
if not src.exists():
|
|
|
|
|
print("no route PDF in the raw store: nothing to send")
|
|
|
|
|
return 0
|
|
|
|
|
tok = forge.token(env)
|
|
|
|
|
if not tok:
|
|
|
|
|
print("ATLAS_BOT_TOKEN is not set: the PDF is not sent")
|
|
|
|
|
return 0
|
|
|
|
|
f = forge.config()
|
|
|
|
|
host = f["url"].rstrip("/")
|
|
|
|
|
genv = {**env, "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_COUNT": "1",
|
|
|
|
|
"GIT_CONFIG_KEY_0": f"http.{host}/.extraheader", "GIT_CONFIG_VALUE_0": f"AUTHORIZATION: token {tok}"}
|
|
|
|
|
new = src.read_bytes()
|
|
|
|
|
tmp = Path(tempfile.mkdtemp(prefix="dragonkind-"))
|
|
|
|
|
try:
|
|
|
|
|
repo = tmp / "repo"
|
|
|
|
|
_git(["clone", "--quiet", "--depth", "1", f"{host}/{BOT_REPO}.git", str(repo)], env=genv)
|
|
|
|
|
dst = repo / BOT_PDF
|
|
|
|
|
old = dst.read_bytes() if dst.exists() else b""
|
|
|
|
|
if old == new:
|
|
|
|
|
print("the bot already has the current route PDF")
|
|
|
|
|
return 0
|
|
|
|
|
gained, lost = text_changes(old, new) if old else (pdf_lines(new), [])
|
|
|
|
|
dst.write_bytes(new)
|
|
|
|
|
_git(["config", "user.name", "Atlas data bot"], cwd=repo)
|
|
|
|
|
_git(["config", "user.email", "atlas-bot@noreply.localhost"], cwd=repo)
|
|
|
|
|
_git(["checkout", "-q", "-b", BRANCH], cwd=repo)
|
|
|
|
|
_git(["add", BOT_PDF], cwd=repo)
|
|
|
|
|
_git(["commit", "-q", "-m", f"Community route PDF: current version ({len(new):,} bytes)"], cwd=repo)
|
|
|
|
|
_git(["push", "-q", "--force", "origin", BRANCH], cwd=repo, env=genv)
|
|
|
|
|
n = forge.open_pr(BRANCH, "Community route PDF changed", body(new, old, gained, lost), env=env,
|
|
|
|
|
repo=BOT_REPO, base="main")
|
|
|
|
|
print(f"route PDF sent: pull request #{n} on {BOT_REPO} ({len(gained)} lines new, {len(lost)} gone)")
|
|
|
|
|
return 0
|
|
|
|
|
finally:
|
|
|
|
|
shutil.rmtree(tmp, ignore_errors=True)
|