Hand-offs without GitHub: bot kit zip, bot zip import, local refresh

- python -m collection export-bot-kit: policy.json, solver.js, graph.js and a
  README with checksums, zipped for the bot
- the bot import takes a folder or a zip (files may sit under any top folder)
- python -m collection refresh: the daily workflow, run locally
- trajectories files are named by date and raw checksum, so two imports on
  one day never overwrite each other

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-10 01:50:12 +00:00
co-authored by Claude Opus 5.5
parent 19209a2ad6
commit 5c49591dae
7 changed files with 174 additions and 2 deletions
+3
View File
@@ -235,3 +235,6 @@ data/raw/
# QA screenshots # QA screenshots
undefined/ undefined/
# hand-off zips and local reports (python -m collection export-bot-kit / refresh)
exports/
inbox/
+6
View File
@@ -73,6 +73,12 @@ This writes `data/bot/scenes.json` and `data/bot/trajectories-<date>.jsonl`: sce
bot's attempts to `runs_clean.csv` (source `bot`). The importer refuses to write anything that looks like prose. bot's attempts to `runs_clean.csv` (source `bot`). The importer refuses to write anything that looks like prose.
See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log. See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
## Running everything locally
`python -m collection refresh` does what the daily GitHub workflow does: checks every source, rebuilds the runs,
the graph, the route model and the statistics. Add `--bot <folder or zip>` to import bot runs, and
`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`).
## Updating with new form data by hand ## Updating with new form data by hand
Download the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel), then: Download the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel), then:
+26 -1
View File
@@ -16,6 +16,7 @@ import shutil
import tempfile import tempfile
import urllib.parse import urllib.parse
import urllib.request import urllib.request
import zipfile
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
@@ -92,6 +93,24 @@ def local_dir(spec, env):
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8") return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
def pack_local(spec, src):
"""The bot's files from a folder (a copy of the bot repo) or a .zip someone sent (the files may sit under any
top folder inside it), packed as one JSON like local_dir."""
src = Path(src).expanduser()
if src.is_dir():
return local_dir(spec, {spec["env"]: str(src)})
files = {rel: None for rel in spec["files"]}
with zipfile.ZipFile(src) as z:
names = [n for n in z.namelist() if not n.endswith("/") and "__MACOSX" not in n]
for rel in spec["files"]:
hits = sorted((n for n in names if n == rel or n.endswith("/" + rel)), key=len)
if hits:
files[rel] = z.read(hits[0]).decode("utf-8")
if not any(files.values()):
raise RuntimeError(f"{src.name}: none of {', '.join(spec['files'])} found inside")
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir} EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
@@ -127,6 +146,12 @@ def extract(name, spec, manifest, env=None, local_file=None):
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed).""" """Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
env = os.environ if env is None else env env = os.environ if env is None else env
ext = spec["extract"].get("ext") or "json" ext = spec["extract"].get("ext") or "json"
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env) kind = spec["extract"]["kind"]
if local_file and kind == "local_dir":
data = pack_local(spec["extract"], local_file) # a bot folder or a .zip someone sent
elif local_file:
data = Path(local_file).expanduser().read_bytes()
else:
data = EXTRACTORS[kind](spec["extract"], env)
changed = store(name, ext, data, manifest) changed = store(name, ext, data, manifest)
return raw_paths(name, ext)[0], changed return raw_paths(name, ext)[0], changed
+82
View File
@@ -0,0 +1,82 @@
"""Hand-offs without GitHub: the zip of atlas files for the bot, and the local equivalent of the daily workflow.
python -m collection export-bot-kit -> exports/atlas-for-bot-<date>-<policy id>.zip
policy.json, solver.js and graph.js (what the bot vendors into vendor/threshing-atlas/), plus a README and
checksums. Send it to the bot's maintainer; they unzip it into vendor/threshing-atlas/.
python -m collection refresh [--bot PATH ...]
Every source (and any bot folders or zips given), then the graph, the route model, the statistics and the
build: what .github/workflows/update-sources.yml does every day.
"""
import hashlib
import json
import shutil
import subprocess
import sys
import zipfile
from datetime import datetime, timezone
from .common import ROOT, path
KIT = {"policy.json": "data/public/policy.json", "solver.js": "core/solver.js", "graph.js": "core/graph.js"}
README = """Threshing Atlas files for the DragonKind bot
============================================
Policy id: {policy}
Made: {made}
Unzip these three files into the bot's vendor/threshing-atlas/ folder, replacing the old ones, and keep them
together: solver.js reads the game from this policy.json.
policy.json the route model (the game, the counts behind every roll, the best routes per colour)
solver.js the advisor: advisor(), recommendOption(), solverHistory()
graph.js trajectoryVariables(): the bot's steps -> the community form's variables
How the bot uses them: docs/BOT_INTEGRATION.md in the atlas repo. Log the policy id above as policyId on each run.
Checksums (sha256):
{sums}
To send runs back: zip the bot's data/runs.jsonl, data/knowledge.json and logs/encounters.jsonl (keep the folder
names) and send the zip. It is imported with `python -m collection refresh --bot <the zip>`; only anonymised
steps and outcomes are kept (no game prose, no account names).
"""
def export_bot_kit(out_dir="exports"):
policy = json.loads(path(KIT["policy.json"]).read_text(encoding="utf-8"))
pid = str(policy.get("id", "unknown"))[:12]
made = datetime.now(timezone.utc).strftime("%Y-%m-%d")
sums = "\n".join(f" {hashlib.sha256(path(src).read_bytes()).hexdigest()} {name}" for name, src in KIT.items())
out = path(out_dir)
out.mkdir(parents=True, exist_ok=True)
zpath = out / f"atlas-for-bot-{made}-{pid}.zip"
with zipfile.ZipFile(zpath, "w", zipfile.ZIP_DEFLATED) as z:
for name, src in KIT.items():
z.writestr(zipfile.ZipInfo(name, (2026, 1, 1, 0, 0, 0)), path(src).read_bytes(), zipfile.ZIP_DEFLATED)
z.writestr(zipfile.ZipInfo("README.txt", (2026, 1, 1, 0, 0, 0)), README.format(policy=pid, made=made, sums=sums))
return zpath
def _step(label, cmd):
print(f"\n== {label}: {' '.join(cmd)}", flush=True)
subprocess.run(cmd, cwd=ROOT, check=True)
def refresh(update_main, bots=()):
"""update -> (graph, policy, stats, build) when something changed. Returns 0 on success."""
py = sys.executable
args = ["update", "--report", str(path("exports/last-report.md"))]
path("exports").mkdir(exist_ok=True)
rc = update_main(args)
for i, b in enumerate(bots):
rc |= update_main(["update", "--source", "bot", "--file", f"bot={b}", "--report", str(path(f"exports/last-report-bot{i + 1}.md"))])
if shutil.which("node"):
_step("route graph", ["node", "analysis/network/build-graph.js"])
else:
print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)")
_step("route model (about 2 minutes)", [py, "planner/build_policy.py"])
_step("statistics", [py, "analysis/stats/site_stats.py"])
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit")
return rc
+20 -1
View File
@@ -4,6 +4,11 @@
For each source (all by default): extract it (or take a local file), and when the raw file changed, For each source (all by default): extract it (or take a local file), and when the raw file changed,
transform it and load the public source file. Then build. Exit code 0 even when a source could not be transform it and load the public source file. Then build. Exit code 0 even when a source could not be
reached: the report says what happened (and the daily workflow opens an issue). reached: the report says what happened (and the daily workflow opens an issue).
python -m collection refresh [--bot PATH ...]
Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then
rebuild the graph, the route model, the statistics and the build.
python -m collection export-bot-kit
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, solver.js and graph.js for the bot.
python -m collection build [--check] python -m collection build [--check]
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml), Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
@@ -46,7 +51,9 @@ def update_source(name, spec, manifest, report, local_file=None, env=None):
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}} old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
lines = mod.describe(old, new) lines = mod.describe(old, new)
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False) wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
wrote |= L.jsonl_file(f"{target}/trajectories-{ctx['changed']}.jsonl", new["trajectories"]) # one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite
tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}"
wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"])
elif target: elif target:
old = L.read_json(target) old = L.read_json(target)
lines = mod.describe(old, new) lines = mod.describe(old, new)
@@ -93,8 +100,20 @@ def main(argv=None):
up.add_argument("--report") up.add_argument("--report")
b = sub.add_parser("build") b = sub.add_parser("build")
b.add_argument("--check", action="store_true") b.add_argument("--check", action="store_true")
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
k.add_argument("--out", default="exports")
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
a = ap.parse_args(argv) a = ap.parse_args(argv)
if a.cmd == "export-bot-kit":
from .etl.handoff import export_bot_kit
print(f"wrote {export_bot_kit(a.out)}")
return 0
if a.cmd == "refresh":
from .etl.handoff import refresh
return refresh(main, a.bot)
if a.cmd == "build": if a.cmd == "build":
changed = build(check=a.check) changed = build(check=a.check)
if a.check and changed: if a.check and changed:
+11
View File
@@ -9,6 +9,17 @@ Data flows both ways:
└──────────── collection --source bot + PR ◄┘ └──────────── collection --source bot + PR ◄┘
``` ```
## Without GitHub: swapping zips
When GitHub Pages is not available, the same files travel as zips:
- **Atlas → bot:** `python -m collection export-bot-kit` writes `exports/atlas-for-bot-<date>-<policy id>.zip`
(`policy.json`, `solver.js`, `graph.js`, a README with checksums). Unzip it into the bot's `vendor/threshing-atlas/`.
- **Bot → atlas:** zip the bot's `data/runs.jsonl`, `data/knowledge.json` and `logs/encounters.jsonl` (folder names
kept; a zip of the whole repo works too) and send it. Put it in the atlas's `inbox/` and run
`python -m collection refresh --bot inbox/<the zip>`: it imports the runs (anonymised: no prose, no account names),
rebuilds everything, and then `export-bot-kit` makes the new zip to send back.
## 1. From the atlas to the bot ## 1. From the atlas to the bot
### What is published ### What is published
+26
View File
@@ -113,6 +113,32 @@ class Bot(unittest.TestCase):
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8")) self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
class Handoff(unittest.TestCase):
def test_bot_zip_imports_like_the_folder(self):
import zipfile
spec = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["extract"]
z = TMP / "ken.zip"
with zipfile.ZipFile(z, "w") as f: # files under a top folder, as a zipped repo would have them
for rel in spec["files"]:
f.write(FX / rel, f"DragonKind-main/{rel}")
self.assertEqual(extract.pack_local(spec, z), extract.pack_local(spec, FX))
empty = TMP / "empty.zip"
with zipfile.ZipFile(empty, "w") as f:
f.writestr("notes.txt", "hi")
with self.assertRaises(RuntimeError):
extract.pack_local(spec, empty)
def test_bot_kit(self):
import zipfile
from collection.etl import handoff
zpath = handoff.export_bot_kit(str(TMP / "exports"))
with zipfile.ZipFile(zpath) as z:
self.assertEqual(sorted(z.namelist()), ["README.txt", "graph.js", "policy.json", "solver.js"])
self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes())
self.assertEqual(json.loads(z.read("policy.json"))["id"],
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
class Sheets(unittest.TestCase): class Sheets(unittest.TestCase):
def test_privacy_rules(self): def test_privacy_rules(self):
p = TMP / "sheet.csv" p = TMP / "sheet.csv"