diff --git a/.gitignore b/.gitignore index 00176ac..4bf0fce 100644 --- a/.gitignore +++ b/.gitignore @@ -235,3 +235,6 @@ data/raw/ # QA screenshots undefined/ +# hand-off zips and local reports (python -m collection export-bot-kit / refresh) +exports/ +inbox/ diff --git a/README.md b/README.md index fb911f3..cd6d093 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,12 @@ This writes `data/bot/scenes.json` and `data/bot/trajectories-.jsonl`: sce bot's attempts to `runs_clean.csv` (source `bot`). The importer refuses to write anything that looks like prose. See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log. +## Running everything locally + +`python -m collection refresh` does what the daily GitHub workflow does: checks every source, rebuilds the runs, +the graph, the route model and the statistics. Add `--bot ` to import bot runs, and +`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`). + ## Updating with new form data by hand Download the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel), then: diff --git a/collection/etl/extract.py b/collection/etl/extract.py index cc2171b..ccfca2d 100644 --- a/collection/etl/extract.py +++ b/collection/etl/extract.py @@ -16,6 +16,7 @@ import shutil import tempfile import urllib.parse import urllib.request +import zipfile from datetime import datetime, timezone from pathlib import Path @@ -92,6 +93,24 @@ def local_dir(spec, env): return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8") +def pack_local(spec, src): + """The bot's files from a folder (a copy of the bot repo) or a .zip someone sent (the files may sit under any + top folder inside it), packed as one JSON like local_dir.""" + src = Path(src).expanduser() + if src.is_dir(): + return local_dir(spec, {spec["env"]: str(src)}) + files = {rel: None for rel in spec["files"]} + with zipfile.ZipFile(src) as z: + names = [n for n in z.namelist() if not n.endswith("/") and "__MACOSX" not in n] + for rel in spec["files"]: + hits = sorted((n for n in names if n == rel or n.endswith("/" + rel)), key=len) + if hits: + files[rel] = z.read(hits[0]).decode("utf-8") + if not any(files.values()): + raise RuntimeError(f"{src.name}: none of {', '.join(spec['files'])} found inside") + return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8") + + EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir} @@ -127,6 +146,12 @@ def extract(name, spec, manifest, env=None, local_file=None): """Fetch one source (or take a local file) into the raw store. Returns (raw path, changed).""" env = os.environ if env is None else env ext = spec["extract"].get("ext") or "json" - data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env) + kind = spec["extract"]["kind"] + if local_file and kind == "local_dir": + data = pack_local(spec["extract"], local_file) # a bot folder or a .zip someone sent + elif local_file: + data = Path(local_file).expanduser().read_bytes() + else: + data = EXTRACTORS[kind](spec["extract"], env) changed = store(name, ext, data, manifest) return raw_paths(name, ext)[0], changed diff --git a/collection/etl/handoff.py b/collection/etl/handoff.py new file mode 100644 index 0000000..3b7bab9 --- /dev/null +++ b/collection/etl/handoff.py @@ -0,0 +1,82 @@ +"""Hand-offs without GitHub: the zip of atlas files for the bot, and the local equivalent of the daily workflow. + + python -m collection export-bot-kit -> exports/atlas-for-bot--.zip + policy.json, solver.js and graph.js (what the bot vendors into vendor/threshing-atlas/), plus a README and + checksums. Send it to the bot's maintainer; they unzip it into vendor/threshing-atlas/. + python -m collection refresh [--bot PATH ...] + Every source (and any bot folders or zips given), then the graph, the route model, the statistics and the + build: what .github/workflows/update-sources.yml does every day. +""" +import hashlib +import json +import shutil +import subprocess +import sys +import zipfile +from datetime import datetime, timezone + +from .common import ROOT, path + +KIT = {"policy.json": "data/public/policy.json", "solver.js": "core/solver.js", "graph.js": "core/graph.js"} + +README = """Threshing Atlas files for the DragonKind bot +============================================ + +Policy id: {policy} +Made: {made} + +Unzip these three files into the bot's vendor/threshing-atlas/ folder, replacing the old ones, and keep them +together: solver.js reads the game from this policy.json. + + policy.json the route model (the game, the counts behind every roll, the best routes per colour) + solver.js the advisor: advisor(), recommendOption(), solverHistory() + graph.js trajectoryVariables(): the bot's steps -> the community form's variables + +How the bot uses them: docs/BOT_INTEGRATION.md in the atlas repo. Log the policy id above as policyId on each run. + +Checksums (sha256): +{sums} + +To send runs back: zip the bot's data/runs.jsonl, data/knowledge.json and logs/encounters.jsonl (keep the folder +names) and send the zip. It is imported with `python -m collection refresh --bot `; only anonymised +steps and outcomes are kept (no game prose, no account names). +""" + + +def export_bot_kit(out_dir="exports"): + policy = json.loads(path(KIT["policy.json"]).read_text(encoding="utf-8")) + pid = str(policy.get("id", "unknown"))[:12] + made = datetime.now(timezone.utc).strftime("%Y-%m-%d") + sums = "\n".join(f" {hashlib.sha256(path(src).read_bytes()).hexdigest()} {name}" for name, src in KIT.items()) + out = path(out_dir) + out.mkdir(parents=True, exist_ok=True) + zpath = out / f"atlas-for-bot-{made}-{pid}.zip" + with zipfile.ZipFile(zpath, "w", zipfile.ZIP_DEFLATED) as z: + for name, src in KIT.items(): + z.writestr(zipfile.ZipInfo(name, (2026, 1, 1, 0, 0, 0)), path(src).read_bytes(), zipfile.ZIP_DEFLATED) + z.writestr(zipfile.ZipInfo("README.txt", (2026, 1, 1, 0, 0, 0)), README.format(policy=pid, made=made, sums=sums)) + return zpath + + +def _step(label, cmd): + print(f"\n== {label}: {' '.join(cmd)}", flush=True) + subprocess.run(cmd, cwd=ROOT, check=True) + + +def refresh(update_main, bots=()): + """update -> (graph, policy, stats, build) when something changed. Returns 0 on success.""" + py = sys.executable + args = ["update", "--report", str(path("exports/last-report.md"))] + path("exports").mkdir(exist_ok=True) + rc = update_main(args) + for i, b in enumerate(bots): + rc |= update_main(["update", "--source", "bot", "--file", f"bot={b}", "--report", str(path(f"exports/last-report-bot{i + 1}.md"))]) + if shutil.which("node"): + _step("route graph", ["node", "analysis/network/build-graph.js"]) + else: + print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)") + _step("route model (about 2 minutes)", [py, "planner/build_policy.py"]) + _step("statistics", [py, "analysis/stats/site_stats.py"]) + _step("runs, generated files and metadata", [py, "-m", "collection", "build"]) + print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit") + return rc diff --git a/collection/run.py b/collection/run.py index ee4402f..ab439d5 100644 --- a/collection/run.py +++ b/collection/run.py @@ -4,6 +4,11 @@ For each source (all by default): extract it (or take a local file), and when the raw file changed, transform it and load the public source file. Then build. Exit code 0 even when a source could not be reached: the report says what happened (and the daily workflow opens an issue). + python -m collection refresh [--bot PATH ...] + Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then + rebuild the graph, the route model, the statistics and the build. + python -m collection export-bot-kit + exports/atlas-for-bot--.zip: policy.json, solver.js and graph.js for the bot. python -m collection build [--check] Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml), and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in @@ -46,7 +51,9 @@ def update_source(name, spec, manifest, report, local_file=None, env=None): old = {"scenes": L.read_json(f"{target}/scenes.json") or {}} lines = mod.describe(old, new) wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False) - wrote |= L.jsonl_file(f"{target}/trajectories-{ctx['changed']}.jsonl", new["trajectories"]) + # one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite + tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}" + wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"]) elif target: old = L.read_json(target) lines = mod.describe(old, new) @@ -93,8 +100,20 @@ def main(argv=None): up.add_argument("--report") b = sub.add_parser("build") b.add_argument("--check", action="store_true") + k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot") + k.add_argument("--out", default="exports") + r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)") + r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)") a = ap.parse_args(argv) + if a.cmd == "export-bot-kit": + from .etl.handoff import export_bot_kit + print(f"wrote {export_bot_kit(a.out)}") + return 0 + if a.cmd == "refresh": + from .etl.handoff import refresh + return refresh(main, a.bot) + if a.cmd == "build": changed = build(check=a.check) if a.check and changed: diff --git a/docs/BOT_INTEGRATION.md b/docs/BOT_INTEGRATION.md index adfcc0a..f2b8d46 100644 --- a/docs/BOT_INTEGRATION.md +++ b/docs/BOT_INTEGRATION.md @@ -9,6 +9,17 @@ Data flows both ways: └──────────── collection --source bot + PR ◄┘ ``` +## Without GitHub: swapping zips + +When GitHub Pages is not available, the same files travel as zips: + +- **Atlas → bot:** `python -m collection export-bot-kit` writes `exports/atlas-for-bot--.zip` + (`policy.json`, `solver.js`, `graph.js`, a README with checksums). Unzip it into the bot's `vendor/threshing-atlas/`. +- **Bot → atlas:** zip the bot's `data/runs.jsonl`, `data/knowledge.json` and `logs/encounters.jsonl` (folder names + kept; a zip of the whole repo works too) and send it. Put it in the atlas's `inbox/` and run + `python -m collection refresh --bot inbox/`: it imports the runs (anonymised: no prose, no account names), + rebuilds everything, and then `export-bot-kit` makes the new zip to send back. + ## 1. From the atlas to the bot ### What is published diff --git a/tests/collection/test_collection.py b/tests/collection/test_collection.py index 9601cc7..87906a1 100644 --- a/tests/collection/test_collection.py +++ b/tests/collection/test_collection.py @@ -113,6 +113,32 @@ class Bot(unittest.TestCase): self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8")) +class Handoff(unittest.TestCase): + def test_bot_zip_imports_like_the_folder(self): + import zipfile + spec = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["extract"] + z = TMP / "ken.zip" + with zipfile.ZipFile(z, "w") as f: # files under a top folder, as a zipped repo would have them + for rel in spec["files"]: + f.write(FX / rel, f"DragonKind-main/{rel}") + self.assertEqual(extract.pack_local(spec, z), extract.pack_local(spec, FX)) + empty = TMP / "empty.zip" + with zipfile.ZipFile(empty, "w") as f: + f.writestr("notes.txt", "hi") + with self.assertRaises(RuntimeError): + extract.pack_local(spec, empty) + + def test_bot_kit(self): + import zipfile + from collection.etl import handoff + zpath = handoff.export_bot_kit(str(TMP / "exports")) + with zipfile.ZipFile(zpath) as z: + self.assertEqual(sorted(z.namelist()), ["README.txt", "graph.js", "policy.json", "solver.js"]) + self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes()) + self.assertEqual(json.loads(z.read("policy.json"))["id"], + json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"]) + + class Sheets(unittest.TestCase): def test_privacy_rules(self): p = TMP / "sheet.csv"