Hand-offs without GitHub: bot kit zip, bot zip import, local refresh
- python -m collection export-bot-kit: policy.json, solver.js, graph.js and a README with checksums, zipped for the bot - the bot import takes a folder or a zip (files may sit under any top folder) - python -m collection refresh: the daily workflow, run locally - trajectories files are named by date and raw checksum, so two imports on one day never overwrite each other Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
19209a2ad6
commit
5c49591dae
@@ -235,3 +235,6 @@ data/raw/
|
||||
|
||||
# QA screenshots
|
||||
undefined/
|
||||
# hand-off zips and local reports (python -m collection export-bot-kit / refresh)
|
||||
exports/
|
||||
inbox/
|
||||
|
||||
@@ -73,6 +73,12 @@ This writes `data/bot/scenes.json` and `data/bot/trajectories-<date>.jsonl`: sce
|
||||
bot's attempts to `runs_clean.csv` (source `bot`). The importer refuses to write anything that looks like prose.
|
||||
See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
|
||||
|
||||
## Running everything locally
|
||||
|
||||
`python -m collection refresh` does what the daily GitHub workflow does: checks every source, rebuilds the runs,
|
||||
the graph, the route model and the statistics. Add `--bot <folder or zip>` to import bot runs, and
|
||||
`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`).
|
||||
|
||||
## Updating with new form data by hand
|
||||
|
||||
Download the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel), then:
|
||||
|
||||
@@ -16,6 +16,7 @@ import shutil
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
@@ -92,6 +93,24 @@ def local_dir(spec, env):
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def pack_local(spec, src):
|
||||
"""The bot's files from a folder (a copy of the bot repo) or a .zip someone sent (the files may sit under any
|
||||
top folder inside it), packed as one JSON like local_dir."""
|
||||
src = Path(src).expanduser()
|
||||
if src.is_dir():
|
||||
return local_dir(spec, {spec["env"]: str(src)})
|
||||
files = {rel: None for rel in spec["files"]}
|
||||
with zipfile.ZipFile(src) as z:
|
||||
names = [n for n in z.namelist() if not n.endswith("/") and "__MACOSX" not in n]
|
||||
for rel in spec["files"]:
|
||||
hits = sorted((n for n in names if n == rel or n.endswith("/" + rel)), key=len)
|
||||
if hits:
|
||||
files[rel] = z.read(hits[0]).decode("utf-8")
|
||||
if not any(files.values()):
|
||||
raise RuntimeError(f"{src.name}: none of {', '.join(spec['files'])} found inside")
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
||||
|
||||
|
||||
@@ -127,6 +146,12 @@ def extract(name, spec, manifest, env=None, local_file=None):
|
||||
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
|
||||
env = os.environ if env is None else env
|
||||
ext = spec["extract"].get("ext") or "json"
|
||||
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env)
|
||||
kind = spec["extract"]["kind"]
|
||||
if local_file and kind == "local_dir":
|
||||
data = pack_local(spec["extract"], local_file) # a bot folder or a .zip someone sent
|
||||
elif local_file:
|
||||
data = Path(local_file).expanduser().read_bytes()
|
||||
else:
|
||||
data = EXTRACTORS[kind](spec["extract"], env)
|
||||
changed = store(name, ext, data, manifest)
|
||||
return raw_paths(name, ext)[0], changed
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Hand-offs without GitHub: the zip of atlas files for the bot, and the local equivalent of the daily workflow.
|
||||
|
||||
python -m collection export-bot-kit -> exports/atlas-for-bot-<date>-<policy id>.zip
|
||||
policy.json, solver.js and graph.js (what the bot vendors into vendor/threshing-atlas/), plus a README and
|
||||
checksums. Send it to the bot's maintainer; they unzip it into vendor/threshing-atlas/.
|
||||
python -m collection refresh [--bot PATH ...]
|
||||
Every source (and any bot folders or zips given), then the graph, the route model, the statistics and the
|
||||
build: what .github/workflows/update-sources.yml does every day.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .common import ROOT, path
|
||||
|
||||
KIT = {"policy.json": "data/public/policy.json", "solver.js": "core/solver.js", "graph.js": "core/graph.js"}
|
||||
|
||||
README = """Threshing Atlas files for the DragonKind bot
|
||||
============================================
|
||||
|
||||
Policy id: {policy}
|
||||
Made: {made}
|
||||
|
||||
Unzip these three files into the bot's vendor/threshing-atlas/ folder, replacing the old ones, and keep them
|
||||
together: solver.js reads the game from this policy.json.
|
||||
|
||||
policy.json the route model (the game, the counts behind every roll, the best routes per colour)
|
||||
solver.js the advisor: advisor(), recommendOption(), solverHistory()
|
||||
graph.js trajectoryVariables(): the bot's steps -> the community form's variables
|
||||
|
||||
How the bot uses them: docs/BOT_INTEGRATION.md in the atlas repo. Log the policy id above as policyId on each run.
|
||||
|
||||
Checksums (sha256):
|
||||
{sums}
|
||||
|
||||
To send runs back: zip the bot's data/runs.jsonl, data/knowledge.json and logs/encounters.jsonl (keep the folder
|
||||
names) and send the zip. It is imported with `python -m collection refresh --bot <the zip>`; only anonymised
|
||||
steps and outcomes are kept (no game prose, no account names).
|
||||
"""
|
||||
|
||||
|
||||
def export_bot_kit(out_dir="exports"):
|
||||
policy = json.loads(path(KIT["policy.json"]).read_text(encoding="utf-8"))
|
||||
pid = str(policy.get("id", "unknown"))[:12]
|
||||
made = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
sums = "\n".join(f" {hashlib.sha256(path(src).read_bytes()).hexdigest()} {name}" for name, src in KIT.items())
|
||||
out = path(out_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
zpath = out / f"atlas-for-bot-{made}-{pid}.zip"
|
||||
with zipfile.ZipFile(zpath, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for name, src in KIT.items():
|
||||
z.writestr(zipfile.ZipInfo(name, (2026, 1, 1, 0, 0, 0)), path(src).read_bytes(), zipfile.ZIP_DEFLATED)
|
||||
z.writestr(zipfile.ZipInfo("README.txt", (2026, 1, 1, 0, 0, 0)), README.format(policy=pid, made=made, sums=sums))
|
||||
return zpath
|
||||
|
||||
|
||||
def _step(label, cmd):
|
||||
print(f"\n== {label}: {' '.join(cmd)}", flush=True)
|
||||
subprocess.run(cmd, cwd=ROOT, check=True)
|
||||
|
||||
|
||||
def refresh(update_main, bots=()):
|
||||
"""update -> (graph, policy, stats, build) when something changed. Returns 0 on success."""
|
||||
py = sys.executable
|
||||
args = ["update", "--report", str(path("exports/last-report.md"))]
|
||||
path("exports").mkdir(exist_ok=True)
|
||||
rc = update_main(args)
|
||||
for i, b in enumerate(bots):
|
||||
rc |= update_main(["update", "--source", "bot", "--file", f"bot={b}", "--report", str(path(f"exports/last-report-bot{i + 1}.md"))])
|
||||
if shutil.which("node"):
|
||||
_step("route graph", ["node", "analysis/network/build-graph.js"])
|
||||
else:
|
||||
print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)")
|
||||
_step("route model (about 2 minutes)", [py, "planner/build_policy.py"])
|
||||
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
||||
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
|
||||
print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit")
|
||||
return rc
|
||||
+20
-1
@@ -4,6 +4,11 @@
|
||||
For each source (all by default): extract it (or take a local file), and when the raw file changed,
|
||||
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
|
||||
reached: the report says what happened (and the daily workflow opens an issue).
|
||||
python -m collection refresh [--bot PATH ...]
|
||||
Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then
|
||||
rebuild the graph, the route model, the statistics and the build.
|
||||
python -m collection export-bot-kit
|
||||
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, solver.js and graph.js for the bot.
|
||||
python -m collection build [--check]
|
||||
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
||||
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
||||
@@ -46,7 +51,9 @@ def update_source(name, spec, manifest, report, local_file=None, env=None):
|
||||
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
||||
lines = mod.describe(old, new)
|
||||
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
||||
wrote |= L.jsonl_file(f"{target}/trajectories-{ctx['changed']}.jsonl", new["trajectories"])
|
||||
# one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite
|
||||
tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}"
|
||||
wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"])
|
||||
elif target:
|
||||
old = L.read_json(target)
|
||||
lines = mod.describe(old, new)
|
||||
@@ -93,8 +100,20 @@ def main(argv=None):
|
||||
up.add_argument("--report")
|
||||
b = sub.add_parser("build")
|
||||
b.add_argument("--check", action="store_true")
|
||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
|
||||
k.add_argument("--out", default="exports")
|
||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
if a.cmd == "export-bot-kit":
|
||||
from .etl.handoff import export_bot_kit
|
||||
print(f"wrote {export_bot_kit(a.out)}")
|
||||
return 0
|
||||
if a.cmd == "refresh":
|
||||
from .etl.handoff import refresh
|
||||
return refresh(main, a.bot)
|
||||
|
||||
if a.cmd == "build":
|
||||
changed = build(check=a.check)
|
||||
if a.check and changed:
|
||||
|
||||
@@ -9,6 +9,17 @@ Data flows both ways:
|
||||
└──────────── collection --source bot + PR ◄┘
|
||||
```
|
||||
|
||||
## Without GitHub: swapping zips
|
||||
|
||||
When GitHub Pages is not available, the same files travel as zips:
|
||||
|
||||
- **Atlas → bot:** `python -m collection export-bot-kit` writes `exports/atlas-for-bot-<date>-<policy id>.zip`
|
||||
(`policy.json`, `solver.js`, `graph.js`, a README with checksums). Unzip it into the bot's `vendor/threshing-atlas/`.
|
||||
- **Bot → atlas:** zip the bot's `data/runs.jsonl`, `data/knowledge.json` and `logs/encounters.jsonl` (folder names
|
||||
kept; a zip of the whole repo works too) and send it. Put it in the atlas's `inbox/` and run
|
||||
`python -m collection refresh --bot inbox/<the zip>`: it imports the runs (anonymised: no prose, no account names),
|
||||
rebuilds everything, and then `export-bot-kit` makes the new zip to send back.
|
||||
|
||||
## 1. From the atlas to the bot
|
||||
|
||||
### What is published
|
||||
|
||||
@@ -113,6 +113,32 @@ class Bot(unittest.TestCase):
|
||||
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
class Handoff(unittest.TestCase):
|
||||
def test_bot_zip_imports_like_the_folder(self):
|
||||
import zipfile
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["extract"]
|
||||
z = TMP / "ken.zip"
|
||||
with zipfile.ZipFile(z, "w") as f: # files under a top folder, as a zipped repo would have them
|
||||
for rel in spec["files"]:
|
||||
f.write(FX / rel, f"DragonKind-main/{rel}")
|
||||
self.assertEqual(extract.pack_local(spec, z), extract.pack_local(spec, FX))
|
||||
empty = TMP / "empty.zip"
|
||||
with zipfile.ZipFile(empty, "w") as f:
|
||||
f.writestr("notes.txt", "hi")
|
||||
with self.assertRaises(RuntimeError):
|
||||
extract.pack_local(spec, empty)
|
||||
|
||||
def test_bot_kit(self):
|
||||
import zipfile
|
||||
from collection.etl import handoff
|
||||
zpath = handoff.export_bot_kit(str(TMP / "exports"))
|
||||
with zipfile.ZipFile(zpath) as z:
|
||||
self.assertEqual(sorted(z.namelist()), ["README.txt", "graph.js", "policy.json", "solver.js"])
|
||||
self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes())
|
||||
self.assertEqual(json.loads(z.read("policy.json"))["id"],
|
||||
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
||||
|
||||
|
||||
class Sheets(unittest.TestCase):
|
||||
def test_privacy_rules(self):
|
||||
p = TMP / "sheet.csv"
|
||||
|
||||
Reference in New Issue
Block a user