Hand-offs without GitHub: bot kit zip, bot zip import, local refresh
- python -m collection export-bot-kit: policy.json, solver.js, graph.js and a README with checksums, zipped for the bot - the bot import takes a folder or a zip (files may sit under any top folder) - python -m collection refresh: the daily workflow, run locally - trajectories files are named by date and raw checksum, so two imports on one day never overwrite each other Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
19209a2ad6
commit
5c49591dae
@@ -235,3 +235,6 @@ data/raw/
|
|||||||
|
|
||||||
# QA screenshots
|
# QA screenshots
|
||||||
undefined/
|
undefined/
|
||||||
|
# hand-off zips and local reports (python -m collection export-bot-kit / refresh)
|
||||||
|
exports/
|
||||||
|
inbox/
|
||||||
|
|||||||
@@ -73,6 +73,12 @@ This writes `data/bot/scenes.json` and `data/bot/trajectories-<date>.jsonl`: sce
|
|||||||
bot's attempts to `runs_clean.csv` (source `bot`). The importer refuses to write anything that looks like prose.
|
bot's attempts to `runs_clean.csv` (source `bot`). The importer refuses to write anything that looks like prose.
|
||||||
See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
|
See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
|
||||||
|
|
||||||
|
## Running everything locally
|
||||||
|
|
||||||
|
`python -m collection refresh` does what the daily GitHub workflow does: checks every source, rebuilds the runs,
|
||||||
|
the graph, the route model and the statistics. Add `--bot <folder or zip>` to import bot runs, and
|
||||||
|
`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`).
|
||||||
|
|
||||||
## Updating with new form data by hand
|
## Updating with new form data by hand
|
||||||
|
|
||||||
Download the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel), then:
|
Download the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel), then:
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import shutil
|
|||||||
import tempfile
|
import tempfile
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
import zipfile
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -92,6 +93,24 @@ def local_dir(spec, env):
|
|||||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def pack_local(spec, src):
|
||||||
|
"""The bot's files from a folder (a copy of the bot repo) or a .zip someone sent (the files may sit under any
|
||||||
|
top folder inside it), packed as one JSON like local_dir."""
|
||||||
|
src = Path(src).expanduser()
|
||||||
|
if src.is_dir():
|
||||||
|
return local_dir(spec, {spec["env"]: str(src)})
|
||||||
|
files = {rel: None for rel in spec["files"]}
|
||||||
|
with zipfile.ZipFile(src) as z:
|
||||||
|
names = [n for n in z.namelist() if not n.endswith("/") and "__MACOSX" not in n]
|
||||||
|
for rel in spec["files"]:
|
||||||
|
hits = sorted((n for n in names if n == rel or n.endswith("/" + rel)), key=len)
|
||||||
|
if hits:
|
||||||
|
files[rel] = z.read(hits[0]).decode("utf-8")
|
||||||
|
if not any(files.values()):
|
||||||
|
raise RuntimeError(f"{src.name}: none of {', '.join(spec['files'])} found inside")
|
||||||
|
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
||||||
|
|
||||||
|
|
||||||
@@ -127,6 +146,12 @@ def extract(name, spec, manifest, env=None, local_file=None):
|
|||||||
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
|
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
|
||||||
env = os.environ if env is None else env
|
env = os.environ if env is None else env
|
||||||
ext = spec["extract"].get("ext") or "json"
|
ext = spec["extract"].get("ext") or "json"
|
||||||
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env)
|
kind = spec["extract"]["kind"]
|
||||||
|
if local_file and kind == "local_dir":
|
||||||
|
data = pack_local(spec["extract"], local_file) # a bot folder or a .zip someone sent
|
||||||
|
elif local_file:
|
||||||
|
data = Path(local_file).expanduser().read_bytes()
|
||||||
|
else:
|
||||||
|
data = EXTRACTORS[kind](spec["extract"], env)
|
||||||
changed = store(name, ext, data, manifest)
|
changed = store(name, ext, data, manifest)
|
||||||
return raw_paths(name, ext)[0], changed
|
return raw_paths(name, ext)[0], changed
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
"""Hand-offs without GitHub: the zip of atlas files for the bot, and the local equivalent of the daily workflow.
|
||||||
|
|
||||||
|
python -m collection export-bot-kit -> exports/atlas-for-bot-<date>-<policy id>.zip
|
||||||
|
policy.json, solver.js and graph.js (what the bot vendors into vendor/threshing-atlas/), plus a README and
|
||||||
|
checksums. Send it to the bot's maintainer; they unzip it into vendor/threshing-atlas/.
|
||||||
|
python -m collection refresh [--bot PATH ...]
|
||||||
|
Every source (and any bot folders or zips given), then the graph, the route model, the statistics and the
|
||||||
|
build: what .github/workflows/update-sources.yml does every day.
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from .common import ROOT, path
|
||||||
|
|
||||||
|
KIT = {"policy.json": "data/public/policy.json", "solver.js": "core/solver.js", "graph.js": "core/graph.js"}
|
||||||
|
|
||||||
|
README = """Threshing Atlas files for the DragonKind bot
|
||||||
|
============================================
|
||||||
|
|
||||||
|
Policy id: {policy}
|
||||||
|
Made: {made}
|
||||||
|
|
||||||
|
Unzip these three files into the bot's vendor/threshing-atlas/ folder, replacing the old ones, and keep them
|
||||||
|
together: solver.js reads the game from this policy.json.
|
||||||
|
|
||||||
|
policy.json the route model (the game, the counts behind every roll, the best routes per colour)
|
||||||
|
solver.js the advisor: advisor(), recommendOption(), solverHistory()
|
||||||
|
graph.js trajectoryVariables(): the bot's steps -> the community form's variables
|
||||||
|
|
||||||
|
How the bot uses them: docs/BOT_INTEGRATION.md in the atlas repo. Log the policy id above as policyId on each run.
|
||||||
|
|
||||||
|
Checksums (sha256):
|
||||||
|
{sums}
|
||||||
|
|
||||||
|
To send runs back: zip the bot's data/runs.jsonl, data/knowledge.json and logs/encounters.jsonl (keep the folder
|
||||||
|
names) and send the zip. It is imported with `python -m collection refresh --bot <the zip>`; only anonymised
|
||||||
|
steps and outcomes are kept (no game prose, no account names).
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def export_bot_kit(out_dir="exports"):
|
||||||
|
policy = json.loads(path(KIT["policy.json"]).read_text(encoding="utf-8"))
|
||||||
|
pid = str(policy.get("id", "unknown"))[:12]
|
||||||
|
made = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
sums = "\n".join(f" {hashlib.sha256(path(src).read_bytes()).hexdigest()} {name}" for name, src in KIT.items())
|
||||||
|
out = path(out_dir)
|
||||||
|
out.mkdir(parents=True, exist_ok=True)
|
||||||
|
zpath = out / f"atlas-for-bot-{made}-{pid}.zip"
|
||||||
|
with zipfile.ZipFile(zpath, "w", zipfile.ZIP_DEFLATED) as z:
|
||||||
|
for name, src in KIT.items():
|
||||||
|
z.writestr(zipfile.ZipInfo(name, (2026, 1, 1, 0, 0, 0)), path(src).read_bytes(), zipfile.ZIP_DEFLATED)
|
||||||
|
z.writestr(zipfile.ZipInfo("README.txt", (2026, 1, 1, 0, 0, 0)), README.format(policy=pid, made=made, sums=sums))
|
||||||
|
return zpath
|
||||||
|
|
||||||
|
|
||||||
|
def _step(label, cmd):
|
||||||
|
print(f"\n== {label}: {' '.join(cmd)}", flush=True)
|
||||||
|
subprocess.run(cmd, cwd=ROOT, check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def refresh(update_main, bots=()):
|
||||||
|
"""update -> (graph, policy, stats, build) when something changed. Returns 0 on success."""
|
||||||
|
py = sys.executable
|
||||||
|
args = ["update", "--report", str(path("exports/last-report.md"))]
|
||||||
|
path("exports").mkdir(exist_ok=True)
|
||||||
|
rc = update_main(args)
|
||||||
|
for i, b in enumerate(bots):
|
||||||
|
rc |= update_main(["update", "--source", "bot", "--file", f"bot={b}", "--report", str(path(f"exports/last-report-bot{i + 1}.md"))])
|
||||||
|
if shutil.which("node"):
|
||||||
|
_step("route graph", ["node", "analysis/network/build-graph.js"])
|
||||||
|
else:
|
||||||
|
print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)")
|
||||||
|
_step("route model (about 2 minutes)", [py, "planner/build_policy.py"])
|
||||||
|
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
||||||
|
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
|
||||||
|
print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit")
|
||||||
|
return rc
|
||||||
+20
-1
@@ -4,6 +4,11 @@
|
|||||||
For each source (all by default): extract it (or take a local file), and when the raw file changed,
|
For each source (all by default): extract it (or take a local file), and when the raw file changed,
|
||||||
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
|
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
|
||||||
reached: the report says what happened (and the daily workflow opens an issue).
|
reached: the report says what happened (and the daily workflow opens an issue).
|
||||||
|
python -m collection refresh [--bot PATH ...]
|
||||||
|
Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then
|
||||||
|
rebuild the graph, the route model, the statistics and the build.
|
||||||
|
python -m collection export-bot-kit
|
||||||
|
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, solver.js and graph.js for the bot.
|
||||||
python -m collection build [--check]
|
python -m collection build [--check]
|
||||||
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
||||||
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
||||||
@@ -46,7 +51,9 @@ def update_source(name, spec, manifest, report, local_file=None, env=None):
|
|||||||
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
||||||
lines = mod.describe(old, new)
|
lines = mod.describe(old, new)
|
||||||
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
||||||
wrote |= L.jsonl_file(f"{target}/trajectories-{ctx['changed']}.jsonl", new["trajectories"])
|
# one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite
|
||||||
|
tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}"
|
||||||
|
wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"])
|
||||||
elif target:
|
elif target:
|
||||||
old = L.read_json(target)
|
old = L.read_json(target)
|
||||||
lines = mod.describe(old, new)
|
lines = mod.describe(old, new)
|
||||||
@@ -93,8 +100,20 @@ def main(argv=None):
|
|||||||
up.add_argument("--report")
|
up.add_argument("--report")
|
||||||
b = sub.add_parser("build")
|
b = sub.add_parser("build")
|
||||||
b.add_argument("--check", action="store_true")
|
b.add_argument("--check", action="store_true")
|
||||||
|
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
|
||||||
|
k.add_argument("--out", default="exports")
|
||||||
|
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||||
|
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||||
a = ap.parse_args(argv)
|
a = ap.parse_args(argv)
|
||||||
|
|
||||||
|
if a.cmd == "export-bot-kit":
|
||||||
|
from .etl.handoff import export_bot_kit
|
||||||
|
print(f"wrote {export_bot_kit(a.out)}")
|
||||||
|
return 0
|
||||||
|
if a.cmd == "refresh":
|
||||||
|
from .etl.handoff import refresh
|
||||||
|
return refresh(main, a.bot)
|
||||||
|
|
||||||
if a.cmd == "build":
|
if a.cmd == "build":
|
||||||
changed = build(check=a.check)
|
changed = build(check=a.check)
|
||||||
if a.check and changed:
|
if a.check and changed:
|
||||||
|
|||||||
@@ -9,6 +9,17 @@ Data flows both ways:
|
|||||||
└──────────── collection --source bot + PR ◄┘
|
└──────────── collection --source bot + PR ◄┘
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Without GitHub: swapping zips
|
||||||
|
|
||||||
|
When GitHub Pages is not available, the same files travel as zips:
|
||||||
|
|
||||||
|
- **Atlas → bot:** `python -m collection export-bot-kit` writes `exports/atlas-for-bot-<date>-<policy id>.zip`
|
||||||
|
(`policy.json`, `solver.js`, `graph.js`, a README with checksums). Unzip it into the bot's `vendor/threshing-atlas/`.
|
||||||
|
- **Bot → atlas:** zip the bot's `data/runs.jsonl`, `data/knowledge.json` and `logs/encounters.jsonl` (folder names
|
||||||
|
kept; a zip of the whole repo works too) and send it. Put it in the atlas's `inbox/` and run
|
||||||
|
`python -m collection refresh --bot inbox/<the zip>`: it imports the runs (anonymised: no prose, no account names),
|
||||||
|
rebuilds everything, and then `export-bot-kit` makes the new zip to send back.
|
||||||
|
|
||||||
## 1. From the atlas to the bot
|
## 1. From the atlas to the bot
|
||||||
|
|
||||||
### What is published
|
### What is published
|
||||||
|
|||||||
@@ -113,6 +113,32 @@ class Bot(unittest.TestCase):
|
|||||||
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
|
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
class Handoff(unittest.TestCase):
|
||||||
|
def test_bot_zip_imports_like_the_folder(self):
|
||||||
|
import zipfile
|
||||||
|
spec = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["extract"]
|
||||||
|
z = TMP / "ken.zip"
|
||||||
|
with zipfile.ZipFile(z, "w") as f: # files under a top folder, as a zipped repo would have them
|
||||||
|
for rel in spec["files"]:
|
||||||
|
f.write(FX / rel, f"DragonKind-main/{rel}")
|
||||||
|
self.assertEqual(extract.pack_local(spec, z), extract.pack_local(spec, FX))
|
||||||
|
empty = TMP / "empty.zip"
|
||||||
|
with zipfile.ZipFile(empty, "w") as f:
|
||||||
|
f.writestr("notes.txt", "hi")
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
extract.pack_local(spec, empty)
|
||||||
|
|
||||||
|
def test_bot_kit(self):
|
||||||
|
import zipfile
|
||||||
|
from collection.etl import handoff
|
||||||
|
zpath = handoff.export_bot_kit(str(TMP / "exports"))
|
||||||
|
with zipfile.ZipFile(zpath) as z:
|
||||||
|
self.assertEqual(sorted(z.namelist()), ["README.txt", "graph.js", "policy.json", "solver.js"])
|
||||||
|
self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes())
|
||||||
|
self.assertEqual(json.loads(z.read("policy.json"))["id"],
|
||||||
|
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
||||||
|
|
||||||
|
|
||||||
class Sheets(unittest.TestCase):
|
class Sheets(unittest.TestCase):
|
||||||
def test_privacy_rules(self):
|
def test_privacy_rules(self):
|
||||||
p = TMP / "sheet.csv"
|
p = TMP / "sheet.csv"
|
||||||
|
|||||||
Reference in New Issue
Block a user