CI / test (pull_request) Canceled after 0s
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen, every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree. - The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and graph.js now use the played steps only. - runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only rows, flagged (no analysis uses them). - Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
238 lines
11 KiB
Python
238 lines
11 KiB
Python
"""Build data/public/experiments.json: the experiment plan the bot follows, updated from the bot's own runs.
|
|
|
|
python planner/build_experiments.py [--out data/public/experiments.json] [--check]
|
|
|
|
Reads the design (planner/experiments.yaml), checks it against the game (core/spec/game.yaml), counts what the bot
|
|
has logged for every arm (data/bot/trajectories-*.jsonl, matched to data/public/runs_clean.csv), and decides each
|
|
experiment's status:
|
|
|
|
running still being assigned
|
|
complete every arm has its planned number of observed rolls
|
|
paused its stop_early rule was met (the plan needs a person)
|
|
|
|
The bot reads the result with assign() in core/solver.js; an experiment that is not running is no longer assigned
|
|
and its share goes to the others. The plan has a content id (logged by the bot as planId).
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import random
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "core"))
|
|
from atlas import model # noqa: E402
|
|
from atlas import runs as runs_io # noqa: E402
|
|
|
|
DESIGN = ROOT / "planner/experiments.yaml"
|
|
OUT = ROOT / "data/public/experiments.json"
|
|
SAMPLING = ("best", "thompson", "uniform")
|
|
|
|
|
|
def design():
|
|
with open(DESIGN, encoding="utf-8") as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def check_options(where, fixed):
|
|
for k, o in (fixed or {}).items():
|
|
if k not in model.DECISIONS or k in ("encounter", "cadet"):
|
|
raise ValueError(f"{where}: '{k}' is not a decision of the game")
|
|
if o not in model.DECISIONS[k]:
|
|
raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})")
|
|
|
|
|
|
def arms(exp, routes=None):
|
|
"""The arms of an experiment, each with the decisions it holds (`fix`): every combination of the factors, or
|
|
the explicit `arms` (one `change` to the route each), or a single arm. A `cycle` arm also carries the decisions
|
|
it varies from attempt to attempt."""
|
|
route = exp.get("route") or {}
|
|
if isinstance(route, str):
|
|
if route not in (routes or {}):
|
|
raise ValueError(f"{exp['id']}: unknown route '{route}'")
|
|
route = routes[route]
|
|
check_options(exp["id"] + " route", route)
|
|
if exp.get("arms"):
|
|
out = []
|
|
for a in exp["arms"]:
|
|
check_options(f"{exp['id']} arm {a['id']}", a.get("change"))
|
|
out.append({"id": a["id"], "change": a.get("change") or {}, "fix": {**route, **(a.get("change") or {})}})
|
|
if len({a["id"] for a in out}) != len(out):
|
|
raise ValueError(f"{exp['id']}: arm ids must be unique")
|
|
return out
|
|
if exp.get("cycle"):
|
|
for k, opts in exp["cycle"].items():
|
|
if k in route:
|
|
raise ValueError(f"{exp['id']}: '{k}' is both cycled and part of the route")
|
|
for o in opts:
|
|
check_options(exp["id"] + " cycle", {k: o})
|
|
return [{"id": "all", "fix": dict(route), "cycle": exp["cycle"]}]
|
|
factors = exp.get("factors") or {}
|
|
for k, opts in factors.items():
|
|
if k in route:
|
|
raise ValueError(f"{exp['id']}: '{k}' is both a factor and part of the route")
|
|
for o in opts:
|
|
check_options(exp["id"] + " factors", {k: o})
|
|
if not factors:
|
|
return [{"id": "all", "fix": dict(route)}]
|
|
out = []
|
|
for combo in itertools.product(*factors.values()):
|
|
levels = dict(zip(factors, combo))
|
|
out.append({"id": " / ".join(f"{k}={o}" for k, o in levels.items()), "levels": levels, "fix": {**route, **levels}})
|
|
return out
|
|
|
|
|
|
def validate(d):
|
|
ids = [e["id"] for e in d["experiments"]]
|
|
if len(set(ids)) != len(ids):
|
|
raise ValueError("experiment ids must be unique")
|
|
for e in d["experiments"]:
|
|
if e.get("starts_after") and e["starts_after"] not in ids:
|
|
raise ValueError(f"{e['id']}: starts_after names no experiment")
|
|
for k in (e.get("when") or {}):
|
|
if k not in (d.get("record") or []):
|
|
raise ValueError(f"{e['id']}: when.{k} is not something the bot records (`record`)")
|
|
if e["target"] not in model.COLOURS:
|
|
raise ValueError(f"{e['id']}: unknown target {e['target']}")
|
|
if e.get("sampling", "best") not in SAMPLING:
|
|
raise ValueError(f"{e['id']}: sampling must be one of {SAMPLING}")
|
|
if e["unit"] not in ("account", "attempt"):
|
|
raise ValueError(f"{e['id']}: unit must be account or attempt")
|
|
if not e.get("share", 0) > 0:
|
|
raise ValueError(f"{e['id']}: share must be positive")
|
|
m = e.get("measures")
|
|
if m and (m["roll"] not in model.CHANCE or m["rare"] not in model.CHANCE[m["roll"]]["outcomes"]):
|
|
raise ValueError(f"{e['id']}: measures {m} is not a roll and outcome of the game")
|
|
if not e.get("ongoing") and not e.get("planned_per_arm"):
|
|
raise ValueError(f"{e['id']}: needs planned_per_arm (or ongoing: true)")
|
|
|
|
|
|
def interval(y, n, seed=1, sims=4000):
|
|
"""Jeffreys 90% range for a rate."""
|
|
if not n:
|
|
return None
|
|
r = random.Random(seed)
|
|
s = sorted(r.betavariate(y + 0.5, n - y + 0.5) for _ in range(sims))
|
|
return {"rate": (y + 0.5) / (n + 1), "lo": s[int(0.05 * sims)], "hi": s[int(0.95 * sims) - 1]}
|
|
|
|
|
|
def logged(bot_dir=ROOT / "data/bot", runs_path=None):
|
|
"""Bot runs that carry an experiment: [(experimentId, arm, prepared run)], each run once."""
|
|
prepared = {r["run_id"]: model.prepare_run(r) for r in runs_io.read(runs_path or runs_io.PATH, sources=["bot"], drop=())}
|
|
seen, out = set(), []
|
|
for f in sorted(glob.glob(str(Path(bot_dir) / "trajectories-*.jsonl"))):
|
|
if "sample" in Path(f).name:
|
|
continue
|
|
with open(f, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
t = json.loads(line)
|
|
meta = t.get("meta") or {}
|
|
if not meta.get("experimentId") or t["id"] in seen:
|
|
continue
|
|
seen.add(t["id"])
|
|
out.append((meta["experimentId"], meta.get("arm"), prepared.get(t["id"])))
|
|
return out
|
|
|
|
|
|
def build(d=None, log=None):
|
|
d = d or design()
|
|
validate(d)
|
|
log = logged() if log is None else log
|
|
exps, status_of = [], {}
|
|
for e in d["experiments"]:
|
|
m = e.get("measures")
|
|
a = arms(e, d.get("routes"))
|
|
tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a}
|
|
for exp_id, arm, v in log:
|
|
if exp_id != e["id"]:
|
|
continue
|
|
if a[0].get("cycle"): # a cycled arm is logged under the combination it played: all one arm here
|
|
arm = a[0]["id"]
|
|
t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0})
|
|
t["runs"] += 1
|
|
y = model.OBSERVE[m["roll"]](v) if m and v else None
|
|
if y is not None:
|
|
t["observed"] += 1
|
|
t["rare"] += y == m["rare"]
|
|
planned = e.get("planned_per_arm")
|
|
# mapping has nothing to measure: its runs are its count
|
|
done = {k: (t["observed"] if m else t["runs"]) for k, t in tally.items() if k in {x["id"] for x in a}}
|
|
total_obs = sum(t["observed"] for t in tally.values())
|
|
total_rare = sum(t["rare"] for t in tally.values())
|
|
status, note = "running", None
|
|
se = e.get("stop_early")
|
|
if e.get("starts_after") and status_of.get(e["starts_after"]) != "complete":
|
|
status, note = "waiting", f"starts when {e['starts_after']} is complete"
|
|
elif not e.get("ongoing") and all(n >= planned for n in done.values()):
|
|
status, note = "complete", f"every arm has its planned {planned}"
|
|
elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]:
|
|
status = "paused"
|
|
note = f"stop_early: {total_rare} rare outcome(s) in {total_obs} observed rolls (rule: at most {se['rare_at_most']} after {se['after']})"
|
|
for x in a:
|
|
t = tally[x["id"]]
|
|
x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})}
|
|
status_of[e["id"]] = status
|
|
exps.append({
|
|
"id": e["id"], "status": status, **({"note": note} if note else {}),
|
|
"question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")},
|
|
"unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}),
|
|
**({"exclusive": True} if e.get("exclusive") else {}),
|
|
**({"when": e["when"]} if e.get("when") else {}),
|
|
"target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"),
|
|
"seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a,
|
|
"progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare,
|
|
**({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}),
|
|
"unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})},
|
|
})
|
|
for s in (d.get("scripts") or {}).values():
|
|
for st in s:
|
|
for k in ("take", "never"):
|
|
re.compile(st.get(k) or "")
|
|
plan = {"schema": 2, "seed": d["seed"], "lifecycle": d["lifecycle"], "record": d.get("record") or [],
|
|
"scripts": d.get("scripts") or {}, "screenPicks": d.get("screen_picks") or [], "experiments": exps}
|
|
plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12]
|
|
return plan
|
|
|
|
|
|
def render(plan):
|
|
return json.dumps(plan, indent=1, ensure_ascii=False) + "\n"
|
|
|
|
|
|
def summary(plan):
|
|
L = [f"experiment plan {plan['id']}"]
|
|
for e in plan["experiments"]:
|
|
p = e["progress"]
|
|
est = p.get("estimate")
|
|
L.append(f" {e['id']:<26} {e['status']:<9} {p['observed']} observed, {p['rare']} rare"
|
|
+ (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "")
|
|
+ (f" [{e['note']}]" if e.get("note") else ""))
|
|
return "\n".join(L)
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--out", default=str(OUT))
|
|
ap.add_argument("--check", action="store_true", help="write nothing; exit 1 if the file is out of date")
|
|
a = ap.parse_args(argv)
|
|
plan = build()
|
|
text = render(plan)
|
|
out = Path(a.out)
|
|
if a.check:
|
|
if not out.exists() or out.read_text(encoding="utf-8") != text:
|
|
print(f"{out} is out of date (run python planner/build_experiments.py)", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
out.write_text(text, encoding="utf-8")
|
|
print(summary(plan))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|