"""Build data/public/experiments.json: the experiment plan the bot follows, updated from the bot's own runs. python planner/build_experiments.py [--out data/public/experiments.json] [--check] Reads the design (planner/experiments.yaml), checks it against the game (core/spec/game.yaml), counts what the bot has logged for every arm (data/bot/trajectories-*.jsonl, matched to data/public/runs_clean.csv), and decides each experiment's status: running still being assigned complete every arm has its planned number of observed rolls paused its stop_early rule was met (the plan needs a person) The bot reads the result with assign() in core/solver.js; an experiment that is not running is no longer assigned and its share goes to the others. The plan has a content id (logged by the bot as planId). """ import argparse import glob import hashlib import itertools import json import random import re import sys from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "core")) from atlas import model # noqa: E402 from atlas import runs as runs_io # noqa: E402 DESIGN = ROOT / "planner/experiments.yaml" OUT = ROOT / "data/public/experiments.json" SAMPLING = ("best", "thompson", "uniform") def design(): with open(DESIGN, encoding="utf-8") as f: return yaml.safe_load(f) def check_options(where, fixed): for k, o in (fixed or {}).items(): if k not in model.DECISIONS or k in ("encounter", "cadet"): raise ValueError(f"{where}: '{k}' is not a decision of the game") if o not in model.DECISIONS[k]: raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})") def arms(exp, routes=None): """The arms of an experiment, each with the decisions it holds (`fix`): every combination of the factors, or the explicit `arms` (one `change` to the route each), or a single arm. A `cycle` arm also carries the decisions it varies from attempt to attempt.""" route = exp.get("route") or {} if isinstance(route, str): if route not in (routes or {}): raise ValueError(f"{exp['id']}: unknown route '{route}'") route = routes[route] check_options(exp["id"] + " route", route) if exp.get("arms"): out = [] for a in exp["arms"]: check_options(f"{exp['id']} arm {a['id']}", a.get("change")) out.append({"id": a["id"], "change": a.get("change") or {}, "fix": {**route, **(a.get("change") or {})}}) if len({a["id"] for a in out}) != len(out): raise ValueError(f"{exp['id']}: arm ids must be unique") return out if exp.get("cycle"): for k, opts in exp["cycle"].items(): if k in route: raise ValueError(f"{exp['id']}: '{k}' is both cycled and part of the route") for o in opts: check_options(exp["id"] + " cycle", {k: o}) return [{"id": "all", "fix": dict(route), "cycle": exp["cycle"]}] factors = exp.get("factors") or {} for k, opts in factors.items(): if k in route: raise ValueError(f"{exp['id']}: '{k}' is both a factor and part of the route") for o in opts: check_options(exp["id"] + " factors", {k: o}) if not factors: return [{"id": "all", "fix": dict(route)}] out = [] for combo in itertools.product(*factors.values()): levels = dict(zip(factors, combo)) out.append({"id": " / ".join(f"{k}={o}" for k, o in levels.items()), "levels": levels, "fix": {**route, **levels}}) return out def validate(d): ids = [e["id"] for e in d["experiments"]] if len(set(ids)) != len(ids): raise ValueError("experiment ids must be unique") for e in d["experiments"]: if e.get("starts_after") and e["starts_after"] not in ids: raise ValueError(f"{e['id']}: starts_after names no experiment") for k in (e.get("when") or {}): if k not in (d.get("record") or []): raise ValueError(f"{e['id']}: when.{k} is not something the bot records (`record`)") if e["target"] not in model.COLOURS: raise ValueError(f"{e['id']}: unknown target {e['target']}") if e.get("sampling", "best") not in SAMPLING: raise ValueError(f"{e['id']}: sampling must be one of {SAMPLING}") if e["unit"] not in ("account", "attempt"): raise ValueError(f"{e['id']}: unit must be account or attempt") if not e.get("share", 0) > 0: raise ValueError(f"{e['id']}: share must be positive") m = e.get("measures") if m and (m["roll"] not in model.CHANCE or m["rare"] not in model.CHANCE[m["roll"]]["outcomes"]): raise ValueError(f"{e['id']}: measures {m} is not a roll and outcome of the game") if not e.get("ongoing") and not e.get("planned_per_arm"): raise ValueError(f"{e['id']}: needs planned_per_arm (or ongoing: true)") def interval(y, n, seed=1, sims=4000): """Jeffreys 90% range for a rate.""" if not n: return None r = random.Random(seed) s = sorted(r.betavariate(y + 0.5, n - y + 0.5) for _ in range(sims)) return {"rate": (y + 0.5) / (n + 1), "lo": s[int(0.05 * sims)], "hi": s[int(0.95 * sims) - 1]} def logged(bot_dir=ROOT / "data/bot", runs_path=None): """Bot runs that carry an experiment: [(experimentId, arm, prepared run)], each run once.""" prepared = {r["run_id"]: model.prepare_run(r) for r in runs_io.read(runs_path or runs_io.PATH, sources=["bot"], drop=())} seen, out = set(), [] for f in sorted(glob.glob(str(Path(bot_dir) / "trajectories-*.jsonl"))): if "sample" in Path(f).name: continue with open(f, encoding="utf-8") as fh: for line in fh: t = json.loads(line) meta = t.get("meta") or {} if not meta.get("experimentId") or t["id"] in seen: continue seen.add(t["id"]) out.append((meta["experimentId"], meta.get("arm"), prepared.get(t["id"]))) return out def build(d=None, log=None): d = d or design() validate(d) log = logged() if log is None else log exps, status_of = [], {} for e in d["experiments"]: m = e.get("measures") a = arms(e, d.get("routes")) tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a} for exp_id, arm, v in log: if exp_id != e["id"]: continue if a[0].get("cycle"): # a cycled arm is logged under the combination it played: all one arm here arm = a[0]["id"] t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0}) t["runs"] += 1 y = model.OBSERVE[m["roll"]](v) if m and v else None if y is not None: t["observed"] += 1 t["rare"] += y == m["rare"] planned = e.get("planned_per_arm") # mapping has nothing to measure: its runs are its count done = {k: (t["observed"] if m else t["runs"]) for k, t in tally.items() if k in {x["id"] for x in a}} total_obs = sum(t["observed"] for t in tally.values()) total_rare = sum(t["rare"] for t in tally.values()) status, note = "running", None se = e.get("stop_early") if e.get("starts_after") and status_of.get(e["starts_after"]) != "complete": status, note = "waiting", f"starts when {e['starts_after']} is complete" elif not e.get("ongoing") and all(n >= planned for n in done.values()): status, note = "complete", f"every arm has its planned {planned}" elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]: status = "paused" note = f"stop_early: {total_rare} rare outcome(s) in {total_obs} observed rolls (rule: at most {se['rare_at_most']} after {se['after']})" for x in a: t = tally[x["id"]] x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})} status_of[e["id"]] = status exps.append({ "id": e["id"], "status": status, **({"note": note} if note else {}), "question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")}, "unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}), **({"exclusive": True} if e.get("exclusive") else {}), **({"when": e["when"]} if e.get("when") else {}), "target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"), "seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a, "progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare, **({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}), "unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})}, }) for s in (d.get("scripts") or {}).values(): for st in s: for k in ("take", "never"): re.compile(st.get(k) or "") plan = {"schema": 2, "seed": d["seed"], "lifecycle": d["lifecycle"], "record": d.get("record") or [], "scripts": d.get("scripts") or {}, "experiments": exps} plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12] return plan def render(plan): return json.dumps(plan, indent=1, ensure_ascii=False) + "\n" def summary(plan): L = [f"experiment plan {plan['id']}"] for e in plan["experiments"]: p = e["progress"] est = p.get("estimate") L.append(f" {e['id']:<26} {e['status']:<9} {p['observed']} observed, {p['rare']} rare" + (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "") + (f" [{e['note']}]" if e.get("note") else "")) return "\n".join(L) def main(argv=None): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--out", default=str(OUT)) ap.add_argument("--check", action="store_true", help="write nothing; exit 1 if the file is out of date") a = ap.parse_args(argv) plan = build() text = render(plan) out = Path(a.out) if a.check: if not out.exists() or out.read_text(encoding="utf-8") != text: print(f"{out} is out of date (run python planner/build_experiments.py)", file=sys.stderr) return 1 return 0 out.write_text(text, encoding="utf-8") print(summary(plan)) return 0 if __name__ == "__main__": sys.exit(main())