"""Build data/public/experiments.json: the experiment plan the bot follows, updated from the bot's own runs. python planner/build_experiments.py [--out data/public/experiments.json] [--check] Reads the design (planner/experiments.yaml), checks it against the game (core/spec/game.yaml), counts what the bot has logged for every arm (data/bot/trajectories-*.jsonl, matched to data/public/runs_clean.csv), and decides each experiment's status: running still being assigned complete every arm has its planned number of observed rolls paused its stop_early rule was met (the plan needs a person) The bot reads the result with assign() in core/solver.js; an experiment that is not running is no longer assigned and its share goes to the others. The plan has a content id (logged by the bot as planId). """ import argparse import glob import hashlib import itertools import json import random import sys from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "core")) from atlas import model # noqa: E402 from atlas import runs as runs_io # noqa: E402 DESIGN = ROOT / "planner/experiments.yaml" OUT = ROOT / "data/public/experiments.json" SAMPLING = ("best", "thompson", "uniform") def design(): with open(DESIGN, encoding="utf-8") as f: return yaml.safe_load(f) def check_options(where, fixed): for k, o in (fixed or {}).items(): if k not in model.DECISIONS or k in ("encounter", "cadet"): raise ValueError(f"{where}: '{k}' is not a decision of the game") if o not in model.DECISIONS[k]: raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})") def arms(exp): """Every combination of the factors, each with the route held on every arm.""" route = exp.get("route") or {} check_options(exp["id"] + " route", route) factors = exp.get("factors") or {} for k, opts in factors.items(): if k in route: raise ValueError(f"{exp['id']}: '{k}' is both a factor and part of the route") for o in opts: check_options(exp["id"] + " factors", {k: o}) if not factors: return [{"id": "all", "fix": dict(route)}] out = [] for combo in itertools.product(*factors.values()): levels = dict(zip(factors, combo)) out.append({"id": " / ".join(f"{k}={o}" for k, o in levels.items()), "levels": levels, "fix": {**route, **levels}}) return out def validate(d): ids = [e["id"] for e in d["experiments"]] if len(set(ids)) != len(ids): raise ValueError("experiment ids must be unique") for e in d["experiments"]: if e["target"] not in model.COLOURS: raise ValueError(f"{e['id']}: unknown target {e['target']}") if e.get("sampling", "best") not in SAMPLING: raise ValueError(f"{e['id']}: sampling must be one of {SAMPLING}") if e["unit"] not in ("account", "attempt"): raise ValueError(f"{e['id']}: unit must be account or attempt") if not e.get("share", 0) > 0: raise ValueError(f"{e['id']}: share must be positive") m = e.get("measures") if m and (m["roll"] not in model.CHANCE or m["rare"] not in model.CHANCE[m["roll"]]["outcomes"]): raise ValueError(f"{e['id']}: measures {m} is not a roll and outcome of the game") if not e.get("ongoing") and not e.get("planned_per_arm"): raise ValueError(f"{e['id']}: needs planned_per_arm (or ongoing: true)") def interval(y, n, seed=1, sims=4000): """Jeffreys 90% range for a rate.""" if not n: return None r = random.Random(seed) s = sorted(r.betavariate(y + 0.5, n - y + 0.5) for _ in range(sims)) return {"rate": (y + 0.5) / (n + 1), "lo": s[int(0.05 * sims)], "hi": s[int(0.95 * sims) - 1]} def logged(bot_dir=ROOT / "data/bot", runs_path=None): """Bot runs that carry an experiment: [(experimentId, arm, prepared run)], each run once.""" prepared = {r["run_id"]: model.prepare_run(r) for r in runs_io.read(runs_path or runs_io.PATH, sources=["bot"], drop=())} seen, out = set(), [] for f in sorted(glob.glob(str(Path(bot_dir) / "trajectories-*.jsonl"))): if "sample" in Path(f).name: continue with open(f, encoding="utf-8") as fh: for line in fh: t = json.loads(line) meta = t.get("meta") or {} if not meta.get("experimentId") or t["id"] in seen: continue seen.add(t["id"]) out.append((meta["experimentId"], meta.get("arm"), prepared.get(t["id"]))) return out def build(d=None, log=None): d = d or design() validate(d) log = logged() if log is None else log exps = [] for e in d["experiments"]: m = e.get("measures") a = arms(e) tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a} for exp_id, arm, v in log: if exp_id != e["id"]: continue t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0}) t["runs"] += 1 y = model.OBSERVE[m["roll"]](v) if m and v else None if y is not None: t["observed"] += 1 t["rare"] += y == m["rare"] planned = e.get("planned_per_arm") # mapping has nothing to measure: its runs are its count done = {k: (t["observed"] if m else t["runs"]) for k, t in tally.items() if k in {x["id"] for x in a}} total_obs = sum(t["observed"] for t in tally.values()) total_rare = sum(t["rare"] for t in tally.values()) status, note = "running", None se = e.get("stop_early") if not e.get("ongoing") and all(n >= planned for n in done.values()): status, note = "complete", f"every arm has its planned {planned}" elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]: status = "paused" note = f"stop_early: {total_rare} rare outcome(s) in {total_obs} observed rolls (rule: at most {se['rare_at_most']} after {se['after']})" for x in a: t = tally[x["id"]] x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})} exps.append({ "id": e["id"], "status": status, **({"note": note} if note else {}), "question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")}, "unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}), **({"exclusive": True} if e.get("exclusive") else {}), "target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"), "seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a, "progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare, **({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}), "unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})}, }) plan = {"schema": 1, "seed": d["seed"], "lifecycle": d["lifecycle"], "experiments": exps} plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12] return plan def render(plan): return json.dumps(plan, indent=1, ensure_ascii=False) + "\n" def summary(plan): L = [f"experiment plan {plan['id']}"] for e in plan["experiments"]: p = e["progress"] est = p.get("estimate") L.append(f" {e['id']:<22} {e['status']:<9} {p['observed']} observed, {p['rare']} rare" + (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "") + (f" [{e['note']}]" if e.get("note") else "")) return "\n".join(L) def main(argv=None): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--out", default=str(OUT)) ap.add_argument("--check", action="store_true", help="write nothing; exit 1 if the file is out of date") a = ap.parse_args(argv) plan = build() text = render(plan) out = Path(a.out) if a.check: if not out.exists() or out.read_text(encoding="utf-8") != text: print(f"{out} is out of date (run python planner/build_experiments.py)", file=sys.stderr) return 1 return 0 out.write_text(text, encoding="utf-8") print(summary(plan)) return 0 if __name__ == "__main__": sys.exit(main())