Files
ekcSchianoandClaude Fable 5.2 f8e4f575f4
CI / test (pull_request) Canceled after 0s
Screen tree, register rows in the master list, and bot runs coded from the picks that were played
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen,
  every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where
  any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree.
- The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and
  graph.js now use the played steps only.
- runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only
  rows, flagged (no analysis uses them).
- Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
2026-11-16 15:50:12 +00:00

238 lines
11 KiB
Python

"""Build data/public/experiments.json: the experiment plan the bot follows, updated from the bot's own runs.
python planner/build_experiments.py [--out data/public/experiments.json] [--check]
Reads the design (planner/experiments.yaml), checks it against the game (core/spec/game.yaml), counts what the bot
has logged for every arm (data/bot/trajectories-*.jsonl, matched to data/public/runs_clean.csv), and decides each
experiment's status:
running still being assigned
complete every arm has its planned number of observed rolls
paused its stop_early rule was met (the plan needs a person)
The bot reads the result with assign() in core/solver.js; an experiment that is not running is no longer assigned
and its share goes to the others. The plan has a content id (logged by the bot as planId).
"""
import argparse
import glob
import hashlib
import itertools
import json
import random
import re
import sys
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "core"))
from atlas import model # noqa: E402
from atlas import runs as runs_io # noqa: E402
DESIGN = ROOT / "planner/experiments.yaml"
OUT = ROOT / "data/public/experiments.json"
SAMPLING = ("best", "thompson", "uniform")
def design():
with open(DESIGN, encoding="utf-8") as f:
return yaml.safe_load(f)
def check_options(where, fixed):
for k, o in (fixed or {}).items():
if k not in model.DECISIONS or k in ("encounter", "cadet"):
raise ValueError(f"{where}: '{k}' is not a decision of the game")
if o not in model.DECISIONS[k]:
raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})")
def arms(exp, routes=None):
"""The arms of an experiment, each with the decisions it holds (`fix`): every combination of the factors, or
the explicit `arms` (one `change` to the route each), or a single arm. A `cycle` arm also carries the decisions
it varies from attempt to attempt."""
route = exp.get("route") or {}
if isinstance(route, str):
if route not in (routes or {}):
raise ValueError(f"{exp['id']}: unknown route '{route}'")
route = routes[route]
check_options(exp["id"] + " route", route)
if exp.get("arms"):
out = []
for a in exp["arms"]:
check_options(f"{exp['id']} arm {a['id']}", a.get("change"))
out.append({"id": a["id"], "change": a.get("change") or {}, "fix": {**route, **(a.get("change") or {})}})
if len({a["id"] for a in out}) != len(out):
raise ValueError(f"{exp['id']}: arm ids must be unique")
return out
if exp.get("cycle"):
for k, opts in exp["cycle"].items():
if k in route:
raise ValueError(f"{exp['id']}: '{k}' is both cycled and part of the route")
for o in opts:
check_options(exp["id"] + " cycle", {k: o})
return [{"id": "all", "fix": dict(route), "cycle": exp["cycle"]}]
factors = exp.get("factors") or {}
for k, opts in factors.items():
if k in route:
raise ValueError(f"{exp['id']}: '{k}' is both a factor and part of the route")
for o in opts:
check_options(exp["id"] + " factors", {k: o})
if not factors:
return [{"id": "all", "fix": dict(route)}]
out = []
for combo in itertools.product(*factors.values()):
levels = dict(zip(factors, combo))
out.append({"id": " / ".join(f"{k}={o}" for k, o in levels.items()), "levels": levels, "fix": {**route, **levels}})
return out
def validate(d):
ids = [e["id"] for e in d["experiments"]]
if len(set(ids)) != len(ids):
raise ValueError("experiment ids must be unique")
for e in d["experiments"]:
if e.get("starts_after") and e["starts_after"] not in ids:
raise ValueError(f"{e['id']}: starts_after names no experiment")
for k in (e.get("when") or {}):
if k not in (d.get("record") or []):
raise ValueError(f"{e['id']}: when.{k} is not something the bot records (`record`)")
if e["target"] not in model.COLOURS:
raise ValueError(f"{e['id']}: unknown target {e['target']}")
if e.get("sampling", "best") not in SAMPLING:
raise ValueError(f"{e['id']}: sampling must be one of {SAMPLING}")
if e["unit"] not in ("account", "attempt"):
raise ValueError(f"{e['id']}: unit must be account or attempt")
if not e.get("share", 0) > 0:
raise ValueError(f"{e['id']}: share must be positive")
m = e.get("measures")
if m and (m["roll"] not in model.CHANCE or m["rare"] not in model.CHANCE[m["roll"]]["outcomes"]):
raise ValueError(f"{e['id']}: measures {m} is not a roll and outcome of the game")
if not e.get("ongoing") and not e.get("planned_per_arm"):
raise ValueError(f"{e['id']}: needs planned_per_arm (or ongoing: true)")
def interval(y, n, seed=1, sims=4000):
"""Jeffreys 90% range for a rate."""
if not n:
return None
r = random.Random(seed)
s = sorted(r.betavariate(y + 0.5, n - y + 0.5) for _ in range(sims))
return {"rate": (y + 0.5) / (n + 1), "lo": s[int(0.05 * sims)], "hi": s[int(0.95 * sims) - 1]}
def logged(bot_dir=ROOT / "data/bot", runs_path=None):
"""Bot runs that carry an experiment: [(experimentId, arm, prepared run)], each run once."""
prepared = {r["run_id"]: model.prepare_run(r) for r in runs_io.read(runs_path or runs_io.PATH, sources=["bot"], drop=())}
seen, out = set(), []
for f in sorted(glob.glob(str(Path(bot_dir) / "trajectories-*.jsonl"))):
if "sample" in Path(f).name:
continue
with open(f, encoding="utf-8") as fh:
for line in fh:
t = json.loads(line)
meta = t.get("meta") or {}
if not meta.get("experimentId") or t["id"] in seen:
continue
seen.add(t["id"])
out.append((meta["experimentId"], meta.get("arm"), prepared.get(t["id"])))
return out
def build(d=None, log=None):
d = d or design()
validate(d)
log = logged() if log is None else log
exps, status_of = [], {}
for e in d["experiments"]:
m = e.get("measures")
a = arms(e, d.get("routes"))
tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a}
for exp_id, arm, v in log:
if exp_id != e["id"]:
continue
if a[0].get("cycle"): # a cycled arm is logged under the combination it played: all one arm here
arm = a[0]["id"]
t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0})
t["runs"] += 1
y = model.OBSERVE[m["roll"]](v) if m and v else None
if y is not None:
t["observed"] += 1
t["rare"] += y == m["rare"]
planned = e.get("planned_per_arm")
# mapping has nothing to measure: its runs are its count
done = {k: (t["observed"] if m else t["runs"]) for k, t in tally.items() if k in {x["id"] for x in a}}
total_obs = sum(t["observed"] for t in tally.values())
total_rare = sum(t["rare"] for t in tally.values())
status, note = "running", None
se = e.get("stop_early")
if e.get("starts_after") and status_of.get(e["starts_after"]) != "complete":
status, note = "waiting", f"starts when {e['starts_after']} is complete"
elif not e.get("ongoing") and all(n >= planned for n in done.values()):
status, note = "complete", f"every arm has its planned {planned}"
elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]:
status = "paused"
note = f"stop_early: {total_rare} rare outcome(s) in {total_obs} observed rolls (rule: at most {se['rare_at_most']} after {se['after']})"
for x in a:
t = tally[x["id"]]
x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})}
status_of[e["id"]] = status
exps.append({
"id": e["id"], "status": status, **({"note": note} if note else {}),
"question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")},
"unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}),
**({"exclusive": True} if e.get("exclusive") else {}),
**({"when": e["when"]} if e.get("when") else {}),
"target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"),
"seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a,
"progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare,
**({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}),
"unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})},
})
for s in (d.get("scripts") or {}).values():
for st in s:
for k in ("take", "never"):
re.compile(st.get(k) or "")
plan = {"schema": 2, "seed": d["seed"], "lifecycle": d["lifecycle"], "record": d.get("record") or [],
"scripts": d.get("scripts") or {}, "screenPicks": d.get("screen_picks") or [], "experiments": exps}
plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12]
return plan
def render(plan):
return json.dumps(plan, indent=1, ensure_ascii=False) + "\n"
def summary(plan):
L = [f"experiment plan {plan['id']}"]
for e in plan["experiments"]:
p = e["progress"]
est = p.get("estimate")
L.append(f" {e['id']:<26} {e['status']:<9} {p['observed']} observed, {p['rare']} rare"
+ (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "")
+ (f" [{e['note']}]" if e.get("note") else ""))
return "\n".join(L)
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--out", default=str(OUT))
ap.add_argument("--check", action="store_true", help="write nothing; exit 1 if the file is out of date")
a = ap.parse_args(argv)
plan = build()
text = render(plan)
out = Path(a.out)
if a.check:
if not out.exists() or out.read_text(encoding="utf-8") != text:
print(f"{out} is out of date (run python planner/build_experiments.py)", file=sys.stderr)
return 1
return 0
out.write_text(text, encoding="utf-8")
print(summary(plan))
return 0
if __name__ == "__main__":
sys.exit(main())