Route model: first attempts are their own shadow roll; experiment plan and automated bot cycle
- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
This commit is contained in:
co-authored by
Claude Fable 5.2
parent
ee07ee143e
commit
ab3664f369
@@ -0,0 +1,66 @@
|
||||
"""The experiment plan (planner/experiments.yaml -> data/public/experiments.json)."""
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "planner"))
|
||||
import build_experiments as B # noqa: E402
|
||||
from atlas import model # noqa: E402
|
||||
|
||||
D = B.design()
|
||||
GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "clearing": "Help your squadmate"}
|
||||
|
||||
|
||||
class Plan(unittest.TestCase):
|
||||
def test_design_is_valid_and_arms_are_every_combination(self):
|
||||
plan = B.build(D, log=[])
|
||||
by = {e["id"]: e for e in plan["experiments"]}
|
||||
self.assertEqual(len(by["first-attempt-shadow"]["arms"]), 4)
|
||||
self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6)
|
||||
for e in plan["experiments"]:
|
||||
self.assertEqual(e["status"], "running")
|
||||
for a in e["arms"]:
|
||||
B.check_options(e["id"], a["fix"])
|
||||
# the Black attempt keeps the shadow's gate open on every arm
|
||||
for a in by["first-attempt-shadow"]["arms"]:
|
||||
self.assertTrue(model.gate_open("shadow", a["fix"]))
|
||||
|
||||
def test_an_unknown_option_is_refused(self):
|
||||
d = copy.deepcopy(D)
|
||||
d["experiments"][0]["factors"]["direction"] = ["North"]
|
||||
with self.assertRaises(ValueError):
|
||||
B.build(d, log=[])
|
||||
|
||||
def test_progress_completes_and_stops_early(self):
|
||||
e0 = D["experiments"][0]
|
||||
arms = [a["id"] for a in B.arms(e0)]
|
||||
seen = {**GATED_TREE, "shadow": True}
|
||||
none = {**GATED_TREE, "shadow": False}
|
||||
# every arm at its planned count, with shadows: complete
|
||||
log = [(e0["id"], a, seen if i < 3 else none) for a in arms for i in range(e0["planned_per_arm"])]
|
||||
e = B.build(D, log=log)["experiments"][0]
|
||||
self.assertEqual(e["status"], "complete")
|
||||
self.assertEqual(e["progress"]["rare"], 12)
|
||||
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 61)
|
||||
# no shadow at all after the stop_early count: paused, for a person to look at
|
||||
log = [(e0["id"], arms[i % 4], none) for i in range(e0["stop_early"]["after"])]
|
||||
e = B.build(D, log=log)["experiments"][0]
|
||||
self.assertEqual(e["status"], "paused")
|
||||
# a run whose roll was not observed (died before the tree) is a run, not an observation
|
||||
e = B.build(D, log=[(e0["id"], arms[0], {"attempt": "First attempt"})])["experiments"][0]
|
||||
self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0))
|
||||
|
||||
def test_the_plan_id_follows_its_content(self):
|
||||
a, b = B.build(D, log=[]), B.build(D, log=[])
|
||||
self.assertEqual(a["id"], b["id"])
|
||||
c = B.build(D, log=[(D["experiments"][0]["id"], "x", None)])
|
||||
self.assertNotEqual(a["id"], c["id"])
|
||||
|
||||
def test_published_plan_is_up_to_date(self):
|
||||
self.assertEqual(B.main(["--check"]), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user