- planner/knowledge.yaml: what the game's developer has said (paraphrased, dated, graded), what the Atlas does with each statement, and the open questions with the experiment that tests each. - analysis/black/evidence.py -> analysis/reports/BLACK.md: counts from the runs. On the same recorded route the shadow shows on 21 of 79 first attempts and 1 of 80 later ones; no single route; Blue then Black after the shadow. - game.yaml: the shadow is a guard (not a roll); what follows it is the Blue, then the Black; on-screen text for the squad's direction screen. - planner/experiments.yaml v2: one exact path on every fresh account (fresh-replicate), then one choice changed at a time (fresh-ablate); shadow-again, second-attempt-same-path, veteran-repeat, veteran-variety; what the bot records about the account; the sequence to follow once the shadow shows. - solver.js: assign() takes account facts, account offsets and cycled arms; recommendOption() answers the squad screen (stay + direction) in one pick. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
91 lines
4.7 KiB
Python
91 lines
4.7 KiB
Python
"""The experiment plan (planner/experiments.yaml -> data/public/experiments.json)."""
|
|
import copy
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "planner"))
|
|
import build_experiments as B # noqa: E402
|
|
from atlas import model # noqa: E402
|
|
|
|
D = B.design()
|
|
GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "clearing": "Help your squadmate"}
|
|
|
|
|
|
class Plan(unittest.TestCase):
|
|
def test_design_is_valid(self):
|
|
plan = B.build(D, log=[])
|
|
by = {e["id"]: e for e in plan["experiments"]}
|
|
# the Black attempt is one exact path, and it keeps the shadow's gate open
|
|
(arm,) = by["fresh-replicate"]["arms"]
|
|
self.assertEqual(arm["fix"], D["routes"]["best-reported"])
|
|
self.assertTrue(model.gate_open("shadow", arm["fix"]))
|
|
self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6)
|
|
for e in plan["experiments"]:
|
|
for a in e["arms"]:
|
|
B.check_options(e["id"], a["fix"])
|
|
B.check_options(e["id"], {k: v[0] for k, v in (a.get("cycle") or {}).items()})
|
|
self.assertEqual(plan["record"], D["record"])
|
|
self.assertEqual(len(plan["scripts"]["shadow"]), 12)
|
|
|
|
def test_ablation_changes_one_thing_at_a_time_and_waits_for_the_replication(self):
|
|
plan = B.build(D, log=[])
|
|
by = {e["id"]: e for e in plan["experiments"]}
|
|
base = D["routes"]["best-reported"]
|
|
self.assertEqual(by["fresh-ablate"]["status"], "waiting")
|
|
for a in by["fresh-ablate"]["arms"]:
|
|
changed = {k for k in a["fix"] if a["fix"][k] != base.get(k)}
|
|
self.assertEqual(changed, set(a["change"]), a["id"])
|
|
self.assertLessEqual(len(changed - {"leave"}), 1, a["id"]) # leaving goes with the company
|
|
self.assertTrue(model.gate_open("shadow", a["fix"]))
|
|
# once the replication has its planned count, the ablation starts
|
|
e0 = next(e for e in D["experiments"] if e["id"] == "fresh-replicate")
|
|
log = [(e0["id"], "all", {**GATED_TREE, "shadow": i < 9}) for i in range(e0["planned_per_arm"])]
|
|
by = {e["id"]: e for e in B.build(D, log=log)["experiments"]}
|
|
self.assertEqual(by["fresh-replicate"]["status"], "complete")
|
|
self.assertEqual(by["fresh-replicate"]["progress"]["rare"], 9)
|
|
self.assertEqual(by["fresh-ablate"]["status"], "running")
|
|
|
|
def test_an_unknown_option_route_or_fact_is_refused(self):
|
|
for edit in (lambda d: d["routes"]["best-reported"].update(direction="North"),
|
|
lambda d: d["experiments"][0].update(route="nowhere"),
|
|
lambda d: d["experiments"][1]["arms"][0]["change"].update(company="Three"),
|
|
lambda d: d["experiments"][2].update(when={"luck": True}),
|
|
lambda d: d["experiments"][1].update(starts_after="nothing")):
|
|
d = copy.deepcopy(D)
|
|
edit(d)
|
|
with self.assertRaises(ValueError):
|
|
B.build(d, log=[])
|
|
|
|
def test_progress(self):
|
|
e0 = D["experiments"][0]
|
|
none = {**GATED_TREE, "shadow": False}
|
|
e = B.build(D, log=[(e0["id"], "all", {**GATED_TREE, "shadow": True})] * 3 + [(e0["id"], "all", none)] * 7)["experiments"][0]
|
|
self.assertEqual((e["status"], e["progress"]["observed"], e["progress"]["rare"]), ("running", 10, 3))
|
|
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 11)
|
|
# a run whose shadow was not observed (died before the tree) is a run, not an observation
|
|
e = B.build(D, log=[(e0["id"], "all", {"attempt": "First attempt"})])["experiments"][0]
|
|
self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0))
|
|
# a cycled experiment logs the combination it played; they all count toward its one arm
|
|
v = B.build(D, log=[("veteran-variety", "company=Alone / direction=East", none)] * 4)
|
|
v = next(e for e in v["experiments"] if e["id"] == "veteran-variety")
|
|
self.assertEqual((v["arms"][0]["progress"]["observed"], v["progress"]["unknownArms"]), (4, []))
|
|
# stop_early pauses an experiment for a person to look at
|
|
d = copy.deepcopy(D)
|
|
d["experiments"][0]["stop_early"] = {"after": 5, "rare_at_most": 0}
|
|
self.assertEqual(B.build(d, log=[(e0["id"], "all", none)] * 5)["experiments"][0]["status"], "paused")
|
|
|
|
def test_the_plan_id_follows_its_content(self):
|
|
a, b = B.build(D, log=[]), B.build(D, log=[])
|
|
self.assertEqual(a["id"], b["id"])
|
|
c = B.build(D, log=[(D["experiments"][0]["id"], "x", None)])
|
|
self.assertNotEqual(a["id"], c["id"])
|
|
|
|
def test_published_plan_is_up_to_date(self):
|
|
self.assertEqual(B.main(["--check"]), 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|