Route model: first attempts are their own shadow roll; experiment plan and automated bot cycle

- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
  depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
  so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
  0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
  experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
This commit is contained in:
ekcSchiano
2026-11-12 03:31:33 +00:00
co-authored by Claude Fable 5.2
parent ee07ee143e
commit ab3664f369
25 changed files with 1938 additions and 573 deletions
+69
View File
@@ -125,3 +125,72 @@ test("Thompson sampling: one posterior draw per seed, with the chance of each pi
for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice);
assert.ok(picks.size > 1, "different seeds explore different options");
});
test("a first attempt has its own odds, exactly as the Python model publishes them", () => {
for (const smp of Object.values(published.samples)) {
for (const [X, t] of Object.entries(smp.targets)) {
for (const obj of ["thisAttempt", "eventually"]) {
const adv = S.advisor(smp, X, obj, { attempt: 1 });
assert.deepStrictEqual(adv.given, { attempt: "First attempt" });
assert.ok(Math.abs(adv.value.x - t.firstAttempt[obj].pX.point) < 1e-12, `${X} ${obj}`);
assert.strictEqual(adv.next({}).choice, t.firstAttempt[obj].tree.choice);
}
}
}
const all = published.samples.all;
const later = S.advisor(all, "Black", "thisAttempt", { attempt: 7 }), dflt = S.advisor(all, "Black", "thisAttempt");
assert.strictEqual(later.value.x, dflt.value.x); // no attempt given: a later attempt
assert.ok(S.advisor(all, "Black", "thisAttempt", { attempt: 1 }).value.x > 5 * later.value.x);
});
test("fix holds decisions to an experiment arm and solves the rest around it", () => {
const all = published.samples.all;
const fix = { direction: "East", scream: "Investigate the scream", clearing: "Help your squadmate", hour: "Tree" };
const adv = S.advisor(all, "Black", "eventually", { attempt: 1, fix });
assert.strictEqual(adv.step({ company: "Alone" }).choice, "East");
assert.deepStrictEqual(Object.keys(adv.step({ company: "Alone" }).options), ["East"]);
assert.ok(adv.value.x <= S.advisor(all, "Black", "eventually", { attempt: 1 }).value.x + 1e-12);
assert.throws(() => S.advisor(all, "Black", "eventually", { fix: { direction: "North" } }), /no option/);
});
test("mapping: uniform sampling picks among the options on the screen, with its propensity", () => {
const all = published.samples.all;
const screen = ["Enter the valley alone", "Enter the valley with one of your squadmates", "Enter the valley with two of your squadmates"];
const picks = new Set();
for (let seed = 1; seed <= 30; seed++) {
const rec = S.recommendOption(S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed }), {}, screen);
assert.ok(Math.abs(rec.propensity - 1 / 3) < 1e-12);
assert.match(S.norm(screen[rec.index]), new RegExp(all.model.game.bot_text.company[rec.choice]));
picks.add(rec.choice);
}
assert.strictEqual(picks.size, 3);
const a = S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed: 4 });
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
});
test("the experiment plan: balanced arms across new accounts, shares after the first attempt", () => {
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
const first = plan.experiments.find((e) => e.id === "first-attempt-shadow");
const count = {};
for (let acc = 0; acc < 4 * 25; acc++) {
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
assert.strictEqual(a.experimentId, "first-attempt-shadow");
assert.strictEqual(a.target, "Black");
assert.strictEqual(a.propensity, 1 / first.arms.length);
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
count[a.arm] = (count[a.arm] || 0) + 1;
S.advisor(published.samples.all, a.target, a.objective, { attempt: 1, fix: a.fix }); // every arm is playable
}
assert.deepStrictEqual(Object.values(count), [25, 25, 25, 25]);
// attempt 2: half the accounts check the shadow again; the others go to the shared experiments
const second = Array.from({ length: 40 }, (_, acc) => S.assign(plan, { accountSeq: acc, attempt: 2 }).experimentId);
assert.strictEqual(second.filter((id) => id === "attempt-2-shadow").length, 20);
const later = {};
for (let acc = 0; acc < 60; acc++) for (let t = 3; t <= 18; t++) { const a = S.assign(plan, { accountSeq: acc, attempt: t }); later[a.experimentId] = (later[a.experimentId] || 0) + 1; }
assert.deepStrictEqual(Object.keys(later).sort(), ["mapping", "meadow-blue-route", "sentinel"]);
assert.ok(Math.abs(later["meadow-blue-route"] / (60 * 16) - 0.6) < 0.06);
// an experiment that is no longer running is not assigned; its share goes to the others
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
assert.notStrictEqual(S.assign(done, { accountSeq: 1, attempt: 5 }).experimentId, "meadow-blue-route");
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
});
+57
View File
@@ -48,6 +48,63 @@ class Bias(unittest.TestCase):
self.assertEqual(model.fit_alpha(T, "meadow")["alpha"], max(model.ALPHA_GRID))
def test_unestimable_bias_can_borrow_another_rolls_estimate(self):
# the shadow was never seen in complete logs, the meadow Blue was: the form's shadow reports count 1/rho(meadow)
cfg = {**CFG, "rolls": {"shadow": "yes", "meadow": "Blue"}, "borrow": {"shadow": "meadow"}}
runs = self.runs(10, 100, 100, 300) + [tree_run("bot", False)] * 100 + \
[tree_run("reddit spreadsheet", True)] * 8 + [tree_run("reddit spreadsheet", False)] * 92
prep = [model.prepare_run(r) for r in runs]
est, names = bias.estimate(runs, prep, cfg, set())
e = est["form"]["shadow"]
self.assertAlmostEqual(e["rho"], est["form"]["meadow"]["rho"])
self.assertIn("assumed", e["used"])
self.assertFalse(e.get("exclude"))
T = model.count_tables(prep, bias.weights(names, prep, est, cfg["rolls"]))
self.assertAlmostEqual(T["shadow"]["1:First attempt"]["yes"], 8 / e["rho"], places=3)
# without `borrow`, the group is left out of the roll
est2, _ = bias.estimate(runs, prep, {**cfg, "borrow": {}}, set())
self.assertTrue(est2["form"]["shadow"]["exclude"])
def tree_run(source, shadow, attempt_group="First attempt"):
r = meadow_run(source, "Orange")
r.update(scream="Investigate & help", resolution="Squadmate's revenge", post_path="Forest", hour="Tree",
branch="Shadow (rare)" if shadow else "Descend slowly: two Greens", outcome_color="Black" if shadow else "Green",
enc_Orange=None, enc_Black="Incinerated" if shadow else None, enc_Green=None if shadow else "Incinerated",
saw_blue_meadow=None, attempt_group=attempt_group, combat_first=None, combat_second=None)
return r
class FirstAttempt(unittest.TestCase):
"""game.yaml `givens`: first and later attempts are different shadow rolls (`distinct`), not pooled."""
def setUp(self):
runs = [tree_run("bot", True)] * 9 + [tree_run("bot", False)] * 91 + [tree_run("bot", False, "Later attempt")] * 900 + \
[tree_run("bot", True, "Later attempt")] * 1
self.T = model.count_tables([model.prepare_run(r) for r in runs])
def test_each_attempt_kind_keeps_its_own_rate(self):
p = model.Params(self.T, "mean", alphas={"shadow": 100000})
gated = {"scream": "Investigate the scream", "clearing": "Help your squadmate"}
later = float(p.chance("shadow", gated)["yes"][0])
first = float(p.having(attempt="First attempt").chance("shadow", gated)["yes"][0])
self.assertAlmostEqual(first, 0.09, delta=0.01) # not dragged to the pooled 1%
self.assertAlmostEqual(later, 0.001, delta=0.001)
self.assertEqual(p.given, {"attempt": "Later attempt"}) # the default: a later attempt
def test_policy_reports_the_first_attempt_separately(self):
from atlas import solver
runs = [tree_run("bot", True)] * 9 + [tree_run("bot", False)] * 91 + [tree_run("bot", False, "Later attempt")] * 900
pol = solver.build_policy([dict(r, enc_Black="Bonded!", outcome="Bonded") if r["branch"] == "Shadow (rare)" else r for r in runs],
draws=6, colours=["Black"], min_support=1)
t = pol["targets"]["Black"]
f = t["firstAttempt"]
self.assertEqual(f["given"], {"attempt": "First attempt"})
self.assertGreater(f["thisAttempt"]["pX"]["point"], 20 * t["thisAttempt"]["pX"]["point"])
self.assertLessEqual(f["thisAttempt"]["pX"]["lo"], f["thisAttempt"]["pX"]["hi"])
self.assertIn("pEventually", f["eventually"])
if __name__ == "__main__":
unittest.main()
+66
View File
@@ -0,0 +1,66 @@
"""The experiment plan (planner/experiments.yaml -> data/public/experiments.json)."""
import copy
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "planner"))
import build_experiments as B # noqa: E402
from atlas import model # noqa: E402
D = B.design()
GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "clearing": "Help your squadmate"}
class Plan(unittest.TestCase):
def test_design_is_valid_and_arms_are_every_combination(self):
plan = B.build(D, log=[])
by = {e["id"]: e for e in plan["experiments"]}
self.assertEqual(len(by["first-attempt-shadow"]["arms"]), 4)
self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6)
for e in plan["experiments"]:
self.assertEqual(e["status"], "running")
for a in e["arms"]:
B.check_options(e["id"], a["fix"])
# the Black attempt keeps the shadow's gate open on every arm
for a in by["first-attempt-shadow"]["arms"]:
self.assertTrue(model.gate_open("shadow", a["fix"]))
def test_an_unknown_option_is_refused(self):
d = copy.deepcopy(D)
d["experiments"][0]["factors"]["direction"] = ["North"]
with self.assertRaises(ValueError):
B.build(d, log=[])
def test_progress_completes_and_stops_early(self):
e0 = D["experiments"][0]
arms = [a["id"] for a in B.arms(e0)]
seen = {**GATED_TREE, "shadow": True}
none = {**GATED_TREE, "shadow": False}
# every arm at its planned count, with shadows: complete
log = [(e0["id"], a, seen if i < 3 else none) for a in arms for i in range(e0["planned_per_arm"])]
e = B.build(D, log=log)["experiments"][0]
self.assertEqual(e["status"], "complete")
self.assertEqual(e["progress"]["rare"], 12)
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 61)
# no shadow at all after the stop_early count: paused, for a person to look at
log = [(e0["id"], arms[i % 4], none) for i in range(e0["stop_early"]["after"])]
e = B.build(D, log=log)["experiments"][0]
self.assertEqual(e["status"], "paused")
# a run whose roll was not observed (died before the tree) is a run, not an observation
e = B.build(D, log=[(e0["id"], arms[0], {"attempt": "First attempt"})])["experiments"][0]
self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0))
def test_the_plan_id_follows_its_content(self):
a, b = B.build(D, log=[]), B.build(D, log=[])
self.assertEqual(a["id"], b["id"])
c = B.build(D, log=[(D["experiments"][0]["id"], "x", None)])
self.assertNotEqual(a["id"], c["id"])
def test_published_plan_is_up_to_date(self):
self.assertEqual(B.main(["--check"]), 0)
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -47,7 +47,7 @@ class GameSpec(unittest.TestCase):
def test_chance_contexts_are_earlier_steps(self):
known = {"company", "left", "direction", "fs", "react", "scream", "clearing", "fight", "dagger", "cadet", "post",
"hour", "shadow", "descend", "red", "valley", "meadow"}
"hour", "shadow", "descend", "red", "valley", "meadow"} | set(G.get("givens") or {})
for k, c in G["chance"].items():
self.assertTrue(set(c["context"]) <= known, k)