The Black is not dice: developer statements, evidence report, and a plan that looks for the rule
- planner/knowledge.yaml: what the game's developer has said (paraphrased, dated, graded), what the Atlas does with each statement, and the open questions with the experiment that tests each. - analysis/black/evidence.py -> analysis/reports/BLACK.md: counts from the runs. On the same recorded route the shadow shows on 21 of 79 first attempts and 1 of 80 later ones; no single route; Blue then Black after the shadow. - game.yaml: the shadow is a guard (not a roll); what follows it is the Blue, then the Black; on-screen text for the squad's direction screen. - planner/experiments.yaml v2: one exact path on every fresh account (fresh-replicate), then one choice changed at a time (fresh-ablate); shadow-again, second-attempt-same-path, veteran-repeat, veteran-variety; what the bot records about the account; the sequence to follow once the shadow shows. - solver.js: assign() takes account facts, account offsets and cycled arms; recommendOption() answers the squad screen (stay + direction) in one pick. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
This commit is contained in:
co-authored by
Claude Fable 5.2
parent
51842e1aa9
commit
d01ec46fcc
+48
-18
@@ -168,29 +168,59 @@ test("mapping: uniform sampling picks among the options on the screen, with its
|
||||
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
|
||||
});
|
||||
|
||||
test("the experiment plan: balanced arms across new accounts, shares after the first attempt", () => {
|
||||
test("the experiment plan: what each attempt of each account is for", () => {
|
||||
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
|
||||
const first = plan.experiments.find((e) => e.id === "first-attempt-shadow");
|
||||
const count = {};
|
||||
for (let acc = 0; acc < 4 * 25; acc++) {
|
||||
const all = published.samples.all;
|
||||
// attempt 1: every new account plays the same exact path (the replication)
|
||||
for (let acc = 0; acc < 30; acc++) {
|
||||
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
|
||||
assert.strictEqual(a.experimentId, "first-attempt-shadow");
|
||||
assert.strictEqual(a.experimentId, "fresh-replicate");
|
||||
assert.strictEqual(a.target, "Black");
|
||||
assert.strictEqual(a.propensity, 1 / first.arms.length);
|
||||
assert.strictEqual(a.fix.direction, "West");
|
||||
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
|
||||
count[a.arm] = (count[a.arm] || 0) + 1;
|
||||
S.advisor(published.samples.all, a.target, a.objective, { attempt: 1, fix: a.fix }); // every arm is playable
|
||||
}
|
||||
assert.deepStrictEqual(Object.values(count), [25, 25, 25, 25]);
|
||||
// attempt 2: half the accounts check the shadow again; the others go to the shared experiments
|
||||
const second = Array.from({ length: 40 }, (_, acc) => S.assign(plan, { accountSeq: acc, attempt: 2 }).experimentId);
|
||||
assert.strictEqual(second.filter((id) => id === "attempt-2-shadow").length, 20);
|
||||
const later = {};
|
||||
for (let acc = 0; acc < 60; acc++) for (let t = 3; t <= 18; t++) { const a = S.assign(plan, { accountSeq: acc, attempt: t }); later[a.experimentId] = (later[a.experimentId] || 0) + 1; }
|
||||
assert.deepStrictEqual(Object.keys(later).sort(), ["mapping", "meadow-blue-route", "sentinel"]);
|
||||
assert.ok(Math.abs(later["meadow-blue-route"] / (60 * 16) - 0.6) < 0.06);
|
||||
// an experiment that is no longer running is not assigned; its share goes to the others
|
||||
const a1 = S.assign(plan, { accountSeq: 0, attempt: 1 });
|
||||
const adv = S.advisor(all, a1.target, a1.objective, { attempt: 1, fix: a1.fix });
|
||||
let h = {};
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const s = adv.step(h);
|
||||
if (s.kind === "decision") { if (s.var === "encounter") break; if (a1.fix[s.var]) assert.strictEqual(s.choice, a1.fix[s.var]); h[s.var] = s.choice; }
|
||||
else if (s.kind === "chance") h[s.var] = s.var === "shadow" ? "no" : Object.keys(s.probs).find((k) => k !== "die");
|
||||
else break;
|
||||
}
|
||||
assert.strictEqual(h.hour, "Tree");
|
||||
// the ablation waits for the replication, then gives every arm the same number of accounts
|
||||
const stage2 = { ...plan, experiments: plan.experiments.map((e) => (e.id === "fresh-replicate" ? { ...e, status: "complete" } : e.id === "fresh-ablate" ? { ...e, status: "running" } : e)) };
|
||||
const count = {};
|
||||
for (let acc = 0; acc < 12 * 5; acc++) { const a = S.assign(stage2, { accountSeq: acc, attempt: 1 }); assert.strictEqual(a.experimentId, "fresh-ablate"); count[a.arm] = (count[a.arm] || 0) + 1; }
|
||||
assert.deepStrictEqual(Object.values(count), Array(12).fill(5));
|
||||
// with squadmates, staying and the direction are one screen
|
||||
const squad = stage2.experiments.find((e) => e.id === "fresh-ablate").arms.find((a) => a.id === "with two squadmates");
|
||||
const rec = S.recommendOption(S.advisor(all, "Black", "eventually", { attempt: 1, fix: squad.fix }), { company: "2 Squadmates" },
|
||||
["Go east with your squad", "Leave your squadmates", "Convince them to head west instead"]);
|
||||
assert.strictEqual(rec.index, 2);
|
||||
assert.deepStrictEqual(rec.also, { direction: "West" });
|
||||
// attempt 2: an account that saw the shadow tries the same path again; half of the others do too
|
||||
const who = (attempt, facts) => Array.from({ length: 40 }, (_, acc) => (S.assign(plan, { accountSeq: acc, attempt, facts }) || {}).experimentId);
|
||||
assert.ok(who(2, { sawShadowBefore: true }).every((id) => id === "shadow-again"));
|
||||
assert.strictEqual(who(2, { sawShadowBefore: false }).filter((id) => id === "second-attempt-same-path").length, 20);
|
||||
// later: two accounts in ten are the long-lived ones, the rest share the research experiments
|
||||
const later = who(6, { sawShadowBefore: false });
|
||||
assert.strictEqual(later.filter((id) => id === "veteran-repeat").length, 4);
|
||||
assert.strictEqual(later.filter((id) => id === "veteran-variety").length, 4);
|
||||
assert.deepStrictEqual([...new Set(later)].sort(), ["mapping", "meadow-blue-route", "sentinel", "veteran-repeat", "veteran-variety"]);
|
||||
// variety: a different combination on every attempt, both gates kept, every one playable
|
||||
const combos = new Set();
|
||||
for (let t = 3; t < 3 + 60; t++) {
|
||||
const a = S.assign(plan, { accountSeq: 11, attempt: t, facts: { sawShadowBefore: false } });
|
||||
assert.strictEqual(a.experimentId, "veteran-variety");
|
||||
assert.strictEqual(a.fix.clearing, "Help your squadmate");
|
||||
combos.add(a.arm);
|
||||
if (t < 8) S.advisor(all, a.target, a.objective, { attempt: t, fix: a.fix });
|
||||
}
|
||||
assert.strictEqual(combos.size, 60);
|
||||
// an experiment that is not running is not assigned; its share goes to the others
|
||||
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
|
||||
assert.notStrictEqual(S.assign(done, { accountSeq: 1, attempt: 5 }).experimentId, "meadow-blue-route");
|
||||
assert.notStrictEqual(S.assign(done, { accountSeq: 5, attempt: 5 }).experimentId, "meadow-blue-route");
|
||||
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
|
||||
});
|
||||
|
||||
@@ -14,43 +14,67 @@ GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "c
|
||||
|
||||
|
||||
class Plan(unittest.TestCase):
|
||||
def test_design_is_valid_and_arms_are_every_combination(self):
|
||||
def test_design_is_valid(self):
|
||||
plan = B.build(D, log=[])
|
||||
by = {e["id"]: e for e in plan["experiments"]}
|
||||
self.assertEqual(len(by["first-attempt-shadow"]["arms"]), 4)
|
||||
# the Black attempt is one exact path, and it keeps the shadow's gate open
|
||||
(arm,) = by["fresh-replicate"]["arms"]
|
||||
self.assertEqual(arm["fix"], D["routes"]["best-reported"])
|
||||
self.assertTrue(model.gate_open("shadow", arm["fix"]))
|
||||
self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6)
|
||||
for e in plan["experiments"]:
|
||||
self.assertEqual(e["status"], "running")
|
||||
for a in e["arms"]:
|
||||
B.check_options(e["id"], a["fix"])
|
||||
# the Black attempt keeps the shadow's gate open on every arm
|
||||
for a in by["first-attempt-shadow"]["arms"]:
|
||||
B.check_options(e["id"], {k: v[0] for k, v in (a.get("cycle") or {}).items()})
|
||||
self.assertEqual(plan["record"], D["record"])
|
||||
self.assertEqual(len(plan["scripts"]["shadow"]), 12)
|
||||
|
||||
def test_ablation_changes_one_thing_at_a_time_and_waits_for_the_replication(self):
|
||||
plan = B.build(D, log=[])
|
||||
by = {e["id"]: e for e in plan["experiments"]}
|
||||
base = D["routes"]["best-reported"]
|
||||
self.assertEqual(by["fresh-ablate"]["status"], "waiting")
|
||||
for a in by["fresh-ablate"]["arms"]:
|
||||
changed = {k for k in a["fix"] if a["fix"][k] != base.get(k)}
|
||||
self.assertEqual(changed, set(a["change"]), a["id"])
|
||||
self.assertLessEqual(len(changed - {"leave"}), 1, a["id"]) # leaving goes with the company
|
||||
self.assertTrue(model.gate_open("shadow", a["fix"]))
|
||||
# once the replication has its planned count, the ablation starts
|
||||
e0 = next(e for e in D["experiments"] if e["id"] == "fresh-replicate")
|
||||
log = [(e0["id"], "all", {**GATED_TREE, "shadow": i < 9}) for i in range(e0["planned_per_arm"])]
|
||||
by = {e["id"]: e for e in B.build(D, log=log)["experiments"]}
|
||||
self.assertEqual(by["fresh-replicate"]["status"], "complete")
|
||||
self.assertEqual(by["fresh-replicate"]["progress"]["rare"], 9)
|
||||
self.assertEqual(by["fresh-ablate"]["status"], "running")
|
||||
|
||||
def test_an_unknown_option_is_refused(self):
|
||||
d = copy.deepcopy(D)
|
||||
d["experiments"][0]["factors"]["direction"] = ["North"]
|
||||
with self.assertRaises(ValueError):
|
||||
B.build(d, log=[])
|
||||
def test_an_unknown_option_route_or_fact_is_refused(self):
|
||||
for edit in (lambda d: d["routes"]["best-reported"].update(direction="North"),
|
||||
lambda d: d["experiments"][0].update(route="nowhere"),
|
||||
lambda d: d["experiments"][1]["arms"][0]["change"].update(company="Three"),
|
||||
lambda d: d["experiments"][2].update(when={"luck": True}),
|
||||
lambda d: d["experiments"][1].update(starts_after="nothing")):
|
||||
d = copy.deepcopy(D)
|
||||
edit(d)
|
||||
with self.assertRaises(ValueError):
|
||||
B.build(d, log=[])
|
||||
|
||||
def test_progress_completes_and_stops_early(self):
|
||||
def test_progress(self):
|
||||
e0 = D["experiments"][0]
|
||||
arms = [a["id"] for a in B.arms(e0)]
|
||||
seen = {**GATED_TREE, "shadow": True}
|
||||
none = {**GATED_TREE, "shadow": False}
|
||||
# every arm at its planned count, with shadows: complete
|
||||
log = [(e0["id"], a, seen if i < 3 else none) for a in arms for i in range(e0["planned_per_arm"])]
|
||||
e = B.build(D, log=log)["experiments"][0]
|
||||
self.assertEqual(e["status"], "complete")
|
||||
self.assertEqual(e["progress"]["rare"], 12)
|
||||
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 61)
|
||||
# no shadow at all after the stop_early count: paused, for a person to look at
|
||||
log = [(e0["id"], arms[i % 4], none) for i in range(e0["stop_early"]["after"])]
|
||||
e = B.build(D, log=log)["experiments"][0]
|
||||
self.assertEqual(e["status"], "paused")
|
||||
# a run whose roll was not observed (died before the tree) is a run, not an observation
|
||||
e = B.build(D, log=[(e0["id"], arms[0], {"attempt": "First attempt"})])["experiments"][0]
|
||||
e = B.build(D, log=[(e0["id"], "all", {**GATED_TREE, "shadow": True})] * 3 + [(e0["id"], "all", none)] * 7)["experiments"][0]
|
||||
self.assertEqual((e["status"], e["progress"]["observed"], e["progress"]["rare"]), ("running", 10, 3))
|
||||
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 11)
|
||||
# a run whose shadow was not observed (died before the tree) is a run, not an observation
|
||||
e = B.build(D, log=[(e0["id"], "all", {"attempt": "First attempt"})])["experiments"][0]
|
||||
self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0))
|
||||
# a cycled experiment logs the combination it played; they all count toward its one arm
|
||||
v = B.build(D, log=[("veteran-variety", "company=Alone / direction=East", none)] * 4)
|
||||
v = next(e for e in v["experiments"] if e["id"] == "veteran-variety")
|
||||
self.assertEqual((v["arms"][0]["progress"]["observed"], v["progress"]["unknownArms"]), (4, []))
|
||||
# stop_early pauses an experiment for a person to look at
|
||||
d = copy.deepcopy(D)
|
||||
d["experiments"][0]["stop_early"] = {"after": 5, "rare_at_most": 0}
|
||||
self.assertEqual(B.build(d, log=[(e0["id"], "all", none)] * 5)["experiments"][0]["status"], "paused")
|
||||
|
||||
def test_the_plan_id_follows_its_content(self):
|
||||
a, b = B.build(D, log=[]), B.build(D, log=[])
|
||||
|
||||
Reference in New Issue
Block a user