Route model: first attempts are their own shadow roll; experiment plan and automated bot cycle
- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
This commit is contained in:
co-authored by
Claude Fable 5.2
parent
ee07ee143e
commit
ab3664f369
@@ -125,3 +125,72 @@ test("Thompson sampling: one posterior draw per seed, with the chance of each pi
|
||||
for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice);
|
||||
assert.ok(picks.size > 1, "different seeds explore different options");
|
||||
});
|
||||
|
||||
test("a first attempt has its own odds, exactly as the Python model publishes them", () => {
|
||||
for (const smp of Object.values(published.samples)) {
|
||||
for (const [X, t] of Object.entries(smp.targets)) {
|
||||
for (const obj of ["thisAttempt", "eventually"]) {
|
||||
const adv = S.advisor(smp, X, obj, { attempt: 1 });
|
||||
assert.deepStrictEqual(adv.given, { attempt: "First attempt" });
|
||||
assert.ok(Math.abs(adv.value.x - t.firstAttempt[obj].pX.point) < 1e-12, `${X} ${obj}`);
|
||||
assert.strictEqual(adv.next({}).choice, t.firstAttempt[obj].tree.choice);
|
||||
}
|
||||
}
|
||||
}
|
||||
const all = published.samples.all;
|
||||
const later = S.advisor(all, "Black", "thisAttempt", { attempt: 7 }), dflt = S.advisor(all, "Black", "thisAttempt");
|
||||
assert.strictEqual(later.value.x, dflt.value.x); // no attempt given: a later attempt
|
||||
assert.ok(S.advisor(all, "Black", "thisAttempt", { attempt: 1 }).value.x > 5 * later.value.x);
|
||||
});
|
||||
|
||||
test("fix holds decisions to an experiment arm and solves the rest around it", () => {
|
||||
const all = published.samples.all;
|
||||
const fix = { direction: "East", scream: "Investigate the scream", clearing: "Help your squadmate", hour: "Tree" };
|
||||
const adv = S.advisor(all, "Black", "eventually", { attempt: 1, fix });
|
||||
assert.strictEqual(adv.step({ company: "Alone" }).choice, "East");
|
||||
assert.deepStrictEqual(Object.keys(adv.step({ company: "Alone" }).options), ["East"]);
|
||||
assert.ok(adv.value.x <= S.advisor(all, "Black", "eventually", { attempt: 1 }).value.x + 1e-12);
|
||||
assert.throws(() => S.advisor(all, "Black", "eventually", { fix: { direction: "North" } }), /no option/);
|
||||
});
|
||||
|
||||
test("mapping: uniform sampling picks among the options on the screen, with its propensity", () => {
|
||||
const all = published.samples.all;
|
||||
const screen = ["Enter the valley alone", "Enter the valley with one of your squadmates", "Enter the valley with two of your squadmates"];
|
||||
const picks = new Set();
|
||||
for (let seed = 1; seed <= 30; seed++) {
|
||||
const rec = S.recommendOption(S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed }), {}, screen);
|
||||
assert.ok(Math.abs(rec.propensity - 1 / 3) < 1e-12);
|
||||
assert.match(S.norm(screen[rec.index]), new RegExp(all.model.game.bot_text.company[rec.choice]));
|
||||
picks.add(rec.choice);
|
||||
}
|
||||
assert.strictEqual(picks.size, 3);
|
||||
const a = S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed: 4 });
|
||||
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
|
||||
});
|
||||
|
||||
test("the experiment plan: balanced arms across new accounts, shares after the first attempt", () => {
|
||||
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
|
||||
const first = plan.experiments.find((e) => e.id === "first-attempt-shadow");
|
||||
const count = {};
|
||||
for (let acc = 0; acc < 4 * 25; acc++) {
|
||||
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
|
||||
assert.strictEqual(a.experimentId, "first-attempt-shadow");
|
||||
assert.strictEqual(a.target, "Black");
|
||||
assert.strictEqual(a.propensity, 1 / first.arms.length);
|
||||
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
|
||||
count[a.arm] = (count[a.arm] || 0) + 1;
|
||||
S.advisor(published.samples.all, a.target, a.objective, { attempt: 1, fix: a.fix }); // every arm is playable
|
||||
}
|
||||
assert.deepStrictEqual(Object.values(count), [25, 25, 25, 25]);
|
||||
// attempt 2: half the accounts check the shadow again; the others go to the shared experiments
|
||||
const second = Array.from({ length: 40 }, (_, acc) => S.assign(plan, { accountSeq: acc, attempt: 2 }).experimentId);
|
||||
assert.strictEqual(second.filter((id) => id === "attempt-2-shadow").length, 20);
|
||||
const later = {};
|
||||
for (let acc = 0; acc < 60; acc++) for (let t = 3; t <= 18; t++) { const a = S.assign(plan, { accountSeq: acc, attempt: t }); later[a.experimentId] = (later[a.experimentId] || 0) + 1; }
|
||||
assert.deepStrictEqual(Object.keys(later).sort(), ["mapping", "meadow-blue-route", "sentinel"]);
|
||||
assert.ok(Math.abs(later["meadow-blue-route"] / (60 * 16) - 0.6) < 0.06);
|
||||
// an experiment that is no longer running is not assigned; its share goes to the others
|
||||
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
|
||||
assert.notStrictEqual(S.assign(done, { accountSeq: 1, attempt: 5 }).experimentId, "meadow-blue-route");
|
||||
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user