Screen tree, register rows in the master list, and bot runs coded from the picks that were played
CI / test (pull_request) Canceled after 0s
CI / test (pull_request) Canceled after 0s
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen, every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree. - The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and graph.js now use the played steps only. - runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only rows, flagged (no analysis uses them). - Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
This commit is contained in:
co-authored by
Claude Fable 5.2
parent
d01ec46fcc
commit
f8e4f575f4
@@ -157,12 +157,28 @@ class Handoff(unittest.TestCase):
|
||||
with self.assertRaises(RuntimeError):
|
||||
extract.pack_local(spec, empty)
|
||||
|
||||
def test_overridden_picks_are_not_played(self):
|
||||
from collection.etl import runs
|
||||
steps = [{"scene": "a", "to": "BURNED", "optionText": "west"}, {"scene": "a", "to": "b", "optionText": "centre"},
|
||||
{"scene": "b", "to": "BURNED", "optionText": "last"}]
|
||||
self.assertEqual([s["optionText"] for s in runs.played(steps)], ["centre", "last"])
|
||||
self.assertEqual(runs.played(steps[1:]), steps[1:]) # a last step that ends the run is kept
|
||||
|
||||
def test_register_rows_are_outcome_only_and_flagged(self):
|
||||
import csv
|
||||
with open(ROOT / "data/public/runs_clean.csv", encoding="utf-8", newline="") as f:
|
||||
reg = [r for r in csv.DictReader(f) if r["source"] == "register"]
|
||||
self.assertTrue(reg)
|
||||
for r in reg:
|
||||
self.assertEqual((r["outcome"], r["company"], r["scream"]), ("Bonded", "Unknown", "Unknown"))
|
||||
self.assertIn("outcome only", r["data_flags"])
|
||||
|
||||
def test_bot_kit(self):
|
||||
import zipfile
|
||||
from collection.etl import handoff
|
||||
zpath = handoff.export_bot_kit(str(TMP / "exports"))
|
||||
with zipfile.ZipFile(zpath) as z:
|
||||
self.assertEqual(sorted(z.namelist()), ["README.txt", "experiments.json", "graph.js", "policy.json", "solver.js"])
|
||||
self.assertEqual(sorted(z.namelist()), ["README.txt", "experiments.json", "graph.js", "policy.json", "solver.js", "tree.json"])
|
||||
self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes())
|
||||
self.assertEqual(json.loads(z.read("policy.json"))["id"],
|
||||
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
||||
|
||||
@@ -224,3 +224,31 @@ test("the experiment plan: what each attempt of each account is for", () => {
|
||||
assert.notStrictEqual(S.assign(done, { accountSeq: 5, attempt: 5 }).experimentId, "meadow-blue-route");
|
||||
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
|
||||
});
|
||||
|
||||
test("screenPick: least-tried fills gaps, Thompson avoids what has killed", () => {
|
||||
const tree = { screens: { s1: { options: {
|
||||
a: { text: "Hold its gaze", n: 40, to: { next: { n: 38 }, DIED: { n: 2 } } },
|
||||
b: { text: "Lower your eyes", n: 40, to: { DIED: { n: 40 } } },
|
||||
c: { text: "Step back", n: 0, to: {} } } } } };
|
||||
const screen = ["Lower your eyes", "Hold its gaze", "Step back", "An option the tree has never seen"];
|
||||
const lt = S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 });
|
||||
assert.ok([2, 3].includes(lt.index) && lt.n === 0);
|
||||
assert.deepStrictEqual(lt, S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 })); // same seed, same pick
|
||||
const picks = {};
|
||||
for (let seed = 1; seed <= 300; seed++) { const p = S.screenPick(tree, "s1", screen, { mode: "thompson", seed }); picks[p.index] = (picks[p.index] || 0) + 1; }
|
||||
assert.ok(!picks[0] || picks[0] < 3, "an option that died 40 times in 40 is almost never drawn");
|
||||
assert.ok(picks[1] > 200 && picks[2] >= 5 && picks[3] >= 5, JSON.stringify(picks)); // mostly the good one; the untested ones still get tried
|
||||
assert.strictEqual(S.screenPick(tree, "unknown-screen", ["x", "y"], { seed: 2 }).n, 0);
|
||||
assert.strictEqual(S.screenPick(tree, "s1", []), null);
|
||||
});
|
||||
|
||||
test("picks the bot logged and then overrode are not part of the run", () => {
|
||||
const G = require("../../core/graph.js");
|
||||
const steps = [
|
||||
{ scene: "a68f681076c6", optionText: "Head west toward the scent", to: "BURNED" },
|
||||
{ scene: "a68f681076c6", optionText: "Head toward the center of the forest", to: "bc4ee4754b9c" },
|
||||
{ scene: "bc4ee4754b9c", optionText: "Lower your eyes in respect", to: "84780dab01e3", tint: "red" }];
|
||||
assert.strictEqual(G.playedSteps(steps).length, 2);
|
||||
assert.strictEqual(G.trajectoryVariables({ steps, outcome: {} }).vars.direction, "Center");
|
||||
assert.strictEqual(G.playedSteps([{ scene: "x", optionText: "only" }]).length, 1); // a run in progress
|
||||
});
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""The screen tree (analysis/tree/build_tree.py) on a small game whose answers are known."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "analysis/tree"))
|
||||
import build_tree as B # noqa: E402
|
||||
|
||||
SCENES = {"start": {"label": "Start", "stage": "S", "options": [{"id": "a", "text": "Left"}, {"id": "b", "text": "Right"}, {"id": "c", "text": "Never tried"}]},
|
||||
"meadow": {"label": "Meadow", "stage": "M", "options": [{"id": "x", "text": "Approach"}]},
|
||||
"blue": {"label": "Blue", "stage": "B", "options": [{"id": "g", "text": "Hold its gaze"}, {"id": "l", "text": "Lower eyes"}]}}
|
||||
|
||||
|
||||
def step(scene, option, to=None, stage="S"):
|
||||
return {"scene": scene, "option": option, "optionText": option, "to": to, "stage": stage}
|
||||
|
||||
|
||||
def run(i, steps, kind, colour=None):
|
||||
return {"id": f"r{i}", "steps": steps, "outcome": {"type": kind, "colour": colour}}
|
||||
|
||||
|
||||
def world():
|
||||
T = []
|
||||
for i in range(100): # Left: the meadow; a Blue waits there 1 time in 10; holding its gaze bonds 8 times in 10
|
||||
s = [step("start", "a", "meadow")]
|
||||
if i % 10 == 0:
|
||||
s += [step("meadow", "x", "blue"), step("blue", "g", "BURNED" if i in (0, 50) else "BONDED:blue")]
|
||||
T.append(run(i, s, "BURNED" if i in (0, 50) else "BONDED", None if i in (0, 50) else "blue"))
|
||||
else:
|
||||
s += [step("meadow", "x", "BURNED", stage="INTENTIONAL_BURN")]
|
||||
T.append(run(i, s, "BURNED"))
|
||||
T.append(run(200, [step("start", "b", "BURNED")], "BURNED")) # Right: died, once
|
||||
T.append(run(201, [step("start", "b", "BURNED"), step("start", "a", "meadow")], "ABORTED")) # overridden pick, then aborted
|
||||
T.append(run(202, [step("blue", "l", "BONDED:blue")], "BONDED", "blue")) # one lucky run with lowered eyes
|
||||
return T
|
||||
|
||||
|
||||
class Tree(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.t = B.build(world(), SCENES)
|
||||
|
||||
def test_counts_kinds_and_overridden_picks(self):
|
||||
S = self.t["screens"]
|
||||
a = S["start"]["options"]["a"]
|
||||
self.assertEqual((a["n"], a["kind"], list(a["to"])), (101, "fixed", ["meadow"])) # 100 runs + the aborted one
|
||||
self.assertLess(a["missed"], 0.01)
|
||||
self.assertEqual(S["start"]["options"]["b"]["n"], 1) # the overridden pick is not counted
|
||||
self.assertEqual(self.t["builtFrom"]["overridden"], 1)
|
||||
self.assertEqual(S["start"]["options"]["c"]["kind"], "untested")
|
||||
x = S["meadow"]["options"]["x"]
|
||||
self.assertEqual(x["kind"], "dice")
|
||||
self.assertAlmostEqual(x["to"]["blue"]["p"], 0.1)
|
||||
self.assertEqual(x["deliberate"], 90) # deliberate burns are real results
|
||||
self.assertTrue(x["to"]["blue"]["lo"] < 0.1 < x["to"]["blue"]["hi"])
|
||||
g = S["blue"]["options"]["g"]
|
||||
self.assertEqual((g["to"]["BONDED:Blue"]["n"], g["to"]["DIED"]["n"]), (8, 2))
|
||||
|
||||
def test_best_known_chance_is_chosen_cautiously(self):
|
||||
best = self.t["best"]["Blue"]
|
||||
# one lucky run with lowered eyes is 100% as observed, but holding its gaze (8 of 10) is the cautious pick
|
||||
self.assertEqual([k["option"] for k in best["picks"]], ["a", "x", "g"])
|
||||
self.assertAlmostEqual(best["p"], 0.1 * 0.8)
|
||||
self.assertLess(best["pCautious"], best["p"])
|
||||
self.assertAlmostEqual(best["ends"]["BONDED:Blue"], 0.08)
|
||||
self.assertAlmostEqual(best["ends"]["DIED"], 0.92)
|
||||
self.assertEqual(self.t["best"]["Black"]["p"], 0.0)
|
||||
|
||||
def test_outcomes_of_any_picks(self):
|
||||
self.assertEqual(B.outcomes(self.t, {"start": "b"}), {"DIED": 1.0})
|
||||
self.assertEqual(B.outcomes(self.t, {"start": "c"}), {"not mapped": 1.0})
|
||||
ends = B.outcomes(self.t, {"start": "a", "blue": "l"})
|
||||
self.assertAlmostEqual(ends["BONDED:Blue"], 0.1)
|
||||
|
||||
def test_coverage_lists_the_gaps(self):
|
||||
c = self.t["coverage"]
|
||||
self.assertEqual((c["screens"], c["options"], c["tested"]), (3, 6, 5))
|
||||
self.assertEqual([u["option"] for u in c["untested"]], ["c"])
|
||||
self.assertEqual(c["endings"], ["BONDED:Blue", "DIED"])
|
||||
|
||||
def test_published_tree_is_up_to_date(self):
|
||||
self.assertEqual(B.main(["--check"]), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user