First attempt cycle #1
@@ -0,0 +1,53 @@
|
||||
# The loop with the bot, every four hours: import the bot's new runs, refit the route model, rebuild the experiment
|
||||
# plan, hand the bot its files, and open a pull request with the new public data (it merges itself once CI passes).
|
||||
# Gitea Actions. Needs a runner on the machine that has the bot's folder (the runner label below), because the
|
||||
# bot's raw data (game prose, account names) never leaves that machine: only the anonymised runs are committed.
|
||||
#
|
||||
# Variables (repository Settings -> Actions -> Variables):
|
||||
# DRAGONKIND_BOT_DIR the bot's folder on the runner, e.g. /home/ken/DragonKind
|
||||
# ATLAS_DELIVER_DIR where the bot reads its files, e.g. /home/ken/DragonKind/vendor/threshing-atlas
|
||||
# Secrets:
|
||||
# DK_ACCOUNT_SALT the salt account names are hashed with (the same one every time)
|
||||
# ATLAS_BOT_TOKEN as in update-sources.yml
|
||||
name: Bot cycle
|
||||
on:
|
||||
schedule:
|
||||
- cron: "41 */4 * * *"
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
cycle:
|
||||
runs-on: bot-host # a runner on the bot's machine; change it to that runner's label
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements.txt
|
||||
- name: Import, refit, re-plan, deliver
|
||||
env:
|
||||
DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }}
|
||||
run: python -m collection cycle --bot "${{ vars.DRAGONKIND_BOT_DIR }}" --deliver "${{ vars.ATLAS_DELIVER_DIR }}"
|
||||
- id: diff
|
||||
run: |
|
||||
git add -A data analysis/reports
|
||||
if git diff --cached --quiet; then echo "changed=false" >> "$GITHUB_OUTPUT"; else echo "changed=true" >> "$GITHUB_OUTPUT"; fi
|
||||
- name: Tests
|
||||
if: steps.diff.outputs.changed == 'true'
|
||||
run: npm test
|
||||
- name: Open a pull request with the new data (it merges itself once CI passes)
|
||||
if: steps.diff.outputs.changed == 'true'
|
||||
env:
|
||||
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||
run: |
|
||||
branch="data/bot-cycle-$(date -u +%Y%m%d-%H%M)"
|
||||
git config user.name "Atlas data bot"
|
||||
git config user.email "atlas-bot@noreply.localhost"
|
||||
git checkout -b "$branch"
|
||||
git commit -m "data: bot cycle (new runs, route model, experiment plan)"
|
||||
git push origin "$branch"
|
||||
python -m collection forge open-pr --head "$branch" --title "Bot cycle: new runs, route model and plan" --body-file exports/last-cycle.md --automerge
|
||||
+555
-524
File diff suppressed because it is too large
Load Diff
+112
-5
@@ -1,11 +1,15 @@
|
||||
"""Hand-offs without GitHub: the zip of atlas files for the bot, and the local equivalent of the daily workflow.
|
||||
|
||||
python -m collection export-bot-kit -> exports/atlas-for-bot-<date>-<policy id>.zip
|
||||
policy.json, solver.js and graph.js (what the bot vendors into vendor/threshing-atlas/), plus a README and
|
||||
policy.json, experiments.json, solver.js and graph.js (what the bot vendors into vendor/threshing-atlas/), plus a README and
|
||||
checksums. Send it to the bot's maintainer; they unzip it into vendor/threshing-atlas/.
|
||||
python -m collection refresh [--bot PATH ...]
|
||||
Every source (and any bot folders or zips given), then the graph, the route model, the statistics and the
|
||||
build: what .github/workflows/update-sources.yml does every day.
|
||||
python -m collection cycle --bot PATH [--deliver DIR] [--community] [--force]
|
||||
The loop with the bot, in one command and safe to run on a timer: import the bot's new runs, refit the
|
||||
route model, rebuild the experiment plan, and put the bot's files where it reads them. Does nothing but
|
||||
deliver when the bot has no new runs.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
@@ -14,10 +18,12 @@ import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .common import ROOT, path
|
||||
|
||||
KIT = {"policy.json": "data/public/policy.json", "solver.js": "core/solver.js", "graph.js": "core/graph.js"}
|
||||
KIT = {"policy.json": "data/public/policy.json", "experiments.json": "data/public/experiments.json",
|
||||
"solver.js": "core/solver.js", "graph.js": "core/graph.js"}
|
||||
|
||||
README = """Threshing Atlas files for the DragonKind bot
|
||||
============================================
|
||||
@@ -25,14 +31,16 @@ README = """Threshing Atlas files for the DragonKind bot
|
||||
Policy id: {policy}
|
||||
Made: {made}
|
||||
|
||||
Unzip these three files into the bot's vendor/threshing-atlas/ folder, replacing the old ones, and keep them
|
||||
Unzip these four files into the bot's vendor/threshing-atlas/ folder, replacing the old ones, and keep them
|
||||
together: solver.js reads the game from this policy.json.
|
||||
|
||||
policy.json the route model (the game, the counts behind every roll, the best routes per colour)
|
||||
solver.js the advisor: advisor(), recommendOption(), solverHistory()
|
||||
experiments.json the experiment plan: what each attempt of each account is for (read it with assign())
|
||||
solver.js the advisor: advisor(), recommendOption(), solverHistory(), assign()
|
||||
graph.js trajectoryVariables(): the bot's steps -> the community form's variables
|
||||
|
||||
How the bot uses them: docs/BOT_INTEGRATION.md in the atlas repo. Log the policy id above as policyId on each run.
|
||||
How the bot uses them: docs/BOT_INTEGRATION.md in the atlas repo. Log the policy id above as policyId on each run,
|
||||
and the plan's id as planId.
|
||||
|
||||
Checksums (sha256):
|
||||
{sums}
|
||||
@@ -76,7 +84,106 @@ def refresh(update_main, bots=()):
|
||||
else:
|
||||
print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)")
|
||||
_step("route model (about 2 minutes)", [py, "planner/build_policy.py"])
|
||||
_step("experiment plan", [py, "planner/build_experiments.py"])
|
||||
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
||||
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
|
||||
print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit")
|
||||
return rc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the automated cycle with the bot
|
||||
|
||||
def _fingerprint():
|
||||
"""What the route model and the plan are built from: the coded runs and the bot's anonymised files."""
|
||||
h = hashlib.sha256()
|
||||
for p in [path("data/public/runs_clean.csv"), *sorted(path("data/bot").glob("*.json*")),
|
||||
path("planner/policy.yaml"), path("planner/experiments.yaml"), path("core/spec/game.yaml")]:
|
||||
if p.exists():
|
||||
h.update(p.name.encode())
|
||||
h.update(p.read_bytes())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _headline():
|
||||
"""The numbers a person checks after a cycle: {label: value}."""
|
||||
out = {}
|
||||
try:
|
||||
pol = json.loads(path("data/public/policy.json").read_text(encoding="utf-8"))
|
||||
out["policy id"] = pol["id"]
|
||||
t = pol["samples"]["all"]["targets"]
|
||||
pct = lambda d: f"{100 * d['mean']:.2f}% ({100 * d['lo']:.2f}-{100 * d['hi']:.2f})" # noqa: E731
|
||||
out["Black, first attempt"] = pct(t["Black"]["firstAttempt"]["thisAttempt"]["pX"])
|
||||
out["Black, later attempt"] = pct(t["Black"]["thisAttempt"]["pX"])
|
||||
out["Blue, per attempt"] = pct(t["Blue"]["thisAttempt"]["pX"])
|
||||
out["runs the model learns from"] = pol["source"]["runs"]
|
||||
out["bot runs"] = pol["source"]["by_source"].get("bot", 0)
|
||||
except (OSError, KeyError, ValueError):
|
||||
pass
|
||||
try:
|
||||
plan = json.loads(path("data/public/experiments.json").read_text(encoding="utf-8"))
|
||||
out["plan id"] = plan["id"]
|
||||
for e in plan["experiments"]:
|
||||
p = e["progress"]
|
||||
out[f"experiment {e['id']}"] = f"{e['status']}: {p['observed']} observed, {p['rare']} rare" + (f" [{e['note']}]" if e.get("note") else "")
|
||||
except (OSError, KeyError, ValueError):
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def deliver(dest):
|
||||
"""Copy the bot's files (KIT) into a folder, each one whole (written beside it, then renamed), with a manifest
|
||||
the bot can check before an attempt: atlas-kit.json = {policyId, planId, made, sha256: {file: sum}}."""
|
||||
dest = Path(dest).expanduser()
|
||||
dest = dest if dest.is_absolute() else path(str(dest))
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
sums = {}
|
||||
for name, src in KIT.items():
|
||||
data = path(src).read_bytes()
|
||||
sums[name] = hashlib.sha256(data).hexdigest()
|
||||
tmp = dest / (name + ".incoming")
|
||||
tmp.write_bytes(data)
|
||||
tmp.replace(dest / name)
|
||||
head = _headline()
|
||||
manifest = {"policyId": head.get("policy id"), "planId": head.get("plan id"),
|
||||
"made": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "sha256": sums}
|
||||
tmp = dest / "atlas-kit.json.incoming"
|
||||
tmp.write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
|
||||
tmp.replace(dest / "atlas-kit.json") # last: a kit whose manifest matches its files is complete
|
||||
return dest
|
||||
|
||||
|
||||
def cycle(update_main, bots, deliver_to=None, community=False, force=False):
|
||||
"""Import the bot's runs -> refit -> re-plan -> deliver. Returns 0 on success. Writes exports/last-cycle.md."""
|
||||
py = sys.executable
|
||||
path("exports").mkdir(exist_ok=True)
|
||||
before, was = _fingerprint(), _headline()
|
||||
rc = 0
|
||||
if community:
|
||||
rc |= update_main(["update", "--report", str(path("exports/last-report.md"))])
|
||||
for i, b in enumerate(bots):
|
||||
rc |= update_main(["update", "--source", "bot", "--file", f"bot={b}", "--report", str(path(f"exports/last-report-bot{i + 1}.md"))])
|
||||
built = all(path(src).exists() for src in KIT.values())
|
||||
changed = force or not built or _fingerprint() != before
|
||||
if changed:
|
||||
if shutil.which("node"):
|
||||
_step("route graph", ["node", "analysis/network/build-graph.js"])
|
||||
_step("route model (a few minutes)", [py, "planner/build_policy.py"])
|
||||
_step("experiment plan", [py, "planner/build_experiments.py"])
|
||||
try:
|
||||
import scipy # noqa: F401
|
||||
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
||||
except ImportError:
|
||||
print("\n(scipy is not installed: data/public/stats.json is not rebuilt; the bot does not use it)")
|
||||
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
|
||||
else:
|
||||
print("\nNo new runs: the route model and the plan are unchanged.")
|
||||
now = _headline()
|
||||
lines = [f"# Bot cycle, {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M')} UTC", "",
|
||||
"The model and the plan were rebuilt." if changed else "No new runs: nothing was rebuilt.", "",
|
||||
"| | Before | Now |", "|---|---|---|"]
|
||||
lines += [f"| {k} | {was.get(k, '-')} | {v} |" for k, v in now.items()]
|
||||
if deliver_to:
|
||||
lines += ["", f"Delivered to `{deliver(deliver_to)}`."]
|
||||
path("exports/last-cycle.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print("\n" + "\n".join(lines))
|
||||
return rc
|
||||
|
||||
+14
-2
@@ -8,7 +8,10 @@
|
||||
Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then
|
||||
rebuild the graph, the route model, the statistics and the build.
|
||||
python -m collection export-bot-kit
|
||||
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, solver.js and graph.js for the bot.
|
||||
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, experiments.json, solver.js and graph.js.
|
||||
python -m collection cycle --bot PATH [--deliver DIR] [--community] [--force]
|
||||
The automated loop with the bot: import its new runs, refit the route model, rebuild the experiment
|
||||
plan (planner/experiments.yaml), and copy the bot's files into DIR. Safe to run on a timer.
|
||||
python -m collection build [--check]
|
||||
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
||||
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
||||
@@ -104,12 +107,21 @@ def main(argv=None):
|
||||
b.add_argument("--check", action="store_true")
|
||||
fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)")
|
||||
fg.add_argument("args", nargs=argparse.REMAINDER)
|
||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
|
||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
||||
k.add_argument("--out", default="exports")
|
||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
||||
c.add_argument("--bot", action="append", default=[], help="the bot's folder, or a .zip of its data (repeatable); default: $DRAGONKIND_BOT_DIR")
|
||||
c.add_argument("--deliver", help="a folder to put the bot's files in (its vendor/threshing-atlas/)")
|
||||
c.add_argument("--community", action="store_true", help="also check the community sources first")
|
||||
c.add_argument("--force", action="store_true", help="rebuild even when there are no new runs")
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
if a.cmd == "cycle":
|
||||
from .etl.handoff import cycle
|
||||
bots = a.bot or ([os.environ["DRAGONKIND_BOT_DIR"]] if os.environ.get("DRAGONKIND_BOT_DIR") else [])
|
||||
return cycle(main, bots, a.deliver, a.community, a.force)
|
||||
if a.cmd == "forge":
|
||||
from .etl import forge
|
||||
return forge.main(a.args)
|
||||
|
||||
+17
-1
@@ -62,6 +62,10 @@ def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000):
|
||||
e = {"rare": y, "n": n, "completeRare": ry, "completeN": rn}
|
||||
if g.get("complete"):
|
||||
e.update(rho=1.0, lo=1.0, hi=1.0, used="complete: counted as reported")
|
||||
elif ry < cfg.get("min_complete_rare", 5) and (cfg.get("borrow") or {}).get(node):
|
||||
# complete logging has (almost) never seen it, so the bias cannot be estimated for this roll: assume
|
||||
# the group over-reports it as much as it over-reports another rare roll (filled in below)
|
||||
e.update(rho=None, borrow=cfg["borrow"][node])
|
||||
elif ry < cfg.get("min_complete_rare", 5):
|
||||
# complete logging has (almost) never seen it: the bias cannot be estimated, so this roll is learned
|
||||
# from complete logging only (its range then shows how little is known)
|
||||
@@ -83,10 +87,22 @@ def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000):
|
||||
for g in groups:
|
||||
for node in rolls:
|
||||
e = out[g["name"]][node]
|
||||
if e.get("rho") is None and not e.get("exclude"):
|
||||
if e.get("rho") is None and not e.get("exclude") and not e.get("borrow"):
|
||||
src = out.get(g.get("same_as") or "", {}).get(node, {})
|
||||
e["rho"] = src.get("rho") or 1.0
|
||||
e["lo"], e["hi"] = src.get("lo", e["rho"]), src.get("hi", e["rho"])
|
||||
# a roll whose bias cannot be estimated borrows the same group's estimate for another roll (`borrow`)
|
||||
for g in groups:
|
||||
for node in rolls:
|
||||
e = out[g["name"]][node]
|
||||
if e.get("borrow"):
|
||||
src = out[g["name"]].get(e["borrow"], {})
|
||||
if src.get("rho"):
|
||||
e.update(rho=src["rho"], lo=src.get("lo"), hi=src.get("hi"),
|
||||
used=f"assumed equal to this group's bias for the {e['borrow']} roll: complete logs have only "
|
||||
f"{e['completeRare']} rare outcomes, too few to estimate it")
|
||||
else:
|
||||
e.update(lo=None, hi=None, exclude=True, used="left out: no bias estimate to borrow")
|
||||
return out, names
|
||||
|
||||
|
||||
|
||||
+24
-5
@@ -28,6 +28,8 @@ ALONE = "(alone)"
|
||||
ALPHA = 20
|
||||
ALPHA_GRID = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 20000, 100000]
|
||||
ENC_CATS = G["encounter_outcomes"]
|
||||
GIVENS = G.get("givens") or {}
|
||||
DEFAULT_GIVEN = {k: g["default"] for k, g in GIVENS.items()}
|
||||
|
||||
|
||||
def enc_colour(ctx):
|
||||
@@ -62,6 +64,8 @@ def prepare_run(r):
|
||||
v = {"company": r["company"] if r["company"] in DECISIONS["company"] else None,
|
||||
"direction": r["direction"] if r["direction"] in DECISIONS["direction"] else None,
|
||||
"fs": r["first_sighting"] if r["first_sighting"] in ("Red", "Brown", "Green", "Orange") else None}
|
||||
# the givens (game.yaml): first or later attempt, from the run's attempt number
|
||||
v["attempt"] = r.get("attempt_group") if r.get("attempt_group") in GIVENS.get("attempt", {}).get("values", ()) else None
|
||||
lf = r.get("left_squadmates")
|
||||
known = lf is not None and not (isinstance(lf, float) and lf != lf) and lf != ""
|
||||
v["left"] = ALONE if v["company"] == "Alone" else (("Leave them" if lf is True or str(lf).lower() == "true" else "Stay together") if v["company"] and known else None)
|
||||
@@ -279,7 +283,7 @@ def fit_alpha(T, node):
|
||||
pairs = []
|
||||
for key in t:
|
||||
L = int(key.split(":")[0])
|
||||
if L == 0:
|
||||
if L <= CHANCE[node].get("distinct", 0): # the top level, and levels that are not pooled (game.yaml)
|
||||
continue
|
||||
ctx = key[key.index(":") + 1:].split("|")
|
||||
parent = f"{L - 1}:" + "|".join(ctx[:L - 1])
|
||||
@@ -368,10 +372,14 @@ class Params:
|
||||
In draw mode each world draws its numbers in the order its own walk of the game first needs them (a branch with
|
||||
probability exactly zero in that world is never walked), so call solver.draw_worlds() before using it.
|
||||
chance(node, state) -> {outcome: array(D)}; encounter(ctx) -> {outcome: array(D)}.
|
||||
|
||||
given: the run's givens (game.yaml `givens`, e.g. first or later attempt); having() gives the same worlds for
|
||||
other givens.
|
||||
"""
|
||||
|
||||
def __init__(self, T, mode="mean", seeds=(), alphas=None):
|
||||
def __init__(self, T, mode="mean", seeds=(), alphas=None, given=None):
|
||||
self.T, self.mode, self.alphas = T, mode, alphas or {}
|
||||
self.given = {**DEFAULT_GIVEN, **(given or {})}
|
||||
self.rngs = [Rng(s) for s in seeds] if mode == "draw" else []
|
||||
self.D = len(self.rngs) if mode == "draw" else 1
|
||||
self.memo = {} # (node, level key) or ("enc", ctx) -> array (K, D)
|
||||
@@ -379,6 +387,14 @@ class Params:
|
||||
self.world = None # while drawing one world (see draw_world), its index
|
||||
self._cache = {}
|
||||
|
||||
def having(self, **given):
|
||||
"""The same odds (and the same posterior draws) for a run with other givens."""
|
||||
import copy
|
||||
p = copy.copy(self)
|
||||
p.given = {**self.given, **given}
|
||||
p._cache = {}
|
||||
return p
|
||||
|
||||
def _levels(self, node, state):
|
||||
sp = CHANCE[node]
|
||||
out = []
|
||||
@@ -408,6 +424,7 @@ class Params:
|
||||
if not gate_open(node, state): # a closed gate: the roll always comes out the same way
|
||||
other = CHANCE[node]["otherwise"]
|
||||
return {k: np.full(self.D, 1.0 if k == other else 0.0) for k in CHANCE[node]["outcomes"]}
|
||||
state = {**self.given, **state}
|
||||
# once every world has been drawn, the answer depends only on the context values: cache it
|
||||
ck = (node, tuple(state.get(k) for k in CHANCE[node]["context"]))
|
||||
if self.world is None and ck in self._cache:
|
||||
@@ -419,13 +436,15 @@ class Params:
|
||||
|
||||
def _chance(self, node, state):
|
||||
cats = CHANCE[node]["outcomes"]
|
||||
al = self.alphas.get(node) or ALPHA
|
||||
fitted = self.alphas.get(node) or ALPHA
|
||||
distinct = CHANCE[node].get("distinct", 0)
|
||||
p = None
|
||||
for key in self._levels(node, state):
|
||||
for L, key in enumerate(self._levels(node, state)):
|
||||
c = self.T[node].get(key, {})
|
||||
parent = p
|
||||
al = 1.0 if L <= distinct else fitted # a level that is not pooled keeps a token prior of one run
|
||||
|
||||
def make_a(d, c=c, parent=parent):
|
||||
def make_a(d, c=c, parent=parent, al=al):
|
||||
if parent is None:
|
||||
return [c.get(k, 0) + 0.5 for k in cats]
|
||||
return [c.get(k, 0) + al * (parent[i] if d is None else parent[i][d]) for i, k in enumerate(cats)]
|
||||
|
||||
+41
-1
@@ -250,13 +250,30 @@ def draw_worlds(T, seeds, alphas, support):
|
||||
"""Posterior draws, one world per seed, each drawn in its own walk of the game (see Params)."""
|
||||
p = model.Params(T, "draw", seeds=seeds, alphas=alphas)
|
||||
p.reached = {}
|
||||
default = dict(p.given)
|
||||
for d in range(p.D):
|
||||
p.world = d
|
||||
_Drawer(p, support, d, p.reached).root()
|
||||
# the same world for runs with other givens (a first attempt): only the rolls that depend on them are new
|
||||
for given in other_givens():
|
||||
p.given = {**default, **given}
|
||||
_Drawer(p, support, d, {}).root()
|
||||
p.given = default
|
||||
p.world = None
|
||||
return p
|
||||
|
||||
|
||||
def other_givens():
|
||||
"""Every non-default value of every given in game.yaml, one at a time: [{"attempt": "First attempt"}]."""
|
||||
return [{k: v} for k, g in model.GIVENS.items() for v in g["values"] if v != g["default"]]
|
||||
|
||||
|
||||
def given_key(given):
|
||||
"""The name a non-default given's results are published under: {"attempt": "First attempt"} -> "firstAttempt"."""
|
||||
words = " ".join(given.values()).split()
|
||||
return words[0].lower() + "".join(w.capitalize() for w in words[1:])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- objectives
|
||||
|
||||
def solve(params, X, support, lam=0.0, record=False, only=None):
|
||||
@@ -364,6 +381,28 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
|
||||
"crossFit": {"inSample": (cross[0]["inSample"] + cross[1]["inSample"]) / 2,
|
||||
"heldOut": (cross[0]["heldOut"] + cross[1]["heldOut"]) / 2},
|
||||
}
|
||||
# Runs with another given (a first attempt). "This attempt" is solved as above. "Eventually": every attempt
|
||||
# after it is an ordinary one, worth the default's ratio (lam), so the best play maximises
|
||||
# P(X now) + P(no bond now) * lam, which is the same score with lam fixed.
|
||||
later = [xi / (xi + oi) if xi + oi else 0.0 for xi, oi in zip(ev_ratio[0], ev_ratio[1])]
|
||||
for given in other_givens():
|
||||
m1, d1 = mean.having(**given), drawp.having(**given)
|
||||
g_one = solve(m1, X, support, record=True)
|
||||
g_ev = solve(m1, X, support, lam=ratio.lam, record=True)
|
||||
e_one = evaluate(d1, X, support, g_one.decide)
|
||||
e_ev = evaluate(d1, X, support, g_ev.decide)
|
||||
|
||||
def gblock(sol, ev, dr, m1=m1):
|
||||
x, o = ev[0], ev[1]
|
||||
return {"pX": {**summarise(x), "point": float(sol.value[0][0])},
|
||||
"pOther": {**summarise(o), "point": float(sol.value[1][0])},
|
||||
"tree": Exporter(m1, X, support, sol, dr).root()}
|
||||
ev_block = gblock(g_ev, e_ev, solve(d1, X, support, lam=d_ratio.lam))
|
||||
x0, o0 = float(g_ev.value[0][0]), float(g_ev.value[1][0])
|
||||
ev_block["pEventually"] = {**summarise([x + (1 - x - o) * r for x, o, r in zip(e_ev[0], e_ev[1], later)]),
|
||||
"point": x0 + (1 - x0 - o0) * float(np.ravel(ratio.lam)[0])}
|
||||
targets[X][given_key(given)] = {"given": given, "thisAttempt": gblock(g_one, e_one, solve(d1, X, support)),
|
||||
"eventually": ev_block}
|
||||
return {
|
||||
"schema": 1,
|
||||
"model": {
|
||||
@@ -371,12 +410,13 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
|
||||
"alphas": alphas, "alphaFit": {k: f["logLik"] for k, f in fitted.items()}, "draws": draws, "seed": seed,
|
||||
"minSupport": min_support,
|
||||
"chanceContext": {k: s["context"] for k, s in model.CHANCE.items()},
|
||||
"given": model.DEFAULT_GIVEN,
|
||||
"decisions": model.DECISIONS, "support": support, "supportCounts": counts,
|
||||
"tables": T,
|
||||
# how much each kind of source over-reports each rare roll, and the correction applied (bias.py)
|
||||
"reportingBias": bias_est,
|
||||
# the game itself (core/spec/game.yaml), so the advisor in core/solver.js can re-solve from any point
|
||||
"game": {k: G[k] for k in ("version", "colours", "decisions", "chance", "encounters", "encounter_outcomes",
|
||||
"game": {k: G[k] for k in ("version", "colours", "givens", "decisions", "chance", "encounters", "encounter_outcomes",
|
||||
"by_colour", "start", "steps", "from_form", "bot_text")},
|
||||
},
|
||||
"data": {"runs": len(runs)},
|
||||
|
||||
+100
-14
@@ -11,6 +11,9 @@
|
||||
*
|
||||
* Public API (unchanged for the bot): advisor(sample, colour, objective, { only }), recommendOption(adv, h, texts),
|
||||
* solverHistory(formVars, seen), makeParams(tables, { alphas, game }), ENCOUNTERS (set from the first sample).
|
||||
* Since November 2026: advisor options `attempt` (the account's attempt number: a first attempt has its own
|
||||
* shadow odds), `fix` (hold some decisions to given options: an experiment arm) and sampling "uniform" (mapping);
|
||||
* assign(plan, { accountSeq, attempt }) reads the experiment plan (experiments.json) and says what an attempt is for.
|
||||
* UMD: window.ThreshSolver in the browser, module.exports in Node. No dependencies.
|
||||
*/
|
||||
(function (root, factory) {
|
||||
@@ -48,22 +51,31 @@
|
||||
}
|
||||
|
||||
/* ---------- the odds: posterior means (or, with rand, one posterior draw) with hierarchical back-off ---------- */
|
||||
function makeParams(T, { alphas = {}, game, rand = null }) {
|
||||
/** The givens of a run (game.yaml `givens`): the defaults, with `attempt` set from an attempt number or name. */
|
||||
function givenFor(game, { attempt = null, given = null } = {}) {
|
||||
const g = Object.fromEntries(Object.entries(game.givens || {}).map(([k, v]) => [k, v.default]));
|
||||
if (attempt != null && game.givens && game.givens.attempt) g.attempt = attempt === 1 || attempt === "1" || attempt === "First attempt" ? "First attempt" : "Later attempt";
|
||||
return { ...g, ...(given || {}) };
|
||||
}
|
||||
// given: the run's givens; memo: pass the same Map to get the same posterior draw for another given
|
||||
function makeParams(T, { alphas = {}, game, rand = null, given = null, memo = new Map() }) {
|
||||
const normalise = (a) => { let sum = 0; for (const x of a) sum += x; return a.map((x) => x / sum); };
|
||||
const estimate = rand ? (a) => dirichlet(a, rand) : normalise;
|
||||
const memo = new Map();
|
||||
const known = given || givenFor(game);
|
||||
const ctxKey = (s, names) => names.map((k) => (s[k] == null ? "?" : s[k])).join("|");
|
||||
function chance(node, state) {
|
||||
function chance(node, walked) {
|
||||
const sp = game.chance[node], cats = sp.outcomes;
|
||||
// a closed gate (game.yaml `requires`): the roll always comes out as `otherwise`
|
||||
if (sp.requires && Object.entries(sp.requires).some(([k, v]) => state[k] !== v)) return Object.fromEntries(cats.map((k) => [k, k === sp.otherwise ? 1 : 0]));
|
||||
if (sp.requires && Object.entries(sp.requires).some(([k, v]) => walked[k] !== v)) return Object.fromEntries(cats.map((k) => [k, k === sp.otherwise ? 1 : 0]));
|
||||
const state = { ...known, ...walked };
|
||||
let p = null;
|
||||
for (let L = 0; L <= sp.context.length; L++) {
|
||||
const names = sp.context.slice(0, L);
|
||||
if (names.some((k) => state[k] == null)) break;
|
||||
const key = L + ":" + ctxKey(state, names), mk = node + "#" + key;
|
||||
if (memo.has(mk)) { p = memo.get(mk); continue; }
|
||||
const c = T[node][key] || {}, al = alphas[node] || ALPHA;
|
||||
// a level that is not pooled (game.yaml `distinct`) keeps a token prior of one run
|
||||
const c = T[node][key] || {}, al = L <= (sp.distinct || 0) ? 1 : alphas[node] || ALPHA;
|
||||
const a = cats.map((k, i) => (c[k] || 0) + (p ? al * p[i] : 0.5));
|
||||
p = estimate(a);
|
||||
memo.set(mk, p);
|
||||
@@ -174,22 +186,50 @@
|
||||
* the deadly roll just passed on the step after it. When h already has a later step but skips an earlier
|
||||
* decision (an older form record, say), the recommended option is assumed for the skipped one.
|
||||
*/
|
||||
function advisor(sample, X, objective = "eventually", { only = null, sampling = "best", seed = 1, draws = 40 } = {}) {
|
||||
function advisor(sample, X, objective = "eventually", { only = null, sampling = "best", seed = 1, draws = 40, attempt = null, given = null, fix = null } = {}) {
|
||||
const m = sample.model, game = m.game;
|
||||
api.game = game; api.ENCOUNTERS = game.by_colour;
|
||||
const solveWith = (params) => (objective === "eventually" ? solveRatio(params, game, m.support, X, only) : solve(params, game, m.support, X, 0, only));
|
||||
// attempt: the account's attempt number (or "First attempt" / "Later attempt"). A first attempt has its own odds
|
||||
// for the rolls that depend on it (the shadow); without it, a later attempt is assumed.
|
||||
const defaults = givenFor(game), known = givenFor(game, { attempt, given });
|
||||
const special = Object.keys(known).some((k) => known[k] !== defaults[k]);
|
||||
// "uniform" (mapping) may take any option the game has, so it is solved without the minimum-support rule
|
||||
const all = Object.fromEntries(Object.entries(game.decisions).map(([k, d]) => [k, d.options]));
|
||||
const base = sampling === "uniform" ? { ...all, encounter: m.support.encounter } : m.support;
|
||||
// fix: { decision: option } holds those decisions to one option (an experiment arm); the rest is solved around it
|
||||
let support = base;
|
||||
if (fix) {
|
||||
support = { ...base };
|
||||
for (const [k, o] of Object.entries(fix)) {
|
||||
if (!game.decisions[k] || !game.decisions[k].options.includes(o)) throw new Error(`fix: ${k} has no option "${o}"`);
|
||||
support[k] = [o];
|
||||
if (game.decisions[k].support_by) for (const key of Object.keys(support)) if (key.startsWith(k + ":")) support[key] = [o];
|
||||
}
|
||||
}
|
||||
// One world (the posterior means, or one draw): the odds for this run's givens, solved for the objective.
|
||||
// "Eventually" on a first attempt: every later attempt is an ordinary one, worth its own ratio (lam), so the
|
||||
// best play now maximises P(X now) + P(no bond now) * lam: the same score with lam fixed.
|
||||
const solveWith = (rand) => {
|
||||
const memo = new Map();
|
||||
const mk = (g) => makeParams(m.tables, { alphas: m.alphas, game, rand, given: g, memo });
|
||||
if (objective !== "eventually") { const params = mk(known); return { params, sol: solve(params, game, support, X, 0, only) }; }
|
||||
if (!special) { const params = mk(known); return { params, sol: solveRatio(params, game, support, X, only) }; }
|
||||
const lam = solveRatio(mk(defaults), game, base, X, only).lam;
|
||||
const params = mk(known);
|
||||
return { params, sol: { ...solve(params, game, support, X, lam, only), lam } };
|
||||
};
|
||||
// "best": the best route under the posterior means. "thompson": the best route of one posterior draw of the odds
|
||||
// (one per attempt: make a new advisor with a new seed for each attempt), so every option is tried about as
|
||||
// often as it is likely to be the best one; as the data firm up, sampling settles on the best route.
|
||||
// "uniform": every option on the screen is equally likely (recommendOption picks; see there).
|
||||
const thompson = sampling === "thompson";
|
||||
const params = makeParams(m.tables, { alphas: m.alphas, game, rand: thompson ? rng(seed) : null });
|
||||
const sol = solveWith(params);
|
||||
const { params, sol } = solveWith(thompson ? rng(seed) : null);
|
||||
// the chance Thompson sampling picks each option at a history, estimated from `draws` other draws: the
|
||||
// propensity to log, so the runs can be re-weighted later
|
||||
let worlds = null;
|
||||
const propensityAt = (key, choice) => {
|
||||
if (!thompson) return 1;
|
||||
if (!worlds) worlds = Array.from({ length: draws }, (_, i) => solveWith(makeParams(m.tables, { alphas: m.alphas, game, rand: rng(seed * 7919 + i + 1) })).choices);
|
||||
if (!worlds) worlds = Array.from({ length: draws }, (_, i) => solveWith(rng(seed * 7919 + i + 1)).sol.choices);
|
||||
return worlds.filter((c) => c.get(key) === choice).length / draws;
|
||||
};
|
||||
const VARS = new Set([...Object.keys(game.decisions), ...Object.keys(game.chance), "encounter"]);
|
||||
@@ -236,7 +276,7 @@
|
||||
throw new Error("the game has no end");
|
||||
}
|
||||
return {
|
||||
colour: X, objective, only, sampling, seed: thompson ? seed : null, value: sol.value, step, params, game,
|
||||
colour: X, objective, only, sampling, seed: sampling === "best" ? null : seed, given: known, fix, value: sol.value, step, params, game,
|
||||
/** attempts recorded at an encounter (genuine attempts only), to show how much data a bond chance rests on */
|
||||
attempts: (ctx) => Object.values(((m.tables.encounter || {})[ctx] || { outcomes: {} }).outcomes).reduce((a, b) => a + b, 0),
|
||||
/** the next decision only (null when the next step is a roll or the run is over) */
|
||||
@@ -258,8 +298,18 @@
|
||||
function recommendOption(adv, h, offered) {
|
||||
const rec = adv.next(h);
|
||||
if (!rec) return null;
|
||||
const pat = ((adv.game.bot_text || {})[rec.decision] || {})[rec.choice];
|
||||
const idx = pat ? offered.map(norm).findIndex((t) => new RegExp(pat).test(t)) : -1;
|
||||
const texts = offered.map(norm), pats = (adv.game.bot_text || {})[rec.decision] || {};
|
||||
// mapping (sampling "uniform"): every option on the screen that the game knows is equally likely, except a
|
||||
// decision held by `fix` and the bonding encounter (where the advisor's own choice stands)
|
||||
if (adv.sampling === "uniform" && rec.decision !== "encounter" && !(adv.fix && adv.fix[rec.decision])) {
|
||||
const on = Object.keys(rec.options).map((o) => [o, pats[o] ? texts.findIndex((t) => new RegExp(pats[o]).test(t)) : -1]).filter(([, i]) => i >= 0);
|
||||
if (on.length) {
|
||||
const [o, i] = on[Math.floor(rng(hash(`${adv.seed}|${rec.decision}|${canon(h)}`))() * on.length)];
|
||||
return { index: i, decision: rec.decision, choice: o, propensity: 1 / on.length, margin: null, options: rec.options, reason: `${rec.label}: ${o} (mapping: 1 of ${on.length} at random)` };
|
||||
}
|
||||
}
|
||||
const pat = pats[rec.choice];
|
||||
const idx = pat ? texts.findIndex((t) => new RegExp(pat).test(t)) : -1;
|
||||
const values = Object.values(rec.options).map((v) => v.pX).sort((a, b) => b - a);
|
||||
return { index: idx < 0 ? null : idx, decision: rec.decision, choice: rec.choice, propensity: rec.propensity,
|
||||
margin: values.length > 1 ? values[0] - values[1] : null,
|
||||
@@ -302,8 +352,44 @@
|
||||
return h;
|
||||
}
|
||||
|
||||
/* ---------- the experiment plan (planner/experiments.yaml, published as experiments.json) ---------- */
|
||||
function hash(s) { let x = 2166136261; for (const ch of String(s)) { x ^= ch.codePointAt(0); x = Math.imul(x, 16777619) >>> 0; } return x || 1; }
|
||||
/** The arm of the n-th unit (0-based) of an experiment: balanced blocks, each block a seeded shuffle of the arms. */
|
||||
function armFor(experiment, n) {
|
||||
const arms = experiment.arms, k = arms.length, rand = rng(hash(`${experiment.id}|${experiment.seed || 0}|${Math.floor(n / k)}`));
|
||||
const order = arms.map((_, i) => i);
|
||||
for (let i = k - 1; i > 0; i--) { const j = Math.floor(rand() * (i + 1)); [order[i], order[j]] = [order[j], order[i]]; }
|
||||
return arms[order[n % k]];
|
||||
}
|
||||
/**
|
||||
* What an attempt is for, under the plan: assign(plan, { accountSeq, attempt }) ->
|
||||
* { experimentId, arm, fix, target, objective, sampling, seed, propensity, burn, planId }, or null when no
|
||||
* experiment covers the attempt (use the account's own goal).
|
||||
* accountSeq: the account's number in the order the accounts were made (0, 1, 2, ...), the same for its whole
|
||||
* life; attempt: the account's attempt number (1 = its first threshing). The answer is the same every time for
|
||||
* the same plan id, so a restart mid-attempt changes nothing. Pass the result to advisor():
|
||||
* advisor(policy.samples.all, a.target, a.objective, { attempt, fix: a.fix, sampling: a.sampling, seed: a.seed })
|
||||
* and log experimentId, arm, planId and the propensity on the run.
|
||||
*/
|
||||
function assign(plan, { accountSeq, attempt }) {
|
||||
let open = (plan.experiments || []).filter((e) => e.status === "running" && attempt >= e.attempts.from && (e.attempts.to == null || attempt <= e.attempts.to)
|
||||
&& (!e.accounts || (accountSeq % e.accounts.every) < e.accounts.take));
|
||||
if (!open.length) return null;
|
||||
if (open.some((e) => e.exclusive)) open = open.filter((e) => e.exclusive); // an exclusive experiment takes its attempts whole
|
||||
const total = open.reduce((s, e) => s + e.share, 0);
|
||||
let u = rng(hash(`${plan.seed || 0}|${accountSeq}|${attempt}`))() * total, exp = open[open.length - 1];
|
||||
for (const e of open) { if (u < e.share) { exp = e; break; } u -= e.share; }
|
||||
// the unit that is randomised: the account (one arm for life) or the attempt (blocks within the account)
|
||||
// (accounts are counted among those the experiment takes, so its blocks stay balanced)
|
||||
const nth = exp.accounts ? Math.floor(accountSeq / exp.accounts.every) * exp.accounts.take + (accountSeq % exp.accounts.every) : accountSeq;
|
||||
const arm = exp.unit === "account" ? armFor(exp, nth) : armFor({ ...exp, seed: `${exp.seed || 0}|${accountSeq}` }, attempt - exp.attempts.from);
|
||||
return { experimentId: exp.id, arm: arm.id, fix: arm.fix || null, target: exp.target, objective: exp.objective || "eventually",
|
||||
sampling: exp.sampling || "best", seed: hash(`${plan.seed || 0}|${exp.id}|${accountSeq}|${attempt}`),
|
||||
propensity: (exp.share / total) / exp.arms.length, burn: exp.burn || null, planId: plan.id };
|
||||
}
|
||||
|
||||
return Object.assign(api, {
|
||||
advisor, recommendOption, solverHistory, norm,
|
||||
advisor, recommendOption, solverHistory, norm, assign, armFor, givenFor,
|
||||
makeParams: (T, { alphas, game } = {}) => makeParams(T, { alphas, game: game || api.game }),
|
||||
/** remember the game of a policy sample (sets ENCOUNTERS) without solving anything */
|
||||
useGame(sample) { api.game = sample.model.game; api.ENCOUNTERS = api.game.by_colour; return api.game; },
|
||||
|
||||
+19
-3
@@ -9,6 +9,16 @@ version: 1
|
||||
# Colours in the order the route model reports them
|
||||
colours: [Red, Orange, Brown, Green, Blue, Black]
|
||||
|
||||
# ---------------------------------------------------------------- givens: what a run starts with
|
||||
# Known before the first choice; neither decided nor rolled. A roll can depend on a given by naming it in its
|
||||
# context. The route model solves the game once per value; `default` is the one used when nothing is said.
|
||||
givens:
|
||||
attempt:
|
||||
label: Attempt
|
||||
# from the run's attempt number (attempt_group in data/public/runs_clean.csv)
|
||||
values: [First attempt, Later attempt]
|
||||
default: Later attempt
|
||||
|
||||
# ---------------------------------------------------------------- decisions: the rider's choices
|
||||
# options: every option the game offers, in a fixed order (ties go to the first). Options seen fewer than
|
||||
# min_support times in the data are not recommended, except where `always_offered` says so.
|
||||
@@ -48,8 +58,10 @@ min_support: 20
|
||||
# ---------------------------------------------------------------- chance events: the game's rolls
|
||||
# outcomes: in a fixed order. context: the earlier steps the odds may depend on, most important first; the
|
||||
# estimate for a full context is shrunk toward the estimate for a shorter one (hierarchical Dirichlet back-off,
|
||||
# strength learned from the data). requires: earlier choices without which the roll always comes out as
|
||||
# `otherwise` (a gate); runs that did not make them say nothing about the roll and are not counted.
|
||||
# strength learned from the data). distinct: the first N context steps are not pooled at all: each of their values
|
||||
# is its own roll (only a token prior of one run ties it to the rest). requires: earlier choices without which the
|
||||
# roll always comes out as `otherwise` (a gate); runs that did not make them say nothing about the roll and are
|
||||
# not counted.
|
||||
chance:
|
||||
fs: { label: First dragon, outcomes: [Red, Brown, Green, Orange], context: [direction, company, left] }
|
||||
# rolls that can kill you: risk_label names the danger when the advisor reports it
|
||||
@@ -59,7 +71,11 @@ chance:
|
||||
shadow:
|
||||
label: Shadow at the tree
|
||||
outcomes: ["yes", "no"]
|
||||
context: [cadet, post, fs, direction, company]
|
||||
# The shadow is close to a first-attempt event (November 2026, form submissions, gated tree runs: 69 shadows in
|
||||
# 748 first attempts, 7 in 1,995 later ones, the same in every week), so first and later attempts are two
|
||||
# different rolls. Routes are then pooled within each.
|
||||
context: [attempt, cadet, post, fs, direction, company]
|
||||
distinct: 1
|
||||
# the "Black Dragon Gates" of the community route PDF (November 2026). The runs agree: in the spreadsheet,
|
||||
# 78 shadows in 4199 tree runs that investigated and helped, 2 (both form submissions) in 646 that did not.
|
||||
requires: { scream: Investigate the scream, clearing: Help your squadmate }
|
||||
|
||||
+2
-2
@@ -138,8 +138,8 @@
|
||||
"description": "The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).",
|
||||
"contentUrl": "https://git.kenschiano.net/YOUR-GITEA-USERNAME/threshing-atlas/raw/branch/main/data/public/policy.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "156c8ac049db20ecbc32cdfa4ea9054bb1575770073d82b1614bd14dfd45b257",
|
||||
"contentSize": "599165 B"
|
||||
"sha256": "5a5baa8b7e740d60c8cbfcb3e07a5b26a9a42fda95bf6d7d155e34b6b9c17d8e",
|
||||
"contentSize": "1082300 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"seed": 20261112,
|
||||
"lifecycle": {
|
||||
"research_days": 3,
|
||||
"retire": true,
|
||||
"long_lived": {
|
||||
"every": 10,
|
||||
"take": 1
|
||||
}
|
||||
},
|
||||
"experiments": [
|
||||
{
|
||||
"id": "first-attempt-shadow",
|
||||
"status": "running",
|
||||
"question": "How often does the shadow appear on an account's first attempt, and does heading West (at the start, at the fork) raise it?",
|
||||
"attempts": {
|
||||
"from": 1,
|
||||
"to": 1
|
||||
},
|
||||
"unit": "account",
|
||||
"share": 1,
|
||||
"exclusive": true,
|
||||
"target": "Black",
|
||||
"objective": "eventually",
|
||||
"sampling": "best",
|
||||
"seed": 20261112,
|
||||
"measures": {
|
||||
"roll": "shadow",
|
||||
"rare": "yes"
|
||||
},
|
||||
"burn": {
|
||||
"reason": "TREE_NO_SHADOW_AUTO_BURN",
|
||||
"when": "no shadow at the tree"
|
||||
},
|
||||
"arms": [
|
||||
{
|
||||
"id": "direction=West / post=West",
|
||||
"levels": {
|
||||
"direction": "West",
|
||||
"post": "West"
|
||||
},
|
||||
"fix": {
|
||||
"scream": "Investigate the scream",
|
||||
"clearing": "Help your squadmate",
|
||||
"hour": "Tree",
|
||||
"direction": "West",
|
||||
"post": "West"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 60
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "direction=West / post=East",
|
||||
"levels": {
|
||||
"direction": "West",
|
||||
"post": "East"
|
||||
},
|
||||
"fix": {
|
||||
"scream": "Investigate the scream",
|
||||
"clearing": "Help your squadmate",
|
||||
"hour": "Tree",
|
||||
"direction": "West",
|
||||
"post": "East"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 60
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "direction=East / post=West",
|
||||
"levels": {
|
||||
"direction": "East",
|
||||
"post": "West"
|
||||
},
|
||||
"fix": {
|
||||
"scream": "Investigate the scream",
|
||||
"clearing": "Help your squadmate",
|
||||
"hour": "Tree",
|
||||
"direction": "East",
|
||||
"post": "West"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 60
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "direction=East / post=East",
|
||||
"levels": {
|
||||
"direction": "East",
|
||||
"post": "East"
|
||||
},
|
||||
"fix": {
|
||||
"scream": "Investigate the scream",
|
||||
"clearing": "Help your squadmate",
|
||||
"hour": "Tree",
|
||||
"direction": "East",
|
||||
"post": "East"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 60
|
||||
}
|
||||
}
|
||||
],
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"unknownArms": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "attempt-2-shadow",
|
||||
"status": "running",
|
||||
"question": "Is the shadow really gone after the first attempt?",
|
||||
"attempts": {
|
||||
"from": 2,
|
||||
"to": 2
|
||||
},
|
||||
"unit": "account",
|
||||
"share": 1,
|
||||
"accounts": {
|
||||
"every": 2,
|
||||
"take": 1
|
||||
},
|
||||
"exclusive": true,
|
||||
"target": "Black",
|
||||
"objective": "eventually",
|
||||
"sampling": "best",
|
||||
"seed": 20261112,
|
||||
"measures": {
|
||||
"roll": "shadow",
|
||||
"rare": "yes"
|
||||
},
|
||||
"burn": {
|
||||
"reason": "TREE_NO_SHADOW_AUTO_BURN",
|
||||
"when": "no shadow at the tree"
|
||||
},
|
||||
"arms": [
|
||||
{
|
||||
"id": "all",
|
||||
"fix": {
|
||||
"scream": "Investigate the scream",
|
||||
"clearing": "Help your squadmate",
|
||||
"hour": "Tree"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 120
|
||||
}
|
||||
}
|
||||
],
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"unknownArms": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "meadow-blue-route",
|
||||
"status": "running",
|
||||
"question": "Does anything before the meadow change how often the Blue waits there?",
|
||||
"attempts": {
|
||||
"from": 2,
|
||||
"to": null
|
||||
},
|
||||
"unit": "attempt",
|
||||
"share": 0.6,
|
||||
"target": "Blue",
|
||||
"objective": "eventually",
|
||||
"sampling": "best",
|
||||
"seed": 20261112,
|
||||
"measures": {
|
||||
"roll": "meadow",
|
||||
"rare": "Blue"
|
||||
},
|
||||
"burn": {
|
||||
"reason": "MEADOW_NO_BLUE_AUTO_BURN",
|
||||
"when": "no Blue in the meadow"
|
||||
},
|
||||
"arms": [
|
||||
{
|
||||
"id": "scream=Investigate the scream / direction=West",
|
||||
"levels": {
|
||||
"scream": "Investigate the scream",
|
||||
"direction": "West"
|
||||
},
|
||||
"fix": {
|
||||
"hour": "Valley",
|
||||
"valley": "Pass the Red (meadow)",
|
||||
"scream": "Investigate the scream",
|
||||
"direction": "West"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 70
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "scream=Investigate the scream / direction=East",
|
||||
"levels": {
|
||||
"scream": "Investigate the scream",
|
||||
"direction": "East"
|
||||
},
|
||||
"fix": {
|
||||
"hour": "Valley",
|
||||
"valley": "Pass the Red (meadow)",
|
||||
"scream": "Investigate the scream",
|
||||
"direction": "East"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 70
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "scream=Investigate the scream / direction=Center",
|
||||
"levels": {
|
||||
"scream": "Investigate the scream",
|
||||
"direction": "Center"
|
||||
},
|
||||
"fix": {
|
||||
"hour": "Valley",
|
||||
"valley": "Pass the Red (meadow)",
|
||||
"scream": "Investigate the scream",
|
||||
"direction": "Center"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 70
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "scream=Keep walking / direction=West",
|
||||
"levels": {
|
||||
"scream": "Keep walking",
|
||||
"direction": "West"
|
||||
},
|
||||
"fix": {
|
||||
"hour": "Valley",
|
||||
"valley": "Pass the Red (meadow)",
|
||||
"scream": "Keep walking",
|
||||
"direction": "West"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 70
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "scream=Keep walking / direction=East",
|
||||
"levels": {
|
||||
"scream": "Keep walking",
|
||||
"direction": "East"
|
||||
},
|
||||
"fix": {
|
||||
"hour": "Valley",
|
||||
"valley": "Pass the Red (meadow)",
|
||||
"scream": "Keep walking",
|
||||
"direction": "East"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 70
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "scream=Keep walking / direction=Center",
|
||||
"levels": {
|
||||
"scream": "Keep walking",
|
||||
"direction": "Center"
|
||||
},
|
||||
"fix": {
|
||||
"hour": "Valley",
|
||||
"valley": "Pass the Red (meadow)",
|
||||
"scream": "Keep walking",
|
||||
"direction": "Center"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 70
|
||||
}
|
||||
}
|
||||
],
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"unknownArms": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "mapping",
|
||||
"status": "running",
|
||||
"question": "What does every branch of the valley lead to?",
|
||||
"attempts": {
|
||||
"from": 2,
|
||||
"to": null
|
||||
},
|
||||
"unit": "attempt",
|
||||
"share": 0.25,
|
||||
"target": "Blue",
|
||||
"objective": "eventually",
|
||||
"sampling": "uniform",
|
||||
"seed": 20261112,
|
||||
"measures": null,
|
||||
"burn": {
|
||||
"reason": "NOT_TARGET_AUTO_BURN",
|
||||
"when": "the encounter is not a Blue or a Black"
|
||||
},
|
||||
"arms": [
|
||||
{
|
||||
"id": "all",
|
||||
"fix": {},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": 600
|
||||
}
|
||||
}
|
||||
],
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"unknownArms": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "sentinel",
|
||||
"status": "running",
|
||||
"question": "Have the game's odds changed?",
|
||||
"attempts": {
|
||||
"from": 2,
|
||||
"to": null
|
||||
},
|
||||
"unit": "attempt",
|
||||
"share": 0.15,
|
||||
"target": "Blue",
|
||||
"objective": "eventually",
|
||||
"sampling": "best",
|
||||
"seed": 20261112,
|
||||
"measures": {
|
||||
"roll": "meadow",
|
||||
"rare": "Blue"
|
||||
},
|
||||
"burn": {
|
||||
"reason": "MEADOW_NO_BLUE_AUTO_BURN",
|
||||
"when": "no Blue in the meadow"
|
||||
},
|
||||
"arms": [
|
||||
{
|
||||
"id": "all",
|
||||
"fix": {
|
||||
"company": "Alone",
|
||||
"direction": "Center",
|
||||
"scream": "Keep walking",
|
||||
"wounded": "Tourniquet",
|
||||
"hour": "Valley",
|
||||
"valley": "Pass the Red (meadow)"
|
||||
},
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"planned": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"progress": {
|
||||
"runs": 0,
|
||||
"observed": 0,
|
||||
"rare": 0,
|
||||
"unknownArms": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"id": "ff265e6f994d"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+58
-5
@@ -9,6 +9,56 @@ Data flows both ways:
|
||||
└──────────── collection --source bot + PR ◄┘
|
||||
```
|
||||
|
||||
## The automated cycle
|
||||
|
||||
One command closes the loop, and it is meant to run on a timer on the machine that has the bot's folder:
|
||||
|
||||
```
|
||||
python -m collection cycle --bot /path/to/DragonKind --deliver /path/to/DragonKind/vendor/threshing-atlas
|
||||
```
|
||||
|
||||
1. **Ingest.** It imports the bot's new runs (anonymised: no prose, no account names; set `DK_ACCOUNT_SALT`, the
|
||||
same value every time, so an account keeps its hash).
|
||||
2. **Refit.** When there are new runs it rebuilds the coded runs, the route model (`policy.json`) and
|
||||
3. **Re-plan:** the experiment plan (`experiments.json`): progress per arm, and which experiments are still
|
||||
running (`planner/experiments.yaml` is the design).
|
||||
4. **Deliver.** It copies `policy.json`, `experiments.json`, `solver.js` and `graph.js` into the `--deliver` folder,
|
||||
each file whole, then writes `atlas-kit.json` (`policyId`, `planId`, checksums) last. The bot should load the
|
||||
kit at the start of an attempt, never in the middle of one, and log both ids on the run.
|
||||
|
||||
With no new runs it only delivers, so running it every hour costs nothing. `exports/last-cycle.md` says what
|
||||
changed (ids, headline odds, experiment progress). `--community` also checks the community sources first.
|
||||
On Gitea, `.gitea/workflows/bot-cycle.yml` runs the same command on a runner that can see the bot's folder and
|
||||
opens the pull request with the new data.
|
||||
|
||||
### What each attempt is for: the plan
|
||||
|
||||
```js
|
||||
const plan = require('../vendor/threshing-atlas/experiments.json');
|
||||
// accountSeq: the account's number in the order accounts were made (0, 1, 2, ...), fixed for its life
|
||||
// attempt: the account's attempt number (1 = its first threshing)
|
||||
const a = Solver.assign(plan, { accountSeq, attempt });
|
||||
// -> { experimentId, arm, fix, target, objective, sampling, seed, propensity, burn, planId } (null: no experiment)
|
||||
const advisor = Solver.advisor(policy.samples.all, a.target, a.objective,
|
||||
{ attempt, fix: a.fix, sampling: a.sampling, seed: a.seed });
|
||||
```
|
||||
|
||||
- **`attempt` matters.** The shadow is close to a first-attempt event, so `advisor(..., { attempt: 1 })` uses the
|
||||
first attempt's odds. Without `attempt` a later attempt is assumed. **Attempt 1 must be played by the bot and
|
||||
logged** (today the bot's records start at attempt 2).
|
||||
- **`fix`** holds some decisions to the arm's options; the advisor plays its best around them.
|
||||
- **`sampling: "uniform"`** (the mapping experiment): `recommendOption()` picks one of the options on the screen
|
||||
at random and returns its propensity.
|
||||
- Log on the run: `experimentId`, `arm`, `planId: plan.id`, `policyId: policy.id`, `accountAttempt`, and the
|
||||
propensity. On the steps an arm fixes, the propensity is `a.propensity`.
|
||||
- `burn` names the deliberate-burn rule the experiment expects (`TREE_NO_SHADOW_AUTO_BURN`,
|
||||
`MEADOW_NO_BLUE_AUTO_BURN`). A genuine Black or Blue bond is always accepted.
|
||||
- The account's life (`plan.lifecycle`): attempt 1 is the Black attempt; then about three days of research
|
||||
attempts; then the account retires, except one in ten, which keeps going so that late attempts are observed.
|
||||
|
||||
An experiment that reaches its planned count becomes `complete`, one that meets its stopping rule becomes `paused`;
|
||||
either way `assign()` stops giving it out and its share goes to the others. Nothing in the bot changes.
|
||||
|
||||
## From the Atlas's Gitea repository
|
||||
|
||||
Both repositories live on git.kenschiano.net. The bot can read the atlas files straight from the repository with a
|
||||
@@ -20,7 +70,8 @@ https://git.kenschiano.net/<owner>/threshing-atlas/raw/branch/main/core/solver.j
|
||||
https://git.kenschiano.net/<owner>/threshing-atlas/raw/branch/main/core/graph.js
|
||||
```
|
||||
|
||||
(header `Authorization: token <token>`). Keep the three files together: solver.js reads the game from policy.json.
|
||||
(header `Authorization: token <token>`), plus `data/public/experiments.json`. Keep the files together: solver.js
|
||||
reads the game from policy.json.
|
||||
|
||||
## Without a shared repository: swapping zips
|
||||
|
||||
@@ -174,9 +225,10 @@ The tie rule below still works for goal accounts; research accounts don't need i
|
||||
- `samples.lowBias`: bulk logs and riders not chasing a rare dragon. Use it for absolute expectations, such as how
|
||||
many attempts a Blue should take. Rare sightings are over-reported in form posts, so `all` is optimistic for Blue.
|
||||
|
||||
Current headline, with the bot's runs and the reporting-bias correction (November 2026): Black about 0.03% per
|
||||
attempt (90% range 0–0.15%: the bot and bulk logs have never seen the shadow in about 700 tree runs), Blue about
|
||||
1.8–2.5% per attempt (40–55 attempts), depending on the sample.
|
||||
Current headline (November 2026): Black about 1.4% on an account's first attempt (95% range 0.7–2.2%; 0.8% in the
|
||||
low-bias sample) and about 0.07% on any later attempt; Blue about 1.9–2.5% per attempt (40–55 attempts), depending
|
||||
on the sample. The first-attempt figure rests on form submissions and an assumed reporting bias (see below): the
|
||||
bot and bulk logs have only 28 first attempts at the tree, and no shadow in about 570 tree runs in all.
|
||||
|
||||
The shadow is gated: it can only appear on runs that investigated the scream and helped the squadmate (the
|
||||
"Black Dragon Gates" of the community route PDF, and the runs agree). The model knows this
|
||||
@@ -184,7 +236,8 @@ The shadow is gated: it can only appear on runs that investigated the scream and
|
||||
|
||||
Form submissions over-report rare sightings: riders report a meadow Blue about 4 times more readily than a meadow
|
||||
without one. The model estimates that bias for each kind of source and corrects for it (`planner/policy.yaml`,
|
||||
`reporting_bias`; the estimates are in `policy.json` under `model.reportingBias`).
|
||||
`reporting_bias`; the estimates are in `policy.json` under `model.reportingBias`). For the shadow the bias cannot be
|
||||
measured (complete logs have never seen one), so it is assumed equal to the meadow Blue's (`borrow`).
|
||||
|
||||
## 2. From the bot to the atlas
|
||||
|
||||
|
||||
+2
-1
@@ -10,7 +10,8 @@
|
||||
"test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js",
|
||||
"test": "npm run test:py && npm run test:unit",
|
||||
"graph": "node analysis/network/build-graph.js",
|
||||
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py",
|
||||
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py",
|
||||
"cycle": "python3 -m collection cycle",
|
||||
"stats": "python3 analysis/stats/site_stats.py",
|
||||
"collect": "python3 -m collection update",
|
||||
"build-data": "python3 -m collection build"
|
||||
|
||||
+6
-3
@@ -7,9 +7,12 @@ The route planner: the best route to every dragon, and what goes back to Ken's b
|
||||
| `build_policy.py` | Solves the route model (`core/atlas/solver.py`, built from the game in `core/spec/game.yaml`) for every colour and both goals ("this attempt" and "eventually"), with 200 posterior draws and a cross-fit check. Writes `data/public/policy.json`, which has a content id so each bot run can record which policy it followed. About 2 minutes. |
|
||||
| `policy_report.py` | Writes the readable version, `analysis/reports/POLICY.md`. |
|
||||
|
||||
| `experiments.yaml` | The experiment plan, written by hand: what each attempt of each bot account is for (the Black attempt on attempt 1, then the meadow Blue experiment, mapping and the sentinel route), with planned counts and stopping rules. |
|
||||
| `build_experiments.py` | Checks the plan against the game, counts what the bot has logged for every arm, sets each experiment's status (running, complete, paused), and writes `data/public/experiments.json` for the bot. |
|
||||
|
||||
Run both with `npm run policy`. The site's valley map and The Odds read `policy.json`; the bot downloads it with
|
||||
`solver.js` and `graph.js` (see `docs/BOT_INTEGRATION.md`).
|
||||
|
||||
**The loop with the bot.** The planner's routes go out to the bot; the bot's runs come back through
|
||||
`collection/bot/`. Experiments the bot should run to fill gaps the community form cannot (for example the
|
||||
answers at each bonding encounter) belong here; `experiments.json` is on the roadmap.
|
||||
**The loop with the bot.** `python -m collection cycle --bot <the bot's folder> --deliver <its vendor folder>`
|
||||
imports the bot's new runs, refits the route model, rebuilds the experiment plan and hands both back, in one
|
||||
command that is safe to run on a timer (`docs/BOT_INTEGRATION.md`, "The automated cycle").
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Build data/public/experiments.json: the experiment plan the bot follows, updated from the bot's own runs.
|
||||
|
||||
python planner/build_experiments.py [--out data/public/experiments.json] [--check]
|
||||
|
||||
Reads the design (planner/experiments.yaml), checks it against the game (core/spec/game.yaml), counts what the bot
|
||||
has logged for every arm (data/bot/trajectories-*.jsonl, matched to data/public/runs_clean.csv), and decides each
|
||||
experiment's status:
|
||||
|
||||
running still being assigned
|
||||
complete every arm has its planned number of observed rolls
|
||||
paused its stop_early rule was met (the plan needs a person)
|
||||
|
||||
The bot reads the result with assign() in core/solver.js; an experiment that is not running is no longer assigned
|
||||
and its share goes to the others. The plan has a content id (logged by the bot as planId).
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from atlas import model # noqa: E402
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
|
||||
DESIGN = ROOT / "planner/experiments.yaml"
|
||||
OUT = ROOT / "data/public/experiments.json"
|
||||
SAMPLING = ("best", "thompson", "uniform")
|
||||
|
||||
|
||||
def design():
|
||||
with open(DESIGN, encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def check_options(where, fixed):
|
||||
for k, o in (fixed or {}).items():
|
||||
if k not in model.DECISIONS or k in ("encounter", "cadet"):
|
||||
raise ValueError(f"{where}: '{k}' is not a decision of the game")
|
||||
if o not in model.DECISIONS[k]:
|
||||
raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})")
|
||||
|
||||
|
||||
def arms(exp):
|
||||
"""Every combination of the factors, each with the route held on every arm."""
|
||||
route = exp.get("route") or {}
|
||||
check_options(exp["id"] + " route", route)
|
||||
factors = exp.get("factors") or {}
|
||||
for k, opts in factors.items():
|
||||
if k in route:
|
||||
raise ValueError(f"{exp['id']}: '{k}' is both a factor and part of the route")
|
||||
for o in opts:
|
||||
check_options(exp["id"] + " factors", {k: o})
|
||||
if not factors:
|
||||
return [{"id": "all", "fix": dict(route)}]
|
||||
out = []
|
||||
for combo in itertools.product(*factors.values()):
|
||||
levels = dict(zip(factors, combo))
|
||||
out.append({"id": " / ".join(f"{k}={o}" for k, o in levels.items()), "levels": levels, "fix": {**route, **levels}})
|
||||
return out
|
||||
|
||||
|
||||
def validate(d):
|
||||
ids = [e["id"] for e in d["experiments"]]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("experiment ids must be unique")
|
||||
for e in d["experiments"]:
|
||||
if e["target"] not in model.COLOURS:
|
||||
raise ValueError(f"{e['id']}: unknown target {e['target']}")
|
||||
if e.get("sampling", "best") not in SAMPLING:
|
||||
raise ValueError(f"{e['id']}: sampling must be one of {SAMPLING}")
|
||||
if e["unit"] not in ("account", "attempt"):
|
||||
raise ValueError(f"{e['id']}: unit must be account or attempt")
|
||||
if not e.get("share", 0) > 0:
|
||||
raise ValueError(f"{e['id']}: share must be positive")
|
||||
m = e.get("measures")
|
||||
if m and (m["roll"] not in model.CHANCE or m["rare"] not in model.CHANCE[m["roll"]]["outcomes"]):
|
||||
raise ValueError(f"{e['id']}: measures {m} is not a roll and outcome of the game")
|
||||
if not e.get("ongoing") and not e.get("planned_per_arm"):
|
||||
raise ValueError(f"{e['id']}: needs planned_per_arm (or ongoing: true)")
|
||||
|
||||
|
||||
def interval(y, n, seed=1, sims=4000):
|
||||
"""Jeffreys 90% range for a rate."""
|
||||
if not n:
|
||||
return None
|
||||
r = random.Random(seed)
|
||||
s = sorted(r.betavariate(y + 0.5, n - y + 0.5) for _ in range(sims))
|
||||
return {"rate": (y + 0.5) / (n + 1), "lo": s[int(0.05 * sims)], "hi": s[int(0.95 * sims) - 1]}
|
||||
|
||||
|
||||
def logged(bot_dir=ROOT / "data/bot", runs_path=None):
|
||||
"""Bot runs that carry an experiment: [(experimentId, arm, prepared run)], each run once."""
|
||||
prepared = {r["run_id"]: model.prepare_run(r) for r in runs_io.read(runs_path or runs_io.PATH, sources=["bot"], drop=())}
|
||||
seen, out = set(), []
|
||||
for f in sorted(glob.glob(str(Path(bot_dir) / "trajectories-*.jsonl"))):
|
||||
if "sample" in Path(f).name:
|
||||
continue
|
||||
with open(f, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
t = json.loads(line)
|
||||
meta = t.get("meta") or {}
|
||||
if not meta.get("experimentId") or t["id"] in seen:
|
||||
continue
|
||||
seen.add(t["id"])
|
||||
out.append((meta["experimentId"], meta.get("arm"), prepared.get(t["id"])))
|
||||
return out
|
||||
|
||||
|
||||
def build(d=None, log=None):
|
||||
d = d or design()
|
||||
validate(d)
|
||||
log = logged() if log is None else log
|
||||
exps = []
|
||||
for e in d["experiments"]:
|
||||
m = e.get("measures")
|
||||
a = arms(e)
|
||||
tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a}
|
||||
for exp_id, arm, v in log:
|
||||
if exp_id != e["id"]:
|
||||
continue
|
||||
t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0})
|
||||
t["runs"] += 1
|
||||
y = model.OBSERVE[m["roll"]](v) if m and v else None
|
||||
if y is not None:
|
||||
t["observed"] += 1
|
||||
t["rare"] += y == m["rare"]
|
||||
planned = e.get("planned_per_arm")
|
||||
# mapping has nothing to measure: its runs are its count
|
||||
done = {k: (t["observed"] if m else t["runs"]) for k, t in tally.items() if k in {x["id"] for x in a}}
|
||||
total_obs = sum(t["observed"] for t in tally.values())
|
||||
total_rare = sum(t["rare"] for t in tally.values())
|
||||
status, note = "running", None
|
||||
se = e.get("stop_early")
|
||||
if not e.get("ongoing") and all(n >= planned for n in done.values()):
|
||||
status, note = "complete", f"every arm has its planned {planned}"
|
||||
elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]:
|
||||
status = "paused"
|
||||
note = f"stop_early: {total_rare} rare outcome(s) in {total_obs} observed rolls (rule: at most {se['rare_at_most']} after {se['after']})"
|
||||
for x in a:
|
||||
t = tally[x["id"]]
|
||||
x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})}
|
||||
exps.append({
|
||||
"id": e["id"], "status": status, **({"note": note} if note else {}),
|
||||
"question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")},
|
||||
"unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}),
|
||||
**({"exclusive": True} if e.get("exclusive") else {}),
|
||||
"target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"),
|
||||
"seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a,
|
||||
"progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare,
|
||||
**({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}),
|
||||
"unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})},
|
||||
})
|
||||
plan = {"schema": 1, "seed": d["seed"], "lifecycle": d["lifecycle"], "experiments": exps}
|
||||
plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12]
|
||||
return plan
|
||||
|
||||
|
||||
def render(plan):
|
||||
return json.dumps(plan, indent=1, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def summary(plan):
|
||||
L = [f"experiment plan {plan['id']}"]
|
||||
for e in plan["experiments"]:
|
||||
p = e["progress"]
|
||||
est = p.get("estimate")
|
||||
L.append(f" {e['id']:<22} {e['status']:<9} {p['observed']} observed, {p['rare']} rare"
|
||||
+ (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "")
|
||||
+ (f" [{e['note']}]" if e.get("note") else ""))
|
||||
return "\n".join(L)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--out", default=str(OUT))
|
||||
ap.add_argument("--check", action="store_true", help="write nothing; exit 1 if the file is out of date")
|
||||
a = ap.parse_args(argv)
|
||||
plan = build()
|
||||
text = render(plan)
|
||||
out = Path(a.out)
|
||||
if a.check:
|
||||
if not out.exists() or out.read_text(encoding="utf-8") != text:
|
||||
print(f"{out} is out of date (run python planner/build_experiments.py)", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
out.write_text(text, encoding="utf-8")
|
||||
print(summary(plan))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
# The experiment plan for Ken's bot: what each attempt of each account is for.
|
||||
#
|
||||
# This file is the design (written by hand). planner/build_experiments.py checks it against the game
|
||||
# (core/spec/game.yaml), counts what the bot has run so far, decides which experiments are still running, and
|
||||
# writes data/public/experiments.json. The bot reads that file with assign() in core/solver.js. The cycle
|
||||
# (python -m collection cycle) rebuilds it after every import of bot runs, so the plan follows the data:
|
||||
# an experiment that reaches its planned count, or its stopping rule, stops being assigned, and its share goes
|
||||
# to the others.
|
||||
#
|
||||
# Rules that keep the results honest:
|
||||
# - counts are planned in advance; an experiment does not stop because its results look good;
|
||||
# - every started attempt is logged, including aborted ones;
|
||||
# - the bot never accepts a bond that is not the experiment's target colour (it burns), and it does accept a
|
||||
# genuine Black or Blue bond.
|
||||
version: 1
|
||||
seed: 20261112 # changes every arm assignment: leave it alone once accounts are running
|
||||
|
||||
# ---------------------------------------------------------------- an account's life
|
||||
lifecycle:
|
||||
# attempt 1 of every new account is the Black attempt (the shadow is close to a first-attempt event), and is
|
||||
# itself an experiment: first-attempt-shadow below
|
||||
research_days: 3 # then the account collects route data for this long (about 18 attempts at 4 hours each)
|
||||
retire: true # and stops, except the long-lived accounts
|
||||
long_lived: { every: 10, take: 1 } # one account in ten keeps running, so attempts past 18 are still observed
|
||||
|
||||
# ---------------------------------------------------------------- the experiments
|
||||
# attempts: which of an account's attempts it covers (from / to, inclusive; no `to` = from then on).
|
||||
# unit: what is randomised. account = one arm for the account's life (needed when only attempt 1 counts);
|
||||
# attempt = arms in balanced blocks within each account.
|
||||
# share: when several experiments cover an attempt, how often each gets it.
|
||||
# exclusive: takes every attempt it covers; the shared experiments get the rest.
|
||||
# accounts: { every: N, take: K } limits it to K accounts in every N (by the order accounts were made).
|
||||
# factors: decisions that are randomised; every combination is an arm. route: decisions held on every arm.
|
||||
# Everything else is the advisor's best play for the target.
|
||||
# sampling: best (default), thompson, or uniform (every option on the screen equally likely: mapping).
|
||||
# measures: the roll and the outcome that are counted. planned_per_arm: how many observed rolls per arm.
|
||||
# stop_early: pause when, after `after` observed rolls in total, at most `rare_at_most` came out rare.
|
||||
experiments:
|
||||
- id: first-attempt-shadow
|
||||
question: How often does the shadow appear on an account's first attempt, and does heading West (at the start, at the fork) raise it?
|
||||
why: >
|
||||
Form submissions show the shadow on 9% of first attempts and 0.3% of later ones, but form submissions
|
||||
over-report it, complete logs have only 28 first attempts, and the bot has none. Among first attempts the
|
||||
form shows more shadows after West (14%) than East (4%), which may be copied guide routes.
|
||||
attempts: { from: 1, to: 1 }
|
||||
unit: account
|
||||
exclusive: true
|
||||
share: 1
|
||||
target: Black
|
||||
objective: eventually
|
||||
factors: { direction: [West, East], post: [West, East] }
|
||||
route: { scream: Investigate the scream, clearing: Help your squadmate, hour: Tree }
|
||||
measures: { roll: shadow, rare: "yes" }
|
||||
planned_per_arm: 60 # 240 accounts: 120 per level of each factor
|
||||
stop_early: { after: 120, rare_at_most: 0 } # no shadow in 120 first attempts rules out 2.5% or more
|
||||
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
|
||||
|
||||
- id: attempt-2-shadow
|
||||
question: Is the shadow really gone after the first attempt?
|
||||
why: The form shows 3 shadows in 295 second attempts (1%), then almost none.
|
||||
attempts: { from: 2, to: 2 }
|
||||
unit: account
|
||||
accounts: { every: 2, take: 1 }
|
||||
exclusive: true
|
||||
share: 1
|
||||
target: Black
|
||||
objective: eventually
|
||||
route: { scream: Investigate the scream, clearing: Help your squadmate, hour: Tree }
|
||||
measures: { roll: shadow, rare: "yes" }
|
||||
planned_per_arm: 120
|
||||
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
|
||||
|
||||
- id: meadow-blue-route
|
||||
question: Does anything before the meadow change how often the Blue waits there?
|
||||
why: >
|
||||
Complete logs show a Blue on about 10% of meadow visits whatever the route, from routes the bot's rules chose.
|
||||
Random assignment is what can confirm that the route does not matter (or find the route where it does).
|
||||
attempts: { from: 2 }
|
||||
unit: attempt
|
||||
share: 0.6
|
||||
target: Blue
|
||||
objective: eventually
|
||||
factors: { scream: [Investigate the scream, Keep walking], direction: [West, East, Center] }
|
||||
route: { hour: Valley, valley: Pass the Red (meadow) }
|
||||
measures: { roll: meadow, rare: Blue }
|
||||
planned_per_arm: 70 # 210 per scream choice, 140 per direction: enough to tell 10% from 20%
|
||||
burn: { reason: MEADOW_NO_BLUE_AUTO_BURN, when: no Blue in the meadow }
|
||||
|
||||
- id: mapping
|
||||
question: What does every branch of the valley lead to?
|
||||
why: Probabilities for every path to every dragon need runs on the paths nobody chooses.
|
||||
attempts: { from: 2 }
|
||||
unit: attempt
|
||||
share: 0.25
|
||||
target: Blue
|
||||
objective: eventually
|
||||
sampling: uniform
|
||||
planned_per_arm: 600
|
||||
burn: { reason: NOT_TARGET_AUTO_BURN, when: the encounter is not a Blue or a Black }
|
||||
|
||||
- id: sentinel
|
||||
question: Have the game's odds changed?
|
||||
why: >
|
||||
One fixed route, run all the time, by accounts that log everything: a change in its rates is a change in the
|
||||
game, not in who reports. It is the most-logged meadow route.
|
||||
attempts: { from: 2 }
|
||||
unit: attempt
|
||||
share: 0.15
|
||||
target: Blue
|
||||
objective: eventually
|
||||
route: { company: Alone, direction: Center, scream: Keep walking, wounded: Tourniquet, hour: Valley, valley: Pass the Red (meadow) }
|
||||
measures: { roll: meadow, rare: Blue }
|
||||
ongoing: true # never completes
|
||||
burn: { reason: MEADOW_NO_BLUE_AUTO_BURN, when: no Blue in the meadow }
|
||||
+7
-1
@@ -25,7 +25,13 @@ reporting_bias:
|
||||
rolls: { shadow: "yes", meadow: Blue } # roll: its rare outcome
|
||||
min_runs: 30 # fewer runs than this in a group (or the complete groups): borrow
|
||||
min_complete_rare: 5 # fewer rare outcomes than this in complete logs: the bias cannot be
|
||||
# estimated, so the roll is learned from complete logs only
|
||||
# estimated, so the roll is learned from complete logs only, unless
|
||||
# `borrow` names another roll for it
|
||||
# An assumption, not an estimate: complete logs have never seen the shadow, so its reporting bias cannot be
|
||||
# measured. Each group is taken to over-report the shadow as much as it over-reports the meadow Blue. Without
|
||||
# this, form submissions are left out of the shadow roll and the first-attempt effect (seen only in them) is
|
||||
# lost. Remove the line to go back to complete logs only; the bot's first-attempt runs will replace it.
|
||||
borrow: { shadow: meadow }
|
||||
groups: # first match wins
|
||||
- { name: bot, when: { source: bot }, complete: true }
|
||||
- { name: bulk logs, when: { source: reddit spreadsheet, entry_not: Form }, complete: true }
|
||||
|
||||
@@ -74,6 +74,15 @@ def main():
|
||||
a, e, l = t["thisAttempt"], t["eventually"], lb["targets"][X]["thisAttempt"]
|
||||
att = e["expectedAttempts"]["mean"]
|
||||
L.append(f"| {X} | {pct(a['pX']['mean'])} ({pct(a['pX']['lo'])}–{pct(a['pX']['hi'])}) | {pct(l['pX']['mean'])} ({pct(l['pX']['lo'])}–{pct(l['pX']['hi'])}) | {pct(t['crossFit']['heldOut'])} | {pct(e['ratio']['mean'])} | {'–' if att is None else js_round(att)} |")
|
||||
L += ["", "## First attempts", ""]
|
||||
L += ["The shadow at the tree is close to a first-attempt event, so an account's first attempt is solved separately (`givens` in `core/spec/game.yaml`). Only the colours that can come from the shadow differ much.", ""]
|
||||
L += ["| Colour | First attempt | Low-bias sample | Later attempt |", "|---|---|---|---|"]
|
||||
for X, t in s["targets"].items():
|
||||
f, fl, a = t["firstAttempt"]["thisAttempt"]["pX"], lb["targets"][X]["firstAttempt"]["thisAttempt"]["pX"], t["thisAttempt"]["pX"]
|
||||
L.append(f"| {X} | {pct(f['mean'])} ({pct(f['lo'])}–{pct(f['hi'])}) | {pct(fl['mean'])} ({pct(fl['lo'])}–{pct(fl['hi'])}) | {pct(a['mean'])} ({pct(a['lo'])}–{pct(a['hi'])}) |")
|
||||
sh = (s["model"].get("reportingBias") or {}).get("form, chasing Blue/Black", {}).get("shadow", {})
|
||||
if sh.get("used", "").startswith("assumed"):
|
||||
L += ["", f"The shadow's reporting bias cannot be measured yet (complete logs have never seen one), so it is assumed equal to the meadow Blue's (`borrow` in `planner/policy.yaml`): form reports of a shadow count 1/{sh['rho']:.1f} each. The bot's first-attempt runs will replace this assumption."]
|
||||
L += ["", "## How much do earlier choices matter?", ""]
|
||||
L += ["Each chance event can depend on the choices before it. The model learns how much from the data (empirical-Bayes concentration; larger means the history matters less):", ""]
|
||||
L += ["| Chance event | Depends on (most important first) | Concentration |", "|---|---|---|"]
|
||||
|
||||
@@ -162,7 +162,7 @@ class Handoff(unittest.TestCase):
|
||||
from collection.etl import handoff
|
||||
zpath = handoff.export_bot_kit(str(TMP / "exports"))
|
||||
with zipfile.ZipFile(zpath) as z:
|
||||
self.assertEqual(sorted(z.namelist()), ["README.txt", "graph.js", "policy.json", "solver.js"])
|
||||
self.assertEqual(sorted(z.namelist()), ["README.txt", "experiments.json", "graph.js", "policy.json", "solver.js"])
|
||||
self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes())
|
||||
self.assertEqual(json.loads(z.read("policy.json"))["id"],
|
||||
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
||||
|
||||
@@ -125,3 +125,72 @@ test("Thompson sampling: one posterior draw per seed, with the chance of each pi
|
||||
for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice);
|
||||
assert.ok(picks.size > 1, "different seeds explore different options");
|
||||
});
|
||||
|
||||
test("a first attempt has its own odds, exactly as the Python model publishes them", () => {
|
||||
for (const smp of Object.values(published.samples)) {
|
||||
for (const [X, t] of Object.entries(smp.targets)) {
|
||||
for (const obj of ["thisAttempt", "eventually"]) {
|
||||
const adv = S.advisor(smp, X, obj, { attempt: 1 });
|
||||
assert.deepStrictEqual(adv.given, { attempt: "First attempt" });
|
||||
assert.ok(Math.abs(adv.value.x - t.firstAttempt[obj].pX.point) < 1e-12, `${X} ${obj}`);
|
||||
assert.strictEqual(adv.next({}).choice, t.firstAttempt[obj].tree.choice);
|
||||
}
|
||||
}
|
||||
}
|
||||
const all = published.samples.all;
|
||||
const later = S.advisor(all, "Black", "thisAttempt", { attempt: 7 }), dflt = S.advisor(all, "Black", "thisAttempt");
|
||||
assert.strictEqual(later.value.x, dflt.value.x); // no attempt given: a later attempt
|
||||
assert.ok(S.advisor(all, "Black", "thisAttempt", { attempt: 1 }).value.x > 5 * later.value.x);
|
||||
});
|
||||
|
||||
test("fix holds decisions to an experiment arm and solves the rest around it", () => {
|
||||
const all = published.samples.all;
|
||||
const fix = { direction: "East", scream: "Investigate the scream", clearing: "Help your squadmate", hour: "Tree" };
|
||||
const adv = S.advisor(all, "Black", "eventually", { attempt: 1, fix });
|
||||
assert.strictEqual(adv.step({ company: "Alone" }).choice, "East");
|
||||
assert.deepStrictEqual(Object.keys(adv.step({ company: "Alone" }).options), ["East"]);
|
||||
assert.ok(adv.value.x <= S.advisor(all, "Black", "eventually", { attempt: 1 }).value.x + 1e-12);
|
||||
assert.throws(() => S.advisor(all, "Black", "eventually", { fix: { direction: "North" } }), /no option/);
|
||||
});
|
||||
|
||||
test("mapping: uniform sampling picks among the options on the screen, with its propensity", () => {
|
||||
const all = published.samples.all;
|
||||
const screen = ["Enter the valley alone", "Enter the valley with one of your squadmates", "Enter the valley with two of your squadmates"];
|
||||
const picks = new Set();
|
||||
for (let seed = 1; seed <= 30; seed++) {
|
||||
const rec = S.recommendOption(S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed }), {}, screen);
|
||||
assert.ok(Math.abs(rec.propensity - 1 / 3) < 1e-12);
|
||||
assert.match(S.norm(screen[rec.index]), new RegExp(all.model.game.bot_text.company[rec.choice]));
|
||||
picks.add(rec.choice);
|
||||
}
|
||||
assert.strictEqual(picks.size, 3);
|
||||
const a = S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed: 4 });
|
||||
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
|
||||
});
|
||||
|
||||
test("the experiment plan: balanced arms across new accounts, shares after the first attempt", () => {
|
||||
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
|
||||
const first = plan.experiments.find((e) => e.id === "first-attempt-shadow");
|
||||
const count = {};
|
||||
for (let acc = 0; acc < 4 * 25; acc++) {
|
||||
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
|
||||
assert.strictEqual(a.experimentId, "first-attempt-shadow");
|
||||
assert.strictEqual(a.target, "Black");
|
||||
assert.strictEqual(a.propensity, 1 / first.arms.length);
|
||||
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
|
||||
count[a.arm] = (count[a.arm] || 0) + 1;
|
||||
S.advisor(published.samples.all, a.target, a.objective, { attempt: 1, fix: a.fix }); // every arm is playable
|
||||
}
|
||||
assert.deepStrictEqual(Object.values(count), [25, 25, 25, 25]);
|
||||
// attempt 2: half the accounts check the shadow again; the others go to the shared experiments
|
||||
const second = Array.from({ length: 40 }, (_, acc) => S.assign(plan, { accountSeq: acc, attempt: 2 }).experimentId);
|
||||
assert.strictEqual(second.filter((id) => id === "attempt-2-shadow").length, 20);
|
||||
const later = {};
|
||||
for (let acc = 0; acc < 60; acc++) for (let t = 3; t <= 18; t++) { const a = S.assign(plan, { accountSeq: acc, attempt: t }); later[a.experimentId] = (later[a.experimentId] || 0) + 1; }
|
||||
assert.deepStrictEqual(Object.keys(later).sort(), ["mapping", "meadow-blue-route", "sentinel"]);
|
||||
assert.ok(Math.abs(later["meadow-blue-route"] / (60 * 16) - 0.6) < 0.06);
|
||||
// an experiment that is no longer running is not assigned; its share goes to the others
|
||||
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
|
||||
assert.notStrictEqual(S.assign(done, { accountSeq: 1, attempt: 5 }).experimentId, "meadow-blue-route");
|
||||
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
|
||||
});
|
||||
|
||||
@@ -48,6 +48,63 @@ class Bias(unittest.TestCase):
|
||||
self.assertEqual(model.fit_alpha(T, "meadow")["alpha"], max(model.ALPHA_GRID))
|
||||
|
||||
|
||||
def test_unestimable_bias_can_borrow_another_rolls_estimate(self):
|
||||
# the shadow was never seen in complete logs, the meadow Blue was: the form's shadow reports count 1/rho(meadow)
|
||||
cfg = {**CFG, "rolls": {"shadow": "yes", "meadow": "Blue"}, "borrow": {"shadow": "meadow"}}
|
||||
runs = self.runs(10, 100, 100, 300) + [tree_run("bot", False)] * 100 + \
|
||||
[tree_run("reddit spreadsheet", True)] * 8 + [tree_run("reddit spreadsheet", False)] * 92
|
||||
prep = [model.prepare_run(r) for r in runs]
|
||||
est, names = bias.estimate(runs, prep, cfg, set())
|
||||
e = est["form"]["shadow"]
|
||||
self.assertAlmostEqual(e["rho"], est["form"]["meadow"]["rho"])
|
||||
self.assertIn("assumed", e["used"])
|
||||
self.assertFalse(e.get("exclude"))
|
||||
T = model.count_tables(prep, bias.weights(names, prep, est, cfg["rolls"]))
|
||||
self.assertAlmostEqual(T["shadow"]["1:First attempt"]["yes"], 8 / e["rho"], places=3)
|
||||
# without `borrow`, the group is left out of the roll
|
||||
est2, _ = bias.estimate(runs, prep, {**cfg, "borrow": {}}, set())
|
||||
self.assertTrue(est2["form"]["shadow"]["exclude"])
|
||||
|
||||
|
||||
def tree_run(source, shadow, attempt_group="First attempt"):
|
||||
r = meadow_run(source, "Orange")
|
||||
r.update(scream="Investigate & help", resolution="Squadmate's revenge", post_path="Forest", hour="Tree",
|
||||
branch="Shadow (rare)" if shadow else "Descend slowly: two Greens", outcome_color="Black" if shadow else "Green",
|
||||
enc_Orange=None, enc_Black="Incinerated" if shadow else None, enc_Green=None if shadow else "Incinerated",
|
||||
saw_blue_meadow=None, attempt_group=attempt_group, combat_first=None, combat_second=None)
|
||||
return r
|
||||
|
||||
|
||||
class FirstAttempt(unittest.TestCase):
|
||||
"""game.yaml `givens`: first and later attempts are different shadow rolls (`distinct`), not pooled."""
|
||||
|
||||
def setUp(self):
|
||||
runs = [tree_run("bot", True)] * 9 + [tree_run("bot", False)] * 91 + [tree_run("bot", False, "Later attempt")] * 900 + \
|
||||
[tree_run("bot", True, "Later attempt")] * 1
|
||||
self.T = model.count_tables([model.prepare_run(r) for r in runs])
|
||||
|
||||
def test_each_attempt_kind_keeps_its_own_rate(self):
|
||||
p = model.Params(self.T, "mean", alphas={"shadow": 100000})
|
||||
gated = {"scream": "Investigate the scream", "clearing": "Help your squadmate"}
|
||||
later = float(p.chance("shadow", gated)["yes"][0])
|
||||
first = float(p.having(attempt="First attempt").chance("shadow", gated)["yes"][0])
|
||||
self.assertAlmostEqual(first, 0.09, delta=0.01) # not dragged to the pooled 1%
|
||||
self.assertAlmostEqual(later, 0.001, delta=0.001)
|
||||
self.assertEqual(p.given, {"attempt": "Later attempt"}) # the default: a later attempt
|
||||
|
||||
def test_policy_reports_the_first_attempt_separately(self):
|
||||
from atlas import solver
|
||||
runs = [tree_run("bot", True)] * 9 + [tree_run("bot", False)] * 91 + [tree_run("bot", False, "Later attempt")] * 900
|
||||
pol = solver.build_policy([dict(r, enc_Black="Bonded!", outcome="Bonded") if r["branch"] == "Shadow (rare)" else r for r in runs],
|
||||
draws=6, colours=["Black"], min_support=1)
|
||||
t = pol["targets"]["Black"]
|
||||
f = t["firstAttempt"]
|
||||
self.assertEqual(f["given"], {"attempt": "First attempt"})
|
||||
self.assertGreater(f["thisAttempt"]["pX"]["point"], 20 * t["thisAttempt"]["pX"]["point"])
|
||||
self.assertLessEqual(f["thisAttempt"]["pX"]["lo"], f["thisAttempt"]["pX"]["hi"])
|
||||
self.assertIn("pEventually", f["eventually"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""The experiment plan (planner/experiments.yaml -> data/public/experiments.json)."""
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "planner"))
|
||||
import build_experiments as B # noqa: E402
|
||||
from atlas import model # noqa: E402
|
||||
|
||||
D = B.design()
|
||||
GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "clearing": "Help your squadmate"}
|
||||
|
||||
|
||||
class Plan(unittest.TestCase):
|
||||
def test_design_is_valid_and_arms_are_every_combination(self):
|
||||
plan = B.build(D, log=[])
|
||||
by = {e["id"]: e for e in plan["experiments"]}
|
||||
self.assertEqual(len(by["first-attempt-shadow"]["arms"]), 4)
|
||||
self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6)
|
||||
for e in plan["experiments"]:
|
||||
self.assertEqual(e["status"], "running")
|
||||
for a in e["arms"]:
|
||||
B.check_options(e["id"], a["fix"])
|
||||
# the Black attempt keeps the shadow's gate open on every arm
|
||||
for a in by["first-attempt-shadow"]["arms"]:
|
||||
self.assertTrue(model.gate_open("shadow", a["fix"]))
|
||||
|
||||
def test_an_unknown_option_is_refused(self):
|
||||
d = copy.deepcopy(D)
|
||||
d["experiments"][0]["factors"]["direction"] = ["North"]
|
||||
with self.assertRaises(ValueError):
|
||||
B.build(d, log=[])
|
||||
|
||||
def test_progress_completes_and_stops_early(self):
|
||||
e0 = D["experiments"][0]
|
||||
arms = [a["id"] for a in B.arms(e0)]
|
||||
seen = {**GATED_TREE, "shadow": True}
|
||||
none = {**GATED_TREE, "shadow": False}
|
||||
# every arm at its planned count, with shadows: complete
|
||||
log = [(e0["id"], a, seen if i < 3 else none) for a in arms for i in range(e0["planned_per_arm"])]
|
||||
e = B.build(D, log=log)["experiments"][0]
|
||||
self.assertEqual(e["status"], "complete")
|
||||
self.assertEqual(e["progress"]["rare"], 12)
|
||||
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 61)
|
||||
# no shadow at all after the stop_early count: paused, for a person to look at
|
||||
log = [(e0["id"], arms[i % 4], none) for i in range(e0["stop_early"]["after"])]
|
||||
e = B.build(D, log=log)["experiments"][0]
|
||||
self.assertEqual(e["status"], "paused")
|
||||
# a run whose roll was not observed (died before the tree) is a run, not an observation
|
||||
e = B.build(D, log=[(e0["id"], arms[0], {"attempt": "First attempt"})])["experiments"][0]
|
||||
self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0))
|
||||
|
||||
def test_the_plan_id_follows_its_content(self):
|
||||
a, b = B.build(D, log=[]), B.build(D, log=[])
|
||||
self.assertEqual(a["id"], b["id"])
|
||||
c = B.build(D, log=[(D["experiments"][0]["id"], "x", None)])
|
||||
self.assertNotEqual(a["id"], c["id"])
|
||||
|
||||
def test_published_plan_is_up_to_date(self):
|
||||
self.assertEqual(B.main(["--check"]), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -47,7 +47,7 @@ class GameSpec(unittest.TestCase):
|
||||
|
||||
def test_chance_contexts_are_earlier_steps(self):
|
||||
known = {"company", "left", "direction", "fs", "react", "scream", "clearing", "fight", "dagger", "cadet", "post",
|
||||
"hour", "shadow", "descend", "red", "valley", "meadow"}
|
||||
"hour", "shadow", "descend", "red", "valley", "meadow"} | set(G.get("givens") or {})
|
||||
for k, c in G["chance"].items():
|
||||
self.assertTrue(set(c["context"]) <= known, k)
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -22,7 +22,7 @@ def world(n=4000, seed=7):
|
||||
run = {"company": "Alone", "direction": ["East", "West", "Center"][i % 3], "first_sighting": ["Red", "Brown"][i % 2],
|
||||
"scream": "Investigate & help", "resolution": ["Killed the bullies yourself", "Squadmate's revenge"][i % 2],
|
||||
"post_path": "Forest", "goal": "Any (for fun)", "malek_intent": "No", "left_squadmates": False,
|
||||
"sighting_action": "Unknown", "combat_first": "(not recorded)", "combat_second": "(not recorded)", "rare_reported": "No",
|
||||
"sighting_action": "Unknown", "attempt_group": "Later attempt", "combat_first": "(not recorded)", "combat_second": "(not recorded)", "rare_reported": "No",
|
||||
**{f"enc_{c}": None for c in ("Red", "Green", "Orange", "Brown", "Blue", "Black")}}
|
||||
run["hour"] = "Tree" if r() < 0.5 else "Valley"
|
||||
if run["hour"] == "Tree":
|
||||
|
||||
Reference in New Issue
Block a user