First attempt cycle #1

Merged
kathleen merged 5 commits from first-attempt-cycle into main 2026-11-16 16:18:10 +00:00
19 changed files with 1588 additions and 231 deletions
Showing only changes of commit d01ec46fcc - Show all commits
+253
View File
@@ -0,0 +1,253 @@
"""What the runs say about the Black dragon: analysis/reports/BLACK.md.
python analysis/black/evidence.py [--check]
Every number in the report is computed here from data/public/runs_clean.csv (and the bot's anonymised
trajectories), so it can be re-run after every import. Nothing is fitted: these are counts. The developer's
statements they are read against are in planner/knowledge.yaml.
"""
import argparse
import collections
import glob
import json
import math
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "core"))
from atlas import runs as runs_io # noqa: E402
OUT = ROOT / "analysis/reports/BLACK.md"
ROUTE = ["company", "direction", "combat_first", "combat_second", "resolution", "post_path"]
ROUTE_NAMES = "company, first direction, first move in the fight, the dagger, the bullies' fate, the fork"
def gated_tree(r):
"""A run that passed both known gates (investigated, helped, climbed the tree) and says whether the shadow showed."""
return r["scream"] == "Investigate & help" and r["hour"] == "Tree" and r["saw_shadow"] is not None
def group(r):
if r["source"] == "bot":
return "bot"
if r["source"] == "reddit spreadsheet":
return "form" if r["entry"] == "Form" else "bulk logs"
return "other"
def tally(rows, key):
d = collections.OrderedDict()
for r in rows:
c = d.setdefault(key(r), [0, 0])
c[0] += bool(r["saw_shadow"])
c[1] += 1
return d
def frac(y, n):
return f"{y} / {n}" + (f" ({100 * y / n:.1f}%)" if n else "")
def upper(n, conf=0.95):
"""The rate that zero in n runs rules out (one-sided)."""
return 1 - (1 - conf) ** (1 / n) if n else 1.0
def mantel_haenszel(strata):
"""strata: [(y1, n1, y0, n0)]. Odds ratio and z of group 1 against group 0, within strata."""
num = den = obs = exp = var = 0.0
for y1, n1, y0, n0 in strata:
N = n1 + n0
if not n1 or not n0:
continue
num += y1 * (n0 - y0) / N
den += y0 * (n1 - y1) / N
m = y1 + y0
obs += y1
exp += n1 * m / N
if N > 1:
var += n1 * n0 * m * (N - m) / (N * N * (N - 1))
return (num / den if den else float("inf")), obs, exp, ((obs - exp) / math.sqrt(var) if var else 0.0)
def last_answer(note):
"""The answer given to the Black, from a rider's free-text note (keywords only; the note is never published).
Riders lower their eyes to the Blue earlier in the same sequence, so only the last thing a note says counts, and
only once the note has reached the Black (or is too short to be about anything else)."""
t = re.sub(r"instead of low\w+[^.>,;]*", "", (note or "").lower())
marks = [m.end() for m in re.finditer(r"threat|branch|south|black", t)]
scope = t if len(t) < 45 else (t[marks[-1]:] if marks else "")
found = [(m.start(), k) for k, pat in (("held its gaze", r"gaze|glare|eye contact"), ("lowered eyes", r"lower"),
("ignored the sound", r"ignore"))
for m in re.finditer(pat, scope)]
if not found: # a long note that never names the Black's moment: use the whole note, gaze only
return "held its gaze" if re.search(r"(hold|held|meet|met)\w* (its|it's|the black'?s?|his|her) (gaze|glare)", t) else None
return max(found)[1]
def blue_mistake(note):
t = (note or "").lower()
if "approach" in t:
return "approached the Blue"
if "warn" in t or "interven" in t:
return "warned the other cadet / intervened"
return None
def build():
runs = runs_io.read()
G = [r for r in runs if gated_tree(r)]
L = ["# The Black dragon: what the runs show", "",
"Written by `analysis/black/evidence.py` from `data/public/runs_clean.csv`. Counts only; nothing is fitted. "
"\"Gated tree runs\" investigated the scream, helped the squadmate and climbed the tree, the two gates the "
"developer has confirmed (`planner/knowledge.yaml`). Form submissions over-report the shadow, so their "
"percentages are too high; comparisons inside the form (first attempt against later, one route against "
"another) are still fair if the over-reporting is the same on both sides.", ""]
# 1. by attempt
L += ["## 1. The shadow by attempt number", "", "| Attempt | Form submissions | Bulk logs | Bot |", "|---|---|---|---|"]
bands = [("1", 1, 1), ("2", 2, 2), ("3", 3, 3), ("4-10", 4, 10), ("11-30", 11, 30), ("31 or more", 31, 10 ** 9)]
for name, lo, hi in bands:
cells = []
for g in ("form", "bulk logs", "bot"):
rs = [r for r in G if group(r) == g and r["attempt"] and lo <= r["attempt"] <= hi]
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
L.append(f"| {name} | " + " | ".join(cells) + " |")
cells = []
for g in ("form", "bulk logs", "bot"):
rs = [r for r in G if group(r) == g and not r["attempt"]]
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
L.append("| not given | " + " | ".join(cells) + " |")
complete_first = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] == 1]
complete_later = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] and r["attempt"] > 1]
L += ["", f"Complete logs (bot and bulk) have {len(complete_first)} first attempts at the tree and "
f"{len(complete_later)} later ones, with {sum(bool(r['saw_shadow']) for r in complete_first + complete_later)} shadows. "
f"Zero in {len(complete_later)} later attempts rules out a rate above {100 * upper(len(complete_later)):.1f}% for them; "
f"zero in {len(complete_first)} first attempts only rules out more than {100 * upper(len(complete_first)):.0f}%.", ""]
# 2. the same route, first against later
F = [r for r in G if group(r) == "form" and r["attempt"]]
by = collections.defaultdict(lambda: [0, 0, 0, 0])
for r in F:
c = by[tuple(r[k] for k in ROUTE)]
i = 0 if r["attempt"] == 1 else 2
c[i] += bool(r["saw_shadow"])
c[i + 1] += 1
orr, obs, exp, z = mantel_haenszel(by.values())
L += ["## 2. The same route on a first attempt and on a later one", "",
f"If the shadow depended only on the choices the form records ({ROUTE_NAMES}), a route would do as well on a "
"later attempt as on a first. It does not.", "",
"| Route | First attempt | Later attempts |", "|---|---|---|"]
top = sorted(by.items(), key=lambda kv: -(kv[1][1] + kv[1][3]))[:8]
for key, (y1, n1, y0, n0) in top:
L.append(f"| {' / '.join(key)} | {frac(y1, n1)} | {frac(y0, n0)} |")
L += ["", f"Across all {len(by)} recorded routes, comparing first and later attempts on the same route: "
f"{obs:.0f} first-attempt shadows where {exp:.1f} would be expected if the attempt did not matter "
f"(odds ratio {orr:.1f}, z = {z:.1f}). First and later attempts choose almost the same routes, so the "
"difference is not in the recorded choices: it is something about the account, or a choice the form does "
"not record.", ""]
# 3. is it decided by route + cadet assignment?
best_key, best = max(((k, v) for k, v in by.items() if v[1] >= 20), key=lambda kv: kv[1][0] / kv[1][1])
B = [r for r in F if r["attempt"] == 1 and tuple(r[k] for k in ROUTE) == best_key]
cells = tally(B, lambda r: (r["wing"], r["section"], r["squad"]))
multi = {k: v for k, v in cells.items() if v[1] >= 2}
mixed = sum(1 for y, n in multi.values() if 0 < y < n)
L += ["## 3. On one route, is the shadow decided by the cadet's wing, section and squad?", "",
f"The developer says the Black is decided by choices and by what the site knows about the account, with no "
f"dice. If the recorded route and the cadet assignment were all of it, every first attempt with the same "
f"route and assignment would agree. On the best-recorded route ({' / '.join(best_key)}; {frac(best[0], best[1])} "
f"on first attempts), {len(multi)} assignments have two or more runs and {mixed} of them disagree (some runs "
"with the shadow, some without). So something else decides it: a choice the form does not record, or a fact "
"about the account other than its assignment. No wing, section or squad stands out:", "",
"| | " + " | ".join(["shadow / runs"]) + " |", "|---|---|"]
for col in ("wing", "section", "squad", "first_sighting", "sighting_action"):
t = tally(B, lambda r, col=col: r[col] or "(blank)")
L.append(f"| {col} | " + ", ".join(f"{k}: {y}/{n}" for k, (y, n) in sorted(t.items(), key=lambda kv: -kv[1][1]) if n >= 3) + " |")
L.append("")
# 4. no single route
S1 = [r for r in G if group(r) == "form" and r["saw_shadow"]]
L += ["## 4. There is no single route", "",
f"The {len(S1)} form runs with a shadow (all attempts) took {len({tuple(r[k] for k in ROUTE) for r in S1})} "
"different recorded routes:", ""]
for col, label in (("company", "Company"), ("direction", "First direction"), ("combat_first", "First move in the fight"),
("combat_second", "The dagger"), ("resolution", "The bullies' fate"), ("post_path", "The fork")):
t = collections.Counter(r[col] for r in S1)
L.append(f"- **{label}:** " + ", ".join(f"{k} {v}" for k, v in t.most_common()))
outside = [r for r in runs if r["saw_shadow"] and not (r["scream"] == "Investigate & help" and r["hour"] == "Tree")]
L += ["", f"Shadows reported without both gates: {len(outside)} (likely mis-entered).", ""]
# 5. after the shadow
SH = [r for r in runs if r["saw_shadow"]]
black = [r for r in SH if r["enc_Black"]]
ans = collections.defaultdict(lambda: [0, 0])
for r in black:
a = last_answer((r["bond_note"] or "") + " " + (r["death_note"] or ""))
if a:
ans[a][0] += r["outcome"] == "Bonded"
ans[a][1] += 1
blue_dead = [r for r in SH if r["enc_Blue"] == "Incinerated"]
mis = collections.Counter(blue_mistake((r["bond_note"] or "") + " " + (r["death_note"] or "")) for r in blue_dead)
L += ["## 5. After the shadow: the Blue, then the Black", "",
"The riders' notes describe one sequence: follow the shadow, a Blue lands and burns another cadet, the Blue "
"leaves, then the Black appears. The Blue is a trial on the way, not an alternative to the Black.", "",
f"- Runs that reached the Black: {len(black)}, bonded {sum(r['outcome'] == 'Bonded' for r in black)}.",
f"- Runs that died at the Blue: {len(blue_dead)}. What their notes say they did: "
+ ", ".join(f"{k} {v}" for k, v in mis.most_common() if k) + ".",
"- The answer given to the Black, where the note says (bonded / runs): "
+ "; ".join(f"{k} {y}/{n}" for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])) + ".", "",
"Holding its gaze is the only answer recorded with a bond, and it still fails about a quarter of the time. "
"The developer says a rider who knows the system bonds with certainty, so those failures are not explained "
"yet: the notes of the riders who died are too short to show what else they did differently.", ""]
# 6. what the bot has tried
L += ["## 6. What the bot has tried", ""]
files = sorted(f for f in glob.glob(str(ROOT / "data/bot/trajectories-*.jsonl")) if "sample" not in f)
seen, acc, order = set(), collections.defaultdict(list), collections.defaultdict(list)
for f in files:
for line in open(f, encoding="utf-8"):
t = json.loads(line)
if t["id"] not in seen:
seen.add(t["id"])
acc[t["meta"].get("accountKey")].append(t["meta"].get("attempt") or 0)
order[t["meta"].get("accountKey")].append((t["meta"].get("attempt") or 0, t["id"]))
by_id = {r["run_id"]: r for r in runs_io.read(sources=["bot"], drop=())}
streak = 0 # the longest run of consecutive attempts on one account with the same gated tree route
for v in order.values():
cur, prev = 0, None
for _, i in sorted(v):
r = by_id.get(i)
s = tuple(r[k] for k in ROUTE + ["sighting_action"]) if r and gated_tree(r) else None
cur = cur + 1 if s is not None and s == prev else (1 if s is not None else 0)
prev, streak = s, max(streak, cur)
botg = [r for r in G if group(r) == "bot"]
routes = collections.Counter(tuple(r[k] for k in ROUTE) for r in botg)
L += [f"{len(seen)} attempts on {len(acc)} accounts; the earliest attempt logged on any account is number "
f"{min((min(v) for v in acc.values() if v), default=0)}. Gated tree runs: {len(botg)}, shadows "
f"{sum(bool(r['saw_shadow']) for r in botg)}, on {len(routes)} recorded routes "
f"(the commonest {routes.most_common(1)[0][1] if routes else 0} times). The bot has not played a first attempt, "
"so it has not yet tested the one situation where riders report the shadow. The most times any account has "
f"repeated one route in a row is {streak}.", ""]
return "\n".join(L) + "\n"
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--check", action="store_true")
a = ap.parse_args(argv)
text = build()
if a.check:
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
print(f"{OUT} is out of date (run python analysis/black/evidence.py)", file=sys.stderr)
return 1
return 0
OUT.write_text(text, encoding="utf-8")
print(f"wrote {OUT.relative_to(ROOT)}")
return 0
if __name__ == "__main__":
sys.exit(main())
+74
View File
@@ -0,0 +1,74 @@
# The Black dragon: what the runs show
Written by `analysis/black/evidence.py` from `data/public/runs_clean.csv`. Counts only; nothing is fitted. "Gated tree runs" investigated the scream, helped the squadmate and climbed the tree, the two gates the developer has confirmed (`planner/knowledge.yaml`). Form submissions over-report the shadow, so their percentages are too high; comparisons inside the form (first attempt against later, one route against another) are still fair if the over-reporting is the same on both sides.
## 1. The shadow by attempt number
| Attempt | Form submissions | Bulk logs | Bot |
|---|---|---|---|
| 1 | 69 / 748 (9.2%) | 0 / 28 (0.0%) | 0 / 0 |
| 2 | 3 / 274 (1.1%) | 0 / 21 (0.0%) | 0 / 4 (0.0%) |
| 3 | 1 / 177 (0.6%) | 0 / 14 (0.0%) | 0 / 5 (0.0%) |
| 4-10 | 1 / 485 (0.2%) | 0 / 102 (0.0%) | 0 / 36 (0.0%) |
| 11-30 | 1 / 524 (0.2%) | 0 / 57 (0.0%) | 0 / 137 (0.0%) |
| 31 or more | 1 / 535 (0.2%) | 0 / 0 | 0 / 54 (0.0%) |
| not given | 11 / 1406 (0.8%) | 0 / 112 (0.0%) | 0 / 0 |
Complete logs (bot and bulk) have 28 first attempts at the tree and 430 later ones, with 0 shadows. Zero in 430 later attempts rules out a rate above 0.7% for them; zero in 28 first attempts only rules out more than 10%.
## 2. The same route on a first attempt and on a later one
If the shadow depended only on the choices the form records (company, first direction, first move in the fight, the dagger, the bullies' fate, the fork), a route would do as well on a later attempt as on a first. It does not.
| Route | First attempt | Later attempts |
|---|---|---|
| Alone / West / Assess the situation / Engage the smaller cadet / Squadmate's revenge / West | 21 / 79 (26.6%) | 1 / 80 (1.2%) |
| Alone / West / Assess the situation / Engage the smaller cadet / Squadmate's revenge / Forest | 4 / 28 (14.3%) | 0 / 91 (0.0%) |
| Alone / Center / Assess the situation / Engage the smaller cadet / Squadmate's revenge / Forest | 4 / 25 (16.0%) | 0 / 70 (0.0%) |
| Alone / Center / Assess the situation / Daggers or not (protect squadmate) / Left bullies to the dragons / Forest | 1 / 28 (3.6%) | 0 / 62 (0.0%) |
| Alone / Center / Assess the situation / Engage the smaller cadet / Squadmate's revenge / West | 1 / 8 (12.5%) | 1 / 80 (1.2%) |
| 1 Squadmate / East / Assess the situation / Engage the smaller cadet / Squadmate's revenge / East | 2 / 28 (7.1%) | 0 / 60 (0.0%) |
| Alone / East / Assess the situation / Engage the smaller cadet / Squadmate's revenge / East | 1 / 33 (3.0%) | 0 / 48 (0.0%) |
| Alone / West / Assess the situation / Engage the smaller cadet / Left bullies to the dragons / West | 2 / 23 (8.7%) | 0 / 53 (0.0%) |
Across all 296 recorded routes, comparing first and later attempts on the same route: 66 first-attempt shadows where 29.4 would be expected if the attempt did not matter (odds ratio 21.5, z = 9.8). First and later attempts choose almost the same routes, so the difference is not in the recorded choices: it is something about the account, or a choice the form does not record.
## 3. On one route, is the shadow decided by the cadet's wing, section and squad?
The developer says the Black is decided by choices and by what the site knows about the account, with no dice. If the recorded route and the cadet assignment were all of it, every first attempt with the same route and assignment would agree. On the best-recorded route (Alone / West / Assess the situation / Engage the smaller cadet / Squadmate's revenge / West; 21 / 79 (26.6%) on first attempts), 20 assignments have two or more runs and 10 of them disagree (some runs with the shadow, some without). So something else decides it: a choice the form does not record, or a fact about the account other than its assignment. No wing, section or squad stands out:
| | shadow / runs |
|---|---|
| wing | Fourth: 5/25, Third: 6/23, First: 6/16, Second: 3/12 |
| section | Claw: 8/27, Tail: 7/26, Flame: 5/23, (blank): 1/3 |
| squad | Second: 10/28, First: 6/26, Third: 4/23 |
| first_sighting | Brown: 10/40, Green: 9/37 |
| sighting_action | Hold still: 10/43, Respect / face it: 4/16, Unknown: 7/13, Weapon: 0/5 |
## 4. There is no single route
The 87 form runs with a shadow (all attempts) took 44 different recorded routes:
- **Company:** Alone 70, 2 Squadmates 9, 1 Squadmate 8
- **First direction:** West 53, Center 20, East 14
- **First move in the fight:** Assess the situation 76, (not recorded) 9, Even the numbers (smaller cadet) 2
- **The dagger:** Engage the smaller cadet 56, (not recorded) 22, Daggers or not (protect squadmate) 9
- **The bullies' fate:** Squadmate's revenge 54, Left bullies to the dragons 22, Killed the bullies yourself 9, Unknown 2
- **The fork:** West 40, Forest 35, East 11, Unknown 1
Shadows reported without both gates: 3 (likely mis-entered).
## 5. After the shadow: the Blue, then the Black
The riders' notes describe one sequence: follow the shadow, a Blue lands and burns another cadet, the Blue leaves, then the Black appears. The Blue is a trial on the way, not an alternative to the Black.
- Runs that reached the Black: 70, bonded 49.
- Runs that died at the Blue: 15. What their notes say they did: approached the Blue 10, warned the other cadet / intervened 4.
- The answer given to the Black, where the note says (bonded / runs): held its gaze 27/39; lowered eyes 0/7; ignored the sound 0/1.
Holding its gaze is the only answer recorded with a bond, and it still fails about a quarter of the time. The developer says a rider who knows the system bonds with certainty, so those failures are not explained yet: the notes of the riders who died are too short to show what else they did differently.
## 6. What the bot has tried
415 attempts on 14 accounts; the earliest attempt logged on any account is number 2. Gated tree runs: 236, shadows 0, on 3 recorded routes (the commonest 87 times). The bot has not played a first attempt, so it has not yet tested the one situation where riders report the shadow. The most times any account has repeated one route in a row is 11.
+16 -16
View File
@@ -541,7 +541,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -565,7 +565,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -590,7 +590,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -614,7 +614,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -643,7 +643,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -667,7 +667,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -692,7 +692,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -716,7 +716,7 @@ Each chance event can depend on the choices before it. The model learns how much
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 25%, Die on purpose 0.0%
- Outcome if you attempt: incinerated 64%, bond Green 25%, bond Orange 6.2%, bond Brown 4.9%
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- Bonding encounter: any (no difference for this colour)
- die on purpose
@@ -927,7 +927,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
@@ -951,7 +951,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
@@ -975,7 +975,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
@@ -1000,7 +1000,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
@@ -1031,7 +1031,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
@@ -1055,7 +1055,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
@@ -1079,7 +1079,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
@@ -1104,7 +1104,7 @@ Each chance event can depend on the choices before it. The model learns how much
- Bonding encounter: any (no difference for this colour)
- die on purpose
- if **yes**:
- Shadow dragon: Black 77%, Blue 23%
- After the shadow: past the Blue to the Black: Black 77%, Blue 23%
- if **Black**:
- **Bonding encounter: Attempt the bond** (still best in 100% of posterior draws). If chosen, then best play after: Attempt the bond 69%, Die on purpose 0.0%
- Outcome if you attempt: bond Black 69%, incinerated 31%
+2
View File
@@ -85,6 +85,7 @@ def refresh(update_main, bots=()):
print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)")
_step("route model (about 2 minutes)", [py, "planner/build_policy.py"])
_step("experiment plan", [py, "planner/build_experiments.py"])
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
_step("statistics", [py, "analysis/stats/site_stats.py"])
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit")
@@ -169,6 +170,7 @@ def cycle(update_main, bots, deliver_to=None, community=False, force=False):
_step("route graph", ["node", "analysis/network/build-graph.js"])
_step("route model (a few minutes)", [py, "planner/build_policy.py"])
_step("experiment plan", [py, "planner/build_experiments.py"])
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
try:
import scipy # noqa: F401
_step("statistics", [py, "analysis/stats/site_stats.py"])
+31 -6
View File
@@ -308,6 +308,14 @@
return { index: i, decision: rec.decision, choice: o, propensity: 1 / on.length, margin: null, options: rec.options, reason: `${rec.label}: ${o} (mapping: 1 of ${on.length} at random)` };
}
}
// With squadmates, staying and the direction are one screen ("Go west with your squadmate", "Convince them to
// head east instead", "Leave your squadmate"): staying is answered with the direction that follows it.
if (rec.decision === "leave" && rec.choice === "Stay together") {
const then = adv.next({ ...h, leave: rec.choice });
const dp = then && then.decision === "direction" ? ((adv.game.bot_text || {}).direction || {})[then.choice] : null;
const di = dp ? texts.findIndex((t) => new RegExp(dp).test(t)) : -1;
if (di >= 0) return { index: di, decision: "leave", choice: rec.choice, also: { direction: then.choice }, propensity: rec.propensity * then.propensity, margin: null, options: rec.options, reason: `${rec.label}: ${rec.choice}, ${then.label}: ${then.choice}` };
}
const pat = pats[rec.choice];
const idx = pat ? texts.findIndex((t) => new RegExp(pat).test(t)) : -1;
const values = Object.values(rec.options).map((v) => v.pX).sort((a, b) => b - a);
@@ -362,18 +370,22 @@
return arms[order[n % k]];
}
/**
* What an attempt is for, under the plan: assign(plan, { accountSeq, attempt }) ->
* What an attempt is for, under the plan: assign(plan, { accountSeq, attempt, facts }) ->
* { experimentId, arm, fix, target, objective, sampling, seed, propensity, burn, planId }, or null when no
* experiment covers the attempt (use the account's own goal).
* accountSeq: the account's number in the order the accounts were made (0, 1, 2, ...), the same for its whole
* life; attempt: the account's attempt number (1 = its first threshing). The answer is the same every time for
* life; attempt: the account's attempt number (1 = its first threshing); facts: what the plan's `when` rules ask
* about the account, e.g. { sawShadowBefore: true }. The answer is the same every time for
* the same plan id, so a restart mid-attempt changes nothing. Pass the result to advisor():
* advisor(policy.samples.all, a.target, a.objective, { attempt, fix: a.fix, sampling: a.sampling, seed: a.seed })
* and log experimentId, arm, planId and the propensity on the run.
*/
function assign(plan, { accountSeq, attempt }) {
function assign(plan, { accountSeq, attempt, facts = {} }) {
const takes = (e) => !e.accounts || ((((accountSeq - (e.accounts.offset || 0)) % e.accounts.every) + e.accounts.every) % e.accounts.every) < e.accounts.take;
// when: facts about the account that must hold (a fact the bot does not pass counts as false)
const holds = (e) => Object.entries(e.when || {}).every(([k, v]) => (facts[k] == null ? false : facts[k]) === v);
let open = (plan.experiments || []).filter((e) => e.status === "running" && attempt >= e.attempts.from && (e.attempts.to == null || attempt <= e.attempts.to)
&& (!e.accounts || (accountSeq % e.accounts.every) < e.accounts.take));
&& takes(e) && holds(e));
if (!open.length) return null;
if (open.some((e) => e.exclusive)) open = open.filter((e) => e.exclusive); // an exclusive experiment takes its attempts whole
const total = open.reduce((s, e) => s + e.share, 0);
@@ -381,8 +393,21 @@
for (const e of open) { if (u < e.share) { exp = e; break; } u -= e.share; }
// the unit that is randomised: the account (one arm for life) or the attempt (blocks within the account)
// (accounts are counted among those the experiment takes, so its blocks stay balanced)
const nth = exp.accounts ? Math.floor(accountSeq / exp.accounts.every) * exp.accounts.take + (accountSeq % exp.accounts.every) : accountSeq;
const arm = exp.unit === "account" ? armFor(exp, nth) : armFor({ ...exp, seed: `${exp.seed || 0}|${accountSeq}` }, attempt - exp.attempts.from);
const off = exp.accounts ? (((accountSeq - (exp.accounts.offset || 0)) % exp.accounts.every) + exp.accounts.every) % exp.accounts.every : 0;
const nth = exp.accounts ? Math.floor(accountSeq / exp.accounts.every) * exp.accounts.take + off : accountSeq;
let arm = exp.unit === "account" ? armFor(exp, nth) : armFor({ ...exp, seed: `${exp.seed || 0}|${accountSeq}` }, attempt - exp.attempts.from);
// cycle: a different combination of these decisions on every attempt of the account, none repeated until all
// are used (the combinations are walked with a step that shares no factor with their number)
if (arm.cycle) {
const keys = Object.keys(arm.cycle), total = keys.reduce((n, k) => n * arm.cycle[k].length, 1);
const gcd = (a, b) => (b ? gcd(b, a % b) : a), h = hash(`${exp.seed || 0}|${exp.id}|${accountSeq}`);
let step = (h % total) || 1;
while (gcd(step, total) !== 1) step = (step % total) + 1;
let i = ((h >>> 8) + (attempt - exp.attempts.from) * step) % total;
const combo = {};
for (const k of keys) { combo[k] = arm.cycle[k][i % arm.cycle[k].length]; i = Math.floor(i / arm.cycle[k].length); }
arm = { id: keys.map((k) => `${k}=${combo[k]}`).join(" / "), fix: { ...arm.fix, ...combo } };
}
return { experimentId: exp.id, arm: arm.id, fix: arm.fix || null, target: exp.target, objective: exp.objective || "eventually",
sampling: exp.sampling || "best", seed: hash(`${plan.seed || 0}|${exp.id}|${accountSeq}|${attempt}`),
propensity: (exp.share / total) / exp.arms.length, burn: exp.burn || null, planId: plan.id };
+12 -2
View File
@@ -70,6 +70,11 @@ chance:
dieHour: { label: Survive the hour, outcomes: [die, live], context: [cadet, post], risk_label: Killed before the hour }
shadow:
label: Shadow at the tree
# Not dice. The developer has said the seed plays no part in whether the Black shows itself: it shows when the
# rider's choices meet its conditions (planner/knowledge.yaml). The conditions are not known yet, so the model
# still treats it as a roll: the number is the share of runs like this one that met them, which is our
# ignorance, not the game's chance. `guard` marks that for the page and the reports.
guard: true
outcomes: ["yes", "no"]
# The shadow is close to a first-attempt event (November 2026, form submissions, gated tree runs: 69 shadows in
# 748 first attempts, 7 in 1,995 later ones, the same in every week), so first and later attempts are two
@@ -80,7 +85,10 @@ chance:
# 78 shadows in 4199 tree runs that investigated and helped, 2 (both form submissions) in 646 that did not.
requires: { scream: Investigate the scream, clearing: Help your squadmate }
otherwise: "no"
shadowColour: { label: Shadow dragon, outcomes: [Black, Blue], context: [] }
# After the shadow comes a Blue (at the redwood), then the Black: riders' notes describe one sequence, and the
# Blue is a trial on the way (analysis/reports/BLACK.md section 5). The form records only which dragon ended the
# run, so this is counted as "reached the Black" against "died at the Blue".
shadowColour: { label: "After the shadow: past the Blue to the Black", outcomes: [Black, Blue], context: [] }
meadow: { label: Meadow dragon, outcomes: [Orange, Brown, Blue], context: [cadet, post, fs, direction, company] }
# Bonding encounters: where each colour can be bonded. A run can only bond the encounter's own colour, except at
@@ -225,7 +233,9 @@ from_form:
# recommendOption() in core/solver.js uses these to point the bot at the recommended option on its screen.
bot_text:
company: { Alone: "^enter the valley alone", "1 Squadmate": "^enter the valley with one", "2 Squadmates": "^enter the valley with two" }
direction: { East: "^head east", West: "^head west", Center: "^head toward the center of the forest" }
leave: { Leave them: "^leave your squadmates?$" } # staying is answered on the same screen as the direction
# alone (or after leaving the squad), and with the squad: going their way or convincing them
direction: { East: "^head east|^go east with your squad|^convince them to head east", West: "^head west|^go west with your squad|^convince them to head west", Center: "^head toward the center of the forest" }
react: { Hold still: "hold still", Respect / face it: "in respect|turn to face it", Retreat / run: "^run and hope", Hide: "^hide", Meet its gaze: "meeting its gaze" }
scream: { Investigate the scream: "^investigate the scream", Keep walking: "^keep walking" }
clearing: { Help your squadmate: "^help your squadmate", Walk away: "^play it safe and leave them" }
+2 -2
View File
@@ -138,8 +138,8 @@
"description": "The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).",
"contentUrl": "https://git.kenschiano.net/kathleen/threshing-atlas/raw/branch/main/data/public/policy.json",
"encodingFormat": "application/json",
"sha256": "5a5baa8b7e740d60c8cbfcb3e07a5b26a9a42fda95bf6d7d155e34b6b9c17d8e",
"contentSize": "1082300 B"
"sha256": "780df39faa06102b519c5d055944d811977a767f29f5c3794b13f22f50a33586",
"contentSize": "1083878 B"
},
{
"@type": "cr:FileObject",
+641 -87
View File
@@ -1,19 +1,92 @@
{
"schema": 1,
"seed": 20261112,
"schema": 2,
"seed": 20261116,
"lifecycle": {
"research_days": 3,
"retire": true,
"long_lived": {
"every": 10,
"take": 1
"take": 2
}
},
"record": [
"accountSeq",
"accountAttempt",
"accountCreatedAt",
"wing",
"section",
"squad",
"sawShadowBefore"
],
"scripts": {
"shadow": [
{
"step": "the tree top",
"take": "shadow",
"never": "green|descend"
},
{
"step": "asked to turn back",
"take": "stay the course|certain you saw",
"never": "second thought|green"
},
{
"step": "asked again",
"take": "intriguing|even if this kills",
"never": "second thought|green"
},
{
"step": "the Blue lands",
"take": "freeze and lower",
"never": "approach|run"
},
{
"step": "another cadet arrives",
"take": "hold (completely )?still",
"never": "warn|turn while|approach"
},
{
"step": "the Blue burns the cadet",
"take": "hold (your )?position",
"never": "interven|approach|warn"
},
{
"step": "the Blue judges",
"take": "head lowered|await.*judg",
"never": "approach|sure"
},
{
"step": "the Blue has left",
"take": "help the .*cadet",
"never": ""
},
{
"step": "the cadet's burns",
"take": "wrap|cut .*sleeve|dagger to cut",
"never": ""
},
{
"step": "moving on",
"take": "head south",
"never": ""
},
{
"step": "a branch snaps",
"take": "turn to face",
"never": "ignore"
},
{
"step": "the Black",
"take": "hold (its|it's|the) gaze|lift your chin",
"never": "lower your eyes"
}
]
},
"experiments": [
{
"id": "first-attempt-shadow",
"id": "fresh-replicate",
"status": "running",
"question": "How often does the shadow appear on an account's first attempt, and does heading West (at the start, at the fork) raise it?",
"question": "On a fresh account, does one exact path always show the shadow, sometimes, or never?",
"attempts": {
"from": 1,
"to": 1
@@ -24,7 +97,7 @@
"target": "Black",
"objective": "eventually",
"sampling": "best",
"seed": 20261112,
"seed": 20261116,
"measures": {
"roll": "shadow",
"rare": "yes"
@@ -35,77 +108,18 @@
},
"arms": [
{
"id": "direction=West / post=West",
"levels": {
"direction": "West",
"post": "West"
},
"id": "all",
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"hour": "Tree",
"direction": "West",
"post": "West"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 60
}
},
{
"id": "direction=West / post=East",
"levels": {
"direction": "West",
"post": "East"
},
"fix": {
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"hour": "Tree",
"direction": "West",
"post": "East"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 60
}
},
{
"id": "direction=East / post=West",
"levels": {
"direction": "East",
"post": "West"
},
"fix": {
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"hour": "Tree",
"direction": "East",
"post": "West"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 60
}
},
{
"id": "direction=East / post=East",
"levels": {
"direction": "East",
"post": "East"
},
"fix": {
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"hour": "Tree",
"direction": "East",
"post": "East"
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
@@ -123,9 +137,392 @@
}
},
{
"id": "attempt-2-shadow",
"id": "fresh-ablate",
"status": "waiting",
"note": "starts when fresh-replicate is complete",
"question": "Which choices on that path does the shadow need?",
"attempts": {
"from": 1,
"to": 1
},
"unit": "account",
"share": 1,
"exclusive": true,
"target": "Black",
"objective": "eventually",
"sampling": "best",
"seed": 20261116,
"measures": {
"roll": "shadow",
"rare": "yes"
},
"burn": {
"reason": "TREE_NO_SHADOW_AUTO_BURN",
"when": "no shadow at the tree"
},
"arms": [
{
"id": "with one squadmate",
"change": {
"company": "1 Squadmate",
"leave": "Stay together"
},
"fix": {
"company": "1 Squadmate",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree",
"leave": "Stay together"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "with two squadmates",
"change": {
"company": "2 Squadmates",
"leave": "Stay together"
},
"fix": {
"company": "2 Squadmates",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree",
"leave": "Stay together"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "leave the squad",
"change": {
"company": "2 Squadmates",
"leave": "Leave them"
},
"fix": {
"company": "2 Squadmates",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree",
"leave": "Leave them"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "east first",
"change": {
"direction": "East"
},
"fix": {
"company": "Alone",
"direction": "East",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "centre first",
"change": {
"direction": "Center"
},
"fix": {
"company": "Alone",
"direction": "Center",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "face the first dragon",
"change": {
"react": "Respect / face it"
},
"fix": {
"company": "Alone",
"direction": "West",
"react": "Respect / face it",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "charge the smaller cadet",
"change": {
"fight": "Charge the smaller cadet"
},
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Charge the smaller cadet",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "charge the bigger cadet",
"change": {
"dagger": "Charge the bigger cadet anyway"
},
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Charge the bigger cadet anyway",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "kill the bullies",
"change": {
"fate": "Killed the bullies yourself"
},
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Killed the bullies yourself",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "leave the bullies",
"change": {
"fate": "Left bullies to the dragons"
},
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Left bullies to the dragons",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "fork to the forest",
"change": {
"post": "Forest"
},
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "Forest",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
},
{
"id": "fork east",
"change": {
"post": "East"
},
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "East",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 15
}
}
],
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"unknownArms": []
}
},
{
"id": "shadow-again",
"status": "running",
"question": "Is the shadow really gone after the first attempt?",
"question": "Does an account that saw the shadow, and died, see it again on the same path?",
"attempts": {
"from": 2,
"to": 4
},
"unit": "account",
"share": 1,
"exclusive": true,
"when": {
"sawShadowBefore": true
},
"target": "Black",
"objective": "eventually",
"sampling": "best",
"seed": 20261116,
"measures": {
"roll": "shadow",
"rare": "yes"
},
"burn": {
"reason": "TREE_NO_SHADOW_AUTO_BURN",
"when": "no shadow at the tree"
},
"arms": [
{
"id": "all",
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 12
}
}
],
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"unknownArms": []
}
},
{
"id": "second-attempt-same-path",
"status": "running",
"question": "Does an account that missed on its first attempt miss again on the same path?",
"attempts": {
"from": 2,
"to": 2
@@ -134,13 +531,139 @@
"share": 1,
"accounts": {
"every": 2,
"take": 1
"take": 1,
"offset": 0
},
"exclusive": true,
"when": {
"sawShadowBefore": false
},
"target": "Black",
"objective": "eventually",
"sampling": "best",
"seed": 20261112,
"seed": 20261116,
"measures": {
"roll": "shadow",
"rare": "yes"
},
"burn": {
"reason": "TREE_NO_SHADOW_AUTO_BURN",
"when": "no shadow at the tree"
},
"arms": [
{
"id": "all",
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 60
}
}
],
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"unknownArms": []
}
},
{
"id": "veteran-repeat",
"status": "running",
"question": "Does an older account open up by repeating one path?",
"attempts": {
"from": 3,
"to": null
},
"unit": "account",
"share": 1,
"accounts": {
"every": 10,
"take": 1,
"offset": 0
},
"exclusive": true,
"when": {
"sawShadowBefore": false
},
"target": "Black",
"objective": "eventually",
"sampling": "best",
"seed": 20261116,
"measures": {
"roll": "shadow",
"rare": "yes"
},
"burn": {
"reason": "TREE_NO_SHADOW_AUTO_BURN",
"when": "no shadow at the tree"
},
"arms": [
{
"id": "all",
"fix": {
"company": "Alone",
"direction": "West",
"react": "Hold still",
"scream": "Investigate the scream",
"clearing": "Help your squadmate",
"fight": "Pause to assess",
"dagger": "Engage the smaller cadet",
"fate": "Squadmate's revenge",
"post": "West",
"hour": "Tree"
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 300
}
}
],
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"unknownArms": []
}
},
{
"id": "veteran-variety",
"status": "running",
"question": "Does an older account open up by varying its choices?",
"attempts": {
"from": 3,
"to": null
},
"unit": "account",
"share": 1,
"accounts": {
"every": 10,
"take": 1,
"offset": 1
},
"exclusive": true,
"when": {
"sawShadowBefore": false
},
"target": "Black",
"objective": "eventually",
"sampling": "best",
"seed": 20261116,
"measures": {
"roll": "shadow",
"rare": "yes"
@@ -157,11 +680,42 @@
"clearing": "Help your squadmate",
"hour": "Tree"
},
"cycle": {
"company": [
"Alone",
"1 Squadmate",
"2 Squadmates"
],
"direction": [
"West",
"East",
"Center"
],
"fight": [
"Pause to assess",
"Charge the smaller cadet",
"Charge between the bigger cadet and your squadmate"
],
"dagger": [
"Engage the smaller cadet",
"Charge the bigger cadet anyway"
],
"fate": [
"Killed the bullies yourself",
"Squadmate's revenge",
"Left bullies to the dragons"
],
"post": [
"Forest",
"West",
"East"
]
},
"progress": {
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 120
"planned": 300
}
}
],
@@ -181,11 +735,11 @@
"to": null
},
"unit": "attempt",
"share": 0.6,
"share": 0.5,
"target": "Blue",
"objective": "eventually",
"sampling": "best",
"seed": 20261112,
"seed": 20261116,
"measures": {
"roll": "meadow",
"rare": "Blue"
@@ -320,17 +874,17 @@
{
"id": "mapping",
"status": "running",
"question": "What does every branch of the valley lead to?",
"question": "What does every option on every screen lead to?",
"attempts": {
"from": 2,
"to": null
},
"unit": "attempt",
"share": 0.25,
"share": 0.35,
"target": "Blue",
"objective": "eventually",
"sampling": "uniform",
"seed": 20261112,
"seed": 20261116,
"measures": null,
"burn": {
"reason": "NOT_TARGET_AUTO_BURN",
@@ -344,7 +898,7 @@
"runs": 0,
"observed": 0,
"rare": 0,
"planned": 600
"planned": 1000
}
}
],
@@ -368,7 +922,7 @@
"target": "Blue",
"objective": "eventually",
"sampling": "best",
"seed": 20261112,
"seed": 20261116,
"measures": {
"roll": "meadow",
"rare": "Blue"
@@ -404,5 +958,5 @@
}
}
],
"id": "ff265e6f994d"
"id": "4f436e3d81c0"
}
File diff suppressed because one or more lines are too long
+38 -11
View File
@@ -5,10 +5,18 @@ The Atlas side is done (branch `first-attempt-cycle`). This is the bot's side. D
## Why
In the community form, the shadow shows on 9% of first attempts at the tree and 0.3% of later ones (69 in 748
against 7 in 1,995, the same in every week). The bot has 236 tree runs and no shadow, all on attempt 2 or later;
it has never logged a first attempt. So: every new account's first attempt is the Black attempt, and it is also
the experiment that measures the first-attempt rate without reporting bias.
Two things changed the plan (16 November).
**The developer says the Black is not dice** (`planner/knowledge.yaml`). The seed plays no part in whether it shows:
it shows when the rider's choices meet its conditions, and a rider who knows them bonds with certainty. There is no
single route; the bullies encounter and the tree are gates; there are a handful of conditions, not dozens.
**The runs say it is close to a first-attempt event** (`analysis/reports/BLACK.md`). On one and the same recorded
route, riders report the shadow on 21 of 79 first attempts and 1 of 80 later ones. The bot has 236 tree runs and no
shadow, all on attempt 2 or later; it has never logged a first attempt.
So the bot stops estimating a rate and starts looking for the rule: one exact path on every fresh account, then
one choice changed at a time. If the answer is not dice, a few accounts per question are enough.
## The five changes
@@ -19,7 +27,7 @@ the experiment that measures the first-attempt rate without reporting bias.
then:
```js
const a = Solver.assign(plan, { accountSeq, attempt }); // what this attempt is for
const a = Solver.assign(plan, { accountSeq, attempt, facts: { sawShadowBefore } }); // what this attempt is for
const adv = Solver.advisor(policy.samples.all, a.target, a.objective,
{ attempt, fix: a.fix, sampling: a.sampling, seed: a.seed });
```
@@ -27,7 +35,7 @@ the experiment that measures the first-attempt rate without reporting bias.
and log `experimentId`, `arm`, `planId`, `policyId`, `accountAttempt` and the propensity on the run. Never
reload the kit in the middle of an attempt.
4. **The account's life** (`plan.lifecycle`): attempt 1 is the Black attempt; then three days of research
attempts; then retire the account, except when `accountSeq % 10 === 0` (those keep running).
attempts; then retire the account, except when `accountSeq % 10` is 0 or 1 (those keep running).
5. **Run the cycle on a timer** on the bot's machine, from a clone of the Atlas:
```
@@ -44,10 +52,29 @@ the experiment that measures the first-attempt rate without reporting bias.
| Attempts | Experiment | What the bot does |
|---|---|---|
| 1 | `first-attempt-shadow` | Investigate, help, tree. Direction and fork are West or East by arm (4 arms, 60 accounts each). Burn when there is no shadow; bond a Black. |
| 2 (half the accounts) | `attempt-2-shadow` | The same tree run once more, to check the shadow is gone. |
| 2 onward, 60% | `meadow-blue-route` | Go to the meadow; investigate or keep walking, and the first direction, by arm (6 arms, 70 each). Burn unless a Blue waits. |
| 2 onward, 25% | `mapping` | Pick at random among the options on each screen (`recommendOption` does it). |
| 2 onward, 15% | `sentinel` | One fixed route, to notice if the game's odds change. |
| 1 | `fresh-replicate` (60 accounts) | One exact path: alone, west, hold still, investigate, help, pause to assess, engage the smaller cadet, squadmate's revenge, fork west, tree. Burn when there is no shadow. |
| 1, afterwards | `fresh-ablate` (12 arms, 15 accounts each) | The same path with one choice changed. Starts by itself when the replication is complete. |
| 2 to 4, accounts that saw the shadow and died | `shadow-again` | The same path again. |
| 2, half of the other accounts | `second-attempt-same-path` | The same path again. |
| 3 onward, one account in ten | `veteran-repeat` | The same path on every attempt, indefinitely. |
| 3 onward, one account in ten | `veteran-variety` | A different combination of company, direction, fight, fate and fork on every attempt, both gates kept. |
| 2 onward, the rest | `meadow-blue-route` 50%, `mapping` 35%, `sentinel` 15% | As before. |
`assign()` now takes the account's facts: `Solver.assign(plan, { accountSeq, attempt, facts: { sawShadowBefore } })`.
## When the shadow shows
The bot has never seen these screens, so `experiments.json` carries the sequence riders report for a bonded Black
(`plan.scripts.shadow`): for each screen in order, a pattern for the option to take and one for options never to
take. In riders' notes, approaching the Blue or warning the other cadet has always been fatal, lowering your eyes to
the Black has never bonded (0 of 7), and holding its gaze bonded 27 of 39. Where a screen matches nothing in the
script, stop and notify a person: do not guess. Log every screen, its options and the pick; the 12 failures while
holding its gaze are unexplained, and the screens before it are where the explanation should be.
## What to record about the account
`plan.record` lists it: `accountSeq`, `accountAttempt`, `accountCreatedAt`, the cadet's `wing`, `section` and
`squad` as the site shows them, and `sawShadowBefore`. If identical paths on fresh accounts disagree about the shadow,
the difference is something the site knows about the account, and these are the candidates we can see.
The plan updates itself from the runs: a finished experiment stops being assigned and its share goes to the others.
+6 -2
View File
@@ -37,7 +37,7 @@ opens the pull request with the new data.
const plan = require('../vendor/threshing-atlas/experiments.json');
// accountSeq: the account's number in the order accounts were made (0, 1, 2, ...), fixed for its life
// attempt: the account's attempt number (1 = its first threshing)
const a = Solver.assign(plan, { accountSeq, attempt });
const a = Solver.assign(plan, { accountSeq, attempt, facts: { sawShadowBefore } });
// -> { experimentId, arm, fix, target, objective, sampling, seed, propensity, burn, planId } (null: no experiment)
const advisor = Solver.advisor(policy.samples.all, a.target, a.objective,
{ attempt, fix: a.fix, sampling: a.sampling, seed: a.seed });
@@ -54,7 +54,11 @@ const advisor = Solver.advisor(policy.samples.all, a.target, a.objective,
- `burn` names the deliberate-burn rule the experiment expects (`TREE_NO_SHADOW_AUTO_BURN`,
`MEADOW_NO_BLUE_AUTO_BURN`). A genuine Black or Blue bond is always accepted.
- The account's life (`plan.lifecycle`): attempt 1 is the Black attempt; then about three days of research
attempts; then the account retires, except one in ten, which keeps going so that late attempts are observed.
attempts; then the account retires, except two in ten, which keep going (one repeating a path, one varying it).
- `plan.record` lists what to log about the account on every run; `plan.scripts.shadow` is the sequence to follow
once the shadow shows (`docs/BOT_CYCLE_FOR_KEN.md`).
- With squadmates, staying together and the direction are one screen; `recommendOption()` answers both at once
and says so in `also`.
An experiment that reaches its planned count becomes `complete`, one that meets its stopping rule becomes `paused`;
either way `assign()` stops giving it out and its share goes to the others. Nothing in the bot changes.
+24
View File
@@ -285,3 +285,27 @@ Branch `first-attempt-cycle` (not merged, not pushed).
the bot that is running, so the bot's side was written as a specification, not against its code.
- **Not done:** the page does not show first-attempt odds yet; Gitea still needs Kathleen's account, the
repository and the push (`docs/REPO_SETTINGS.md`); git.kenschiano.net answers (Gitea 1.27.0).
## 11. Update, 2026-11-16: the developer's statements, and looking for the Black's rule
- **Source.** The developer's answers in a community Discord channel (135 forwarded messages, exported
2026-11-16). The export is private raw data; `planner/knowledge.yaml` holds the statements paraphrased, dated and
graded (said / hint / declined), with what the Atlas does with each and the open questions.
- **What it changes.** The Black is not dice: the seed plays no part, it shows when the rider's choices meet its
conditions, and knowledge gives certainty. The bullies encounter and the tree are gates; there is no single route;
about five conditions. The Blue is dice from the seed. No event is rarer than 1 in 20. Runs are never forgotten
and never hurt.
- **What the runs add** (`analysis/black/evidence.py` -> `analysis/reports/BLACK.md`, rebuilt every cycle): on the
same recorded route, 21 of 79 first attempts show the shadow against 1 of 80 later ones (across routes: odds ratio
21.5, z = 9.8); on that route, runs with the same wing, section and squad still disagree; shadows are on 44
different recorded routes; after the shadow comes a Blue, then the Black; holding the Black's gaze bonded 27 of
39, lowering eyes 0 of 7.
- **Model.** The shadow is marked `guard: true` in `game.yaml`: the model's number is the share of runs that met
unknown conditions, not a chance. `shadowColour` is "past the Blue to the Black".
- **Plan (version 2).** Attempt 1 on every fresh account is one exact path (`fresh-replicate`, 60 accounts), then
one choice changed at a time (`fresh-ablate`, starts by itself). `shadow-again`, `second-attempt-same-path`,
`veteran-repeat` and `veteran-variety` test fresh against veteran. `assign()` takes account facts; the plan carries
what to record and the sequence to follow once the shadow shows.
- **Not known.** What differs between fresh accounts that see the shadow and those that do not on the same route;
what "twenty" counts; why holding the Black's gaze fails three times in ten.
- **To tidy.** One public `bond_note` holds a rider's handle ("Run 116 - ..."), in a row marked as a first attempt.
+1 -1
View File
@@ -10,7 +10,7 @@
"test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js",
"test": "npm run test:py && npm run test:unit",
"graph": "node analysis/network/build-graph.js",
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py",
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py",
"cycle": "python3 -m collection cycle",
"stats": "python3 analysis/stats/site_stats.py",
"collect": "python3 -m collection update",
+1
View File
@@ -7,6 +7,7 @@ The route planner: the best route to every dragon, and what goes back to Ken's b
| `build_policy.py` | Solves the route model (`core/atlas/solver.py`, built from the game in `core/spec/game.yaml`) for every colour and both goals ("this attempt" and "eventually"), with 200 posterior draws and a cross-fit check. Writes `data/public/policy.json`, which has a content id so each bot run can record which policy it followed. About 2 minutes. |
| `policy_report.py` | Writes the readable version, `analysis/reports/POLICY.md`. |
| `knowledge.yaml` | What the game's developer has said about the mechanics (paraphrased, dated, graded said / hint / declined), what the Atlas does with each statement, and the open questions with the experiment that tests each. |
| `experiments.yaml` | The experiment plan, written by hand: what each attempt of each bot account is for (the Black attempt on attempt 1, then the meadow Blue experiment, mapping and the sentinel route), with planned counts and stopping rules. |
| `build_experiments.py` | Checks the plan against the game, counts what the bot has logged for every arm, sets each experiment's status (running, complete, paused), and writes `data/public/experiments.json` for the bot. |
+45 -7
View File
@@ -19,6 +19,7 @@ import hashlib
import itertools
import json
import random
import re
import sys
from pathlib import Path
@@ -47,10 +48,31 @@ def check_options(where, fixed):
raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})")
def arms(exp):
"""Every combination of the factors, each with the route held on every arm."""
def arms(exp, routes=None):
"""The arms of an experiment, each with the decisions it holds (`fix`): every combination of the factors, or
the explicit `arms` (one `change` to the route each), or a single arm. A `cycle` arm also carries the decisions
it varies from attempt to attempt."""
route = exp.get("route") or {}
if isinstance(route, str):
if route not in (routes or {}):
raise ValueError(f"{exp['id']}: unknown route '{route}'")
route = routes[route]
check_options(exp["id"] + " route", route)
if exp.get("arms"):
out = []
for a in exp["arms"]:
check_options(f"{exp['id']} arm {a['id']}", a.get("change"))
out.append({"id": a["id"], "change": a.get("change") or {}, "fix": {**route, **(a.get("change") or {})}})
if len({a["id"] for a in out}) != len(out):
raise ValueError(f"{exp['id']}: arm ids must be unique")
return out
if exp.get("cycle"):
for k, opts in exp["cycle"].items():
if k in route:
raise ValueError(f"{exp['id']}: '{k}' is both cycled and part of the route")
for o in opts:
check_options(exp["id"] + " cycle", {k: o})
return [{"id": "all", "fix": dict(route), "cycle": exp["cycle"]}]
factors = exp.get("factors") or {}
for k, opts in factors.items():
if k in route:
@@ -71,6 +93,11 @@ def validate(d):
if len(set(ids)) != len(ids):
raise ValueError("experiment ids must be unique")
for e in d["experiments"]:
if e.get("starts_after") and e["starts_after"] not in ids:
raise ValueError(f"{e['id']}: starts_after names no experiment")
for k in (e.get("when") or {}):
if k not in (d.get("record") or []):
raise ValueError(f"{e['id']}: when.{k} is not something the bot records (`record`)")
if e["target"] not in model.COLOURS:
raise ValueError(f"{e['id']}: unknown target {e['target']}")
if e.get("sampling", "best") not in SAMPLING:
@@ -117,14 +144,16 @@ def build(d=None, log=None):
d = d or design()
validate(d)
log = logged() if log is None else log
exps = []
exps, status_of = [], {}
for e in d["experiments"]:
m = e.get("measures")
a = arms(e)
a = arms(e, d.get("routes"))
tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a}
for exp_id, arm, v in log:
if exp_id != e["id"]:
continue
if a[0].get("cycle"): # a cycled arm is logged under the combination it played: all one arm here
arm = a[0]["id"]
t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0})
t["runs"] += 1
y = model.OBSERVE[m["roll"]](v) if m and v else None
@@ -138,7 +167,9 @@ def build(d=None, log=None):
total_rare = sum(t["rare"] for t in tally.values())
status, note = "running", None
se = e.get("stop_early")
if not e.get("ongoing") and all(n >= planned for n in done.values()):
if e.get("starts_after") and status_of.get(e["starts_after"]) != "complete":
status, note = "waiting", f"starts when {e['starts_after']} is complete"
elif not e.get("ongoing") and all(n >= planned for n in done.values()):
status, note = "complete", f"every arm has its planned {planned}"
elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]:
status = "paused"
@@ -146,18 +177,25 @@ def build(d=None, log=None):
for x in a:
t = tally[x["id"]]
x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})}
status_of[e["id"]] = status
exps.append({
"id": e["id"], "status": status, **({"note": note} if note else {}),
"question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")},
"unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}),
**({"exclusive": True} if e.get("exclusive") else {}),
**({"when": e["when"]} if e.get("when") else {}),
"target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"),
"seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a,
"progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare,
**({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}),
"unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})},
})
plan = {"schema": 1, "seed": d["seed"], "lifecycle": d["lifecycle"], "experiments": exps}
for s in (d.get("scripts") or {}).values():
for st in s:
for k in ("take", "never"):
re.compile(st.get(k) or "")
plan = {"schema": 2, "seed": d["seed"], "lifecycle": d["lifecycle"], "record": d.get("record") or [],
"scripts": d.get("scripts") or {}, "experiments": exps}
plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12]
return plan
@@ -171,7 +209,7 @@ def summary(plan):
for e in plan["experiments"]:
p = e["progress"]
est = p.get("estimate")
L.append(f" {e['id']:<22} {e['status']:<9} {p['observed']} observed, {p['rare']} rare"
L.append(f" {e['id']:<26} {e['status']:<9} {p['observed']} observed, {p['rare']} rare"
+ (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "")
+ (f" [{e['note']}]" if e.get("note") else ""))
return "\n".join(L)
+173 -53
View File
@@ -1,85 +1,209 @@
# The experiment plan for Ken's bot: what each attempt of each account is for.
#
# This file is the design (written by hand). planner/build_experiments.py checks it against the game
# (core/spec/game.yaml), counts what the bot has run so far, decides which experiments are still running, and
# writes data/public/experiments.json. The bot reads that file with assign() in core/solver.js. The cycle
# (python -m collection cycle) rebuilds it after every import of bot runs, so the plan follows the data:
# an experiment that reaches its planned count, or its stopping rule, stops being assigned, and its share goes
# to the others.
# (core/spec/game.yaml), counts what the bot has run so far, decides which experiments are running, and writes
# data/public/experiments.json. The bot reads that file with assign() in core/solver.js. The cycle
# (python -m collection cycle) rebuilds it after every import of bot runs, so the plan follows the data.
#
# Version 2 (2026-11-16). The developer has said the Black is not dice: it shows when a rider's choices meet its
# conditions (planner/knowledge.yaml), and riders report it almost only on a first attempt, on many different
# routes (analysis/reports/BLACK.md). So the Black experiments no longer estimate a rate. They look for the rule:
# hold one exact path fixed on fresh accounts, see whether the answer is always the same, then change one choice
# at a time. Because the answer is not dice, a few accounts per question are enough.
#
# Rules that keep the results honest:
# - counts are planned in advance; an experiment does not stop because its results look good;
# - every started attempt is logged, including aborted ones;
# - every started attempt is logged, including aborted ones, and every screen with its options and the pick;
# - the bot never accepts a bond that is not the experiment's target colour (it burns), and it does accept a
# genuine Black or Blue bond.
version: 1
seed: 20261112 # changes every arm assignment: leave it alone once accounts are running
version: 2
seed: 20261116 # changes every arm assignment: leave it alone once accounts are running
# ---------------------------------------------------------------- an account's life
lifecycle:
# attempt 1 of every new account is the Black attempt (the shadow is close to a first-attempt event), and is
# itself an experiment: first-attempt-shadow below
# attempt 1 of every new account is the Black attempt, on one exact path (fresh-replicate, then fresh-ablate)
research_days: 3 # then the account collects route data for this long (about 18 attempts at 4 hours each)
retire: true # and stops, except the long-lived accounts
long_lived: { every: 10, take: 1 } # one account in ten keeps running, so attempts past 18 are still observed
long_lived: { every: 10, take: 2 } # two accounts in ten keep running (veteran-repeat and veteran-variety)
# What the bot records about the account on every run, beyond the steps: the Black's conditions rest on choices and
# on what the site knows about the account, so anything that could be such a fact is kept.
record:
- accountSeq # the account's number, in the order the accounts were made
- accountAttempt # 1 = the account's first threshing
- accountCreatedAt # when the account was made (UTC)
- wing # the cadet's assignment, as the site shows it
- section
- squad
- sawShadowBefore # whether any earlier run of the account showed the shadow
# ---------------------------------------------------------------- named paths
routes:
# The recorded route with the most first-attempt shadows in the community form: 21 in 79 first attempts, 1 in 80
# later ones (BLACK.md section 2).
best-reported:
company: Alone
direction: West
react: Hold still
scream: Investigate the scream
clearing: Help your squadmate
fight: Pause to assess
dagger: Engage the smaller cadet
fate: Squadmate's revenge
post: West
hour: Tree
# ---------------------------------------------------------------- what to do once the shadow shows
# The sequence in the notes of riders who bonded the Black (BLACK.md section 5), as keywords of the option to
# take on each screen, in order. The bot has never seen these screens, and a shadow must not be wasted: where a
# screen matches nothing here it should stop and notify a person, not guess.
scripts:
shadow:
- { step: the tree top, take: "shadow", never: "green|descend" }
- { step: asked to turn back, take: "stay the course|certain you saw", never: "second thought|green" }
- { step: asked again, take: "intriguing|even if this kills", never: "second thought|green" }
- { step: the Blue lands, take: "freeze and lower", never: "approach|run" }
- { step: another cadet arrives, take: "hold (completely )?still", never: "warn|turn while|approach" }
- { step: the Blue burns the cadet, take: "hold (your )?position", never: "interven|approach|warn" }
- { step: the Blue judges, take: "head lowered|await.*judg", never: "approach|sure" }
- { step: the Blue has left, take: "help the .*cadet", never: "" }
- { step: the cadet's burns, take: "wrap|cut .*sleeve|dagger to cut", never: "" }
- { step: moving on, take: "head south", never: "" }
- { step: a branch snaps, take: "turn to face", never: "ignore" }
- { step: the Black, take: "hold (its|it's|the) gaze|lift your chin", never: "lower your eyes" }
# ---------------------------------------------------------------- the experiments
# attempts: which of an account's attempts it covers (from / to, inclusive; no `to` = from then on).
# unit: what is randomised. account = one arm for the account's life (needed when only attempt 1 counts);
# attempt = arms in balanced blocks within each account.
# share: when several experiments cover an attempt, how often each gets it.
# exclusive: takes every attempt it covers; the shared experiments get the rest.
# accounts: { every: N, take: K } limits it to K accounts in every N (by the order accounts were made).
# factors: decisions that are randomised; every combination is an arm. route: decisions held on every arm.
# Everything else is the advisor's best play for the target.
# sampling: best (default), thompson, or uniform (every option on the screen equally likely: mapping).
# measures: the roll and the outcome that are counted. planned_per_arm: how many observed rolls per arm.
# stop_early: pause when, after `after` observed rolls in total, at most `rare_at_most` came out rare.
# unit: what is randomised. account = one arm for the account's life; attempt = balanced blocks within an account.
# exclusive: takes every attempt it covers; the shared experiments (share) split the rest.
# accounts: { every: N, take: K, offset: J } limits it to K accounts in every N, starting at the J-th.
# when: facts about the account that must hold (from `record`).
# starts_after: waits until that experiment is complete.
# route: decisions held on every arm (a name from `routes`, or a mapping). factors: every combination is an arm.
# arms: explicit arms, each a `change` to the route (one choice at a time). cycle: a different combination of
# these decisions on every attempt, never repeating until all are used.
# measures: the roll or guard and the outcome that are counted. planned_per_arm: observed outcomes per arm.
experiments:
- id: first-attempt-shadow
question: How often does the shadow appear on an account's first attempt, and does heading West (at the start, at the fork) raise it?
- id: fresh-replicate
question: On a fresh account, does one exact path always show the shadow, sometimes, or never?
why: >
Form submissions show the shadow on 9% of first attempts and 0.3% of later ones, but form submissions
over-report it, complete logs have only 28 first attempts, and the bot has none. Among first attempts the
form shows more shadows after West (14%) than East (4%), which may be copied guide routes.
If the Black is decided by this run's choices alone, identical choices on fresh accounts must agree: all
shadow or none. If they disagree, fresh accounts differ in something the site knows and we do not record
yet, and comparing the accounts that saw it with those that did not is how to find it.
attempts: { from: 1, to: 1 }
unit: account
exclusive: true
share: 1
target: Black
objective: eventually
factors: { direction: [West, East], post: [West, East] }
route: { scream: Investigate the scream, clearing: Help your squadmate, hour: Tree }
route: best-reported
measures: { roll: shadow, rare: "yes" }
planned_per_arm: 60 # 240 accounts: 120 per level of each factor
stop_early: { after: 120, rare_at_most: 0 } # no shadow in 120 first attempts rules out 2.5% or more
planned_per_arm: 60 # if riders' 27% is three to four times too high, about 4 to 16 shadows
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
- id: attempt-2-shadow
question: Is the shadow really gone after the first attempt?
why: The form shows 3 shadows in 295 second attempts (1%), then almost none.
attempts: { from: 2, to: 2 }
- id: fresh-ablate
question: Which choices on that path does the shadow need?
why: Change one choice, keep the rest. The developer hints at about five conditions, two of them the known gates.
starts_after: fresh-replicate
attempts: { from: 1, to: 1 }
unit: account
accounts: { every: 2, take: 1 }
exclusive: true
share: 1
target: Black
objective: eventually
route: { scream: Investigate the scream, clearing: Help your squadmate, hour: Tree }
route: best-reported
arms:
- { id: with one squadmate, change: { company: 1 Squadmate, leave: Stay together } }
- { id: with two squadmates, change: { company: 2 Squadmates, leave: Stay together } }
- { id: leave the squad, change: { company: 2 Squadmates, leave: Leave them } }
- { id: east first, change: { direction: East } }
- { id: centre first, change: { direction: Center } }
- { id: face the first dragon, change: { react: Respect / face it } }
- { id: charge the smaller cadet, change: { fight: Charge the smaller cadet } }
- { id: charge the bigger cadet, change: { dagger: Charge the bigger cadet anyway } }
- { id: kill the bullies, change: { fate: Killed the bullies yourself } }
- { id: leave the bullies, change: { fate: Left bullies to the dragons } }
- { id: fork to the forest, change: { post: Forest } }
- { id: fork east, change: { post: East } }
measures: { roll: shadow, rare: "yes" }
planned_per_arm: 120
planned_per_arm: 15 # set from fresh-replicate: enough that "none" is not bad luck at the rate found there
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
- id: shadow-again
question: Does an account that saw the shadow, and died, see it again on the same path?
why: The cleanest test of fresh against veteran. About three in ten riders who reach the Black die there.
attempts: { from: 2, to: 4 }
when: { sawShadowBefore: true }
unit: account
exclusive: true
share: 1
target: Black
route: best-reported
measures: { roll: shadow, rare: "yes" }
planned_per_arm: 12
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
- id: second-attempt-same-path
question: Does an account that missed on its first attempt miss again on the same path?
why: The form shows 21 in 79 on first attempts and 1 in 80 later on this route; the bot can check it on identical paths.
attempts: { from: 2, to: 2 }
when: { sawShadowBefore: false }
accounts: { every: 2, take: 1, offset: 0 }
unit: account
exclusive: true
share: 1
target: Black
route: best-reported
measures: { roll: shadow, rare: "yes" }
planned_per_arm: 60
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
- id: veteran-repeat
question: Does an older account open up by repeating one path?
why: >
The developer says a run never hurts and names twenty as a count that matters. No bot account has repeated
one route more than 11 times in a row.
attempts: { from: 3 }
when: { sawShadowBefore: false }
accounts: { every: 10, take: 1, offset: 0 }
unit: account
exclusive: true
share: 1
target: Black
route: best-reported
measures: { roll: shadow, rare: "yes" }
planned_per_arm: 300
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
- id: veteran-variety
question: Does an older account open up by varying its choices?
why: The opposite of veteran-repeat, on the next account. A different combination every attempt, both gates kept.
attempts: { from: 3 }
when: { sawShadowBefore: false }
accounts: { every: 10, take: 1, offset: 1 }
unit: account
exclusive: true
share: 1
target: Black
route: { scream: Investigate the scream, clearing: Help your squadmate, hour: Tree }
cycle:
company: [Alone, 1 Squadmate, 2 Squadmates]
direction: [West, East, Center]
fight: [Pause to assess, Charge the smaller cadet, Charge between the bigger cadet and your squadmate]
dagger: [Engage the smaller cadet, Charge the bigger cadet anyway]
fate: [Killed the bullies yourself, Squadmate's revenge, Left bullies to the dragons]
post: [Forest, West, East]
measures: { roll: shadow, rare: "yes" }
planned_per_arm: 300
burn: { reason: TREE_NO_SHADOW_AUTO_BURN, when: no shadow at the tree }
- id: meadow-blue-route
question: Does anything before the meadow change how often the Blue waits there?
why: >
Complete logs show a Blue on about 10% of meadow visits whatever the route, from routes the bot's rules chose.
Random assignment is what can confirm that the route does not matter (or find the route where it does).
The developer says the Blue is dice from the seed. Complete logs show a Blue on about 10% of meadow visits
whatever the route; random assignment confirms it or finds the scene where the odds differ.
attempts: { from: 2 }
unit: attempt
share: 0.6
share: 0.5
target: Blue
objective: eventually
factors: { scream: [Investigate the scream, Keep walking], direction: [West, East, Center] }
route: { hour: Valley, valley: Pass the Red (meadow) }
measures: { roll: meadow, rare: Blue }
@@ -87,28 +211,24 @@ experiments:
burn: { reason: MEADOW_NO_BLUE_AUTO_BURN, when: no Blue in the meadow }
- id: mapping
question: What does every branch of the valley lead to?
why: Probabilities for every path to every dragon need runs on the paths nobody chooses.
question: What does every option on every screen lead to?
why: Probabilities for every path need runs on the paths nobody chooses. The game has about 24 million paths, so this maps screens, not paths.
attempts: { from: 2 }
unit: attempt
share: 0.25
share: 0.35
target: Blue
objective: eventually
sampling: uniform
planned_per_arm: 600
planned_per_arm: 1000
burn: { reason: NOT_TARGET_AUTO_BURN, when: the encounter is not a Blue or a Black }
- id: sentinel
question: Have the game's odds changed?
why: >
One fixed route, run all the time, by accounts that log everything: a change in its rates is a change in the
game, not in who reports. It is the most-logged meadow route.
why: One fixed route, run all the time by accounts that log everything. It is the most-logged meadow route.
attempts: { from: 2 }
unit: attempt
share: 0.15
target: Blue
objective: eventually
route: { company: Alone, direction: Center, scream: Keep walking, wounded: Tourniquet, hour: Valley, valley: Pass the Red (meadow) }
measures: { roll: meadow, rare: Blue }
ongoing: true # never completes
ongoing: true
burn: { reason: MEADOW_NO_BLUE_AUTO_BURN, when: no Blue in the meadow }
+171
View File
@@ -0,0 +1,171 @@
# What the game's developer has said about how Dragonkind works, and what the Atlas does with each statement.
#
# Source: the developer's answers in a community Discord channel, forwarded there between 2026-10-19 and
# 2026-11-13 and exported on 2026-11-16. The export is private raw data (never committed: data/raw/). The
# statements below are paraphrased, with the date of the original message; no member is named.
#
# strength: said the developer stated it plainly
# hint said indirectly, or with a caveat ("I think", "maybe", a number offered half-seriously)
# declined the developer refused to answer (the community's "taps the sign"); recorded so that nobody
# reads a refusal as a yes or a no
# use: what in the Atlas depends on it. `open` lists what is still unknown and the experiment that tests it
# (planner/experiments.yaml). analysis/reports/BLACK.md is what the runs themselves show.
version: 1
engine:
- id: deterministic-engine
date: 2026-10-29
strength: said
statement: >
The game is a seeded, guarded, acyclic, deterministic transition system. The scene a reader stands in is a
function of the story and the path so far; the choices offered are a function of the story, the path, the
run's seed, and facts the site knows about the account. A run is deterministic once its seed is fixed.
use: The route model's rolls are fixed constants per scene, not quantities that drift; the game is a tree.
- id: seed
date: 2026-10-21
strength: said
statement: >
Each run gets a new id from the database; a keyed hash of it is the run's seed, the source of all the run's
randomness. The dice are in effect thrown before the first choice. The seed cannot be seen or influenced
from the browser, and neither time of day nor timing matters.
use: No experiment varies time of day or timing. Nothing a rider does before a run changes its dice.
- id: draws
date: 2026-11-04
strength: said
statement: A scene that uses the seed carries a chance out of 10,000 of an event.
use: Each roll is one fixed number per scene. Pooling a roll across routes that reach the same scene is right.
- id: one-in-twenty
date: 2026-10-19
strength: said
statement: No single event has worse than 1-in-20 odds; rare outcomes come from several events stacking.
use: >
A roll estimated below 5% is suspect: either several events are being counted as one, or it is not a roll.
The shadow (0.07% on later attempts) is the case in point: it is not a roll (see black/not-random).
- id: facts
date: 2026-10-21
strength: said
statement: >
The "facts" are what a web server knows about the request and the account. Neither part of the email address
is one. The site keeps little about readers, collects no analytics and does not fingerprint.
use: The bot records what it can see of the account on every run (planner/experiments.yaml, `record`).
- id: runs-remembered
date: 2026-10-29
strength: said
statement: A run is a small database entry and there is no reason ever to forget one.
use: An account's earlier runs can be among the facts. The bot keeps every account's full history.
- id: choice-order
date: 2026-10-25
strength: said
statement: The order of the choices on a screen is shuffled by the seed and belongs to the interface, not the engine.
use: Screen position is never analysed; options are matched by their text.
- id: size
date: 2026-10-25
strength: said
statement: There are about 24 million distinct paths; random play is bad at bonding.
use: Mapping cannot enumerate paths. It records each screen's options and where each leads.
- id: no-squad-bias
date: 2026-10-29
strength: said
statement: There is no squad bias toward the ability to bond anything.
use: Wing, section and squad are recorded but not expected to matter (BLACK.md section 3 agrees).
blue:
- id: blue-is-dice
date: 2026-11-04
strength: said
statement: >
Whether a Blue appears is a dice roll from the seed, like a loot drop. Someone who says they could
brute-force a Blue is right. Seeing one, the stacked odds of the bonding steps still kill most riders.
use: The meadow Blue stays a roll. Repeated attempts are the method; the gain is in the bonding answers.
black:
- id: not-random
date: 2026-11-04
strength: said
statement: >
The seed plays no part in whether the Black shows itself (repeated 2026-10-21, 2026-11-04, 2026-11-13). Its
conditions rest entirely on the rider's own choices. A rider who meets all of them gets the Black, and one
who knows the system can bond a Black with certainty, no luck involved.
use: >
The shadow is a guard, not a roll (core/spec/game.yaml `guard`). The model's "chance of the shadow" is the
share of runs like this one that met conditions we do not know yet, not the game's dice. Experiments on it
look for the rule (hold everything fixed, change one thing), not for a rate.
- id: gates
date: 2026-10-28
strength: said
statement: >
A rider must navigate the bullies encounter, and the tree is also a gate. There is no single route, and
there never will be a map that takes a rider down one specific route. The path does not change with time.
use: game.yaml `requires` on the shadow. BLACK.md section 4 (shadows on 44 different recorded routes) agrees.
- id: few-factors
date: 2026-10-28
strength: hint
statement: There are not forty factors; five "sounds like" a tolerable number.
use: The search is for a handful of conditions. Not relied on as an exact count.
- id: twenty
date: 2026-11-03
strength: hint
statement: Twenty is a discrete count the community would have a handle on once it understood how the Black is seen.
use: Unexplained. No analysis assumes what is counted.
- id: runs-never-hurt
date: 2026-11-07
strength: said
statement: >
Repeating the same path does not hurt an account's chances, so any given run either does nothing for them or
improves them. With a solution in hand there is no bad reputation to undo; the only risk is bonding
something unintended. Dooming an account at creation would be an accessibility problem.
use: >
Research attempts cannot spoil an account for the Black. Hard to square with section 2 of BLACK.md (the same
route works on a first attempt and almost never later), which is why the fresh-against-veteran experiments
come first.
- id: fresh-and-veteran
date: 2026-11-11
strength: hint
statement: >
Asked whether a rider with perfect knowledge would find the Black more easily on a first attempt or a later
one, the developer came close to answering and said the question was near a solution. Later: the community
is exploring what the developer would explore; the hard part is that an older account gives no feedback
until "it" happens; the task is to test the apparent success of a fresh account against the environment of
a veteran one.
use: Experiments fresh-replicate, shadow-again, second-attempt-same-path, veteran-repeat, veteran-variety.
- id: who-you-walk-with
date: 2026-11-07
strength: hint
statement: >
The valley might look different depending on who a rider walks in with, who they leave behind and who they
choose to fight; the dragons assess riders as riders assess dragons.
use: Company, leaving the squad, and the fight are the first decisions the ablation changes.
- id: death-does-not-matter
date: 2026-11-10
strength: said
statement: How a rider dies, the death message, and the scene a rider dies at are not factors in later runs.
use: Deliberate burns on research accounts are not a confound.
- id: order-does-not-matter
date: 2026-11-04
strength: hint
statement: As the developer understood the question, order does not matter.
use: None yet (the question it answered is not in the export).
declined: # refused, so neither confirmed nor denied
- Whether the conditions are only the answers to the questions (2026-10-25).
- Whether the shadow at the tree is the only way to meet a Black (2026-10-26).
- Whether choices repeated across runs raise or lower the chance (2026-10-21); later answered in part (runs-never-hurt).
- Whether older accounts can recover (2026-10-21).
- Whether there are hidden scores for personality, reputation or the like (2026-11-07, 2026-11-09, 2026-11-10).
- Whether the link riders see between cadet assignment and the Black is coincidence (2026-11-04).
open:
- question: On a fresh account, does one exact path always show the shadow, or only sometimes?
why: Always would mean the rule is in this run's choices. Sometimes would mean accounts differ in a fact we do not record.
experiment: fresh-replicate
- question: Which choices on that path are needed?
experiment: fresh-ablate
- question: Does an account that saw the shadow see it again on the same path?
experiment: shadow-again
- question: Does an account that missed on its first attempt miss again on the same path?
experiment: second-attempt-same-path
- question: Does an older account open up by repeating one path, or by varying its choices?
experiment: veteran-repeat, veteran-variety
- question: Holding the Black's gaze bonds about seven times in ten. What decides the rest?
why: The developer says knowledge gives certainty, so it should not be dice.
experiment: every screen after the shadow is logged; the bot follows the reported sequence (experiments.yaml `scripts`).
+48 -18
View File
@@ -168,29 +168,59 @@ test("mapping: uniform sampling picks among the options on the screen, with its
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
});
test("the experiment plan: balanced arms across new accounts, shares after the first attempt", () => {
test("the experiment plan: what each attempt of each account is for", () => {
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
const first = plan.experiments.find((e) => e.id === "first-attempt-shadow");
const count = {};
for (let acc = 0; acc < 4 * 25; acc++) {
const all = published.samples.all;
// attempt 1: every new account plays the same exact path (the replication)
for (let acc = 0; acc < 30; acc++) {
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
assert.strictEqual(a.experimentId, "first-attempt-shadow");
assert.strictEqual(a.experimentId, "fresh-replicate");
assert.strictEqual(a.target, "Black");
assert.strictEqual(a.propensity, 1 / first.arms.length);
assert.strictEqual(a.fix.direction, "West");
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
count[a.arm] = (count[a.arm] || 0) + 1;
S.advisor(published.samples.all, a.target, a.objective, { attempt: 1, fix: a.fix }); // every arm is playable
}
assert.deepStrictEqual(Object.values(count), [25, 25, 25, 25]);
// attempt 2: half the accounts check the shadow again; the others go to the shared experiments
const second = Array.from({ length: 40 }, (_, acc) => S.assign(plan, { accountSeq: acc, attempt: 2 }).experimentId);
assert.strictEqual(second.filter((id) => id === "attempt-2-shadow").length, 20);
const later = {};
for (let acc = 0; acc < 60; acc++) for (let t = 3; t <= 18; t++) { const a = S.assign(plan, { accountSeq: acc, attempt: t }); later[a.experimentId] = (later[a.experimentId] || 0) + 1; }
assert.deepStrictEqual(Object.keys(later).sort(), ["mapping", "meadow-blue-route", "sentinel"]);
assert.ok(Math.abs(later["meadow-blue-route"] / (60 * 16) - 0.6) < 0.06);
// an experiment that is no longer running is not assigned; its share goes to the others
const a1 = S.assign(plan, { accountSeq: 0, attempt: 1 });
const adv = S.advisor(all, a1.target, a1.objective, { attempt: 1, fix: a1.fix });
let h = {};
for (let i = 0; i < 20; i++) {
const s = adv.step(h);
if (s.kind === "decision") { if (s.var === "encounter") break; if (a1.fix[s.var]) assert.strictEqual(s.choice, a1.fix[s.var]); h[s.var] = s.choice; }
else if (s.kind === "chance") h[s.var] = s.var === "shadow" ? "no" : Object.keys(s.probs).find((k) => k !== "die");
else break;
}
assert.strictEqual(h.hour, "Tree");
// the ablation waits for the replication, then gives every arm the same number of accounts
const stage2 = { ...plan, experiments: plan.experiments.map((e) => (e.id === "fresh-replicate" ? { ...e, status: "complete" } : e.id === "fresh-ablate" ? { ...e, status: "running" } : e)) };
const count = {};
for (let acc = 0; acc < 12 * 5; acc++) { const a = S.assign(stage2, { accountSeq: acc, attempt: 1 }); assert.strictEqual(a.experimentId, "fresh-ablate"); count[a.arm] = (count[a.arm] || 0) + 1; }
assert.deepStrictEqual(Object.values(count), Array(12).fill(5));
// with squadmates, staying and the direction are one screen
const squad = stage2.experiments.find((e) => e.id === "fresh-ablate").arms.find((a) => a.id === "with two squadmates");
const rec = S.recommendOption(S.advisor(all, "Black", "eventually", { attempt: 1, fix: squad.fix }), { company: "2 Squadmates" },
["Go east with your squad", "Leave your squadmates", "Convince them to head west instead"]);
assert.strictEqual(rec.index, 2);
assert.deepStrictEqual(rec.also, { direction: "West" });
// attempt 2: an account that saw the shadow tries the same path again; half of the others do too
const who = (attempt, facts) => Array.from({ length: 40 }, (_, acc) => (S.assign(plan, { accountSeq: acc, attempt, facts }) || {}).experimentId);
assert.ok(who(2, { sawShadowBefore: true }).every((id) => id === "shadow-again"));
assert.strictEqual(who(2, { sawShadowBefore: false }).filter((id) => id === "second-attempt-same-path").length, 20);
// later: two accounts in ten are the long-lived ones, the rest share the research experiments
const later = who(6, { sawShadowBefore: false });
assert.strictEqual(later.filter((id) => id === "veteran-repeat").length, 4);
assert.strictEqual(later.filter((id) => id === "veteran-variety").length, 4);
assert.deepStrictEqual([...new Set(later)].sort(), ["mapping", "meadow-blue-route", "sentinel", "veteran-repeat", "veteran-variety"]);
// variety: a different combination on every attempt, both gates kept, every one playable
const combos = new Set();
for (let t = 3; t < 3 + 60; t++) {
const a = S.assign(plan, { accountSeq: 11, attempt: t, facts: { sawShadowBefore: false } });
assert.strictEqual(a.experimentId, "veteran-variety");
assert.strictEqual(a.fix.clearing, "Help your squadmate");
combos.add(a.arm);
if (t < 8) S.advisor(all, a.target, a.objective, { attempt: t, fix: a.fix });
}
assert.strictEqual(combos.size, 60);
// an experiment that is not running is not assigned; its share goes to the others
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
assert.notStrictEqual(S.assign(done, { accountSeq: 1, attempt: 5 }).experimentId, "meadow-blue-route");
assert.notStrictEqual(S.assign(done, { accountSeq: 5, attempt: 5 }).experimentId, "meadow-blue-route");
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
});
+49 -25
View File
@@ -14,43 +14,67 @@ GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "c
class Plan(unittest.TestCase):
def test_design_is_valid_and_arms_are_every_combination(self):
def test_design_is_valid(self):
plan = B.build(D, log=[])
by = {e["id"]: e for e in plan["experiments"]}
self.assertEqual(len(by["first-attempt-shadow"]["arms"]), 4)
# the Black attempt is one exact path, and it keeps the shadow's gate open
(arm,) = by["fresh-replicate"]["arms"]
self.assertEqual(arm["fix"], D["routes"]["best-reported"])
self.assertTrue(model.gate_open("shadow", arm["fix"]))
self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6)
for e in plan["experiments"]:
self.assertEqual(e["status"], "running")
for a in e["arms"]:
B.check_options(e["id"], a["fix"])
# the Black attempt keeps the shadow's gate open on every arm
for a in by["first-attempt-shadow"]["arms"]:
B.check_options(e["id"], {k: v[0] for k, v in (a.get("cycle") or {}).items()})
self.assertEqual(plan["record"], D["record"])
self.assertEqual(len(plan["scripts"]["shadow"]), 12)
def test_ablation_changes_one_thing_at_a_time_and_waits_for_the_replication(self):
plan = B.build(D, log=[])
by = {e["id"]: e for e in plan["experiments"]}
base = D["routes"]["best-reported"]
self.assertEqual(by["fresh-ablate"]["status"], "waiting")
for a in by["fresh-ablate"]["arms"]:
changed = {k for k in a["fix"] if a["fix"][k] != base.get(k)}
self.assertEqual(changed, set(a["change"]), a["id"])
self.assertLessEqual(len(changed - {"leave"}), 1, a["id"]) # leaving goes with the company
self.assertTrue(model.gate_open("shadow", a["fix"]))
# once the replication has its planned count, the ablation starts
e0 = next(e for e in D["experiments"] if e["id"] == "fresh-replicate")
log = [(e0["id"], "all", {**GATED_TREE, "shadow": i < 9}) for i in range(e0["planned_per_arm"])]
by = {e["id"]: e for e in B.build(D, log=log)["experiments"]}
self.assertEqual(by["fresh-replicate"]["status"], "complete")
self.assertEqual(by["fresh-replicate"]["progress"]["rare"], 9)
self.assertEqual(by["fresh-ablate"]["status"], "running")
def test_an_unknown_option_is_refused(self):
d = copy.deepcopy(D)
d["experiments"][0]["factors"]["direction"] = ["North"]
with self.assertRaises(ValueError):
B.build(d, log=[])
def test_an_unknown_option_route_or_fact_is_refused(self):
for edit in (lambda d: d["routes"]["best-reported"].update(direction="North"),
lambda d: d["experiments"][0].update(route="nowhere"),
lambda d: d["experiments"][1]["arms"][0]["change"].update(company="Three"),
lambda d: d["experiments"][2].update(when={"luck": True}),
lambda d: d["experiments"][1].update(starts_after="nothing")):
d = copy.deepcopy(D)
edit(d)
with self.assertRaises(ValueError):
B.build(d, log=[])
def test_progress_completes_and_stops_early(self):
def test_progress(self):
e0 = D["experiments"][0]
arms = [a["id"] for a in B.arms(e0)]
seen = {**GATED_TREE, "shadow": True}
none = {**GATED_TREE, "shadow": False}
# every arm at its planned count, with shadows: complete
log = [(e0["id"], a, seen if i < 3 else none) for a in arms for i in range(e0["planned_per_arm"])]
e = B.build(D, log=log)["experiments"][0]
self.assertEqual(e["status"], "complete")
self.assertEqual(e["progress"]["rare"], 12)
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 61)
# no shadow at all after the stop_early count: paused, for a person to look at
log = [(e0["id"], arms[i % 4], none) for i in range(e0["stop_early"]["after"])]
e = B.build(D, log=log)["experiments"][0]
self.assertEqual(e["status"], "paused")
# a run whose roll was not observed (died before the tree) is a run, not an observation
e = B.build(D, log=[(e0["id"], arms[0], {"attempt": "First attempt"})])["experiments"][0]
e = B.build(D, log=[(e0["id"], "all", {**GATED_TREE, "shadow": True})] * 3 + [(e0["id"], "all", none)] * 7)["experiments"][0]
self.assertEqual((e["status"], e["progress"]["observed"], e["progress"]["rare"]), ("running", 10, 3))
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 11)
# a run whose shadow was not observed (died before the tree) is a run, not an observation
e = B.build(D, log=[(e0["id"], "all", {"attempt": "First attempt"})])["experiments"][0]
self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0))
# a cycled experiment logs the combination it played; they all count toward its one arm
v = B.build(D, log=[("veteran-variety", "company=Alone / direction=East", none)] * 4)
v = next(e for e in v["experiments"] if e["id"] == "veteran-variety")
self.assertEqual((v["arms"][0]["progress"]["observed"], v["progress"]["unknownArms"]), (4, []))
# stop_early pauses an experiment for a person to look at
d = copy.deepcopy(D)
d["experiments"][0]["stop_early"] = {"after": 5, "rare_at_most": 0}
self.assertEqual(B.build(d, log=[(e0["id"], "all", none)] * 5)["experiments"][0]["status"], "paused")
def test_the_plan_id_follows_its_content(self):
a, b = B.build(D, log=[]), B.build(D, log=[])