Files
threshing-atlas/analysis/black/evidence.py
T
ekcSchianoandClaude Fable 5.2 d01ec46fcc The Black is not dice: developer statements, evidence report, and a plan that looks for the rule
- planner/knowledge.yaml: what the game's developer has said (paraphrased, dated, graded), what the Atlas does
  with each statement, and the open questions with the experiment that tests each.
- analysis/black/evidence.py -> analysis/reports/BLACK.md: counts from the runs. On the same recorded route the
  shadow shows on 21 of 79 first attempts and 1 of 80 later ones; no single route; Blue then Black after the shadow.
- game.yaml: the shadow is a guard (not a roll); what follows it is the Blue, then the Black; on-screen text for
  the squad's direction screen.
- planner/experiments.yaml v2: one exact path on every fresh account (fresh-replicate), then one choice changed
  at a time (fresh-ablate); shadow-again, second-attempt-same-path, veteran-repeat, veteran-variety; what the bot
  records about the account; the sequence to follow once the shadow shows.
- solver.js: assign() takes account facts, account offsets and cycled arms; recommendOption() answers the
  squad screen (stay + direction) in one pick.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
2026-11-16 15:24:03 +00:00

254 lines
14 KiB
Python

"""What the runs say about the Black dragon: analysis/reports/BLACK.md.
python analysis/black/evidence.py [--check]
Every number in the report is computed here from data/public/runs_clean.csv (and the bot's anonymised
trajectories), so it can be re-run after every import. Nothing is fitted: these are counts. The developer's
statements they are read against are in planner/knowledge.yaml.
"""
import argparse
import collections
import glob
import json
import math
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "core"))
from atlas import runs as runs_io # noqa: E402
OUT = ROOT / "analysis/reports/BLACK.md"
ROUTE = ["company", "direction", "combat_first", "combat_second", "resolution", "post_path"]
ROUTE_NAMES = "company, first direction, first move in the fight, the dagger, the bullies' fate, the fork"
def gated_tree(r):
"""A run that passed both known gates (investigated, helped, climbed the tree) and says whether the shadow showed."""
return r["scream"] == "Investigate & help" and r["hour"] == "Tree" and r["saw_shadow"] is not None
def group(r):
if r["source"] == "bot":
return "bot"
if r["source"] == "reddit spreadsheet":
return "form" if r["entry"] == "Form" else "bulk logs"
return "other"
def tally(rows, key):
d = collections.OrderedDict()
for r in rows:
c = d.setdefault(key(r), [0, 0])
c[0] += bool(r["saw_shadow"])
c[1] += 1
return d
def frac(y, n):
return f"{y} / {n}" + (f" ({100 * y / n:.1f}%)" if n else "")
def upper(n, conf=0.95):
"""The rate that zero in n runs rules out (one-sided)."""
return 1 - (1 - conf) ** (1 / n) if n else 1.0
def mantel_haenszel(strata):
"""strata: [(y1, n1, y0, n0)]. Odds ratio and z of group 1 against group 0, within strata."""
num = den = obs = exp = var = 0.0
for y1, n1, y0, n0 in strata:
N = n1 + n0
if not n1 or not n0:
continue
num += y1 * (n0 - y0) / N
den += y0 * (n1 - y1) / N
m = y1 + y0
obs += y1
exp += n1 * m / N
if N > 1:
var += n1 * n0 * m * (N - m) / (N * N * (N - 1))
return (num / den if den else float("inf")), obs, exp, ((obs - exp) / math.sqrt(var) if var else 0.0)
def last_answer(note):
"""The answer given to the Black, from a rider's free-text note (keywords only; the note is never published).
Riders lower their eyes to the Blue earlier in the same sequence, so only the last thing a note says counts, and
only once the note has reached the Black (or is too short to be about anything else)."""
t = re.sub(r"instead of low\w+[^.>,;]*", "", (note or "").lower())
marks = [m.end() for m in re.finditer(r"threat|branch|south|black", t)]
scope = t if len(t) < 45 else (t[marks[-1]:] if marks else "")
found = [(m.start(), k) for k, pat in (("held its gaze", r"gaze|glare|eye contact"), ("lowered eyes", r"lower"),
("ignored the sound", r"ignore"))
for m in re.finditer(pat, scope)]
if not found: # a long note that never names the Black's moment: use the whole note, gaze only
return "held its gaze" if re.search(r"(hold|held|meet|met)\w* (its|it's|the black'?s?|his|her) (gaze|glare)", t) else None
return max(found)[1]
def blue_mistake(note):
t = (note or "").lower()
if "approach" in t:
return "approached the Blue"
if "warn" in t or "interven" in t:
return "warned the other cadet / intervened"
return None
def build():
runs = runs_io.read()
G = [r for r in runs if gated_tree(r)]
L = ["# The Black dragon: what the runs show", "",
"Written by `analysis/black/evidence.py` from `data/public/runs_clean.csv`. Counts only; nothing is fitted. "
"\"Gated tree runs\" investigated the scream, helped the squadmate and climbed the tree, the two gates the "
"developer has confirmed (`planner/knowledge.yaml`). Form submissions over-report the shadow, so their "
"percentages are too high; comparisons inside the form (first attempt against later, one route against "
"another) are still fair if the over-reporting is the same on both sides.", ""]
# 1. by attempt
L += ["## 1. The shadow by attempt number", "", "| Attempt | Form submissions | Bulk logs | Bot |", "|---|---|---|---|"]
bands = [("1", 1, 1), ("2", 2, 2), ("3", 3, 3), ("4-10", 4, 10), ("11-30", 11, 30), ("31 or more", 31, 10 ** 9)]
for name, lo, hi in bands:
cells = []
for g in ("form", "bulk logs", "bot"):
rs = [r for r in G if group(r) == g and r["attempt"] and lo <= r["attempt"] <= hi]
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
L.append(f"| {name} | " + " | ".join(cells) + " |")
cells = []
for g in ("form", "bulk logs", "bot"):
rs = [r for r in G if group(r) == g and not r["attempt"]]
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
L.append("| not given | " + " | ".join(cells) + " |")
complete_first = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] == 1]
complete_later = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] and r["attempt"] > 1]
L += ["", f"Complete logs (bot and bulk) have {len(complete_first)} first attempts at the tree and "
f"{len(complete_later)} later ones, with {sum(bool(r['saw_shadow']) for r in complete_first + complete_later)} shadows. "
f"Zero in {len(complete_later)} later attempts rules out a rate above {100 * upper(len(complete_later)):.1f}% for them; "
f"zero in {len(complete_first)} first attempts only rules out more than {100 * upper(len(complete_first)):.0f}%.", ""]
# 2. the same route, first against later
F = [r for r in G if group(r) == "form" and r["attempt"]]
by = collections.defaultdict(lambda: [0, 0, 0, 0])
for r in F:
c = by[tuple(r[k] for k in ROUTE)]
i = 0 if r["attempt"] == 1 else 2
c[i] += bool(r["saw_shadow"])
c[i + 1] += 1
orr, obs, exp, z = mantel_haenszel(by.values())
L += ["## 2. The same route on a first attempt and on a later one", "",
f"If the shadow depended only on the choices the form records ({ROUTE_NAMES}), a route would do as well on a "
"later attempt as on a first. It does not.", "",
"| Route | First attempt | Later attempts |", "|---|---|---|"]
top = sorted(by.items(), key=lambda kv: -(kv[1][1] + kv[1][3]))[:8]
for key, (y1, n1, y0, n0) in top:
L.append(f"| {' / '.join(key)} | {frac(y1, n1)} | {frac(y0, n0)} |")
L += ["", f"Across all {len(by)} recorded routes, comparing first and later attempts on the same route: "
f"{obs:.0f} first-attempt shadows where {exp:.1f} would be expected if the attempt did not matter "
f"(odds ratio {orr:.1f}, z = {z:.1f}). First and later attempts choose almost the same routes, so the "
"difference is not in the recorded choices: it is something about the account, or a choice the form does "
"not record.", ""]
# 3. is it decided by route + cadet assignment?
best_key, best = max(((k, v) for k, v in by.items() if v[1] >= 20), key=lambda kv: kv[1][0] / kv[1][1])
B = [r for r in F if r["attempt"] == 1 and tuple(r[k] for k in ROUTE) == best_key]
cells = tally(B, lambda r: (r["wing"], r["section"], r["squad"]))
multi = {k: v for k, v in cells.items() if v[1] >= 2}
mixed = sum(1 for y, n in multi.values() if 0 < y < n)
L += ["## 3. On one route, is the shadow decided by the cadet's wing, section and squad?", "",
f"The developer says the Black is decided by choices and by what the site knows about the account, with no "
f"dice. If the recorded route and the cadet assignment were all of it, every first attempt with the same "
f"route and assignment would agree. On the best-recorded route ({' / '.join(best_key)}; {frac(best[0], best[1])} "
f"on first attempts), {len(multi)} assignments have two or more runs and {mixed} of them disagree (some runs "
"with the shadow, some without). So something else decides it: a choice the form does not record, or a fact "
"about the account other than its assignment. No wing, section or squad stands out:", "",
"| | " + " | ".join(["shadow / runs"]) + " |", "|---|---|"]
for col in ("wing", "section", "squad", "first_sighting", "sighting_action"):
t = tally(B, lambda r, col=col: r[col] or "(blank)")
L.append(f"| {col} | " + ", ".join(f"{k}: {y}/{n}" for k, (y, n) in sorted(t.items(), key=lambda kv: -kv[1][1]) if n >= 3) + " |")
L.append("")
# 4. no single route
S1 = [r for r in G if group(r) == "form" and r["saw_shadow"]]
L += ["## 4. There is no single route", "",
f"The {len(S1)} form runs with a shadow (all attempts) took {len({tuple(r[k] for k in ROUTE) for r in S1})} "
"different recorded routes:", ""]
for col, label in (("company", "Company"), ("direction", "First direction"), ("combat_first", "First move in the fight"),
("combat_second", "The dagger"), ("resolution", "The bullies' fate"), ("post_path", "The fork")):
t = collections.Counter(r[col] for r in S1)
L.append(f"- **{label}:** " + ", ".join(f"{k} {v}" for k, v in t.most_common()))
outside = [r for r in runs if r["saw_shadow"] and not (r["scream"] == "Investigate & help" and r["hour"] == "Tree")]
L += ["", f"Shadows reported without both gates: {len(outside)} (likely mis-entered).", ""]
# 5. after the shadow
SH = [r for r in runs if r["saw_shadow"]]
black = [r for r in SH if r["enc_Black"]]
ans = collections.defaultdict(lambda: [0, 0])
for r in black:
a = last_answer((r["bond_note"] or "") + " " + (r["death_note"] or ""))
if a:
ans[a][0] += r["outcome"] == "Bonded"
ans[a][1] += 1
blue_dead = [r for r in SH if r["enc_Blue"] == "Incinerated"]
mis = collections.Counter(blue_mistake((r["bond_note"] or "") + " " + (r["death_note"] or "")) for r in blue_dead)
L += ["## 5. After the shadow: the Blue, then the Black", "",
"The riders' notes describe one sequence: follow the shadow, a Blue lands and burns another cadet, the Blue "
"leaves, then the Black appears. The Blue is a trial on the way, not an alternative to the Black.", "",
f"- Runs that reached the Black: {len(black)}, bonded {sum(r['outcome'] == 'Bonded' for r in black)}.",
f"- Runs that died at the Blue: {len(blue_dead)}. What their notes say they did: "
+ ", ".join(f"{k} {v}" for k, v in mis.most_common() if k) + ".",
"- The answer given to the Black, where the note says (bonded / runs): "
+ "; ".join(f"{k} {y}/{n}" for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])) + ".", "",
"Holding its gaze is the only answer recorded with a bond, and it still fails about a quarter of the time. "
"The developer says a rider who knows the system bonds with certainty, so those failures are not explained "
"yet: the notes of the riders who died are too short to show what else they did differently.", ""]
# 6. what the bot has tried
L += ["## 6. What the bot has tried", ""]
files = sorted(f for f in glob.glob(str(ROOT / "data/bot/trajectories-*.jsonl")) if "sample" not in f)
seen, acc, order = set(), collections.defaultdict(list), collections.defaultdict(list)
for f in files:
for line in open(f, encoding="utf-8"):
t = json.loads(line)
if t["id"] not in seen:
seen.add(t["id"])
acc[t["meta"].get("accountKey")].append(t["meta"].get("attempt") or 0)
order[t["meta"].get("accountKey")].append((t["meta"].get("attempt") or 0, t["id"]))
by_id = {r["run_id"]: r for r in runs_io.read(sources=["bot"], drop=())}
streak = 0 # the longest run of consecutive attempts on one account with the same gated tree route
for v in order.values():
cur, prev = 0, None
for _, i in sorted(v):
r = by_id.get(i)
s = tuple(r[k] for k in ROUTE + ["sighting_action"]) if r and gated_tree(r) else None
cur = cur + 1 if s is not None and s == prev else (1 if s is not None else 0)
prev, streak = s, max(streak, cur)
botg = [r for r in G if group(r) == "bot"]
routes = collections.Counter(tuple(r[k] for k in ROUTE) for r in botg)
L += [f"{len(seen)} attempts on {len(acc)} accounts; the earliest attempt logged on any account is number "
f"{min((min(v) for v in acc.values() if v), default=0)}. Gated tree runs: {len(botg)}, shadows "
f"{sum(bool(r['saw_shadow']) for r in botg)}, on {len(routes)} recorded routes "
f"(the commonest {routes.most_common(1)[0][1] if routes else 0} times). The bot has not played a first attempt, "
"so it has not yet tested the one situation where riders report the shadow. The most times any account has "
f"repeated one route in a row is {streak}.", ""]
return "\n".join(L) + "\n"
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--check", action="store_true")
a = ap.parse_args(argv)
text = build()
if a.check:
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
print(f"{OUT} is out of date (run python analysis/black/evidence.py)", file=sys.stderr)
return 1
return 0
OUT.write_text(text, encoding="utf-8")
print(f"wrote {OUT.relative_to(ROOT)}")
return 0
if __name__ == "__main__":
sys.exit(main())