- planner/knowledge.yaml: what the game's developer has said (paraphrased, dated, graded), what the Atlas does with each statement, and the open questions with the experiment that tests each. - analysis/black/evidence.py -> analysis/reports/BLACK.md: counts from the runs. On the same recorded route the shadow shows on 21 of 79 first attempts and 1 of 80 later ones; no single route; Blue then Black after the shadow. - game.yaml: the shadow is a guard (not a roll); what follows it is the Blue, then the Black; on-screen text for the squad's direction screen. - planner/experiments.yaml v2: one exact path on every fresh account (fresh-replicate), then one choice changed at a time (fresh-ablate); shadow-again, second-attempt-same-path, veteran-repeat, veteran-variety; what the bot records about the account; the sequence to follow once the shadow shows. - solver.js: assign() takes account facts, account offsets and cycled arms; recommendOption() answers the squad screen (stay + direction) in one pick. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
254 lines
14 KiB
Python
254 lines
14 KiB
Python
"""What the runs say about the Black dragon: analysis/reports/BLACK.md.
|
|
|
|
python analysis/black/evidence.py [--check]
|
|
|
|
Every number in the report is computed here from data/public/runs_clean.csv (and the bot's anonymised
|
|
trajectories), so it can be re-run after every import. Nothing is fitted: these are counts. The developer's
|
|
statements they are read against are in planner/knowledge.yaml.
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import glob
|
|
import json
|
|
import math
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "core"))
|
|
from atlas import runs as runs_io # noqa: E402
|
|
|
|
OUT = ROOT / "analysis/reports/BLACK.md"
|
|
ROUTE = ["company", "direction", "combat_first", "combat_second", "resolution", "post_path"]
|
|
ROUTE_NAMES = "company, first direction, first move in the fight, the dagger, the bullies' fate, the fork"
|
|
|
|
|
|
def gated_tree(r):
|
|
"""A run that passed both known gates (investigated, helped, climbed the tree) and says whether the shadow showed."""
|
|
return r["scream"] == "Investigate & help" and r["hour"] == "Tree" and r["saw_shadow"] is not None
|
|
|
|
|
|
def group(r):
|
|
if r["source"] == "bot":
|
|
return "bot"
|
|
if r["source"] == "reddit spreadsheet":
|
|
return "form" if r["entry"] == "Form" else "bulk logs"
|
|
return "other"
|
|
|
|
|
|
def tally(rows, key):
|
|
d = collections.OrderedDict()
|
|
for r in rows:
|
|
c = d.setdefault(key(r), [0, 0])
|
|
c[0] += bool(r["saw_shadow"])
|
|
c[1] += 1
|
|
return d
|
|
|
|
|
|
def frac(y, n):
|
|
return f"{y} / {n}" + (f" ({100 * y / n:.1f}%)" if n else "")
|
|
|
|
|
|
def upper(n, conf=0.95):
|
|
"""The rate that zero in n runs rules out (one-sided)."""
|
|
return 1 - (1 - conf) ** (1 / n) if n else 1.0
|
|
|
|
|
|
def mantel_haenszel(strata):
|
|
"""strata: [(y1, n1, y0, n0)]. Odds ratio and z of group 1 against group 0, within strata."""
|
|
num = den = obs = exp = var = 0.0
|
|
for y1, n1, y0, n0 in strata:
|
|
N = n1 + n0
|
|
if not n1 or not n0:
|
|
continue
|
|
num += y1 * (n0 - y0) / N
|
|
den += y0 * (n1 - y1) / N
|
|
m = y1 + y0
|
|
obs += y1
|
|
exp += n1 * m / N
|
|
if N > 1:
|
|
var += n1 * n0 * m * (N - m) / (N * N * (N - 1))
|
|
return (num / den if den else float("inf")), obs, exp, ((obs - exp) / math.sqrt(var) if var else 0.0)
|
|
|
|
|
|
def last_answer(note):
|
|
"""The answer given to the Black, from a rider's free-text note (keywords only; the note is never published).
|
|
Riders lower their eyes to the Blue earlier in the same sequence, so only the last thing a note says counts, and
|
|
only once the note has reached the Black (or is too short to be about anything else)."""
|
|
t = re.sub(r"instead of low\w+[^.>,;]*", "", (note or "").lower())
|
|
marks = [m.end() for m in re.finditer(r"threat|branch|south|black", t)]
|
|
scope = t if len(t) < 45 else (t[marks[-1]:] if marks else "")
|
|
found = [(m.start(), k) for k, pat in (("held its gaze", r"gaze|glare|eye contact"), ("lowered eyes", r"lower"),
|
|
("ignored the sound", r"ignore"))
|
|
for m in re.finditer(pat, scope)]
|
|
if not found: # a long note that never names the Black's moment: use the whole note, gaze only
|
|
return "held its gaze" if re.search(r"(hold|held|meet|met)\w* (its|it's|the black'?s?|his|her) (gaze|glare)", t) else None
|
|
return max(found)[1]
|
|
|
|
|
|
def blue_mistake(note):
|
|
t = (note or "").lower()
|
|
if "approach" in t:
|
|
return "approached the Blue"
|
|
if "warn" in t or "interven" in t:
|
|
return "warned the other cadet / intervened"
|
|
return None
|
|
|
|
|
|
def build():
|
|
runs = runs_io.read()
|
|
G = [r for r in runs if gated_tree(r)]
|
|
L = ["# The Black dragon: what the runs show", "",
|
|
"Written by `analysis/black/evidence.py` from `data/public/runs_clean.csv`. Counts only; nothing is fitted. "
|
|
"\"Gated tree runs\" investigated the scream, helped the squadmate and climbed the tree, the two gates the "
|
|
"developer has confirmed (`planner/knowledge.yaml`). Form submissions over-report the shadow, so their "
|
|
"percentages are too high; comparisons inside the form (first attempt against later, one route against "
|
|
"another) are still fair if the over-reporting is the same on both sides.", ""]
|
|
|
|
# 1. by attempt
|
|
L += ["## 1. The shadow by attempt number", "", "| Attempt | Form submissions | Bulk logs | Bot |", "|---|---|---|---|"]
|
|
bands = [("1", 1, 1), ("2", 2, 2), ("3", 3, 3), ("4-10", 4, 10), ("11-30", 11, 30), ("31 or more", 31, 10 ** 9)]
|
|
for name, lo, hi in bands:
|
|
cells = []
|
|
for g in ("form", "bulk logs", "bot"):
|
|
rs = [r for r in G if group(r) == g and r["attempt"] and lo <= r["attempt"] <= hi]
|
|
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
|
|
L.append(f"| {name} | " + " | ".join(cells) + " |")
|
|
cells = []
|
|
for g in ("form", "bulk logs", "bot"):
|
|
rs = [r for r in G if group(r) == g and not r["attempt"]]
|
|
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
|
|
L.append("| not given | " + " | ".join(cells) + " |")
|
|
complete_first = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] == 1]
|
|
complete_later = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] and r["attempt"] > 1]
|
|
L += ["", f"Complete logs (bot and bulk) have {len(complete_first)} first attempts at the tree and "
|
|
f"{len(complete_later)} later ones, with {sum(bool(r['saw_shadow']) for r in complete_first + complete_later)} shadows. "
|
|
f"Zero in {len(complete_later)} later attempts rules out a rate above {100 * upper(len(complete_later)):.1f}% for them; "
|
|
f"zero in {len(complete_first)} first attempts only rules out more than {100 * upper(len(complete_first)):.0f}%.", ""]
|
|
|
|
# 2. the same route, first against later
|
|
F = [r for r in G if group(r) == "form" and r["attempt"]]
|
|
by = collections.defaultdict(lambda: [0, 0, 0, 0])
|
|
for r in F:
|
|
c = by[tuple(r[k] for k in ROUTE)]
|
|
i = 0 if r["attempt"] == 1 else 2
|
|
c[i] += bool(r["saw_shadow"])
|
|
c[i + 1] += 1
|
|
orr, obs, exp, z = mantel_haenszel(by.values())
|
|
L += ["## 2. The same route on a first attempt and on a later one", "",
|
|
f"If the shadow depended only on the choices the form records ({ROUTE_NAMES}), a route would do as well on a "
|
|
"later attempt as on a first. It does not.", "",
|
|
"| Route | First attempt | Later attempts |", "|---|---|---|"]
|
|
top = sorted(by.items(), key=lambda kv: -(kv[1][1] + kv[1][3]))[:8]
|
|
for key, (y1, n1, y0, n0) in top:
|
|
L.append(f"| {' / '.join(key)} | {frac(y1, n1)} | {frac(y0, n0)} |")
|
|
L += ["", f"Across all {len(by)} recorded routes, comparing first and later attempts on the same route: "
|
|
f"{obs:.0f} first-attempt shadows where {exp:.1f} would be expected if the attempt did not matter "
|
|
f"(odds ratio {orr:.1f}, z = {z:.1f}). First and later attempts choose almost the same routes, so the "
|
|
"difference is not in the recorded choices: it is something about the account, or a choice the form does "
|
|
"not record.", ""]
|
|
|
|
# 3. is it decided by route + cadet assignment?
|
|
best_key, best = max(((k, v) for k, v in by.items() if v[1] >= 20), key=lambda kv: kv[1][0] / kv[1][1])
|
|
B = [r for r in F if r["attempt"] == 1 and tuple(r[k] for k in ROUTE) == best_key]
|
|
cells = tally(B, lambda r: (r["wing"], r["section"], r["squad"]))
|
|
multi = {k: v for k, v in cells.items() if v[1] >= 2}
|
|
mixed = sum(1 for y, n in multi.values() if 0 < y < n)
|
|
L += ["## 3. On one route, is the shadow decided by the cadet's wing, section and squad?", "",
|
|
f"The developer says the Black is decided by choices and by what the site knows about the account, with no "
|
|
f"dice. If the recorded route and the cadet assignment were all of it, every first attempt with the same "
|
|
f"route and assignment would agree. On the best-recorded route ({' / '.join(best_key)}; {frac(best[0], best[1])} "
|
|
f"on first attempts), {len(multi)} assignments have two or more runs and {mixed} of them disagree (some runs "
|
|
"with the shadow, some without). So something else decides it: a choice the form does not record, or a fact "
|
|
"about the account other than its assignment. No wing, section or squad stands out:", "",
|
|
"| | " + " | ".join(["shadow / runs"]) + " |", "|---|---|"]
|
|
for col in ("wing", "section", "squad", "first_sighting", "sighting_action"):
|
|
t = tally(B, lambda r, col=col: r[col] or "(blank)")
|
|
L.append(f"| {col} | " + ", ".join(f"{k}: {y}/{n}" for k, (y, n) in sorted(t.items(), key=lambda kv: -kv[1][1]) if n >= 3) + " |")
|
|
L.append("")
|
|
|
|
# 4. no single route
|
|
S1 = [r for r in G if group(r) == "form" and r["saw_shadow"]]
|
|
L += ["## 4. There is no single route", "",
|
|
f"The {len(S1)} form runs with a shadow (all attempts) took {len({tuple(r[k] for k in ROUTE) for r in S1})} "
|
|
"different recorded routes:", ""]
|
|
for col, label in (("company", "Company"), ("direction", "First direction"), ("combat_first", "First move in the fight"),
|
|
("combat_second", "The dagger"), ("resolution", "The bullies' fate"), ("post_path", "The fork")):
|
|
t = collections.Counter(r[col] for r in S1)
|
|
L.append(f"- **{label}:** " + ", ".join(f"{k} {v}" for k, v in t.most_common()))
|
|
outside = [r for r in runs if r["saw_shadow"] and not (r["scream"] == "Investigate & help" and r["hour"] == "Tree")]
|
|
L += ["", f"Shadows reported without both gates: {len(outside)} (likely mis-entered).", ""]
|
|
|
|
# 5. after the shadow
|
|
SH = [r for r in runs if r["saw_shadow"]]
|
|
black = [r for r in SH if r["enc_Black"]]
|
|
ans = collections.defaultdict(lambda: [0, 0])
|
|
for r in black:
|
|
a = last_answer((r["bond_note"] or "") + " " + (r["death_note"] or ""))
|
|
if a:
|
|
ans[a][0] += r["outcome"] == "Bonded"
|
|
ans[a][1] += 1
|
|
blue_dead = [r for r in SH if r["enc_Blue"] == "Incinerated"]
|
|
mis = collections.Counter(blue_mistake((r["bond_note"] or "") + " " + (r["death_note"] or "")) for r in blue_dead)
|
|
L += ["## 5. After the shadow: the Blue, then the Black", "",
|
|
"The riders' notes describe one sequence: follow the shadow, a Blue lands and burns another cadet, the Blue "
|
|
"leaves, then the Black appears. The Blue is a trial on the way, not an alternative to the Black.", "",
|
|
f"- Runs that reached the Black: {len(black)}, bonded {sum(r['outcome'] == 'Bonded' for r in black)}.",
|
|
f"- Runs that died at the Blue: {len(blue_dead)}. What their notes say they did: "
|
|
+ ", ".join(f"{k} {v}" for k, v in mis.most_common() if k) + ".",
|
|
"- The answer given to the Black, where the note says (bonded / runs): "
|
|
+ "; ".join(f"{k} {y}/{n}" for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])) + ".", "",
|
|
"Holding its gaze is the only answer recorded with a bond, and it still fails about a quarter of the time. "
|
|
"The developer says a rider who knows the system bonds with certainty, so those failures are not explained "
|
|
"yet: the notes of the riders who died are too short to show what else they did differently.", ""]
|
|
|
|
# 6. what the bot has tried
|
|
L += ["## 6. What the bot has tried", ""]
|
|
files = sorted(f for f in glob.glob(str(ROOT / "data/bot/trajectories-*.jsonl")) if "sample" not in f)
|
|
seen, acc, order = set(), collections.defaultdict(list), collections.defaultdict(list)
|
|
for f in files:
|
|
for line in open(f, encoding="utf-8"):
|
|
t = json.loads(line)
|
|
if t["id"] not in seen:
|
|
seen.add(t["id"])
|
|
acc[t["meta"].get("accountKey")].append(t["meta"].get("attempt") or 0)
|
|
order[t["meta"].get("accountKey")].append((t["meta"].get("attempt") or 0, t["id"]))
|
|
by_id = {r["run_id"]: r for r in runs_io.read(sources=["bot"], drop=())}
|
|
streak = 0 # the longest run of consecutive attempts on one account with the same gated tree route
|
|
for v in order.values():
|
|
cur, prev = 0, None
|
|
for _, i in sorted(v):
|
|
r = by_id.get(i)
|
|
s = tuple(r[k] for k in ROUTE + ["sighting_action"]) if r and gated_tree(r) else None
|
|
cur = cur + 1 if s is not None and s == prev else (1 if s is not None else 0)
|
|
prev, streak = s, max(streak, cur)
|
|
botg = [r for r in G if group(r) == "bot"]
|
|
routes = collections.Counter(tuple(r[k] for k in ROUTE) for r in botg)
|
|
L += [f"{len(seen)} attempts on {len(acc)} accounts; the earliest attempt logged on any account is number "
|
|
f"{min((min(v) for v in acc.values() if v), default=0)}. Gated tree runs: {len(botg)}, shadows "
|
|
f"{sum(bool(r['saw_shadow']) for r in botg)}, on {len(routes)} recorded routes "
|
|
f"(the commonest {routes.most_common(1)[0][1] if routes else 0} times). The bot has not played a first attempt, "
|
|
"so it has not yet tested the one situation where riders report the shadow. The most times any account has "
|
|
f"repeated one route in a row is {streak}.", ""]
|
|
return "\n".join(L) + "\n"
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--check", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
text = build()
|
|
if a.check:
|
|
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
|
print(f"{OUT} is out of date (run python analysis/black/evidence.py)", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
OUT.write_text(text, encoding="utf-8")
|
|
print(f"wrote {OUT.relative_to(ROOT)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|