CI / test (pull_request) Successful in 42s
- analysis/black/evidence.py: Reddit comments and "Report a run" are their own groups. The attempt table gets a column for each once it has a gated tree run (Reddit: 4 runs, 3 with a shadow; reported: none yet). A note says Reddit describes 17 shadows but only 3 say both gates and 3 the attempt, and that, like form posts, they show what shadow runs look like, not how often the shadow comes. black.json: by_attempt["Reddit comments"], reddit_shadows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
304 lines
17 KiB
Python
304 lines
17 KiB
Python
"""What the runs say about the Black dragon: analysis/reports/BLACK.md.
|
|
|
|
python analysis/black/evidence.py [--check]
|
|
|
|
Every number in the report is computed here from data/public/runs_clean.csv (and the bot's anonymised
|
|
trajectories), so it can be re-run after every import. Nothing is fitted: these are counts. The developer's
|
|
statements they are read against are in planner/knowledge.yaml.
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import glob
|
|
import json
|
|
import math
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "core"))
|
|
from atlas import runs as runs_io # noqa: E402
|
|
|
|
OUT = ROOT / "analysis/reports/BLACK.md"
|
|
OUT_JSON = ROOT / "data/public/black.json" # the same numbers, for the page
|
|
ROUTE = ["company", "direction", "combat_first", "combat_second", "resolution", "post_path"]
|
|
ROUTE_NAMES = "company, first direction, first move in the fight, the dagger, the bullies' fate, the fork"
|
|
|
|
|
|
def gated_tree(r):
|
|
"""A run that passed both known gates (investigated, helped, climbed the tree) and says whether the shadow showed."""
|
|
return r["scream"] == "Investigate & help" and r["hour"] == "Tree" and r["saw_shadow"] is not None
|
|
|
|
|
|
def group(r):
|
|
if r["source"] == "bot":
|
|
return "bot"
|
|
if r["source"] == "reddit spreadsheet":
|
|
return "form" if r["entry"] == "Form" else "bulk logs"
|
|
if r["source"] == "reddit comment":
|
|
return "Reddit comments"
|
|
if r["source"] == "reported":
|
|
return "reported"
|
|
return "other"
|
|
|
|
|
|
GROUP_NAMES = {"form": "Form submissions", "bulk logs": "Bulk logs", "bot": "Bot", "Reddit comments": "Reddit comments",
|
|
"reported": "Reported on the Atlas"}
|
|
|
|
|
|
def tally(rows, key):
|
|
d = collections.OrderedDict()
|
|
for r in rows:
|
|
c = d.setdefault(key(r), [0, 0])
|
|
c[0] += bool(r["saw_shadow"])
|
|
c[1] += 1
|
|
return d
|
|
|
|
|
|
def frac(y, n):
|
|
return f"{y} / {n}" + (f" ({100 * y / n:.1f}%)" if n else "")
|
|
|
|
|
|
def upper(n, conf=0.95):
|
|
"""The rate that zero in n runs rules out (one-sided)."""
|
|
return 1 - (1 - conf) ** (1 / n) if n else 1.0
|
|
|
|
|
|
def mantel_haenszel(strata):
|
|
"""strata: [(y1, n1, y0, n0)]. Odds ratio and z of group 1 against group 0, within strata."""
|
|
num = den = obs = exp = var = 0.0
|
|
for y1, n1, y0, n0 in strata:
|
|
N = n1 + n0
|
|
if not n1 or not n0:
|
|
continue
|
|
num += y1 * (n0 - y0) / N
|
|
den += y0 * (n1 - y1) / N
|
|
m = y1 + y0
|
|
obs += y1
|
|
exp += n1 * m / N
|
|
if N > 1:
|
|
var += n1 * n0 * m * (N - m) / (N * N * (N - 1))
|
|
return (num / den if den else float("inf")), obs, exp, ((obs - exp) / math.sqrt(var) if var else 0.0)
|
|
|
|
|
|
def last_answer(note):
|
|
"""The answer given to the Black, from a rider's free-text note (keywords only; the note is never published).
|
|
Riders lower their eyes to the Blue earlier in the same sequence, so only the last thing a note says counts, and
|
|
only once the note has reached the Black (or is too short to be about anything else)."""
|
|
t = re.sub(r"instead of low\w+[^.>,;]*", "", (note or "").lower())
|
|
marks = [m.end() for m in re.finditer(r"threat|branch|south|black", t)]
|
|
scope = t if len(t) < 45 else (t[marks[-1]:] if marks else "")
|
|
found = [(m.start(), k) for k, pat in (("held its gaze", r"gaze|glare|eye contact"), ("lowered eyes", r"lower"),
|
|
("ignored the sound", r"ignore"))
|
|
for m in re.finditer(pat, scope)]
|
|
if not found: # a long note that never names the Black's moment: use the whole note, gaze only
|
|
return "held its gaze" if re.search(r"(hold|held|meet|met)\w* (its|it's|the black'?s?|his|her) (gaze|glare)", t) else None
|
|
return max(found)[1]
|
|
|
|
|
|
def blue_mistake(note):
|
|
t = (note or "").lower()
|
|
if "approach" in t:
|
|
return "approached the Blue"
|
|
if "warn" in t or "interven" in t:
|
|
return "warned the other cadet / intervened"
|
|
return None
|
|
|
|
|
|
def build():
|
|
"""(the report as Markdown, its key numbers as a dict for the page)."""
|
|
runs = runs_io.read()
|
|
G = [r for r in runs if gated_tree(r)]
|
|
facts = {"schema": 1, "by_attempt": {}}
|
|
L = ["# The Black dragon: what the runs show", "",
|
|
"Written by `analysis/black/evidence.py` from `data/public/runs_clean.csv`. Counts only; nothing is fitted. "
|
|
"\"Gated tree runs\" investigated the scream, helped the squadmate and climbed the tree, the two gates the "
|
|
"developer has confirmed (`planner/knowledge.yaml`). Form submissions over-report the shadow, so their "
|
|
"percentages are too high; comparisons inside the form (first attempt against later, one route against "
|
|
"another) are still fair if the over-reporting is the same on both sides.", ""]
|
|
|
|
# 1. by attempt
|
|
# the self-reported sources (Reddit comments, "Report a run") get a column once they have a gated tree run
|
|
groups = ["form", "bulk logs", "bot"] + [g for g in ("Reddit comments", "reported") if any(group(r) == g for r in G)]
|
|
L += ["## 1. The shadow by attempt number", "", "| Attempt | " + " | ".join(GROUP_NAMES[g] for g in groups) + " |",
|
|
"|---" * (len(groups) + 1) + "|"]
|
|
bands = [("1", 1, 1), ("2", 2, 2), ("3", 3, 3), ("4-10", 4, 10), ("11-30", 11, 30), ("31 or more", 31, 10 ** 9)]
|
|
for name, lo, hi in bands:
|
|
cells = []
|
|
for g in groups:
|
|
rs = [r for r in G if group(r) == g and r["attempt"] and lo <= r["attempt"] <= hi]
|
|
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
|
|
facts["by_attempt"].setdefault(g, {})[name] = [sum(bool(r["saw_shadow"]) for r in rs), len(rs)]
|
|
L.append(f"| {name} | " + " | ".join(cells) + " |")
|
|
cells = []
|
|
for g in groups:
|
|
rs = [r for r in G if group(r) == g and not r["attempt"]]
|
|
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
|
|
L.append("| not given | " + " | ".join(cells) + " |")
|
|
complete_first = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] == 1]
|
|
complete_later = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] and r["attempt"] > 1]
|
|
L += ["", f"Complete logs (bot and bulk) have {len(complete_first)} first attempts at the tree and "
|
|
f"{len(complete_later)} later ones, with {sum(bool(r['saw_shadow']) for r in complete_first + complete_later)} shadows. "
|
|
f"Zero in {len(complete_later)} later attempts rules out a rate above {100 * upper(len(complete_later)):.1f}% for them; "
|
|
f"zero in {len(complete_first)} first attempts only rules out more than {100 * upper(len(complete_first)):.0f}%.", ""]
|
|
rc = [r for r in runs if r["source"] == "reddit comment" and r["saw_shadow"]]
|
|
if rc:
|
|
rg = [r for r in G if group(r) == "Reddit comments" and r["saw_shadow"]]
|
|
facts["reddit_shadows"] = {"all": len(rc), "gated": len(rg), "first_attempt": sum(r["attempt"] == 1 for r in rc)}
|
|
L += [f"Reddit comments describe {len(rc)} shadows in all, but a comment rarely says how the run began: only "
|
|
f"{len(rg)} say the rider helped their squadmate and climbed the tree (the table above), and "
|
|
f"{sum(r['attempt'] == 1 for r in rc)} say it was a first attempt. Like form submissions, they are posted by "
|
|
"choice, so they say what shadow runs look like, not how often the shadow comes.", ""]
|
|
|
|
# 2. the same route, first against later
|
|
F = [r for r in G if group(r) == "form" and r["attempt"]]
|
|
by = collections.defaultdict(lambda: [0, 0, 0, 0])
|
|
for r in F:
|
|
c = by[tuple(r[k] for k in ROUTE)]
|
|
i = 0 if r["attempt"] == 1 else 2
|
|
c[i] += bool(r["saw_shadow"])
|
|
c[i + 1] += 1
|
|
orr, obs, exp, z = mantel_haenszel(by.values())
|
|
facts["same_route"] = {"routes": len(by), "first_shadows": round(obs), "expected": round(exp, 1),
|
|
"odds_ratio": round(orr, 1) if math.isfinite(orr) else None, # JSON has no Infinity
|
|
"z": round(z, 1)}
|
|
facts["form_first"] = [sum(c[0] for c in by.values()), sum(c[1] for c in by.values())]
|
|
facts["form_later"] = [sum(c[2] for c in by.values()), sum(c[3] for c in by.values())]
|
|
L += ["## 2. The same route on a first attempt and on a later one", "",
|
|
f"If the shadow depended only on the choices the form records ({ROUTE_NAMES}), a route would do as well on a "
|
|
"later attempt as on a first. It does not.", "",
|
|
"| Route | First attempt | Later attempts |", "|---|---|---|"]
|
|
top = sorted(by.items(), key=lambda kv: -(kv[1][1] + kv[1][3]))[:8]
|
|
for key, (y1, n1, y0, n0) in top:
|
|
L.append(f"| {' / '.join(key)} | {frac(y1, n1)} | {frac(y0, n0)} |")
|
|
L += ["", f"Across all {len(by)} recorded routes, comparing first and later attempts on the same route: "
|
|
f"{obs:.0f} first-attempt shadows where {exp:.1f} would be expected if the attempt did not matter "
|
|
f"(odds ratio {f'{orr:.1f}' if math.isfinite(orr) else 'unbounded: no later-attempt shadow at all'}, z = {z:.1f}). First and later attempts choose almost the same routes, so the "
|
|
"difference is not in the recorded choices: it is something about the account, or a choice the form does "
|
|
"not record.", ""]
|
|
|
|
# 3. is it decided by route + cadet assignment?
|
|
best_key, best = max(((k, v) for k, v in by.items() if v[1] >= 20), key=lambda kv: kv[1][0] / kv[1][1])
|
|
facts["best_route"] = {"route": list(best_key), "first": [best[0], best[1]], "later": [best[2], best[3]]}
|
|
B = [r for r in F if r["attempt"] == 1 and tuple(r[k] for k in ROUTE) == best_key]
|
|
cells = tally(B, lambda r: (r["wing"], r["section"], r["squad"]))
|
|
multi = {k: v for k, v in cells.items() if v[1] >= 2}
|
|
mixed = sum(1 for y, n in multi.values() if 0 < y < n)
|
|
L += ["## 3. On one route, is the shadow decided by the cadet's wing, section and squad?", "",
|
|
f"The developer says the Black is decided by choices and by what the site knows about the account, with no "
|
|
f"dice. If the recorded route and the cadet assignment were all of it, every first attempt with the same "
|
|
f"route and assignment would agree. On the best-recorded route ({' / '.join(best_key)}; {frac(best[0], best[1])} "
|
|
f"on first attempts), {len(multi)} assignments have two or more runs and {mixed} of them disagree (some runs "
|
|
"with the shadow, some without). So something else decides it: a choice the form does not record, or a fact "
|
|
"about the account other than its assignment. No wing, section or squad stands out:", "",
|
|
"| | " + " | ".join(["shadow / runs"]) + " |", "|---|---|"]
|
|
for col in ("wing", "section", "squad", "first_sighting", "sighting_action"):
|
|
t = tally(B, lambda r, col=col: r[col] or "(blank)")
|
|
L.append(f"| {col} | " + ", ".join(f"{k}: {y}/{n}" for k, (y, n) in sorted(t.items(), key=lambda kv: -kv[1][1]) if n >= 3) + " |")
|
|
L.append("")
|
|
|
|
# 4. no single route
|
|
S1 = [r for r in G if group(r) == "form" and r["saw_shadow"]]
|
|
L += ["## 4. There is no single route", "",
|
|
f"The {len(S1)} form runs with a shadow (all attempts) took {len({tuple(r[k] for k in ROUTE) for r in S1})} "
|
|
"different recorded routes:", ""]
|
|
for col, label in (("company", "Company"), ("direction", "First direction"), ("combat_first", "First move in the fight"),
|
|
("combat_second", "The dagger"), ("resolution", "The bullies' fate"), ("post_path", "The fork")):
|
|
t = collections.Counter(r[col] for r in S1)
|
|
L.append(f"- **{label}:** " + ", ".join(f"{k} {v}" for k, v in t.most_common()))
|
|
# a gate counts as missed only when the run answers it otherwise; a run that does not say (a short Reddit
|
|
# comment, a blank form answer) is not a contradiction
|
|
def stated(v):
|
|
return bool(v) and v != "Unknown" and not str(v).startswith("(")
|
|
|
|
def contradicts(r):
|
|
return (stated(r["scream"]) and r["scream"] != "Investigate & help") or (stated(r["hour"]) and r["hour"] != "Tree")
|
|
outside = [r for r in runs if r["saw_shadow"] and contradicts(r)]
|
|
unstated = [r for r in runs if r["saw_shadow"] and not contradicts(r)
|
|
and not (r["scream"] == "Investigate & help" and r["hour"] == "Tree")]
|
|
facts["shadow_routes"] = {"runs": len(S1), "routes": len({tuple(r[k] for k in ROUTE) for r in S1}),
|
|
"off_gate": len(outside), "gates_unstated": len(unstated)}
|
|
L += ["", f"Shadows reported with an answer that contradicts a gate: {len(outside)} (likely mis-entered or misread). "
|
|
f"Another {len(unstated)} do not say whether they passed both gates.", ""]
|
|
|
|
# 5. after the shadow
|
|
SH = [r for r in runs if r["saw_shadow"]]
|
|
black = [r for r in SH if r["enc_Black"]]
|
|
ans = collections.defaultdict(lambda: [0, 0])
|
|
for r in black:
|
|
a = last_answer((r["bond_note"] or "") + " " + (r["death_note"] or ""))
|
|
if a:
|
|
ans[a][0] += r["outcome"] == "Bonded"
|
|
ans[a][1] += 1
|
|
blue_dead = [r for r in SH if r["enc_Blue"] == "Incinerated"]
|
|
facts["after_shadow"] = {"reached_black": len(black), "bonded_black": sum(r["outcome"] == "Bonded" for r in black),
|
|
"died_at_blue": len(blue_dead),
|
|
"answers": {k: [y, n] for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])}}
|
|
mis = collections.Counter(blue_mistake((r["bond_note"] or "") + " " + (r["death_note"] or "")) for r in blue_dead)
|
|
L += ["## 5. After the shadow: the Blue, then the Black", "",
|
|
"The riders' notes describe one sequence: follow the shadow, a Blue lands and burns another cadet, the Blue "
|
|
"leaves, then the Black appears. The Blue is a trial on the way, not an alternative to the Black.", "",
|
|
f"- Runs that reached the Black: {len(black)}, bonded {sum(r['outcome'] == 'Bonded' for r in black)}.",
|
|
f"- Runs that died at the Blue: {len(blue_dead)}. What their notes say they did: "
|
|
+ ", ".join(f"{k} {v}" for k, v in mis.most_common() if k) + ".",
|
|
"- The answer given to the Black, where the note says (bonded / runs): "
|
|
+ "; ".join(f"{k} {y}/{n}" for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])) + ".", "",
|
|
"Holding its gaze is the only answer recorded with a bond, and it still fails about a quarter of the time. "
|
|
"The developer says a rider who knows the system bonds with certainty, so those failures are not explained "
|
|
"yet: the notes of the riders who died are too short to show what else they did differently.", ""]
|
|
|
|
# 6. what the bot has tried
|
|
L += ["## 6. What the bot has tried", ""]
|
|
files = sorted(f for f in glob.glob(str(ROOT / "data/bot/trajectories-*.jsonl")) if "sample" not in f)
|
|
seen, acc, order = set(), collections.defaultdict(list), collections.defaultdict(list)
|
|
for f in files:
|
|
for line in open(f, encoding="utf-8"):
|
|
t = json.loads(line)
|
|
if t["id"] not in seen:
|
|
seen.add(t["id"])
|
|
acc[t["meta"].get("accountKey")].append(t["meta"].get("attempt") or 0)
|
|
order[t["meta"].get("accountKey")].append((t["meta"].get("attempt") or 0, t["id"]))
|
|
by_id = {r["run_id"]: r for r in runs_io.read(sources=["bot"], drop=())}
|
|
streak = 0 # the longest run of consecutive attempts on one account with the same gated tree route
|
|
for v in order.values():
|
|
cur, prev = 0, None
|
|
for _, i in sorted(v):
|
|
r = by_id.get(i)
|
|
s = tuple(r[k] for k in ROUTE + ["sighting_action"]) if r and gated_tree(r) else None
|
|
cur = cur + 1 if s is not None and s == prev else (1 if s is not None else 0)
|
|
prev, streak = s, max(streak, cur)
|
|
botg = [r for r in G if group(r) == "bot"]
|
|
facts["bot"] = {"attempts": len(seen), "accounts": len(acc), "gated_tree": len(botg),
|
|
"shadows": sum(bool(r["saw_shadow"]) for r in botg),
|
|
"earliest_attempt": min((min(v) for v in acc.values() if v), default=0)}
|
|
routes = collections.Counter(tuple(r[k] for k in ROUTE) for r in botg)
|
|
L += [f"{len(seen)} attempts on {len(acc)} accounts; the earliest attempt logged on any account is number "
|
|
f"{min((min(v) for v in acc.values() if v), default=0)}. Gated tree runs: {len(botg)}, shadows "
|
|
f"{sum(bool(r['saw_shadow']) for r in botg)}, on {len(routes)} recorded routes "
|
|
f"(the commonest {routes.most_common(1)[0][1] if routes else 0} times). The bot has not played a first attempt, "
|
|
"so it has not yet tested the one situation where riders report the shadow. The most times any account has "
|
|
f"repeated one route in a row is {streak}.", ""]
|
|
return "\n".join(L) + "\n", facts
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--check", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
text, facts = build()
|
|
data = json.dumps(facts, ensure_ascii=False, indent=1, allow_nan=False) + "\n"
|
|
if a.check:
|
|
for path, want in ((OUT, text), (OUT_JSON, data)):
|
|
if not path.exists() or path.read_text(encoding="utf-8") != want:
|
|
print(f"{path} is out of date (run python analysis/black/evidence.py)", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
OUT.write_text(text, encoding="utf-8")
|
|
OUT_JSON.write_text(data, encoding="utf-8")
|
|
print(f"wrote {OUT.relative_to(ROOT)} and {OUT_JSON.relative_to(ROOT)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|