"""What the runs say about the Black dragon: analysis/reports/BLACK.md. python analysis/black/evidence.py [--check] Every number in the report is computed here from data/public/runs_clean.csv (and the bot's anonymised trajectories), so it can be re-run after every import. Nothing is fitted: these are counts. The developer's statements they are read against are in planner/knowledge.yaml. """ import argparse import collections import glob import json import math import re import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "core")) from atlas import runs as runs_io # noqa: E402 OUT = ROOT / "analysis/reports/BLACK.md" OUT_JSON = ROOT / "data/public/black.json" # the same numbers, for the page ROUTE = ["company", "direction", "combat_first", "combat_second", "resolution", "post_path"] ROUTE_NAMES = "company, first direction, first move in the fight, the dagger, the bullies' fate, the fork" def gated_tree(r): """A run that passed both known gates (investigated, helped, climbed the tree) and says whether the shadow showed.""" return r["scream"] == "Investigate & help" and r["hour"] == "Tree" and r["saw_shadow"] is not None def group(r): if r["source"] == "bot": return "bot" if r["source"] == "reddit spreadsheet": return "form" if r["entry"] == "Form" else "bulk logs" return "other" def tally(rows, key): d = collections.OrderedDict() for r in rows: c = d.setdefault(key(r), [0, 0]) c[0] += bool(r["saw_shadow"]) c[1] += 1 return d def frac(y, n): return f"{y} / {n}" + (f" ({100 * y / n:.1f}%)" if n else "") def upper(n, conf=0.95): """The rate that zero in n runs rules out (one-sided).""" return 1 - (1 - conf) ** (1 / n) if n else 1.0 def mantel_haenszel(strata): """strata: [(y1, n1, y0, n0)]. Odds ratio and z of group 1 against group 0, within strata.""" num = den = obs = exp = var = 0.0 for y1, n1, y0, n0 in strata: N = n1 + n0 if not n1 or not n0: continue num += y1 * (n0 - y0) / N den += y0 * (n1 - y1) / N m = y1 + y0 obs += y1 exp += n1 * m / N if N > 1: var += n1 * n0 * m * (N - m) / (N * N * (N - 1)) return (num / den if den else float("inf")), obs, exp, ((obs - exp) / math.sqrt(var) if var else 0.0) def last_answer(note): """The answer given to the Black, from a rider's free-text note (keywords only; the note is never published). Riders lower their eyes to the Blue earlier in the same sequence, so only the last thing a note says counts, and only once the note has reached the Black (or is too short to be about anything else).""" t = re.sub(r"instead of low\w+[^.>,;]*", "", (note or "").lower()) marks = [m.end() for m in re.finditer(r"threat|branch|south|black", t)] scope = t if len(t) < 45 else (t[marks[-1]:] if marks else "") found = [(m.start(), k) for k, pat in (("held its gaze", r"gaze|glare|eye contact"), ("lowered eyes", r"lower"), ("ignored the sound", r"ignore")) for m in re.finditer(pat, scope)] if not found: # a long note that never names the Black's moment: use the whole note, gaze only return "held its gaze" if re.search(r"(hold|held|meet|met)\w* (its|it's|the black'?s?|his|her) (gaze|glare)", t) else None return max(found)[1] def blue_mistake(note): t = (note or "").lower() if "approach" in t: return "approached the Blue" if "warn" in t or "interven" in t: return "warned the other cadet / intervened" return None def build(): """(the report as Markdown, its key numbers as a dict for the page).""" runs = runs_io.read() G = [r for r in runs if gated_tree(r)] facts = {"schema": 1, "by_attempt": {}} L = ["# The Black dragon: what the runs show", "", "Written by `analysis/black/evidence.py` from `data/public/runs_clean.csv`. Counts only; nothing is fitted. " "\"Gated tree runs\" investigated the scream, helped the squadmate and climbed the tree, the two gates the " "developer has confirmed (`planner/knowledge.yaml`). Form submissions over-report the shadow, so their " "percentages are too high; comparisons inside the form (first attempt against later, one route against " "another) are still fair if the over-reporting is the same on both sides.", ""] # 1. by attempt L += ["## 1. The shadow by attempt number", "", "| Attempt | Form submissions | Bulk logs | Bot |", "|---|---|---|---|"] bands = [("1", 1, 1), ("2", 2, 2), ("3", 3, 3), ("4-10", 4, 10), ("11-30", 11, 30), ("31 or more", 31, 10 ** 9)] for name, lo, hi in bands: cells = [] for g in ("form", "bulk logs", "bot"): rs = [r for r in G if group(r) == g and r["attempt"] and lo <= r["attempt"] <= hi] cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs))) facts["by_attempt"].setdefault(g, {})[name] = [sum(bool(r["saw_shadow"]) for r in rs), len(rs)] L.append(f"| {name} | " + " | ".join(cells) + " |") cells = [] for g in ("form", "bulk logs", "bot"): rs = [r for r in G if group(r) == g and not r["attempt"]] cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs))) L.append("| not given | " + " | ".join(cells) + " |") complete_first = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] == 1] complete_later = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] and r["attempt"] > 1] L += ["", f"Complete logs (bot and bulk) have {len(complete_first)} first attempts at the tree and " f"{len(complete_later)} later ones, with {sum(bool(r['saw_shadow']) for r in complete_first + complete_later)} shadows. " f"Zero in {len(complete_later)} later attempts rules out a rate above {100 * upper(len(complete_later)):.1f}% for them; " f"zero in {len(complete_first)} first attempts only rules out more than {100 * upper(len(complete_first)):.0f}%.", ""] # 2. the same route, first against later F = [r for r in G if group(r) == "form" and r["attempt"]] by = collections.defaultdict(lambda: [0, 0, 0, 0]) for r in F: c = by[tuple(r[k] for k in ROUTE)] i = 0 if r["attempt"] == 1 else 2 c[i] += bool(r["saw_shadow"]) c[i + 1] += 1 orr, obs, exp, z = mantel_haenszel(by.values()) facts["same_route"] = {"routes": len(by), "first_shadows": round(obs), "expected": round(exp, 1), "odds_ratio": round(orr, 1) if math.isfinite(orr) else None, # JSON has no Infinity "z": round(z, 1)} facts["form_first"] = [sum(c[0] for c in by.values()), sum(c[1] for c in by.values())] facts["form_later"] = [sum(c[2] for c in by.values()), sum(c[3] for c in by.values())] L += ["## 2. The same route on a first attempt and on a later one", "", f"If the shadow depended only on the choices the form records ({ROUTE_NAMES}), a route would do as well on a " "later attempt as on a first. It does not.", "", "| Route | First attempt | Later attempts |", "|---|---|---|"] top = sorted(by.items(), key=lambda kv: -(kv[1][1] + kv[1][3]))[:8] for key, (y1, n1, y0, n0) in top: L.append(f"| {' / '.join(key)} | {frac(y1, n1)} | {frac(y0, n0)} |") L += ["", f"Across all {len(by)} recorded routes, comparing first and later attempts on the same route: " f"{obs:.0f} first-attempt shadows where {exp:.1f} would be expected if the attempt did not matter " f"(odds ratio {f'{orr:.1f}' if math.isfinite(orr) else 'unbounded: no later-attempt shadow at all'}, z = {z:.1f}). First and later attempts choose almost the same routes, so the " "difference is not in the recorded choices: it is something about the account, or a choice the form does " "not record.", ""] # 3. is it decided by route + cadet assignment? best_key, best = max(((k, v) for k, v in by.items() if v[1] >= 20), key=lambda kv: kv[1][0] / kv[1][1]) facts["best_route"] = {"route": list(best_key), "first": [best[0], best[1]], "later": [best[2], best[3]]} B = [r for r in F if r["attempt"] == 1 and tuple(r[k] for k in ROUTE) == best_key] cells = tally(B, lambda r: (r["wing"], r["section"], r["squad"])) multi = {k: v for k, v in cells.items() if v[1] >= 2} mixed = sum(1 for y, n in multi.values() if 0 < y < n) L += ["## 3. On one route, is the shadow decided by the cadet's wing, section and squad?", "", f"The developer says the Black is decided by choices and by what the site knows about the account, with no " f"dice. If the recorded route and the cadet assignment were all of it, every first attempt with the same " f"route and assignment would agree. On the best-recorded route ({' / '.join(best_key)}; {frac(best[0], best[1])} " f"on first attempts), {len(multi)} assignments have two or more runs and {mixed} of them disagree (some runs " "with the shadow, some without). So something else decides it: a choice the form does not record, or a fact " "about the account other than its assignment. No wing, section or squad stands out:", "", "| | " + " | ".join(["shadow / runs"]) + " |", "|---|---|"] for col in ("wing", "section", "squad", "first_sighting", "sighting_action"): t = tally(B, lambda r, col=col: r[col] or "(blank)") L.append(f"| {col} | " + ", ".join(f"{k}: {y}/{n}" for k, (y, n) in sorted(t.items(), key=lambda kv: -kv[1][1]) if n >= 3) + " |") L.append("") # 4. no single route S1 = [r for r in G if group(r) == "form" and r["saw_shadow"]] L += ["## 4. There is no single route", "", f"The {len(S1)} form runs with a shadow (all attempts) took {len({tuple(r[k] for k in ROUTE) for r in S1})} " "different recorded routes:", ""] for col, label in (("company", "Company"), ("direction", "First direction"), ("combat_first", "First move in the fight"), ("combat_second", "The dagger"), ("resolution", "The bullies' fate"), ("post_path", "The fork")): t = collections.Counter(r[col] for r in S1) L.append(f"- **{label}:** " + ", ".join(f"{k} {v}" for k, v in t.most_common())) # a gate counts as missed only when the run answers it otherwise; a run that does not say (a short Reddit # comment, a blank form answer) is not a contradiction def stated(v): return bool(v) and v != "Unknown" and not str(v).startswith("(") def contradicts(r): return (stated(r["scream"]) and r["scream"] != "Investigate & help") or (stated(r["hour"]) and r["hour"] != "Tree") outside = [r for r in runs if r["saw_shadow"] and contradicts(r)] unstated = [r for r in runs if r["saw_shadow"] and not contradicts(r) and not (r["scream"] == "Investigate & help" and r["hour"] == "Tree")] facts["shadow_routes"] = {"runs": len(S1), "routes": len({tuple(r[k] for k in ROUTE) for r in S1}), "off_gate": len(outside), "gates_unstated": len(unstated)} L += ["", f"Shadows reported with an answer that contradicts a gate: {len(outside)} (likely mis-entered or misread). " f"Another {len(unstated)} do not say whether they passed both gates.", ""] # 5. after the shadow SH = [r for r in runs if r["saw_shadow"]] black = [r for r in SH if r["enc_Black"]] ans = collections.defaultdict(lambda: [0, 0]) for r in black: a = last_answer((r["bond_note"] or "") + " " + (r["death_note"] or "")) if a: ans[a][0] += r["outcome"] == "Bonded" ans[a][1] += 1 blue_dead = [r for r in SH if r["enc_Blue"] == "Incinerated"] facts["after_shadow"] = {"reached_black": len(black), "bonded_black": sum(r["outcome"] == "Bonded" for r in black), "died_at_blue": len(blue_dead), "answers": {k: [y, n] for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])}} mis = collections.Counter(blue_mistake((r["bond_note"] or "") + " " + (r["death_note"] or "")) for r in blue_dead) L += ["## 5. After the shadow: the Blue, then the Black", "", "The riders' notes describe one sequence: follow the shadow, a Blue lands and burns another cadet, the Blue " "leaves, then the Black appears. The Blue is a trial on the way, not an alternative to the Black.", "", f"- Runs that reached the Black: {len(black)}, bonded {sum(r['outcome'] == 'Bonded' for r in black)}.", f"- Runs that died at the Blue: {len(blue_dead)}. What their notes say they did: " + ", ".join(f"{k} {v}" for k, v in mis.most_common() if k) + ".", "- The answer given to the Black, where the note says (bonded / runs): " + "; ".join(f"{k} {y}/{n}" for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])) + ".", "", "Holding its gaze is the only answer recorded with a bond, and it still fails about a quarter of the time. " "The developer says a rider who knows the system bonds with certainty, so those failures are not explained " "yet: the notes of the riders who died are too short to show what else they did differently.", ""] # 6. what the bot has tried L += ["## 6. What the bot has tried", ""] files = sorted(f for f in glob.glob(str(ROOT / "data/bot/trajectories-*.jsonl")) if "sample" not in f) seen, acc, order = set(), collections.defaultdict(list), collections.defaultdict(list) for f in files: for line in open(f, encoding="utf-8"): t = json.loads(line) if t["id"] not in seen: seen.add(t["id"]) acc[t["meta"].get("accountKey")].append(t["meta"].get("attempt") or 0) order[t["meta"].get("accountKey")].append((t["meta"].get("attempt") or 0, t["id"])) by_id = {r["run_id"]: r for r in runs_io.read(sources=["bot"], drop=())} streak = 0 # the longest run of consecutive attempts on one account with the same gated tree route for v in order.values(): cur, prev = 0, None for _, i in sorted(v): r = by_id.get(i) s = tuple(r[k] for k in ROUTE + ["sighting_action"]) if r and gated_tree(r) else None cur = cur + 1 if s is not None and s == prev else (1 if s is not None else 0) prev, streak = s, max(streak, cur) botg = [r for r in G if group(r) == "bot"] facts["bot"] = {"attempts": len(seen), "accounts": len(acc), "gated_tree": len(botg), "shadows": sum(bool(r["saw_shadow"]) for r in botg), "earliest_attempt": min((min(v) for v in acc.values() if v), default=0)} routes = collections.Counter(tuple(r[k] for k in ROUTE) for r in botg) L += [f"{len(seen)} attempts on {len(acc)} accounts; the earliest attempt logged on any account is number " f"{min((min(v) for v in acc.values() if v), default=0)}. Gated tree runs: {len(botg)}, shadows " f"{sum(bool(r['saw_shadow']) for r in botg)}, on {len(routes)} recorded routes " f"(the commonest {routes.most_common(1)[0][1] if routes else 0} times). The bot has not played a first attempt, " "so it has not yet tested the one situation where riders report the shadow. The most times any account has " f"repeated one route in a row is {streak}.", ""] return "\n".join(L) + "\n", facts def main(argv=None): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--check", action="store_true") a = ap.parse_args(argv) text, facts = build() data = json.dumps(facts, ensure_ascii=False, indent=1, allow_nan=False) + "\n" if a.check: for path, want in ((OUT, text), (OUT_JSON, data)): if not path.exists() or path.read_text(encoding="utf-8") != want: print(f"{path} is out of date (run python analysis/black/evidence.py)", file=sys.stderr) return 1 return 0 OUT.write_text(text, encoding="utf-8") OUT_JSON.write_text(data, encoding="utf-8") print(f"wrote {OUT.relative_to(ROOT)} and {OUT_JSON.relative_to(ROOT)}") return 0 if __name__ == "__main__": sys.exit(main())