The Black is not dice: developer statements, evidence report, and a plan that looks for the rule
- planner/knowledge.yaml: what the game's developer has said (paraphrased, dated, graded), what the Atlas does with each statement, and the open questions with the experiment that tests each. - analysis/black/evidence.py -> analysis/reports/BLACK.md: counts from the runs. On the same recorded route the shadow shows on 21 of 79 first attempts and 1 of 80 later ones; no single route; Blue then Black after the shadow. - game.yaml: the shadow is a guard (not a roll); what follows it is the Blue, then the Black; on-screen text for the squad's direction screen. - planner/experiments.yaml v2: one exact path on every fresh account (fresh-replicate), then one choice changed at a time (fresh-ablate); shadow-again, second-attempt-same-path, veteran-repeat, veteran-variety; what the bot records about the account; the sequence to follow once the shadow shows. - solver.js: assign() takes account facts, account offsets and cycled arms; recommendOption() answers the squad screen (stay + direction) in one pick. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
This commit is contained in:
co-authored by
Claude Fable 5.2
parent
51842e1aa9
commit
d01ec46fcc
@@ -0,0 +1,253 @@
|
||||
"""What the runs say about the Black dragon: analysis/reports/BLACK.md.
|
||||
|
||||
python analysis/black/evidence.py [--check]
|
||||
|
||||
Every number in the report is computed here from data/public/runs_clean.csv (and the bot's anonymised
|
||||
trajectories), so it can be re-run after every import. Nothing is fitted: these are counts. The developer's
|
||||
statements they are read against are in planner/knowledge.yaml.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import glob
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
|
||||
OUT = ROOT / "analysis/reports/BLACK.md"
|
||||
ROUTE = ["company", "direction", "combat_first", "combat_second", "resolution", "post_path"]
|
||||
ROUTE_NAMES = "company, first direction, first move in the fight, the dagger, the bullies' fate, the fork"
|
||||
|
||||
|
||||
def gated_tree(r):
|
||||
"""A run that passed both known gates (investigated, helped, climbed the tree) and says whether the shadow showed."""
|
||||
return r["scream"] == "Investigate & help" and r["hour"] == "Tree" and r["saw_shadow"] is not None
|
||||
|
||||
|
||||
def group(r):
|
||||
if r["source"] == "bot":
|
||||
return "bot"
|
||||
if r["source"] == "reddit spreadsheet":
|
||||
return "form" if r["entry"] == "Form" else "bulk logs"
|
||||
return "other"
|
||||
|
||||
|
||||
def tally(rows, key):
|
||||
d = collections.OrderedDict()
|
||||
for r in rows:
|
||||
c = d.setdefault(key(r), [0, 0])
|
||||
c[0] += bool(r["saw_shadow"])
|
||||
c[1] += 1
|
||||
return d
|
||||
|
||||
|
||||
def frac(y, n):
|
||||
return f"{y} / {n}" + (f" ({100 * y / n:.1f}%)" if n else "")
|
||||
|
||||
|
||||
def upper(n, conf=0.95):
|
||||
"""The rate that zero in n runs rules out (one-sided)."""
|
||||
return 1 - (1 - conf) ** (1 / n) if n else 1.0
|
||||
|
||||
|
||||
def mantel_haenszel(strata):
|
||||
"""strata: [(y1, n1, y0, n0)]. Odds ratio and z of group 1 against group 0, within strata."""
|
||||
num = den = obs = exp = var = 0.0
|
||||
for y1, n1, y0, n0 in strata:
|
||||
N = n1 + n0
|
||||
if not n1 or not n0:
|
||||
continue
|
||||
num += y1 * (n0 - y0) / N
|
||||
den += y0 * (n1 - y1) / N
|
||||
m = y1 + y0
|
||||
obs += y1
|
||||
exp += n1 * m / N
|
||||
if N > 1:
|
||||
var += n1 * n0 * m * (N - m) / (N * N * (N - 1))
|
||||
return (num / den if den else float("inf")), obs, exp, ((obs - exp) / math.sqrt(var) if var else 0.0)
|
||||
|
||||
|
||||
def last_answer(note):
|
||||
"""The answer given to the Black, from a rider's free-text note (keywords only; the note is never published).
|
||||
Riders lower their eyes to the Blue earlier in the same sequence, so only the last thing a note says counts, and
|
||||
only once the note has reached the Black (or is too short to be about anything else)."""
|
||||
t = re.sub(r"instead of low\w+[^.>,;]*", "", (note or "").lower())
|
||||
marks = [m.end() for m in re.finditer(r"threat|branch|south|black", t)]
|
||||
scope = t if len(t) < 45 else (t[marks[-1]:] if marks else "")
|
||||
found = [(m.start(), k) for k, pat in (("held its gaze", r"gaze|glare|eye contact"), ("lowered eyes", r"lower"),
|
||||
("ignored the sound", r"ignore"))
|
||||
for m in re.finditer(pat, scope)]
|
||||
if not found: # a long note that never names the Black's moment: use the whole note, gaze only
|
||||
return "held its gaze" if re.search(r"(hold|held|meet|met)\w* (its|it's|the black'?s?|his|her) (gaze|glare)", t) else None
|
||||
return max(found)[1]
|
||||
|
||||
|
||||
def blue_mistake(note):
|
||||
t = (note or "").lower()
|
||||
if "approach" in t:
|
||||
return "approached the Blue"
|
||||
if "warn" in t or "interven" in t:
|
||||
return "warned the other cadet / intervened"
|
||||
return None
|
||||
|
||||
|
||||
def build():
|
||||
runs = runs_io.read()
|
||||
G = [r for r in runs if gated_tree(r)]
|
||||
L = ["# The Black dragon: what the runs show", "",
|
||||
"Written by `analysis/black/evidence.py` from `data/public/runs_clean.csv`. Counts only; nothing is fitted. "
|
||||
"\"Gated tree runs\" investigated the scream, helped the squadmate and climbed the tree, the two gates the "
|
||||
"developer has confirmed (`planner/knowledge.yaml`). Form submissions over-report the shadow, so their "
|
||||
"percentages are too high; comparisons inside the form (first attempt against later, one route against "
|
||||
"another) are still fair if the over-reporting is the same on both sides.", ""]
|
||||
|
||||
# 1. by attempt
|
||||
L += ["## 1. The shadow by attempt number", "", "| Attempt | Form submissions | Bulk logs | Bot |", "|---|---|---|---|"]
|
||||
bands = [("1", 1, 1), ("2", 2, 2), ("3", 3, 3), ("4-10", 4, 10), ("11-30", 11, 30), ("31 or more", 31, 10 ** 9)]
|
||||
for name, lo, hi in bands:
|
||||
cells = []
|
||||
for g in ("form", "bulk logs", "bot"):
|
||||
rs = [r for r in G if group(r) == g and r["attempt"] and lo <= r["attempt"] <= hi]
|
||||
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
|
||||
L.append(f"| {name} | " + " | ".join(cells) + " |")
|
||||
cells = []
|
||||
for g in ("form", "bulk logs", "bot"):
|
||||
rs = [r for r in G if group(r) == g and not r["attempt"]]
|
||||
cells.append(frac(sum(bool(r["saw_shadow"]) for r in rs), len(rs)))
|
||||
L.append("| not given | " + " | ".join(cells) + " |")
|
||||
complete_first = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] == 1]
|
||||
complete_later = [r for r in G if group(r) in ("bot", "bulk logs") and r["attempt"] and r["attempt"] > 1]
|
||||
L += ["", f"Complete logs (bot and bulk) have {len(complete_first)} first attempts at the tree and "
|
||||
f"{len(complete_later)} later ones, with {sum(bool(r['saw_shadow']) for r in complete_first + complete_later)} shadows. "
|
||||
f"Zero in {len(complete_later)} later attempts rules out a rate above {100 * upper(len(complete_later)):.1f}% for them; "
|
||||
f"zero in {len(complete_first)} first attempts only rules out more than {100 * upper(len(complete_first)):.0f}%.", ""]
|
||||
|
||||
# 2. the same route, first against later
|
||||
F = [r for r in G if group(r) == "form" and r["attempt"]]
|
||||
by = collections.defaultdict(lambda: [0, 0, 0, 0])
|
||||
for r in F:
|
||||
c = by[tuple(r[k] for k in ROUTE)]
|
||||
i = 0 if r["attempt"] == 1 else 2
|
||||
c[i] += bool(r["saw_shadow"])
|
||||
c[i + 1] += 1
|
||||
orr, obs, exp, z = mantel_haenszel(by.values())
|
||||
L += ["## 2. The same route on a first attempt and on a later one", "",
|
||||
f"If the shadow depended only on the choices the form records ({ROUTE_NAMES}), a route would do as well on a "
|
||||
"later attempt as on a first. It does not.", "",
|
||||
"| Route | First attempt | Later attempts |", "|---|---|---|"]
|
||||
top = sorted(by.items(), key=lambda kv: -(kv[1][1] + kv[1][3]))[:8]
|
||||
for key, (y1, n1, y0, n0) in top:
|
||||
L.append(f"| {' / '.join(key)} | {frac(y1, n1)} | {frac(y0, n0)} |")
|
||||
L += ["", f"Across all {len(by)} recorded routes, comparing first and later attempts on the same route: "
|
||||
f"{obs:.0f} first-attempt shadows where {exp:.1f} would be expected if the attempt did not matter "
|
||||
f"(odds ratio {orr:.1f}, z = {z:.1f}). First and later attempts choose almost the same routes, so the "
|
||||
"difference is not in the recorded choices: it is something about the account, or a choice the form does "
|
||||
"not record.", ""]
|
||||
|
||||
# 3. is it decided by route + cadet assignment?
|
||||
best_key, best = max(((k, v) for k, v in by.items() if v[1] >= 20), key=lambda kv: kv[1][0] / kv[1][1])
|
||||
B = [r for r in F if r["attempt"] == 1 and tuple(r[k] for k in ROUTE) == best_key]
|
||||
cells = tally(B, lambda r: (r["wing"], r["section"], r["squad"]))
|
||||
multi = {k: v for k, v in cells.items() if v[1] >= 2}
|
||||
mixed = sum(1 for y, n in multi.values() if 0 < y < n)
|
||||
L += ["## 3. On one route, is the shadow decided by the cadet's wing, section and squad?", "",
|
||||
f"The developer says the Black is decided by choices and by what the site knows about the account, with no "
|
||||
f"dice. If the recorded route and the cadet assignment were all of it, every first attempt with the same "
|
||||
f"route and assignment would agree. On the best-recorded route ({' / '.join(best_key)}; {frac(best[0], best[1])} "
|
||||
f"on first attempts), {len(multi)} assignments have two or more runs and {mixed} of them disagree (some runs "
|
||||
"with the shadow, some without). So something else decides it: a choice the form does not record, or a fact "
|
||||
"about the account other than its assignment. No wing, section or squad stands out:", "",
|
||||
"| | " + " | ".join(["shadow / runs"]) + " |", "|---|---|"]
|
||||
for col in ("wing", "section", "squad", "first_sighting", "sighting_action"):
|
||||
t = tally(B, lambda r, col=col: r[col] or "(blank)")
|
||||
L.append(f"| {col} | " + ", ".join(f"{k}: {y}/{n}" for k, (y, n) in sorted(t.items(), key=lambda kv: -kv[1][1]) if n >= 3) + " |")
|
||||
L.append("")
|
||||
|
||||
# 4. no single route
|
||||
S1 = [r for r in G if group(r) == "form" and r["saw_shadow"]]
|
||||
L += ["## 4. There is no single route", "",
|
||||
f"The {len(S1)} form runs with a shadow (all attempts) took {len({tuple(r[k] for k in ROUTE) for r in S1})} "
|
||||
"different recorded routes:", ""]
|
||||
for col, label in (("company", "Company"), ("direction", "First direction"), ("combat_first", "First move in the fight"),
|
||||
("combat_second", "The dagger"), ("resolution", "The bullies' fate"), ("post_path", "The fork")):
|
||||
t = collections.Counter(r[col] for r in S1)
|
||||
L.append(f"- **{label}:** " + ", ".join(f"{k} {v}" for k, v in t.most_common()))
|
||||
outside = [r for r in runs if r["saw_shadow"] and not (r["scream"] == "Investigate & help" and r["hour"] == "Tree")]
|
||||
L += ["", f"Shadows reported without both gates: {len(outside)} (likely mis-entered).", ""]
|
||||
|
||||
# 5. after the shadow
|
||||
SH = [r for r in runs if r["saw_shadow"]]
|
||||
black = [r for r in SH if r["enc_Black"]]
|
||||
ans = collections.defaultdict(lambda: [0, 0])
|
||||
for r in black:
|
||||
a = last_answer((r["bond_note"] or "") + " " + (r["death_note"] or ""))
|
||||
if a:
|
||||
ans[a][0] += r["outcome"] == "Bonded"
|
||||
ans[a][1] += 1
|
||||
blue_dead = [r for r in SH if r["enc_Blue"] == "Incinerated"]
|
||||
mis = collections.Counter(blue_mistake((r["bond_note"] or "") + " " + (r["death_note"] or "")) for r in blue_dead)
|
||||
L += ["## 5. After the shadow: the Blue, then the Black", "",
|
||||
"The riders' notes describe one sequence: follow the shadow, a Blue lands and burns another cadet, the Blue "
|
||||
"leaves, then the Black appears. The Blue is a trial on the way, not an alternative to the Black.", "",
|
||||
f"- Runs that reached the Black: {len(black)}, bonded {sum(r['outcome'] == 'Bonded' for r in black)}.",
|
||||
f"- Runs that died at the Blue: {len(blue_dead)}. What their notes say they did: "
|
||||
+ ", ".join(f"{k} {v}" for k, v in mis.most_common() if k) + ".",
|
||||
"- The answer given to the Black, where the note says (bonded / runs): "
|
||||
+ "; ".join(f"{k} {y}/{n}" for k, (y, n) in sorted(ans.items(), key=lambda kv: -kv[1][1])) + ".", "",
|
||||
"Holding its gaze is the only answer recorded with a bond, and it still fails about a quarter of the time. "
|
||||
"The developer says a rider who knows the system bonds with certainty, so those failures are not explained "
|
||||
"yet: the notes of the riders who died are too short to show what else they did differently.", ""]
|
||||
|
||||
# 6. what the bot has tried
|
||||
L += ["## 6. What the bot has tried", ""]
|
||||
files = sorted(f for f in glob.glob(str(ROOT / "data/bot/trajectories-*.jsonl")) if "sample" not in f)
|
||||
seen, acc, order = set(), collections.defaultdict(list), collections.defaultdict(list)
|
||||
for f in files:
|
||||
for line in open(f, encoding="utf-8"):
|
||||
t = json.loads(line)
|
||||
if t["id"] not in seen:
|
||||
seen.add(t["id"])
|
||||
acc[t["meta"].get("accountKey")].append(t["meta"].get("attempt") or 0)
|
||||
order[t["meta"].get("accountKey")].append((t["meta"].get("attempt") or 0, t["id"]))
|
||||
by_id = {r["run_id"]: r for r in runs_io.read(sources=["bot"], drop=())}
|
||||
streak = 0 # the longest run of consecutive attempts on one account with the same gated tree route
|
||||
for v in order.values():
|
||||
cur, prev = 0, None
|
||||
for _, i in sorted(v):
|
||||
r = by_id.get(i)
|
||||
s = tuple(r[k] for k in ROUTE + ["sighting_action"]) if r and gated_tree(r) else None
|
||||
cur = cur + 1 if s is not None and s == prev else (1 if s is not None else 0)
|
||||
prev, streak = s, max(streak, cur)
|
||||
botg = [r for r in G if group(r) == "bot"]
|
||||
routes = collections.Counter(tuple(r[k] for k in ROUTE) for r in botg)
|
||||
L += [f"{len(seen)} attempts on {len(acc)} accounts; the earliest attempt logged on any account is number "
|
||||
f"{min((min(v) for v in acc.values() if v), default=0)}. Gated tree runs: {len(botg)}, shadows "
|
||||
f"{sum(bool(r['saw_shadow']) for r in botg)}, on {len(routes)} recorded routes "
|
||||
f"(the commonest {routes.most_common(1)[0][1] if routes else 0} times). The bot has not played a first attempt, "
|
||||
"so it has not yet tested the one situation where riders report the shadow. The most times any account has "
|
||||
f"repeated one route in a row is {streak}.", ""]
|
||||
return "\n".join(L) + "\n"
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--check", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
text = build()
|
||||
if a.check:
|
||||
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
||||
print(f"{OUT} is out of date (run python analysis/black/evidence.py)", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
OUT.write_text(text, encoding="utf-8")
|
||||
print(f"wrote {OUT.relative_to(ROOT)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user