Stable run ids (fingerprints) for the form and the register; tests for the site statistics (#25)
CI / test (pull_request) Successful in 43s

- collection/etl/runs.py fingerprints(): a form or register run's id is now <prefix>-<hash of the row's non-empty
  answers, timestamp included> instead of its row number, so deleting or re-sorting rows in the sheet moves no
  other run's id, and a column added later changes none. The 32 rows that are exact copies of another row get
  -2, -3 (identical, so which is which does not matter). A row edited in the sheet gets a new id. Every id in
  runs_clean.csv and overlap.json changes once; nothing else does (no statistic or model reads the ids).
- tests/core/test_site_stats.py (9 tests): measures.yaml conditions, source groups and presets, bond
  denominators without deliberate deaths (cards, encounters, route map), the riders' paths, and a JSON-safe
  stats.json, on a fixed sample of the published runs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-19 20:16:16 +00:00
co-authored by Claude Opus 5.5
parent ca411e8315
commit 1dfa8fb4c3
7 changed files with 9979 additions and 9818 deletions
+26 -5
View File
@@ -4,7 +4,9 @@ One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label. the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
""" """
import collections
import functools import functools
import hashlib
import json import json
import re import re
from pathlib import Path from pathlib import Path
@@ -51,19 +53,37 @@ def encounter_columns(row, intentional=False):
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:` # ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
def fingerprints(records, prefix):
"""Stable ids for rows that have no id of their own (the form, the register): `<prefix>-<hash of the row's
non-empty answers, timestamp included>`, so deleting or re-sorting rows in the sheet changes no other row's id,
and a column added later (empty in old rows) changes none. Rows that are exact copies of each other get -2, -3...
in the order they come; they are identical, so which copy is which does not matter. A row edited in the sheet
gets a new id: it is a different record."""
seen = collections.Counter()
out = []
for rec in records:
answers = {str(k): v for k, v in rec.items() if v is not None and v != "" and v == v} # v == v: not NaN
h = hashlib.sha1(json.dumps(answers, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")).hexdigest()[:12]
seen[h] += 1
out.append(f"{prefix}-{h}" + (f"-{seen[h]}" if seen[h] > 1 else ""))
return out
def spreadsheet(name, spec): def spreadsheet(name, spec):
"""The community form: core/spec/form.yaml + core/atlas/coding.py, plus the game-rule checks every source that """The community form: core/spec/form.yaml + core/atlas/coding.py, plus the game-rule checks every source that
answers the form's questions gets (rule_flags; a flag the coder already set is not repeated).""" answers the form's questions gets (rule_flags; a flag the coder already set is not repeated)."""
from atlas import coding from atlas import coding
df, missing = coding.code_frame(coding.read_export(path(spec["load"]))) raw = coding.read_export(path(spec["load"]))
df, missing = coding.code_frame(raw)
if missing: if missing:
print(f"WARNING: {name}: questions not found in the sheet: {missing}") print(f"WARNING: {name}: questions not found in the sheet: {missing}")
ids = fingerprints(raw.astype(object).where(raw.notna(), None).to_dict("records"), "sheet")
df["attempt"] = df["attempt"].astype("Int64") df["attempt"] = df["attempt"].astype("Int64")
rows = [] rows = []
for rec in df.astype(object).where(df.notna(), None).to_dict("records"): for i, rec in enumerate(df.astype(object).where(df.notna(), None).to_dict("records")):
n = rec.pop("run_id") rec.pop("run_id") # the coder's row number: shifts when rows are deleted or re-sorted
entry = rec.pop("source") entry = rec.pop("source")
row = {**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry} row = {**{c: None for c in COLUMNS}, **rec, "run_id": ids[i], "source": spec["run_source"], "entry": entry}
have = [f for f in (row["data_flags"] or "").split("; ") if f] have = [f for f in (row["data_flags"] or "").split("; ") if f]
row["data_flags"] = "; ".join(have + [f for f in rule_flags(row) if f not in have]) row["data_flags"] = "; ".join(have + [f for f in rule_flags(row) if f not in have])
rows.append(row) rows.append(row)
@@ -99,6 +119,7 @@ def register(name, spec):
"colour": ("colour",), "tail": ("tail",)}.items()} "colour": ("colour",), "tail": ("tail",)}.items()}
values = {c: DICT["columns"][c].get("values") or [] for c in ("wing", "section", "squad", "outcome_color", "tail")} values = {c: DICT["columns"][c].get("values") or [] for c in ("wing", "section", "squad", "outcome_color", "tail")}
rows = [] rows = []
ids = fingerprints(data["rows"], "register")
for n, r in enumerate(data["rows"], 1): for n, r in enumerate(data["rows"], 1):
get = lambda k: (str(r.get(h[k])).strip() if h[k] and r.get(h[k]) is not None else None) # noqa: E731 get = lambda k: (str(r.get(h[k])).strip() if h[k] and r.get(h[k]) is not None else None) # noqa: E731
flags = ["outcome only (register): the same run may also be in the spreadsheet"] flags = ["outcome only (register): the same run may also be in the spreadsheet"]
@@ -107,7 +128,7 @@ def register(name, spec):
flags.append("colour not recognised") flags.append("colour not recognised")
colour = None colour = None
tail = (get("tail") or "").replace(" ", "").capitalize() or None tail = (get("tail") or "").replace(" ", "").capitalize() or None
row = blank_row(run_id=f"register-{n}", source=spec["run_source"], date=(get("when") or "")[:10] or None, row = blank_row(run_id=ids[n - 1], source=spec["run_source"], date=(get("when") or "")[:10] or None,
outcome="Bonded", outcome_color=colour, tail=tail if tail in values["tail"] else None) outcome="Bonded", outcome_color=colour, tail=tail if tail in values["tail"] else None)
for c in ("wing", "section", "squad"): for c in ("wing", "section", "squad"):
if get(c) and (not values[c] or get(c) in values[c]): # section is free text in the dictionary if get(c) and (not values[c] or get(c) in values[c]): # section is free text in the dictionary
+5 -5
View File
@@ -103,8 +103,8 @@
"description": "Every run from every source, one per row, coded into the same columns (collection/etl/runs.py; the form's rules are core/spec/form.yaml). The `source` column says where each run came from.", "description": "Every run from every source, one per row, coded into the same columns (collection/etl/runs.py; the form's rules are core/spec/form.yaml). The `source` column says where each run came from.",
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/runs_clean.csv", "contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/runs_clean.csv",
"encodingFormat": "text/csv", "encodingFormat": "text/csv",
"sha256": "b5e121cf5deab5f8bc1420b34a5bc434d6f933c7452e200eed4891c35de45122", "sha256": "24321aa853d696b4a06226e28bac5e7843759dc3393721ef020d7f5c6beadf4e",
"contentSize": "3748768 B" "contentSize": "3828261 B"
}, },
{ {
"@type": "cr:FileObject", "@type": "cr:FileObject",
@@ -133,8 +133,8 @@
"description": "How many Reddit comment runs may also be in the community form (the same answers within a day), against what chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.", "description": "How many Reddit comment runs may also be in the community form (the same answers within a day), against what chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.",
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/overlap.json", "contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/overlap.json",
"encodingFormat": "application/json", "encodingFormat": "application/json",
"sha256": "04280a52f028334451c505ddc35004d98cc1f4595b2294c3746a09e34454e8b0", "sha256": "e7511cccf9e8379d8e892639cb38772269ac228756b984e9e0d4d8e9a583a9dd",
"contentSize": "1976 B" "contentSize": "2544 B"
}, },
{ {
"@type": "cr:FileObject", "@type": "cr:FileObject",
@@ -201,7 +201,7 @@
"@type": "cr:Field", "@type": "cr:Field",
"@id": "runs/run_id", "@id": "runs/run_id",
"name": "run_id", "name": "run_id",
"description": "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>.", "description": "Unique id of the run, stable when rows are deleted or re-sorted: sheet-<hash of the form row's answers and timestamp>, register-<hash of the register row>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>.",
"dataType": "sc:Text", "dataType": "sc:Text",
"source": { "source": {
"fileObject": { "fileObject": {
+1 -1
View File
@@ -75,7 +75,7 @@ files:
# The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take # The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take
# (empty cells are always allowed and mean "does not apply" or "not recorded"). # (empty cells are always allowed and mean "does not apply" or "not recorded").
columns: columns:
run_id: { type: text, required: true, description: "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>." } run_id: { type: text, required: true, description: "Unique id of the run, stable when rows are deleted or re-sorted: sheet-<hash of the form row's answers and timestamp>, register-<hash of the register row>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>." }
source: source:
type: text type: text
required: true required: true
+67 -67
View File
@@ -21,113 +21,113 @@
{ {
"comment": "comment-pdjr7gm", "comment": "comment-pdjr7gm",
"form": [ "form": [
"sheet-8", "sheet-e42d078df15a",
"sheet-38" "sheet-d930d7c3533d"
] ]
}, },
{ {
"comment": "comment-pdq6tsw", "comment": "comment-pdq6tsw",
"form": [ "form": [
"sheet-168", "sheet-e588ea82d6f9",
"sheet-190", "sheet-410e2db3f2df",
"sheet-398" "sheet-af47d3ca8ef8"
] ]
}, },
{ {
"comment": "comment-pe0kwol", "comment": "comment-pe0kwol",
"form": [ "form": [
"sheet-224" "sheet-bea7c3c25e85"
] ]
}, },
{ {
"comment": "comment-pe2l6mz", "comment": "comment-pe2l6mz",
"form": [ "form": [
"sheet-5", "sheet-7a526f9f5956",
"sheet-25", "sheet-8191a69798de",
"sheet-107", "sheet-0d56c45763b7",
"sheet-292", "sheet-c51873588c6d",
"sheet-393", "sheet-b8d6d0a2ce4c",
"sheet-471", "sheet-34e592789d1b",
"sheet-483", "sheet-66e8fda4be49",
"sheet-514", "sheet-1922bb0d7397",
"sheet-520" "sheet-37f31b85732d"
] ]
}, },
{ {
"comment": "comment-pe2n3j1", "comment": "comment-pe2n3j1",
"form": [ "form": [
"sheet-195", "sheet-807397c3d7e2",
"sheet-276" "sheet-ff44440c3570"
] ]
}, },
{ {
"comment": "comment-pev3uge", "comment": "comment-pev3uge",
"form": [ "form": [
"sheet-870", "sheet-9c848ba4c9a9",
"sheet-916", "sheet-65c505bb553d",
"sheet-934", "sheet-93077866d5b5",
"sheet-960", "sheet-f4ba657cd34c",
"sheet-961", "sheet-2fa18d22a1eb",
"sheet-962", "sheet-4602a99ee254",
"sheet-975", "sheet-5320d71ea890",
"sheet-984", "sheet-55b901b0c666",
"sheet-999", "sheet-94b17052d8c3",
"sheet-1008", "sheet-8924c5206376",
"sheet-1009", "sheet-81124318fbca",
"sheet-1019", "sheet-1346c5ddbeb6",
"sheet-1023", "sheet-c8670abfd8b3",
"sheet-1068", "sheet-913d430db3ed",
"sheet-1069", "sheet-2fa9c70f091d",
"sheet-1077", "sheet-19e5e9069e3b",
"sheet-1081", "sheet-19ecd172b18a",
"sheet-1089", "sheet-62484835c1c0",
"sheet-1097", "sheet-86e585f7994e",
"sheet-1101", "sheet-ad2265128708",
"sheet-1110", "sheet-f65261b3b1f7",
"sheet-1129", "sheet-cead6ddb0272",
"sheet-1194", "sheet-dd7216ed25e8",
"sheet-1200", "sheet-60a2410b997e",
"sheet-1201", "sheet-93303aaac8a1",
"sheet-1235", "sheet-aac64def431c",
"sheet-1238", "sheet-a62003707841",
"sheet-1257", "sheet-76d43e939528",
"sheet-1259", "sheet-ca945bc0e4f7",
"sheet-1264", "sheet-83270750d93e",
"sheet-1271", "sheet-7cb7c22a757e",
"sheet-1278", "sheet-1598950786a0",
"sheet-1283", "sheet-0d44ea821771",
"sheet-1290", "sheet-cc3275c111d3",
"sheet-1292" "sheet-899c141a899c"
] ]
}, },
{ {
"comment": "comment-pf22tpj", "comment": "comment-pf22tpj",
"form": [ "form": [
"sheet-1031", "sheet-3fdbb179c709",
"sheet-1042", "sheet-08a79a6ae907",
"sheet-1048", "sheet-73d9e41fee8d",
"sheet-1140", "sheet-172fec1dedc4",
"sheet-1142", "sheet-792cc1632e55",
"sheet-1143", "sheet-e97b3363e32f",
"sheet-1144", "sheet-d81d1b4b8c65",
"sheet-1171", "sheet-bcb8e109f973",
"sheet-1261", "sheet-8d6f20b09b5e",
"sheet-1262", "sheet-3848c3443c4e",
"sheet-1303", "sheet-ff87178ef296",
"sheet-1304", "sheet-5819e3882bec",
"sheet-1364" "sheet-aba89201ba1e"
] ]
}, },
{ {
"comment": "comment-pfhmw5d", "comment": "comment-pfhmw5d",
"form": [ "form": [
"sheet-1504" "sheet-b03b79966bdb"
] ]
}, },
{ {
"comment": "comment-pgzz9hc", "comment": "comment-pgzz9hc",
"form": [ "form": [
"sheet-1714" "sheet-344751923fc3"
] ]
} }
] ]
+9740 -9740
View File
File diff suppressed because it is too large Load Diff
+13
View File
@@ -359,6 +359,19 @@ class RedditComments(unittest.TestCase):
self.assertIn("THRESH-X1", text) self.assertIn("THRESH-X1", text)
self.assertTrue(text.startswith(review_reddit.MARKER)) self.assertTrue(text.startswith(review_reddit.MARKER))
def test_fingerprint_ids_are_stable(self):
a = {"Timestamp": "2026-10-04 11:49:07", "Q1": "Alone", "Q2": None}
b = {"Timestamp": "2026-10-04 11:50:00", "Q1": "West"}
dup = {"Timestamp": "Data Import", "Q1": "Alone"}
ids = runs.fingerprints([a, b, dup, dict(dup)], "sheet")
self.assertEqual(len(set(ids)), 4)
self.assertTrue(all(i.startswith("sheet-") for i in ids))
self.assertTrue(ids[3].endswith("-2") and ids[3].startswith(ids[2])) # an exact copy gets -2
self.assertEqual(runs.fingerprints([b, a], "sheet"), [ids[1], ids[0]]) # re-sorting moves no id
self.assertEqual(runs.fingerprints([b], "sheet"), [ids[1]]) # deleting a row moves no other id
self.assertEqual(runs.fingerprints([{**a, "Q9": ""}], "sheet"), [ids[0]]) # a new, empty column changes none
self.assertNotEqual(runs.fingerprints([{**a, "Q1": "West"}], "sheet"), [ids[0]]) # an edited row is new
def test_delivered_not_extracted(self): def test_delivered_not_extracted(self):
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"] spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
with self.assertRaises(extract.Skip): with self.assertRaises(extract.Skip):
+127
View File
@@ -0,0 +1,127 @@
"""The page's statistics (analysis/stats/site_stats.py): the conditions of measures.yaml, how runs are grouped by
where they came from, the presets, deliberate deaths, and the invariants of stats.json on a sample of the real runs."""
import random
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "analysis/stats"))
sys.path.insert(0, str(ROOT / "core"))
import site_stats as SS # noqa: E402
from atlas import runs as runs_io # noqa: E402
DEATHS = ("Incinerated", "Died before the hour")
def sample(per_source=300, seed=7):
"""A fixed sample of the published runs, every statistics source represented, impossible runs left out."""
rng = random.Random(seed)
runs = [r for r in runs_io.read(drop=()) if not runs_io.impossible(r)]
out = []
for s in sorted({r["source"] for r in runs}):
rs = [r for r in runs if r["source"] == s]
out += rng.sample(rs, min(per_source, len(rs)))
return out
class Conditions(unittest.TestCase):
def test_match(self):
r = {"hour": "Tree", "branch": "Shadow (rare)", "outcome": "Bonded", "_deliberate": False}
self.assertTrue(SS.match(r, {"hour": "Tree"}))
self.assertTrue(SS.match(r, {"hour": {"in": ["Tree", "Valley"]}, "branch": {"exists": True}}))
self.assertFalse(SS.match(r, {"hour": {"not": "Tree"}}))
self.assertTrue(SS.match(r, {"any": [{"hour": "Valley"}, {"outcome": "Bonded"}]}))
self.assertFalse(SS.match({**r, "branch": None}, {"branch": {"exists": True}}))
# deliberate deaths: `{not: true}` keeps runs that were never marked (a test row, another caller)
self.assertTrue(SS.match(r, {"_deliberate": {"not": True}}))
self.assertTrue(SS.match({"hour": "Tree"}, {"_deliberate": {"not": True}}))
self.assertFalse(SS.match({**r, "_deliberate": True}, {"_deliberate": {"not": True}}))
def test_bond_pools_leave_out_deliberate_deaths(self):
for t, T in SS.TARGETS.items():
if t.startswith("bond_"):
self.assertEqual(T["pool"].get("_deliberate"), {"not": True}, t)
self.assertEqual(SS.OUTCOMES["bonded_Red"]["denom"].get("_deliberate"), {"not": True})
class Sources(unittest.TestCase):
def test_groups_and_presets(self):
form = {"source": "reddit spreadsheet", "entry": "Form", "goal": "Red", "data_flags": ""}
bulk = {"source": "reddit spreadsheet", "entry": "Data Import", "goal": "Red", "data_flags": ""}
bot = {"source": "bot", "entry": None, "goal": "Black", "data_flags": ""}
com = {"source": "reddit comment", "entry": None, "goal": None, "data_flags": "read from a Reddit comment"}
self.assertEqual([SS.entry_group(r) for r in (form, bulk, bot, com)],
["Form", "Data Import", "DragonKind bot", "Reddit comments"])
self.assertEqual([SS.stratum(r) for r in (bulk, bot, com)], ["Bulk logs (imports)", "DragonKind bot", "Reddit comments"])
runs = [form, bulk, bot, com]
f = lambda **kw: SS.apply_filters(runs, {"source": "all", "goal": "all", "excludeFlagged": False, **kw}) # noqa: E731
self.assertEqual(f(source="form"), [form])
self.assertEqual(f(source="bulk"), [bulk]) # the bot and Reddit are not bulk logs
self.assertEqual(f(source="bot"), [bot])
self.assertEqual(f(source="reddit"), [com])
self.assertEqual(f(goal="only_rare"), [bot])
self.assertEqual(f(excludeFlagged=True), [form, bulk, bot])
class Build(unittest.TestCase):
@classmethod
def setUpClass(cls):
every = sample()
runs = [r for r in every if r["source"] in SS.M["sources"]]
for r in runs:
r["_deliberate"] = SS.deliberate(r)
cls.runs, cls.every = runs, every
cls.out = SS.build(runs, len(runs), None, every_run=every)
def test_presets(self):
self.assertEqual(set(self.out["presets"]), set(SS.M["presets"]))
for pid, p in self.out["presets"].items():
self.assertEqual(set(p), {"label", "analyse", "routes", "map", "valley", "overview"}, pid)
self.assertEqual(set(self.out["presets"]["bot"]["analyse"]["meta"]["sources"]), {"DragonKind bot"})
self.assertEqual(set(self.out["presets"]["reddit"]["analyse"]["meta"]["sources"]), {"Reddit comments"})
def test_overview_counts(self):
o = self.out["presets"]["all"]["overview"]
ended = [r for r in self.runs if r["outcome"] in ("Bonded",) + DEATHS]
self.assertEqual(o["cadets"] + o["deliberate"], len(ended))
self.assertEqual(o["deliberate"], sum(1 for r in ended if r["_deliberate"]))
self.assertEqual(o["bonded"] + o["dead"], o["cadets"])
def test_map_bond_denominators_skip_deliberate_deaths(self):
m = self.out["presets"]["all"]["map"]
for i, nd in enumerate(m["nodes"]):
for t, bt in m["by_target"].items():
n = bt["nodes"][i]["n"]
if t.startswith("bond_"):
self.assertLessEqual(n, nd["n"])
else:
self.assertEqual(n, nd["n"], t)
# somewhere in the sample a deliberate death reached the map, so some bond denominator is smaller
self.assertTrue(any(bt["nodes"][i]["n"] < nd["n"] for t, bt in m["by_target"].items() if t.startswith("bond_")
for i, nd in enumerate(m["nodes"])))
def test_encounters_leave_out_deliberate_deaths(self):
for e, spec in zip(self.out["presets"]["all"]["analyse"]["encounters"], SS.M["encounters"]):
col = spec["column"]
n = sum(1 for r in self.runs if SS.match(r, spec["where"]) and r.get(col) in ("Bonded!", "Incinerated")
and not r["_deliberate"])
self.assertEqual(e["overall"]["n"], n, e["context"])
def test_paths(self):
p = self.out["paths"]
self.assertEqual([lay["key"] for lay in p["layers"]], [lay["key"] for lay in SS.M["path_layers"]])
for lay in p["layers"]:
self.assertEqual(sum(lay["counts"].values()), lay["runs"], lay["key"])
self.assertEqual(set(lay["lines"]), {"lived", "chose", "failed"})
bot = next(lay for lay in p["layers"] if lay["key"] == "bot")
self.assertGreater(bot["counts"]["chose"], 0) # the bot burns on purpose: drawn as chosen deaths
self.assertEqual(set(p["unrouted"]), set(SS.M.get("path_unrouted", [])))
def test_json_safe(self):
import json
json.dumps(self.out, allow_nan=False) # the page cannot parse NaN or Infinity
if __name__ == "__main__":
unittest.main()