Merge pull request 'Stable run ids (fingerprints) for the form and the register; tests for the site statistics' (#46) from atlas/fingerprint-ids into main
Publish the site (pages branch) / publish (push) Successful in 5s
CI / test (push) Successful in 2m3s

This commit was merged in pull request #46.
This commit is contained in:
2026-11-19 20:23:21 +00:00
7 changed files with 9979 additions and 9818 deletions
+26 -5
View File
@@ -4,7 +4,9 @@ One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
"""
import collections
import functools
import hashlib
import json
import re
from pathlib import Path
@@ -51,19 +53,37 @@ def encounter_columns(row, intentional=False):
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
def fingerprints(records, prefix):
"""Stable ids for rows that have no id of their own (the form, the register): `<prefix>-<hash of the row's
non-empty answers, timestamp included>`, so deleting or re-sorting rows in the sheet changes no other row's id,
and a column added later (empty in old rows) changes none. Rows that are exact copies of each other get -2, -3...
in the order they come; they are identical, so which copy is which does not matter. A row edited in the sheet
gets a new id: it is a different record."""
seen = collections.Counter()
out = []
for rec in records:
answers = {str(k): v for k, v in rec.items() if v is not None and v != "" and v == v} # v == v: not NaN
h = hashlib.sha1(json.dumps(answers, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")).hexdigest()[:12]
seen[h] += 1
out.append(f"{prefix}-{h}" + (f"-{seen[h]}" if seen[h] > 1 else ""))
return out
def spreadsheet(name, spec):
"""The community form: core/spec/form.yaml + core/atlas/coding.py, plus the game-rule checks every source that
answers the form's questions gets (rule_flags; a flag the coder already set is not repeated)."""
from atlas import coding
df, missing = coding.code_frame(coding.read_export(path(spec["load"])))
raw = coding.read_export(path(spec["load"]))
df, missing = coding.code_frame(raw)
if missing:
print(f"WARNING: {name}: questions not found in the sheet: {missing}")
ids = fingerprints(raw.astype(object).where(raw.notna(), None).to_dict("records"), "sheet")
df["attempt"] = df["attempt"].astype("Int64")
rows = []
for rec in df.astype(object).where(df.notna(), None).to_dict("records"):
n = rec.pop("run_id")
for i, rec in enumerate(df.astype(object).where(df.notna(), None).to_dict("records")):
rec.pop("run_id") # the coder's row number: shifts when rows are deleted or re-sorted
entry = rec.pop("source")
row = {**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry}
row = {**{c: None for c in COLUMNS}, **rec, "run_id": ids[i], "source": spec["run_source"], "entry": entry}
have = [f for f in (row["data_flags"] or "").split("; ") if f]
row["data_flags"] = "; ".join(have + [f for f in rule_flags(row) if f not in have])
rows.append(row)
@@ -99,6 +119,7 @@ def register(name, spec):
"colour": ("colour",), "tail": ("tail",)}.items()}
values = {c: DICT["columns"][c].get("values") or [] for c in ("wing", "section", "squad", "outcome_color", "tail")}
rows = []
ids = fingerprints(data["rows"], "register")
for n, r in enumerate(data["rows"], 1):
get = lambda k: (str(r.get(h[k])).strip() if h[k] and r.get(h[k]) is not None else None) # noqa: E731
flags = ["outcome only (register): the same run may also be in the spreadsheet"]
@@ -107,7 +128,7 @@ def register(name, spec):
flags.append("colour not recognised")
colour = None
tail = (get("tail") or "").replace(" ", "").capitalize() or None
row = blank_row(run_id=f"register-{n}", source=spec["run_source"], date=(get("when") or "")[:10] or None,
row = blank_row(run_id=ids[n - 1], source=spec["run_source"], date=(get("when") or "")[:10] or None,
outcome="Bonded", outcome_color=colour, tail=tail if tail in values["tail"] else None)
for c in ("wing", "section", "squad"):
if get(c) and (not values[c] or get(c) in values[c]): # section is free text in the dictionary
+5 -5
View File
@@ -103,8 +103,8 @@
"description": "Every run from every source, one per row, coded into the same columns (collection/etl/runs.py; the form's rules are core/spec/form.yaml). The `source` column says where each run came from.",
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/runs_clean.csv",
"encodingFormat": "text/csv",
"sha256": "b5e121cf5deab5f8bc1420b34a5bc434d6f933c7452e200eed4891c35de45122",
"contentSize": "3748768 B"
"sha256": "24321aa853d696b4a06226e28bac5e7843759dc3393721ef020d7f5c6beadf4e",
"contentSize": "3828261 B"
},
{
"@type": "cr:FileObject",
@@ -133,8 +133,8 @@
"description": "How many Reddit comment runs may also be in the community form (the same answers within a day), against what chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.",
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/overlap.json",
"encodingFormat": "application/json",
"sha256": "04280a52f028334451c505ddc35004d98cc1f4595b2294c3746a09e34454e8b0",
"contentSize": "1976 B"
"sha256": "e7511cccf9e8379d8e892639cb38772269ac228756b984e9e0d4d8e9a583a9dd",
"contentSize": "2544 B"
},
{
"@type": "cr:FileObject",
@@ -201,7 +201,7 @@
"@type": "cr:Field",
"@id": "runs/run_id",
"name": "run_id",
"description": "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>.",
"description": "Unique id of the run, stable when rows are deleted or re-sorted: sheet-<hash of the form row's answers and timestamp>, register-<hash of the register row>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>.",
"dataType": "sc:Text",
"source": {
"fileObject": {
+1 -1
View File
@@ -75,7 +75,7 @@ files:
# The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take
# (empty cells are always allowed and mean "does not apply" or "not recorded").
columns:
run_id: { type: text, required: true, description: "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>." }
run_id: { type: text, required: true, description: "Unique id of the run, stable when rows are deleted or re-sorted: sheet-<hash of the form row's answers and timestamp>, register-<hash of the register row>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>." }
source:
type: text
required: true
+67 -67
View File
@@ -21,113 +21,113 @@
{
"comment": "comment-pdjr7gm",
"form": [
"sheet-8",
"sheet-38"
"sheet-e42d078df15a",
"sheet-d930d7c3533d"
]
},
{
"comment": "comment-pdq6tsw",
"form": [
"sheet-168",
"sheet-190",
"sheet-398"
"sheet-e588ea82d6f9",
"sheet-410e2db3f2df",
"sheet-af47d3ca8ef8"
]
},
{
"comment": "comment-pe0kwol",
"form": [
"sheet-224"
"sheet-bea7c3c25e85"
]
},
{
"comment": "comment-pe2l6mz",
"form": [
"sheet-5",
"sheet-25",
"sheet-107",
"sheet-292",
"sheet-393",
"sheet-471",
"sheet-483",
"sheet-514",
"sheet-520"
"sheet-7a526f9f5956",
"sheet-8191a69798de",
"sheet-0d56c45763b7",
"sheet-c51873588c6d",
"sheet-b8d6d0a2ce4c",
"sheet-34e592789d1b",
"sheet-66e8fda4be49",
"sheet-1922bb0d7397",
"sheet-37f31b85732d"
]
},
{
"comment": "comment-pe2n3j1",
"form": [
"sheet-195",
"sheet-276"
"sheet-807397c3d7e2",
"sheet-ff44440c3570"
]
},
{
"comment": "comment-pev3uge",
"form": [
"sheet-870",
"sheet-916",
"sheet-934",
"sheet-960",
"sheet-961",
"sheet-962",
"sheet-975",
"sheet-984",
"sheet-999",
"sheet-1008",
"sheet-1009",
"sheet-1019",
"sheet-1023",
"sheet-1068",
"sheet-1069",
"sheet-1077",
"sheet-1081",
"sheet-1089",
"sheet-1097",
"sheet-1101",
"sheet-1110",
"sheet-1129",
"sheet-1194",
"sheet-1200",
"sheet-1201",
"sheet-1235",
"sheet-1238",
"sheet-1257",
"sheet-1259",
"sheet-1264",
"sheet-1271",
"sheet-1278",
"sheet-1283",
"sheet-1290",
"sheet-1292"
"sheet-9c848ba4c9a9",
"sheet-65c505bb553d",
"sheet-93077866d5b5",
"sheet-f4ba657cd34c",
"sheet-2fa18d22a1eb",
"sheet-4602a99ee254",
"sheet-5320d71ea890",
"sheet-55b901b0c666",
"sheet-94b17052d8c3",
"sheet-8924c5206376",
"sheet-81124318fbca",
"sheet-1346c5ddbeb6",
"sheet-c8670abfd8b3",
"sheet-913d430db3ed",
"sheet-2fa9c70f091d",
"sheet-19e5e9069e3b",
"sheet-19ecd172b18a",
"sheet-62484835c1c0",
"sheet-86e585f7994e",
"sheet-ad2265128708",
"sheet-f65261b3b1f7",
"sheet-cead6ddb0272",
"sheet-dd7216ed25e8",
"sheet-60a2410b997e",
"sheet-93303aaac8a1",
"sheet-aac64def431c",
"sheet-a62003707841",
"sheet-76d43e939528",
"sheet-ca945bc0e4f7",
"sheet-83270750d93e",
"sheet-7cb7c22a757e",
"sheet-1598950786a0",
"sheet-0d44ea821771",
"sheet-cc3275c111d3",
"sheet-899c141a899c"
]
},
{
"comment": "comment-pf22tpj",
"form": [
"sheet-1031",
"sheet-1042",
"sheet-1048",
"sheet-1140",
"sheet-1142",
"sheet-1143",
"sheet-1144",
"sheet-1171",
"sheet-1261",
"sheet-1262",
"sheet-1303",
"sheet-1304",
"sheet-1364"
"sheet-3fdbb179c709",
"sheet-08a79a6ae907",
"sheet-73d9e41fee8d",
"sheet-172fec1dedc4",
"sheet-792cc1632e55",
"sheet-e97b3363e32f",
"sheet-d81d1b4b8c65",
"sheet-bcb8e109f973",
"sheet-8d6f20b09b5e",
"sheet-3848c3443c4e",
"sheet-ff87178ef296",
"sheet-5819e3882bec",
"sheet-aba89201ba1e"
]
},
{
"comment": "comment-pfhmw5d",
"form": [
"sheet-1504"
"sheet-b03b79966bdb"
]
},
{
"comment": "comment-pgzz9hc",
"form": [
"sheet-1714"
"sheet-344751923fc3"
]
}
]
+9740 -9740
View File
File diff suppressed because it is too large Load Diff
+13
View File
@@ -359,6 +359,19 @@ class RedditComments(unittest.TestCase):
self.assertIn("THRESH-X1", text)
self.assertTrue(text.startswith(review_reddit.MARKER))
def test_fingerprint_ids_are_stable(self):
a = {"Timestamp": "2026-10-04 11:49:07", "Q1": "Alone", "Q2": None}
b = {"Timestamp": "2026-10-04 11:50:00", "Q1": "West"}
dup = {"Timestamp": "Data Import", "Q1": "Alone"}
ids = runs.fingerprints([a, b, dup, dict(dup)], "sheet")
self.assertEqual(len(set(ids)), 4)
self.assertTrue(all(i.startswith("sheet-") for i in ids))
self.assertTrue(ids[3].endswith("-2") and ids[3].startswith(ids[2])) # an exact copy gets -2
self.assertEqual(runs.fingerprints([b, a], "sheet"), [ids[1], ids[0]]) # re-sorting moves no id
self.assertEqual(runs.fingerprints([b], "sheet"), [ids[1]]) # deleting a row moves no other id
self.assertEqual(runs.fingerprints([{**a, "Q9": ""}], "sheet"), [ids[0]]) # a new, empty column changes none
self.assertNotEqual(runs.fingerprints([{**a, "Q1": "West"}], "sheet"), [ids[0]]) # an edited row is new
def test_delivered_not_extracted(self):
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
with self.assertRaises(extract.Skip):
+127
View File
@@ -0,0 +1,127 @@
"""The page's statistics (analysis/stats/site_stats.py): the conditions of measures.yaml, how runs are grouped by
where they came from, the presets, deliberate deaths, and the invariants of stats.json on a sample of the real runs."""
import random
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "analysis/stats"))
sys.path.insert(0, str(ROOT / "core"))
import site_stats as SS # noqa: E402
from atlas import runs as runs_io # noqa: E402
DEATHS = ("Incinerated", "Died before the hour")
def sample(per_source=300, seed=7):
"""A fixed sample of the published runs, every statistics source represented, impossible runs left out."""
rng = random.Random(seed)
runs = [r for r in runs_io.read(drop=()) if not runs_io.impossible(r)]
out = []
for s in sorted({r["source"] for r in runs}):
rs = [r for r in runs if r["source"] == s]
out += rng.sample(rs, min(per_source, len(rs)))
return out
class Conditions(unittest.TestCase):
def test_match(self):
r = {"hour": "Tree", "branch": "Shadow (rare)", "outcome": "Bonded", "_deliberate": False}
self.assertTrue(SS.match(r, {"hour": "Tree"}))
self.assertTrue(SS.match(r, {"hour": {"in": ["Tree", "Valley"]}, "branch": {"exists": True}}))
self.assertFalse(SS.match(r, {"hour": {"not": "Tree"}}))
self.assertTrue(SS.match(r, {"any": [{"hour": "Valley"}, {"outcome": "Bonded"}]}))
self.assertFalse(SS.match({**r, "branch": None}, {"branch": {"exists": True}}))
# deliberate deaths: `{not: true}` keeps runs that were never marked (a test row, another caller)
self.assertTrue(SS.match(r, {"_deliberate": {"not": True}}))
self.assertTrue(SS.match({"hour": "Tree"}, {"_deliberate": {"not": True}}))
self.assertFalse(SS.match({**r, "_deliberate": True}, {"_deliberate": {"not": True}}))
def test_bond_pools_leave_out_deliberate_deaths(self):
for t, T in SS.TARGETS.items():
if t.startswith("bond_"):
self.assertEqual(T["pool"].get("_deliberate"), {"not": True}, t)
self.assertEqual(SS.OUTCOMES["bonded_Red"]["denom"].get("_deliberate"), {"not": True})
class Sources(unittest.TestCase):
def test_groups_and_presets(self):
form = {"source": "reddit spreadsheet", "entry": "Form", "goal": "Red", "data_flags": ""}
bulk = {"source": "reddit spreadsheet", "entry": "Data Import", "goal": "Red", "data_flags": ""}
bot = {"source": "bot", "entry": None, "goal": "Black", "data_flags": ""}
com = {"source": "reddit comment", "entry": None, "goal": None, "data_flags": "read from a Reddit comment"}
self.assertEqual([SS.entry_group(r) for r in (form, bulk, bot, com)],
["Form", "Data Import", "DragonKind bot", "Reddit comments"])
self.assertEqual([SS.stratum(r) for r in (bulk, bot, com)], ["Bulk logs (imports)", "DragonKind bot", "Reddit comments"])
runs = [form, bulk, bot, com]
f = lambda **kw: SS.apply_filters(runs, {"source": "all", "goal": "all", "excludeFlagged": False, **kw}) # noqa: E731
self.assertEqual(f(source="form"), [form])
self.assertEqual(f(source="bulk"), [bulk]) # the bot and Reddit are not bulk logs
self.assertEqual(f(source="bot"), [bot])
self.assertEqual(f(source="reddit"), [com])
self.assertEqual(f(goal="only_rare"), [bot])
self.assertEqual(f(excludeFlagged=True), [form, bulk, bot])
class Build(unittest.TestCase):
@classmethod
def setUpClass(cls):
every = sample()
runs = [r for r in every if r["source"] in SS.M["sources"]]
for r in runs:
r["_deliberate"] = SS.deliberate(r)
cls.runs, cls.every = runs, every
cls.out = SS.build(runs, len(runs), None, every_run=every)
def test_presets(self):
self.assertEqual(set(self.out["presets"]), set(SS.M["presets"]))
for pid, p in self.out["presets"].items():
self.assertEqual(set(p), {"label", "analyse", "routes", "map", "valley", "overview"}, pid)
self.assertEqual(set(self.out["presets"]["bot"]["analyse"]["meta"]["sources"]), {"DragonKind bot"})
self.assertEqual(set(self.out["presets"]["reddit"]["analyse"]["meta"]["sources"]), {"Reddit comments"})
def test_overview_counts(self):
o = self.out["presets"]["all"]["overview"]
ended = [r for r in self.runs if r["outcome"] in ("Bonded",) + DEATHS]
self.assertEqual(o["cadets"] + o["deliberate"], len(ended))
self.assertEqual(o["deliberate"], sum(1 for r in ended if r["_deliberate"]))
self.assertEqual(o["bonded"] + o["dead"], o["cadets"])
def test_map_bond_denominators_skip_deliberate_deaths(self):
m = self.out["presets"]["all"]["map"]
for i, nd in enumerate(m["nodes"]):
for t, bt in m["by_target"].items():
n = bt["nodes"][i]["n"]
if t.startswith("bond_"):
self.assertLessEqual(n, nd["n"])
else:
self.assertEqual(n, nd["n"], t)
# somewhere in the sample a deliberate death reached the map, so some bond denominator is smaller
self.assertTrue(any(bt["nodes"][i]["n"] < nd["n"] for t, bt in m["by_target"].items() if t.startswith("bond_")
for i, nd in enumerate(m["nodes"])))
def test_encounters_leave_out_deliberate_deaths(self):
for e, spec in zip(self.out["presets"]["all"]["analyse"]["encounters"], SS.M["encounters"]):
col = spec["column"]
n = sum(1 for r in self.runs if SS.match(r, spec["where"]) and r.get(col) in ("Bonded!", "Incinerated")
and not r["_deliberate"])
self.assertEqual(e["overall"]["n"], n, e["context"])
def test_paths(self):
p = self.out["paths"]
self.assertEqual([lay["key"] for lay in p["layers"]], [lay["key"] for lay in SS.M["path_layers"]])
for lay in p["layers"]:
self.assertEqual(sum(lay["counts"].values()), lay["runs"], lay["key"])
self.assertEqual(set(lay["lines"]), {"lived", "chose", "failed"})
bot = next(lay for lay in p["layers"] if lay["key"] == "bot")
self.assertGreater(bot["counts"]["chose"], 0) # the bot burns on purpose: drawn as chosen deaths
self.assertEqual(set(p["unrouted"]), set(SS.M.get("path_unrouted", [])))
def test_json_safe(self):
import json
json.dumps(self.out, allow_nan=False) # the page cannot parse NaN or Infinity
if __name__ == "__main__":
unittest.main()