Stable run ids (fingerprints) for the form and the register; tests for the site statistics (#25)
CI / test (pull_request) Successful in 43s
CI / test (pull_request) Successful in 43s
- collection/etl/runs.py fingerprints(): a form or register run's id is now <prefix>-<hash of the row's non-empty answers, timestamp included> instead of its row number, so deleting or re-sorting rows in the sheet moves no other run's id, and a column added later changes none. The 32 rows that are exact copies of another row get -2, -3 (identical, so which is which does not matter). A row edited in the sheet gets a new id. Every id in runs_clean.csv and overlap.json changes once; nothing else does (no statistic or model reads the ids). - tests/core/test_site_stats.py (9 tests): measures.yaml conditions, source groups and presets, bond denominators without deliberate deaths (cards, encounters, route map), the riders' paths, and a JSON-safe stats.json, on a fixed sample of the published runs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ca411e8315
commit
1dfa8fb4c3
+26
-5
@@ -4,7 +4,9 @@ One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by
|
||||
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
|
||||
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
|
||||
"""
|
||||
import collections
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -51,19 +53,37 @@ def encounter_columns(row, intentional=False):
|
||||
|
||||
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
|
||||
|
||||
def fingerprints(records, prefix):
|
||||
"""Stable ids for rows that have no id of their own (the form, the register): `<prefix>-<hash of the row's
|
||||
non-empty answers, timestamp included>`, so deleting or re-sorting rows in the sheet changes no other row's id,
|
||||
and a column added later (empty in old rows) changes none. Rows that are exact copies of each other get -2, -3...
|
||||
in the order they come; they are identical, so which copy is which does not matter. A row edited in the sheet
|
||||
gets a new id: it is a different record."""
|
||||
seen = collections.Counter()
|
||||
out = []
|
||||
for rec in records:
|
||||
answers = {str(k): v for k, v in rec.items() if v is not None and v != "" and v == v} # v == v: not NaN
|
||||
h = hashlib.sha1(json.dumps(answers, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")).hexdigest()[:12]
|
||||
seen[h] += 1
|
||||
out.append(f"{prefix}-{h}" + (f"-{seen[h]}" if seen[h] > 1 else ""))
|
||||
return out
|
||||
|
||||
|
||||
def spreadsheet(name, spec):
|
||||
"""The community form: core/spec/form.yaml + core/atlas/coding.py, plus the game-rule checks every source that
|
||||
answers the form's questions gets (rule_flags; a flag the coder already set is not repeated)."""
|
||||
from atlas import coding
|
||||
df, missing = coding.code_frame(coding.read_export(path(spec["load"])))
|
||||
raw = coding.read_export(path(spec["load"]))
|
||||
df, missing = coding.code_frame(raw)
|
||||
if missing:
|
||||
print(f"WARNING: {name}: questions not found in the sheet: {missing}")
|
||||
ids = fingerprints(raw.astype(object).where(raw.notna(), None).to_dict("records"), "sheet")
|
||||
df["attempt"] = df["attempt"].astype("Int64")
|
||||
rows = []
|
||||
for rec in df.astype(object).where(df.notna(), None).to_dict("records"):
|
||||
n = rec.pop("run_id")
|
||||
for i, rec in enumerate(df.astype(object).where(df.notna(), None).to_dict("records")):
|
||||
rec.pop("run_id") # the coder's row number: shifts when rows are deleted or re-sorted
|
||||
entry = rec.pop("source")
|
||||
row = {**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry}
|
||||
row = {**{c: None for c in COLUMNS}, **rec, "run_id": ids[i], "source": spec["run_source"], "entry": entry}
|
||||
have = [f for f in (row["data_flags"] or "").split("; ") if f]
|
||||
row["data_flags"] = "; ".join(have + [f for f in rule_flags(row) if f not in have])
|
||||
rows.append(row)
|
||||
@@ -99,6 +119,7 @@ def register(name, spec):
|
||||
"colour": ("colour",), "tail": ("tail",)}.items()}
|
||||
values = {c: DICT["columns"][c].get("values") or [] for c in ("wing", "section", "squad", "outcome_color", "tail")}
|
||||
rows = []
|
||||
ids = fingerprints(data["rows"], "register")
|
||||
for n, r in enumerate(data["rows"], 1):
|
||||
get = lambda k: (str(r.get(h[k])).strip() if h[k] and r.get(h[k]) is not None else None) # noqa: E731
|
||||
flags = ["outcome only (register): the same run may also be in the spreadsheet"]
|
||||
@@ -107,7 +128,7 @@ def register(name, spec):
|
||||
flags.append("colour not recognised")
|
||||
colour = None
|
||||
tail = (get("tail") or "").replace(" ", "").capitalize() or None
|
||||
row = blank_row(run_id=f"register-{n}", source=spec["run_source"], date=(get("when") or "")[:10] or None,
|
||||
row = blank_row(run_id=ids[n - 1], source=spec["run_source"], date=(get("when") or "")[:10] or None,
|
||||
outcome="Bonded", outcome_color=colour, tail=tail if tail in values["tail"] else None)
|
||||
for c in ("wing", "section", "squad"):
|
||||
if get(c) and (not values[c] or get(c) in values[c]): # section is free text in the dictionary
|
||||
|
||||
Reference in New Issue
Block a user