Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)
Moves only; no behaviour changes. Every file now lives under the part of the project it belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow. core/ engine.js, graph.js, solver.js, python/clean.py collection/ sources/ (form, register, PDF check, daily update), bot/ (importer) analysis/ stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md) planner/ build-policy.js, policy-report.js site/ atlas/ (the page), build.js, qa/ tests/ core/ (unit), parity/ (JS vs Python), fixtures/ Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and policy.json are byte-identical; the built site differs only in comments that name file paths. The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged. Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now default to a temp folder) and test.txt, and drops the unused npm workspaces. Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
dcd5117373
commit
432e04c2c7
@@ -0,0 +1,37 @@
|
||||
"""Compare factor tables and models between the JS engine and the Python reference."""
|
||||
import json, sys
|
||||
js = json.load(open("tests/out/results_js.json"))
|
||||
py = json.load(open("tests/out/results_py.json"))
|
||||
TOL = 1e-6
|
||||
fails = 0
|
||||
def close(a, b, rel=TOL):
|
||||
if a is None or b is None:
|
||||
return a is None and b is None
|
||||
return abs(a - b) <= rel * max(1, abs(a), abs(b))
|
||||
for key, ft in py["factors"].items():
|
||||
jrows = {(r["factor"], r["level"]): r for r in js["factors"][key]["rows"]}
|
||||
for r in ft["rows"]:
|
||||
j = jrows.get((r["factor"], r["level"]))
|
||||
if j is None:
|
||||
print("missing in JS:", key, r["factor"], r["level"]); fails += 1; continue
|
||||
for f in ("k", "n", "p", "lo", "hi", "or", "or_lo", "or_hi", "fisher_p", "q"):
|
||||
if (f in r) != (f in j) or (f in r and not close(r[f], j[f], 1e-5)):
|
||||
print("diff", key, r["factor"], r["level"], f, r.get(f), j.get(f)); fails += 1
|
||||
for key, m in py["models"].items():
|
||||
jm = js["models"][key]
|
||||
if m["n"] != jm["n"] or m["events"] != jm["events"]:
|
||||
print("model size diff", key); fails += 1
|
||||
jt = {t["term"]: t for t in jm["terms"]}
|
||||
for t in m["terms"]:
|
||||
j = jt.get(t["term"])
|
||||
if not j:
|
||||
print("term missing", key, t["term"]); fails += 1; continue
|
||||
for f in ("or", "lo", "hi"):
|
||||
if not close(t[f], j[f], 1e-4):
|
||||
print("model diff", key, t["term"], f, t[f], j[f]); fails += 1
|
||||
if abs(t["p"] - j["p"]) > 1e-6:
|
||||
print("model p diff", key, t["term"], t["p"], j["p"]); fails += 1
|
||||
n_rows = sum(len(f["rows"]) for f in py["factors"].values())
|
||||
n_terms = sum(len(m["terms"]) for m in py["models"].values())
|
||||
print(f"checked {n_rows} factor rows and {n_terms} model terms: {'ALL MATCH' if not fails else str(fails) + ' differences'}")
|
||||
sys.exit(1 if fails else 0)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Recompute route rankings in Python (crude Fisher and stratified Mantel-Haenszel) and compare with engine.js."""
|
||||
import json, subprocess, sys
|
||||
sys.path.insert(0, "analysis/stats")
|
||||
import pandas as pd
|
||||
from stats_core import odds_ratio, mantel_haenszel, bh
|
||||
d = pd.DataFrame(json.load(open("tests/out/runs_js.json")))
|
||||
js = json.loads(subprocess.check_output(["node", "-e", """
|
||||
const E=require('./core/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
|
||||
const {runs}=E.codeRows(raw.rows,raw.headers);const o={};for(const t of ['see_Blue','bond_Blue','see_shadow','bond_Green']){o[t]=E.rankRoutes(runs,t,10)}
|
||||
console.log(JSON.stringify(o));"""]))
|
||||
MISSING = {"Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)", "(not asked/blank)", "Unknown/lost count", "Blank/not asked"}
|
||||
miss = lambda v: not isinstance(v, str) or v == "" or v in MISSING # noqa: E731
|
||||
def stage(r):
|
||||
# a route is only ranked from runs whose every step is known (mirrors rankRoutes in engine.js)
|
||||
if r.scream.startswith("DIED") or miss(r.scream) or miss(r.resolution): cad = None # noqa
|
||||
else: cad = ("Help → " + r.resolution) if r.scream == "Investigate & help" else (r.scream + " → " + r.resolution)
|
||||
hour = {"Tree": "Climb the tree", "Valley": "Valley centre"}.get(r.hour)
|
||||
fs = None if r.scream == "DIED at first sighting" or miss(r.first_sighting) else r.first_sighting
|
||||
post_null = not isinstance(r.post_path, str)
|
||||
post = None if post_null or miss(r.post_path) else r.post_path
|
||||
vals = [None if miss(r.company) else r.company, None if miss(r.direction) else r.direction, fs, cad, post, hour]
|
||||
if any(v is None for i, v in enumerate(vals) if not (i == 4 and post_null)): return None
|
||||
return " › ".join(v if v else "—" for v in vals)
|
||||
def stratum(r):
|
||||
if r.source != "Form": return "bulk"
|
||||
return {"Blue": "blue", "Black": "black", "Blue or Black": "blue-or-black"}.get(r.goal, "other")
|
||||
d["route"] = d.apply(stage, axis=1); d["stratum"] = d.apply(stratum, axis=1)
|
||||
d["branch"] = d.branch.where(d.branch.notna(), None)
|
||||
T = {"see_Blue": (lambda x: x.branch == "Passed the Red: meadow", lambda x: x.saw_blue_meadow),
|
||||
"bond_Blue": (lambda x: x.hour == "Valley", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Blue")),
|
||||
"see_shadow": (lambda x: (x.hour == "Tree") & x.branch.notna(), lambda x: x.saw_shadow),
|
||||
"bond_Green": (lambda x: x.hour == "Tree", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Green"))}
|
||||
fails = 0
|
||||
for t, (pool, ev) in T.items():
|
||||
P = d[pool(d)].copy(); P["e"] = ev(P).astype(bool)
|
||||
K, N = int(P.e.sum()), len(P)
|
||||
rows = {}
|
||||
for route, g in P[P.route.notna()].groupby("route"):
|
||||
if len(g) < 10: continue
|
||||
k, n = int(g.e.sum()), len(g)
|
||||
orr = odds_ratio(k, n - k, K - k, N - n - (K - k))
|
||||
tabs = []
|
||||
for s, sg in P.groupby("stratum"):
|
||||
inr = sg.route == route
|
||||
a = int((inr & sg.e).sum()); nin = int(inr.sum()); c = int((~inr & sg.e).sum()); nout = len(sg) - nin
|
||||
tabs.append([a, nin - a, c, nout - c])
|
||||
rows[route] = (k, n, orr, mantel_haenszel(tabs))
|
||||
jr = {r["route"]: r for r in js[t]["rows"]}
|
||||
if set(jr) != set(rows): print(t, "route sets differ", set(jr) ^ set(rows)); fails += 1
|
||||
for route, (k, n, orr, mh) in rows.items():
|
||||
j = jr.get(route)
|
||||
if not j: continue
|
||||
if (k, n) != (j["k"], j["n"]) or abs(orr[0] - j["or"]) > 1e-6 * max(1, orr[0]) or abs(orr[3] - j["fisher_p"]) > 1e-6:
|
||||
print("crude diff", t, route); fails += 1
|
||||
if mh is None:
|
||||
if j["mh_or"] is not None: print("mh none diff", t, route); fails += 1
|
||||
elif j["mh_or"] is None or abs(mh[0] - j["mh_or"]) > 1e-6 * max(1, mh[0]) or abs(mh[3] - j["mh_p"]) > 1e-5:
|
||||
print("mh diff", t, route, mh, j["mh_or"], j["mh_p"]); fails += 1
|
||||
print(f"route rankings for {len(T)} outcomes: {'ALL MATCH' if not fails else str(fails) + ' differences'}")
|
||||
sys.exit(1 if fails else 0)
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Check that the JS engine (core/engine.js) and the Python reference (python/clean.py) code every run identically."""
|
||||
import json, sys
|
||||
import pandas as pd
|
||||
js = pd.DataFrame(json.load(open(sys.argv[1] if len(sys.argv) > 1 else "tests/out/runs_js.json")))
|
||||
py = pd.read_csv(sys.argv[2] if len(sys.argv) > 2 else "data/derived/runs_clean.csv", keep_default_na=False, na_values=[""])
|
||||
def norm(v):
|
||||
if v is None or (not isinstance(v, str) and pd.isna(v)) or v == "":
|
||||
return None
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return str(int(v))
|
||||
if isinstance(v, bool):
|
||||
return str(v)
|
||||
return str(v)
|
||||
bad = {}
|
||||
for c in py.columns:
|
||||
a, b = [norm(v) for v in py[c]], [norm(v) for v in js[c]]
|
||||
d = [(i, x, y) for i, (x, y) in enumerate(zip(a, b)) if x != y]
|
||||
if d:
|
||||
bad[c] = (len(d), d[:3])
|
||||
print("runs:", len(py), len(js), "| mismatching columns:", bad or "NONE")
|
||||
sys.exit(1 if bad else 0)
|
||||
@@ -0,0 +1,13 @@
|
||||
// node tests/parity/run_engine.js data/public/form_responses.json out_dir
|
||||
// Runs the browser engine in Node and writes coded runs + results, for comparison with Python.
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const E = require("../../core/engine.js");
|
||||
const [, , src = "data/public/form_responses.json", outDir = "tests/out"] = process.argv;
|
||||
const raw = JSON.parse(fs.readFileSync(src, "utf8"));
|
||||
const { runs, missing } = E.codeRows(raw.rows, raw.headers);
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(outDir, "runs_js.json"), JSON.stringify(runs));
|
||||
const res = E.analyse(runs, {});
|
||||
fs.writeFileSync(path.join(outDir, "results_js.json"), JSON.stringify(res, null, 1));
|
||||
console.log(`${runs.length} runs coded; missing columns: ${missing.join(", ") || "none"}`);
|
||||
Reference in New Issue
Block a user