Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)

Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 19:46:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcd5117373
commit 432e04c2c7
66 changed files with 201 additions and 2695 deletions
@@ -2,10 +2,10 @@
const test = require("node:test");
const assert = require("node:assert");
const path = require("path");
const { importBot, assertPublicSafe } = require("../packages/ingest/src/bot");
const G = require("../packages/core/src/graph");
const { importBot, assertPublicSafe } = require("../../collection/bot/bot");
const G = require("../../core/graph");
const fx = path.join(__dirname, "fixtures/bot");
const fx = path.join(__dirname, "../fixtures/bot");
const { scenes, trajectories } = importBot({ dataDir: path.join(fx, "data"), logsDir: path.join(fx, "logs"), salt: "test-salt" });
test("one trajectory per attempt, duplicates and repeated burns collapsed", () => {
@@ -1,7 +1,7 @@
// node --test tests/solver.test.js
// node --test tests/core/solver.test.js
const test = require("node:test");
const assert = require("node:assert");
const S = require("../packages/core/src/solver");
const S = require("../../core/solver");
// A synthetic world: the meadow shows Blue 30% of the time, the shadow appears 2% of the time,
// facing the valley Red always bonds it, the Greens burn half the time and bond Brown (rescue) a quarter of the time.
@@ -1,11 +1,11 @@
"""Recompute route rankings in Python (crude Fisher and stratified Mantel-Haenszel) and compare with engine.js."""
import json, subprocess, sys
sys.path.insert(0, "research/python")
sys.path.insert(0, "analysis/stats")
import pandas as pd
from stats_core import odds_ratio, mantel_haenszel, bh
d = pd.DataFrame(json.load(open("tests/out/runs_js.json")))
js = json.loads(subprocess.check_output(["node", "-e", """
const E=require('./packages/core/src/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
const E=require('./core/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
const {runs}=E.codeRows(raw.rows,raw.headers);const o={};for(const t of ['see_Blue','bond_Blue','see_shadow','bond_Green']){o[t]=E.rankRoutes(runs,t,10)}
console.log(JSON.stringify(o));"""]))
MISSING = {"Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)", "(not asked/blank)", "Unknown/lost count", "Blank/not asked"}
@@ -1,4 +1,4 @@
"""Check that the JS engine (packages/core/src/engine.js) and the Python reference (python/clean.py) code every run identically."""
"""Check that the JS engine (core/engine.js) and the Python reference (python/clean.py) code every run identically."""
import json, sys
import pandas as pd
js = pd.DataFrame(json.load(open(sys.argv[1] if len(sys.argv) > 1 else "tests/out/runs_js.json")))
@@ -1,8 +1,8 @@
// node tests/run_engine.js data/public/form_responses.json out_dir
// node tests/parity/run_engine.js data/public/form_responses.json out_dir
// Runs the browser engine in Node and writes coded runs + results, for comparison with Python.
const fs = require("fs");
const path = require("path");
const E = require("../packages/core/src/engine.js");
const E = require("../../core/engine.js");
const [, , src = "data/public/form_responses.json", outDir = "tests/out"] = process.argv;
const raw = JSON.parse(fs.readFileSync(src, "utf8"));
const { runs, missing } = E.codeRows(raw.rows, raw.headers);