Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)

Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 19:46:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcd5117373
commit 432e04c2c7
66 changed files with 201 additions and 2695 deletions
+846
View File
@@ -0,0 +1,846 @@
/*
* engine.js — Threshing route analysis engine.
*
* One file, used two ways:
* - in the browser (the route map page loads it and recomputes everything from the raw form export)
* - in Node (`node tests/parity/run_engine.js data.json`), so results can be reproduced and checked against
* the Python reference implementation in python/.
*
* Pipeline: raw form rows -> codeRow() (one coded run per row) -> analyse() -> results object
* coded runs + a target outcome -> buildGraph() -> nodes/links for the route map
*
* Every rule here is described in plain language in the Methods section of the page.
*/
(function (root, factory) {
if (typeof module === "object" && module.exports) module.exports = factory();
else root.ThreshEngine = factory();
})(typeof self !== "undefined" ? self : this, function () {
"use strict";
// Free-text answers sometimes contain a respondent's email address; never display or export it.
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
const redact = (v) => (v == null ? v : String(v).replace(EMAIL_RE, "[email removed]"));
const COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"];
// Answers that carry no information about the choice ("Unsure", blank, a question the form did not ask yet,
// conflicting answers). They are kept in the coded data for transparency, but every analysis treats them as
// missing: they are never shown or compared as if they were a choice.
const MISSING = new Set(["Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)",
"(not asked/blank)", "Unknown/lost count", "Blank/not asked"]);
const isMissing = (v) => v === null || v === undefined || v === "" || MISSING.has(v);
// ---------------------------------------------------------------- column detection
const COLUMN_KEYS = {
timestamp: ["timestamp"],
wing_First: ["wing and section", "[first]"],
wing_Second: ["wing and section", "[second]"],
wing_Third: ["wing and section", "[third]"],
wing_Fourth: ["wing and section", "[fourth]"],
squad: ["what squad"],
attempt: ["attempt number"],
company: ["who accompanied"],
dir_East: ["initial direction", "[east"],
dir_West: ["initial direction", "[west"],
dir_Center: ["initial direction", "[forest center"],
cadet: ["outcome of your cadet encounter"],
post: ["path did you take after the bully"],
hour: ["after walking an hour"],
enc_Red: ["attempt to bond", "[red"],
enc_Green: ["attempt to bond", "[green"],
enc_Orange: ["attempt to bond", "[orange"],
enc_Brown: ["attempt to bond", "[brown"],
enc_Blue: ["attempt to bond", "[blue"],
enc_Black: ["attempt to bond", "[black"],
death_note: ["those who died"],
bond_note: ["those who bonded"],
goal: ["colour dragon are you trying"],
combat: ["combat choice"],
// Questions added in the redesigned form (late October 2026). "!x" means the header must NOT contain x.
first_dir: ["what was your first direction"],
first_colour: ["colour was your initial dragon sighting", "!["],
leg_West: ["colour was your initial dragon sighting", "[west"],
leg_Center: ["colour was your initial dragon sighting", "[forest center"],
sighting_action: ["action did you take during the first dragon sighting"],
cadet_actions: ["during your cadet encounter"],
goal_v2: ["why are you doing this run"],
malek: ["seek malek"],
rare: ["encounter a rare dragon"],
tail: ["what tail does your dragon have"],
};
// A key is only reported missing when neither it nor its replacement in the other form version was found.
const ALTERNATIVE = {
dir_East: "first_dir", dir_West: "first_dir", dir_Center: "first_dir", first_dir: "dir_Center", first_colour: "dir_Center",
goal: "goal_v2", goal_v2: "goal", combat: "cadet_actions", cadet_actions: "combat",
};
const OPTIONAL = new Set(["leg_West", "leg_Center", "sighting_action", "malek", "rare", "tail"]);
function findColumns(headers) {
const C = {};
for (const [key, words] of Object.entries(COLUMN_KEYS)) {
const hit = headers.find((h) => words.every((w) => w[0] === "!" ? !String(h).toLowerCase().includes(w.slice(1)) : String(h).toLowerCase().includes(w)));
if (hit !== undefined) C[key] = hit;
}
const missing = Object.keys(COLUMN_KEYS).filter((k) => !(k in C) && !OPTIONAL.has(k) && !(ALTERNATIVE[k] in C));
return { C, missing };
}
// ---------------------------------------------------------------- coding rules
const CADET_MAP = {
"Bullies > You dealt with them": ["Investigate & help", "Killed the bullies yourself"],
"Bullies > Squadmate's revenge": ["Investigate & help", "Squadmate's revenge"],
"Bullies > Left to the Dragons": ["Investigate & help", "Left bullies to the dragons"],
"Bullies > Assessed the situation": ["Investigate & help", "Unknown"],
"Injured Cadet > Tourniquet": ["Ignore the scream", "Tourniquet"],
"Injured Cadet > Left them there": ["Ignore the scream", "Left injured cadet"],
"Injured Cadet > Took their weapons": ["Ignore the scream", "Took their weapons"],
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", "Left them to it"],
"You died in combat": ["Investigate & help", "DIED in combat"],
"N/A - You died with initial dragon sighting": ["DIED at first sighting", "DIED at first sighting"],
};
const COMBAT_MAP = {
"Bullies > Assessed the situation": "Assess the situation",
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)":
"Put yourself in the middle (bigger cadet)",
};
// Redesigned form: the cadet encounter is asked as a path of actions plus a separate outcome.
// value: [scream, first combat move, second combat move, resolution (when the action decides it)]
const CADET_ACTIONS = {
"Bullies > Assess > Unknown": ["Investigate & help", "Assess the situation", null, null],
"Bullies > Assess > Smaller Cadet": ["Investigate & help", "Assess the situation", "Engage the smaller cadet", null],
"Bullies > Assess > Daggers or not": ["Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", null],
"Bullies > Unknown": ["Investigate & help", null, null, null],
"Bullies > Smaller Cadet (Did not assess)": ["Investigate & help", "Even the numbers (smaller cadet)", null, null],
"Bullies > Bigger Cadet (Did not assess)": ["Investigate & help", "Put yourself in the middle (bigger cadet)", null, null],
"Injured Cadet > Tourniquet": ["Ignore the scream", null, null, "Tourniquet"],
"Injured Cadet > Left them there": ["Ignore the scream", null, null, "Left injured cadet"],
"Injured Cadet > Took their weapons": ["Ignore the scream", null, null, "Took their weapons"],
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", null, null, "Left them to it"],
"N/A - You died with initial dragon sighting": ["DIED at first sighting", null, null, "DIED at first sighting"],
"Unsure": ["Unknown", null, null, "Unknown"],
};
const CADET_OUTCOME = {
"Squadmate's revenge": "Squadmate's revenge",
"You ended the bullies yourself": "Killed the bullies yourself",
"Left to the Dragons": "Left bullies to the dragons",
"You died in combat": "DIED in combat",
};
const DIRECTION_MAP = { "Forest Center": "Center", "West / Scent": "West", "East / Roar": "East" };
const GOAL_V2 = {
"Hoping for a Rare Dragon (Blue or Black)": "Blue or Black",
"Trying to bond the Red dragon under the tree": "Red (tree)",
"Daily Run Assignment data": "Daily run data",
"Just here for fun, and maybe a dragon": "Any (for fun)",
};
const SIGHTING_ACTION = {
"Hold Ground / Stay Still": "Hold still", "Respect / Face it": "Respect / face it",
"Retreat / Run / Back Away": "Retreat / run", "Hide / Duck": "Hide", "Draw / Drop Weapon": "Weapon",
"Meet Gaze": "Meet its gaze", "Protect / Help Sqaudmate": "Protect squadmate", "Approach": "Approach",
"Stand on your own": "Stand alone", "Unsure": "Unknown", "Unsure - You don't remember": "Unknown",
};
const RARE_MAP = { "Yes - A blue dragon in the valley": "Blue (valley)", "Yes - Shadow!!!": "Shadow", "No": "No" };
const RARE_GOALS = ["Blue", "Black", "Blue or Black"];
const isRareGoal = (goal) => RARE_GOALS.includes(goal);
const HOUR_MAP = {
"Continued to Valley Centre": "Valley",
"Climbed the tallest tree": "Tree",
"N/A - You're dead": "DIED before the hour",
};
// First-sighting pools per (company, direction) from the community PDF. Only used to FLAG rows.
const SIGHTING_POOL = {
"Alone|East": ["Red", "Orange"],
"Alone|West": ["Red", "Brown", "Green"],
"Alone|Center": ["Red", "Brown"],
"1 Squadmate|East": ["Green", "Orange", "Red"],
"1 Squadmate|West": ["Red", "Brown", "Green"],
"1 Squadmate|Center": ["Red", "Brown"],
"2 Squadmates|East": ["Red", "Orange"],
"2 Squadmates|West": ["Green", "Brown", "Red"],
"2 Squadmates|Center": ["Red", "Brown"],
};
const str = (v) => {
if (v === null || v === undefined) return null;
if (typeof v === "number" && isNaN(v)) return null;
const t = String(v).trim();
return t === "" || t === "nan" || t === "NaN" ? null : t;
};
// Form submissions carry a real timestamp; rows bulk-pasted into the sheet carry a label
// ("Data Import", "Manual Upload", "Historical import") instead.
function parseTimestamp(v) {
if (v instanceof Date && !isNaN(v)) return v;
if (typeof v === "number" && v > 40000 && v < 80000) return new Date(Date.UTC(1899, 11, 30) + Math.floor(v) * 86400000); // Excel serial date
const t = str(v);
if (!t) return null;
let m = t.match(/^(\d{4})-(\d{2})-(\d{2})/);
if (m) return new Date(Date.UTC(+m[1], +m[2] - 1, +m[3]));
m = t.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})/); // Google Sheets CSV: M/D/YYYY
if (m) return new Date(Date.UTC(+m[3], +m[1] - 1, +m[2]));
return null;
}
const isoDate = (d) => d.toISOString().slice(0, 10);
function parseAttempt(v) {
const t0 = str(v);
if (!t0) return null;
const t = t0.toLowerCase();
if (t.includes("lost count")) return null;
const words = { first: 1, second: 2, third: 3, fourth: 4, fifth: 5 };
for (const [w, n] of Object.entries(words)) if (t.startsWith(w)) return n;
const m = t.match(/\d+/);
return m ? parseInt(m[0], 10) : null;
}
function codeRow(row, C) {
const g = (k) => (k in C ? str(row[C[k]]) : null);
const flags = [];
const tsRaw = "timestamp" in C ? row[C.timestamp] : null;
const ts = parseTimestamp(tsRaw);
const source = ts ? "Form" : str(tsRaw) || "Unknown import";
const date = ts ? isoDate(ts) : null;
const wings = ["First", "Second", "Third", "Fourth"].filter((w) => g("wing_" + w));
const wing = wings.length === 1 ? wings[0] : wings.length ? "Multiple" : "Unknown";
const section = wings.length === 1 ? g("wing_" + wings[0]) : null;
if (wings.length > 1) flags.push("multiple wings");
let direction, first;
if ("first_dir" in C) {
// Redesigned form: one direction question and one colour question. The older per-direction colour
// columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
direction = DIRECTION_MAP[g("first_dir")] || null;
first = g("first_colour");
const legacy = ["West", "Center"].filter((d) => g("leg_" + d));
if (!direction && legacy.length === 1) {
direction = legacy[0];
flags.push("direction taken from the older sighting column");
}
const leg = direction ? g("leg_" + direction) : null;
if (!first && leg) first = leg;
else if (first && leg && first !== leg && first !== "Unsure" && leg !== "Unsure")
flags.push("first sighting differs between the old and new columns");
if (!direction) {
direction = "Unknown";
flags.push("direction missing");
}
} else {
const dirs = ["East", "West", "Center"].filter((d) => g("dir_" + d));
if (dirs.length === 1) {
direction = dirs[0];
first = g("dir_" + dirs[0]);
} else {
direction = dirs.length ? "Multiple" : "Unknown";
first = null;
flags.push("direction " + (dirs.length ? "multiple" : "missing"));
}
}
const companyRaw = g("company") || "Unknown";
const leftSquad = / > Left them$/.test(companyRaw);
const company = leftSquad ? companyRaw.replace(/ > Left them$/, "") : companyRaw;
// Leaving your squadmates sends you into the solo pools, so either pool is plausible.
const pool = (SIGHTING_POOL[company + "|" + direction] || []).concat(leftSquad ? SIGHTING_POOL["Alone|" + direction] || [] : []);
if (first && first !== "Unsure" && pool.length && !pool.includes(first))
flags.push(`first sighting ${first} not in documented pool for ${company}/${direction}`);
const cadetRaw = g("cadet");
let scream, resolution, combat = null, combat2 = null;
if ("cadet_actions" in C) {
const act = g("cadet_actions");
const a = CADET_ACTIONS[act] || ["Unknown", null, null, "Unknown"];
if (act && !CADET_ACTIONS[act]) flags.push("unrecognised cadet answer");
[scream, combat, combat2] = a;
const out = CADET_OUTCOME[cadetRaw] || null;
if (a[3] !== null) {
resolution = a[3];
if (out === "DIED in combat" && !resolution.startsWith("DIED")) flags.push("died in combat on a path without combat");
} else resolution = out || "Unknown";
} else {
[scream, resolution] = CADET_MAP[cadetRaw] || ["Unknown", "Unknown"];
if (cadetRaw && !CADET_MAP[cadetRaw]) flags.push("unrecognised cadet answer");
combat = g("combat") ? COMBAT_MAP[g("combat")] || null : null;
}
const postRaw = g("post");
let post = null;
if (scream === "Investigate & help" && !resolution.startsWith("DIED"))
post = ["Forest", "West", "East"].includes(postRaw) ? postRaw : "Unknown";
else if (scream === "Ignore the scream" || scream === "Investigate & leave") post = "(no fork)";
const diedEarly = scream.startsWith("DIED") || resolution.startsWith("DIED");
let hour = HOUR_MAP[g("hour")] || null;
if (diedEarly) hour = null;
else if (hour === null) hour = "Unknown";
const enc = {};
for (const c of COLORS) enc[c] = g("enc_" + c);
const rare = RARE_MAP[g("rare")] || null;
const bonded = COLORS.filter((c) => enc[c] === "Bonded!");
const burned = COLORS.filter((c) => enc[c] === "Incinerated");
let branch = null;
if (hour === "Tree") {
if (enc.Blue || enc.Black || rare === "Shadow") branch = "Shadow (rare)";
else if (enc.Green) branch = "Descend slowly: two Greens";
else if (enc.Red) branch = "Descend quickly: Red";
else if (enc.Orange || enc.Brown) {
branch = "Descend slowly: two Greens";
flags.push("tree run with Orange/Brown but no Green recorded");
}
} else if (hour === "Valley") {
if (enc.Green) flags.push("Green recorded on a Valley run (no Greens in the valley)");
const meadow = ["Orange", "Brown", "Blue"].some((c) => enc[c]) || rare === "Blue (valley)";
if (meadow || enc.Red === "Changed my mind") branch = "Passed the Red: meadow";
else if (enc.Red) branch = "Valley Red";
}
let outcome, outcomeColor = null;
if (diedEarly) outcome = "Died before the hour";
else if (bonded.length === 1) { outcome = "Bonded"; outcomeColor = bonded[0]; }
else if (bonded.length > 1) { outcome = "Ambiguous"; flags.push("more than one dragon marked Bonded"); }
else if (burned.length) {
const order = hour !== "Valley"
? ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
: ["Red", "Orange", "Brown", "Blue", "Green", "Black"];
const inOrder = order.filter((c) => burned.includes(c));
outcome = "Incinerated";
outcomeColor = inOrder[inOrder.length - 1];
if (burned.length > 1) flags.push("more than one dragon marked Incinerated");
} else if (hour === "DIED before the hour") outcome = "Died before the hour";
else outcome = "No outcome recorded";
const rescued = enc.Green === "Saved by another" && outcome === "Bonded" && ["Orange", "Brown"].includes(outcomeColor);
const attempt = parseAttempt(g("attempt"));
if ((rare === "Blue (valley)" && hour === "Tree") || (rare === "Shadow" && hour === "Valley"))
flags.push("rare-dragon answer does not match the path");
const goalRaw = g("goal_v2");
const goal = g("goal") || (goalRaw && goalRaw !== "Unknown" ? GOAL_V2[goalRaw] || goalRaw : null) || "(not asked/blank)";
return {
source, date, wing, section, squad: g("squad") || "Unknown", attempt,
// "I've lost count" means many attempts, so it is a later attempt, not missing
attempt_group: attempt === 1 ? "First attempt" : attempt > 1 || /lost count/i.test(g("attempt") || "") ? "Later attempt" : "Unknown",
goal, company, left_squadmates: leftSquad, direction, first_sighting: first || "Unknown",
sighting_action: "sighting_action" in C ? SIGHTING_ACTION[g("sighting_action")] || g("sighting_action") || "Unknown" : "(not asked)",
scream, resolution, combat_first: combat || "(not recorded)", combat_second: combat2 || "(not recorded)",
post_path: post, hour, branch,
enc_Red: enc.Red, enc_Green: enc.Green, enc_Orange: enc.Orange, enc_Brown: enc.Brown,
enc_Blue: enc.Blue, enc_Black: enc.Black,
outcome, outcome_color: outcomeColor, rescued,
saw_blue_meadow: !!(hour === "Valley" && (enc.Blue || rare === "Blue (valley)")),
saw_shadow: !!(hour === "Tree" && (enc.Blue || enc.Black || rare === "Shadow")),
rare_reported: "rare" in C ? rare || "Unknown" : "(not asked)",
malek_intent: "malek" in C ? g("malek") || "Unknown" : "(not asked)",
tail: g("tail"),
data_flags: flags.join("; "),
death_note: redact(g("death_note")), bond_note: redact(g("bond_note")),
};
}
function codeRows(rows, headers) {
if (!headers) {
const seen = new Set();
rows.forEach((r) => Object.keys(r).forEach((k) => seen.add(k)));
headers = [...seen];
}
const { C, missing } = findColumns(headers);
const runs = rows.map((r, i) => Object.assign({ run_id: i + 1 }, codeRow(r, C)));
return { runs, missing };
}
// ---------------------------------------------------------------- statistics
const Z95 = 1.959963984540054;
function wilson(k, n) {
if (!n) return { k, n, p: null, lo: null, hi: null };
const z = Z95, p = k / n, d = 1 + (z * z) / n;
const c = (p + (z * z) / (2 * n)) / d;
const h = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / d;
return { k, n, p, lo: Math.max(0, c - h), hi: Math.min(1, c + h) };
}
const LOGF = [0];
function logFact(n) {
for (let i = LOGF.length; i <= n; i++) LOGF[i] = LOGF[i - 1] + Math.log(i);
return LOGF[n];
}
// Two-sided Fisher exact test: sum of probabilities of all tables (same margins) no more likely
// than the observed one (same rule as scipy.stats.fisher_exact, with a 1e-7 relative tolerance).
function fisher(a, b, c, d) {
const r1 = a + b, r2 = c + d, c1 = a + c, n = r1 + r2;
const lp = (x) => logFact(r1) + logFact(r2) + logFact(c1) + logFact(n - c1) -
logFact(n) - logFact(x) - logFact(r1 - x) - logFact(c1 - x) - logFact(r2 - c1 + x);
const lo = Math.max(0, c1 - r2), hi = Math.min(r1, c1);
const pObs = lp(a);
let p = 0;
for (let x = lo; x <= hi; x++) {
const q = lp(x);
if (q <= pObs + 1e-7 * Math.abs(pObs) + 1e-12) p += Math.exp(q);
}
return Math.min(1, p);
}
function oddsRatio(a, b, c, d) {
const corrected = a === 0 || b === 0 || c === 0 || d === 0;
const k = corrected ? 0.5 : 0;
const A = a + k, B = b + k, Cc = c + k, D = d + k;
const or = (A * D) / (B * Cc);
const se = Math.sqrt(1 / A + 1 / B + 1 / Cc + 1 / D);
return {
or, or_lo: Math.exp(Math.log(or) - Z95 * se), or_hi: Math.exp(Math.log(or) + Z95 * se),
fisher_p: fisher(a, b, c, d), corrected,
};
}
// Mantel-Haenszel common odds ratio across strata (tables [a,b,c,d]), Robins-Breslow-Greenland 95% CI,
// and the Cochran-Mantel-Haenszel chi-square test (1 df, with continuity correction).
function mantelHaenszel(tables) {
let R = 0, S = 0, PR = 0, PSQR = 0, QS = 0, sumA = 0, sumE = 0, sumV = 0;
for (const [a, b, c, d] of tables) {
const n = a + b + c + d;
if (n < 2) continue;
const r = (a * d) / n, s_ = (b * c) / n, P = (a + d) / n, Q = (b + c) / n;
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_;
const m1 = a + b, m0 = c + d, n1 = a + c, n0 = b + d;
sumA += a; sumE += (m1 * n1) / n;
sumV += (m1 * m0 * n1 * n0) / (n * n * (n - 1));
}
if (!R || !S) return { mh_or: null, mh_lo: null, mh_hi: null, mh_p: null };
const or = R / S;
const varLog = PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S);
const se = Math.sqrt(varLog);
const chi = sumV > 0 ? Math.pow(Math.max(0, Math.abs(sumA - sumE) - 0.5), 2) / sumV : 0;
return { mh_or: or, mh_lo: Math.exp(Math.log(or) - Z95 * se), mh_hi: Math.exp(Math.log(or) + Z95 * se), mh_p: normSf2(Math.sqrt(chi)) };
}
function bh(ps) {
const m = ps.length, order = ps.map((p, i) => [p, i]).sort((x, y) => x[0] - y[0]);
const q = new Array(m);
let prev = 1;
for (let r = m; r >= 1; r--) {
const [p, i] = order[r - 1];
prev = Math.min(prev, (p * m) / r);
q[i] = prev;
}
return q;
}
// Normal CDF (Abramowitz-Stegun 7.1.26 is too coarse for small p; use erfc via continued fraction)
function normSf2(z) { // two-sided p for |z|
z = Math.abs(z);
const t = 1 / (1 + 0.5 * z / Math.SQRT2);
const x = z / Math.SQRT2;
const tau = t * Math.exp(-x * x - 1.26551223 + t * (1.00002368 + t * (0.37409196 + t * (0.09678418 +
t * (-0.18628806 + t * (0.27886807 + t * (-1.13520398 + t * (1.48851587 +
t * (-0.82215223 + t * 0.17087277)))))))));
return Math.min(1, tau); // erfc(x) = 2*P(Z>z)
}
function solve(A, b) { // Gaussian elimination with partial pivoting
const n = b.length, M = A.map((r, i) => r.concat([b[i]]));
for (let i = 0; i < n; i++) {
let p = i;
for (let r = i + 1; r < n; r++) if (Math.abs(M[r][i]) > Math.abs(M[p][i])) p = r;
[M[i], M[p]] = [M[p], M[i]];
for (let r = i + 1; r < n; r++) {
const f = M[r][i] / M[i][i];
for (let c = i; c <= n; c++) M[r][c] -= f * M[i][c];
}
}
const x = new Array(n).fill(0);
for (let i = n - 1; i >= 0; i--) {
let s = M[i][n];
for (let c = i + 1; c < n; c++) s -= M[i][c] * x[c];
x[i] = s / M[i][i];
}
return x;
}
function invert(A) {
const n = A.length;
return A.map((_, j) => solve(A, A.map((__, i) => (i === j ? 1 : 0)))).reduce((acc, col, j) => {
col.forEach((v, i) => (acc[i][j] = v));
return acc;
}, A.map(() => new Array(n).fill(0)));
}
// Logistic regression, Newton-Raphson with an L2 penalty (lambda) on all slopes (not the intercept).
function logisticRidge(X, y, lam = 1, iters = 100) {
const n = X.length, k = X[0].length;
let beta = new Array(k).fill(0);
const info = (b) => {
const H = Array.from({ length: k }, () => new Array(k).fill(0));
const grad = new Array(k).fill(0);
for (let i = 0; i < n; i++) {
let eta = 0;
for (let j = 0; j < k; j++) eta += X[i][j] * b[j];
const mu = 1 / (1 + Math.exp(-eta)), w = mu * (1 - mu);
for (let j = 0; j < k; j++) {
grad[j] += X[i][j] * (y[i] - mu);
if (!X[i][j]) continue;
for (let l = 0; l < k; l++) H[j][l] += X[i][j] * X[i][l] * w;
}
}
for (let j = 1; j < k; j++) { H[j][j] += lam; grad[j] -= lam * b[j]; }
return { H, grad };
};
for (let it = 0; it < iters; it++) {
const { H, grad } = info(beta);
const step = solve(H, grad);
beta = beta.map((v, j) => v + step[j]);
if (Math.max(...step.map(Math.abs)) < 1e-10) break;
}
const { H } = info(beta);
const Hi = invert(H);
const se = Hi.map((r, j) => Math.sqrt(r[j]));
return { beta, se };
}
// ---------------------------------------------------------------- analysis helpers
const rate = (runs, pred) => wilson(runs.filter(pred).length, runs.length);
const groupBy = (arr, key) => arr.reduce((m, r) => {
const k = typeof key === "function" ? key(r) : r[key];
(m[k] = m[k] || []).push(r);
return m;
}, {});
const sortedKeys = (o) => Object.keys(o).sort();
const stratum = (r) => r.source !== "Form" ? "Bulk logs (imports)" :
r.goal === "Blue" ? "Form: chasing Blue" : r.goal === "Black" ? "Form: chasing Black" :
r.goal === "Blue or Black" ? "Form: chasing Blue or Black" : "Form: other/blank goal";
const goal2 = (r) => isRareGoal(r.goal) ? r.goal : r.goal === "(not asked/blank)" ? "Blank/not asked" : "Other colour/any";
// Denominators for each outcome ("who could have had this outcome?")
const OUTCOMES = {
blue_meadow: {
label: "Blue appears in the meadow", denom: (r) => r.branch === "Passed the Red: meadow", event: (r) => r.saw_blue_meadow,
},
shadow: {
label: "Shadow option appears at the tree", denom: (r) => r.hour === "Tree" && r.branch !== null, event: (r) => r.saw_shadow,
},
bond_blue_meadow: {
label: "Bond, given you approached the meadow Blue", denom: (r) => r.branch === "Passed the Red: meadow" && ["Bonded!", "Incinerated"].includes(r.enc_Blue),
event: (r) => r.enc_Blue === "Bonded!",
},
reach_hour: {
label: "Survive to the hour (tree/valley choice)", denom: () => true, event: (r) => r.hour === "Tree" || r.hour === "Valley",
},
};
for (const c of COLORS) {
OUTCOMES["bonded_" + c] = {
label: `Run ends bonded to ${c}`, denom: (r) => r.hour === "Tree" || r.hour === "Valley",
event: (r) => r.outcome === "Bonded" && r.outcome_color === c,
};
}
const FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal", "sighting_action", "combat_second", "malek_intent"];
// Encounter contexts: which recorded dragon attempt belongs to which in-game encounter.
const CONTEXTS = [
["Red (tree, descend quickly)", (r) => r.hour === "Tree" && r.branch === "Descend quickly: Red", "enc_Red", "Red"],
["Green (two Greens)", (r) => r.hour === "Tree" && r.branch === "Descend slowly: two Greens", "enc_Green", "Green"],
["Blue (shadow)", (r) => r.hour === "Tree" && r.branch === "Shadow (rare)", "enc_Blue", "Blue"],
["Black (shadow)", (r) => r.hour === "Tree" && r.branch === "Shadow (rare)", "enc_Black", "Black"],
["Red (valley)", (r) => r.hour === "Valley" && r.branch === "Valley Red", "enc_Red", "Red"],
["Orange (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Orange", "Orange"],
["Brown (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Brown", "Brown"],
["Blue (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Blue", "Blue"],
];
function applyFilters(runs, f = {}) {
return runs.filter((r) => {
if (f.source === "form" && r.source !== "Form") return false;
if (f.source === "bulk" && r.source === "Form") return false;
if (f.goal === "exclude_rare" && isRareGoal(r.goal)) return false;
if (f.goal === "only_rare" && !isRareGoal(r.goal)) return false;
if (f.excludeFlagged && r.data_flags) return false;
return true;
});
}
function factorTable(runs, outKey, minN = 5) {
const O = OUTCOMES[outKey];
const D = runs.filter(O.denom);
const total = D.length, events = D.filter(O.event).length;
const rows = [], missing = {};
for (const f of FACTORS) {
const groups = groupBy(D.filter((r) => !isMissing(r[f])), f);
missing[f] = D.filter((r) => isMissing(r[f])).length;
const nf = Object.values(groups).reduce((s, g) => s + g.length, 0);
const ef = Object.values(groups).reduce((s, g) => s + g.filter(O.event).length, 0);
for (const lv of sortedKeys(groups)) {
const g = groups[lv], k = g.filter(O.event).length, n = g.length;
const row = Object.assign({ factor: f, level: lv }, wilson(k, n));
const rk = ef - k, rn = nf - n; // rest of the same factor
if (n >= minN && rn >= minN && ef > 0 && ef < nf) Object.assign(row, oddsRatio(k, n - k, rk, rn - rk));
rows.push(row);
}
}
const tested = rows.filter((r) => r.fisher_p !== undefined);
bh(tested.map((r) => r.fisher_p)).forEach((q, i) => (tested[i].q = q));
return { outcome: outKey, label: O.label, overall: wilson(events, total), rows, missing };
}
// Multivariable model: one row per run in the outcome's denominator; categorical factors dummy-coded
// against their most common level.
function model(runs, outKey, factors, lam = 1) {
const O = OUTCOMES[outKey];
const D = runs.filter(O.denom).map((r) => {
const o = { y: O.event(r) ? 1 : 0 };
for (const [name, fn] of Object.entries(factors)) o[name] = fn(r);
return o;
}).filter((o) => Object.keys(factors).every((f) => o[f] !== null && o[f] !== undefined));
if (D.length < 20) return null;
const cols = [["(intercept)", () => 1]], refs = {};
for (const f of Object.keys(factors)) {
const counts = groupBy(D, f);
const levels = sortedKeys(counts).sort((a, b) => counts[b].length - counts[a].length || (a < b ? -1 : 1));
refs[f] = levels[0];
for (const lv of levels.slice(1)) cols.push([`${f} = ${lv}`, (o) => (o[f] === lv ? 1 : 0), f, lv, counts[lv].length]);
}
const X = D.map((o) => cols.map((c) => c[1](o)));
const y = D.map((o) => o.y);
const { beta, se } = logisticRidge(X, y, lam);
const terms = cols.slice(1).map((c, i) => {
const b = beta[i + 1], s = se[i + 1];
return { term: c[0], factor: c[2], level: c[3], n: c[4], or: Math.exp(b), lo: Math.exp(b - Z95 * s), hi: Math.exp(b + Z95 * s), p: normSf2(b / s) };
});
return { outcome: outKey, label: O.label, n: D.length, events: y.reduce((a, b) => a + b, 0), refs, lambda: lam, terms };
}
const routeKey = (r) => [r.company, r.direction, r.first_sighting, r.scream, r.resolution, r.post_path || "—", r.hour || "—"];
const ROUTE_FIELDS = ["company", "direction", "first_sighting", "scream", "resolution", "post_path", "hour"];
function analyse(allRuns, filters = {}) {
const runs = applyFilters(allRuns, filters);
const res = { filters, n: runs.length };
// meta / data quality
const flagCounts = {};
runs.forEach((r) => r.data_flags && r.data_flags.split("; ").forEach((f) => {
const key = f.startsWith("first sighting") ? "first sighting outside documented pool" : f;
flagCounts[key] = (flagCounts[key] || 0) + 1;
}));
const dates = runs.map((r) => r.date).filter(Boolean).sort();
res.meta = {
sources: Object.fromEntries(Object.entries(groupBy(runs, "source")).map(([k, v]) => [k, v.length])),
date_min: dates[0] || null, date_max: dates[dates.length - 1] || null,
n_flagged: runs.filter((r) => r.data_flags).length, flag_counts: flagCounts,
outcomes: Object.fromEntries(Object.entries(groupBy(runs, (r) => r.outcome + (r.outcome_color ? " " + r.outcome_color : ""))).map(([k, v]) => [k, v.length])),
};
// chance events, overall and by stratum
res.chance = {};
for (const key of ["blue_meadow", "shadow"]) {
const O = OUTCOMES[key], D = runs.filter(O.denom);
res.chance[key] = {
label: O.label, overall: rate(D, O.event),
by_stratum: Object.fromEntries(sortedKeys(groupBy(D, stratum)).map((s) => [s, rate(groupBy(D, stratum)[s], O.event)])),
by_goal: Object.fromEntries(sortedKeys(groupBy(D, "goal")).map((s) => [s, rate(groupBy(D, "goal")[s], O.event)])),
by_attempt: Object.fromEntries(sortedKeys(groupBy(D, "attempt_group")).map((s) => [s, rate(groupBy(D, "attempt_group")[s], O.event)])),
};
}
// first sighting pools
const fs = groupBy(runs.filter((r) => !isMissing(r.direction) && !isMissing(r.company)), (r) => r.company + "|" + r.direction);
res.first_sighting = sortedKeys(fs).map((k) => {
const g = fs[k], cnt = groupBy(g, "first_sighting");
return { key: k, n: g.length, colors: Object.fromEntries(sortedKeys(cnt).map((c) => [c, cnt[c].length])) };
});
// bonding encounters
res.encounters = CONTEXTS.map(([name, where, col, color]) => {
const A = runs.filter((r) => where(r) && ["Bonded!", "Incinerated"].includes(r[col]));
const win = (r) => r[col] === "Bonded!";
const intent = A.filter((r) => r.goal === color || r.goal.startsWith("Any") || (r.goal === "Blue or Black" && (color === "Blue" || color === "Black")));
const byS = groupBy(A, stratum);
return {
context: name, color, overall: rate(A, win), intent_matched: rate(intent, win),
by_stratum: Object.fromEntries(sortedKeys(byS).map((s) => [s, rate(byS[s], win)])),
};
});
const greens = runs.filter((r) => r.hour === "Tree" && r.branch === "Descend slowly: two Greens" && r.enc_Green);
res.green_rescue = {
green_encounters: greens.length,
bonded_green: greens.filter((r) => r.enc_Green === "Bonded!").length,
rescued_orange: greens.filter((r) => r.rescued && r.outcome_color === "Orange").length,
rescued_brown: greens.filter((r) => r.rescued && r.outcome_color === "Brown").length,
incinerated: greens.filter((r) => r.outcome === "Incinerated").length,
};
// factor tables
res.factors = {};
for (const key of Object.keys(OUTCOMES)) res.factors[key] = factorTable(runs, key);
// adjusted models
const first2 = (r) => (["Red", "Brown", "Orange", "Green"].includes(r.first_sighting) ? r.first_sighting : null);
const known = (f) => (r) => (isMissing(r[f]) ? null : r[f]);
const src2 = (r) => (r.source === "Form" ? "Form submission" : "Bulk log (import)");
res.models = {
blue_meadow: model(runs, "blue_meadow", {
source: src2, goal: goal2, direction: known("direction"), first_sighting: first2,
resolution: known("resolution"), company: known("company"),
}),
shadow: model(runs, "shadow", {
source: src2, goal: goal2, attempt: known("attempt_group"), direction: known("direction"),
}),
bond_blue_meadow: model(runs, "bond_blue_meadow", {
source: src2, goal: goal2, direction: known("direction"), resolution: known("resolution"),
}),
};
// daily trend (form submissions only, since imports have no date)
const byDate = groupBy(runs.filter((r) => r.date), "date");
res.daily = sortedKeys(byDate).map((d) => {
const g = byDate[d];
const m = g.filter(OUTCOMES.blue_meadow.denom), t = g.filter(OUTCOMES.shadow.denom);
return { date: d, runs: g.length, meadow_n: m.length, blue_k: m.filter((r) => r.saw_blue_meadow).length, tree_n: t.length, shadow_k: t.filter((r) => r.saw_shadow).length };
});
// routes (choices up to the hour) with outcome rates
const R = groupBy(runs.filter((r) => r.hour === "Tree" || r.hour === "Valley"), (r) => routeKey(r).join(" › "));
res.routes = sortedKeys(R).map((k) => {
const g = R[k], m = g.filter(OUTCOMES.blue_meadow.denom), t = g.filter(OUTCOMES.shadow.denom);
const o = { route: k, parts: routeKey(g[0]), n: g.length, blue_meadow: rate(m, (r) => r.saw_blue_meadow), shadow: rate(t, (r) => r.saw_shadow) };
for (const c of COLORS) o["bonded_" + c] = rate(g, (r) => r.outcome === "Bonded" && r.outcome_color === c);
o.bonded_any = rate(g, (r) => r.outcome === "Bonded");
return o;
});
return res;
}
// ---------------------------------------------------------------- route-map graph
const STAGES = [
{ key: "company", label: "Company" },
{ key: "direction", label: "Direction" },
{ key: "first_sighting", label: "First sighting" },
{ key: "cadet", label: "Cadet encounter" },
{ key: "post_path", label: "Fork in the road" },
{ key: "hour", label: "After an hour" },
{ key: "encounter", label: "Encounter" },
{ key: "end", label: "Outcome" },
];
const CHANCE_STAGES = new Set(["first_sighting", "encounter"]); // decided (partly) by the game, not the player
function stageValue(r, key) {
switch (key) {
case "company": return isMissing(r.company) ? null : r.company;
case "direction": return isMissing(r.direction) ? null : r.direction;
case "first_sighting": return r.scream === "DIED at first sighting" || isMissing(r.first_sighting) ? null : r.first_sighting;
case "cadet":
if (r.scream.startsWith("DIED") || isMissing(r.scream) || isMissing(r.resolution)) return null;
return r.scream === "Investigate & help" ? "Help → " + r.resolution : r.scream + " → " + r.resolution;
case "post_path": return isMissing(r.post_path) ? null : r.post_path;
case "hour": return r.hour === "Tree" ? "Climb the tree" : r.hour === "Valley" ? "Valley centre" : null;
case "encounter": {
if (!r.branch) return null;
if (r.hour === "Tree") {
if (r.branch === "Shadow (rare)") return "Tree: shadow → Blue/Black";
if (r.branch === "Descend quickly: Red") return "Tree: Red below";
return "Tree: two Greens";
}
if (r.branch === "Valley Red") return "Valley: bond the Red";
if (r.enc_Blue) return "Meadow: Blue";
if (r.enc_Orange && r.enc_Brown) return "Meadow: Orange/Brown";
if (r.enc_Orange) return "Meadow: Orange";
if (r.enc_Brown) return "Meadow: Brown";
return "Meadow (no attempt recorded)";
}
case "end":
if (r.outcome === "Bonded") return "Bonded " + r.outcome_color;
if (r.outcome === "Incinerated") return "Incinerated";
if (r.outcome === "Died before the hour") return "Died before the hour";
return null;
}
return null;
}
// pool = which runs could have reached this outcome at all (used by rankRoutes as the comparison group)
const BOTH = (r) => r.hour === "Tree" || r.hour === "Valley";
const TREE = (r) => r.hour === "Tree";
const VALLEY = (r) => r.hour === "Valley";
const TARGETS = {
bond_Blue: { label: "Bond Blue", color: "Blue", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Blue", pool: VALLEY },
bond_Black: { label: "Bond Black", color: "Black", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Black", pool: TREE },
see_Blue: { label: "Meet a Blue (meadow)", color: "Blue", fn: (r) => r.saw_blue_meadow, pool: (r) => r.branch === "Passed the Red: meadow" },
see_shadow: { label: "Get the tree shadow", color: "Black", fn: (r) => r.saw_shadow, pool: (r) => r.hour === "Tree" && r.branch !== null },
bond_Red: { label: "Bond Red", color: "Red", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Red", pool: BOTH },
bond_Green: { label: "Bond Green", color: "Green", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Green", pool: TREE },
bond_Orange: { label: "Bond Orange", color: "Orange", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Orange", pool: BOTH },
bond_Brown: { label: "Bond Brown", color: "Brown", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Brown", pool: BOTH },
bond_any: { label: "Bond any dragon", color: null, fn: (r) => r.outcome === "Bonded", pool: BOTH },
};
// conditions: { stageKey: Set(values) } — a run passes if, for every stage with a condition, its value is in the set.
function buildGraph(runs, targetKey, conditions = {}) {
const T = TARGETS[targetKey].fn;
const sel = runs.filter((r) => Object.entries(conditions).every(([k, set]) => !set.size || set.has(stageValue(r, k))));
const nodes = new Map(), links = new Map();
const node = (si, v) => {
const id = si + "|" + v;
if (!nodes.has(id)) nodes.set(id, { id, stage: si, stageKey: STAGES[si].key, value: v, n: 0, k: 0 });
return nodes.get(id);
};
for (const r of sel) {
const hit = T(r) ? 1 : 0;
let prev = null;
STAGES.forEach((s, si) => {
const v = stageValue(r, s.key);
if (v === null || v === undefined) return;
const nd = node(si, v);
nd.n++; nd.k += hit;
if (prev) {
const id = prev.id + "→" + nd.id;
if (!links.has(id)) links.set(id, { id, source: prev.id, target: nd.id, n: 0, k: 0, chance: CHANCE_STAGES.has(s.key) });
const l = links.get(id); l.n++; l.k += hit;
}
prev = nd;
});
}
const out = (o) => Object.assign(o, wilson(o.k, o.n));
return {
stages: STAGES, selected: sel.length, target: targetKey,
nodes: [...nodes.values()].map(out), links: [...links.values()].map(out),
};
}
// Rank full routes (every choice up to the hour) for a target outcome.
// Each route with at least minN runs is compared with all other runs by Fisher's exact test;
// q-values use Benjamini-Hochberg across all ranked routes. Ranking uses the lower 95% Wilson
// bound, so a route needs both a high rate and enough runs to rank high.
function rankRoutes(runs, targetKey, minN = 10, denomStage = null) {
const T = TARGETS[targetKey].fn;
const keyStages = STAGES.slice(0, 6).map((s) => s.key);
const pool = runs.filter(TARGETS[targetKey].pool);
const totK = pool.filter(T).length, totN = pool.length;
// a route is only ranked from runs whose every step is known; runs with a gap still count in the comparison group
const groups = groupBy(pool.filter((r) => keyStages.every((k) => stageValue(r, k) || (k === "post_path" && r.post_path === null))),
(r) => keyStages.map((k) => stageValue(r, k) || "—").join(" › "));
const rows = Object.entries(groups).filter(([, g]) => g.length >= minN).map(([key, g]) => {
const k = g.filter(T).length, n = g.length;
const o = Object.assign({ route: key, values: keyStages.map((s) => stageValue(g[0], s)) }, wilson(k, n));
Object.assign(o, oddsRatio(k, n - k, totK - k, totN - n - (totK - k)));
// same comparison, adjusted for data source / chase-goal stratum (reporting bias)
const inRoute = new Set(g);
const tabs = Object.values(groupBy(pool, stratum)).map((sg) => {
const a = sg.filter((r) => inRoute.has(r) && T(r)).length, nin = sg.filter((r) => inRoute.has(r)).length;
const c = sg.filter((r) => !inRoute.has(r) && T(r)).length, nout = sg.length - nin;
return [a, nin - a, c, nout - c];
});
Object.assign(o, mantelHaenszel(tabs));
return o;
});
bh(rows.map((r) => r.fisher_p)).forEach((q, i) => (rows[i].q = q));
const mhIdx = rows.map((r, i) => [r.mh_p, i]).filter(([p]) => p !== null);
bh(mhIdx.map(([p]) => p)).forEach((q, j) => (rows[mhIdx[j][1]].mh_q = q));
rows.sort((a, b) => b.lo - a.lo || b.n - a.n);
return { target: targetKey, overall: wilson(totK, totN), minN, n_routes_tested: rows.length, rows };
}
return {
COLORS, MISSING, isMissing, COLUMN_KEYS, STAGES, TARGETS, OUTCOMES, FACTORS, ROUTE_FIELDS, rankRoutes,
findColumns, codeRow, codeRows, parseTimestamp,
wilson, fisher, oddsRatio, bh, logisticRidge, mantelHaenszel,
applyFilters, analyse, factorTable, model, buildGraph, stageValue, stratum,
};
});