Files
threshing-atlas/packages/core/src/engine.js
T
ekcSchianoandClaude Opus 5.5 c3d067b88c Threshing Atlas: monorepo, route graph, bot importer, daily source checks
- apps/atlas: the web page (valley map, route flow, Dragon Register)
- packages/core: coding rules and statistics (engine.js) and the shared route
  graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk,
  Beta credible intervals
- packages/ingest: DragonKind bot importer to anonymised trajectories (no prose,
  no account identifiers); data/bot holds Ken's sample
- research/python: independent reference implementation + update_sources.py
- tests: JS/Python parity checks and graph/importer unit tests
- CI, GitHub Pages deploy, daily check of the community sheets and route PDF
- Email addresses in free-text form answers are redacted; the raw spreadsheet
  is not committed
- MIT code licence, CC BY 4.0 data licence

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-01 21:54:54 +00:00

732 lines
35 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/*
* engine.js — Threshing route analysis engine.
*
* One file, used two ways:
* - in the browser (the route map page loads it and recomputes everything from the raw form export)
* - in Node (`node tests/run_engine.js data.json`), so results can be reproduced and checked against
* the Python reference implementation in python/.
*
* Pipeline: raw form rows -> codeRow() (one coded run per row) -> analyse() -> results object
* coded runs + a target outcome -> buildGraph() -> nodes/links for the route map
*
* Every rule here is described in plain language in the Methods section of the page.
*/
(function (root, factory) {
if (typeof module === "object" && module.exports) module.exports = factory();
else root.ThreshEngine = factory();
})(typeof self !== "undefined" ? self : this, function () {
"use strict";
// Free-text answers sometimes contain a respondent's email address; never display or export it.
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
const redact = (v) => (v == null ? v : String(v).replace(EMAIL_RE, "[email removed]"));
const COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"];
// ---------------------------------------------------------------- column detection
const COLUMN_KEYS = {
timestamp: ["timestamp"],
wing_First: ["wing and section", "[first]"],
wing_Second: ["wing and section", "[second]"],
wing_Third: ["wing and section", "[third]"],
wing_Fourth: ["wing and section", "[fourth]"],
squad: ["what squad"],
attempt: ["attempt number"],
company: ["who accompanied"],
dir_East: ["initial direction", "[east"],
dir_West: ["initial direction", "[west"],
dir_Center: ["initial direction", "[forest center"],
cadet: ["outcome of your cadet encounter"],
post: ["path did you take after the bully"],
hour: ["after walking an hour"],
enc_Red: ["attempt to bond", "[red"],
enc_Green: ["attempt to bond", "[green"],
enc_Orange: ["attempt to bond", "[orange"],
enc_Brown: ["attempt to bond", "[brown"],
enc_Blue: ["attempt to bond", "[blue"],
enc_Black: ["attempt to bond", "[black"],
death_note: ["those who died"],
bond_note: ["those who bonded"],
goal: ["colour dragon are you trying"],
combat: ["combat choice"],
};
function findColumns(headers) {
const C = {};
for (const [key, words] of Object.entries(COLUMN_KEYS)) {
const hit = headers.find((h) => words.every((w) => String(h).toLowerCase().includes(w)));
if (hit !== undefined) C[key] = hit;
}
const missing = Object.keys(COLUMN_KEYS).filter((k) => !(k in C));
return { C, missing };
}
// ---------------------------------------------------------------- coding rules
const CADET_MAP = {
"Bullies > You dealt with them": ["Investigate & help", "Killed the bullies yourself"],
"Bullies > Squadmate's revenge": ["Investigate & help", "Squadmate's revenge"],
"Bullies > Left to the Dragons": ["Investigate & help", "Left bullies to the dragons"],
"Bullies > Assessed the situation": ["Investigate & help", "Unknown"],
"Injured Cadet > Tourniquet": ["Ignore the scream", "Tourniquet"],
"Injured Cadet > Left them there": ["Ignore the scream", "Left injured cadet"],
"Injured Cadet > Took their weapons": ["Ignore the scream", "Took their weapons"],
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", "Left them to it"],
"You died in combat": ["Investigate & help", "DIED in combat"],
"N/A - You died with initial dragon sighting": ["DIED at first sighting", "DIED at first sighting"],
};
const COMBAT_MAP = {
"Bullies > Assessed the situation": "Assess the situation",
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)":
"Put yourself in the middle (bigger cadet)",
};
const HOUR_MAP = {
"Continued to Valley Centre": "Valley",
"Climbed the tallest tree": "Tree",
"N/A - You're dead": "DIED before the hour",
};
// First-sighting pools per (company, direction) from the community PDF. Only used to FLAG rows.
const SIGHTING_POOL = {
"Alone|East": ["Red", "Orange"],
"Alone|West": ["Red", "Brown", "Green"],
"Alone|Center": ["Red", "Brown"],
"1 Squadmate|East": ["Green", "Orange", "Red"],
"1 Squadmate|West": ["Red", "Brown", "Green"],
"1 Squadmate|Center": ["Red", "Brown"],
"2 Squadmates|East": ["Red", "Orange"],
"2 Squadmates|West": ["Green", "Brown", "Red"],
"2 Squadmates|Center": ["Red", "Brown"],
};
const str = (v) => {
if (v === null || v === undefined) return null;
if (typeof v === "number" && isNaN(v)) return null;
const t = String(v).trim();
return t === "" || t === "nan" || t === "NaN" ? null : t;
};
// Form submissions carry a real timestamp; rows bulk-pasted into the sheet carry a label
// ("Data Import", "Manual Upload", "Historical import") instead.
function parseTimestamp(v) {
if (v instanceof Date && !isNaN(v)) return v;
if (typeof v === "number" && v > 40000 && v < 80000) return new Date(Date.UTC(1899, 11, 30) + Math.floor(v) * 86400000); // Excel serial date
const t = str(v);
if (!t) return null;
let m = t.match(/^(\d{4})-(\d{2})-(\d{2})/);
if (m) return new Date(Date.UTC(+m[1], +m[2] - 1, +m[3]));
m = t.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})/); // Google Sheets CSV: M/D/YYYY
if (m) return new Date(Date.UTC(+m[3], +m[1] - 1, +m[2]));
return null;
}
const isoDate = (d) => d.toISOString().slice(0, 10);
function parseAttempt(v) {
const t0 = str(v);
if (!t0) return null;
const t = t0.toLowerCase();
if (t.includes("lost count")) return null;
const words = { first: 1, second: 2, third: 3, fourth: 4, fifth: 5 };
for (const [w, n] of Object.entries(words)) if (t.startsWith(w)) return n;
const m = t.match(/\d+/);
return m ? parseInt(m[0], 10) : null;
}
function codeRow(row, C) {
const g = (k) => (k in C ? str(row[C[k]]) : null);
const flags = [];
const tsRaw = "timestamp" in C ? row[C.timestamp] : null;
const ts = parseTimestamp(tsRaw);
const source = ts ? "Form" : str(tsRaw) || "Unknown import";
const date = ts ? isoDate(ts) : null;
const wings = ["First", "Second", "Third", "Fourth"].filter((w) => g("wing_" + w));
const wing = wings.length === 1 ? wings[0] : wings.length ? "Multiple" : "Unknown";
const section = wings.length === 1 ? g("wing_" + wings[0]) : null;
if (wings.length > 1) flags.push("multiple wings");
const dirs = ["East", "West", "Center"].filter((d) => g("dir_" + d));
let direction, first;
if (dirs.length === 1) {
direction = dirs[0];
first = g("dir_" + dirs[0]);
} else {
direction = dirs.length ? "Multiple" : "Unknown";
first = null;
flags.push("direction " + (dirs.length ? "multiple" : "missing"));
}
const company = g("company") || "Unknown";
const pool = SIGHTING_POOL[company + "|" + direction];
if (first && first !== "Unsure" && pool && !pool.includes(first))
flags.push(`first sighting ${first} not in documented pool for ${company}/${direction}`);
const cadetRaw = g("cadet");
const [scream, resolution] = CADET_MAP[cadetRaw] || ["Unknown", "Unknown"];
if (cadetRaw && !CADET_MAP[cadetRaw]) flags.push("unrecognised cadet answer");
const combat = g("combat") ? COMBAT_MAP[g("combat")] || null : null;
const postRaw = g("post");
let post = null;
if (scream === "Investigate & help" && !resolution.startsWith("DIED"))
post = ["Forest", "West", "East"].includes(postRaw) ? postRaw : "Unknown";
else if (scream === "Ignore the scream" || scream === "Investigate & leave") post = "(no fork)";
const diedEarly = scream.startsWith("DIED") || resolution.startsWith("DIED");
let hour = HOUR_MAP[g("hour")] || null;
if (diedEarly) hour = null;
else if (hour === null) hour = "Unknown";
const enc = {};
for (const c of COLORS) enc[c] = g("enc_" + c);
const bonded = COLORS.filter((c) => enc[c] === "Bonded!");
const burned = COLORS.filter((c) => enc[c] === "Incinerated");
let branch = null;
if (hour === "Tree") {
if (enc.Blue || enc.Black) branch = "Shadow (rare)";
else if (enc.Green) branch = "Descend slowly: two Greens";
else if (enc.Red) branch = "Descend quickly: Red";
else if (enc.Orange || enc.Brown) {
branch = "Descend slowly: two Greens";
flags.push("tree run with Orange/Brown but no Green recorded");
}
} else if (hour === "Valley") {
if (enc.Green) flags.push("Green recorded on a Valley run (no Greens in the valley)");
const meadow = ["Orange", "Brown", "Blue"].some((c) => enc[c]);
if (meadow || enc.Red === "Changed my mind") branch = "Passed the Red: meadow";
else if (enc.Red) branch = "Valley Red";
}
let outcome, outcomeColor = null;
if (diedEarly) outcome = "Died before the hour";
else if (bonded.length === 1) { outcome = "Bonded"; outcomeColor = bonded[0]; }
else if (bonded.length > 1) { outcome = "Ambiguous"; flags.push("more than one dragon marked Bonded"); }
else if (burned.length) {
const order = hour !== "Valley"
? ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
: ["Red", "Orange", "Brown", "Blue", "Green", "Black"];
const inOrder = order.filter((c) => burned.includes(c));
outcome = "Incinerated";
outcomeColor = inOrder[inOrder.length - 1];
if (burned.length > 1) flags.push("more than one dragon marked Incinerated");
} else if (hour === "DIED before the hour") outcome = "Died before the hour";
else outcome = "No outcome recorded";
const rescued = enc.Green === "Saved by another" && outcome === "Bonded" && ["Orange", "Brown"].includes(outcomeColor);
const attempt = parseAttempt(g("attempt"));
return {
source, date, wing, section, squad: g("squad") || "Unknown", attempt,
attempt_group: attempt === 1 ? "First attempt" : attempt > 1 ? "Later attempt" : "Unknown/lost count",
goal: g("goal") || "(not asked/blank)", company, direction, first_sighting: first || "Unknown",
scream, resolution, combat_first: combat || "(not recorded)", post_path: post, hour, branch,
enc_Red: enc.Red, enc_Green: enc.Green, enc_Orange: enc.Orange, enc_Brown: enc.Brown,
enc_Blue: enc.Blue, enc_Black: enc.Black,
outcome, outcome_color: outcomeColor, rescued,
saw_blue_meadow: !!(hour === "Valley" && enc.Blue),
saw_shadow: !!(hour === "Tree" && (enc.Blue || enc.Black)),
data_flags: flags.join("; "),
death_note: redact(g("death_note")), bond_note: redact(g("bond_note")),
};
}
function codeRows(rows, headers) {
if (!headers) {
const seen = new Set();
rows.forEach((r) => Object.keys(r).forEach((k) => seen.add(k)));
headers = [...seen];
}
const { C, missing } = findColumns(headers);
const runs = rows.map((r, i) => Object.assign({ run_id: i + 1 }, codeRow(r, C)));
return { runs, missing };
}
// ---------------------------------------------------------------- statistics
const Z95 = 1.959963984540054;
function wilson(k, n) {
if (!n) return { k, n, p: null, lo: null, hi: null };
const z = Z95, p = k / n, d = 1 + (z * z) / n;
const c = (p + (z * z) / (2 * n)) / d;
const h = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / d;
return { k, n, p, lo: Math.max(0, c - h), hi: Math.min(1, c + h) };
}
const LOGF = [0];
function logFact(n) {
for (let i = LOGF.length; i <= n; i++) LOGF[i] = LOGF[i - 1] + Math.log(i);
return LOGF[n];
}
// Two-sided Fisher exact test: sum of probabilities of all tables (same margins) no more likely
// than the observed one (same rule as scipy.stats.fisher_exact, with a 1e-7 relative tolerance).
function fisher(a, b, c, d) {
const r1 = a + b, r2 = c + d, c1 = a + c, n = r1 + r2;
const lp = (x) => logFact(r1) + logFact(r2) + logFact(c1) + logFact(n - c1) -
logFact(n) - logFact(x) - logFact(r1 - x) - logFact(c1 - x) - logFact(r2 - c1 + x);
const lo = Math.max(0, c1 - r2), hi = Math.min(r1, c1);
const pObs = lp(a);
let p = 0;
for (let x = lo; x <= hi; x++) {
const q = lp(x);
if (q <= pObs + 1e-7 * Math.abs(pObs) + 1e-12) p += Math.exp(q);
}
return Math.min(1, p);
}
function oddsRatio(a, b, c, d) {
const corrected = a === 0 || b === 0 || c === 0 || d === 0;
const k = corrected ? 0.5 : 0;
const A = a + k, B = b + k, Cc = c + k, D = d + k;
const or = (A * D) / (B * Cc);
const se = Math.sqrt(1 / A + 1 / B + 1 / Cc + 1 / D);
return {
or, or_lo: Math.exp(Math.log(or) - Z95 * se), or_hi: Math.exp(Math.log(or) + Z95 * se),
fisher_p: fisher(a, b, c, d), corrected,
};
}
// Mantel-Haenszel common odds ratio across strata (tables [a,b,c,d]), Robins-Breslow-Greenland 95% CI,
// and the Cochran-Mantel-Haenszel chi-square test (1 df, with continuity correction).
function mantelHaenszel(tables) {
let R = 0, S = 0, PR = 0, PSQR = 0, QS = 0, sumA = 0, sumE = 0, sumV = 0;
for (const [a, b, c, d] of tables) {
const n = a + b + c + d;
if (n < 2) continue;
const r = (a * d) / n, s_ = (b * c) / n, P = (a + d) / n, Q = (b + c) / n;
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_;
const m1 = a + b, m0 = c + d, n1 = a + c, n0 = b + d;
sumA += a; sumE += (m1 * n1) / n;
sumV += (m1 * m0 * n1 * n0) / (n * n * (n - 1));
}
if (!R || !S) return { mh_or: null, mh_lo: null, mh_hi: null, mh_p: null };
const or = R / S;
const varLog = PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S);
const se = Math.sqrt(varLog);
const chi = sumV > 0 ? Math.pow(Math.max(0, Math.abs(sumA - sumE) - 0.5), 2) / sumV : 0;
return { mh_or: or, mh_lo: Math.exp(Math.log(or) - Z95 * se), mh_hi: Math.exp(Math.log(or) + Z95 * se), mh_p: normSf2(Math.sqrt(chi)) };
}
function bh(ps) {
const m = ps.length, order = ps.map((p, i) => [p, i]).sort((x, y) => x[0] - y[0]);
const q = new Array(m);
let prev = 1;
for (let r = m; r >= 1; r--) {
const [p, i] = order[r - 1];
prev = Math.min(prev, (p * m) / r);
q[i] = prev;
}
return q;
}
// Normal CDF (Abramowitz-Stegun 7.1.26 is too coarse for small p; use erfc via continued fraction)
function normSf2(z) { // two-sided p for |z|
z = Math.abs(z);
const t = 1 / (1 + 0.5 * z / Math.SQRT2);
const x = z / Math.SQRT2;
const tau = t * Math.exp(-x * x - 1.26551223 + t * (1.00002368 + t * (0.37409196 + t * (0.09678418 +
t * (-0.18628806 + t * (0.27886807 + t * (-1.13520398 + t * (1.48851587 +
t * (-0.82215223 + t * 0.17087277)))))))));
return Math.min(1, tau); // erfc(x) = 2*P(Z>z)
}
function solve(A, b) { // Gaussian elimination with partial pivoting
const n = b.length, M = A.map((r, i) => r.concat([b[i]]));
for (let i = 0; i < n; i++) {
let p = i;
for (let r = i + 1; r < n; r++) if (Math.abs(M[r][i]) > Math.abs(M[p][i])) p = r;
[M[i], M[p]] = [M[p], M[i]];
for (let r = i + 1; r < n; r++) {
const f = M[r][i] / M[i][i];
for (let c = i; c <= n; c++) M[r][c] -= f * M[i][c];
}
}
const x = new Array(n).fill(0);
for (let i = n - 1; i >= 0; i--) {
let s = M[i][n];
for (let c = i + 1; c < n; c++) s -= M[i][c] * x[c];
x[i] = s / M[i][i];
}
return x;
}
function invert(A) {
const n = A.length;
return A.map((_, j) => solve(A, A.map((__, i) => (i === j ? 1 : 0)))).reduce((acc, col, j) => {
col.forEach((v, i) => (acc[i][j] = v));
return acc;
}, A.map(() => new Array(n).fill(0)));
}
// Logistic regression, Newton-Raphson with an L2 penalty (lambda) on all slopes (not the intercept).
function logisticRidge(X, y, lam = 1, iters = 100) {
const n = X.length, k = X[0].length;
let beta = new Array(k).fill(0);
const info = (b) => {
const H = Array.from({ length: k }, () => new Array(k).fill(0));
const grad = new Array(k).fill(0);
for (let i = 0; i < n; i++) {
let eta = 0;
for (let j = 0; j < k; j++) eta += X[i][j] * b[j];
const mu = 1 / (1 + Math.exp(-eta)), w = mu * (1 - mu);
for (let j = 0; j < k; j++) {
grad[j] += X[i][j] * (y[i] - mu);
if (!X[i][j]) continue;
for (let l = 0; l < k; l++) H[j][l] += X[i][j] * X[i][l] * w;
}
}
for (let j = 1; j < k; j++) { H[j][j] += lam; grad[j] -= lam * b[j]; }
return { H, grad };
};
for (let it = 0; it < iters; it++) {
const { H, grad } = info(beta);
const step = solve(H, grad);
beta = beta.map((v, j) => v + step[j]);
if (Math.max(...step.map(Math.abs)) < 1e-10) break;
}
const { H } = info(beta);
const Hi = invert(H);
const se = Hi.map((r, j) => Math.sqrt(r[j]));
return { beta, se };
}
// ---------------------------------------------------------------- analysis helpers
const rate = (runs, pred) => wilson(runs.filter(pred).length, runs.length);
const groupBy = (arr, key) => arr.reduce((m, r) => {
const k = typeof key === "function" ? key(r) : r[key];
(m[k] = m[k] || []).push(r);
return m;
}, {});
const sortedKeys = (o) => Object.keys(o).sort();
const stratum = (r) => r.source !== "Form" ? "Bulk logs (imports)" :
r.goal === "Blue" ? "Form: chasing Blue" : r.goal === "Black" ? "Form: chasing Black" : "Form: other/blank goal";
const goal2 = (r) => ["Blue", "Black"].includes(r.goal) ? r.goal : r.goal === "(not asked/blank)" ? "Blank/not asked" : "Other colour/any";
// Denominators for each outcome ("who could have had this outcome?")
const OUTCOMES = {
blue_meadow: {
label: "Blue appears in the meadow", denom: (r) => r.branch === "Passed the Red: meadow", event: (r) => r.saw_blue_meadow,
},
shadow: {
label: "Shadow option appears at the tree", denom: (r) => r.hour === "Tree" && r.branch !== null, event: (r) => r.saw_shadow,
},
bond_blue_meadow: {
label: "Bond, given you approached the meadow Blue", denom: (r) => r.branch === "Passed the Red: meadow" && ["Bonded!", "Incinerated"].includes(r.enc_Blue),
event: (r) => r.enc_Blue === "Bonded!",
},
reach_hour: {
label: "Survive to the hour (tree/valley choice)", denom: () => true, event: (r) => r.hour === "Tree" || r.hour === "Valley",
},
};
for (const c of COLORS) {
OUTCOMES["bonded_" + c] = {
label: `Run ends bonded to ${c}`, denom: (r) => r.hour === "Tree" || r.hour === "Valley",
event: (r) => r.outcome === "Bonded" && r.outcome_color === c,
};
}
const FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal"];
// Encounter contexts: which recorded dragon attempt belongs to which in-game encounter.
const CONTEXTS = [
["Red (tree, descend quickly)", (r) => r.hour === "Tree" && r.branch === "Descend quickly: Red", "enc_Red", "Red"],
["Green (two Greens)", (r) => r.hour === "Tree" && r.branch === "Descend slowly: two Greens", "enc_Green", "Green"],
["Blue (shadow)", (r) => r.hour === "Tree" && r.branch === "Shadow (rare)", "enc_Blue", "Blue"],
["Black (shadow)", (r) => r.hour === "Tree" && r.branch === "Shadow (rare)", "enc_Black", "Black"],
["Red (valley)", (r) => r.hour === "Valley" && r.branch === "Valley Red", "enc_Red", "Red"],
["Orange (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Orange", "Orange"],
["Brown (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Brown", "Brown"],
["Blue (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Blue", "Blue"],
];
function applyFilters(runs, f = {}) {
return runs.filter((r) => {
if (f.source === "form" && r.source !== "Form") return false;
if (f.source === "bulk" && r.source === "Form") return false;
if (f.goal === "exclude_rare" && ["Blue", "Black"].includes(r.goal)) return false;
if (f.goal === "only_rare" && !["Blue", "Black"].includes(r.goal)) return false;
if (f.excludeFlagged && r.data_flags) return false;
return true;
});
}
function factorTable(runs, outKey, minN = 5) {
const O = OUTCOMES[outKey];
const D = runs.filter(O.denom);
const total = D.length, events = D.filter(O.event).length;
const rows = [];
for (const f of FACTORS) {
const groups = groupBy(D.filter((r) => r[f] !== null && r[f] !== undefined), f);
const nf = Object.values(groups).reduce((s, g) => s + g.length, 0);
const ef = Object.values(groups).reduce((s, g) => s + g.filter(O.event).length, 0);
for (const lv of sortedKeys(groups)) {
const g = groups[lv], k = g.filter(O.event).length, n = g.length;
const row = Object.assign({ factor: f, level: lv }, wilson(k, n));
const rk = ef - k, rn = nf - n; // rest of the same factor
if (n >= minN && rn >= minN && ef > 0 && ef < nf) Object.assign(row, oddsRatio(k, n - k, rk, rn - rk));
rows.push(row);
}
}
const tested = rows.filter((r) => r.fisher_p !== undefined);
bh(tested.map((r) => r.fisher_p)).forEach((q, i) => (tested[i].q = q));
return { outcome: outKey, label: O.label, overall: wilson(events, total), rows };
}
// Multivariable model: one row per run in the outcome's denominator; categorical factors dummy-coded
// against their most common level.
function model(runs, outKey, factors, lam = 1) {
const O = OUTCOMES[outKey];
const D = runs.filter(O.denom).map((r) => {
const o = { y: O.event(r) ? 1 : 0 };
for (const [name, fn] of Object.entries(factors)) o[name] = fn(r);
return o;
}).filter((o) => Object.keys(factors).every((f) => o[f] !== null && o[f] !== undefined));
if (D.length < 20) return null;
const cols = [["(intercept)", () => 1]], refs = {};
for (const f of Object.keys(factors)) {
const counts = groupBy(D, f);
const levels = sortedKeys(counts).sort((a, b) => counts[b].length - counts[a].length || (a < b ? -1 : 1));
refs[f] = levels[0];
for (const lv of levels.slice(1)) cols.push([`${f} = ${lv}`, (o) => (o[f] === lv ? 1 : 0), f, lv, counts[lv].length]);
}
const X = D.map((o) => cols.map((c) => c[1](o)));
const y = D.map((o) => o.y);
const { beta, se } = logisticRidge(X, y, lam);
const terms = cols.slice(1).map((c, i) => {
const b = beta[i + 1], s = se[i + 1];
return { term: c[0], factor: c[2], level: c[3], n: c[4], or: Math.exp(b), lo: Math.exp(b - Z95 * s), hi: Math.exp(b + Z95 * s), p: normSf2(b / s) };
});
return { outcome: outKey, label: O.label, n: D.length, events: y.reduce((a, b) => a + b, 0), refs, lambda: lam, terms };
}
const routeKey = (r) => [r.company, r.direction, r.first_sighting, r.scream, r.resolution, r.post_path || "—", r.hour || "—"];
const ROUTE_FIELDS = ["company", "direction", "first_sighting", "scream", "resolution", "post_path", "hour"];
function analyse(allRuns, filters = {}) {
const runs = applyFilters(allRuns, filters);
const res = { filters, n: runs.length };
// meta / data quality
const flagCounts = {};
runs.forEach((r) => r.data_flags && r.data_flags.split("; ").forEach((f) => {
const key = f.startsWith("first sighting") ? "first sighting outside documented pool" : f;
flagCounts[key] = (flagCounts[key] || 0) + 1;
}));
const dates = runs.map((r) => r.date).filter(Boolean).sort();
res.meta = {
sources: Object.fromEntries(Object.entries(groupBy(runs, "source")).map(([k, v]) => [k, v.length])),
date_min: dates[0] || null, date_max: dates[dates.length - 1] || null,
n_flagged: runs.filter((r) => r.data_flags).length, flag_counts: flagCounts,
outcomes: Object.fromEntries(Object.entries(groupBy(runs, (r) => r.outcome + (r.outcome_color ? " " + r.outcome_color : ""))).map(([k, v]) => [k, v.length])),
};
// chance events, overall and by stratum
res.chance = {};
for (const key of ["blue_meadow", "shadow"]) {
const O = OUTCOMES[key], D = runs.filter(O.denom);
res.chance[key] = {
label: O.label, overall: rate(D, O.event),
by_stratum: Object.fromEntries(sortedKeys(groupBy(D, stratum)).map((s) => [s, rate(groupBy(D, stratum)[s], O.event)])),
by_goal: Object.fromEntries(sortedKeys(groupBy(D, "goal")).map((s) => [s, rate(groupBy(D, "goal")[s], O.event)])),
by_attempt: Object.fromEntries(sortedKeys(groupBy(D, "attempt_group")).map((s) => [s, rate(groupBy(D, "attempt_group")[s], O.event)])),
};
}
// first sighting pools
const fs = groupBy(runs.filter((r) => r.direction !== "Multiple" && r.direction !== "Unknown"), (r) => r.company + "|" + r.direction);
res.first_sighting = sortedKeys(fs).map((k) => {
const g = fs[k], cnt = groupBy(g, "first_sighting");
return { key: k, n: g.length, colors: Object.fromEntries(sortedKeys(cnt).map((c) => [c, cnt[c].length])) };
});
// bonding encounters
res.encounters = CONTEXTS.map(([name, where, col, color]) => {
const A = runs.filter((r) => where(r) && ["Bonded!", "Incinerated"].includes(r[col]));
const win = (r) => r[col] === "Bonded!";
const intent = A.filter((r) => r.goal === color || r.goal.startsWith("Any"));
const byS = groupBy(A, stratum);
return {
context: name, color, overall: rate(A, win), intent_matched: rate(intent, win),
by_stratum: Object.fromEntries(sortedKeys(byS).map((s) => [s, rate(byS[s], win)])),
};
});
const greens = runs.filter((r) => r.hour === "Tree" && r.branch === "Descend slowly: two Greens" && r.enc_Green);
res.green_rescue = {
green_encounters: greens.length,
bonded_green: greens.filter((r) => r.enc_Green === "Bonded!").length,
rescued_orange: greens.filter((r) => r.rescued && r.outcome_color === "Orange").length,
rescued_brown: greens.filter((r) => r.rescued && r.outcome_color === "Brown").length,
incinerated: greens.filter((r) => r.outcome === "Incinerated").length,
};
// factor tables
res.factors = {};
for (const key of Object.keys(OUTCOMES)) res.factors[key] = factorTable(runs, key);
// adjusted models
const first2 = (r) => ["Red", "Brown", "Orange", "Green"].includes(r.first_sighting) ? r.first_sighting : "Unsure/unknown";
const src2 = (r) => (r.source === "Form" ? "Form submission" : "Bulk log (import)");
res.models = {
blue_meadow: model(runs, "blue_meadow", {
source: src2, goal: goal2, direction: (r) => r.direction, first_sighting: first2,
resolution: (r) => r.resolution, company: (r) => (r.company === "Unknown" ? null : r.company),
}),
shadow: model(runs, "shadow", {
source: src2, goal: goal2, attempt: (r) => r.attempt_group, direction: (r) => r.direction,
}),
bond_blue_meadow: model(runs, "bond_blue_meadow", {
source: src2, goal: goal2, direction: (r) => r.direction, resolution: (r) => r.resolution,
}),
};
// daily trend (form submissions only, since imports have no date)
const byDate = groupBy(runs.filter((r) => r.date), "date");
res.daily = sortedKeys(byDate).map((d) => {
const g = byDate[d];
const m = g.filter(OUTCOMES.blue_meadow.denom), t = g.filter(OUTCOMES.shadow.denom);
return { date: d, runs: g.length, meadow_n: m.length, blue_k: m.filter((r) => r.saw_blue_meadow).length, tree_n: t.length, shadow_k: t.filter((r) => r.saw_shadow).length };
});
// routes (choices up to the hour) with outcome rates
const R = groupBy(runs.filter((r) => r.hour === "Tree" || r.hour === "Valley"), (r) => routeKey(r).join(" › "));
res.routes = sortedKeys(R).map((k) => {
const g = R[k], m = g.filter(OUTCOMES.blue_meadow.denom), t = g.filter(OUTCOMES.shadow.denom);
const o = { route: k, parts: routeKey(g[0]), n: g.length, blue_meadow: rate(m, (r) => r.saw_blue_meadow), shadow: rate(t, (r) => r.saw_shadow) };
for (const c of COLORS) o["bonded_" + c] = rate(g, (r) => r.outcome === "Bonded" && r.outcome_color === c);
o.bonded_any = rate(g, (r) => r.outcome === "Bonded");
return o;
});
return res;
}
// ---------------------------------------------------------------- route-map graph
const STAGES = [
{ key: "company", label: "Company" },
{ key: "direction", label: "Direction" },
{ key: "first_sighting", label: "First sighting" },
{ key: "cadet", label: "Cadet encounter" },
{ key: "post_path", label: "Fork in the road" },
{ key: "hour", label: "After an hour" },
{ key: "encounter", label: "Encounter" },
{ key: "end", label: "Outcome" },
];
const CHANCE_STAGES = new Set(["first_sighting", "encounter"]); // decided (partly) by the game, not the player
function stageValue(r, key) {
switch (key) {
case "company": return r.company;
case "direction": return r.direction;
case "first_sighting": return r.scream === "DIED at first sighting" ? null : r.first_sighting;
case "cadet":
if (r.scream.startsWith("DIED")) return null;
return r.scream === "Investigate & help" ? "Help → " + r.resolution : r.scream + " → " + r.resolution;
case "post_path": return r.post_path;
case "hour": return r.hour === "Tree" ? "Climb the tree" : r.hour === "Valley" ? "Valley centre" : null;
case "encounter": {
if (!r.branch) return null;
if (r.hour === "Tree") {
if (r.branch === "Shadow (rare)") return "Tree: shadow → Blue/Black";
if (r.branch === "Descend quickly: Red") return "Tree: Red below";
return "Tree: two Greens";
}
if (r.branch === "Valley Red") return "Valley: bond the Red";
if (r.enc_Blue) return "Meadow: Blue";
if (r.enc_Orange && r.enc_Brown) return "Meadow: Orange/Brown";
if (r.enc_Orange) return "Meadow: Orange";
if (r.enc_Brown) return "Meadow: Brown";
return "Meadow (no attempt recorded)";
}
case "end":
if (r.outcome === "Bonded") return "Bonded " + r.outcome_color;
if (r.outcome === "Incinerated") return "Incinerated";
if (r.outcome === "Died before the hour") return "Died before the hour";
return null;
}
return null;
}
// pool = which runs could have reached this outcome at all (used by rankRoutes as the comparison group)
const BOTH = (r) => r.hour === "Tree" || r.hour === "Valley";
const TREE = (r) => r.hour === "Tree";
const VALLEY = (r) => r.hour === "Valley";
const TARGETS = {
bond_Blue: { label: "Bond Blue", color: "Blue", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Blue", pool: VALLEY },
bond_Black: { label: "Bond Black", color: "Black", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Black", pool: TREE },
see_Blue: { label: "Meet a Blue (meadow)", color: "Blue", fn: (r) => r.saw_blue_meadow, pool: (r) => r.branch === "Passed the Red: meadow" },
see_shadow: { label: "Get the tree shadow", color: "Black", fn: (r) => r.saw_shadow, pool: (r) => r.hour === "Tree" && r.branch !== null },
bond_Red: { label: "Bond Red", color: "Red", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Red", pool: BOTH },
bond_Green: { label: "Bond Green", color: "Green", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Green", pool: TREE },
bond_Orange: { label: "Bond Orange", color: "Orange", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Orange", pool: BOTH },
bond_Brown: { label: "Bond Brown", color: "Brown", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Brown", pool: BOTH },
bond_any: { label: "Bond any dragon", color: null, fn: (r) => r.outcome === "Bonded", pool: BOTH },
};
// conditions: { stageKey: Set(values) } — a run passes if, for every stage with a condition, its value is in the set.
function buildGraph(runs, targetKey, conditions = {}) {
const T = TARGETS[targetKey].fn;
const sel = runs.filter((r) => Object.entries(conditions).every(([k, set]) => !set.size || set.has(stageValue(r, k))));
const nodes = new Map(), links = new Map();
const node = (si, v) => {
const id = si + "|" + v;
if (!nodes.has(id)) nodes.set(id, { id, stage: si, stageKey: STAGES[si].key, value: v, n: 0, k: 0 });
return nodes.get(id);
};
for (const r of sel) {
const hit = T(r) ? 1 : 0;
let prev = null;
STAGES.forEach((s, si) => {
const v = stageValue(r, s.key);
if (v === null || v === undefined) return;
const nd = node(si, v);
nd.n++; nd.k += hit;
if (prev) {
const id = prev.id + "→" + nd.id;
if (!links.has(id)) links.set(id, { id, source: prev.id, target: nd.id, n: 0, k: 0, chance: CHANCE_STAGES.has(s.key) });
const l = links.get(id); l.n++; l.k += hit;
}
prev = nd;
});
}
const out = (o) => Object.assign(o, wilson(o.k, o.n));
return {
stages: STAGES, selected: sel.length, target: targetKey,
nodes: [...nodes.values()].map(out), links: [...links.values()].map(out),
};
}
// Rank full routes (every choice up to the hour) for a target outcome.
// Each route with at least minN runs is compared with all other runs by Fisher's exact test;
// q-values use Benjamini-Hochberg across all ranked routes. Ranking uses the lower 95% Wilson
// bound, so a route needs both a high rate and enough runs to rank high.
function rankRoutes(runs, targetKey, minN = 10, denomStage = null) {
const T = TARGETS[targetKey].fn;
const keyStages = STAGES.slice(0, 6).map((s) => s.key);
const pool = runs.filter(TARGETS[targetKey].pool);
const totK = pool.filter(T).length, totN = pool.length;
const groups = groupBy(pool, (r) => keyStages.map((k) => stageValue(r, k) || "—").join(" › "));
const rows = Object.entries(groups).filter(([, g]) => g.length >= minN).map(([key, g]) => {
const k = g.filter(T).length, n = g.length;
const o = Object.assign({ route: key, values: keyStages.map((s) => stageValue(g[0], s)) }, wilson(k, n));
Object.assign(o, oddsRatio(k, n - k, totK - k, totN - n - (totK - k)));
// same comparison, adjusted for data source / chase-goal stratum (reporting bias)
const inRoute = new Set(g);
const tabs = Object.values(groupBy(pool, stratum)).map((sg) => {
const a = sg.filter((r) => inRoute.has(r) && T(r)).length, nin = sg.filter((r) => inRoute.has(r)).length;
const c = sg.filter((r) => !inRoute.has(r) && T(r)).length, nout = sg.length - nin;
return [a, nin - a, c, nout - c];
});
Object.assign(o, mantelHaenszel(tabs));
return o;
});
bh(rows.map((r) => r.fisher_p)).forEach((q, i) => (rows[i].q = q));
const mhIdx = rows.map((r, i) => [r.mh_p, i]).filter(([p]) => p !== null);
bh(mhIdx.map(([p]) => p)).forEach((q, j) => (rows[mhIdx[j][1]].mh_q = q));
rows.sort((a, b) => b.lo - a.lo || b.n - a.n);
return { target: targetKey, overall: wilson(totK, totN), minN, n_routes_tested: rows.length, rows };
}
return {
COLORS, COLUMN_KEYS, STAGES, TARGETS, OUTCOMES, FACTORS, ROUTE_FIELDS, rankRoutes,
findColumns, codeRow, codeRows, parseTimestamp,
wilson, fisher, oddsRatio, bh, logisticRidge, mantelHaenszel,
applyFilters, analyse, factorTable, model, buildGraph, stageValue, stratum,
};
});