Files
threshing-atlas/core/engine.js
T
432e04c2c7 Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)
Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 19:46:33 +00:00

847 lines
43 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/*
* engine.js — Threshing route analysis engine.
*
* One file, used two ways:
* - in the browser (the route map page loads it and recomputes everything from the raw form export)
* - in Node (`node tests/parity/run_engine.js data.json`), so results can be reproduced and checked against
* the Python reference implementation in python/.
*
* Pipeline: raw form rows -> codeRow() (one coded run per row) -> analyse() -> results object
* coded runs + a target outcome -> buildGraph() -> nodes/links for the route map
*
* Every rule here is described in plain language in the Methods section of the page.
*/
(function (root, factory) {
if (typeof module === "object" && module.exports) module.exports = factory();
else root.ThreshEngine = factory();
})(typeof self !== "undefined" ? self : this, function () {
"use strict";
// Free-text answers sometimes contain a respondent's email address; never display or export it.
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
const redact = (v) => (v == null ? v : String(v).replace(EMAIL_RE, "[email removed]"));
const COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"];
// Answers that carry no information about the choice ("Unsure", blank, a question the form did not ask yet,
// conflicting answers). They are kept in the coded data for transparency, but every analysis treats them as
// missing: they are never shown or compared as if they were a choice.
const MISSING = new Set(["Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)",
"(not asked/blank)", "Unknown/lost count", "Blank/not asked"]);
const isMissing = (v) => v === null || v === undefined || v === "" || MISSING.has(v);
// ---------------------------------------------------------------- column detection
const COLUMN_KEYS = {
timestamp: ["timestamp"],
wing_First: ["wing and section", "[first]"],
wing_Second: ["wing and section", "[second]"],
wing_Third: ["wing and section", "[third]"],
wing_Fourth: ["wing and section", "[fourth]"],
squad: ["what squad"],
attempt: ["attempt number"],
company: ["who accompanied"],
dir_East: ["initial direction", "[east"],
dir_West: ["initial direction", "[west"],
dir_Center: ["initial direction", "[forest center"],
cadet: ["outcome of your cadet encounter"],
post: ["path did you take after the bully"],
hour: ["after walking an hour"],
enc_Red: ["attempt to bond", "[red"],
enc_Green: ["attempt to bond", "[green"],
enc_Orange: ["attempt to bond", "[orange"],
enc_Brown: ["attempt to bond", "[brown"],
enc_Blue: ["attempt to bond", "[blue"],
enc_Black: ["attempt to bond", "[black"],
death_note: ["those who died"],
bond_note: ["those who bonded"],
goal: ["colour dragon are you trying"],
combat: ["combat choice"],
// Questions added in the redesigned form (late October 2026). "!x" means the header must NOT contain x.
first_dir: ["what was your first direction"],
first_colour: ["colour was your initial dragon sighting", "!["],
leg_West: ["colour was your initial dragon sighting", "[west"],
leg_Center: ["colour was your initial dragon sighting", "[forest center"],
sighting_action: ["action did you take during the first dragon sighting"],
cadet_actions: ["during your cadet encounter"],
goal_v2: ["why are you doing this run"],
malek: ["seek malek"],
rare: ["encounter a rare dragon"],
tail: ["what tail does your dragon have"],
};
// A key is only reported missing when neither it nor its replacement in the other form version was found.
const ALTERNATIVE = {
dir_East: "first_dir", dir_West: "first_dir", dir_Center: "first_dir", first_dir: "dir_Center", first_colour: "dir_Center",
goal: "goal_v2", goal_v2: "goal", combat: "cadet_actions", cadet_actions: "combat",
};
const OPTIONAL = new Set(["leg_West", "leg_Center", "sighting_action", "malek", "rare", "tail"]);
function findColumns(headers) {
const C = {};
for (const [key, words] of Object.entries(COLUMN_KEYS)) {
const hit = headers.find((h) => words.every((w) => w[0] === "!" ? !String(h).toLowerCase().includes(w.slice(1)) : String(h).toLowerCase().includes(w)));
if (hit !== undefined) C[key] = hit;
}
const missing = Object.keys(COLUMN_KEYS).filter((k) => !(k in C) && !OPTIONAL.has(k) && !(ALTERNATIVE[k] in C));
return { C, missing };
}
// ---------------------------------------------------------------- coding rules
const CADET_MAP = {
"Bullies > You dealt with them": ["Investigate & help", "Killed the bullies yourself"],
"Bullies > Squadmate's revenge": ["Investigate & help", "Squadmate's revenge"],
"Bullies > Left to the Dragons": ["Investigate & help", "Left bullies to the dragons"],
"Bullies > Assessed the situation": ["Investigate & help", "Unknown"],
"Injured Cadet > Tourniquet": ["Ignore the scream", "Tourniquet"],
"Injured Cadet > Left them there": ["Ignore the scream", "Left injured cadet"],
"Injured Cadet > Took their weapons": ["Ignore the scream", "Took their weapons"],
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", "Left them to it"],
"You died in combat": ["Investigate & help", "DIED in combat"],
"N/A - You died with initial dragon sighting": ["DIED at first sighting", "DIED at first sighting"],
};
const COMBAT_MAP = {
"Bullies > Assessed the situation": "Assess the situation",
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)":
"Put yourself in the middle (bigger cadet)",
};
// Redesigned form: the cadet encounter is asked as a path of actions plus a separate outcome.
// value: [scream, first combat move, second combat move, resolution (when the action decides it)]
const CADET_ACTIONS = {
"Bullies > Assess > Unknown": ["Investigate & help", "Assess the situation", null, null],
"Bullies > Assess > Smaller Cadet": ["Investigate & help", "Assess the situation", "Engage the smaller cadet", null],
"Bullies > Assess > Daggers or not": ["Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", null],
"Bullies > Unknown": ["Investigate & help", null, null, null],
"Bullies > Smaller Cadet (Did not assess)": ["Investigate & help", "Even the numbers (smaller cadet)", null, null],
"Bullies > Bigger Cadet (Did not assess)": ["Investigate & help", "Put yourself in the middle (bigger cadet)", null, null],
"Injured Cadet > Tourniquet": ["Ignore the scream", null, null, "Tourniquet"],
"Injured Cadet > Left them there": ["Ignore the scream", null, null, "Left injured cadet"],
"Injured Cadet > Took their weapons": ["Ignore the scream", null, null, "Took their weapons"],
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", null, null, "Left them to it"],
"N/A - You died with initial dragon sighting": ["DIED at first sighting", null, null, "DIED at first sighting"],
"Unsure": ["Unknown", null, null, "Unknown"],
};
const CADET_OUTCOME = {
"Squadmate's revenge": "Squadmate's revenge",
"You ended the bullies yourself": "Killed the bullies yourself",
"Left to the Dragons": "Left bullies to the dragons",
"You died in combat": "DIED in combat",
};
const DIRECTION_MAP = { "Forest Center": "Center", "West / Scent": "West", "East / Roar": "East" };
const GOAL_V2 = {
"Hoping for a Rare Dragon (Blue or Black)": "Blue or Black",
"Trying to bond the Red dragon under the tree": "Red (tree)",
"Daily Run Assignment data": "Daily run data",
"Just here for fun, and maybe a dragon": "Any (for fun)",
};
const SIGHTING_ACTION = {
"Hold Ground / Stay Still": "Hold still", "Respect / Face it": "Respect / face it",
"Retreat / Run / Back Away": "Retreat / run", "Hide / Duck": "Hide", "Draw / Drop Weapon": "Weapon",
"Meet Gaze": "Meet its gaze", "Protect / Help Sqaudmate": "Protect squadmate", "Approach": "Approach",
"Stand on your own": "Stand alone", "Unsure": "Unknown", "Unsure - You don't remember": "Unknown",
};
const RARE_MAP = { "Yes - A blue dragon in the valley": "Blue (valley)", "Yes - Shadow!!!": "Shadow", "No": "No" };
const RARE_GOALS = ["Blue", "Black", "Blue or Black"];
const isRareGoal = (goal) => RARE_GOALS.includes(goal);
const HOUR_MAP = {
"Continued to Valley Centre": "Valley",
"Climbed the tallest tree": "Tree",
"N/A - You're dead": "DIED before the hour",
};
// First-sighting pools per (company, direction) from the community PDF. Only used to FLAG rows.
const SIGHTING_POOL = {
"Alone|East": ["Red", "Orange"],
"Alone|West": ["Red", "Brown", "Green"],
"Alone|Center": ["Red", "Brown"],
"1 Squadmate|East": ["Green", "Orange", "Red"],
"1 Squadmate|West": ["Red", "Brown", "Green"],
"1 Squadmate|Center": ["Red", "Brown"],
"2 Squadmates|East": ["Red", "Orange"],
"2 Squadmates|West": ["Green", "Brown", "Red"],
"2 Squadmates|Center": ["Red", "Brown"],
};
const str = (v) => {
if (v === null || v === undefined) return null;
if (typeof v === "number" && isNaN(v)) return null;
const t = String(v).trim();
return t === "" || t === "nan" || t === "NaN" ? null : t;
};
// Form submissions carry a real timestamp; rows bulk-pasted into the sheet carry a label
// ("Data Import", "Manual Upload", "Historical import") instead.
function parseTimestamp(v) {
if (v instanceof Date && !isNaN(v)) return v;
if (typeof v === "number" && v > 40000 && v < 80000) return new Date(Date.UTC(1899, 11, 30) + Math.floor(v) * 86400000); // Excel serial date
const t = str(v);
if (!t) return null;
let m = t.match(/^(\d{4})-(\d{2})-(\d{2})/);
if (m) return new Date(Date.UTC(+m[1], +m[2] - 1, +m[3]));
m = t.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})/); // Google Sheets CSV: M/D/YYYY
if (m) return new Date(Date.UTC(+m[3], +m[1] - 1, +m[2]));
return null;
}
const isoDate = (d) => d.toISOString().slice(0, 10);
function parseAttempt(v) {
const t0 = str(v);
if (!t0) return null;
const t = t0.toLowerCase();
if (t.includes("lost count")) return null;
const words = { first: 1, second: 2, third: 3, fourth: 4, fifth: 5 };
for (const [w, n] of Object.entries(words)) if (t.startsWith(w)) return n;
const m = t.match(/\d+/);
return m ? parseInt(m[0], 10) : null;
}
function codeRow(row, C) {
const g = (k) => (k in C ? str(row[C[k]]) : null);
const flags = [];
const tsRaw = "timestamp" in C ? row[C.timestamp] : null;
const ts = parseTimestamp(tsRaw);
const source = ts ? "Form" : str(tsRaw) || "Unknown import";
const date = ts ? isoDate(ts) : null;
const wings = ["First", "Second", "Third", "Fourth"].filter((w) => g("wing_" + w));
const wing = wings.length === 1 ? wings[0] : wings.length ? "Multiple" : "Unknown";
const section = wings.length === 1 ? g("wing_" + wings[0]) : null;
if (wings.length > 1) flags.push("multiple wings");
let direction, first;
if ("first_dir" in C) {
// Redesigned form: one direction question and one colour question. The older per-direction colour
// columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
direction = DIRECTION_MAP[g("first_dir")] || null;
first = g("first_colour");
const legacy = ["West", "Center"].filter((d) => g("leg_" + d));
if (!direction && legacy.length === 1) {
direction = legacy[0];
flags.push("direction taken from the older sighting column");
}
const leg = direction ? g("leg_" + direction) : null;
if (!first && leg) first = leg;
else if (first && leg && first !== leg && first !== "Unsure" && leg !== "Unsure")
flags.push("first sighting differs between the old and new columns");
if (!direction) {
direction = "Unknown";
flags.push("direction missing");
}
} else {
const dirs = ["East", "West", "Center"].filter((d) => g("dir_" + d));
if (dirs.length === 1) {
direction = dirs[0];
first = g("dir_" + dirs[0]);
} else {
direction = dirs.length ? "Multiple" : "Unknown";
first = null;
flags.push("direction " + (dirs.length ? "multiple" : "missing"));
}
}
const companyRaw = g("company") || "Unknown";
const leftSquad = / > Left them$/.test(companyRaw);
const company = leftSquad ? companyRaw.replace(/ > Left them$/, "") : companyRaw;
// Leaving your squadmates sends you into the solo pools, so either pool is plausible.
const pool = (SIGHTING_POOL[company + "|" + direction] || []).concat(leftSquad ? SIGHTING_POOL["Alone|" + direction] || [] : []);
if (first && first !== "Unsure" && pool.length && !pool.includes(first))
flags.push(`first sighting ${first} not in documented pool for ${company}/${direction}`);
const cadetRaw = g("cadet");
let scream, resolution, combat = null, combat2 = null;
if ("cadet_actions" in C) {
const act = g("cadet_actions");
const a = CADET_ACTIONS[act] || ["Unknown", null, null, "Unknown"];
if (act && !CADET_ACTIONS[act]) flags.push("unrecognised cadet answer");
[scream, combat, combat2] = a;
const out = CADET_OUTCOME[cadetRaw] || null;
if (a[3] !== null) {
resolution = a[3];
if (out === "DIED in combat" && !resolution.startsWith("DIED")) flags.push("died in combat on a path without combat");
} else resolution = out || "Unknown";
} else {
[scream, resolution] = CADET_MAP[cadetRaw] || ["Unknown", "Unknown"];
if (cadetRaw && !CADET_MAP[cadetRaw]) flags.push("unrecognised cadet answer");
combat = g("combat") ? COMBAT_MAP[g("combat")] || null : null;
}
const postRaw = g("post");
let post = null;
if (scream === "Investigate & help" && !resolution.startsWith("DIED"))
post = ["Forest", "West", "East"].includes(postRaw) ? postRaw : "Unknown";
else if (scream === "Ignore the scream" || scream === "Investigate & leave") post = "(no fork)";
const diedEarly = scream.startsWith("DIED") || resolution.startsWith("DIED");
let hour = HOUR_MAP[g("hour")] || null;
if (diedEarly) hour = null;
else if (hour === null) hour = "Unknown";
const enc = {};
for (const c of COLORS) enc[c] = g("enc_" + c);
const rare = RARE_MAP[g("rare")] || null;
const bonded = COLORS.filter((c) => enc[c] === "Bonded!");
const burned = COLORS.filter((c) => enc[c] === "Incinerated");
let branch = null;
if (hour === "Tree") {
if (enc.Blue || enc.Black || rare === "Shadow") branch = "Shadow (rare)";
else if (enc.Green) branch = "Descend slowly: two Greens";
else if (enc.Red) branch = "Descend quickly: Red";
else if (enc.Orange || enc.Brown) {
branch = "Descend slowly: two Greens";
flags.push("tree run with Orange/Brown but no Green recorded");
}
} else if (hour === "Valley") {
if (enc.Green) flags.push("Green recorded on a Valley run (no Greens in the valley)");
const meadow = ["Orange", "Brown", "Blue"].some((c) => enc[c]) || rare === "Blue (valley)";
if (meadow || enc.Red === "Changed my mind") branch = "Passed the Red: meadow";
else if (enc.Red) branch = "Valley Red";
}
let outcome, outcomeColor = null;
if (diedEarly) outcome = "Died before the hour";
else if (bonded.length === 1) { outcome = "Bonded"; outcomeColor = bonded[0]; }
else if (bonded.length > 1) { outcome = "Ambiguous"; flags.push("more than one dragon marked Bonded"); }
else if (burned.length) {
const order = hour !== "Valley"
? ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
: ["Red", "Orange", "Brown", "Blue", "Green", "Black"];
const inOrder = order.filter((c) => burned.includes(c));
outcome = "Incinerated";
outcomeColor = inOrder[inOrder.length - 1];
if (burned.length > 1) flags.push("more than one dragon marked Incinerated");
} else if (hour === "DIED before the hour") outcome = "Died before the hour";
else outcome = "No outcome recorded";
const rescued = enc.Green === "Saved by another" && outcome === "Bonded" && ["Orange", "Brown"].includes(outcomeColor);
const attempt = parseAttempt(g("attempt"));
if ((rare === "Blue (valley)" && hour === "Tree") || (rare === "Shadow" && hour === "Valley"))
flags.push("rare-dragon answer does not match the path");
const goalRaw = g("goal_v2");
const goal = g("goal") || (goalRaw && goalRaw !== "Unknown" ? GOAL_V2[goalRaw] || goalRaw : null) || "(not asked/blank)";
return {
source, date, wing, section, squad: g("squad") || "Unknown", attempt,
// "I've lost count" means many attempts, so it is a later attempt, not missing
attempt_group: attempt === 1 ? "First attempt" : attempt > 1 || /lost count/i.test(g("attempt") || "") ? "Later attempt" : "Unknown",
goal, company, left_squadmates: leftSquad, direction, first_sighting: first || "Unknown",
sighting_action: "sighting_action" in C ? SIGHTING_ACTION[g("sighting_action")] || g("sighting_action") || "Unknown" : "(not asked)",
scream, resolution, combat_first: combat || "(not recorded)", combat_second: combat2 || "(not recorded)",
post_path: post, hour, branch,
enc_Red: enc.Red, enc_Green: enc.Green, enc_Orange: enc.Orange, enc_Brown: enc.Brown,
enc_Blue: enc.Blue, enc_Black: enc.Black,
outcome, outcome_color: outcomeColor, rescued,
saw_blue_meadow: !!(hour === "Valley" && (enc.Blue || rare === "Blue (valley)")),
saw_shadow: !!(hour === "Tree" && (enc.Blue || enc.Black || rare === "Shadow")),
rare_reported: "rare" in C ? rare || "Unknown" : "(not asked)",
malek_intent: "malek" in C ? g("malek") || "Unknown" : "(not asked)",
tail: g("tail"),
data_flags: flags.join("; "),
death_note: redact(g("death_note")), bond_note: redact(g("bond_note")),
};
}
function codeRows(rows, headers) {
if (!headers) {
const seen = new Set();
rows.forEach((r) => Object.keys(r).forEach((k) => seen.add(k)));
headers = [...seen];
}
const { C, missing } = findColumns(headers);
const runs = rows.map((r, i) => Object.assign({ run_id: i + 1 }, codeRow(r, C)));
return { runs, missing };
}
// ---------------------------------------------------------------- statistics
const Z95 = 1.959963984540054;
function wilson(k, n) {
if (!n) return { k, n, p: null, lo: null, hi: null };
const z = Z95, p = k / n, d = 1 + (z * z) / n;
const c = (p + (z * z) / (2 * n)) / d;
const h = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / d;
return { k, n, p, lo: Math.max(0, c - h), hi: Math.min(1, c + h) };
}
const LOGF = [0];
function logFact(n) {
for (let i = LOGF.length; i <= n; i++) LOGF[i] = LOGF[i - 1] + Math.log(i);
return LOGF[n];
}
// Two-sided Fisher exact test: sum of probabilities of all tables (same margins) no more likely
// than the observed one (same rule as scipy.stats.fisher_exact, with a 1e-7 relative tolerance).
function fisher(a, b, c, d) {
const r1 = a + b, r2 = c + d, c1 = a + c, n = r1 + r2;
const lp = (x) => logFact(r1) + logFact(r2) + logFact(c1) + logFact(n - c1) -
logFact(n) - logFact(x) - logFact(r1 - x) - logFact(c1 - x) - logFact(r2 - c1 + x);
const lo = Math.max(0, c1 - r2), hi = Math.min(r1, c1);
const pObs = lp(a);
let p = 0;
for (let x = lo; x <= hi; x++) {
const q = lp(x);
if (q <= pObs + 1e-7 * Math.abs(pObs) + 1e-12) p += Math.exp(q);
}
return Math.min(1, p);
}
function oddsRatio(a, b, c, d) {
const corrected = a === 0 || b === 0 || c === 0 || d === 0;
const k = corrected ? 0.5 : 0;
const A = a + k, B = b + k, Cc = c + k, D = d + k;
const or = (A * D) / (B * Cc);
const se = Math.sqrt(1 / A + 1 / B + 1 / Cc + 1 / D);
return {
or, or_lo: Math.exp(Math.log(or) - Z95 * se), or_hi: Math.exp(Math.log(or) + Z95 * se),
fisher_p: fisher(a, b, c, d), corrected,
};
}
// Mantel-Haenszel common odds ratio across strata (tables [a,b,c,d]), Robins-Breslow-Greenland 95% CI,
// and the Cochran-Mantel-Haenszel chi-square test (1 df, with continuity correction).
function mantelHaenszel(tables) {
let R = 0, S = 0, PR = 0, PSQR = 0, QS = 0, sumA = 0, sumE = 0, sumV = 0;
for (const [a, b, c, d] of tables) {
const n = a + b + c + d;
if (n < 2) continue;
const r = (a * d) / n, s_ = (b * c) / n, P = (a + d) / n, Q = (b + c) / n;
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_;
const m1 = a + b, m0 = c + d, n1 = a + c, n0 = b + d;
sumA += a; sumE += (m1 * n1) / n;
sumV += (m1 * m0 * n1 * n0) / (n * n * (n - 1));
}
if (!R || !S) return { mh_or: null, mh_lo: null, mh_hi: null, mh_p: null };
const or = R / S;
const varLog = PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S);
const se = Math.sqrt(varLog);
const chi = sumV > 0 ? Math.pow(Math.max(0, Math.abs(sumA - sumE) - 0.5), 2) / sumV : 0;
return { mh_or: or, mh_lo: Math.exp(Math.log(or) - Z95 * se), mh_hi: Math.exp(Math.log(or) + Z95 * se), mh_p: normSf2(Math.sqrt(chi)) };
}
function bh(ps) {
const m = ps.length, order = ps.map((p, i) => [p, i]).sort((x, y) => x[0] - y[0]);
const q = new Array(m);
let prev = 1;
for (let r = m; r >= 1; r--) {
const [p, i] = order[r - 1];
prev = Math.min(prev, (p * m) / r);
q[i] = prev;
}
return q;
}
// Normal CDF (Abramowitz-Stegun 7.1.26 is too coarse for small p; use erfc via continued fraction)
function normSf2(z) { // two-sided p for |z|
z = Math.abs(z);
const t = 1 / (1 + 0.5 * z / Math.SQRT2);
const x = z / Math.SQRT2;
const tau = t * Math.exp(-x * x - 1.26551223 + t * (1.00002368 + t * (0.37409196 + t * (0.09678418 +
t * (-0.18628806 + t * (0.27886807 + t * (-1.13520398 + t * (1.48851587 +
t * (-0.82215223 + t * 0.17087277)))))))));
return Math.min(1, tau); // erfc(x) = 2*P(Z>z)
}
function solve(A, b) { // Gaussian elimination with partial pivoting
const n = b.length, M = A.map((r, i) => r.concat([b[i]]));
for (let i = 0; i < n; i++) {
let p = i;
for (let r = i + 1; r < n; r++) if (Math.abs(M[r][i]) > Math.abs(M[p][i])) p = r;
[M[i], M[p]] = [M[p], M[i]];
for (let r = i + 1; r < n; r++) {
const f = M[r][i] / M[i][i];
for (let c = i; c <= n; c++) M[r][c] -= f * M[i][c];
}
}
const x = new Array(n).fill(0);
for (let i = n - 1; i >= 0; i--) {
let s = M[i][n];
for (let c = i + 1; c < n; c++) s -= M[i][c] * x[c];
x[i] = s / M[i][i];
}
return x;
}
function invert(A) {
const n = A.length;
return A.map((_, j) => solve(A, A.map((__, i) => (i === j ? 1 : 0)))).reduce((acc, col, j) => {
col.forEach((v, i) => (acc[i][j] = v));
return acc;
}, A.map(() => new Array(n).fill(0)));
}
// Logistic regression, Newton-Raphson with an L2 penalty (lambda) on all slopes (not the intercept).
function logisticRidge(X, y, lam = 1, iters = 100) {
const n = X.length, k = X[0].length;
let beta = new Array(k).fill(0);
const info = (b) => {
const H = Array.from({ length: k }, () => new Array(k).fill(0));
const grad = new Array(k).fill(0);
for (let i = 0; i < n; i++) {
let eta = 0;
for (let j = 0; j < k; j++) eta += X[i][j] * b[j];
const mu = 1 / (1 + Math.exp(-eta)), w = mu * (1 - mu);
for (let j = 0; j < k; j++) {
grad[j] += X[i][j] * (y[i] - mu);
if (!X[i][j]) continue;
for (let l = 0; l < k; l++) H[j][l] += X[i][j] * X[i][l] * w;
}
}
for (let j = 1; j < k; j++) { H[j][j] += lam; grad[j] -= lam * b[j]; }
return { H, grad };
};
for (let it = 0; it < iters; it++) {
const { H, grad } = info(beta);
const step = solve(H, grad);
beta = beta.map((v, j) => v + step[j]);
if (Math.max(...step.map(Math.abs)) < 1e-10) break;
}
const { H } = info(beta);
const Hi = invert(H);
const se = Hi.map((r, j) => Math.sqrt(r[j]));
return { beta, se };
}
// ---------------------------------------------------------------- analysis helpers
const rate = (runs, pred) => wilson(runs.filter(pred).length, runs.length);
const groupBy = (arr, key) => arr.reduce((m, r) => {
const k = typeof key === "function" ? key(r) : r[key];
(m[k] = m[k] || []).push(r);
return m;
}, {});
const sortedKeys = (o) => Object.keys(o).sort();
const stratum = (r) => r.source !== "Form" ? "Bulk logs (imports)" :
r.goal === "Blue" ? "Form: chasing Blue" : r.goal === "Black" ? "Form: chasing Black" :
r.goal === "Blue or Black" ? "Form: chasing Blue or Black" : "Form: other/blank goal";
const goal2 = (r) => isRareGoal(r.goal) ? r.goal : r.goal === "(not asked/blank)" ? "Blank/not asked" : "Other colour/any";
// Denominators for each outcome ("who could have had this outcome?")
const OUTCOMES = {
blue_meadow: {
label: "Blue appears in the meadow", denom: (r) => r.branch === "Passed the Red: meadow", event: (r) => r.saw_blue_meadow,
},
shadow: {
label: "Shadow option appears at the tree", denom: (r) => r.hour === "Tree" && r.branch !== null, event: (r) => r.saw_shadow,
},
bond_blue_meadow: {
label: "Bond, given you approached the meadow Blue", denom: (r) => r.branch === "Passed the Red: meadow" && ["Bonded!", "Incinerated"].includes(r.enc_Blue),
event: (r) => r.enc_Blue === "Bonded!",
},
reach_hour: {
label: "Survive to the hour (tree/valley choice)", denom: () => true, event: (r) => r.hour === "Tree" || r.hour === "Valley",
},
};
for (const c of COLORS) {
OUTCOMES["bonded_" + c] = {
label: `Run ends bonded to ${c}`, denom: (r) => r.hour === "Tree" || r.hour === "Valley",
event: (r) => r.outcome === "Bonded" && r.outcome_color === c,
};
}
const FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal", "sighting_action", "combat_second", "malek_intent"];
// Encounter contexts: which recorded dragon attempt belongs to which in-game encounter.
const CONTEXTS = [
["Red (tree, descend quickly)", (r) => r.hour === "Tree" && r.branch === "Descend quickly: Red", "enc_Red", "Red"],
["Green (two Greens)", (r) => r.hour === "Tree" && r.branch === "Descend slowly: two Greens", "enc_Green", "Green"],
["Blue (shadow)", (r) => r.hour === "Tree" && r.branch === "Shadow (rare)", "enc_Blue", "Blue"],
["Black (shadow)", (r) => r.hour === "Tree" && r.branch === "Shadow (rare)", "enc_Black", "Black"],
["Red (valley)", (r) => r.hour === "Valley" && r.branch === "Valley Red", "enc_Red", "Red"],
["Orange (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Orange", "Orange"],
["Brown (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Brown", "Brown"],
["Blue (meadow)", (r) => r.hour === "Valley" && r.branch === "Passed the Red: meadow", "enc_Blue", "Blue"],
];
function applyFilters(runs, f = {}) {
return runs.filter((r) => {
if (f.source === "form" && r.source !== "Form") return false;
if (f.source === "bulk" && r.source === "Form") return false;
if (f.goal === "exclude_rare" && isRareGoal(r.goal)) return false;
if (f.goal === "only_rare" && !isRareGoal(r.goal)) return false;
if (f.excludeFlagged && r.data_flags) return false;
return true;
});
}
function factorTable(runs, outKey, minN = 5) {
const O = OUTCOMES[outKey];
const D = runs.filter(O.denom);
const total = D.length, events = D.filter(O.event).length;
const rows = [], missing = {};
for (const f of FACTORS) {
const groups = groupBy(D.filter((r) => !isMissing(r[f])), f);
missing[f] = D.filter((r) => isMissing(r[f])).length;
const nf = Object.values(groups).reduce((s, g) => s + g.length, 0);
const ef = Object.values(groups).reduce((s, g) => s + g.filter(O.event).length, 0);
for (const lv of sortedKeys(groups)) {
const g = groups[lv], k = g.filter(O.event).length, n = g.length;
const row = Object.assign({ factor: f, level: lv }, wilson(k, n));
const rk = ef - k, rn = nf - n; // rest of the same factor
if (n >= minN && rn >= minN && ef > 0 && ef < nf) Object.assign(row, oddsRatio(k, n - k, rk, rn - rk));
rows.push(row);
}
}
const tested = rows.filter((r) => r.fisher_p !== undefined);
bh(tested.map((r) => r.fisher_p)).forEach((q, i) => (tested[i].q = q));
return { outcome: outKey, label: O.label, overall: wilson(events, total), rows, missing };
}
// Multivariable model: one row per run in the outcome's denominator; categorical factors dummy-coded
// against their most common level.
function model(runs, outKey, factors, lam = 1) {
const O = OUTCOMES[outKey];
const D = runs.filter(O.denom).map((r) => {
const o = { y: O.event(r) ? 1 : 0 };
for (const [name, fn] of Object.entries(factors)) o[name] = fn(r);
return o;
}).filter((o) => Object.keys(factors).every((f) => o[f] !== null && o[f] !== undefined));
if (D.length < 20) return null;
const cols = [["(intercept)", () => 1]], refs = {};
for (const f of Object.keys(factors)) {
const counts = groupBy(D, f);
const levels = sortedKeys(counts).sort((a, b) => counts[b].length - counts[a].length || (a < b ? -1 : 1));
refs[f] = levels[0];
for (const lv of levels.slice(1)) cols.push([`${f} = ${lv}`, (o) => (o[f] === lv ? 1 : 0), f, lv, counts[lv].length]);
}
const X = D.map((o) => cols.map((c) => c[1](o)));
const y = D.map((o) => o.y);
const { beta, se } = logisticRidge(X, y, lam);
const terms = cols.slice(1).map((c, i) => {
const b = beta[i + 1], s = se[i + 1];
return { term: c[0], factor: c[2], level: c[3], n: c[4], or: Math.exp(b), lo: Math.exp(b - Z95 * s), hi: Math.exp(b + Z95 * s), p: normSf2(b / s) };
});
return { outcome: outKey, label: O.label, n: D.length, events: y.reduce((a, b) => a + b, 0), refs, lambda: lam, terms };
}
const routeKey = (r) => [r.company, r.direction, r.first_sighting, r.scream, r.resolution, r.post_path || "—", r.hour || "—"];
const ROUTE_FIELDS = ["company", "direction", "first_sighting", "scream", "resolution", "post_path", "hour"];
function analyse(allRuns, filters = {}) {
const runs = applyFilters(allRuns, filters);
const res = { filters, n: runs.length };
// meta / data quality
const flagCounts = {};
runs.forEach((r) => r.data_flags && r.data_flags.split("; ").forEach((f) => {
const key = f.startsWith("first sighting") ? "first sighting outside documented pool" : f;
flagCounts[key] = (flagCounts[key] || 0) + 1;
}));
const dates = runs.map((r) => r.date).filter(Boolean).sort();
res.meta = {
sources: Object.fromEntries(Object.entries(groupBy(runs, "source")).map(([k, v]) => [k, v.length])),
date_min: dates[0] || null, date_max: dates[dates.length - 1] || null,
n_flagged: runs.filter((r) => r.data_flags).length, flag_counts: flagCounts,
outcomes: Object.fromEntries(Object.entries(groupBy(runs, (r) => r.outcome + (r.outcome_color ? " " + r.outcome_color : ""))).map(([k, v]) => [k, v.length])),
};
// chance events, overall and by stratum
res.chance = {};
for (const key of ["blue_meadow", "shadow"]) {
const O = OUTCOMES[key], D = runs.filter(O.denom);
res.chance[key] = {
label: O.label, overall: rate(D, O.event),
by_stratum: Object.fromEntries(sortedKeys(groupBy(D, stratum)).map((s) => [s, rate(groupBy(D, stratum)[s], O.event)])),
by_goal: Object.fromEntries(sortedKeys(groupBy(D, "goal")).map((s) => [s, rate(groupBy(D, "goal")[s], O.event)])),
by_attempt: Object.fromEntries(sortedKeys(groupBy(D, "attempt_group")).map((s) => [s, rate(groupBy(D, "attempt_group")[s], O.event)])),
};
}
// first sighting pools
const fs = groupBy(runs.filter((r) => !isMissing(r.direction) && !isMissing(r.company)), (r) => r.company + "|" + r.direction);
res.first_sighting = sortedKeys(fs).map((k) => {
const g = fs[k], cnt = groupBy(g, "first_sighting");
return { key: k, n: g.length, colors: Object.fromEntries(sortedKeys(cnt).map((c) => [c, cnt[c].length])) };
});
// bonding encounters
res.encounters = CONTEXTS.map(([name, where, col, color]) => {
const A = runs.filter((r) => where(r) && ["Bonded!", "Incinerated"].includes(r[col]));
const win = (r) => r[col] === "Bonded!";
const intent = A.filter((r) => r.goal === color || r.goal.startsWith("Any") || (r.goal === "Blue or Black" && (color === "Blue" || color === "Black")));
const byS = groupBy(A, stratum);
return {
context: name, color, overall: rate(A, win), intent_matched: rate(intent, win),
by_stratum: Object.fromEntries(sortedKeys(byS).map((s) => [s, rate(byS[s], win)])),
};
});
const greens = runs.filter((r) => r.hour === "Tree" && r.branch === "Descend slowly: two Greens" && r.enc_Green);
res.green_rescue = {
green_encounters: greens.length,
bonded_green: greens.filter((r) => r.enc_Green === "Bonded!").length,
rescued_orange: greens.filter((r) => r.rescued && r.outcome_color === "Orange").length,
rescued_brown: greens.filter((r) => r.rescued && r.outcome_color === "Brown").length,
incinerated: greens.filter((r) => r.outcome === "Incinerated").length,
};
// factor tables
res.factors = {};
for (const key of Object.keys(OUTCOMES)) res.factors[key] = factorTable(runs, key);
// adjusted models
const first2 = (r) => (["Red", "Brown", "Orange", "Green"].includes(r.first_sighting) ? r.first_sighting : null);
const known = (f) => (r) => (isMissing(r[f]) ? null : r[f]);
const src2 = (r) => (r.source === "Form" ? "Form submission" : "Bulk log (import)");
res.models = {
blue_meadow: model(runs, "blue_meadow", {
source: src2, goal: goal2, direction: known("direction"), first_sighting: first2,
resolution: known("resolution"), company: known("company"),
}),
shadow: model(runs, "shadow", {
source: src2, goal: goal2, attempt: known("attempt_group"), direction: known("direction"),
}),
bond_blue_meadow: model(runs, "bond_blue_meadow", {
source: src2, goal: goal2, direction: known("direction"), resolution: known("resolution"),
}),
};
// daily trend (form submissions only, since imports have no date)
const byDate = groupBy(runs.filter((r) => r.date), "date");
res.daily = sortedKeys(byDate).map((d) => {
const g = byDate[d];
const m = g.filter(OUTCOMES.blue_meadow.denom), t = g.filter(OUTCOMES.shadow.denom);
return { date: d, runs: g.length, meadow_n: m.length, blue_k: m.filter((r) => r.saw_blue_meadow).length, tree_n: t.length, shadow_k: t.filter((r) => r.saw_shadow).length };
});
// routes (choices up to the hour) with outcome rates
const R = groupBy(runs.filter((r) => r.hour === "Tree" || r.hour === "Valley"), (r) => routeKey(r).join(" › "));
res.routes = sortedKeys(R).map((k) => {
const g = R[k], m = g.filter(OUTCOMES.blue_meadow.denom), t = g.filter(OUTCOMES.shadow.denom);
const o = { route: k, parts: routeKey(g[0]), n: g.length, blue_meadow: rate(m, (r) => r.saw_blue_meadow), shadow: rate(t, (r) => r.saw_shadow) };
for (const c of COLORS) o["bonded_" + c] = rate(g, (r) => r.outcome === "Bonded" && r.outcome_color === c);
o.bonded_any = rate(g, (r) => r.outcome === "Bonded");
return o;
});
return res;
}
// ---------------------------------------------------------------- route-map graph
const STAGES = [
{ key: "company", label: "Company" },
{ key: "direction", label: "Direction" },
{ key: "first_sighting", label: "First sighting" },
{ key: "cadet", label: "Cadet encounter" },
{ key: "post_path", label: "Fork in the road" },
{ key: "hour", label: "After an hour" },
{ key: "encounter", label: "Encounter" },
{ key: "end", label: "Outcome" },
];
const CHANCE_STAGES = new Set(["first_sighting", "encounter"]); // decided (partly) by the game, not the player
function stageValue(r, key) {
switch (key) {
case "company": return isMissing(r.company) ? null : r.company;
case "direction": return isMissing(r.direction) ? null : r.direction;
case "first_sighting": return r.scream === "DIED at first sighting" || isMissing(r.first_sighting) ? null : r.first_sighting;
case "cadet":
if (r.scream.startsWith("DIED") || isMissing(r.scream) || isMissing(r.resolution)) return null;
return r.scream === "Investigate & help" ? "Help → " + r.resolution : r.scream + " → " + r.resolution;
case "post_path": return isMissing(r.post_path) ? null : r.post_path;
case "hour": return r.hour === "Tree" ? "Climb the tree" : r.hour === "Valley" ? "Valley centre" : null;
case "encounter": {
if (!r.branch) return null;
if (r.hour === "Tree") {
if (r.branch === "Shadow (rare)") return "Tree: shadow → Blue/Black";
if (r.branch === "Descend quickly: Red") return "Tree: Red below";
return "Tree: two Greens";
}
if (r.branch === "Valley Red") return "Valley: bond the Red";
if (r.enc_Blue) return "Meadow: Blue";
if (r.enc_Orange && r.enc_Brown) return "Meadow: Orange/Brown";
if (r.enc_Orange) return "Meadow: Orange";
if (r.enc_Brown) return "Meadow: Brown";
return "Meadow (no attempt recorded)";
}
case "end":
if (r.outcome === "Bonded") return "Bonded " + r.outcome_color;
if (r.outcome === "Incinerated") return "Incinerated";
if (r.outcome === "Died before the hour") return "Died before the hour";
return null;
}
return null;
}
// pool = which runs could have reached this outcome at all (used by rankRoutes as the comparison group)
const BOTH = (r) => r.hour === "Tree" || r.hour === "Valley";
const TREE = (r) => r.hour === "Tree";
const VALLEY = (r) => r.hour === "Valley";
const TARGETS = {
bond_Blue: { label: "Bond Blue", color: "Blue", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Blue", pool: VALLEY },
bond_Black: { label: "Bond Black", color: "Black", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Black", pool: TREE },
see_Blue: { label: "Meet a Blue (meadow)", color: "Blue", fn: (r) => r.saw_blue_meadow, pool: (r) => r.branch === "Passed the Red: meadow" },
see_shadow: { label: "Get the tree shadow", color: "Black", fn: (r) => r.saw_shadow, pool: (r) => r.hour === "Tree" && r.branch !== null },
bond_Red: { label: "Bond Red", color: "Red", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Red", pool: BOTH },
bond_Green: { label: "Bond Green", color: "Green", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Green", pool: TREE },
bond_Orange: { label: "Bond Orange", color: "Orange", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Orange", pool: BOTH },
bond_Brown: { label: "Bond Brown", color: "Brown", fn: (r) => r.outcome === "Bonded" && r.outcome_color === "Brown", pool: BOTH },
bond_any: { label: "Bond any dragon", color: null, fn: (r) => r.outcome === "Bonded", pool: BOTH },
};
// conditions: { stageKey: Set(values) } — a run passes if, for every stage with a condition, its value is in the set.
function buildGraph(runs, targetKey, conditions = {}) {
const T = TARGETS[targetKey].fn;
const sel = runs.filter((r) => Object.entries(conditions).every(([k, set]) => !set.size || set.has(stageValue(r, k))));
const nodes = new Map(), links = new Map();
const node = (si, v) => {
const id = si + "|" + v;
if (!nodes.has(id)) nodes.set(id, { id, stage: si, stageKey: STAGES[si].key, value: v, n: 0, k: 0 });
return nodes.get(id);
};
for (const r of sel) {
const hit = T(r) ? 1 : 0;
let prev = null;
STAGES.forEach((s, si) => {
const v = stageValue(r, s.key);
if (v === null || v === undefined) return;
const nd = node(si, v);
nd.n++; nd.k += hit;
if (prev) {
const id = prev.id + "→" + nd.id;
if (!links.has(id)) links.set(id, { id, source: prev.id, target: nd.id, n: 0, k: 0, chance: CHANCE_STAGES.has(s.key) });
const l = links.get(id); l.n++; l.k += hit;
}
prev = nd;
});
}
const out = (o) => Object.assign(o, wilson(o.k, o.n));
return {
stages: STAGES, selected: sel.length, target: targetKey,
nodes: [...nodes.values()].map(out), links: [...links.values()].map(out),
};
}
// Rank full routes (every choice up to the hour) for a target outcome.
// Each route with at least minN runs is compared with all other runs by Fisher's exact test;
// q-values use Benjamini-Hochberg across all ranked routes. Ranking uses the lower 95% Wilson
// bound, so a route needs both a high rate and enough runs to rank high.
function rankRoutes(runs, targetKey, minN = 10, denomStage = null) {
const T = TARGETS[targetKey].fn;
const keyStages = STAGES.slice(0, 6).map((s) => s.key);
const pool = runs.filter(TARGETS[targetKey].pool);
const totK = pool.filter(T).length, totN = pool.length;
// a route is only ranked from runs whose every step is known; runs with a gap still count in the comparison group
const groups = groupBy(pool.filter((r) => keyStages.every((k) => stageValue(r, k) || (k === "post_path" && r.post_path === null))),
(r) => keyStages.map((k) => stageValue(r, k) || "—").join(" › "));
const rows = Object.entries(groups).filter(([, g]) => g.length >= minN).map(([key, g]) => {
const k = g.filter(T).length, n = g.length;
const o = Object.assign({ route: key, values: keyStages.map((s) => stageValue(g[0], s)) }, wilson(k, n));
Object.assign(o, oddsRatio(k, n - k, totK - k, totN - n - (totK - k)));
// same comparison, adjusted for data source / chase-goal stratum (reporting bias)
const inRoute = new Set(g);
const tabs = Object.values(groupBy(pool, stratum)).map((sg) => {
const a = sg.filter((r) => inRoute.has(r) && T(r)).length, nin = sg.filter((r) => inRoute.has(r)).length;
const c = sg.filter((r) => !inRoute.has(r) && T(r)).length, nout = sg.length - nin;
return [a, nin - a, c, nout - c];
});
Object.assign(o, mantelHaenszel(tabs));
return o;
});
bh(rows.map((r) => r.fisher_p)).forEach((q, i) => (rows[i].q = q));
const mhIdx = rows.map((r, i) => [r.mh_p, i]).filter(([p]) => p !== null);
bh(mhIdx.map(([p]) => p)).forEach((q, j) => (rows[mhIdx[j][1]].mh_q = q));
rows.sort((a, b) => b.lo - a.lo || b.n - a.n);
return { target: targetKey, overall: wilson(totK, totN), minN, n_routes_tested: rows.length, rows };
}
return {
COLORS, MISSING, isMissing, COLUMN_KEYS, STAGES, TARGETS, OUTCOMES, FACTORS, ROUTE_FIELDS, rankRoutes,
findColumns, codeRow, codeRows, parseTimestamp,
wilson, fisher, oddsRatio, bh, logisticRidge, mantelHaenszel,
applyFilters, analyse, factorTable, model, buildGraph, stageValue, stratum,
};
});