Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
61bc50583c
commit
3b24a11f5a
+1
-1
@@ -187,7 +187,7 @@
|
||||
}
|
||||
const FACTOR_LABEL = { company: "Company", direction: "Direction", first_sighting: "First sighting", scream: "Scream choice", resolution: "Cadet encounter",
|
||||
combat_first: "First combat move", post_path: "Fork in the road", hour: "After an hour", wing: "Wing", squad: "Squad",
|
||||
attempt_group: "Attempt", source: "Data source", goal: "Chasing", attempt: "Attempt", period: "Date",
|
||||
attempt_group: "Attempt", entry: "Data source", source: "Data source", goal: "Chasing", attempt: "Attempt", period: "Date",
|
||||
sighting_action: "Reaction to the first dragon", combat_second: "Second combat move", malek_intent: "Planned to seek Malek", };
|
||||
const labelFactor = (f) => FACTOR_LABEL[f] || f;
|
||||
|
||||
|
||||
+31
-2
@@ -64,7 +64,7 @@ button, .filebtn, .btnlike { cursor: pointer; }
|
||||
.btnlike { display: inline-block; text-decoration: none; }
|
||||
button:hover, .filebtn:hover { border-color: var(--accent); }
|
||||
button:focus-visible, select:focus-visible, input:focus-visible, .filebtn:focus-within, a:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
||||
button.primary { background: var(--accent); color: var(--accent-ink); border-color: var(--accent); font-weight: 600; }
|
||||
button.primary, a.btnlike.primary { background: var(--accent); color: var(--accent-ink); border-color: var(--accent); font-weight: 600; }
|
||||
.seg { display: inline-flex; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; }
|
||||
.seg button { border: 0; border-radius: 0; border-right: 1px solid var(--line); }
|
||||
.seg button:last-child { border-right: 0; }
|
||||
@@ -436,6 +436,7 @@ a { color: var(--accent); text-underline-offset: 2px; }
|
||||
<button class="tab" role="tab" id="tab-dragons" aria-controls="panel-dragons" aria-selected="false">The Dragons</button>
|
||||
<button class="tab" role="tab" id="tab-deep" aria-controls="panel-deep" aria-selected="false">Deep Dive</button>
|
||||
<button class="tab" role="tab" id="tab-data" aria-controls="panel-data" aria-selected="false">Data & Methods</button>
|
||||
<button class="tab" role="tab" id="tab-report" aria-controls="panel-report" aria-selected="false">Report a run</button>
|
||||
</nav>
|
||||
</div>
|
||||
</header>
|
||||
@@ -603,7 +604,7 @@ a { color: var(--accent); text-underline-offset: 2px; }
|
||||
<section id="data-sec" aria-labelledby="h-data">
|
||||
<div class="section-head"><span class="eyebrow">Where the numbers come from</span><h2 id="h-data">Data</h2></div>
|
||||
<div class="data-status" id="data-status"><span class="loading">Loading data…</span></div>
|
||||
<p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>. Every column is described in <a href="data/dictionary.yaml">data/dictionary.yaml</a>, and <a href="data/croissant.json">data/croissant.json</a> carries the same description in the Croissant metadata format, with a checksum for every file.</p>
|
||||
<p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>: every run from every source (the community spreadsheet, runs reported here, result pages posted in the Reddit megathreads and the DragonKind bot), with a <code>source</code> column. The statistics on this page use the community spreadsheet's runs; the other sources are coded from different questions and join the statistics once they have been reviewed. Every column is described in <a href="data/dictionary.yaml">data/dictionary.yaml</a>, and <a href="data/croissant.json">data/croissant.json</a> carries the same description in the Croissant metadata format, with a checksum for every file.</p>
|
||||
</section>
|
||||
<section id="share" aria-labelledby="h-share">
|
||||
<div class="section-head"><span class="eyebrow">Open by design</span><h2 id="h-share">Share & reproduce</h2>
|
||||
@@ -684,6 +685,34 @@ adjusted OR = exp(βⱼ), 95% CI = exp(βⱼ ± 1.96·SEⱼ)</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<div class="tabpanel" id="panel-report" role="tabpanel" aria-labelledby="tab-report" hidden>
|
||||
<!-- the links here are the sources in collection/pipeline.yaml -->
|
||||
<section id="report-sec" aria-labelledby="h-report">
|
||||
<div class="section-head"><span class="eyebrow">Every run helps</span><h2 id="h-report">Report a run</h2>
|
||||
<p class="muted">Every run anyone logs ends up in the Atlas's open data, <a href="data/runs_clean.csv" download>runs_clean.csv</a>, which anyone can download and reuse. Deaths are as useful as bonds: the odds on this page are only as good as the runs behind them. Please report each run in one place only.</p></div>
|
||||
<div class="grid2">
|
||||
<div class="panel"><h3>Report it on the Atlas</h3>
|
||||
<p class="small">A short form with the game's own choices (a GitHub issue, so you need a free GitHub account). Reports show up in the data the next day with the source <b>reported</b>. Reports are public: please leave out your account name.</p>
|
||||
<p><a class="btnlike primary" href="https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" target="_blank" rel="noopener">Report a run</a></p>
|
||||
</div>
|
||||
<div class="panel"><h3>Or use the community form</h3>
|
||||
<p class="small">The r/fourthwing megathreads collect runs with u/ClumsyBadger's Google Form, and its answers go into a public spreadsheet. The Atlas reads that spreadsheet every day (source <b>reddit spreadsheet</b>), so a run logged there is counted here too.</p>
|
||||
<p class="btnrow"><a class="btnlike" href="https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0" target="_blank" rel="noopener">The spreadsheet</a>
|
||||
<a class="btnlike" href="https://www.reddit.com/r/fourthwing/comments/1wz5cnj/" target="_blank" rel="noopener">The megathread (with the form)</a></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel"><h3>Where the Atlas's runs come from</h3>
|
||||
<table class="small"><thead><tr><th scope="col">Source</th><th scope="col">What it is</th></tr></thead><tbody>
|
||||
<tr><td>reddit spreadsheet</td><td>The community form and spreadsheet linked from the r/fourthwing megathreads.</td></tr>
|
||||
<tr><td>reported</td><td>Runs reported here, with "Report a run".</td></tr>
|
||||
<tr><td>reddit thread</td><td>Dragonkind result pages posted in the megathreads (the dragon is known, the route is not).</td></tr>
|
||||
<tr><td>bot</td><td>Runs logged by the DragonKind bot.</td></tr>
|
||||
<tr><td>reddit pdf</td><td>The community route PDF: a map of the choices, used to check the game's routes (it holds no runs).</td></tr>
|
||||
</tbody></table>
|
||||
<p class="small muted">What every column means: <a href="data/dictionary.yaml">data/dictionary.yaml</a>.</p>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer><div class="wrap"><p><b>Data:</b> the community <a href="https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0" target="_blank" rel="noopener">"Threshing Routes" responses</a> compiled by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, with percentages in the <a href="https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view" target="_blank" rel="noopener">route PDF</a> by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, <a href="https://www.reddit.com/user/Old_Regular3018/" target="_blank" rel="noopener">u/Old_Regular3018</a> and <a href="https://www.reddit.com/user/Smart_Trifle_2099/" target="_blank" rel="noopener">u/Smart_Trifle_2099</a>; the <a href="https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ" target="_blank" rel="noopener">Unofficial Dragon Register</a>; and the r/fourthwing bonding megathreads (<a href="https://www.reddit.com/r/fourthwing/comments/1wvwfp9/" target="_blank" rel="noopener">Part 3</a>, <a href="https://www.reddit.com/r/fourthwing/comments/1wz5cnj/" target="_blank" rel="noopener">Part 4</a>). Code and data: <a href="https://github.com/ekcSchiano/threshing-atlas" target="_blank" rel="noopener">github.com/ekcSchiano/threshing-atlas</a> (MIT, data CC BY 4.0).</p>
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@
|
||||
function rng(seed) { let s = seed >>> 0; return () => ((s = (s * 1664525 + 1013904223) >>> 0) / 4294967296); }
|
||||
|
||||
// ------------------------------------------------------------ tabs
|
||||
const TABS = ["valley", "odds", "dragons", "deep", "data"];
|
||||
const TABS = ["valley", "odds", "dragons", "deep", "data", "report"];
|
||||
// old links (#overview, #flow, #planner ...) still land somewhere sensible
|
||||
const ALIAS = { overview: "odds", planner: "valley", flow: "deep", rare: "deep", share: "data" };
|
||||
const tabOf = (hash) => { const t = (hash || "").replace(/^#/, "").split("/")[0]; return TABS.includes(t) ? t : ALIAS[t] || null; };
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ copy("core/graph.js", "graph.js");
|
||||
copy("core/solver.js", "solver.js");
|
||||
// The numbers the page shows (all computed in Python) and the published data
|
||||
for (const f of ["stats.json", "register.json", "graph.json", "policy.json"]) copy(`data/public/${f}`, `data/${f}`);
|
||||
copy("data/derived/runs_clean.csv", "data/runs_clean.csv");
|
||||
copy("data/public/runs_clean.csv", "data/runs_clean.csv");
|
||||
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
|
||||
// Source files shown in the page's "view the code" panel
|
||||
copy("core/atlas/coding.py", "python/coding.py");
|
||||
|
||||
+2
-2
@@ -12,7 +12,7 @@ const BASE = "http://localhost:8765/";
|
||||
const p = await b.newPage({ viewport: vp });
|
||||
p.on("pageerror", (e) => log.errors.push(`${vpName} pageerror: ${e.message}`));
|
||||
p.on("console", (m) => { if (m.type() === "error" && !/net::ERR|Failed to load resource/.test(m.text())) log.errors.push(`${vpName} console: ${m.text()}`); });
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data"]) {
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data", "report"]) {
|
||||
await p.goto(BASE + "#" + t); await p.waitForTimeout(t === "valley" ? 2500 : 1800);
|
||||
// open every <details> so hidden text is reviewed too
|
||||
await p.$$eval(`#panel-${t} details`, (ds) => ds.forEach((d) => (d.open = true)));
|
||||
@@ -34,7 +34,7 @@ const BASE = "http://localhost:8765/";
|
||||
// click every button in every tab (desktop), checking for errors after each
|
||||
const p = await b.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
p.on("pageerror", (e) => log.errors.push(`click pageerror: ${e.message}`));
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data"]) {
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data", "report"]) {
|
||||
await p.goto(BASE + "#" + t); await p.waitForTimeout(2000);
|
||||
const n = await p.$$eval(`#panel-${t} button`, (bs) => bs.length);
|
||||
let clicked = 0;
|
||||
|
||||
Reference in New Issue
Block a user