Merge pull request #15 from ekcSchiano/collection/etl
Data collection as an ETL pipeline; every source in runs_clean.csv; Report a run
This commit is contained in:
@@ -0,0 +1,216 @@
|
||||
# Written by collection/etl/report_form.py from collection/specs/report_form.yaml: edit that file, then run `python -m collection build`.
|
||||
name: Report a run
|
||||
description: Log one threshing run for the Threshing Atlas. Reports are public and become rows of runs_clean.csv.
|
||||
title: Run report
|
||||
labels:
|
||||
- run report
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
|
||||
Every answer here is **public** (this is a GitHub issue) and goes into the Atlas's open data as source "reported".
|
||||
Please don't include your DragonKind account name or anything personal.
|
||||
You can also use the community Google Form linked from the r/fourthwing megathread; please report each run only once.
|
||||
- type: dropdown
|
||||
id: goal
|
||||
attributes:
|
||||
label: Which dragon were you trying to bond?
|
||||
options:
|
||||
- Red
|
||||
- Green
|
||||
- Orange
|
||||
- Brown
|
||||
- Blue
|
||||
- Black
|
||||
- Any, just for fun
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: input
|
||||
id: attempt
|
||||
attributes:
|
||||
label: Which attempt was this for your account? (a number; 1 = your first)
|
||||
placeholder: e.g. 3
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: company
|
||||
attributes:
|
||||
label: Who entered the valley with you?
|
||||
options:
|
||||
- Alone
|
||||
- One squadmate
|
||||
- Two squadmates
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: left_squadmates
|
||||
attributes:
|
||||
label: If you went with squadmates, did you leave them?
|
||||
options:
|
||||
- Yes, I left them
|
||||
- No, we stayed together
|
||||
- I went alone
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: direction
|
||||
attributes:
|
||||
label: Which way did you go first?
|
||||
options:
|
||||
- East (toward the roar)
|
||||
- West (toward the scent)
|
||||
- Toward the forest centre
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: first_sighting
|
||||
attributes:
|
||||
label: Which dragon found you first?
|
||||
options:
|
||||
- Red
|
||||
- Brown
|
||||
- Green
|
||||
- Orange
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: sighting_action
|
||||
attributes:
|
||||
label: What did you do when it found you?
|
||||
options:
|
||||
- Hold still
|
||||
- Lower your eyes in respect / turn to face it
|
||||
- Run
|
||||
- Hide
|
||||
- Draw a weapon
|
||||
- Meet its gaze
|
||||
- Protect your squadmate
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: scream
|
||||
attributes:
|
||||
label: The scream
|
||||
options:
|
||||
- I died at the first dragon
|
||||
- Investigated, and helped my squadmate
|
||||
- Investigated, and walked away
|
||||
- Kept walking (ignored it)
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: combat_first
|
||||
attributes:
|
||||
label: 'If you helped: your first move in the fight'
|
||||
options:
|
||||
- Pause to assess
|
||||
- Charge the smaller cadet
|
||||
- Charge between the bigger cadet and your squadmate
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: combat_second
|
||||
attributes:
|
||||
label: 'If you paused to assess: the smaller cadet''s dagger'
|
||||
options:
|
||||
- Engage the smaller cadet
|
||||
- Charge the bigger cadet anyway
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: resolution
|
||||
attributes:
|
||||
label: How did the cadet encounter end?
|
||||
options:
|
||||
- I died in the fight
|
||||
- 'Bullies: killed them myself'
|
||||
- 'Bullies: my squadmate''s revenge'
|
||||
- 'Bullies: left them to the dragons'
|
||||
- 'Wounded cadet: tourniquet'
|
||||
- 'Wounded cadet: left them'
|
||||
- 'Wounded cadet: took their weapons'
|
||||
- I walked away from the clearing
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: post_path
|
||||
attributes:
|
||||
label: 'After the fight: which way did you walk?'
|
||||
options:
|
||||
- Deeper into the forest
|
||||
- West
|
||||
- East
|
||||
- There was no fork
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: hour
|
||||
attributes:
|
||||
label: After an hour's walk
|
||||
options:
|
||||
- Climbed the tallest tree
|
||||
- Went to the valley centre
|
||||
- I died before the hour
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: branch
|
||||
attributes:
|
||||
label: Which bonding encounter did you reach?
|
||||
options:
|
||||
- 'Tree: the shadow (Blue or Black)'
|
||||
- 'Tree: climbed down quickly, jumped on the Red'
|
||||
- 'Tree: climbed down quickly, let the Red pass'
|
||||
- 'Tree: climbed down slowly to the two Greens'
|
||||
- 'Valley: faced the Red'
|
||||
- 'Valley: passed the Red, the meadow'
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: outcome
|
||||
attributes:
|
||||
label: How did the run end?
|
||||
options:
|
||||
- Bonded
|
||||
- Incinerated at a bonding encounter
|
||||
- Died before the hour
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: outcome_color
|
||||
attributes:
|
||||
label: Which dragon bonded you (or incinerated you)?
|
||||
options:
|
||||
- Red
|
||||
- Green
|
||||
- Orange
|
||||
- Brown
|
||||
- Blue
|
||||
- Black
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: tail
|
||||
attributes:
|
||||
label: 'If you bonded: the dragon''s tail'
|
||||
options:
|
||||
- Clubtail
|
||||
- Daggertail
|
||||
- Morningstartail
|
||||
- Scorpiontail
|
||||
- Swordtail
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
@@ -22,16 +22,55 @@ jobs:
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements.txt
|
||||
# The raw store (data/raw: the current and previous raw file of each source) is never committed. Between runs
|
||||
# it lives in the Actions cache, encrypted with the RAW_ARCHIVE_KEY secret, so only the previous version is
|
||||
# kept and nobody without the key can read it (the register has rider names). Without the secret there is no
|
||||
# archive: each run starts from the manifest alone.
|
||||
- name: Restore the private raw archive
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/raw-archive
|
||||
key: raw-archive-none
|
||||
restore-keys: raw-archive-
|
||||
- name: Decrypt it
|
||||
env:
|
||||
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
|
||||
run: |
|
||||
f="$RUNNER_TEMP/raw-archive/raw.tgz.enc"
|
||||
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
|
||||
fi
|
||||
# extract -> transform -> load for every source, then the combined runs (collection/pipeline.yaml)
|
||||
- id: check
|
||||
run: python collection/sources/update_sources.py --report "$RUNNER_TEMP/report.md"
|
||||
- name: Rebuild graph and run tests
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
|
||||
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
|
||||
run: python -m collection update --report "$RUNNER_TEMP/report.md"
|
||||
- name: Rebuild the analysis and run tests
|
||||
if: steps.check.outputs.changed == 'true'
|
||||
run: |
|
||||
node analysis/network/build-graph.js
|
||||
npm run policy
|
||||
npm run stats
|
||||
npm run metadata
|
||||
python -m collection build
|
||||
npm test
|
||||
- name: Encrypt the raw store
|
||||
id: archive
|
||||
env:
|
||||
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
|
||||
run: |
|
||||
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -d data/raw ]; then
|
||||
mkdir -p "$RUNNER_TEMP/raw-archive"
|
||||
tar -czf - -C data raw | openssl enc -aes-256-cbc -pbkdf2 -salt -pass env:RAW_ARCHIVE_KEY -out "$RUNNER_TEMP/raw-archive/raw.tgz.enc"
|
||||
echo "save=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Save it (a new copy only when a raw file changed)
|
||||
if: steps.archive.outputs.save == 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/raw-archive
|
||||
key: raw-archive-${{ hashFiles('data/manifest.json') }}
|
||||
# The PR is opened with ATLAS_BOT_TOKEN (a fine-grained token, see docs/REPO_SETTINGS.md) when it is set:
|
||||
# changes made with the built-in GITHUB_TOKEN do not start other workflows, so its merge would not
|
||||
# redeploy the site. Without the secret it falls back to GITHUB_TOKEN and the PR waits for a manual merge.
|
||||
|
||||
+3
-3
@@ -227,11 +227,11 @@ __pycache__/
|
||||
*.pyc
|
||||
.env
|
||||
.DS_Store
|
||||
# Raw register export contains rider names: keep it local, publish only data/public/register.json
|
||||
data/raw/register/
|
||||
# The raw store (collection/pipeline.yaml): current and previous raw files of every source. Never published: the
|
||||
# register has rider names, the bot's data has game prose and account ids, the PDF is not ours to redistribute.
|
||||
data/raw/
|
||||
# Third-party guide PDF: link to it, do not redistribute
|
||||
*.pdf
|
||||
data/raw/form/*.xlsx
|
||||
|
||||
# QA screenshots
|
||||
undefined/
|
||||
|
||||
@@ -14,18 +14,21 @@ core/ the shared model every part uses. spec/ holds the declarative ru
|
||||
form; game.yaml: the game); atlas/ is the Python package that reads them (coding, route model).
|
||||
solver.js (the route advisor) and graph.js (the bot's text) are the only JavaScript, because
|
||||
they run in the browser and in Ken's bot.
|
||||
collection/ getting data in: sources/ (the form sheet, the Dragon Register, the route PDF check, the daily
|
||||
update job) and bot/ (the DragonKind bot importer). Reddit is not collected yet.
|
||||
collection/ getting data in, as extract -> transform -> load: pipeline.yaml declares every source (the
|
||||
community spreadsheet, Reddit threads, "Report a run", the bot, the route PDF, the register);
|
||||
`python -m collection` runs it and builds data/public/runs_clean.csv from all of them.
|
||||
analysis/ making sense of it: stats/ (measures.yaml and site_stats.py: every number on the page), network/ (builds
|
||||
data/public/graph.json), reports/ (generated reports such as POLICY.md)
|
||||
planner/ the route planner: builds data/public/policy.json and its report; the bot uses it for advice
|
||||
and experiments (docs/BOT_INTEGRATION.md)
|
||||
site/ the Atlas web page (site/atlas/), its build script (site/build.js) and QA checks (site/qa/)
|
||||
data/ raw/ (source exports, not committed), public/ (what the page and the bot load), derived/
|
||||
(coded runs, PDF fingerprint), bot/ (anonymised bot runs), sources.json (download links)
|
||||
data/ public/ (everything published, including runs_clean.csv), dictionary.yaml (every column),
|
||||
croissant.json (dataset metadata), manifest.json (current raw versions), bot/ (anonymised bot
|
||||
runs), derived/ (PDF fingerprint), raw/ (current + previous raw files; never committed)
|
||||
docs/ guides that cross sections: privacy and licences, repo settings, bot data protocol, bot
|
||||
integration, colours, roadmap
|
||||
tests/ core/ (rule, statistics, route model and advisor tests), parity/ (tools to compare two policies), fixtures/
|
||||
tests/ core/ (rule, statistics, route model and advisor tests), collection/ (the pipeline), parity/
|
||||
(tools to compare two policies), fixtures/
|
||||
```
|
||||
|
||||
## Run it locally
|
||||
@@ -34,17 +37,18 @@ Requirements: Node 18+, Python 3.9+ (`pip install -r requirements.txt`).
|
||||
|
||||
```
|
||||
npm run dev # builds dist/ and serves it at http://localhost:8000
|
||||
npm test # coding rules, statistics, route model, game spec, advisor and bot-text tests
|
||||
npm test # pipeline, coding rules, statistics, route model, game spec, advisor and bot-text tests
|
||||
```
|
||||
|
||||
## New data from the community sources
|
||||
|
||||
A GitHub Action (`.github/workflows/update-sources.yml`) runs every day. It downloads the Threshing Routes
|
||||
responses, the Dragon Register and the route PDF, and if anything changed it rebuilds the data, runs the
|
||||
tests and opens a pull request that says what changed: new responses, new or renamed questions, and for
|
||||
the PDF the pages and lines that are new. The PDF itself is never committed, only a fingerprint
|
||||
(`data/derived/pdf_fingerprint.json`). If a link stops working, the Action opens an issue. You can also
|
||||
run it by hand from the Actions tab, or locally with `npm run check-sources`.
|
||||
A GitHub Action (`.github/workflows/update-sources.yml`) runs the collection pipeline every day
|
||||
(`python -m collection update`, see `collection/README.md`). It checks every source in `collection/pipeline.yaml`,
|
||||
and if anything changed it rebuilds the runs and the analysis, runs the tests and opens a pull request that says
|
||||
what changed: new responses, new or renamed questions, new run reports, new result links on Reddit, and for the PDF
|
||||
the pages and lines that are new. The PDF itself is never committed, only a fingerprint. If a source cannot be
|
||||
reached, or new answers do not match `data/dictionary.yaml`, the Action opens an issue instead. Locally:
|
||||
`npm run collect`.
|
||||
|
||||
## Best routes (the solver)
|
||||
|
||||
@@ -60,29 +64,27 @@ shows) may depend on earlier choices; how much is learned from the data. See the
|
||||
The private DragonKind bot's files are converted locally, and only the anonymised result is committed:
|
||||
|
||||
```
|
||||
DK_ACCOUNT_SALT=... npm run import-bot -- --bot /path/to/DragonKind --tag 2026-11-01
|
||||
DRAGONKIND_BOT_DIR=/path/to/DragonKind DK_ACCOUNT_SALT=... python -m collection update --source bot
|
||||
npm run graph
|
||||
```
|
||||
|
||||
This writes `data/bot/scenes.json` and `data/bot/trajectories-<tag>.jsonl`: scene fingerprints, short labels,
|
||||
choice text and outcomes, with no game prose and no account names. The importer refuses to write anything that
|
||||
looks like prose. See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
|
||||
This writes `data/bot/scenes.json` and `data/bot/trajectories-<date>.jsonl`: scene fingerprints, short labels
|
||||
(`collection/specs/bot.yaml`), choice text and outcomes, with no game prose and no account names, and adds the
|
||||
bot's attempts to `runs_clean.csv` (source `bot`). The importer refuses to write anything that looks like prose.
|
||||
See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
|
||||
|
||||
## Updating with new form data by hand
|
||||
|
||||
In the browser: open the page and use **Load a newer export** with the responses spreadsheet
|
||||
(Google Sheets → File → Download → Microsoft Excel). Everything recomputes. Questions are matched by
|
||||
wording; a renamed question is listed as "not found", and then `COLUMN_KEYS` in
|
||||
`core/spec/form.yaml` needs updating.
|
||||
|
||||
To update the bundled snapshot, put the new export in `data/raw/form/`, then:
|
||||
Download the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel), then:
|
||||
|
||||
```
|
||||
python collection/sources/export_raw.py data/raw/form/NEW_EXPORT.xlsx data/public/form_responses.json
|
||||
PYTHONPATH=core python -m atlas.coding data/public/form_responses.json data/derived/runs_clean.csv
|
||||
python -m collection update --source reddit_spreadsheet --file reddit_spreadsheet=path/to/export.xlsx
|
||||
npm test
|
||||
```
|
||||
|
||||
Questions are matched by wording; a renamed question is listed in the report, and then `core/spec/form.yaml`
|
||||
needs updating.
|
||||
|
||||
## Changing a coding rule
|
||||
|
||||
The tables (how each question is found, what each answer means, which answers count as missing) are in
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Build data/public/graph.json from every source: community form runs + DragonKind bot trajectories.
|
||||
// Build data/public/graph.json from every run in data/public/runs_clean.csv (every source), plus the bot's scene graph.
|
||||
// node analysis/network/build-graph.js
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
@@ -6,7 +6,7 @@ const G = require("../../core/graph.js");
|
||||
const root = path.resolve(__dirname, "../..");
|
||||
const rel = (p) => path.join(root, p);
|
||||
|
||||
/** The coded runs written by the Python coding rules (python -m atlas.coding -> data/derived/runs_clean.csv). */
|
||||
/** The coded runs of every source, written by `python -m collection build`. */
|
||||
function readCSV(file) {
|
||||
const text = fs.readFileSync(file, "utf8"), rows = [];
|
||||
let row = [], cell = "", q = false;
|
||||
@@ -25,8 +25,10 @@ function readCSV(file) {
|
||||
const [head, ...body] = rows;
|
||||
return body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] === "" || r[i] === undefined ? null : r[i]])));
|
||||
}
|
||||
const runs = readCSV(rel("data/derived/runs_clean.csv"));
|
||||
const records = runs.map((r) => ({ source: r.source === "Form" ? "form" : "form-bulk", vars: G.formRunVariables(r) }));
|
||||
const runs = readCSV(rel("data/public/runs_clean.csv"));
|
||||
// spreadsheet runs keep the split between form submissions and bulk logs
|
||||
const sourceKey = (r) => (r.source === "reddit spreadsheet" ? (r.entry === "Form" ? "form" : "form-bulk") : r.source);
|
||||
const records = runs.map((r) => ({ source: sourceKey(r), vars: G.formRunVariables(r) }));
|
||||
|
||||
const botDir = rel("data/bot");
|
||||
const scenes = fs.existsSync(path.join(botDir, "scenes.json")) ? JSON.parse(fs.readFileSync(path.join(botDir, "scenes.json"), "utf8")) : {};
|
||||
@@ -34,12 +36,6 @@ const trajs = [];
|
||||
for (const f of fs.readdirSync(botDir).filter((f) => /^trajectories-.*\.jsonl$/.test(f)).sort()) {
|
||||
for (const line of fs.readFileSync(path.join(botDir, f), "utf8").split("\n")) if (line.trim()) trajs.push(JSON.parse(line));
|
||||
}
|
||||
const unmapped = {};
|
||||
for (const t of trajs) {
|
||||
const { vars, unmapped: u } = G.trajectoryVariables(t);
|
||||
records.push({ source: "bot", vars });
|
||||
for (const x of u) unmapped[x] = (unmapped[x] || 0) + 1;
|
||||
}
|
||||
|
||||
const graph = {
|
||||
sources: records.reduce((m, r) => ((m[r.source] = (m[r.source] || 0) + 1), m), {}),
|
||||
@@ -48,5 +44,3 @@ const graph = {
|
||||
};
|
||||
fs.writeFileSync(rel("data/public/graph.json"), JSON.stringify(graph));
|
||||
console.log(`graph.json: ${JSON.stringify(graph.sources)}, ${Object.keys(graph.variables.nodes).length} variable nodes, ${Object.keys(graph.scenes).length} scenes`);
|
||||
const um = Object.keys(unmapped).filter((t) => !/^(head deeper|investigate the scream|lower your|show your bravery|hide behind|run and hope|reds are too|jump|daggers or not|engage the smaller)/i.test(t));
|
||||
if (um.length) console.log("bot choices with no form equivalent:", um);
|
||||
|
||||
@@ -6,6 +6,11 @@
|
||||
# (not blank), {any: [condition, ...]} (at least one). Several fields in one condition must all hold.
|
||||
version: 1
|
||||
|
||||
# Which runs of data/public/runs_clean.csv the statistics use (its `source` column). Runs from the other sources
|
||||
# (reported, bot, Reddit result pages) are in the published file but not in these statistics yet: they are coded
|
||||
# from different questions, and the bot follows a strategy, so pooling them needs its own review.
|
||||
sources: [reddit spreadsheet]
|
||||
|
||||
colours: [Red, Green, Orange, Brown, Blue, Black]
|
||||
|
||||
# Outcomes for the factor tables and models: who could have had the outcome (denom), and the outcome (event).
|
||||
@@ -34,7 +39,7 @@ bonded_outcome:
|
||||
|
||||
# Factors compared in the factor tables (each answer against the other answers to the same question)
|
||||
factors: [company, direction, first_sighting, scream, resolution, combat_first, post_path, hour, wing, squad,
|
||||
attempt_group, source, goal, sighting_action, combat_second, malek_intent]
|
||||
attempt_group, entry, goal, sighting_action, combat_second, malek_intent]
|
||||
|
||||
# Route targets: the outcome (event) and which runs could have reached it at all (pool)
|
||||
targets:
|
||||
|
||||
@@ -19,6 +19,7 @@ ROOT = HERE.parents[1]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
sys.path.insert(0, str(HERE))
|
||||
from atlas import coding, model # noqa: E402
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
from stats_core import Z95, bh, logistic_ridge, mantel_haenszel, odds_ratio, p_from_z # noqa: E402
|
||||
from stats_core import wilson as _wilson # noqa: E402
|
||||
|
||||
@@ -95,7 +96,7 @@ def is_rare(goal):
|
||||
|
||||
|
||||
def stratum(r):
|
||||
if r["source"] != "Form":
|
||||
if r["entry"] != "Form":
|
||||
return "Bulk logs (imports)"
|
||||
g = r["goal"]
|
||||
return {"Blue": "Form: chasing Blue", "Black": "Form: chasing Black", "Blue or Black": "Form: chasing Blue or Black"}.get(g, "Form: other/blank goal")
|
||||
@@ -107,9 +108,9 @@ def goal2(r):
|
||||
|
||||
def apply_filters(runs, f):
|
||||
def ok(r):
|
||||
if f.get("source") == "form" and r["source"] != "Form":
|
||||
if f.get("source") == "form" and r["entry"] != "Form":
|
||||
return False
|
||||
if f.get("source") == "bulk" and r["source"] == "Form":
|
||||
if f.get("source") == "bulk" and r["entry"] == "Form":
|
||||
return False
|
||||
if f.get("goal") == "exclude_rare" and is_rare(r["goal"]):
|
||||
return False
|
||||
@@ -196,7 +197,7 @@ def analyse(all_runs, filters):
|
||||
dates = sorted(r["date"] for r in runs if r.get("date"))
|
||||
outs = group_by(runs, lambda r: r["outcome"] + (" " + r["outcome_color"] if r.get("outcome_color") else ""))
|
||||
res["meta"] = {
|
||||
"sources": {k: len(v) for k, v in group_by(runs, "source").items()},
|
||||
"sources": {k: len(v) for k, v in group_by(runs, "entry").items()},
|
||||
"date_min": dates[0] if dates else None, "date_max": dates[-1] if dates else None,
|
||||
"n_flagged": sum(1 for r in runs if r["data_flags"]), "flag_counts": flag_counts,
|
||||
"outcomes": {k: len(v) for k, v in outs.items()},
|
||||
@@ -208,7 +209,7 @@ def analyse(all_runs, filters):
|
||||
D = [r for r in runs if match(r, O["denom"])]
|
||||
ev = lambda r, O=O: match(r, O["event"]) # noqa: E731
|
||||
by = lambda k: {s: rate(g, ev) for s, g in sorted(group_by(D, k).items())} # noqa: E731
|
||||
form = [r for r in D if r["source"] == "Form"]
|
||||
form = [r for r in D if r["entry"] == "Form"]
|
||||
res["chance"][key] = {"label": O["label"], "overall": rate(D, ev), "by_stratum": by(stratum), "by_goal": by("goal"),
|
||||
"by_attempt": by("attempt_group"), "form_overall": rate(form, ev)}
|
||||
|
||||
@@ -239,7 +240,7 @@ def analyse(all_runs, filters):
|
||||
|
||||
first2 = lambda r: r["first_sighting"] if r["first_sighting"] in ("Red", "Brown", "Orange", "Green") else None # noqa: E731
|
||||
known = lambda f: (lambda r: None if is_missing(r.get(f)) else r[f]) # noqa: E731
|
||||
src2 = lambda r: "Form submission" if r["source"] == "Form" else "Bulk log (import)" # noqa: E731
|
||||
src2 = lambda r: "Form submission" if r["entry"] == "Form" else "Bulk log (import)" # noqa: E731
|
||||
res["models"] = {
|
||||
"blue_meadow": fit_model(runs, "blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"),
|
||||
"first_sighting": first2, "resolution": known("resolution"), "company": known("company")}),
|
||||
@@ -256,7 +257,7 @@ def analyse(all_runs, filters):
|
||||
"blue": W(bk, len(m)), "shadow": W(sk, len(t))})
|
||||
|
||||
res["source_rates"] = {key: {s: rate([r for r in g if match(r, OUTCOMES[key]["denom"])], lambda r, k=key: match(r, OUTCOMES[k]["event"]))
|
||||
for s, g in group_by(runs, "source").items()} for key in ("blue_meadow", "shadow")}
|
||||
for s, g in group_by(runs, "entry").items()} for key in ("blue_meadow", "shadow")}
|
||||
return res
|
||||
|
||||
|
||||
@@ -508,23 +509,25 @@ def build(runs, rows, exported=None):
|
||||
return clean(out)
|
||||
|
||||
|
||||
def coded_runs(path):
|
||||
df, _ = coding.load(path)
|
||||
df = df.drop(columns=["run_id"])
|
||||
return df.astype(object).where(df.notna(), None).to_dict("records")
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--data", default=str(ROOT / "data/public/form_responses.json"))
|
||||
ap.add_argument("--data", default=str(runs_io.PATH))
|
||||
ap.add_argument("--out", default=str(ROOT / "data/public/stats.json"))
|
||||
a = ap.parse_args(argv)
|
||||
raw = json.load(open(a.data, encoding="utf-8"))
|
||||
runs = coded_runs(a.data)
|
||||
out = build(runs, len(raw["rows"]), raw.get("exported"))
|
||||
runs = runs_io.read(a.data, sources=M["sources"])
|
||||
out = build(runs, len(runs), exported())
|
||||
Path(a.out).write_text(json.dumps(out, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
|
||||
print(f"{a.out}: {len(runs)} runs, {len(out['presets'])} presets, {Path(a.out).stat().st_size / 1e6:.1f} MB")
|
||||
|
||||
|
||||
def exported():
|
||||
"""When the community spreadsheet last changed (data/manifest.json), else the date in its public file."""
|
||||
m = ROOT / "data/manifest.json"
|
||||
entry = json.loads(m.read_text(encoding="utf-8")).get("reddit_spreadsheet") if m.exists() else None
|
||||
if entry:
|
||||
return entry["changed"]
|
||||
return json.loads((ROOT / "data/public/form_responses.json").read_text(encoding="utf-8")).get("exported")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
+50
-13
@@ -1,19 +1,56 @@
|
||||
# collection
|
||||
|
||||
Getting data in. Everything here turns a source into the shared run format (see `core/`) or into public files
|
||||
under `data/`.
|
||||
Getting data in, as a strict **extract → transform → load** pipeline. Every source is declared in
|
||||
[`pipeline.yaml`](pipeline.yaml); the code only carries those declarations out.
|
||||
|
||||
| Source | Code | Output |
|
||||
```
|
||||
python -m collection update # every source: extract, then transform + load if it changed; then build
|
||||
python -m collection update --source reported --source register
|
||||
python -m collection update --file reddit_spreadsheet=~/Downloads/export.xlsx # a local file instead of the download
|
||||
python -m collection build # code every public source file into data/public/runs_clean.csv
|
||||
python -m collection build --check # nothing written; exit 1 if anything is out of date (CI runs this)
|
||||
```
|
||||
|
||||
## The stages
|
||||
|
||||
| Stage | Does | Never does |
|
||||
|---|---|---|
|
||||
| Community "Threshing Routes" form (Google Sheet) | `sources/export_raw.py`, then `core/atlas/coding.py` | `data/public/form_responses.json`, `data/derived/runs_clean.csv` |
|
||||
| Unofficial Dragon Register (Google Sheet) | `sources/export_register.py` | `data/public/register.json` (rider names stay local) |
|
||||
| Community route PDF | `sources/update_sources.py` | `data/derived/pdf_fingerprint.json` only; the PDF is never committed |
|
||||
| DragonKind bot (Ken) | `bot/import-bot.js`, `bot/bot.js` | `data/bot/scenes.json`, `data/bot/trajectories-<tag>.jsonl` |
|
||||
| Reddit megathreads | not collected yet | — |
|
||||
| **extract** (`etl/extract.py`) | Fetches the source as it is now into `data/raw/<source>/current.<ext>`. The version it replaces becomes `previous.<ext>`; nothing older is kept. `data/manifest.json` records the current raw file's sha256 and the day it last changed. | Change the data. |
|
||||
| **transform** (`etl/transform/`) | Turns the raw file into the public source file, applying the source's privacy rules. A pure function of the raw file: no network, no clock. | Write files. |
|
||||
| **validate** (`etl/validate.py`) | Checks the combined runs against `data/dictionary.yaml` (types, every value listed, unique ids). A failure stops the load and is reported. | Fix anything silently. |
|
||||
| **load** (`etl/load.py`) | The only stage that writes public files. Byte-stable: the same input always gives the same bytes, and a file is only rewritten when its bytes change. | Publish anything raw. |
|
||||
|
||||
`sources/update_sources.py` runs every day from `.github/workflows/update-sources.yml`. It downloads the sheets
|
||||
and the PDF (links in `data/sources.json`), rebuilds the data if anything changed, and opens a pull request that
|
||||
merges itself when CI passes. A broken link opens an issue.
|
||||
`build` then codes every public source file into one run per row (`etl/runs.py`, one coder per source) and writes
|
||||
`data/public/runs_clean.csv`, plus the files generated from the specs: the "Report a run" issue form, the bot
|
||||
crosswalk inside `core/graph.js`, and `data/croissant.json`. Every input to `build` is committed, so CI rebuilds
|
||||
it exactly.
|
||||
|
||||
The bot importer only ever writes fingerprints, short labels, choice text and outcomes. It refuses anything that
|
||||
looks like game prose or an account identifier. What the bot should log is in `docs/BOT_DATA_PROTOCOL.md`.
|
||||
## The sources
|
||||
|
||||
| `source` in runs_clean.csv | Pipeline source | Extract | Public source file |
|
||||
|---|---|---|---|
|
||||
| reddit spreadsheet | `reddit_spreadsheet`: u/ClumsyBadger's Threshing Routes form and spreadsheet, linked from the megathreads | the sheet's .xlsx export | `data/public/form_responses.json` (emails redacted), coded by `core/spec/form.yaml` |
|
||||
| reddit thread | `reddit_thread`: dragonkind.com result pages posted in the r/fourthwing megathreads | the threads, through the official Reddit API (needs `REDDIT_CLIENT_ID` / `REDDIT_CLIENT_SECRET`) | `manual/reddit_results.yaml`, checked by hand; the extract lists result links not in it yet |
|
||||
| reported | `reported`: the Atlas's "Report a run" form (a GitHub issue form, `specs/report_form.yaml`) | issues labelled `run report`, through the GitHub API | `data/public/reports.json` (answers only, no usernames) |
|
||||
| bot | `bot`: Ken's DragonKind bot | a local copy of the bot's private repo (`DRAGONKIND_BOT_DIR`) | `data/bot/scenes.json`, `data/bot/trajectories-<date>.jsonl` (no prose, accounts hashed with `DK_ACCOUNT_SALT`; `specs/bot.yaml`) |
|
||||
| reddit pdf | `reddit_pdf`: the community route PDF | the PDF | `data/derived/pdf_fingerprint.json` only: hashes, never the text |
|
||||
| — | `register`: the Unofficial Dragon Register | the sheet's .xlsx export | `data/public/register.json` (rider names and photos dropped) |
|
||||
|
||||
The route PDF is a flowchart of the choices, not a log of runs, so it adds no rows to `runs_clean.csv`; its
|
||||
fingerprint tells us when the routes it documents change. The register lists bonded dragons without the route that
|
||||
led there, so it adds no rows either.
|
||||
|
||||
## Raw files and privacy
|
||||
|
||||
`data/raw/` is git-ignored and never published. It holds the current raw file of each source and the one before
|
||||
it; older versions are not kept. The daily workflow keeps that store between runs in the Actions cache, encrypted
|
||||
with the `RAW_ARCHIVE_KEY` secret (see `docs/REPO_SETTINGS.md`). Privacy rules sit with each source in
|
||||
`pipeline.yaml`: emails redacted, register names and photos dropped, the bot's fields refused if they look like
|
||||
prose or account ids, GitHub and Reddit usernames never kept.
|
||||
|
||||
## Adding a source
|
||||
|
||||
1. Add a block to `pipeline.yaml` (extract kind, transform, privacy, load target, and `run_source` / `runs` if it
|
||||
has runs).
|
||||
2. If it needs a new transform or coder, add one module to `etl/transform/` and one function to `etl/runs.py`.
|
||||
3. Add its `source` value to `data/dictionary.yaml`, then `python -m collection build` and `npm test`.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from collection.run import main
|
||||
|
||||
raise SystemExit(main())
|
||||
@@ -1,211 +0,0 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Import DragonKind bot data into anonymised trajectories and a scene catalogue.
|
||||
*
|
||||
* Inputs (all from the PRIVATE bot repo, read locally, never committed):
|
||||
* data/knowledge.json scene atlas: nodes (prose, choices, tint, backdrop) and edges
|
||||
* data/runs.jsonl one record per attempt (newer bot versions)
|
||||
* logs/encounters.jsonl every decision the bot made, with the options on screen
|
||||
*
|
||||
* Outputs (public, CC BY 4.0): no game prose, only fingerprints, short labels and choice text.
|
||||
* scenes.json { sceneId: { stage, tint, backdrop, label, options: [{ id, text }] } }
|
||||
* trajectories.jsonl one line per attempt: { id, source, meta, steps: [...], outcome }
|
||||
*/
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const sha1 = (s, n = 12) => crypto.createHash('sha1').update(String(s)).digest('hex').slice(0, n);
|
||||
|
||||
// Must match normalize() in the bot's src/strategy.js, so fingerprints line up.
|
||||
function normalize(text) {
|
||||
return String(text || '')
|
||||
.toLowerCase()
|
||||
.replace(/[‘’‛`']/g, "'")
|
||||
.replace(/[“”"]/g, '"')
|
||||
.replace(/[–—-]/g, ' ')
|
||||
.replace(/…/g, ' ')
|
||||
.replace(/[^a-z0-9'\s]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// Must match computeSceneFingerprint() in the bot's src/learner.js.
|
||||
function sceneId(prose, choices) {
|
||||
const proseNorm = normalize(prose).slice(0, 500);
|
||||
const sorted = choices.map(normalize).filter(Boolean).sort();
|
||||
return sha1(`${proseNorm} ### ${sorted.join(' || ')}`);
|
||||
}
|
||||
|
||||
const optionId = (text) => sha1(normalize(text), 10);
|
||||
|
||||
const readJsonl = (file) => {
|
||||
if (!file || !fs.existsSync(file)) return [];
|
||||
return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter((l) => l.trim()).map((l) => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
};
|
||||
|
||||
/**
|
||||
* Short public labels. Keys are scene fingerprints; values are our own paraphrases, not game prose.
|
||||
* Unknown scenes fall back to "<stage> · <tint>" and are listed by the importer so labels can be added.
|
||||
*/
|
||||
const LABELS = {
|
||||
db7786392626: 'Start: choose company',
|
||||
a68f681076c6: 'Valley edge: choose direction',
|
||||
bc4ee4754b9c: 'First sighting: Red (forest centre)',
|
||||
abe3011378ac: 'First sighting: Brown (forest centre)',
|
||||
'84780dab01e3': 'Survived the first sighting',
|
||||
d25f2759ace0: 'Deeper in the forest: a scream',
|
||||
f72199f24b91: 'Clearing: squadmate vs two bullies',
|
||||
b209bacacec3: 'Bully fight: first move',
|
||||
fbe0f0bea700: 'Bully fight: smaller cadet has a dagger',
|
||||
a2368b0dc212: 'Blocked the bigger cadet: bullies’ fate',
|
||||
f4b4944d3f3b: 'Beat the smaller cadet: bullies’ fate',
|
||||
'0cfd06cf8939': 'After the fight: choose a direction',
|
||||
'0a561d53ad19': 'An hour later: valley or tree',
|
||||
'96b68612eaf4': 'Top of the tree (shadow check)',
|
||||
'10bc7353b8eb': 'Descended quickly: a Red below',
|
||||
c62f1f1a2b3a: 'Let the Red pass: two Greens in the clearing',
|
||||
'07e24681e3b3': 'Descended slowly: two Greens in the clearing'
|
||||
};
|
||||
|
||||
function labelFor(id, node) {
|
||||
if (LABELS[id]) return LABELS[id];
|
||||
const bits = [node.stage || 'Scene'];
|
||||
if (node.tint) bits.push(node.tint);
|
||||
return bits.join(' · ');
|
||||
}
|
||||
|
||||
/** Scene catalogue from knowledge.json (and, for scenes only in the log, from the log). */
|
||||
function buildScenes(knowledge, encounters) {
|
||||
const scenes = {};
|
||||
for (const [id, n] of Object.entries((knowledge && knowledge.nodes) || {})) {
|
||||
scenes[id] = {
|
||||
stage: n.stage || 'UNKNOWN',
|
||||
tint: n.tint || null,
|
||||
backdrop: n.backdrop || null,
|
||||
label: labelFor(id, n),
|
||||
options: Object.values(n.choices || {}).map((c) => ({ id: optionId(c.label || c.choiceNorm), text: c.label || c.choiceNorm }))
|
||||
};
|
||||
}
|
||||
for (const e of encounters) {
|
||||
if (!Array.isArray(e.choices) || !e.prose) continue;
|
||||
const id = e.nodeId || sceneId(e.prose, e.choices);
|
||||
if (scenes[id]) continue;
|
||||
const stage = (e.decision && e.decision.stage) || e.stage || 'UNKNOWN';
|
||||
scenes[id] = { stage, tint: null, backdrop: null, label: labelFor(id, { stage }), options: e.choices.map((t) => ({ id: optionId(t), text: t })) };
|
||||
}
|
||||
return scenes;
|
||||
}
|
||||
|
||||
const CHOICE_EVENTS = new Set(['DECISION', 'MANUAL_DECISION', 'MANUAL_EDGE_PICK']);
|
||||
|
||||
/**
|
||||
* Trajectories from the encounter log: group events by attempt, keep the decisions that were acted on,
|
||||
* and close each attempt at its first BURNED / DRAGON_BONDED event. Repeated BURNED events with no
|
||||
* decisions in between (the cooldown screen being re-detected) are ignored.
|
||||
*/
|
||||
function trajectoriesFromEncounters(encounters, scenes) {
|
||||
const byAttempt = new Map();
|
||||
for (const e of encounters) {
|
||||
const key = e.account ? `${e.account}:${e.attempt}` : String(e.attempt);
|
||||
if (!byAttempt.has(key)) byAttempt.set(key, []);
|
||||
byAttempt.get(key).push(e);
|
||||
}
|
||||
const out = [];
|
||||
for (const [key, events] of byAttempt) {
|
||||
events.sort((a, b) => String(a.ts).localeCompare(String(b.ts)));
|
||||
const steps = [];
|
||||
let outcome = null;
|
||||
for (const e of events) {
|
||||
if (outcome) break;
|
||||
if (CHOICE_EVENTS.has(e.event)) {
|
||||
const d = e.decision || {};
|
||||
const text = d.choiceText || e.choiceText;
|
||||
if (d.action && d.action !== 'choose') continue; // paused, nothing happened
|
||||
if (!text || !Array.isArray(e.choices)) continue;
|
||||
const scene = e.nodeId || sceneId(e.prose || '', e.choices);
|
||||
const manual = e.event !== 'DECISION';
|
||||
steps.push({
|
||||
scene,
|
||||
stage: d.stage || (scenes[scene] && scenes[scene].stage) || 'UNKNOWN',
|
||||
tint: (scenes[scene] && scenes[scene].tint) || null,
|
||||
option: optionId(text),
|
||||
optionText: text,
|
||||
offered: e.choices.map(optionId),
|
||||
rule: d.rule || null,
|
||||
decisionMode: manual ? 'manual' : d.source || null,
|
||||
manual,
|
||||
at: e.ts
|
||||
});
|
||||
} else if (e.event === 'BURNED' && steps.length) {
|
||||
outcome = { type: 'BURNED', colour: null, intentional: Boolean(e.intentional) };
|
||||
} else if (e.event === 'DRAGON_BONDED' && steps.length) {
|
||||
outcome = { type: 'BONDED', colour: e.dragon ? String(e.dragon).toLowerCase() : null, intentional: false };
|
||||
}
|
||||
}
|
||||
// A decision repeated on the same scene (the bot re-detecting the screen) counts once.
|
||||
const dedup = steps.filter((s, i) => !(i > 0 && steps[i - 1].scene === s.scene && steps[i - 1].option === s.option));
|
||||
for (let i = 0; i < dedup.length; i++) dedup[i].to = i + 1 < dedup.length ? dedup[i + 1].scene : null;
|
||||
if (!dedup.length) continue;
|
||||
const last = dedup[dedup.length - 1];
|
||||
if (outcome) last.to = outcome.type === 'BURNED' ? 'BURNED' : `BONDED:${outcome.colour || 'unknown'}`;
|
||||
out.push({
|
||||
id: `bot-${sha1(`${key}:${dedup[0].at}`, 10)}`,
|
||||
source: 'bot',
|
||||
meta: { attempt: Number(String(key).split(':').pop()) || null, startedAt: dedup[0].at, accountKey: null, target: null, decisionMode: null },
|
||||
steps: dedup,
|
||||
outcome: outcome || { type: 'ABORTED', colour: null, intentional: false }
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Fill run-level metadata (target colour, mode) from runs.jsonl where an attempt is covered by both. */
|
||||
function attachRunMeta(trajs, runs, salt) {
|
||||
for (const r of runs) {
|
||||
const t = trajs.find((x) => x.meta.attempt === r.attempt && Math.abs(Date.parse(x.meta.startedAt) - Date.parse(r.startedAt)) < 6 * 3600e3);
|
||||
if (!t) continue;
|
||||
t.meta.target = r.targetDragon || null;
|
||||
t.meta.decisionMode = r.decisionMode || null;
|
||||
t.meta.policyId = r.policyId || null;
|
||||
t.meta.experimentId = r.experimentId || null;
|
||||
t.meta.arm = r.arm || null;
|
||||
if (r.accountKey) t.meta.accountKey = r.accountKey;
|
||||
else if (r.accountId && salt) t.meta.accountKey = sha1(`${salt}:${r.accountId}`, 10);
|
||||
if (r.outcome === 'BONDED' && r.dragonColor) t.outcome = { type: 'BONDED', colour: String(r.dragonColor).toLowerCase(), intentional: false };
|
||||
if (r.intentionalBurn) t.outcome.intentional = true;
|
||||
}
|
||||
return trajs;
|
||||
}
|
||||
|
||||
function importBot({ dataDir, logsDir, salt = process.env.DK_ACCOUNT_SALT || null }) {
|
||||
const knowledgeFile = path.join(dataDir, 'knowledge.json');
|
||||
const knowledge = fs.existsSync(knowledgeFile) ? JSON.parse(fs.readFileSync(knowledgeFile, 'utf8')) : { nodes: {} };
|
||||
const encounters = readJsonl(path.join(logsDir, 'encounters.jsonl'));
|
||||
const runs = readJsonl(path.join(dataDir, 'runs.jsonl'));
|
||||
const scenes = buildScenes(knowledge, encounters);
|
||||
const trajectories = attachRunMeta(trajectoriesFromEncounters(encounters, scenes), runs, salt);
|
||||
const unlabeled = Object.entries(scenes).filter(([id]) => !LABELS[id]).map(([id, s]) => `${id} (${s.stage}${s.tint ? `, ${s.tint}` : ''})`);
|
||||
return { scenes, trajectories, unlabeled };
|
||||
}
|
||||
|
||||
/** Guard: nothing that looks like game prose or an account identifier may leave this module. */
|
||||
function assertPublicSafe(obj) {
|
||||
const banned = ['prose', 'proseNorm', 'proseExcerpt', 'email', 'password', 'accountId', 'accountName'];
|
||||
const walk = (v, p) => {
|
||||
if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${p}[${i}]`));
|
||||
if (v && typeof v === 'object') {
|
||||
for (const [k, x] of Object.entries(v)) {
|
||||
if (banned.includes(k)) throw new Error(`refusing to export field "${k}" at ${p}`);
|
||||
walk(x, `${p}.${k}`);
|
||||
}
|
||||
} else if (typeof v === 'string' && v.length > 200) {
|
||||
throw new Error(`refusing to export a ${v.length}-character string at ${p} (looks like prose)`);
|
||||
}
|
||||
};
|
||||
walk(obj, '$');
|
||||
}
|
||||
|
||||
module.exports = { normalize, sceneId, optionId, buildScenes, trajectoriesFromEncounters, attachRunMeta, importBot, assertPublicSafe, LABELS };
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
/**
|
||||
* Convert a local copy of the DragonKind bot's data into the public, anonymised format.
|
||||
*
|
||||
* node collection/bot/import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag 2026-11-01]
|
||||
*
|
||||
* Reads <bot>/data/knowledge.json, <bot>/data/runs.jsonl and <bot>/logs/encounters.jsonl.
|
||||
* Writes <out>/scenes.json (merged with any existing catalogue) and <out>/trajectories-<tag>.jsonl.
|
||||
* Set DK_ACCOUNT_SALT in the environment to hash account ids; never commit the salt.
|
||||
*/
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { importBot, assertPublicSafe } = require('./bot');
|
||||
|
||||
const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {
|
||||
if (a.startsWith('--')) acc.push([a.slice(2), all[i + 1]]);
|
||||
return acc;
|
||||
}, []));
|
||||
if (!args.bot) {
|
||||
console.error('usage: import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag YYYY-MM-DD]');
|
||||
process.exit(2);
|
||||
}
|
||||
const out = path.resolve(args.out || 'data/bot');
|
||||
const tag = args.tag || new Date().toISOString().slice(0, 10);
|
||||
|
||||
const { scenes, trajectories, unlabeled } = importBot({
|
||||
dataDir: path.join(args.bot, 'data'),
|
||||
logsDir: path.join(args.bot, 'logs')
|
||||
});
|
||||
assertPublicSafe(scenes);
|
||||
assertPublicSafe(trajectories);
|
||||
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
const sceneFile = path.join(out, 'scenes.json');
|
||||
const existing = fs.existsSync(sceneFile) ? JSON.parse(fs.readFileSync(sceneFile, 'utf8')) : {};
|
||||
for (const [id, s] of Object.entries(scenes)) {
|
||||
const old = existing[id];
|
||||
if (!old) { existing[id] = s; continue; }
|
||||
const ids = new Set(old.options.map((o) => o.id));
|
||||
for (const o of s.options) if (!ids.has(o.id)) old.options.push(o);
|
||||
old.tint = old.tint || s.tint;
|
||||
old.backdrop = old.backdrop || s.backdrop;
|
||||
if (old.label.includes('·') && !s.label.includes('·')) old.label = s.label;
|
||||
}
|
||||
const sorted = Object.fromEntries(Object.entries(existing).sort(([a], [b]) => a.localeCompare(b)));
|
||||
fs.writeFileSync(sceneFile, `${JSON.stringify(sorted, null, 1)}\n`);
|
||||
const trajFile = path.join(out, `trajectories-${tag}.jsonl`);
|
||||
fs.writeFileSync(trajFile, trajectories.map((t) => JSON.stringify(t)).join('\n') + '\n');
|
||||
|
||||
const outcomes = trajectories.reduce((m, t) => ((m[t.outcome.type] = (m[t.outcome.type] || 0) + 1), m), {});
|
||||
console.log(`${Object.keys(scenes).length} scenes, ${trajectories.length} attempts ${JSON.stringify(outcomes)}`);
|
||||
console.log(`wrote ${path.relative(process.cwd(), sceneFile)} and ${path.relative(process.cwd(), trajFile)}`);
|
||||
if (unlabeled.length) console.log(`scenes without a short label (add them to LABELS in collection/bot/bot.js):\n ${unlabeled.join('\n ')}`);
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Files written from the declarative specs, so each rule has one source of truth:
|
||||
.github/ISSUE_TEMPLATE/report-a-run.yml from collection/specs/report_form.yaml (report_form.render)
|
||||
the CROSSWALK block of core/graph.js from collection/specs/bot.yaml (Ken's bot uses it at run time)
|
||||
"""
|
||||
import json
|
||||
|
||||
from .common import path, read_yaml
|
||||
|
||||
GRAPH_JS = "core/graph.js"
|
||||
BEGIN = " // BEGIN CROSSWALK"
|
||||
END = " // END CROSSWALK\n"
|
||||
|
||||
|
||||
def graph_js():
|
||||
"""core/graph.js with its CROSSWALK block rewritten from the bot spec."""
|
||||
text = path(GRAPH_JS).read_text(encoding="utf-8")
|
||||
a = text.index(BEGIN)
|
||||
a = text.index("\n", a) + 1
|
||||
b = text.index(END, a)
|
||||
rules = read_yaml("collection/specs/bot.yaml")["crosswalk"]
|
||||
lines = [" const CROSSWALK = ["]
|
||||
for i, (rx, col, val) in enumerate(rules):
|
||||
sep = "," if i < len(rules) - 1 else ""
|
||||
lines.append(f" [/{rx.replace('/', chr(92) + '/')}/, {json.dumps(col)}, {json.dumps(val, ensure_ascii=False)}]{sep}")
|
||||
lines.append(" ];")
|
||||
return text[:a] + "\n".join(lines) + "\n" + text[b:]
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Paths, the pipeline spec, and byte-stable writers shared by every stage."""
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(ROOT / "core") not in sys.path:
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
|
||||
|
||||
def path(p):
|
||||
return ROOT / p
|
||||
|
||||
|
||||
def read_yaml(p):
|
||||
with open(path(p), encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def pipeline():
|
||||
return read_yaml("collection/pipeline.yaml")
|
||||
|
||||
|
||||
def sha256_bytes(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(p):
|
||||
h = hashlib.sha256()
|
||||
with open(p, "rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def json_bytes(obj, compact=True):
|
||||
if compact:
|
||||
return (json.dumps(obj, ensure_ascii=False, separators=(",", ":"))).encode("utf-8")
|
||||
return (json.dumps(obj, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def jsonl_bytes(rows):
|
||||
return "".join(json.dumps(r, ensure_ascii=False, separators=(",", ":")) + "\n" for r in rows).encode("utf-8")
|
||||
|
||||
|
||||
def read_jsonl(p):
|
||||
p = Path(p)
|
||||
if not p.exists():
|
||||
return []
|
||||
out = []
|
||||
for line in p.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip():
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
class Report:
|
||||
"""What a pipeline run did, as Markdown lines (the daily pull request's body)."""
|
||||
|
||||
def __init__(self):
|
||||
self.lines = []
|
||||
self.changed = False
|
||||
self.failed = []
|
||||
|
||||
def add(self, line):
|
||||
self.lines.append(line)
|
||||
print(line)
|
||||
|
||||
def text(self, title):
|
||||
return "\n".join([f"## {title}", ""] + self.lines) + "\n"
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Extract: fetch each source as it is now into the raw store. No transformation happens here.
|
||||
|
||||
The raw store is data/raw/<source>/ (git-ignored, never published):
|
||||
current.<ext> the latest version
|
||||
previous.<ext> the version before it (kept only in case it is needed; the daily workflow keeps it in an
|
||||
encrypted, private cache). Nothing older is kept and old versions are not hashed.
|
||||
data/manifest.json records, for each source, the sha256 of the current raw file and the date it last changed.
|
||||
|
||||
Each extractor returns the raw bytes, or raises Skip when the source cannot be reached from here (a missing
|
||||
secret, the bot's private repo not on this machine).
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .common import path, sha256_bytes
|
||||
|
||||
USER_AGENT = "threshing-atlas-collector/1.0 (+https://github.com/ekcSchiano/threshing-atlas)"
|
||||
RAW_DIR = path("data/raw")
|
||||
|
||||
|
||||
class Skip(Exception):
|
||||
"""The source cannot be extracted here; not an error."""
|
||||
|
||||
|
||||
def _get(url, headers=None, data=None, timeout=60):
|
||||
req = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT, **(headers or {})})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- extractors, by `kind`
|
||||
|
||||
def http(spec, env):
|
||||
data = _get(spec["url"])
|
||||
head = data[:15].lower()
|
||||
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
||||
raise RuntimeError(f"{spec['url']} returned a web page, not a file (is the link still shared publicly?)")
|
||||
return data
|
||||
|
||||
|
||||
def reddit(spec, env):
|
||||
"""The threads' comments through the official Reddit API (app-only OAuth: a "script" app's id and secret)."""
|
||||
cid, secret = (env.get(k) for k in spec["secrets"])
|
||||
if not (cid and secret):
|
||||
raise Skip(f"set the {' and '.join(spec['secrets'])} secrets to read the threads through the Reddit API")
|
||||
auth = base64.b64encode(f"{cid}:{secret}".encode()).decode()
|
||||
token = json.loads(_get("https://www.reddit.com/api/v1/access_token",
|
||||
headers={"Authorization": f"Basic {auth}"},
|
||||
data=b"grant_type=client_credentials"))["access_token"]
|
||||
threads = {}
|
||||
for tid in spec["threads"]:
|
||||
threads[tid] = json.loads(_get(f"https://oauth.reddit.com/comments/{tid}?limit=500&depth=10&raw_json=1",
|
||||
headers={"Authorization": f"Bearer {token}"}))
|
||||
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def github_issues(spec, env):
|
||||
"""Every issue with the label (open and closed), through the GitHub REST API."""
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
label = urllib.parse.quote(spec["label"])
|
||||
issues, page = [], 1
|
||||
while True:
|
||||
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
|
||||
batch = json.loads(_get(url, headers=headers))
|
||||
issues += [i for i in batch if "pull_request" not in i]
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
issues.sort(key=lambda i: i["number"])
|
||||
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def local_dir(spec, env):
|
||||
"""The bot's files, from a local copy of its private repo (DRAGONKIND_BOT_DIR), packed as one JSON."""
|
||||
root = env.get(spec["env"])
|
||||
if not root:
|
||||
raise Skip(f"the bot's repo is private: set {spec['env']} to a local copy to import it")
|
||||
files = {}
|
||||
for rel in spec["files"]:
|
||||
p = os.path.join(root, rel)
|
||||
files[rel] = Path(p).read_text(encoding="utf-8") if os.path.exists(p) else None
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the raw store
|
||||
|
||||
def raw_paths(name, ext):
|
||||
d = RAW_DIR / name
|
||||
return d / f"current.{ext}", d / f"previous.{ext}"
|
||||
|
||||
|
||||
def store(name, ext, data, manifest, today=None):
|
||||
"""Keep `data` as the current raw file. Returns True when it differs from the last one seen."""
|
||||
current, previous = raw_paths(name, ext)
|
||||
digest = sha256_bytes(data)
|
||||
entry = manifest.get(name) or {}
|
||||
changed = entry.get("sha256") != digest
|
||||
current.parent.mkdir(parents=True, exist_ok=True)
|
||||
on_disk = sha256_bytes(current.read_bytes()) if current.exists() else None
|
||||
if on_disk and on_disk != digest:
|
||||
shutil.copyfile(current, previous) # keep one previous version; anything older is gone
|
||||
if on_disk != digest:
|
||||
fd, tmp = tempfile.mkstemp(dir=current.parent)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(tmp, current)
|
||||
if changed:
|
||||
manifest[name] = {"sha256": digest, "bytes": len(data),
|
||||
"changed": today or datetime.now(timezone.utc).strftime("%Y-%m-%d")}
|
||||
return changed
|
||||
|
||||
|
||||
def extract(name, spec, manifest, env=None, local_file=None):
|
||||
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
|
||||
env = os.environ if env is None else env
|
||||
ext = spec["extract"].get("ext") or "json"
|
||||
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env)
|
||||
changed = store(name, ext, data, manifest)
|
||||
return raw_paths(name, ext)[0], changed
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Load: the only stage that writes public files. Writes are byte-stable (the same table gives the same bytes)
|
||||
and only happen when the bytes change, so git shows exactly what changed."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from .common import json_bytes, jsonl_bytes, path
|
||||
|
||||
|
||||
def public(obj):
|
||||
"""Drop transient keys (those starting with "_"), which are for the report only."""
|
||||
if isinstance(obj, dict):
|
||||
return {k: public(v) for k, v in obj.items() if not str(k).startswith("_")}
|
||||
if isinstance(obj, list):
|
||||
return [public(v) for v in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def read_json(rel):
|
||||
p = path(rel)
|
||||
return json.loads(p.read_text(encoding="utf-8")) if p.exists() else None
|
||||
|
||||
|
||||
def write_bytes(rel, data, check=False):
|
||||
"""Write when different. Returns True when the file changed (or, with check, would change)."""
|
||||
p = path(rel)
|
||||
if p.exists() and p.read_bytes() == data:
|
||||
return False
|
||||
if not check:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(data)
|
||||
return True
|
||||
|
||||
|
||||
def json_file(rel, obj, check=False, compact=True):
|
||||
return write_bytes(rel, json_bytes(public(obj), compact), check)
|
||||
|
||||
|
||||
def jsonl_file(rel, rows, check=False):
|
||||
return write_bytes(rel, jsonl_bytes(rows), check)
|
||||
|
||||
|
||||
def csv_bytes(rows, columns):
|
||||
"""True/False for booleans, empty for missing, everything else as text."""
|
||||
buf = io.StringIO()
|
||||
w = csv.writer(buf, lineterminator="\n")
|
||||
w.writerow(columns)
|
||||
for r in rows:
|
||||
w.writerow(["" if r.get(c) is None else r[c] for c in columns])
|
||||
return buf.getvalue().encode("utf-8")
|
||||
|
||||
|
||||
def csv_file(rel, rows, columns, check=False):
|
||||
return write_bytes(rel, csv_bytes(rows, columns), check)
|
||||
@@ -3,20 +3,13 @@
|
||||
Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each
|
||||
column means, so tools (and people) can load and cite the data without reading this repo's code.
|
||||
|
||||
python3 collection/publish_metadata.py # write data/croissant.json
|
||||
python3 collection/publish_metadata.py --check # exit 1 if data/croissant.json is out of date
|
||||
Written by `python -m collection build` (a test checks it is up to date).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from .common import ROOT, pipeline, read_yaml, sha256_file
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DICTIONARY = ROOT / "data" / "dictionary.yaml"
|
||||
SOURCES = ROOT / "data" / "sources.json"
|
||||
OUT = ROOT / "data" / "croissant.json"
|
||||
OUT = "data/croissant.json"
|
||||
|
||||
CONTEXT = {
|
||||
"@language": "en",
|
||||
@@ -57,17 +50,11 @@ CONTEXT = {
|
||||
DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"}
|
||||
|
||||
|
||||
def sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
sha256 = sha256_file
|
||||
|
||||
|
||||
def load_dictionary():
|
||||
with open(DICTIONARY, encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
return read_yaml("data/dictionary.yaml")
|
||||
|
||||
|
||||
def field_description(spec):
|
||||
@@ -80,8 +67,7 @@ def field_description(spec):
|
||||
def build(dictionary=None):
|
||||
d = dictionary or load_dictionary()
|
||||
ds = d["dataset"]
|
||||
with open(SOURCES, encoding="utf-8") as f:
|
||||
sources = json.load(f)
|
||||
sources = pipeline()["sources"]
|
||||
|
||||
distribution = []
|
||||
for key, spec in d["files"].items():
|
||||
@@ -124,9 +110,10 @@ def build(dictionary=None):
|
||||
"citeAs": ds["citation"],
|
||||
"isLiveDataset": True,
|
||||
"isBasedOn": [
|
||||
{"@type": "sc:CreativeWork", "name": s["name"], "url": s["view"]}
|
||||
for key, s in sources.items()
|
||||
if key != "pdf" # the route PDF is only fingerprinted, never redistributed
|
||||
{"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))}
|
||||
for s in sources.values()
|
||||
# the route PDF is only fingerprinted, never redistributed; the bot's repo is private
|
||||
if (s.get("privacy") or {}).get("redistribute", True) and s.get("links")
|
||||
],
|
||||
"distribution": distribution,
|
||||
"recordSet": [
|
||||
@@ -134,7 +121,7 @@ def build(dictionary=None):
|
||||
"@type": "cr:RecordSet",
|
||||
"@id": "runs",
|
||||
"name": "runs",
|
||||
"description": "One coded threshing run per row (data/derived/runs_clean.csv).",
|
||||
"description": "One coded threshing run per row, from every source (data/public/runs_clean.csv).",
|
||||
"key": {"@id": "runs/run_id"},
|
||||
"field": fields,
|
||||
}
|
||||
@@ -144,20 +131,3 @@ def build(dictionary=None):
|
||||
|
||||
def render(doc):
|
||||
return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def main(argv):
|
||||
text = render(build())
|
||||
if "--check" in argv:
|
||||
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
||||
print("data/croissant.json is out of date: run python3 collection/publish_metadata.py", file=sys.stderr)
|
||||
return 1
|
||||
print("data/croissant.json is up to date")
|
||||
return 0
|
||||
OUT.write_text(text, encoding="utf-8")
|
||||
print(f"wrote {OUT.relative_to(ROOT)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,60 @@
|
||||
"""The "Report a run" GitHub issue form, written from collection/specs/report_form.yaml, and the parser for the
|
||||
issues it creates (GitHub renders each answer as "### <question>" followed by the answer)."""
|
||||
import re
|
||||
|
||||
import yaml
|
||||
|
||||
from .common import read_yaml
|
||||
|
||||
|
||||
class _Dumper(yaml.SafeDumper):
|
||||
pass
|
||||
|
||||
|
||||
def _str(dumper, s):
|
||||
# multi-line text (the intro) as a readable block
|
||||
return dumper.represent_scalar("tag:yaml.org,2002:str", s, style="|" if "\n" in s else None)
|
||||
|
||||
|
||||
_Dumper.add_representer(str, _str)
|
||||
|
||||
SPEC_PATH = "collection/specs/report_form.yaml"
|
||||
TEMPLATE = ".github/ISSUE_TEMPLATE/report-a-run.yml"
|
||||
NOT_SURE = "Not sure"
|
||||
NO_RESPONSE = {"", "_No response_", "None", NOT_SURE}
|
||||
|
||||
|
||||
def spec():
|
||||
return read_yaml(SPEC_PATH)
|
||||
|
||||
|
||||
def render(s=None):
|
||||
s = s or spec()
|
||||
body = [{"type": "markdown", "attributes": {"value": s["intro"]}}]
|
||||
for f in s["fields"]:
|
||||
if f.get("type") == "integer":
|
||||
item = {"type": "input", "id": f["id"], "attributes": {"label": f["label"], "placeholder": "e.g. 3"}}
|
||||
else:
|
||||
opts = list(f["options"])
|
||||
if not f.get("required"):
|
||||
opts.append(NOT_SURE)
|
||||
item = {"type": "dropdown", "id": f["id"], "attributes": {"label": f["label"], "options": opts}}
|
||||
item["validations"] = {"required": bool(f.get("required"))}
|
||||
body.append(item)
|
||||
form = {"name": s["name"], "description": s["description"], "title": s["title"], "labels": [s["label"]], "body": body}
|
||||
head = f"# Written by collection/etl/report_form.py from {SPEC_PATH}: edit that file, then run `python -m collection build`.\n"
|
||||
return head + yaml.dump(form, Dumper=_Dumper, sort_keys=False, allow_unicode=True, width=1000)
|
||||
|
||||
|
||||
def parse_body(body, s=None):
|
||||
"""Issue body -> {field id: chosen option label or text}. Unanswered questions are left out."""
|
||||
s = s or spec()
|
||||
by_label = {f["label"].strip(): f["id"] for f in s["fields"]}
|
||||
out = {}
|
||||
for block in re.split(r"^### ", body or "", flags=re.M)[1:]:
|
||||
q, _, a = block.partition("\n")
|
||||
fid = by_label.get(q.strip())
|
||||
a = a.strip()
|
||||
if fid and a not in NO_RESPONSE:
|
||||
out[fid] = a
|
||||
return out
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Build: every public source file -> one coded run per row, for data/public/runs_clean.csv.
|
||||
|
||||
One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by the columns of data/dictionary.yaml;
|
||||
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
|
||||
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from .common import path, read_jsonl, read_yaml
|
||||
from . import report_form
|
||||
|
||||
DICT = read_yaml("data/dictionary.yaml")
|
||||
COLUMNS = list(DICT["columns"])
|
||||
COLOURS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
|
||||
|
||||
# what "not known" is, per column, for sources that did not ask (the community form's coding uses the same)
|
||||
NOT_KNOWN = {
|
||||
"wing": "Unknown", "squad": "Unknown", "attempt_group": "Unknown", "goal": "(not asked/blank)",
|
||||
"company": "Unknown", "direction": "Unknown", "first_sighting": "Unknown", "sighting_action": "Unknown",
|
||||
"scream": "Unknown", "resolution": "Unknown", "combat_first": "(not recorded)", "combat_second": "(not recorded)",
|
||||
"post_path": "Unknown", "hour": "Unknown", "rare_reported": "(not asked)", "malek_intent": "(not asked)",
|
||||
"outcome": "No outcome recorded", "data_flags": "",
|
||||
}
|
||||
|
||||
|
||||
def blank_row(**values):
|
||||
row = {c: None for c in COLUMNS}
|
||||
row.update(values)
|
||||
for c, v in NOT_KNOWN.items():
|
||||
if row[c] is None:
|
||||
row[c] = v
|
||||
return row
|
||||
|
||||
|
||||
def attempt_group(n):
|
||||
return "First attempt" if n == 1 else "Later attempt" if n and n > 1 else "Unknown"
|
||||
|
||||
|
||||
def encounter_columns(row, intentional=False):
|
||||
"""The bonding-grid column of the dragon that ended the run."""
|
||||
col = row["outcome_color"]
|
||||
if col and not intentional:
|
||||
if row["outcome"] == "Bonded":
|
||||
row["enc_" + col] = "Bonded!"
|
||||
elif row["outcome"] == "Incinerated":
|
||||
row["enc_" + col] = "Incinerated"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
|
||||
|
||||
def spreadsheet(name, spec):
|
||||
"""The community form: core/spec/form.yaml + core/atlas/coding.py, unchanged."""
|
||||
from atlas import coding
|
||||
df, missing = coding.code_frame(coding.read_export(path(spec["load"])))
|
||||
if missing:
|
||||
print(f"WARNING: {name}: questions not found in the sheet: {missing}")
|
||||
df["attempt"] = df["attempt"].astype("Int64")
|
||||
rows = []
|
||||
for rec in df.astype(object).where(df.notna(), None).to_dict("records"):
|
||||
n = rec.pop("run_id")
|
||||
entry = rec.pop("source")
|
||||
rows.append({**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry})
|
||||
return rows
|
||||
|
||||
|
||||
def reddit_results(name, spec):
|
||||
"""Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown."""
|
||||
rows = []
|
||||
for r in read_yaml(spec["transform"]["manual"])["results"]:
|
||||
row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded",
|
||||
outcome_color=r["colour"], tail=r["tail"])
|
||||
encounter_columns(row)
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def reports(name, spec):
|
||||
"""Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml)."""
|
||||
form = report_form.spec()
|
||||
fields = {f["id"]: f for f in form["fields"]}
|
||||
p = path(spec["load"])
|
||||
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []}
|
||||
rows = []
|
||||
for rep in data["reports"]:
|
||||
vals, flags = {}, []
|
||||
for fid, answer in rep["answers"].items():
|
||||
f = fields.get(fid)
|
||||
if not f:
|
||||
continue
|
||||
if f.get("type") == "integer":
|
||||
m = re.search(r"\d+", answer)
|
||||
vals[f["column"]] = int(m.group()) if m else None
|
||||
elif answer in f["options"]:
|
||||
vals[f["column"]] = f["options"][answer]
|
||||
else:
|
||||
flags.append(f"unrecognised answer to {fid}")
|
||||
row = blank_row(run_id=f"reported-{rep['number']}", source=spec["run_source"], date=rep.get("opened"), **vals)
|
||||
row["attempt_group"] = attempt_group(row["attempt"])
|
||||
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
|
||||
row["hour"] = "DIED before the hour"
|
||||
branch = row["branch"]
|
||||
if branch:
|
||||
tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens")
|
||||
if row["hour"] == "Unknown":
|
||||
row["hour"] = "Tree" if tree else "Valley"
|
||||
elif row["hour"] != ("Tree" if tree else "Valley"):
|
||||
flags.append("encounter does not match the tree/valley answer")
|
||||
row["saw_shadow"] = branch == "Shadow (rare)" if tree else None
|
||||
if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue":
|
||||
row["saw_blue_meadow"] = True
|
||||
if row["outcome"] == "Bonded" and not row["outcome_color"]:
|
||||
flags.append("bonded, colour not given")
|
||||
encounter_columns(row)
|
||||
row["data_flags"] = "; ".join(flags)
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _norm(text):
|
||||
from .transform.bot import normalize
|
||||
return normalize(text)
|
||||
|
||||
|
||||
def bot_trajectories(spec):
|
||||
"""Every published trajectory (data/bot/trajectories-*.jsonl), each attempt once."""
|
||||
seen, out = set(), []
|
||||
for f in sorted(Path(path(spec["load"])).glob("trajectories-*.jsonl")):
|
||||
for t in read_jsonl(f):
|
||||
if t["id"] not in seen:
|
||||
seen.add(t["id"])
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def bot_variables(traj, crosswalk):
|
||||
"""A bot attempt -> columns, through the crosswalk in collection/specs/bot.yaml. First match wins."""
|
||||
v, unmapped = {}, []
|
||||
for s in traj["steps"]:
|
||||
t = _norm(s["optionText"])
|
||||
hit = next((c for c in crosswalk if c[0].search(t)), None)
|
||||
if hit:
|
||||
v.setdefault(hit[1], hit[2])
|
||||
else:
|
||||
unmapped.append(s["optionText"])
|
||||
# the first dragon is the first tinted scene once the direction is chosen
|
||||
if v.get("direction") and not v.get("first_sighting") and s.get("tint"):
|
||||
v["first_sighting"] = s["tint"].capitalize()
|
||||
return v, unmapped
|
||||
|
||||
|
||||
def bot(name, spec):
|
||||
cw = [(re.compile(rx), col, val) for rx, col, val in read_yaml(spec["transform"]["spec"])["crosswalk"]]
|
||||
rows = []
|
||||
for traj in bot_trajectories(spec):
|
||||
v, _ = bot_variables(traj, cw)
|
||||
red = v.pop("red", None)
|
||||
v.pop("scream_investigate", None)
|
||||
if red == "Let it pass (two Greens)":
|
||||
v["branch"] = "Descend quickly: let the Red pass"
|
||||
o = traj["outcome"]
|
||||
meta = traj.get("meta") or {}
|
||||
flags = []
|
||||
if o["type"] == "BONDED":
|
||||
outcome, colour = "Bonded", (o.get("colour") or "").capitalize() or None
|
||||
elif o["type"] == "BURNED":
|
||||
last_tint = traj["steps"][-1].get("tint")
|
||||
outcome = "Incinerated" if v.get("hour") else "Died before the hour"
|
||||
colour = last_tint.capitalize() if last_tint and v.get("hour") else None
|
||||
if o.get("intentional"):
|
||||
flags.append("deliberate burn (the bot chose to die)")
|
||||
else:
|
||||
outcome, colour = "No outcome recorded", None
|
||||
target = (meta.get("target") or "").capitalize() or None
|
||||
row = blank_row(run_id=traj["id"], source=spec["run_source"], date=(meta.get("startedAt") or "")[:10] or None,
|
||||
attempt=meta.get("attempt"), goal=target if target in COLOURS else None,
|
||||
outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v)
|
||||
row["attempt_group"] = attempt_group(row["attempt"])
|
||||
encounter_columns(row, intentional=bool(o.get("intentional")))
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reports": reports, "bot": bot}
|
||||
|
||||
|
||||
def build_runs(pipe):
|
||||
rows = []
|
||||
for name in pipe["runs"]["order"]:
|
||||
spec = pipe["sources"][name]
|
||||
if spec.get("runs"):
|
||||
rows += CODERS[spec["runs"]](name, spec)
|
||||
return rows
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Named transforms (pipeline.yaml `transform: {name: ...}`). Each module has
|
||||
transform(raw_path, spec, ctx) -> the public source file's content (pure: no network, no clock)
|
||||
describe(old, new) -> report lines comparing the previous public file with the new one
|
||||
"""
|
||||
from . import bot, pdf, reddit, reports, sheet
|
||||
|
||||
TRANSFORMS = {"sheet": sheet, "pdf_fingerprint": pdf, "bot": bot, "reddit_results": reddit, "reports": reports}
|
||||
@@ -0,0 +1,239 @@
|
||||
"""The DragonKind bot's private data -> anonymised trajectories and a scene catalogue (Python port of bot.js).
|
||||
|
||||
Raw input (from the private bot repo, packed by extract.local_dir): data/knowledge.json (scene atlas: prose,
|
||||
choices, tint, backdrop), data/runs.jsonl (one record per attempt) and logs/encounters.jsonl (every decision the
|
||||
bot made, with the options on screen). Nothing of the prose or the account ids leaves this module: the output
|
||||
holds scene fingerprints, short labels (collection/specs/bot.yaml), choice text and outcomes, and check_public()
|
||||
refuses anything else.
|
||||
|
||||
Output: {"scenes": {sceneId: {stage, tint, backdrop, label, options: [{id, text}]}},
|
||||
"trajectories": [{id, source, meta, steps, outcome}], "unlabeled": [...]}
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from ..common import read_yaml
|
||||
|
||||
SPEC = read_yaml("collection/specs/bot.yaml")
|
||||
_NORMALIZE = [(re.compile(a), b) for a, b in SPEC["normalize"]]
|
||||
CHOICE_EVENTS = set(SPEC["choice_events"])
|
||||
|
||||
|
||||
def sha1(s, n=12):
|
||||
return hashlib.sha1(str(s).encode("utf-8")).hexdigest()[:n]
|
||||
|
||||
|
||||
def normalize(text):
|
||||
"""Must match normalize() in the bot's src/strategy.js, so fingerprints line up."""
|
||||
t = str(text or "").lower()
|
||||
for rx, rep in _NORMALIZE:
|
||||
t = rx.sub(rep, t)
|
||||
return t.strip()
|
||||
|
||||
|
||||
def scene_id(prose, choices):
|
||||
"""Must match computeSceneFingerprint() in the bot's src/learner.js."""
|
||||
prose_norm = normalize(prose)[:500]
|
||||
opts = sorted(c for c in (normalize(c) for c in choices) if c)
|
||||
return sha1(f"{prose_norm} ### {' || '.join(opts)}")
|
||||
|
||||
|
||||
def option_id(text):
|
||||
return sha1(normalize(text), 10)
|
||||
|
||||
|
||||
def label_for(sid, node):
|
||||
if sid in SPEC["labels"]:
|
||||
return SPEC["labels"][sid]
|
||||
bits = [node.get("stage") or "Scene"]
|
||||
if node.get("tint"):
|
||||
bits.append(node["tint"])
|
||||
return " · ".join(bits)
|
||||
|
||||
|
||||
def _jsonl(text):
|
||||
out = []
|
||||
for line in (text or "").splitlines():
|
||||
if line.strip():
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def build_scenes(knowledge, encounters):
|
||||
scenes = {}
|
||||
for sid, n in ((knowledge or {}).get("nodes") or {}).items():
|
||||
scenes[sid] = {
|
||||
"stage": n.get("stage") or "UNKNOWN", "tint": n.get("tint"), "backdrop": n.get("backdrop"),
|
||||
"label": label_for(sid, n),
|
||||
"options": [{"id": option_id(c.get("label") or c.get("choiceNorm")), "text": c.get("label") or c.get("choiceNorm")}
|
||||
for c in (n.get("choices") or {}).values()],
|
||||
}
|
||||
for e in encounters:
|
||||
if not isinstance(e.get("choices"), list) or not e.get("prose"):
|
||||
continue
|
||||
sid = e.get("nodeId") or scene_id(e["prose"], e["choices"])
|
||||
if sid in scenes:
|
||||
continue
|
||||
stage = (e.get("decision") or {}).get("stage") or e.get("stage") or "UNKNOWN"
|
||||
scenes[sid] = {"stage": stage, "tint": None, "backdrop": None, "label": label_for(sid, {"stage": stage}),
|
||||
"options": [{"id": option_id(t), "text": t} for t in e["choices"]]}
|
||||
return scenes
|
||||
|
||||
|
||||
def trajectories_from_encounters(encounters, scenes):
|
||||
"""Group the log by attempt, keep the decisions acted on, and close each attempt at its first BURNED or
|
||||
DRAGON_BONDED event. Repeated BURNED events with no decision between them are ignored."""
|
||||
by_attempt = {}
|
||||
for e in encounters:
|
||||
key = f"{e['account']}:{e.get('attempt')}" if e.get("account") else str(e.get("attempt"))
|
||||
by_attempt.setdefault(key, []).append(e)
|
||||
out = []
|
||||
for key, events in by_attempt.items():
|
||||
events = sorted(events, key=lambda e: str(e.get("ts")))
|
||||
steps, outcome = [], None
|
||||
for e in events:
|
||||
if outcome:
|
||||
break
|
||||
if e.get("event") in CHOICE_EVENTS:
|
||||
d = e.get("decision") or {}
|
||||
text = d.get("choiceText") or e.get("choiceText")
|
||||
if d.get("action") and d["action"] != "choose":
|
||||
continue # paused, nothing happened
|
||||
if not text or not isinstance(e.get("choices"), list):
|
||||
continue
|
||||
scene = e.get("nodeId") or scene_id(e.get("prose") or "", e["choices"])
|
||||
manual = e["event"] != "DECISION"
|
||||
sc = scenes.get(scene) or {}
|
||||
steps.append({
|
||||
"scene": scene, "stage": d.get("stage") or sc.get("stage") or "UNKNOWN", "tint": sc.get("tint"),
|
||||
"option": option_id(text), "optionText": text, "offered": [option_id(c) for c in e["choices"]],
|
||||
"rule": d.get("rule"), "decisionMode": "manual" if manual else d.get("source"),
|
||||
"manual": manual, "at": e.get("ts"),
|
||||
})
|
||||
elif e.get("event") == "BURNED" and steps:
|
||||
outcome = {"type": "BURNED", "colour": None, "intentional": bool(e.get("intentional"))}
|
||||
elif e.get("event") == "DRAGON_BONDED" and steps:
|
||||
outcome = {"type": "BONDED", "colour": str(e["dragon"]).lower() if e.get("dragon") else None, "intentional": False}
|
||||
# a decision repeated on the same scene (the bot re-detecting the screen) counts once
|
||||
dedup = [s for i, s in enumerate(steps)
|
||||
if not (i > 0 and steps[i - 1]["scene"] == s["scene"] and steps[i - 1]["option"] == s["option"])]
|
||||
for i, s in enumerate(dedup):
|
||||
s["to"] = dedup[i + 1]["scene"] if i + 1 < len(dedup) else None
|
||||
if not dedup:
|
||||
continue
|
||||
if outcome:
|
||||
dedup[-1]["to"] = "BURNED" if outcome["type"] == "BURNED" else f"BONDED:{outcome['colour'] or 'unknown'}"
|
||||
try:
|
||||
attempt = int(float(key.split(":")[-1])) or None
|
||||
except ValueError:
|
||||
attempt = None
|
||||
started = dedup[0]["at"]
|
||||
out.append({
|
||||
"id": "bot-" + sha1(f"{key}:{started}", 10),
|
||||
"source": "bot",
|
||||
"meta": {"attempt": attempt, "startedAt": dedup[0]["at"], "accountKey": None, "target": None, "decisionMode": None},
|
||||
"steps": dedup,
|
||||
"outcome": outcome or {"type": "ABORTED", "colour": None, "intentional": False},
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _ms(ts):
|
||||
try:
|
||||
return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() * 1000
|
||||
except ValueError:
|
||||
return float("nan")
|
||||
|
||||
|
||||
def attach_run_meta(trajs, runs, salt):
|
||||
"""Run-level metadata (target colour, mode) from runs.jsonl, where an attempt is in both."""
|
||||
window = SPEC["match_hours"] * 3600e3
|
||||
for r in runs:
|
||||
t = next((x for x in trajs if x["meta"]["attempt"] == r.get("attempt")
|
||||
and abs(_ms(x["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window), None)
|
||||
if not t:
|
||||
continue
|
||||
m = t["meta"]
|
||||
m["target"] = r.get("targetDragon")
|
||||
m["decisionMode"] = r.get("decisionMode")
|
||||
m["policyId"] = r.get("policyId")
|
||||
m["experimentId"] = r.get("experimentId")
|
||||
m["arm"] = r.get("arm")
|
||||
if r.get("accountKey"):
|
||||
m["accountKey"] = r["accountKey"]
|
||||
elif r.get("accountId") and salt:
|
||||
m["accountKey"] = sha1(f"{salt}:{r['accountId']}", 10)
|
||||
if r.get("outcome") == "BONDED" and r.get("dragonColor"):
|
||||
t["outcome"] = {"type": "BONDED", "colour": str(r["dragonColor"]).lower(), "intentional": False}
|
||||
if r.get("intentionalBurn"):
|
||||
t["outcome"]["intentional"] = True
|
||||
return trajs
|
||||
|
||||
|
||||
def import_bot(knowledge, encounters, runs, salt=None):
|
||||
scenes = build_scenes(knowledge, encounters)
|
||||
trajectories = attach_run_meta(trajectories_from_encounters(encounters, scenes), runs, salt)
|
||||
unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]]
|
||||
return scenes, trajectories, unlabeled
|
||||
|
||||
|
||||
def check_public(obj, refuse, max_string):
|
||||
"""Nothing that looks like game prose or an account identifier may leave this module."""
|
||||
def walk(v, p):
|
||||
if isinstance(v, list):
|
||||
for i, x in enumerate(v):
|
||||
walk(x, f"{p}[{i}]")
|
||||
elif isinstance(v, dict):
|
||||
for k, x in v.items():
|
||||
if k in refuse:
|
||||
raise ValueError(f'refusing to export field "{k}" at {p}')
|
||||
walk(x, f"{p}.{k}")
|
||||
elif isinstance(v, str) and len(v) > max_string:
|
||||
raise ValueError(f"refusing to export a {len(v)}-character string at {p} (looks like prose)")
|
||||
walk(obj, "$")
|
||||
|
||||
|
||||
def merge_scenes(existing, scenes):
|
||||
"""Add new scenes and new options to the published catalogue; never drop what is already there."""
|
||||
out = json.loads(json.dumps(existing or {}))
|
||||
for sid, s in scenes.items():
|
||||
old = out.get(sid)
|
||||
if not old:
|
||||
out[sid] = s
|
||||
continue
|
||||
ids = {o["id"] for o in old["options"]}
|
||||
old["options"] += [o for o in s["options"] if o["id"] not in ids]
|
||||
old["tint"] = old.get("tint") or s["tint"]
|
||||
old["backdrop"] = old.get("backdrop") or s["backdrop"]
|
||||
if "·" in old["label"] and "·" not in s["label"]:
|
||||
old["label"] = s["label"]
|
||||
return dict(sorted(out.items()))
|
||||
|
||||
|
||||
def transform(raw_path, spec, ctx):
|
||||
with open(raw_path, encoding="utf-8") as f:
|
||||
files = json.load(f)
|
||||
k = files.get("data/knowledge.json")
|
||||
knowledge = json.loads(k) if k else {"nodes": {}}
|
||||
encounters = _jsonl(files.get("logs/encounters.jsonl"))
|
||||
runs = _jsonl(files.get("data/runs.jsonl"))
|
||||
privacy = spec.get("privacy") or {}
|
||||
scenes, trajectories, unlabeled = import_bot(knowledge, encounters, runs, ctx.get("env", {}).get(privacy.get("account_salt_env", "")))
|
||||
for obj in (scenes, trajectories):
|
||||
check_public(obj, set(privacy.get("refuse_fields", [])), privacy.get("max_string", 200))
|
||||
return {"scenes": scenes, "trajectories": trajectories, "_unlabeled": unlabeled}
|
||||
|
||||
|
||||
def describe(old, new):
|
||||
outcomes = {}
|
||||
for t in new["trajectories"]:
|
||||
outcomes[t["outcome"]["type"]] = outcomes.get(t["outcome"]["type"], 0) + 1
|
||||
lines = [f"{len(new['scenes'])} scenes, {len(new['trajectories'])} attempts {json.dumps(outcomes)}"]
|
||||
if new.get("_unlabeled"):
|
||||
lines.append(" - Scenes without a short label (add them to collection/specs/bot.yaml): " + ", ".join(new["_unlabeled"]))
|
||||
return lines
|
||||
@@ -0,0 +1,51 @@
|
||||
"""The route PDF -> its fingerprint only. The PDF is never redistributed: the public file holds hashes of the file,
|
||||
of each page's text and of each line, so a change can be detected and located without keeping the text."""
|
||||
import hashlib
|
||||
|
||||
|
||||
def _sha(text, n):
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:n]
|
||||
|
||||
|
||||
def pdf_lines(path):
|
||||
from pypdf import PdfReader
|
||||
pages = []
|
||||
for page in PdfReader(str(path)).pages:
|
||||
text = page.extract_text() or ""
|
||||
pages.append([" ".join(line.split()) for line in text.splitlines() if line.strip()])
|
||||
return pages
|
||||
|
||||
|
||||
def transform(raw_path, spec, ctx):
|
||||
with open(raw_path, "rb") as f:
|
||||
data = f.read()
|
||||
pages = pdf_lines(raw_path)
|
||||
return {
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"bytes": len(data),
|
||||
"pages": len(pages),
|
||||
"page_hashes": [_sha("\n".join(p), 16) for p in pages],
|
||||
"line_hashes": sorted({_sha(line.lower(), 12) for p in pages for line in p}),
|
||||
"checked": ctx.get("changed"),
|
||||
# transient: the new lines' text, for the report only (load drops every key starting with "_")
|
||||
"_lines": [line for p in pages for line in p],
|
||||
}
|
||||
|
||||
|
||||
def describe(old, new):
|
||||
if old is None:
|
||||
return [f"fingerprint recorded for the first time ({new['pages']} pages)"]
|
||||
if old.get("page_hashes") == new["page_hashes"]:
|
||||
return ["the file changed but its text did not (layout or images only). Worth a look."]
|
||||
changed = [i + 1 for i, h in enumerate(new["page_hashes"])
|
||||
if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
|
||||
old_lines = set(old["line_hashes"])
|
||||
new_lines = [line for line in new.get("_lines", []) if _sha(line.lower(), 12) not in old_lines]
|
||||
lines = [f"changed. {old['pages']} → {new['pages']} pages; pages with new text: {', '.join(map(str, changed)) or 'none'}; "
|
||||
f"{len(new_lines)} new lines, {len(old_lines - set(new['line_hashes']))} lines gone."]
|
||||
if new_lines:
|
||||
lines.append(" - New lines (update core/spec/game.yaml or the crosswalks if they add options):")
|
||||
lines += [f" - {line}" for line in new_lines[:60]]
|
||||
if len(new_lines) > 60:
|
||||
lines.append(f" - … and {len(new_lines) - 60} more")
|
||||
return lines
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Reddit megathreads (raw: the threads' comment trees from the official API) -> dragonkind.com result links that
|
||||
are not yet in collection/manual/reddit_results.yaml. Usernames and comment text are never kept: only the result
|
||||
ids and the comment permalinks, for the report, so a person can read the result page and add it by hand."""
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from ..common import read_yaml
|
||||
|
||||
RESULT_RE = re.compile(r"dragonkind\.com/results/(THRESH-[A-Z0-9]+)", re.I)
|
||||
|
||||
|
||||
def _comments(node):
|
||||
"""Every comment in a listing tree."""
|
||||
if isinstance(node, list):
|
||||
for x in node:
|
||||
yield from _comments(x)
|
||||
elif isinstance(node, dict):
|
||||
data = node.get("data") or {}
|
||||
if node.get("kind") in ("t1", "t3"):
|
||||
yield data
|
||||
for key in ("children", "replies"):
|
||||
if isinstance(data.get(key), (dict, list)):
|
||||
yield from _comments(data[key])
|
||||
|
||||
|
||||
def transform(raw_path, spec, ctx):
|
||||
threads = json.loads(Path(raw_path).read_text(encoding="utf-8"))
|
||||
known = {r["id"].upper() for r in read_yaml(spec["transform"]["manual"])["results"]}
|
||||
found = {}
|
||||
for tid, tree in sorted(threads.items()):
|
||||
for c in _comments(tree):
|
||||
for m in RESULT_RE.finditer((c.get("body") or "") + " " + (c.get("selftext") or "") + " " + (c.get("url") or "")):
|
||||
rid = m.group(1).upper()
|
||||
if rid not in known and rid not in found:
|
||||
found[rid] = "https://www.reddit.com" + (c.get("permalink") or f"/comments/{tid}/")
|
||||
return {"new_results": [{"id": k, "comment": v} for k, v in sorted(found.items())]}
|
||||
|
||||
|
||||
def describe(old, new):
|
||||
rs = new["new_results"]
|
||||
if not rs:
|
||||
return ["no new result links in the threads"]
|
||||
return [f"{len(rs)} result link(s) not yet in collection/manual/reddit_results.yaml (read each page, then add it):"] + \
|
||||
[f" - https://dragonkind.com/results/{r['id']} (posted at {r['comment']})" for r in rs[:40]]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Run reports (raw: the "run report" issues from the GitHub API) -> data/public/reports.json.
|
||||
|
||||
Kept per report: the issue number, the date it was opened, and the answers (the option labels chosen). GitHub
|
||||
usernames and any other text are dropped. Reports closed as "not planned" or carrying a reject label (e.g.
|
||||
invalid, duplicate) are left out, so a maintainer removes a bad report by closing it that way."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .. import report_form
|
||||
|
||||
|
||||
def transform(raw_path, spec, ctx):
|
||||
issues = json.loads(Path(raw_path).read_text(encoding="utf-8"))
|
||||
reject = set(spec["transform"].get("reject_labels", []))
|
||||
form = report_form.spec()
|
||||
reports = []
|
||||
for i in sorted(issues, key=lambda i: i["number"]):
|
||||
labels = {lb["name"] if isinstance(lb, dict) else lb for lb in i.get("labels") or []}
|
||||
if labels & reject or i.get("state_reason") == "not_planned":
|
||||
continue
|
||||
answers = report_form.parse_body(i.get("body"), form)
|
||||
if not answers:
|
||||
continue
|
||||
reports.append({"number": i["number"], "opened": (i.get("created_at") or "")[:10] or None, "answers": answers})
|
||||
return {"source": spec["label"], "reports": reports}
|
||||
|
||||
|
||||
def describe(old, new):
|
||||
o = {r["number"] for r in (old or {}).get("reports", [])}
|
||||
n = {r["number"] for r in new["reports"]}
|
||||
return [f"{len(n)} report(s) ({len(n - o)} new, {len(o - n)} removed)"]
|
||||
@@ -0,0 +1,75 @@
|
||||
"""A Google Sheets export (.xlsx or .csv) -> the public source file: the sheet's own questions and answers, with the
|
||||
source's privacy rules applied. No recoding happens here (that is core/atlas/coding.py, at build time)."""
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from ..common import read_yaml
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
|
||||
def load_sheet(path, sheet_contains="form responses"):
|
||||
if str(path).lower().endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if sheet_contains in n.lower()), xl.sheet_names[0])
|
||||
return pd.read_excel(xl, sheet)
|
||||
|
||||
|
||||
def cell(v, redact=True):
|
||||
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
|
||||
return None
|
||||
if isinstance(v, (datetime, pd.Timestamp)):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
if isinstance(v, str) and redact:
|
||||
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
|
||||
return v
|
||||
|
||||
|
||||
def extra_results(manual):
|
||||
"""Dragons read from result pages linked on Reddit (collection/manual/reddit_results.yaml)."""
|
||||
return [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{r['id']}",
|
||||
"name": r["name"], "colour": r["colour"], "tail": r["tail"], "sex": r["sex"], "age": r["age"],
|
||||
"lineage": r["lineage"]} for r in read_yaml(manual)["results"]]
|
||||
|
||||
|
||||
def transform(raw_path, spec, ctx):
|
||||
t, privacy = spec["transform"], spec.get("privacy") or {}
|
||||
raw = load_sheet(raw_path, t.get("sheet_contains", "form responses"))
|
||||
drop = [d.lower() for d in privacy.get("drop_columns_matching", [])]
|
||||
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
|
||||
redact = privacy.get("redact_emails", True)
|
||||
rows = []
|
||||
for _, r in raw.iterrows():
|
||||
row = {}
|
||||
for k, v in r.items():
|
||||
c = cell(v, redact)
|
||||
if c is not None:
|
||||
row[str(k)] = c
|
||||
rows.append(row)
|
||||
out = {"source": spec["label"], "exported": ctx.get("changed"),
|
||||
# every header, even one blank in every row, so the coding rules see every question
|
||||
"headers": [str(c) for c in raw.columns], "rows": rows}
|
||||
if t.get("extra"):
|
||||
out["extra"] = extra_results(t["extra"])
|
||||
return out
|
||||
|
||||
|
||||
def describe(old, new):
|
||||
"""One report line (plus details) comparing the previous public file with the new one."""
|
||||
if old is None:
|
||||
return [f"first export: {len(new['rows'])} rows"]
|
||||
key = lambda r: repr(sorted(r.items())) # noqa: E731
|
||||
o, n = {key(r) for r in old["rows"]}, {key(r) for r in new["rows"]}
|
||||
lines = [f"{len(old['rows'])} → {len(new['rows'])} rows ({len(n - o)} new or edited, {len(o - n)} removed or edited)"]
|
||||
new_q = [h for h in new["headers"] if h not in old["headers"]]
|
||||
gone_q = [h for h in old["headers"] if h not in new["headers"]]
|
||||
if new_q:
|
||||
lines.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}")
|
||||
if gone_q:
|
||||
lines.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
|
||||
return lines
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Validate: a table is checked against data/dictionary.yaml before it is loaded. A failure stops the load."""
|
||||
import re
|
||||
|
||||
CHECKS = {
|
||||
"integer": lambda v: isinstance(v, int) and not isinstance(v, bool) or bool(re.fullmatch(r"-?\d+", str(v))),
|
||||
"boolean": lambda v: isinstance(v, bool) or v in ("True", "False"),
|
||||
"date": lambda v: bool(re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(v))),
|
||||
"text": lambda v: isinstance(v, str),
|
||||
}
|
||||
|
||||
|
||||
class Invalid(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def problems(rows, dictionary, header=None):
|
||||
"""Every way the rows break the dictionary, as readable lines (empty when the table is valid)."""
|
||||
cols = dictionary["columns"]
|
||||
out = []
|
||||
if header is not None and list(header) != list(cols):
|
||||
out.append(f"columns differ from the dictionary: {list(header)} vs {list(cols)}")
|
||||
ids = [r.get("run_id") for r in rows]
|
||||
if len(set(ids)) != len(ids):
|
||||
out.append("run_id is not unique")
|
||||
for name, col in cols.items():
|
||||
vals = [r.get(name) for r in rows]
|
||||
present = [v for v in vals if v is not None and v != ""]
|
||||
bad_type = sorted({str(v) for v in present if not CHECKS[col["type"]](v)})
|
||||
if bad_type:
|
||||
out.append(f"{name}: not {col['type']}: {bad_type[:5]}")
|
||||
if "values" in col:
|
||||
allowed = {str(v) for v in col["values"]}
|
||||
bad = sorted({str(v) for v in present if str(v) not in allowed})
|
||||
if bad:
|
||||
out.append(f"{name}: values not in the dictionary: {bad[:10]}")
|
||||
if col.get("required") and len(present) != len(vals):
|
||||
out.append(f"{name}: required, but {len(vals) - len(present)} rows are empty")
|
||||
return out
|
||||
|
||||
|
||||
def check(rows, dictionary, what="table"):
|
||||
errs = problems(rows, dictionary)
|
||||
if errs:
|
||||
raise Invalid(f"{what} does not match data/dictionary.yaml:\n " + "\n ".join(errs))
|
||||
@@ -0,0 +1,19 @@
|
||||
# Dragons from dragonkind.com result pages that riders linked in the r/fourthwing megathreads, read by hand.
|
||||
# Each is one run in runs_clean.csv (source "reddit thread": bonded, colour and tail known, route not recorded) and
|
||||
# appears under "extra" in data/public/register.json. The reddit_thread source lists result links found in the
|
||||
# threads that are not here yet; add them after reading the result page. No usernames are kept.
|
||||
version: 1
|
||||
read: "2026-10-19"
|
||||
results:
|
||||
- { id: THRESH-VFRQVM5K3B0CP, name: Thiduach, colour: Blue, tail: Swordtail, sex: Female, age: 51, lineage: Gormfaileas }
|
||||
- { id: THRESH-3M4P9MNFQGBS8, name: Caenuis, colour: Blue, tail: Swordtail, sex: Female, age: 38, lineage: Gormfaileas }
|
||||
- { id: THRESH-5JQDJ21G1BYKR, name: Oavuagh, colour: Green, tail: Morningstartail, sex: Female, age: 43, lineage: Uaineloidsig }
|
||||
- { id: THRESH-5KRAAVFB3D1MG, name: Geagios, colour: Blue, tail: Clubtail, sex: Male, age: 99, lineage: Gormfaileas }
|
||||
- { id: THRESH-B5HSM00J4A2Z4, name: Fiedheil, colour: Black, tail: Clubtail, sex: Male, age: 32, lineage: Moerlaighen }
|
||||
- { id: THRESH-BMPM2GAPK58JT, name: Teirfua, colour: Green, tail: Morningstartail, sex: Female, age: 68, lineage: Eifyrielle }
|
||||
- { id: THRESH-MCQ8D6GAGF4K8, name: Caollo, colour: Blue, tail: Morningstartail, sex: Female, age: 93, lineage: Nyghmuiron }
|
||||
- { id: THRESH-Q0XDBFMNBEB86, name: Aenminn, colour: Red, tail: Swordtail, sex: Male, age: 59, lineage: Berradhyr }
|
||||
- { id: THRESH-W99DB29MG16QA, name: Meinso, colour: Green, tail: Clubtail, sex: Female, age: 63, lineage: Eifyrielle }
|
||||
- { id: THRESH-XCJCY7HW7QSSP, name: Einmial, colour: Blue, tail: Clubtail, sex: Male, age: 58, lineage: Gormfaileas }
|
||||
- { id: THRESH-XR6G68VPGT48M, name: Recuagh, colour: Brown, tail: Morningstartail, sex: Female, age: 55, lineage: Vormalgh }
|
||||
- { id: THRESH-YQC97TK34214E, name: Dannidh, colour: Orange, tail: Clubtail, sex: Male, age: 61, lineage: Cariaslog }
|
||||
@@ -0,0 +1,89 @@
|
||||
# Data collection as extract -> transform -> load, one block per source. collection/run.py reads this file.
|
||||
#
|
||||
# extract fetches the source as it is now. The raw file goes to data/raw/<source>/current.<ext> (git-ignored,
|
||||
# never published); the version it replaces is kept as previous.<ext>, and the daily workflow keeps
|
||||
# that previous copy in an encrypted, private cache. Nothing older is kept.
|
||||
# transform turns the raw file into the public source file: a pure function of the raw file (no network, no
|
||||
# clock), with the source's privacy rules applied. Named transforms live in collection/etl/transform/.
|
||||
# load writes the public source file (byte-stable: same input, same bytes) and reports what changed.
|
||||
#
|
||||
# Then `build` codes every public source file into one run per row (runs: the coder) and writes
|
||||
# data/public/runs_clean.csv: every run from every source, with a `source` column. Each table is checked against
|
||||
# data/dictionary.yaml before it is written. Every input to `build` is committed, so CI can rebuild it exactly.
|
||||
version: 1
|
||||
|
||||
raw_dir: data/raw
|
||||
manifest: data/manifest.json # for each source: the sha256 of the current raw file and when it last changed
|
||||
|
||||
runs:
|
||||
path: data/public/runs_clean.csv
|
||||
# sources in this order; within a source, the order the coder returns
|
||||
order: [reddit_spreadsheet, reddit_thread, reported, bot, reddit_pdf]
|
||||
|
||||
sources:
|
||||
reddit_spreadsheet:
|
||||
label: Threshing Routes form and spreadsheet (u/ClumsyBadger, linked from the r/fourthwing megathreads)
|
||||
run_source: reddit spreadsheet
|
||||
links:
|
||||
sheet: https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0
|
||||
form: null # the megathread's Google Form (responder link); shown on "Report a run" when set
|
||||
extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0/export?format=xlsx", ext: xlsx }
|
||||
transform: { name: sheet, sheet_contains: form responses }
|
||||
privacy: { redact_emails: true }
|
||||
load: data/public/form_responses.json
|
||||
runs: spreadsheet # core/spec/form.yaml + core/atlas/coding.py
|
||||
|
||||
register:
|
||||
label: Unofficial Dragon Register (which dragon each rider bonded)
|
||||
links: { sheet: https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ }
|
||||
extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ/export?format=xlsx", ext: xlsx }
|
||||
transform: { name: sheet, sheet_contains: form responses, extra: collection/manual/reddit_results.yaml }
|
||||
privacy: { redact_emails: true, drop_columns_matching: [photo, cadet name] } # no rider names, no photos
|
||||
load: data/public/register.json
|
||||
runs: null # bonded dragons without the route that led there
|
||||
|
||||
reddit_pdf:
|
||||
label: Community route PDF (u/ClumsyBadger)
|
||||
run_source: reddit pdf
|
||||
links: { view: https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view }
|
||||
extract: { kind: http, url: "https://drive.google.com/uc?export=download&id=17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg", ext: pdf }
|
||||
transform: { name: pdf_fingerprint }
|
||||
privacy: { redistribute: false } # only hashes of the file, its pages and its lines are kept
|
||||
load: data/derived/pdf_fingerprint.json
|
||||
runs: null # a flowchart of the routes, not a log of runs (see collection/README.md)
|
||||
|
||||
reddit_thread:
|
||||
label: Runs posted in the r/fourthwing Dragonkind megathreads
|
||||
run_source: reddit thread
|
||||
links:
|
||||
part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/
|
||||
part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/
|
||||
# the official Reddit API (app-only OAuth); skipped with a note when the secrets are not set
|
||||
extract: { kind: reddit, threads: [1wvwfp9, 1wz5cnj], secrets: [REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET], ext: json }
|
||||
# lists dragonkind.com result links that are not in the manual file yet; usernames are never kept
|
||||
transform: { name: reddit_results, manual: collection/manual/reddit_results.yaml }
|
||||
load: null # the public source file is the hand-checked collection/manual/reddit_results.yaml
|
||||
runs: reddit_results
|
||||
|
||||
reported:
|
||||
label: Runs reported on the Atlas ("Report a run", a GitHub issue form)
|
||||
run_source: reported
|
||||
links: { report: "https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" }
|
||||
extract: { kind: github_issues, repo: ekcSchiano/threshing-atlas, label: run report, ext: json }
|
||||
# GitHub usernames are dropped; issues closed as "not planned" or labelled "invalid" are left out
|
||||
transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] }
|
||||
load: data/public/reports.json
|
||||
runs: reports
|
||||
|
||||
bot:
|
||||
label: DragonKind bot runs (Ken's private bot, imported locally)
|
||||
run_source: bot
|
||||
# the bot's repo is private: this step runs only on a machine with a copy of it
|
||||
extract: { kind: local_dir, env: DRAGONKIND_BOT_DIR, files: [data/knowledge.json, data/runs.jsonl, logs/encounters.jsonl] }
|
||||
transform: { name: bot, spec: collection/specs/bot.yaml }
|
||||
privacy:
|
||||
account_salt_env: DK_ACCOUNT_SALT # account ids are hashed with this salt, which is never committed
|
||||
refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName]
|
||||
max_string: 200 # anything longer looks like game prose
|
||||
load: data/bot # scenes.json (merged) + trajectories-<date>.jsonl
|
||||
runs: bot
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Data collection: extract -> transform -> load for every source, then build the combined runs.
|
||||
|
||||
python -m collection update [--source NAME ...] [--file NAME=PATH ...] [--report FILE]
|
||||
For each source (all by default): extract it (or take a local file), and when the raw file changed,
|
||||
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
|
||||
reached: the report says what happened (and the daily workflow opens an issue).
|
||||
python -m collection build [--check]
|
||||
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
||||
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
||||
core/graph.js, and data/croissant.json. With --check nothing is written; exit 1
|
||||
if any of them is out of date (CI runs this).
|
||||
|
||||
Sources and their rules: collection/pipeline.yaml. What each column means: data/dictionary.yaml.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .etl import extract as E
|
||||
from .etl import load as L
|
||||
from .etl import codegen, metadata, report_form, runs, validate
|
||||
from .etl.common import Report, path, pipeline
|
||||
from .etl.transform import TRANSFORMS
|
||||
|
||||
|
||||
def update_source(name, spec, manifest, report, local_file=None, env=None):
|
||||
env = os.environ if env is None else env
|
||||
try:
|
||||
raw, changed = E.extract(name, spec, manifest, env, local_file)
|
||||
except E.Skip as e:
|
||||
report.add(f"- **{spec['label']}:** skipped ({e}).")
|
||||
return False
|
||||
except Exception as e: # one broken source must not block the others
|
||||
report.failed.append(name)
|
||||
report.add(f"- **{spec['label']}:** could not extract ({type(e).__name__}: {e}).")
|
||||
return False
|
||||
if not changed:
|
||||
report.add(f"- **{spec['label']}:** no change.")
|
||||
return False
|
||||
mod = TRANSFORMS[spec["transform"]["name"]]
|
||||
ctx = {"changed": manifest[name]["changed"], "env": env}
|
||||
new = mod.transform(raw, spec, ctx)
|
||||
target = spec.get("load")
|
||||
if name == "bot":
|
||||
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
||||
lines = mod.describe(old, new)
|
||||
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
||||
wrote |= L.jsonl_file(f"{target}/trajectories-{ctx['changed']}.jsonl", new["trajectories"])
|
||||
elif target:
|
||||
old = L.read_json(target)
|
||||
lines = mod.describe(old, new)
|
||||
wrote = L.json_file(target, new, compact=not target.startswith("data/derived"))
|
||||
else: # nothing to load: the report is the output (e.g. Reddit result links to add by hand)
|
||||
lines, wrote = mod.describe(None, new), False
|
||||
report.add(f"- **{spec['label']}:** {lines[0]}")
|
||||
for line in lines[1:]:
|
||||
report.add(line)
|
||||
return wrote
|
||||
|
||||
|
||||
def build(check=False, report=None):
|
||||
"""Combined runs, issue form and metadata. Returns the list of files that changed (or would change)."""
|
||||
pipe = pipeline()
|
||||
rows = runs.build_runs(pipe)
|
||||
validate.check(rows, runs.DICT, "runs_clean.csv")
|
||||
out = []
|
||||
if L.csv_file(pipe["runs"]["path"], rows, runs.COLUMNS, check):
|
||||
out.append(pipe["runs"]["path"])
|
||||
if L.write_bytes(report_form.TEMPLATE, report_form.render().encode("utf-8"), check):
|
||||
out.append(report_form.TEMPLATE)
|
||||
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
|
||||
out.append(codegen.GRAPH_JS)
|
||||
# the metadata hashes the files above, so it is built from what is (or would be) on disk
|
||||
if not check or not out:
|
||||
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
|
||||
out.append(metadata.OUT)
|
||||
counts = {}
|
||||
for r in rows:
|
||||
counts[r["source"]] = counts.get(r["source"], 0) + 1
|
||||
msg = f"runs_clean.csv: {len(rows)} runs ({', '.join(f'{k} {v}' for k, v in counts.items())})"
|
||||
(report.add if report else print)(f"- **Combined runs:** {msg}")
|
||||
return out
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(prog="python -m collection", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
up = sub.add_parser("update")
|
||||
up.add_argument("--source", action="append", help="only these sources (repeatable)")
|
||||
up.add_argument("--file", action="append", default=[], help="NAME=PATH: use a local file instead of extracting")
|
||||
up.add_argument("--report")
|
||||
b = sub.add_parser("build")
|
||||
b.add_argument("--check", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
if a.cmd == "build":
|
||||
changed = build(check=a.check)
|
||||
if a.check and changed:
|
||||
print("out of date (run `python -m collection build`): " + ", ".join(changed), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
pipe = pipeline()
|
||||
manifest = L.read_json(pipe["manifest"]) or {}
|
||||
files = dict(f.split("=", 1) for f in a.file)
|
||||
report = Report()
|
||||
for name, spec in pipe["sources"].items():
|
||||
if a.source and name not in a.source:
|
||||
continue
|
||||
report.changed |= update_source(name, spec, manifest, report, files.get(name))
|
||||
L.json_file(pipe["manifest"], dict(sorted(manifest.items())), compact=False)
|
||||
try:
|
||||
report.changed |= bool(build(report=report))
|
||||
except validate.Invalid as e: # new answers the coding rules do not know yet: report them, load nothing
|
||||
report.failed.append("build")
|
||||
report.add(f"- **Combined runs not rebuilt:** {e}")
|
||||
report.add(" - Add the new answers to `core/spec/form.yaml` (or `data/dictionary.yaml`), then run `python -m collection build`.")
|
||||
text = report.text(f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
||||
if a.report:
|
||||
with open(a.report, "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
if os.environ.get("GITHUB_OUTPUT"):
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
f.write(f"changed={'true' if report.changed else 'false'}\nfailed={','.join(report.failed)}\n")
|
||||
return 0
|
||||
@@ -1,49 +0,0 @@
|
||||
"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads.
|
||||
|
||||
python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json
|
||||
|
||||
The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page
|
||||
applies exactly the same cleaning rules to it as it would to a freshly uploaded export).
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
|
||||
def load_raw(path):
|
||||
if str(path).lower().endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
return pd.read_excel(xl, sheet)
|
||||
|
||||
|
||||
def cell(v):
|
||||
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
|
||||
return None
|
||||
if isinstance(v, (datetime, pd.Timestamp)):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
if isinstance(v, str):
|
||||
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
|
||||
return v
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json"
|
||||
raw = load_raw(src)
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
# keep every header present even if a row is blank there, so column detection sees all questions
|
||||
headers = [str(c) for c in raw.columns]
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
|
||||
"headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
|
||||
print(f"{len(rows)} rows -> {out}")
|
||||
@@ -1,43 +0,0 @@
|
||||
"""export_register.py — convert the "Unofficial Dragon Register" .xlsx/.csv export into web/data/register.json.
|
||||
|
||||
python export_register.py ../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx ../../data/public/register.json
|
||||
|
||||
Raw rows are kept as-is, except that photo links and rider names are dropped. Dragons read from public dragonkind.com result pages
|
||||
linked in the Reddit megathreads are stored separately under "extra" with their source URL.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from export_raw import cell, load_raw
|
||||
|
||||
# Read from dragonkind.com/results/<id> pages linked in the Part 3 and Part 4 megathreads (2026-10-19).
|
||||
REDDIT_RESULTS = [
|
||||
("THRESH-VFRQVM5K3B0CP", "Thiduach", "Blue", "Swordtail", "Female", 51, "Gormfaileas"),
|
||||
("THRESH-3M4P9MNFQGBS8", "Caenuis", "Blue", "Swordtail", "Female", 38, "Gormfaileas"),
|
||||
("THRESH-5JQDJ21G1BYKR", "Oavuagh", "Green", "Morningstartail", "Female", 43, "Uaineloidsig"),
|
||||
("THRESH-5KRAAVFB3D1MG", "Geagios", "Blue", "Clubtail", "Male", 99, "Gormfaileas"),
|
||||
("THRESH-B5HSM00J4A2Z4", "Fiedheil", "Black", "Clubtail", "Male", 32, "Moerlaighen"),
|
||||
("THRESH-BMPM2GAPK58JT", "Teirfua", "Green", "Morningstartail", "Female", 68, "Eifyrielle"),
|
||||
("THRESH-MCQ8D6GAGF4K8", "Caollo", "Blue", "Morningstartail", "Female", 93, "Nyghmuiron"),
|
||||
("THRESH-Q0XDBFMNBEB86", "Aenminn", "Red", "Swordtail", "Male", 59, "Berradhyr"),
|
||||
("THRESH-W99DB29MG16QA", "Meinso", "Green", "Clubtail", "Female", 63, "Eifyrielle"),
|
||||
("THRESH-XCJCY7HW7QSSP", "Einmial", "Blue", "Clubtail", "Male", 58, "Gormfaileas"),
|
||||
("THRESH-XR6G68VPGT48M", "Recuagh", "Brown", "Morningstartail", "Female", 55, "Vormalgh"),
|
||||
("THRESH-YQC97TK34214E", "Dannidh", "Orange", "Clubtail", "Male", 61, "Cariaslog"),
|
||||
]
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/register.json"
|
||||
raw = load_raw(src)
|
||||
raw = raw[[c for c in raw.columns if "photo" not in str(c).lower() and "cadet name" not in str(c).lower()]]
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
extra = [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{i}",
|
||||
"name": n, "colour": c, "tail": t, "sex": s, "age": a, "lineage": l} for i, n, c, t, s, a, l in REDDIT_RESULTS]
|
||||
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
|
||||
"headers": [str(c) for c in raw.columns], "rows": rows, "extra": extra},
|
||||
open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
|
||||
print(f"{len(rows)} register rows + {len(extra)} Reddit result pages -> {out}")
|
||||
@@ -1,185 +0,0 @@
|
||||
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
|
||||
|
||||
python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
|
||||
|
||||
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
|
||||
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
|
||||
which is handy for testing or when a download link changes.
|
||||
|
||||
What it does:
|
||||
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
|
||||
when the responses changed, and reports how many rows were added or edited.
|
||||
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
|
||||
The raw register export is never written into the repo.
|
||||
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
|
||||
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
|
||||
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
|
||||
|
||||
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, os.path.join(ROOT, "core"))
|
||||
from export_raw import cell, load_raw # noqa: E402
|
||||
|
||||
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
|
||||
|
||||
|
||||
def sha(data, n=None):
|
||||
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
|
||||
return h[:n] if n else h
|
||||
|
||||
|
||||
def download(url, dest):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
|
||||
data = r.read()
|
||||
f.write(data)
|
||||
head = data[:15].lower()
|
||||
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
||||
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
|
||||
return dest
|
||||
|
||||
|
||||
def sheet_rows(path, drop=()):
|
||||
raw = load_raw(path)
|
||||
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
return [str(c) for c in raw.columns], rows
|
||||
|
||||
|
||||
def diff_rows(old, new):
|
||||
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
|
||||
o, n = {key(r) for r in old}, {key(r) for r in new}
|
||||
return len(n - o), len(o - n)
|
||||
|
||||
|
||||
def update_form(src, report):
|
||||
out = P("data", "public", "form_responses.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src)
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE)
|
||||
from atlas import coding # noqa: E402
|
||||
coding.main([out, P("data", "derived", "runs_clean.csv")])
|
||||
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
|
||||
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
|
||||
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
|
||||
shutil.copyfile(src, raw_copy)
|
||||
new_q = [h for h in headers if h not in current["headers"]]
|
||||
gone_q = [h for h in current["headers"] if h not in headers]
|
||||
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
|
||||
f"({added} new or edited, {removed} removed or edited).")
|
||||
if new_q:
|
||||
report.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}")
|
||||
if gone_q:
|
||||
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
|
||||
return True
|
||||
|
||||
|
||||
def update_register(src, report):
|
||||
out = P("data", "public", "register.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("collection", "sources", "export_register.py"), src, out], check=True, cwd=HERE)
|
||||
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
|
||||
return True
|
||||
|
||||
|
||||
def pdf_lines(path):
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError:
|
||||
sys.exit("pypdf is required: pip install -r requirements.txt")
|
||||
pages = []
|
||||
for page in PdfReader(path).pages:
|
||||
text = page.extract_text() or ""
|
||||
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
|
||||
return pages
|
||||
|
||||
|
||||
def update_pdf(src, report):
|
||||
out = P("data", "derived", "pdf_fingerprint.json")
|
||||
old = json.load(open(out)) if os.path.exists(out) else None
|
||||
data = open(src, "rb").read()
|
||||
pages = pdf_lines(src)
|
||||
fp = {
|
||||
"sha256": sha(data),
|
||||
"bytes": len(data),
|
||||
"pages": len(pages),
|
||||
"page_hashes": [sha("\n".join(p), 16) for p in pages],
|
||||
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
|
||||
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
|
||||
}
|
||||
if old and old["sha256"] == fp["sha256"]:
|
||||
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
|
||||
return False
|
||||
if old and old.get("page_hashes") == fp["page_hashes"]:
|
||||
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
|
||||
elif old:
|
||||
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
|
||||
old_lines = set(old["line_hashes"])
|
||||
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
|
||||
removed = len(old_lines - set(fp["line_hashes"]))
|
||||
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
|
||||
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
|
||||
if new_lines:
|
||||
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
|
||||
report.extend(f" - {l}" for l in new_lines[:60])
|
||||
if len(new_lines) > 60:
|
||||
report.append(f" - … and {len(new_lines) - 60} more")
|
||||
else:
|
||||
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
|
||||
json.dump(fp, open(out, "w"), indent=1)
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--report", default=None)
|
||||
ap.add_argument("--form")
|
||||
ap.add_argument("--register")
|
||||
ap.add_argument("--pdf")
|
||||
args = ap.parse_args()
|
||||
sources = json.load(open(P("data", "sources.json")))
|
||||
report, changed, failed = [], False, []
|
||||
tmp = tempfile.mkdtemp(prefix="sources-")
|
||||
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
|
||||
for key, local, fn in steps:
|
||||
try:
|
||||
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
|
||||
changed |= fn(os.path.abspath(path), report)
|
||||
except Exception as e: # keep going: one broken link should not block the others
|
||||
failed.append(key)
|
||||
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
|
||||
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
|
||||
text = "\n".join([f"## {head}", ""] + report) + "\n"
|
||||
print(text)
|
||||
if args.report:
|
||||
open(args.report, "w", encoding="utf-8").write(text)
|
||||
if os.environ.get("GITHUB_OUTPUT"):
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
# The DragonKind bot: how its screens and choices become public data and runs. collection/etl/transform/bot.py.
|
||||
#
|
||||
# The bot's raw data (game prose, account ids) stays on the machine that runs it. What is published:
|
||||
# scene fingerprints, the short labels below (our own paraphrases, not game prose), choice text and outcomes.
|
||||
version: 1
|
||||
|
||||
# Must match normalize() in the bot's src/strategy.js, so fingerprints line up (applied in this order).
|
||||
normalize:
|
||||
- ["[\u2018\u2019\u201b`']", "'"]
|
||||
- ["[\u201c\u201d\"]", "\""]
|
||||
- ["[\u2013\u2014-]", " "]
|
||||
- ["\u2026", " "]
|
||||
- ["[^a-z0-9'\\s]", " "]
|
||||
- ["\\s+", " "]
|
||||
|
||||
# Events in the bot's log that record a choice it acted on
|
||||
choice_events: [DECISION, MANUAL_DECISION, MANUAL_EDGE_PICK]
|
||||
# runs.jsonl and the log are matched when the same attempt starts within this many hours
|
||||
match_hours: 6
|
||||
|
||||
# Short public labels, by scene fingerprint. A scene without one is labelled "<stage> · <tint>" and listed by the
|
||||
# importer so a label can be added here.
|
||||
labels:
|
||||
"db7786392626": "Start: choose company"
|
||||
"a68f681076c6": "Valley edge: choose direction"
|
||||
"bc4ee4754b9c": "First sighting: Red (forest centre)"
|
||||
"abe3011378ac": "First sighting: Brown (forest centre)"
|
||||
"84780dab01e3": "Survived the first sighting"
|
||||
"d25f2759ace0": "Deeper in the forest: a scream"
|
||||
"f72199f24b91": "Clearing: squadmate vs two bullies"
|
||||
"b209bacacec3": "Bully fight: first move"
|
||||
"fbe0f0bea700": "Bully fight: smaller cadet has a dagger"
|
||||
"a2368b0dc212": "Blocked the bigger cadet: bullies’ fate"
|
||||
"f4b4944d3f3b": "Beat the smaller cadet: bullies’ fate"
|
||||
"0cfd06cf8939": "After the fight: choose a direction"
|
||||
"0a561d53ad19": "An hour later: valley or tree"
|
||||
"96b68612eaf4": "Top of the tree (shadow check)"
|
||||
"10bc7353b8eb": "Descended quickly: a Red below"
|
||||
"c62f1f1a2b3a": "Let the Red pass: two Greens in the clearing"
|
||||
"07e24681e3b3": "Descended slowly: two Greens in the clearing"
|
||||
|
||||
# Crosswalk: a bot choice (option text, normalised as in core/graph.js norm()) -> a column of the coded runs.
|
||||
# First match wins; a column keeps its first value. Unmatched choices are listed, never guessed.
|
||||
# Two keys are not columns: scream_investigate (the scream step on its own) and red (the Red below the tree).
|
||||
crosswalk:
|
||||
- ["^enter the valley alone", company, "Alone"]
|
||||
- ["^enter the valley with one of your squadmates", company, "1 Squadmate"]
|
||||
- ["^enter the valley with two of your squadmates", company, "2 Squadmates"]
|
||||
- ["^head east toward the roar", direction, "East"]
|
||||
- ["^head west toward the scent", direction, "West"]
|
||||
- ["^head toward the center of the forest", direction, "Center"]
|
||||
- ["^keep walking the cadets are just as dangerous", scream, "Ignore the scream"]
|
||||
- ["^run and hope", sighting_action, "Retreat / run"]
|
||||
- ["hold still", sighting_action, "Hold still"]
|
||||
- ["lower your eyes in respect|lower your eyes and turn to face it", sighting_action, "Respect / face it"]
|
||||
- ["meeting its gaze", sighting_action, "Meet its gaze"]
|
||||
- ["^hide behind a tree", sighting_action, "Hide"]
|
||||
- ["^investigate the scream", scream_investigate, true]
|
||||
- ["^help your squadmate", scream, "Investigate & help"]
|
||||
- ["^play it safe and leave them", scream, "Investigate & leave"]
|
||||
- ["^pause to assess the situation", combat_first, "Assess the situation"]
|
||||
- ["^charge the other cadet", combat_first, "Even the numbers (smaller cadet)"]
|
||||
- ["^unsheathe your weapon and charge between", combat_first, "Put yourself in the middle (bigger cadet)"]
|
||||
- ["^engage the smaller cadet", combat_second, "Engage the smaller cadet"]
|
||||
- ["^daggers or not", combat_second, "Daggers or not (protect squadmate)"]
|
||||
- ["^end the lives of the bullies", resolution, "Killed the bullies yourself"]
|
||||
- ["^allow your squadmate to exact", resolution, "Squadmate's revenge"]
|
||||
- ["^leave the bullies alive", resolution, "Left bullies to the dragons"]
|
||||
- ["tourniquet", resolution, "Tourniquet"]
|
||||
# the wounded cadet's other two choices (the bot has not logged this screen yet, so these are kept loose)
|
||||
- ["leave (them|him|her) to die", resolution, "Left injured cadet"]
|
||||
- ["take (their|his|her) weapons?", resolution, "Took their weapons"]
|
||||
- ["^walk toward the east", post_path, "East"]
|
||||
- ["^walk toward the west", post_path, "West"]
|
||||
- ["^walk farther into the forest", post_path, "Forest"]
|
||||
- ["^continue toward the center of the valley", hour, "Valley"]
|
||||
- ["^find the tallest tree", hour, "Tree"]
|
||||
- ["^descend quickly", branch, "Descend quickly: Red"]
|
||||
- ["^climb down slowly to pursue the green", branch, "Descend slowly: two Greens"]
|
||||
- ["^reds are too hot tempered", red, "Let it pass (two Greens)"]
|
||||
- ["^jump (now|in a few seconds)", red, "Jump onto the Red"]
|
||||
@@ -0,0 +1,118 @@
|
||||
# "Report a run": the questions of the GitHub issue form, and how each answer becomes a column of runs_clean.csv.
|
||||
#
|
||||
# collection/etl/report_form.py writes .github/ISSUE_TEMPLATE/report-a-run.yml from this file (a test checks it is
|
||||
# up to date), and collection/etl/runs.py codes every report with it. Each option is "label shown: column value";
|
||||
# an option with value null (or no answer) means "not known". Values must be listed in data/dictionary.yaml.
|
||||
version: 1
|
||||
|
||||
name: Report a run
|
||||
description: Log one threshing run for the Threshing Atlas. Reports are public and become rows of runs_clean.csv.
|
||||
title: "Run report"
|
||||
label: run report
|
||||
intro: |
|
||||
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
|
||||
Every answer here is **public** (this is a GitHub issue) and goes into the Atlas's open data as source "reported".
|
||||
Please don't include your DragonKind account name or anything personal.
|
||||
You can also use the community Google Form linked from the r/fourthwing megathread; please report each run only once.
|
||||
|
||||
fields:
|
||||
- id: goal
|
||||
label: Which dragon were you trying to bond?
|
||||
column: goal
|
||||
options: { Red: Red, Green: Green, Orange: Orange, Brown: Brown, Blue: Blue, Black: Black, "Any, just for fun": Any (for fun) }
|
||||
- id: attempt
|
||||
label: Which attempt was this for your account? (a number; 1 = your first)
|
||||
column: attempt
|
||||
type: integer
|
||||
- id: company
|
||||
label: Who entered the valley with you?
|
||||
column: company
|
||||
required: true
|
||||
options: { Alone: Alone, One squadmate: 1 Squadmate, Two squadmates: 2 Squadmates }
|
||||
- id: left_squadmates
|
||||
label: If you went with squadmates, did you leave them?
|
||||
column: left_squadmates
|
||||
options: { "Yes, I left them": true, "No, we stayed together": false, "I went alone": null }
|
||||
- id: direction
|
||||
label: Which way did you go first?
|
||||
column: direction
|
||||
required: true
|
||||
options: { East (toward the roar): East, West (toward the scent): West, Toward the forest centre: Center }
|
||||
- id: first_sighting
|
||||
label: Which dragon found you first?
|
||||
column: first_sighting
|
||||
options: { Red: Red, Brown: Brown, Green: Green, Orange: Orange }
|
||||
- id: sighting_action
|
||||
label: What did you do when it found you?
|
||||
column: sighting_action
|
||||
options:
|
||||
Hold still: Hold still
|
||||
Lower your eyes in respect / turn to face it: Respect / face it
|
||||
Run: Retreat / run
|
||||
Hide: Hide
|
||||
Draw a weapon: Weapon
|
||||
Meet its gaze: Meet its gaze
|
||||
Protect your squadmate: Protect squadmate
|
||||
- id: scream
|
||||
label: The scream
|
||||
column: scream
|
||||
required: true
|
||||
options:
|
||||
I died at the first dragon: DIED at first sighting
|
||||
Investigated, and helped my squadmate: Investigate & help
|
||||
Investigated, and walked away: Investigate & leave
|
||||
Kept walking (ignored it): Ignore the scream
|
||||
- id: combat_first
|
||||
label: "If you helped: your first move in the fight"
|
||||
column: combat_first
|
||||
options:
|
||||
Pause to assess: Assess the situation
|
||||
Charge the smaller cadet: Even the numbers (smaller cadet)
|
||||
Charge between the bigger cadet and your squadmate: Put yourself in the middle (bigger cadet)
|
||||
- id: combat_second
|
||||
label: "If you paused to assess: the smaller cadet's dagger"
|
||||
column: combat_second
|
||||
options: { Engage the smaller cadet: Engage the smaller cadet, Charge the bigger cadet anyway: Daggers or not (protect squadmate) }
|
||||
- id: resolution
|
||||
label: How did the cadet encounter end?
|
||||
column: resolution
|
||||
options:
|
||||
I died in the fight: DIED in combat
|
||||
"Bullies: killed them myself": Killed the bullies yourself
|
||||
"Bullies: my squadmate's revenge": Squadmate's revenge
|
||||
"Bullies: left them to the dragons": Left bullies to the dragons
|
||||
"Wounded cadet: tourniquet": Tourniquet
|
||||
"Wounded cadet: left them": Left injured cadet
|
||||
"Wounded cadet: took their weapons": Took their weapons
|
||||
"I walked away from the clearing": Left them to it
|
||||
- id: post_path
|
||||
label: "After the fight: which way did you walk?"
|
||||
column: post_path
|
||||
options: { Deeper into the forest: Forest, West: West, East: East, There was no fork: (no fork) }
|
||||
- id: hour
|
||||
label: After an hour's walk
|
||||
column: hour
|
||||
options: { Climbed the tallest tree: Tree, Went to the valley centre: Valley, I died before the hour: DIED before the hour }
|
||||
- id: branch
|
||||
label: Which bonding encounter did you reach?
|
||||
column: branch
|
||||
options:
|
||||
"Tree: the shadow (Blue or Black)": Shadow (rare)
|
||||
"Tree: climbed down quickly, jumped on the Red": "Descend quickly: Red"
|
||||
"Tree: climbed down quickly, let the Red pass": "Descend quickly: let the Red pass"
|
||||
"Tree: climbed down slowly to the two Greens": "Descend slowly: two Greens"
|
||||
"Valley: faced the Red": Valley Red
|
||||
"Valley: passed the Red, the meadow": "Passed the Red: meadow"
|
||||
- id: outcome
|
||||
label: How did the run end?
|
||||
column: outcome
|
||||
required: true
|
||||
options: { Bonded: Bonded, Incinerated at a bonding encounter: Incinerated, Died before the hour: Died before the hour }
|
||||
- id: outcome_color
|
||||
label: Which dragon bonded you (or incinerated you)?
|
||||
column: outcome_color
|
||||
options: { Red: Red, Green: Green, Orange: Orange, Brown: Brown, Blue: Blue, Black: Black }
|
||||
- id: tail
|
||||
label: "If you bonded: the dragon's tail"
|
||||
column: tail
|
||||
options: { Clubtail: Clubtail, Daggertail: Daggertail, Morningstartail: Morningstartail, Scorpiontail: Scorpiontail, Swordtail: Swordtail }
|
||||
@@ -4,9 +4,10 @@ The tables (how each question is found, what every answer means, which answers c
|
||||
core/spec/form.yaml. This module holds the logic that combines answers into a run: which way the rider went,
|
||||
which branch they took after the hour, how the run ended, and which rows look inconsistent (data_flags).
|
||||
|
||||
python -m atlas.coding [form_responses.json|.xlsx|.csv] [runs_clean.csv]
|
||||
python -m atlas.coding [form_responses.json|.xlsx|.csv] [out.csv]
|
||||
|
||||
Defaults: data/public/form_responses.json -> data/derived/runs_clean.csv
|
||||
Prints a summary of the coded export (default: data/public/form_responses.json), and writes the coded runs to
|
||||
out.csv when given. The published runs (every source) are written by `python -m collection build`.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
@@ -276,7 +277,8 @@ def read_export(path):
|
||||
"""An export as a DataFrame: the bundled JSON (data/public/form_responses.json), a CSV or the .xlsx."""
|
||||
p = str(path).lower()
|
||||
if p.endswith(".json"):
|
||||
d = json.load(open(path, encoding="utf-8"))
|
||||
with open(path, encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
return pd.DataFrame(d["rows"], columns=d["headers"])
|
||||
if p.endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
@@ -300,14 +302,15 @@ def load(path):
|
||||
def main(argv=None):
|
||||
argv = sys.argv[1:] if argv is None else argv
|
||||
src = argv[0] if argv else ROOT / "data/public/form_responses.json"
|
||||
out = argv[1] if len(argv) > 1 else ROOT / "data/derived/runs_clean.csv"
|
||||
out = argv[1] if len(argv) > 1 else None
|
||||
df, missing = load(src)
|
||||
if missing:
|
||||
print("WARNING: columns not found:", missing)
|
||||
# a whole-number column with gaps would otherwise be written as 1.0, 2.0, ... (data/dictionary.yaml: integer)
|
||||
df["attempt"] = df["attempt"].astype("Int64")
|
||||
if out:
|
||||
df.to_csv(out, index=False)
|
||||
print(f"{len(df)} runs -> {out}")
|
||||
print(f"{len(df)} runs" + (f" -> {out}" if out else ""))
|
||||
print(df.outcome.value_counts().to_string())
|
||||
print("rows with flags:", (df["data_flags"] != "").sum())
|
||||
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Read data/public/runs_clean.csv (written by `python -m collection build`) as typed records.
|
||||
|
||||
Every analysis reads the runs from this one published file. Types come from data/dictionary.yaml: integers and
|
||||
booleans are parsed, empty cells are None (data_flags is "" when a row has no flags).
|
||||
"""
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
PATH = ROOT / "data/public/runs_clean.csv"
|
||||
with open(ROOT / "data/dictionary.yaml", encoding="utf-8") as _f:
|
||||
TYPES = {k: v["type"] for k, v in yaml.safe_load(_f)["columns"].items()}
|
||||
|
||||
|
||||
def _parse(col, v):
|
||||
if v == "":
|
||||
return "" if col == "data_flags" else None
|
||||
t = TYPES.get(col, "text")
|
||||
if t == "integer":
|
||||
return int(v)
|
||||
if t == "boolean":
|
||||
return v == "True"
|
||||
return v
|
||||
|
||||
|
||||
def read(path=PATH, sources=None, drop=("run_id",)):
|
||||
"""Typed records, optionally only from some sources (the `source` column)."""
|
||||
with open(path, encoding="utf-8", newline="") as f:
|
||||
rows = [{k: _parse(k, v) for k, v in r.items() if k not in drop} for r in csv.DictReader(f)]
|
||||
return [r for r in rows if sources is None or r["source"] in sources]
|
||||
+4
-3
@@ -6,8 +6,8 @@
|
||||
* or a terminal (BURNED, BONDED:<colour>). Chance shows up as one option leading to several scenes,
|
||||
* e.g. "head toward the centre" -> a Red OR a Brown first sighting.
|
||||
* 2. Decision-variable graph (all sources): the coarse variables the community form asks about
|
||||
* (company, direction, first sighting, ...). Bot trajectories are mapped onto it with CROSSWALK, so bot and
|
||||
* form runs can be counted side by side, per source.
|
||||
* (company, direction, first sighting, ...). trajectoryVariables() maps a bot trajectory onto it with CROSSWALK
|
||||
* (written from collection/specs/bot.yaml; the Python importer uses the same file), for the bot's advisor.
|
||||
*
|
||||
* Trajectories keep the full ordered history of each attempt, so history-dependent (non-Markov) models can be fitted
|
||||
* later without changing the data format.
|
||||
@@ -79,6 +79,7 @@
|
||||
|
||||
/* ---------- Crosswalk: bot option text -> community form variables ---------- */
|
||||
|
||||
// BEGIN CROSSWALK: written by `python -m collection build` from collection/specs/bot.yaml (edit that file)
|
||||
const CROSSWALK = [
|
||||
[/^enter the valley alone/, "company", "Alone"],
|
||||
[/^enter the valley with one of your squadmates/, "company", "1 Squadmate"],
|
||||
@@ -104,7 +105,6 @@
|
||||
[/^allow your squadmate to exact/, "resolution", "Squadmate's revenge"],
|
||||
[/^leave the bullies alive/, "resolution", "Left bullies to the dragons"],
|
||||
[/tourniquet/, "resolution", "Tourniquet"],
|
||||
// the wounded cadet's other two choices (the bot has not logged this screen yet, so these are kept loose)
|
||||
[/leave (them|him|her) to die/, "resolution", "Left injured cadet"],
|
||||
[/take (their|his|her) weapons?/, "resolution", "Took their weapons"],
|
||||
[/^walk toward the east/, "post_path", "East"],
|
||||
@@ -117,6 +117,7 @@
|
||||
[/^reds are too hot tempered/, "red", "Let it pass (two Greens)"],
|
||||
[/^jump (now|in a few seconds)/, "red", "Jump onto the Red"]
|
||||
];
|
||||
// END CROSSWALK
|
||||
|
||||
/** Map one bot trajectory onto the form's decision variables. Unmapped choices are listed, not guessed. */
|
||||
function trajectoryVariables(traj) {
|
||||
|
||||
+18
-11
@@ -1,18 +1,25 @@
|
||||
# Data sources
|
||||
|
||||
| Source | Where it lives here | How to update |
|
||||
|---|---|---|
|
||||
| Community "Threshing Routes" Google Form (u/ClumsyBadger) | `raw/form/*.xlsx` → `public/form_responses.json` | Download the sheet as .xlsx, then `python collection/sources/export_raw.py` |
|
||||
| Community route PDF (u/ClumsyBadger) | not redistributed; linked from the Reddit megathreads | A scene crosswalk will be kept in `derived/` |
|
||||
| Unofficial Dragon Register | `raw/register/` (git-ignored, has rider names) → `public/register.json` | `python collection/sources/export_register.py` |
|
||||
| Reddit megathreads (r/fourthwing, Dragonkind bonding Parts 3 and 4) | qualitative; routes cited in the valley map | manual |
|
||||
| DragonKind bot exports | `bot/runs-YYYY-MM-DD.jsonl` | by pull request, see `docs/BOT_DATA_PROTOCOL.md` |
|
||||
| Discord | not yet | planned, with admin consent and anonymised usernames |
|
||||
Every source is declared in `collection/pipeline.yaml` and collected by `python -m collection update`
|
||||
(see `collection/README.md`). Every run from every source is in `public/runs_clean.csv`, with a `source` column.
|
||||
|
||||
| Source (`source` column) | Where it lives here |
|
||||
|---|---|
|
||||
| Community "Threshing Routes" form and spreadsheet, u/ClumsyBadger (`reddit spreadsheet`) | `public/form_responses.json` (emails redacted) |
|
||||
| Result pages posted in the r/fourthwing megathreads (`reddit thread`) | `collection/manual/reddit_results.yaml` |
|
||||
| The Atlas's "Report a run" form (`reported`) | `public/reports.json` (answers only, no usernames) |
|
||||
| DragonKind bot exports (`bot`) | `bot/scenes.json`, `bot/trajectories-<date>.jsonl` (anonymised; see `docs/BOT_DATA_PROTOCOL.md`) |
|
||||
| Community route PDF, u/ClumsyBadger (`reddit pdf`) | not redistributed: `derived/pdf_fingerprint.json` only; it maps the choices and holds no runs |
|
||||
| Unofficial Dragon Register | `public/register.json` (rider names and photos dropped); no runs |
|
||||
| Discord | not yet: planned, with admin consent and anonymised usernames |
|
||||
|
||||
Raw files (`raw/`, git-ignored) are never published: the current version of each source and the one before it.
|
||||
`manifest.json` records the current raw file's checksum and when it last changed.
|
||||
|
||||
## What each column means
|
||||
|
||||
`data/dictionary.yaml` describes every published file and every column of `data/derived/runs_clean.csv`: its type,
|
||||
`data/dictionary.yaml` describes every published file and every column of `data/public/runs_clean.csv`: its type,
|
||||
what it means and every value it can take. `data/croissant.json` says the same in the
|
||||
[Croissant](https://docs.mlcommons.org/croissant/) metadata format (with a sha256 checksum for every file), so
|
||||
dataset tools can load and cite the data. After changing the data or the dictionary, run `npm run metadata`; the
|
||||
tests fail if a column or value is not described or the metadata is out of date.
|
||||
dataset tools can load and cite the data. `python -m collection build` rewrites it; the tests fail if a column or
|
||||
value is not described or the metadata is out of date.
|
||||
|
||||
+50
-15
@@ -67,13 +67,23 @@
|
||||
"isBasedOn": [
|
||||
{
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Threshing Routes form responses (u/ClumsyBadger)",
|
||||
"name": "Threshing Routes form and spreadsheet (u/ClumsyBadger, linked from the r/fourthwing megathreads)",
|
||||
"url": "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0"
|
||||
},
|
||||
{
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Unofficial Dragon Register responses",
|
||||
"name": "Unofficial Dragon Register (which dragon each rider bonded)",
|
||||
"url": "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ"
|
||||
},
|
||||
{
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Runs posted in the r/fourthwing Dragonkind megathreads",
|
||||
"url": "https://www.reddit.com/r/fourthwing/comments/1wvwfp9/"
|
||||
},
|
||||
{
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Runs reported on the Atlas (\"Report a run\", a GitHub issue form)",
|
||||
"url": "https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml"
|
||||
}
|
||||
],
|
||||
"distribution": [
|
||||
@@ -81,11 +91,11 @@
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "runs_clean",
|
||||
"name": "runs_clean",
|
||||
"description": "One coded run per row of the community form (core/atlas/coding.py, rules in core/spec/form.yaml).",
|
||||
"description": "Every run from every source, one per row, coded into the same columns (collection/etl/runs.py; the form's rules are core/spec/form.yaml). The `source` column says where each run came from.",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/runs_clean.csv",
|
||||
"encodingFormat": "text/csv",
|
||||
"sha256": "ef34b36881b6bc0f57453e425b1c7d76155e6572463aff0d7e89b9857823f7db",
|
||||
"contentSize": "1991643 B"
|
||||
"sha256": "766d8708dad77327534063df42cd4ff806ab707497618ab5263cf3dce98a30c7",
|
||||
"contentSize": "2146368 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
@@ -97,6 +107,16 @@
|
||||
"sha256": "4c5791f78ce81f5e88f1fb99d933acd06d8223ff9a573e96b6fcb3a6fd9c8d22",
|
||||
"contentSize": "8293395 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "reports",
|
||||
"name": "reports",
|
||||
"description": "Runs reported through the Atlas's \"Report a run\" form (GitHub issues), answers only, no usernames.",
|
||||
"contentUrl": "https://github.com/ekcSchiano/threshing-atlas/blob/main/data/public/reports.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "d42eb04b9923eb1d2a8e5c356452f52daab78293be5a29a448cfb69c974eab30",
|
||||
"contentSize": "92 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "register",
|
||||
@@ -114,8 +134,8 @@
|
||||
"description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/stats.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "d48b1da5176d6e85f6ee093cfb4f50dfbc6f71cfa7fec8ecb49a7c32c14f6d53",
|
||||
"contentSize": "4269825 B"
|
||||
"sha256": "dc79c0762e5c27423f9fa2086f06aa7019b0aec6ae41d6a6dcbe59a1016cffab",
|
||||
"contentSize": "4269546 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
@@ -131,11 +151,11 @@
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "graph",
|
||||
"name": "graph",
|
||||
"description": "The route graph over every source (form and bot), counted per source (analysis/network/build-graph.js).",
|
||||
"description": "The route graph over every run in runs_clean.csv, counted per source (analysis/network/build-graph.js).",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/graph.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "043227ff7d858fa8e73e7447f4cd7171bb216ab8d203b036df056d8ed542e446",
|
||||
"contentSize": "48634 B"
|
||||
"sha256": "72ef6fa6d6edc3a53026eff3f9cf39a883f7a709eacaaf21351c9ca1251a986a",
|
||||
"contentSize": "48783 B"
|
||||
}
|
||||
],
|
||||
"recordSet": [
|
||||
@@ -143,7 +163,7 @@
|
||||
"@type": "cr:RecordSet",
|
||||
"@id": "runs",
|
||||
"name": "runs",
|
||||
"description": "One coded threshing run per row (data/derived/runs_clean.csv).",
|
||||
"description": "One coded threshing run per row, from every source (data/public/runs_clean.csv).",
|
||||
"key": {
|
||||
"@id": "runs/run_id"
|
||||
},
|
||||
@@ -152,8 +172,8 @@
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/run_id",
|
||||
"name": "run_id",
|
||||
"description": "Row number in the form export (1 = first row).",
|
||||
"dataType": "sc:Integer",
|
||||
"description": "Unique id of the run: sheet-<row of the form export>, thread-<result id>, reported-<issue number> or bot-<hash>.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
@@ -167,7 +187,7 @@
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/source",
|
||||
"name": "source",
|
||||
"description": "\"Form\" for a form submission (it has a timestamp). Bulk-pasted logs keep the label they were pasted with; they record every run of the players who kept them, so they are the least selectively reported rows. Values: Form; Data Import; BulkUpload3; Manual Upload; Historical import; Unknown import.",
|
||||
"description": "Where the run came from. \"reddit spreadsheet\": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. \"reddit thread\": a dragonkind.com result page posted in a megathread (bonded, route not recorded). \"reported\": the Atlas's own \"Report a run\" form. \"bot\": an attempt logged by the DragonKind bot. \"reddit pdf\": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). Values: reddit spreadsheet; reddit thread; reported; bot; reddit pdf.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
@@ -178,11 +198,26 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/entry",
|
||||
"name": "entry",
|
||||
"description": "For spreadsheet runs, the sheet's own row label: \"Form\" for a form submission (it has a timestamp); bulk-pasted logs keep the label they were pasted with. Bulk logs record every run of the players who kept them, so they are the least selectively reported rows. Empty for other sources. Values: Form; Data Import; BulkUpload3; Manual Upload; Historical import; Unknown import.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "entry"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/date",
|
||||
"name": "date",
|
||||
"description": "Date of a form submission (YYYY-MM-DD). Empty for bulk-pasted rows.",
|
||||
"description": "Date of the run or its report (YYYY-MM-DD): the form submission, the issue, or the bot attempt. Empty for bulk-pasted rows and result pages.",
|
||||
"dataType": "sc:Date",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+26
-9
@@ -1,7 +1,7 @@
|
||||
# Data dictionary: every published file and every column of the coded runs.
|
||||
#
|
||||
# collection/publish_metadata.py builds data/croissant.json (Croissant, the MLCommons metadata format) from this
|
||||
# file, data/sources.json and the licences. tests/core/test_dictionary.py checks that every column of the coded runs
|
||||
# collection/etl/metadata.py builds data/croissant.json (Croissant, the MLCommons metadata format) from this
|
||||
# file, collection/pipeline.yaml and the licences. tests/core/test_dictionary.py checks that every column of the coded runs
|
||||
# is described here and that every value it takes is listed.
|
||||
#
|
||||
# Missing answers: "Unknown", "Unsure", "(not asked)" and the other values in core/spec/form.yaml missing_values carry
|
||||
@@ -25,16 +25,22 @@ dataset:
|
||||
|
||||
files:
|
||||
runs_clean:
|
||||
path: data/derived/runs_clean.csv
|
||||
path: data/public/runs_clean.csv
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/runs_clean.csv
|
||||
format: text/csv
|
||||
description: One coded run per row of the community form (core/atlas/coding.py, rules in core/spec/form.yaml).
|
||||
description: >-
|
||||
Every run from every source, one per row, coded into the same columns (collection/etl/runs.py; the form's
|
||||
rules are core/spec/form.yaml). The `source` column says where each run came from.
|
||||
form_responses:
|
||||
path: data/public/form_responses.json
|
||||
format: application/json
|
||||
description: >-
|
||||
The community "Threshing Routes" form export as published (headers and rows), with email addresses in
|
||||
free-text answers redacted. The source of runs_clean.
|
||||
reports:
|
||||
path: data/public/reports.json
|
||||
format: application/json
|
||||
description: Runs reported through the Atlas's "Report a run" form (GitHub issues), answers only, no usernames.
|
||||
register:
|
||||
path: data/public/register.json
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/register.json
|
||||
@@ -58,19 +64,30 @@ files:
|
||||
path: data/public/graph.json
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/graph.json
|
||||
format: application/json
|
||||
description: The route graph over every source (form and bot), counted per source (analysis/network/build-graph.js).
|
||||
description: The route graph over every run in runs_clean.csv, counted per source (analysis/network/build-graph.js).
|
||||
|
||||
# The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take
|
||||
# (empty cells are always allowed and mean "does not apply" or "not recorded").
|
||||
columns:
|
||||
run_id: { type: integer, description: Row number in the form export (1 = first row). }
|
||||
run_id: { type: text, required: true, description: "Unique id of the run: sheet-<row of the form export>, thread-<result id>, reported-<issue number> or bot-<hash>." }
|
||||
source:
|
||||
type: text
|
||||
required: true
|
||||
description: >-
|
||||
"Form" for a form submission (it has a timestamp). Bulk-pasted logs keep the label they were pasted with;
|
||||
they record every run of the players who kept them, so they are the least selectively reported rows.
|
||||
Where the run came from. "reddit spreadsheet": the community Threshing Routes form and spreadsheet linked from
|
||||
the r/fourthwing megathreads. "reddit thread": a dragonkind.com result page posted in a megathread (bonded,
|
||||
route not recorded). "reported": the Atlas's own "Report a run" form. "bot": an attempt logged by the
|
||||
DragonKind bot. "reddit pdf": the community route PDF (a flowchart of the routes, not a log of runs, so it
|
||||
adds no rows today).
|
||||
values: [reddit spreadsheet, reddit thread, reported, bot, reddit pdf]
|
||||
entry:
|
||||
type: text
|
||||
description: >-
|
||||
For spreadsheet runs, the sheet's own row label: "Form" for a form submission (it has a timestamp);
|
||||
bulk-pasted logs keep the label they were pasted with. Bulk logs record every run of the players who kept
|
||||
them, so they are the least selectively reported rows. Empty for other sources.
|
||||
values: [Form, Data Import, BulkUpload3, Manual Upload, Historical import, Unknown import]
|
||||
date: { type: date, description: Date of a form submission (YYYY-MM-DD). Empty for bulk-pasted rows. }
|
||||
date: { type: date, description: "Date of the run or its report (YYYY-MM-DD): the form submission, the issue, or the bot attempt. Empty for bulk-pasted rows and result pages." }
|
||||
wing: { type: text, description: The rider's wing, from the form., values: [First, Second, Third, Fourth, Multiple, Unknown] }
|
||||
section: { type: text, description: The rider's section within the wing (free text as answered). }
|
||||
squad: { type: text, description: The rider's squad., values: [First, Second, Third, "Don't know", Unknown] }
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"reported": {
|
||||
"sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945",
|
||||
"bytes": 2,
|
||||
"changed": "2026-11-04"
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
{"source":"Runs reported on the Atlas (\"Report a run\", a GitHub issue form)","reports":[]}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"form": {
|
||||
"name": "Threshing Routes form responses (u/ClumsyBadger)",
|
||||
"view": "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0",
|
||||
"download": "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0/export?format=xlsx",
|
||||
"filename": "Threshing_Routes_Responses.xlsx"
|
||||
},
|
||||
"register": {
|
||||
"name": "Unofficial Dragon Register responses",
|
||||
"view": "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ",
|
||||
"download": "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ/export?format=xlsx",
|
||||
"filename": "Unofficial_Dragon_Register_Responses.xlsx"
|
||||
},
|
||||
"pdf": {
|
||||
"name": "Community route PDF (u/ClumsyBadger)",
|
||||
"view": "https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view",
|
||||
"download": "https://drive.google.com/uc?export=download&id=17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg",
|
||||
"filename": "Threshing_Options.pdf"
|
||||
}
|
||||
}
|
||||
@@ -125,7 +125,7 @@ rescue branch (the wounded cadet) to be tried in both directions in phase 1.
|
||||
|
||||
## 4. Getting data to the atlas
|
||||
|
||||
- The atlas repo already has the exporter: `npm run import-bot -- --bot /path/to/DragonKind`. It reads
|
||||
- The atlas repo already has the exporter: `DRAGONKIND_BOT_DIR=/path/to/DragonKind python -m collection update --source bot`. It reads
|
||||
`data/knowledge.json`, `data/runs.jsonl` and `logs/encounters.jsonl` from the bot folder and writes
|
||||
`data/bot/scenes.json` and `data/bot/trajectories-<date>.jsonl`, with:
|
||||
- prose removed (scene fingerprints, short labels, choice text and outcomes only). The importer
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
Data flows both ways:
|
||||
|
||||
```
|
||||
community form, register, PDF ──┐
|
||||
community sources, reports ─────┐
|
||||
├─► threshing-atlas ──► policy.json + solver.js ──► bot (advice per screen)
|
||||
bot runs (anonymised) ──────────┘ ▲ │
|
||||
└──────────── import-bot + pull request ◄──┘
|
||||
└──────────── collection --source bot + PR ◄┘
|
||||
```
|
||||
|
||||
## 1. From the atlas to the bot
|
||||
@@ -129,7 +129,7 @@ Current headline (from `analysis/reports/POLICY.md`): Black about 1.7% per attem
|
||||
|
||||
## 2. From the bot to the atlas
|
||||
|
||||
Already in place: `npm run import-bot -- --bot /path/to/DragonKind` in the atlas repo converts the bot's
|
||||
Already in place: `DRAGONKIND_BOT_DIR=/path/to/DragonKind python -m collection update --source bot` in the atlas repo converts the bot's
|
||||
`knowledge.json`, `runs.jsonl` and `logs/encounters.jsonl` into anonymised files under `data/bot/`, and refuses to
|
||||
write prose or account names. Commit them by pull request; CI rebuilds the graph and checks everything.
|
||||
|
||||
@@ -147,4 +147,4 @@ What would make the bot's data far more useful (details in `docs/BOT_DATA_PROTOC
|
||||
- The model learns how much history matters from the data. Today it finds that the meadow colour does not depend on
|
||||
earlier choices at all, and the shadow only weakly. The bot's randomised runs are what can confirm or overturn that.
|
||||
- The policy is only as good as the crosswalk between the bot's screen text and the form's variables. If the game
|
||||
changes its wording, `OPTION_PATTERNS` in `solver.js` and `CROSSWALK` in `graph.js` need updating.
|
||||
changes its wording, `OPTION_PATTERNS` in `solver.js` and the crosswalk in `collection/specs/bot.yaml` (written into `graph.js` by `python -m collection build`) need updating.
|
||||
|
||||
+14
-4
@@ -13,7 +13,7 @@ Rules every part of the project follows. If a change would break one, stop and a
|
||||
- **The community route PDF:** never committed or redistributed. Only a fingerprint
|
||||
(`data/derived/pdf_fingerprint.json`) so we can tell when it changes, and a link to the original.
|
||||
- **Bot data:** only scene fingerprints, short labels, choice text and outcomes. Never game prose, never account
|
||||
names. The importer (`collection/bot/bot.js`) refuses to write anything that looks like either.
|
||||
names. The importer (`collection/etl/transform/bot.py`, rules in `collection/pipeline.yaml`) refuses to write anything that looks like either.
|
||||
|
||||
## The bot
|
||||
|
||||
@@ -25,10 +25,20 @@ Rules every part of the project follows. If a change would break one, stop and a
|
||||
## Sources we do not scrape
|
||||
|
||||
- **Discord:** not without the server admins' consent; never with a personal account token.
|
||||
- **Reddit:** not collected yet. When it is, it should go through Reddit's official API with an app key, not page
|
||||
scraping.
|
||||
- **Reddit:** only through Reddit's official API with an app key (`REDDIT_CLIENT_ID` / `REDDIT_CLIENT_SECRET`),
|
||||
never by page scraping. Only dragonkind.com result links are taken from the threads; usernames and comment text
|
||||
are never kept, and each result page is read and added by hand (`collection/manual/reddit_results.yaml`).
|
||||
|
||||
## Run reports
|
||||
|
||||
"Report a run" creates a public GitHub issue. The pipeline keeps only the issue number, the date and the answers
|
||||
(`data/public/reports.json`); GitHub usernames are dropped. The form asks people not to include account names. A
|
||||
report closed as "not planned" or labelled `invalid` or `duplicate` is left out of the data.
|
||||
|
||||
## Secrets
|
||||
|
||||
- `ATLAS_BOT_TOKEN` (repository secret) lets the daily data update open and auto-merge its pull request
|
||||
(`docs/REPO_SETTINGS.md`). Tokens go in secrets only: never in a file, issue, chat or commit.
|
||||
(`docs/REPO_SETTINGS.md`).
|
||||
- `RAW_ARCHIVE_KEY` encrypts the raw store (the current and previous raw file of each source, which include the
|
||||
register's rider names) in the Actions cache between daily runs. Without it nothing is archived.
|
||||
- `REDDIT_CLIENT_ID` / `REDDIT_CLIENT_SECRET` let the daily update read the megathreads through the Reddit API. Tokens go in secrets only: never in a file, issue, chat or commit.
|
||||
|
||||
@@ -59,3 +59,15 @@ merge.
|
||||
If Ken's bot should also open pull requests here (for example to send new bot runs), create a **second** token
|
||||
with the same settings and store it as a secret in the bot's own repository. Separate tokens can be revoked
|
||||
separately, and a leak in one place does not expose the other.
|
||||
|
||||
## Data collection secrets (optional)
|
||||
|
||||
Both are repository secrets (Settings → Secrets and variables → Actions → New repository secret). Without them the
|
||||
daily update still runs and says in its report what it skipped.
|
||||
|
||||
| Secret | What it is for | How to make it |
|
||||
|---|---|---|
|
||||
| `RAW_ARCHIVE_KEY` | Encrypts the private raw store (current and previous raw file of each source) kept in the Actions cache between runs. | Any long random string, e.g. the output of `openssl rand -base64 32`. Keep a copy somewhere safe: without it the archive cannot be read. |
|
||||
| `REDDIT_CLIENT_ID`, `REDDIT_CLIENT_SECRET` | Reading the r/fourthwing megathreads through the official Reddit API. | reddit.com/prefs/apps → "create another app…" → type **script**, any name, redirect uri `http://localhost`. The id is under the app name; the secret is labelled "secret". |
|
||||
|
||||
The "Report a run" form adds the label `run report` to each report; the label must exist (Issues → Labels).
|
||||
|
||||
+13
-1
@@ -39,6 +39,18 @@ share.
|
||||
small JS advisor (`core/solver.js`, about 250 lines) reads the game and the Python-built model from `policy.json`
|
||||
and replaces the 900-line JS solver; `engine.js` is gone.
|
||||
5. **Done.** `data/dictionary.yaml` describes every published file and every column of the coded runs (type, meaning,
|
||||
allowed values); `collection/publish_metadata.py` writes `data/croissant.json` (Croissant 1.0) from it, with a
|
||||
allowed values); `collection/etl/metadata.py` writes `data/croissant.json` (Croissant 1.0) from it, with a
|
||||
sha256 for every file. `tests/core/test_dictionary.py` fails when a column or value is undescribed or the metadata
|
||||
is stale. It also caught `attempt` being written as `1.0`, `2.0`: now a whole number.
|
||||
|
||||
## Data collection review (November 2026)
|
||||
|
||||
Collection is an extract → transform → load pipeline declared in `collection/pipeline.yaml` (`python -m collection`).
|
||||
`data/public/runs_clean.csv` holds every run from every source with a `source` column (reddit spreadsheet, reddit
|
||||
thread, reported, bot, reddit pdf). The bot importer is Python (`collection/etl/transform/bot.py`, identical output to
|
||||
the old `bot.js`), and its crosswalk is `collection/specs/bot.yaml`, written into `core/graph.js` for the bot.
|
||||
Reddit goes through the official API; "Report a run" is a GitHub issue form generated from
|
||||
`collection/specs/report_form.yaml`. Only the previous raw version is kept, encrypted, in the Actions cache.
|
||||
|
||||
Still to do: decide which non-spreadsheet sources join the statistics and the route model (`analysis/stats/measures.yaml`
|
||||
`sources`); the bot follows a strategy, so its runs need weighting rather than pooling.
|
||||
|
||||
+3
-4
@@ -6,15 +6,14 @@
|
||||
"scripts": {
|
||||
"build": "node site/build.js",
|
||||
"dev": "node site/build.js && python3 -m http.server 8000 -d dist",
|
||||
"test:py": "PYTHONPATH=core python3 -m atlas.coding && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py'",
|
||||
"test:py": "python3 -m collection build --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'",
|
||||
"test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js",
|
||||
"test": "npm run test:py && npm run test:unit",
|
||||
"graph": "node analysis/network/build-graph.js",
|
||||
"import-bot": "node collection/bot/import-bot.js",
|
||||
"check-sources": "python3 collection/sources/update_sources.py",
|
||||
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py",
|
||||
"stats": "python3 analysis/stats/site_stats.py",
|
||||
"metadata": "python3 collection/publish_metadata.py"
|
||||
"collect": "python3 -m collection update",
|
||||
"build-data": "python3 -m collection build"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
|
||||
+12
-11
@@ -15,7 +15,14 @@ from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from atlas import coding, model, solver # noqa: E402
|
||||
from atlas import model, solver # noqa: E402
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
|
||||
import yaml # noqa: E402
|
||||
|
||||
# the same runs the site's statistics use (analysis/stats/measures.yaml `sources`)
|
||||
with open(ROOT / "analysis/stats/measures.yaml", encoding="utf-8") as _f:
|
||||
SOURCES = yaml.safe_load(_f)["sources"]
|
||||
|
||||
|
||||
def clean(o):
|
||||
@@ -31,24 +38,18 @@ def clean(o):
|
||||
return o
|
||||
|
||||
|
||||
def coded_runs(path):
|
||||
df, _ = coding.load(path)
|
||||
runs = df.drop(columns=["run_id"]).astype(object).where(df.drop(columns=["run_id"]).notna(), None).to_dict("records")
|
||||
return runs
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--draws", type=int, default=200)
|
||||
ap.add_argument("--data", default=str(ROOT / "data/public/form_responses.json"))
|
||||
ap.add_argument("--data", default=str(runs_io.PATH))
|
||||
ap.add_argument("--out", default=str(ROOT / "data/public/policy.json"))
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
rows = len(json.load(open(a.data, encoding="utf-8"))["rows"])
|
||||
runs = coded_runs(a.data)
|
||||
runs = runs_io.read(a.data, sources=SOURCES)
|
||||
rows = len(runs)
|
||||
samples = {
|
||||
"all": runs,
|
||||
"lowBias": [r for r in runs if r["source"] != "Form" or r["goal"] not in model.RARE_GOALS],
|
||||
"lowBias": [r for r in runs if r["entry"] != "Form" or r["goal"] not in model.RARE_GOALS],
|
||||
}
|
||||
t0 = time.time()
|
||||
out = {"schema": 1, "source": {"form_rows": rows}, "samples": {}}
|
||||
|
||||
+1
-1
@@ -187,7 +187,7 @@
|
||||
}
|
||||
const FACTOR_LABEL = { company: "Company", direction: "Direction", first_sighting: "First sighting", scream: "Scream choice", resolution: "Cadet encounter",
|
||||
combat_first: "First combat move", post_path: "Fork in the road", hour: "After an hour", wing: "Wing", squad: "Squad",
|
||||
attempt_group: "Attempt", source: "Data source", goal: "Chasing", attempt: "Attempt", period: "Date",
|
||||
attempt_group: "Attempt", entry: "Data source", source: "Data source", goal: "Chasing", attempt: "Attempt", period: "Date",
|
||||
sighting_action: "Reaction to the first dragon", combat_second: "Second combat move", malek_intent: "Planned to seek Malek", };
|
||||
const labelFactor = (f) => FACTOR_LABEL[f] || f;
|
||||
|
||||
|
||||
+31
-2
@@ -64,7 +64,7 @@ button, .filebtn, .btnlike { cursor: pointer; }
|
||||
.btnlike { display: inline-block; text-decoration: none; }
|
||||
button:hover, .filebtn:hover { border-color: var(--accent); }
|
||||
button:focus-visible, select:focus-visible, input:focus-visible, .filebtn:focus-within, a:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
||||
button.primary { background: var(--accent); color: var(--accent-ink); border-color: var(--accent); font-weight: 600; }
|
||||
button.primary, a.btnlike.primary { background: var(--accent); color: var(--accent-ink); border-color: var(--accent); font-weight: 600; }
|
||||
.seg { display: inline-flex; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; }
|
||||
.seg button { border: 0; border-radius: 0; border-right: 1px solid var(--line); }
|
||||
.seg button:last-child { border-right: 0; }
|
||||
@@ -436,6 +436,7 @@ a { color: var(--accent); text-underline-offset: 2px; }
|
||||
<button class="tab" role="tab" id="tab-dragons" aria-controls="panel-dragons" aria-selected="false">The Dragons</button>
|
||||
<button class="tab" role="tab" id="tab-deep" aria-controls="panel-deep" aria-selected="false">Deep Dive</button>
|
||||
<button class="tab" role="tab" id="tab-data" aria-controls="panel-data" aria-selected="false">Data & Methods</button>
|
||||
<button class="tab" role="tab" id="tab-report" aria-controls="panel-report" aria-selected="false">Report a run</button>
|
||||
</nav>
|
||||
</div>
|
||||
</header>
|
||||
@@ -603,7 +604,7 @@ a { color: var(--accent); text-underline-offset: 2px; }
|
||||
<section id="data-sec" aria-labelledby="h-data">
|
||||
<div class="section-head"><span class="eyebrow">Where the numbers come from</span><h2 id="h-data">Data</h2></div>
|
||||
<div class="data-status" id="data-status"><span class="loading">Loading data…</span></div>
|
||||
<p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>. Every column is described in <a href="data/dictionary.yaml">data/dictionary.yaml</a>, and <a href="data/croissant.json">data/croissant.json</a> carries the same description in the Croissant metadata format, with a checksum for every file.</p>
|
||||
<p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>: every run from every source (the community spreadsheet, runs reported here, result pages posted in the Reddit megathreads and the DragonKind bot), with a <code>source</code> column. The statistics on this page use the community spreadsheet's runs; the other sources are coded from different questions and join the statistics once they have been reviewed. Every column is described in <a href="data/dictionary.yaml">data/dictionary.yaml</a>, and <a href="data/croissant.json">data/croissant.json</a> carries the same description in the Croissant metadata format, with a checksum for every file.</p>
|
||||
</section>
|
||||
<section id="share" aria-labelledby="h-share">
|
||||
<div class="section-head"><span class="eyebrow">Open by design</span><h2 id="h-share">Share & reproduce</h2>
|
||||
@@ -684,6 +685,34 @@ adjusted OR = exp(βⱼ), 95% CI = exp(βⱼ ± 1.96·SEⱼ)</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<div class="tabpanel" id="panel-report" role="tabpanel" aria-labelledby="tab-report" hidden>
|
||||
<!-- the links here are the sources in collection/pipeline.yaml -->
|
||||
<section id="report-sec" aria-labelledby="h-report">
|
||||
<div class="section-head"><span class="eyebrow">Every run helps</span><h2 id="h-report">Report a run</h2>
|
||||
<p class="muted">Every run anyone logs ends up in the Atlas's open data, <a href="data/runs_clean.csv" download>runs_clean.csv</a>, which anyone can download and reuse. Deaths are as useful as bonds: the odds on this page are only as good as the runs behind them. Please report each run in one place only.</p></div>
|
||||
<div class="grid2">
|
||||
<div class="panel"><h3>Report it on the Atlas</h3>
|
||||
<p class="small">A short form with the game's own choices (a GitHub issue, so you need a free GitHub account). Reports show up in the data the next day with the source <b>reported</b>. Reports are public: please leave out your account name.</p>
|
||||
<p><a class="btnlike primary" href="https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" target="_blank" rel="noopener">Report a run</a></p>
|
||||
</div>
|
||||
<div class="panel"><h3>Or use the community form</h3>
|
||||
<p class="small">The r/fourthwing megathreads collect runs with u/ClumsyBadger's Google Form, and its answers go into a public spreadsheet. The Atlas reads that spreadsheet every day (source <b>reddit spreadsheet</b>), so a run logged there is counted here too.</p>
|
||||
<p class="btnrow"><a class="btnlike" href="https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0" target="_blank" rel="noopener">The spreadsheet</a>
|
||||
<a class="btnlike" href="https://www.reddit.com/r/fourthwing/comments/1wz5cnj/" target="_blank" rel="noopener">The megathread (with the form)</a></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel"><h3>Where the Atlas's runs come from</h3>
|
||||
<table class="small"><thead><tr><th scope="col">Source</th><th scope="col">What it is</th></tr></thead><tbody>
|
||||
<tr><td>reddit spreadsheet</td><td>The community form and spreadsheet linked from the r/fourthwing megathreads.</td></tr>
|
||||
<tr><td>reported</td><td>Runs reported here, with "Report a run".</td></tr>
|
||||
<tr><td>reddit thread</td><td>Dragonkind result pages posted in the megathreads (the dragon is known, the route is not).</td></tr>
|
||||
<tr><td>bot</td><td>Runs logged by the DragonKind bot.</td></tr>
|
||||
<tr><td>reddit pdf</td><td>The community route PDF: a map of the choices, used to check the game's routes (it holds no runs).</td></tr>
|
||||
</tbody></table>
|
||||
<p class="small muted">What every column means: <a href="data/dictionary.yaml">data/dictionary.yaml</a>.</p>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer><div class="wrap"><p><b>Data:</b> the community <a href="https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0" target="_blank" rel="noopener">"Threshing Routes" responses</a> compiled by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, with percentages in the <a href="https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view" target="_blank" rel="noopener">route PDF</a> by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, <a href="https://www.reddit.com/user/Old_Regular3018/" target="_blank" rel="noopener">u/Old_Regular3018</a> and <a href="https://www.reddit.com/user/Smart_Trifle_2099/" target="_blank" rel="noopener">u/Smart_Trifle_2099</a>; the <a href="https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ" target="_blank" rel="noopener">Unofficial Dragon Register</a>; and the r/fourthwing bonding megathreads (<a href="https://www.reddit.com/r/fourthwing/comments/1wvwfp9/" target="_blank" rel="noopener">Part 3</a>, <a href="https://www.reddit.com/r/fourthwing/comments/1wz5cnj/" target="_blank" rel="noopener">Part 4</a>). Code and data: <a href="https://github.com/ekcSchiano/threshing-atlas" target="_blank" rel="noopener">github.com/ekcSchiano/threshing-atlas</a> (MIT, data CC BY 4.0).</p>
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@
|
||||
function rng(seed) { let s = seed >>> 0; return () => ((s = (s * 1664525 + 1013904223) >>> 0) / 4294967296); }
|
||||
|
||||
// ------------------------------------------------------------ tabs
|
||||
const TABS = ["valley", "odds", "dragons", "deep", "data"];
|
||||
const TABS = ["valley", "odds", "dragons", "deep", "data", "report"];
|
||||
// old links (#overview, #flow, #planner ...) still land somewhere sensible
|
||||
const ALIAS = { overview: "odds", planner: "valley", flow: "deep", rare: "deep", share: "data" };
|
||||
const tabOf = (hash) => { const t = (hash || "").replace(/^#/, "").split("/")[0]; return TABS.includes(t) ? t : ALIAS[t] || null; };
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ copy("core/graph.js", "graph.js");
|
||||
copy("core/solver.js", "solver.js");
|
||||
// The numbers the page shows (all computed in Python) and the published data
|
||||
for (const f of ["stats.json", "register.json", "graph.json", "policy.json"]) copy(`data/public/${f}`, `data/${f}`);
|
||||
copy("data/derived/runs_clean.csv", "data/runs_clean.csv");
|
||||
copy("data/public/runs_clean.csv", "data/runs_clean.csv");
|
||||
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
|
||||
// Source files shown in the page's "view the code" panel
|
||||
copy("core/atlas/coding.py", "python/coding.py");
|
||||
|
||||
+2
-2
@@ -12,7 +12,7 @@ const BASE = "http://localhost:8765/";
|
||||
const p = await b.newPage({ viewport: vp });
|
||||
p.on("pageerror", (e) => log.errors.push(`${vpName} pageerror: ${e.message}`));
|
||||
p.on("console", (m) => { if (m.type() === "error" && !/net::ERR|Failed to load resource/.test(m.text())) log.errors.push(`${vpName} console: ${m.text()}`); });
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data"]) {
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data", "report"]) {
|
||||
await p.goto(BASE + "#" + t); await p.waitForTimeout(t === "valley" ? 2500 : 1800);
|
||||
// open every <details> so hidden text is reviewed too
|
||||
await p.$$eval(`#panel-${t} details`, (ds) => ds.forEach((d) => (d.open = true)));
|
||||
@@ -34,7 +34,7 @@ const BASE = "http://localhost:8765/";
|
||||
// click every button in every tab (desktop), checking for errors after each
|
||||
const p = await b.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
p.on("pageerror", (e) => log.errors.push(`click pageerror: ${e.message}`));
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data"]) {
|
||||
for (const t of ["valley", "odds", "dragons", "deep", "data", "report"]) {
|
||||
await p.goto(BASE + "#" + t); await p.waitForTimeout(2000);
|
||||
const n = await p.$$eval(`#panel-${t} button`, (bs) => bs.length);
|
||||
let clicked = 0;
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""The collection pipeline: extract's raw store, every transform, the coders, validation and the generated files."""
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from collection import run # noqa: E402
|
||||
from collection.etl import codegen, extract, report_form, runs, validate # noqa: E402
|
||||
from collection.etl.common import read_yaml # noqa: E402
|
||||
from collection.etl.transform import bot, reddit, reports, sheet # noqa: E402
|
||||
|
||||
FX = ROOT / "tests" / "fixtures" / "bot"
|
||||
TMP = Path(tempfile.mkdtemp(prefix="collection-test-"))
|
||||
|
||||
|
||||
def fixture_import(salt="test-salt"):
|
||||
kn = json.loads((FX / "data/knowledge.json").read_text(encoding="utf-8"))
|
||||
enc = bot._jsonl((FX / "logs/encounters.jsonl").read_text(encoding="utf-8"))
|
||||
rn = bot._jsonl((FX / "data/runs.jsonl").read_text(encoding="utf-8"))
|
||||
return bot.import_bot(kn, enc, rn, salt)
|
||||
|
||||
|
||||
class RawStore(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.old = extract.RAW_DIR
|
||||
extract.RAW_DIR = TMP / "raw"
|
||||
shutil.rmtree(extract.RAW_DIR, ignore_errors=True)
|
||||
|
||||
def tearDown(self):
|
||||
extract.RAW_DIR = self.old
|
||||
|
||||
def test_current_and_one_previous_only(self):
|
||||
m = {}
|
||||
self.assertTrue(extract.store("s", "txt", b"one", m, today="2026-11-01"))
|
||||
self.assertFalse(extract.store("s", "txt", b"one", m, today="2026-11-02"))
|
||||
self.assertEqual(m["s"]["changed"], "2026-11-01")
|
||||
self.assertTrue(extract.store("s", "txt", b"two", m, today="2026-11-03"))
|
||||
self.assertTrue(extract.store("s", "txt", b"three", m, today="2026-11-04"))
|
||||
cur, prev = extract.raw_paths("s", "txt")
|
||||
self.assertEqual((cur.read_bytes(), prev.read_bytes()), (b"three", b"two"))
|
||||
self.assertEqual(sorted(p.name for p in cur.parent.iterdir()), ["current.txt", "previous.txt"])
|
||||
self.assertEqual(set(m["s"]), {"sha256", "bytes", "changed"}) # the current version only
|
||||
|
||||
def test_missing_secrets_skip_not_fail(self):
|
||||
with self.assertRaises(extract.Skip):
|
||||
extract.reddit({"secrets": ["A", "B"], "threads": []}, {})
|
||||
with self.assertRaises(extract.Skip):
|
||||
extract.local_dir({"env": "DRAGONKIND_BOT_DIR", "files": []}, {})
|
||||
|
||||
|
||||
class Bot(unittest.TestCase):
|
||||
def test_same_output_as_the_javascript_importer(self):
|
||||
scenes, trajs, unlabeled = fixture_import()
|
||||
expected = json.loads((FX / "expected.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(json.dumps(scenes, ensure_ascii=False), json.dumps(expected["scenes"], ensure_ascii=False))
|
||||
self.assertEqual(json.dumps(trajs, ensure_ascii=False), json.dumps(expected["trajectories"], ensure_ascii=False))
|
||||
self.assertEqual(unlabeled, expected["unlabeled"])
|
||||
|
||||
def test_one_trajectory_per_attempt(self):
|
||||
_, (t1, t2), _ = fixture_import()
|
||||
self.assertEqual(len(t1["steps"]), 3)
|
||||
self.assertEqual(t1["steps"][-1]["to"], "BURNED")
|
||||
self.assertEqual(t1["steps"][0]["to"], t1["steps"][1]["scene"])
|
||||
self.assertEqual(t2["steps"][-1]["to"], "BONDED:green")
|
||||
self.assertTrue(t2["steps"][0]["manual"])
|
||||
|
||||
def test_accounts_hashed_and_no_prose(self):
|
||||
scenes, trajs, _ = fixture_import()
|
||||
self.assertRegex(trajs[0]["meta"]["accountKey"], r"^[0-9a-f]{10}$")
|
||||
text = json.dumps({"s": scenes, "t": trajs})
|
||||
self.assertNotIn("acct-1", text)
|
||||
self.assertNotIn("Synthetic", text)
|
||||
P = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["privacy"]
|
||||
bot.check_public(scenes, set(P["refuse_fields"]), P["max_string"])
|
||||
with self.assertRaises(ValueError):
|
||||
bot.check_public({"prose": "x"}, set(P["refuse_fields"]), 200)
|
||||
with self.assertRaises(ValueError):
|
||||
bot.check_public({"note": "y" * 201}, set(), 200)
|
||||
|
||||
def test_crosswalk_matches_graph_js(self):
|
||||
"""The bot's advisor maps choices with core/graph.js; the importer with bot.yaml. Same answers."""
|
||||
trajs = runs.bot_trajectories(read_yaml("collection/pipeline.yaml")["sources"]["bot"])
|
||||
trajs += json.loads((FX / "expected.json").read_text(encoding="utf-8"))["trajectories"]
|
||||
script = ("const G=require('./core/graph.js');let s='';process.stdin.on('data',d=>s+=d).on('end',()=>"
|
||||
"console.log(JSON.stringify(JSON.parse(s).map(t=>G.trajectoryVariables(t).vars))))")
|
||||
js = json.loads(subprocess.run(["node", "-e", script], input=json.dumps(trajs), capture_output=True,
|
||||
text=True, cwd=ROOT, check=True).stdout)
|
||||
import re
|
||||
cw = [(re.compile(rx), c, v) for rx, c, v in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
|
||||
for t, j in zip(trajs, js):
|
||||
py, _ = runs.bot_variables(t, cw)
|
||||
self.assertEqual(py, {k: v for k, v in j.items() if k not in ("outcome", "outcome_color")})
|
||||
|
||||
def test_every_bot_choice_is_mapped(self):
|
||||
"""Every choice the bot has seen maps onto a column, except answers inside a bonding encounter."""
|
||||
import re
|
||||
cw = [re.compile(rx) for rx, _, _ in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
|
||||
scenes = json.loads((ROOT / "data/bot/scenes.json").read_text(encoding="utf-8"))
|
||||
unmapped = [f"{s['label']}: {o['text']}" for s in scenes.values()
|
||||
if len(s["options"]) > 1 and "two Greens in the clearing" not in s["label"]
|
||||
for o in s["options"] if not any(rx.search(bot.normalize(o["text"])) for rx in cw)]
|
||||
self.assertEqual(unmapped, [])
|
||||
|
||||
def test_graph_js_crosswalk_is_generated(self):
|
||||
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
class Sheets(unittest.TestCase):
|
||||
def test_privacy_rules(self):
|
||||
p = TMP / "sheet.csv"
|
||||
p.write_text("Timestamp,Cadet name,Photo upload,Notes\n2026-10-01,Violet,http://x,mail me at a@b.co\n", encoding="utf-8")
|
||||
spec = {"label": "t", "transform": {"name": "sheet"},
|
||||
"privacy": {"redact_emails": True, "drop_columns_matching": ["photo", "cadet name"]}}
|
||||
out = sheet.transform(p, spec, {"changed": "2026-11-01"})
|
||||
self.assertEqual(out["headers"], ["Timestamp", "Notes"])
|
||||
self.assertEqual(out["rows"], [{"Timestamp": "2026-10-01", "Notes": "mail me at [email removed]"}])
|
||||
self.assertEqual(out["exported"], "2026-11-01")
|
||||
|
||||
|
||||
class Reddit(unittest.TestCase):
|
||||
def test_new_result_links_only_and_no_usernames(self):
|
||||
known = read_yaml("collection/manual/reddit_results.yaml")["results"][0]["id"]
|
||||
tree = [{"kind": "Listing", "data": {"children": [
|
||||
{"kind": "t1", "data": {"author": "someone", "permalink": "/r/fourthwing/comments/x/y/c1/",
|
||||
"body": f"got mine! https://dragonkind.com/results/{known} and https://dragonkind.com/results/THRESH-NEW123",
|
||||
"replies": {"kind": "Listing", "data": {"children": [
|
||||
{"kind": "t1", "data": {"author": "other", "permalink": "/c2/", "body": "dragonkind.com/results/thresh-new456"}}]}}}}]}}]
|
||||
p = TMP / "reddit.json"
|
||||
p.write_text(json.dumps({"x": tree}), encoding="utf-8")
|
||||
out = reddit.transform(p, {"transform": {"manual": "collection/manual/reddit_results.yaml"}}, {})
|
||||
self.assertEqual([r["id"] for r in out["new_results"]], ["THRESH-NEW123", "THRESH-NEW456"])
|
||||
self.assertNotIn("someone", json.dumps(out))
|
||||
|
||||
|
||||
class Reports(unittest.TestCase):
|
||||
def issue(self, number, answers, **kw):
|
||||
form = report_form.spec()
|
||||
by_id = {f["id"]: f for f in form["fields"]}
|
||||
body = "\n\n".join(f"### {f['label']}\n\n{answers.get(f['id'], '_No response_')}" for f in by_id.values())
|
||||
return {"number": number, "body": body, "created_at": "2026-11-05T10:00:00Z", "user": {"login": "rider"},
|
||||
"labels": [{"name": "run report"}], "state": "open", **kw}
|
||||
|
||||
def test_issue_form_is_generated(self):
|
||||
self.assertEqual(report_form.render(), (ROOT / report_form.TEMPLATE).read_text(encoding="utf-8"))
|
||||
|
||||
def test_every_option_is_a_dictionary_value(self):
|
||||
cols = read_yaml("data/dictionary.yaml")["columns"]
|
||||
for f in report_form.spec()["fields"]:
|
||||
self.assertIn(f["column"], cols)
|
||||
for v in (f.get("options") or {}).values():
|
||||
if v is not None and "values" in cols[f["column"]]:
|
||||
self.assertIn(str(v), cols[f["column"]]["values"], f["id"])
|
||||
|
||||
def test_reports_become_valid_runs(self):
|
||||
good = self.issue(7, {"company": "Alone", "direction": "Toward the forest centre", "first_sighting": "Red",
|
||||
"scream": "Investigated, and helped my squadmate", "branch": "Valley: passed the Red, the meadow",
|
||||
"outcome": "Bonded", "outcome_color": "Blue", "tail": "Clubtail", "attempt": "3rd"})
|
||||
spam = self.issue(8, {"company": "Alone"}, state="closed", state_reason="not_planned")
|
||||
dup = self.issue(9, {"company": "Alone"}, labels=[{"name": "run report"}, {"name": "duplicate"}])
|
||||
raw = TMP / "issues.json"
|
||||
raw.write_text(json.dumps([good, spam, dup]), encoding="utf-8")
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reported"]
|
||||
out = reports.transform(raw, spec, {})
|
||||
self.assertEqual([r["number"] for r in out["reports"]], [7])
|
||||
self.assertNotIn("rider", json.dumps(out))
|
||||
target = TMP / "reports.json"
|
||||
target.write_text(json.dumps(out), encoding="utf-8")
|
||||
rows = runs.reports("reported", {**spec, "load": str(target)})
|
||||
r = rows[0]
|
||||
self.assertEqual((r["run_id"], r["source"], r["date"]), ("reported-7", "reported", "2026-11-05"))
|
||||
self.assertEqual((r["company"], r["direction"], r["hour"], r["attempt"]), ("Alone", "Center", "Valley", 3))
|
||||
self.assertEqual((r["outcome"], r["outcome_color"], r["enc_Blue"], r["saw_blue_meadow"]), ("Bonded", "Blue", "Bonded!", True))
|
||||
self.assertEqual(validate.problems(rows, runs.DICT), [])
|
||||
|
||||
|
||||
class Build(unittest.TestCase):
|
||||
def test_published_files_are_up_to_date(self):
|
||||
self.assertEqual(run.build(check=True), [], "run `python -m collection build`")
|
||||
|
||||
def test_every_source_value_is_described(self):
|
||||
pipe = read_yaml("collection/pipeline.yaml")
|
||||
described = runs.DICT["columns"]["source"]["values"]
|
||||
for name, spec in pipe["sources"].items():
|
||||
if spec.get("run_source"):
|
||||
self.assertIn(spec["run_source"], described, name)
|
||||
|
||||
def test_validation_catches_unknown_values(self):
|
||||
row = runs.blank_row(run_id="x", source="reported", company="Five squadmates")
|
||||
self.assertTrue(any("company" in p for p in validate.problems([row], runs.DICT)))
|
||||
self.assertTrue(any("unique" in p for p in validate.problems([row, row], runs.DICT)))
|
||||
|
||||
def test_spreadsheet_rows_keep_their_entry_label(self):
|
||||
with open(ROOT / "data/public/runs_clean.csv", encoding="utf-8", newline="") as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
sheet_rows = [r for r in rows if r["source"] == "reddit spreadsheet"]
|
||||
self.assertTrue(sheet_rows and all(r["entry"] and r["run_id"].startswith("sheet-") for r in sheet_rows))
|
||||
self.assertTrue(all(not r["entry"] for r in rows if r["source"] != "reddit spreadsheet"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,39 +2,11 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert");
|
||||
const path = require("path");
|
||||
const { importBot, assertPublicSafe } = require("../../collection/bot/bot");
|
||||
const G = require("../../core/graph");
|
||||
|
||||
const fx = path.join(__dirname, "../fixtures/bot");
|
||||
const { scenes, trajectories } = importBot({ dataDir: path.join(fx, "data"), logsDir: path.join(fx, "logs"), salt: "test-salt" });
|
||||
|
||||
test("one trajectory per attempt, duplicates and repeated burns collapsed", () => {
|
||||
assert.strictEqual(trajectories.length, 2);
|
||||
const [t1, t2] = trajectories;
|
||||
assert.strictEqual(t1.steps.length, 3);
|
||||
assert.deepStrictEqual(t1.steps.map((s) => s.to).slice(-1), ["BURNED"]);
|
||||
assert.strictEqual(t1.steps[0].to, t1.steps[1].scene);
|
||||
assert.strictEqual(t2.outcome.type, "BONDED");
|
||||
assert.strictEqual(t2.steps.at(-1).to, "BONDED:green");
|
||||
assert.strictEqual(t2.steps[0].manual, true, "manual picks are flagged");
|
||||
});
|
||||
|
||||
test("run metadata attached, account id hashed with the salt", () => {
|
||||
const t1 = trajectories[0];
|
||||
assert.strictEqual(t1.meta.target, "black");
|
||||
assert.strictEqual(t1.meta.decisionMode, "explore");
|
||||
assert.strictEqual(t1.outcome.intentional, true);
|
||||
assert.match(t1.meta.accountKey, /^[0-9a-f]{10}$/);
|
||||
assert.ok(!JSON.stringify(trajectories).includes("acct-1"));
|
||||
});
|
||||
|
||||
test("public export contains no prose", () => {
|
||||
assertPublicSafe(scenes);
|
||||
assertPublicSafe(trajectories);
|
||||
assert.ok(!JSON.stringify({ scenes, trajectories }).includes("Synthetic"));
|
||||
assert.throws(() => assertPublicSafe({ prose: "x" }));
|
||||
assert.throws(() => assertPublicSafe({ note: "y".repeat(201) }));
|
||||
});
|
||||
// the bot importer's output on tests/fixtures/bot (the importer is Python now: tests/collection/test_bot.py checks
|
||||
// that it still produces exactly this file)
|
||||
const { scenes, trajectories } = require(path.join(__dirname, "../fixtures/bot/expected.json"));
|
||||
|
||||
test("crosswalk maps bot choices to form variables", () => {
|
||||
const { vars } = G.trajectoryVariables(trajectories[0]);
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
"""The data dictionary describes every published column and value, and data/croissant.json matches it."""
|
||||
import csv
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
spec = importlib.util.spec_from_file_location("publish_metadata", ROOT / "collection" / "publish_metadata.py")
|
||||
publish_metadata = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(publish_metadata)
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from collection.etl import metadata as publish_metadata # noqa: E402
|
||||
|
||||
with open(ROOT / "data" / "dictionary.yaml", encoding="utf-8") as f:
|
||||
DICT = yaml.safe_load(f)
|
||||
with open(ROOT / "data" / "derived" / "runs_clean.csv", encoding="utf-8", newline="") as f:
|
||||
with open(ROOT / "data" / "public" / "runs_clean.csv", encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
HEADER = reader.fieldnames
|
||||
ROWS = list(reader)
|
||||
@@ -61,7 +60,7 @@ class Croissant(unittest.TestCase):
|
||||
def test_croissant_is_up_to_date(self):
|
||||
expected = publish_metadata.render(publish_metadata.build(DICT))
|
||||
actual = (ROOT / "data" / "croissant.json").read_text(encoding="utf-8")
|
||||
self.assertEqual(actual, expected, "run python3 collection/publish_metadata.py")
|
||||
self.assertEqual(actual, expected, "run python -m collection build")
|
||||
|
||||
def test_fields_and_checksums(self):
|
||||
doc = json.loads((ROOT / "data" / "croissant.json").read_text(encoding="utf-8"))
|
||||
|
||||
Vendored
+178
@@ -0,0 +1,178 @@
|
||||
{
|
||||
"scenes": {
|
||||
"b0fa671066ec": {
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"backdrop": null,
|
||||
"label": "TEST",
|
||||
"options": [
|
||||
{
|
||||
"id": "c18f80dd1e",
|
||||
"text": "Enter the valley alone"
|
||||
},
|
||||
{
|
||||
"id": "3da794f136",
|
||||
"text": "Enter the valley with one of your squadmates"
|
||||
}
|
||||
]
|
||||
},
|
||||
"e0b5530601be": {
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"backdrop": null,
|
||||
"label": "TEST",
|
||||
"options": [
|
||||
{
|
||||
"id": "5f7eefac34",
|
||||
"text": "Head east toward the roar"
|
||||
},
|
||||
{
|
||||
"id": "05d19abb5d",
|
||||
"text": "Head toward the center of the forest"
|
||||
}
|
||||
]
|
||||
},
|
||||
"1c86a35c091f": {
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"backdrop": null,
|
||||
"label": "TEST",
|
||||
"options": [
|
||||
{
|
||||
"id": "a31cbcb659",
|
||||
"text": "Lower your eyes in respect"
|
||||
},
|
||||
{
|
||||
"id": "1859b3cd4a",
|
||||
"text": "Hide behind a tree"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"trajectories": [
|
||||
{
|
||||
"id": "bot-9176f0ecbe",
|
||||
"source": "bot",
|
||||
"meta": {
|
||||
"attempt": 1,
|
||||
"startedAt": "2026-01-01T00:00:00.000Z",
|
||||
"accountKey": "7bd9b15fe8",
|
||||
"target": "black",
|
||||
"decisionMode": "explore",
|
||||
"policyId": null,
|
||||
"experimentId": null,
|
||||
"arm": null
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"scene": "b0fa671066ec",
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"option": "c18f80dd1e",
|
||||
"optionText": "Enter the valley alone",
|
||||
"offered": [
|
||||
"c18f80dd1e",
|
||||
"3da794f136"
|
||||
],
|
||||
"rule": "R",
|
||||
"decisionMode": "guide",
|
||||
"manual": false,
|
||||
"at": "2026-01-01T00:00:00.000Z",
|
||||
"to": "e0b5530601be"
|
||||
},
|
||||
{
|
||||
"scene": "e0b5530601be",
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"option": "05d19abb5d",
|
||||
"optionText": "Head toward the center of the forest",
|
||||
"offered": [
|
||||
"5f7eefac34",
|
||||
"05d19abb5d"
|
||||
],
|
||||
"rule": "R",
|
||||
"decisionMode": "guide",
|
||||
"manual": false,
|
||||
"at": "2026-01-01T00:00:02.000Z",
|
||||
"to": "1c86a35c091f"
|
||||
},
|
||||
{
|
||||
"scene": "1c86a35c091f",
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"option": "1859b3cd4a",
|
||||
"optionText": "Hide behind a tree",
|
||||
"offered": [
|
||||
"a31cbcb659",
|
||||
"1859b3cd4a"
|
||||
],
|
||||
"rule": "R",
|
||||
"decisionMode": "guide",
|
||||
"manual": false,
|
||||
"at": "2026-01-01T00:00:03.000Z",
|
||||
"to": "BURNED"
|
||||
}
|
||||
],
|
||||
"outcome": {
|
||||
"type": "BURNED",
|
||||
"colour": null,
|
||||
"intentional": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "bot-b34ab7e2df",
|
||||
"source": "bot",
|
||||
"meta": {
|
||||
"attempt": 2,
|
||||
"startedAt": "2026-01-01T00:00:07.000Z",
|
||||
"accountKey": null,
|
||||
"target": null,
|
||||
"decisionMode": null
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"scene": "b0fa671066ec",
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"option": "3da794f136",
|
||||
"optionText": "Enter the valley with one of your squadmates",
|
||||
"offered": [
|
||||
"c18f80dd1e",
|
||||
"3da794f136"
|
||||
],
|
||||
"rule": null,
|
||||
"decisionMode": "manual",
|
||||
"manual": true,
|
||||
"at": "2026-01-01T00:00:07.000Z",
|
||||
"to": "e0b5530601be"
|
||||
},
|
||||
{
|
||||
"scene": "e0b5530601be",
|
||||
"stage": "TEST",
|
||||
"tint": null,
|
||||
"option": "5f7eefac34",
|
||||
"optionText": "Head east toward the roar",
|
||||
"offered": [
|
||||
"5f7eefac34",
|
||||
"05d19abb5d"
|
||||
],
|
||||
"rule": "R",
|
||||
"decisionMode": "guide",
|
||||
"manual": false,
|
||||
"at": "2026-01-01T00:00:08.000Z",
|
||||
"to": "BONDED:green"
|
||||
}
|
||||
],
|
||||
"outcome": {
|
||||
"type": "BONDED",
|
||||
"colour": "green",
|
||||
"intentional": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"unlabeled": [
|
||||
"b0fa671066ec (TEST)",
|
||||
"e0b5530601be (TEST)",
|
||||
"1c86a35c091f (TEST)"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user