CI / test (pull_request) Canceled after 0s
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen, every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree. - The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and graph.js now use the played steps only. - runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only rows, flagged (no analysis uses them). - Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
100 lines
6.1 KiB
YAML
100 lines
6.1 KiB
YAML
# Data collection as extract -> transform -> load, one block per source. collection/run.py reads this file.
|
|
#
|
|
# extract fetches the source as it is now. The raw file goes to data/raw/<source>/current.<ext> (git-ignored,
|
|
# never published); the version it replaces is kept as previous.<ext>, and the daily workflow keeps
|
|
# that previous copy in an encrypted, private cache. Nothing older is kept.
|
|
# transform turns the raw file into the public source file: a pure function of the raw file (no network, no
|
|
# clock), with the source's privacy rules applied. Named transforms live in collection/etl/transform/.
|
|
# load writes the public source file (byte-stable: same input, same bytes) and reports what changed.
|
|
#
|
|
# Then `build` codes every public source file into one run per row (runs: the coder) and writes
|
|
# data/public/runs_clean.csv: every run from every source, with a `source` column. Each table is checked against
|
|
# data/dictionary.yaml before it is written. Every input to `build` is committed, so CI can rebuild it exactly.
|
|
version: 1
|
|
|
|
raw_dir: data/raw
|
|
manifest: data/manifest.json # for each source: the sha256 of the current raw file and when it last changed
|
|
|
|
runs:
|
|
path: data/public/runs_clean.csv
|
|
# sources in this order; within a source, the order the coder returns
|
|
order: [reddit_spreadsheet, reddit_thread, register, reported, bot, reddit_pdf]
|
|
|
|
sources:
|
|
reddit_spreadsheet:
|
|
label: Threshing Routes form and spreadsheet (u/ClumsyBadger, linked from the r/fourthwing megathreads)
|
|
run_source: reddit spreadsheet
|
|
links:
|
|
sheet: https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0
|
|
form: https://forms.gle/DRpj9tG8gb5E4Hcu7 # the Threshing Routes Google Form the megathreads link to
|
|
extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0/export?format=xlsx", ext: xlsx }
|
|
transform: { name: sheet, sheet_contains: form responses }
|
|
privacy: { redact_emails: true }
|
|
load: data/public/form_responses.json
|
|
runs: spreadsheet # core/spec/form.yaml + core/atlas/coding.py
|
|
|
|
register:
|
|
label: Unofficial Dragon Register (which dragon each rider bonded)
|
|
run_source: register
|
|
links: { sheet: https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ }
|
|
extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ/export?format=xlsx", ext: xlsx }
|
|
transform: { name: sheet, sheet_contains: form responses, extra: collection/manual/reddit_results.yaml }
|
|
privacy: { redact_emails: true, drop_columns_matching: [photo, cadet name] } # no rider names, no photos
|
|
load: data/public/register.json
|
|
# bonded dragons without the route that led there: in the master list as outcome-only runs, flagged, because
|
|
# a rider may have logged the same run in the community form (no analysis uses them: planner/policy.yaml and
|
|
# analysis/stats/measures.yaml name their sources)
|
|
runs: register
|
|
|
|
reddit_pdf:
|
|
label: Community route PDF (u/ClumsyBadger)
|
|
run_source: reddit pdf
|
|
links: { view: https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view }
|
|
extract: { kind: http, url: "https://drive.google.com/uc?export=download&id=17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg", ext: pdf }
|
|
transform: { name: pdf_fingerprint }
|
|
privacy: { redistribute: false } # only hashes of the file, its pages and its lines are kept
|
|
load: data/derived/pdf_fingerprint.json
|
|
runs: null # a flowchart of the routes, not a log of runs (see collection/README.md)
|
|
|
|
reddit_thread:
|
|
label: Runs posted in the r/fourthwing Dragonkind megathreads
|
|
run_source: reddit thread
|
|
links:
|
|
part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/
|
|
part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/
|
|
# the official Reddit API (app-only OAuth); skipped with a note when the secrets are not set
|
|
extract: { kind: reddit, threads: [1wvwfp9, 1wz5cnj], secrets: [REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET], ext: json }
|
|
# lists dragonkind.com result links that are not in the manual file yet; usernames are never kept
|
|
transform: { name: reddit_results, manual: collection/manual/reddit_results.yaml }
|
|
load: null # the public source file is the hand-checked collection/manual/reddit_results.yaml
|
|
runs: reddit_results
|
|
|
|
reported:
|
|
label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository)
|
|
run_source: reported
|
|
# the repository's host is forge.yaml (Gitea or GitHub); its "new issue" link is made from it
|
|
extract: { kind: forge_issues, label: run report, ext: json }
|
|
# usernames are dropped; issues closed as "not planned" (GitHub) or labelled invalid/duplicate are left out
|
|
transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] }
|
|
load: data/public/reports.json
|
|
runs: reports
|
|
|
|
bot:
|
|
label: DragonKind bot runs (Ken's private bot, imported locally)
|
|
run_source: bot
|
|
# the bot's repo is private: this step runs only on a machine with a copy of it
|
|
# data/accounts/<account>/ and logs/accounts/<account>/: one folder per game account (account names are
|
|
# replaced by salted hashes); the top-level files are the bot's original single account
|
|
extract:
|
|
kind: local_dir
|
|
env: DRAGONKIND_BOT_DIR
|
|
files: [data/knowledge.json, data/runs.jsonl, logs/encounters.jsonl, data/accounts/*/runs.jsonl, logs/accounts/*/encounters.jsonl]
|
|
transform: { name: bot, spec: collection/specs/bot.yaml }
|
|
privacy:
|
|
account_salt_env: DK_ACCOUNT_SALT # account names are hashed with this salt (or, if unset, one kept in
|
|
# data/raw/bot/account-salt.txt); never committed
|
|
refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName]
|
|
max_string: 200 # anything longer looks like game prose
|
|
load: data/bot # scenes.json (merged) + trajectories-<date>.jsonl
|
|
runs: bot
|