# Data collection as extract -> transform -> load, one block per source. collection/run.py reads this file. # # extract fetches the source as it is now. The raw file goes to data/raw//current. (git-ignored, # never published); the version it replaces is kept as previous., and the daily workflow keeps # that previous copy in an encrypted, private cache. Nothing older is kept. # transform turns the raw file into the public source file: a pure function of the raw file (no network, no # clock), with the source's privacy rules applied. Named transforms live in collection/etl/transform/. # load writes the public source file (byte-stable: same input, same bytes) and reports what changed. # # Then `build` codes every public source file into one run per row (runs: the coder) and writes # data/public/runs_clean.csv: every run from every source, with a `source` column. Each table is checked against # data/dictionary.yaml before it is written. Every input to `build` is committed, so CI can rebuild it exactly. version: 1 raw_dir: data/raw manifest: data/manifest.json # for each source: the sha256 of the current raw file and when it last changed runs: path: data/public/runs_clean.csv # sources in this order; within a source, the order the coder returns order: [reddit_spreadsheet, reddit_thread, reddit_comments, register, reported, bot, reddit_pdf] sources: reddit_spreadsheet: label: Threshing Routes form and spreadsheet (u/ClumsyBadger, linked from the r/fourthwing megathreads) run_source: reddit spreadsheet links: sheet: https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0 form: https://forms.gle/DRpj9tG8gb5E4Hcu7 # the Threshing Routes Google Form the megathreads link to extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0/export?format=xlsx", ext: xlsx } transform: { name: sheet, sheet_contains: form responses } privacy: { redact_emails: true } load: data/public/form_responses.json runs: spreadsheet # core/spec/form.yaml + core/atlas/coding.py register: label: Unofficial Dragon Register (which dragon each rider bonded) run_source: register links: { sheet: https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ } extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ/export?format=xlsx", ext: xlsx } transform: { name: sheet, sheet_contains: form responses, extra: collection/manual/reddit_results.yaml } privacy: { redact_emails: true, drop_columns_matching: [photo, cadet name] } # no rider names, no photos load: data/public/register.json # bonded dragons without the route that led there: in the master list as outcome-only runs, flagged, because # a rider may have logged the same run in the community form (no analysis uses them: planner/policy.yaml and # analysis/stats/measures.yaml name their sources) runs: register reddit_pdf: label: Community route PDF (u/ClumsyBadger) run_source: reddit pdf links: { view: https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view } extract: { kind: http, url: "https://drive.google.com/uc?export=download&id=17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg", ext: pdf } transform: { name: pdf_fingerprint } privacy: { redistribute: false } # only hashes of the file, its pages and its lines are kept load: data/derived/pdf_fingerprint.json runs: null # a flowchart of the routes, not a log of runs (see collection/README.md) reddit_thread: cite: [part3, part4] # both megathreads, in the dataset's isBasedOn label: Runs posted in the r/fourthwing Dragonkind megathreads run_source: reddit thread links: part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/ part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/ # the official Reddit API (app-only OAuth); skipped with a note when the secrets are not set extract: { kind: reddit, threads: [1wvwfp9, 1wz5cnj], secrets: [REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET], ext: json } # lists dragonkind.com result links that are not in the manual file yet; usernames are never kept transform: { name: reddit_results, manual: collection/manual/reddit_results.yaml } load: null # the public source file is the hand-checked collection/manual/reddit_results.yaml runs: reddit_results reddit_comments: cite: [part3, part4] # both megathreads, in the dataset's isBasedOn label: Runs riders describe in the r/fourthwing Dragonkind megathread comments run_source: reddit comment links: part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/ part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/ scraper: https://git.kenschiano.net/DragonKind/threshing-reddit # delivered, not extracted: the threshing-reddit scraper (its own repository) reads the threads every 6 hours and # opens a pull request here that changes data/public/reddit_comments.json. Each comment is read into the # "Report a run" answers (option labels of specs/report_form.yaml); only the comment id, the day and the # answers are kept: no usernames, no comment text. `update` leaves this source alone. extract: { kind: delivered, by: threshing-reddit } transform: null load: data/public/reddit_comments.json # hand corrections (reject / fix / add a run), applied when the runs are built manual: collection/manual/reddit_comments.yaml runs: reddit_comments reported: label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository) run_source: reported # the repository's host is forge.yaml (Gitea or GitHub); its "new issue" link is made from it extract: { kind: forge_issues, label: run report, ext: json } # usernames are dropped; issues closed as "not planned" (GitHub) or labelled invalid/duplicate are left out transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] } load: data/public/reports.json runs: reports bot: label: DragonKind bot runs (Ken's private bot, imported locally) run_source: bot # the bot's repo is private: this step runs only on a machine with a copy of it # data/accounts// and logs/accounts//: one folder per game account (account names are # replaced by salted hashes); the top-level files are the bot's original single account extract: kind: local_dir env: DRAGONKIND_BOT_DIR files: [data/knowledge.json, data/runs.jsonl, logs/encounters.jsonl, data/accounts/*/runs.jsonl, logs/accounts/*/encounters.jsonl] transform: { name: bot, spec: collection/specs/bot.yaml } privacy: account_salt_env: DK_ACCOUNT_SALT # account names are hashed with this salt (or, if unset, one kept in # data/raw/bot/account-salt.txt); never committed refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName] max_string: 200 # anything longer looks like game prose load: data/bot # scenes.json and trajectories-bot.jsonl, both merged with what is published runs: bot