- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
90 lines
5.5 KiB
YAML
90 lines
5.5 KiB
YAML
# Data collection as extract -> transform -> load, one block per source. collection/run.py reads this file.
|
|
#
|
|
# extract fetches the source as it is now. The raw file goes to data/raw/<source>/current.<ext> (git-ignored,
|
|
# never published); the version it replaces is kept as previous.<ext>, and the daily workflow keeps
|
|
# that previous copy in an encrypted, private cache. Nothing older is kept.
|
|
# transform turns the raw file into the public source file: a pure function of the raw file (no network, no
|
|
# clock), with the source's privacy rules applied. Named transforms live in collection/etl/transform/.
|
|
# load writes the public source file (byte-stable: same input, same bytes) and reports what changed.
|
|
#
|
|
# Then `build` codes every public source file into one run per row (runs: the coder) and writes
|
|
# data/public/runs_clean.csv: every run from every source, with a `source` column. Each table is checked against
|
|
# data/dictionary.yaml before it is written. Every input to `build` is committed, so CI can rebuild it exactly.
|
|
version: 1
|
|
|
|
raw_dir: data/raw
|
|
manifest: data/manifest.json # for each source: the sha256 of the current raw file and when it last changed
|
|
|
|
runs:
|
|
path: data/public/runs_clean.csv
|
|
# sources in this order; within a source, the order the coder returns
|
|
order: [reddit_spreadsheet, reddit_thread, reported, bot, reddit_pdf]
|
|
|
|
sources:
|
|
reddit_spreadsheet:
|
|
label: Threshing Routes form and spreadsheet (u/ClumsyBadger, linked from the r/fourthwing megathreads)
|
|
run_source: reddit spreadsheet
|
|
links:
|
|
sheet: https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0
|
|
form: null # the megathread's Google Form (responder link); shown on "Report a run" when set
|
|
extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0/export?format=xlsx", ext: xlsx }
|
|
transform: { name: sheet, sheet_contains: form responses }
|
|
privacy: { redact_emails: true }
|
|
load: data/public/form_responses.json
|
|
runs: spreadsheet # core/spec/form.yaml + core/atlas/coding.py
|
|
|
|
register:
|
|
label: Unofficial Dragon Register (which dragon each rider bonded)
|
|
links: { sheet: https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ }
|
|
extract: { kind: http, url: "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ/export?format=xlsx", ext: xlsx }
|
|
transform: { name: sheet, sheet_contains: form responses, extra: collection/manual/reddit_results.yaml }
|
|
privacy: { redact_emails: true, drop_columns_matching: [photo, cadet name] } # no rider names, no photos
|
|
load: data/public/register.json
|
|
runs: null # bonded dragons without the route that led there
|
|
|
|
reddit_pdf:
|
|
label: Community route PDF (u/ClumsyBadger)
|
|
run_source: reddit pdf
|
|
links: { view: https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view }
|
|
extract: { kind: http, url: "https://drive.google.com/uc?export=download&id=17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg", ext: pdf }
|
|
transform: { name: pdf_fingerprint }
|
|
privacy: { redistribute: false } # only hashes of the file, its pages and its lines are kept
|
|
load: data/derived/pdf_fingerprint.json
|
|
runs: null # a flowchart of the routes, not a log of runs (see collection/README.md)
|
|
|
|
reddit_thread:
|
|
label: Runs posted in the r/fourthwing Dragonkind megathreads
|
|
run_source: reddit thread
|
|
links:
|
|
part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/
|
|
part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/
|
|
# the official Reddit API (app-only OAuth); skipped with a note when the secrets are not set
|
|
extract: { kind: reddit, threads: [1wvwfp9, 1wz5cnj], secrets: [REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET], ext: json }
|
|
# lists dragonkind.com result links that are not in the manual file yet; usernames are never kept
|
|
transform: { name: reddit_results, manual: collection/manual/reddit_results.yaml }
|
|
load: null # the public source file is the hand-checked collection/manual/reddit_results.yaml
|
|
runs: reddit_results
|
|
|
|
reported:
|
|
label: Runs reported on the Atlas ("Report a run", a GitHub issue form)
|
|
run_source: reported
|
|
links: { report: "https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" }
|
|
extract: { kind: github_issues, repo: ekcSchiano/threshing-atlas, label: run report, ext: json }
|
|
# GitHub usernames are dropped; issues closed as "not planned" or labelled "invalid" are left out
|
|
transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] }
|
|
load: data/public/reports.json
|
|
runs: reports
|
|
|
|
bot:
|
|
label: DragonKind bot runs (Ken's private bot, imported locally)
|
|
run_source: bot
|
|
# the bot's repo is private: this step runs only on a machine with a copy of it
|
|
extract: { kind: local_dir, env: DRAGONKIND_BOT_DIR, files: [data/knowledge.json, data/runs.jsonl, logs/encounters.jsonl] }
|
|
transform: { name: bot, spec: collection/specs/bot.yaml }
|
|
privacy:
|
|
account_salt_env: DK_ACCOUNT_SALT # account ids are hashed with this salt, which is never committed
|
|
refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName]
|
|
max_string: 200 # anything longer looks like game prose
|
|
load: data/bot # scenes.json (merged) + trajectories-<date>.jsonl
|
|
runs: bot
|