- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
103 lines
4.4 KiB
YAML
103 lines
4.4 KiB
YAML
name: Check sources for new data
|
|
on:
|
|
schedule:
|
|
- cron: "23 10 * * *" # daily
|
|
workflow_dispatch:
|
|
permissions:
|
|
contents: write
|
|
pull-requests: write
|
|
issues: write
|
|
jobs:
|
|
check:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
# secrets cannot be tested in `if:` directly
|
|
HAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN != '' }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 20
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.12"
|
|
- run: pip install -r requirements.txt
|
|
# The raw store (data/raw: the current and previous raw file of each source) is never committed. Between runs
|
|
# it lives in the Actions cache, encrypted with the RAW_ARCHIVE_KEY secret, so only the previous version is
|
|
# kept and nobody without the key can read it (the register has rider names). Without the secret there is no
|
|
# archive: each run starts from the manifest alone.
|
|
- name: Restore the private raw archive
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ${{ runner.temp }}/raw-archive
|
|
key: raw-archive-none
|
|
restore-keys: raw-archive-
|
|
- name: Decrypt it
|
|
env:
|
|
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
|
|
run: |
|
|
f="$RUNNER_TEMP/raw-archive/raw.tgz.enc"
|
|
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
|
|
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
|
|
fi
|
|
# extract -> transform -> load for every source, then the combined runs (collection/pipeline.yaml)
|
|
- id: check
|
|
env:
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
|
|
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
|
|
run: python -m collection update --report "$RUNNER_TEMP/report.md"
|
|
- name: Rebuild the analysis and run tests
|
|
if: steps.check.outputs.changed == 'true'
|
|
run: |
|
|
node analysis/network/build-graph.js
|
|
npm run policy
|
|
npm run stats
|
|
python -m collection build
|
|
npm test
|
|
- name: Encrypt the raw store
|
|
id: archive
|
|
env:
|
|
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
|
|
run: |
|
|
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -d data/raw ]; then
|
|
mkdir -p "$RUNNER_TEMP/raw-archive"
|
|
tar -czf - -C data raw | openssl enc -aes-256-cbc -pbkdf2 -salt -pass env:RAW_ARCHIVE_KEY -out "$RUNNER_TEMP/raw-archive/raw.tgz.enc"
|
|
echo "save=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
- name: Save it (a new copy only when a raw file changed)
|
|
if: steps.archive.outputs.save == 'true'
|
|
uses: actions/cache/save@v4
|
|
with:
|
|
path: ${{ runner.temp }}/raw-archive
|
|
key: raw-archive-${{ hashFiles('data/manifest.json') }}
|
|
# The PR is opened with ATLAS_BOT_TOKEN (a fine-grained token, see docs/REPO_SETTINGS.md) when it is set:
|
|
# changes made with the built-in GITHUB_TOKEN do not start other workflows, so its merge would not
|
|
# redeploy the site. Without the secret it falls back to GITHUB_TOKEN and the PR waits for a manual merge.
|
|
- name: Open a pull request with the new data
|
|
id: pr
|
|
if: steps.check.outputs.changed == 'true'
|
|
uses: peter-evans/create-pull-request@v6
|
|
with:
|
|
token: ${{ secrets.ATLAS_BOT_TOKEN || github.token }}
|
|
branch: data/source-update
|
|
delete-branch: true
|
|
commit-message: "data: update from community sources"
|
|
title: "New data from the community sources"
|
|
body-path: ${{ runner.temp }}/report.md
|
|
labels: data
|
|
- name: Merge it automatically once CI passes
|
|
if: steps.pr.outputs.pull-request-number != '' && env.HAS_BOT_TOKEN == 'true'
|
|
env:
|
|
GH_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
|
run: gh pr merge "${{ steps.pr.outputs.pull-request-number }}" --auto --squash
|
|
- name: Report a broken source link
|
|
if: steps.check.outputs.failed != ''
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
title="Source check failed: ${{ steps.check.outputs.failed }}"
|
|
if ! gh issue list --state open --search "$title in:title" --json number --jq '.[0].number' | grep -q .; then
|
|
gh issue create --title "$title" --body-file "$RUNNER_TEMP/report.md"
|
|
fi
|