Files
threshing-atlas/collection/etl/extract.py
T
ekcSchianoandClaude Opus 5.5 3b24a11f5a Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-04 02:46:48 +00:00

133 lines
5.6 KiB
Python

"""Extract: fetch each source as it is now into the raw store. No transformation happens here.
The raw store is data/raw/<source>/ (git-ignored, never published):
current.<ext> the latest version
previous.<ext> the version before it (kept only in case it is needed; the daily workflow keeps it in an
encrypted, private cache). Nothing older is kept and old versions are not hashed.
data/manifest.json records, for each source, the sha256 of the current raw file and the date it last changed.
Each extractor returns the raw bytes, or raises Skip when the source cannot be reached from here (a missing
secret, the bot's private repo not on this machine).
"""
import base64
import json
import os
import shutil
import tempfile
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from .common import path, sha256_bytes
USER_AGENT = "threshing-atlas-collector/1.0 (+https://github.com/ekcSchiano/threshing-atlas)"
RAW_DIR = path("data/raw")
class Skip(Exception):
"""The source cannot be extracted here; not an error."""
def _get(url, headers=None, data=None, timeout=60):
req = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT, **(headers or {})})
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
# ---------------------------------------------------------------- extractors, by `kind`
def http(spec, env):
data = _get(spec["url"])
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{spec['url']} returned a web page, not a file (is the link still shared publicly?)")
return data
def reddit(spec, env):
"""The threads' comments through the official Reddit API (app-only OAuth: a "script" app's id and secret)."""
cid, secret = (env.get(k) for k in spec["secrets"])
if not (cid and secret):
raise Skip(f"set the {' and '.join(spec['secrets'])} secrets to read the threads through the Reddit API")
auth = base64.b64encode(f"{cid}:{secret}".encode()).decode()
token = json.loads(_get("https://www.reddit.com/api/v1/access_token",
headers={"Authorization": f"Basic {auth}"},
data=b"grant_type=client_credentials"))["access_token"]
threads = {}
for tid in spec["threads"]:
threads[tid] = json.loads(_get(f"https://oauth.reddit.com/comments/{tid}?limit=500&depth=10&raw_json=1",
headers={"Authorization": f"Bearer {token}"}))
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
def github_issues(spec, env):
"""Every issue with the label (open and closed), through the GitHub REST API."""
headers = {"Accept": "application/vnd.github+json"}
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
if token:
headers["Authorization"] = f"Bearer {token}"
label = urllib.parse.quote(spec["label"])
issues, page = [], 1
while True:
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
batch = json.loads(_get(url, headers=headers))
issues += [i for i in batch if "pull_request" not in i]
if len(batch) < 100:
break
page += 1
issues.sort(key=lambda i: i["number"])
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
def local_dir(spec, env):
"""The bot's files, from a local copy of its private repo (DRAGONKIND_BOT_DIR), packed as one JSON."""
root = env.get(spec["env"])
if not root:
raise Skip(f"the bot's repo is private: set {spec['env']} to a local copy to import it")
files = {}
for rel in spec["files"]:
p = os.path.join(root, rel)
files[rel] = Path(p).read_text(encoding="utf-8") if os.path.exists(p) else None
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
# ---------------------------------------------------------------- the raw store
def raw_paths(name, ext):
d = RAW_DIR / name
return d / f"current.{ext}", d / f"previous.{ext}"
def store(name, ext, data, manifest, today=None):
"""Keep `data` as the current raw file. Returns True when it differs from the last one seen."""
current, previous = raw_paths(name, ext)
digest = sha256_bytes(data)
entry = manifest.get(name) or {}
changed = entry.get("sha256") != digest
current.parent.mkdir(parents=True, exist_ok=True)
on_disk = sha256_bytes(current.read_bytes()) if current.exists() else None
if on_disk and on_disk != digest:
shutil.copyfile(current, previous) # keep one previous version; anything older is gone
if on_disk != digest:
fd, tmp = tempfile.mkstemp(dir=current.parent)
with os.fdopen(fd, "wb") as f:
f.write(data)
os.replace(tmp, current)
if changed:
manifest[name] = {"sha256": digest, "bytes": len(data),
"changed": today or datetime.now(timezone.utc).strftime("%Y-%m-%d")}
return changed
def extract(name, spec, manifest, env=None, local_file=None):
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
env = os.environ if env is None else env
ext = spec["extract"].get("ext") or "json"
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env)
changed = store(name, ext, data, manifest)
return raw_paths(name, ext)[0], changed