Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
61bc50583c
commit
3b24a11f5a
@@ -0,0 +1,132 @@
|
||||
"""Extract: fetch each source as it is now into the raw store. No transformation happens here.
|
||||
|
||||
The raw store is data/raw/<source>/ (git-ignored, never published):
|
||||
current.<ext> the latest version
|
||||
previous.<ext> the version before it (kept only in case it is needed; the daily workflow keeps it in an
|
||||
encrypted, private cache). Nothing older is kept and old versions are not hashed.
|
||||
data/manifest.json records, for each source, the sha256 of the current raw file and the date it last changed.
|
||||
|
||||
Each extractor returns the raw bytes, or raises Skip when the source cannot be reached from here (a missing
|
||||
secret, the bot's private repo not on this machine).
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .common import path, sha256_bytes
|
||||
|
||||
USER_AGENT = "threshing-atlas-collector/1.0 (+https://github.com/ekcSchiano/threshing-atlas)"
|
||||
RAW_DIR = path("data/raw")
|
||||
|
||||
|
||||
class Skip(Exception):
|
||||
"""The source cannot be extracted here; not an error."""
|
||||
|
||||
|
||||
def _get(url, headers=None, data=None, timeout=60):
|
||||
req = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT, **(headers or {})})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- extractors, by `kind`
|
||||
|
||||
def http(spec, env):
|
||||
data = _get(spec["url"])
|
||||
head = data[:15].lower()
|
||||
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
||||
raise RuntimeError(f"{spec['url']} returned a web page, not a file (is the link still shared publicly?)")
|
||||
return data
|
||||
|
||||
|
||||
def reddit(spec, env):
|
||||
"""The threads' comments through the official Reddit API (app-only OAuth: a "script" app's id and secret)."""
|
||||
cid, secret = (env.get(k) for k in spec["secrets"])
|
||||
if not (cid and secret):
|
||||
raise Skip(f"set the {' and '.join(spec['secrets'])} secrets to read the threads through the Reddit API")
|
||||
auth = base64.b64encode(f"{cid}:{secret}".encode()).decode()
|
||||
token = json.loads(_get("https://www.reddit.com/api/v1/access_token",
|
||||
headers={"Authorization": f"Basic {auth}"},
|
||||
data=b"grant_type=client_credentials"))["access_token"]
|
||||
threads = {}
|
||||
for tid in spec["threads"]:
|
||||
threads[tid] = json.loads(_get(f"https://oauth.reddit.com/comments/{tid}?limit=500&depth=10&raw_json=1",
|
||||
headers={"Authorization": f"Bearer {token}"}))
|
||||
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def github_issues(spec, env):
|
||||
"""Every issue with the label (open and closed), through the GitHub REST API."""
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
label = urllib.parse.quote(spec["label"])
|
||||
issues, page = [], 1
|
||||
while True:
|
||||
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
|
||||
batch = json.loads(_get(url, headers=headers))
|
||||
issues += [i for i in batch if "pull_request" not in i]
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
issues.sort(key=lambda i: i["number"])
|
||||
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def local_dir(spec, env):
|
||||
"""The bot's files, from a local copy of its private repo (DRAGONKIND_BOT_DIR), packed as one JSON."""
|
||||
root = env.get(spec["env"])
|
||||
if not root:
|
||||
raise Skip(f"the bot's repo is private: set {spec['env']} to a local copy to import it")
|
||||
files = {}
|
||||
for rel in spec["files"]:
|
||||
p = os.path.join(root, rel)
|
||||
files[rel] = Path(p).read_text(encoding="utf-8") if os.path.exists(p) else None
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the raw store
|
||||
|
||||
def raw_paths(name, ext):
|
||||
d = RAW_DIR / name
|
||||
return d / f"current.{ext}", d / f"previous.{ext}"
|
||||
|
||||
|
||||
def store(name, ext, data, manifest, today=None):
|
||||
"""Keep `data` as the current raw file. Returns True when it differs from the last one seen."""
|
||||
current, previous = raw_paths(name, ext)
|
||||
digest = sha256_bytes(data)
|
||||
entry = manifest.get(name) or {}
|
||||
changed = entry.get("sha256") != digest
|
||||
current.parent.mkdir(parents=True, exist_ok=True)
|
||||
on_disk = sha256_bytes(current.read_bytes()) if current.exists() else None
|
||||
if on_disk and on_disk != digest:
|
||||
shutil.copyfile(current, previous) # keep one previous version; anything older is gone
|
||||
if on_disk != digest:
|
||||
fd, tmp = tempfile.mkstemp(dir=current.parent)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(tmp, current)
|
||||
if changed:
|
||||
manifest[name] = {"sha256": digest, "bytes": len(data),
|
||||
"changed": today or datetime.now(timezone.utc).strftime("%Y-%m-%d")}
|
||||
return changed
|
||||
|
||||
|
||||
def extract(name, spec, manifest, env=None, local_file=None):
|
||||
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
|
||||
env = os.environ if env is None else env
|
||||
ext = spec["extract"].get("ext") or "json"
|
||||
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env)
|
||||
changed = store(name, ext, data, manifest)
|
||||
return raw_paths(name, ext)[0], changed
|
||||
Reference in New Issue
Block a user