Files
threshing-atlas/collection/etl/extract.py
kschianoandClaude Opus 5.5 197c16dac0
CI / test (pull_request) Successful in 46s
Receive runs from the Reddit megathread comments (source "reddit comment")
The threshing-reddit scraper (DragonKind/threshing-reddit) reads the runs
riders describe in the r/fourthwing megathreads and delivers them here by
pull request as data/public/reddit_comments.json: each run is the "Report a
run" answers (option labels of collection/specs/report_form.yaml), with the
comment id and day only; no usernames, no comment text.

- pipeline.yaml: a reddit_comments source of extract kind "delivered" (the
  atlas fetches nothing; `update` skips it with a note)
- runs.py: the coder, sharing one answers_row() with the reported coder;
  every row is flagged as read from a Reddit comment (and "not checked by
  hand" until corrected); a result page a comment already carries is not
  counted twice as a "reddit thread" row
- collection/manual/reddit_comments.yaml: reject / fix / add, applied at
  build time
- dictionary, SOURCES, README, croissant: the new source

The route model and statistics do not use the source until it is named in
planner/policy.yaml / analysis/stats/measures.yaml.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:06:14 -05:00

159 lines
7.0 KiB
Python

"""Extract: fetch each source as it is now into the raw store. No transformation happens here.
The raw store is data/raw/<source>/ (git-ignored, never published):
current.<ext> the latest version
previous.<ext> the version before it (kept only in case it is needed; the daily workflow keeps it in an
encrypted, private cache). Nothing older is kept and old versions are not hashed.
data/manifest.json records, for each source, the sha256 of the current raw file and the date it last changed.
Each extractor returns the raw bytes, or raises Skip when the source cannot be reached from here (a missing
secret, the bot's private repo not on this machine).
"""
import base64
import fnmatch
import glob
import json
import os
import shutil
import tempfile
import urllib.parse
import urllib.request
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from .common import path, sha256_bytes
USER_AGENT = "threshing-atlas-collector/1.0 (+https://github.com/ekcSchiano/threshing-atlas)"
RAW_DIR = path("data/raw")
class Skip(Exception):
"""The source cannot be extracted here; not an error."""
def _get(url, headers=None, data=None, timeout=60):
req = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT, **(headers or {})})
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
# ---------------------------------------------------------------- extractors, by `kind`
def http(spec, env):
data = _get(spec["url"])
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{spec['url']} returned a web page, not a file (is the link still shared publicly?)")
return data
def reddit(spec, env):
"""The threads' comments through the official Reddit API (app-only OAuth: a "script" app's id and secret)."""
cid, secret = (env.get(k) for k in spec["secrets"])
if not (cid and secret):
raise Skip(f"set the {' and '.join(spec['secrets'])} secrets to read the threads through the Reddit API")
auth = base64.b64encode(f"{cid}:{secret}".encode()).decode()
token = json.loads(_get("https://www.reddit.com/api/v1/access_token",
headers={"Authorization": f"Basic {auth}"},
data=b"grant_type=client_credentials"))["access_token"]
threads = {}
for tid in spec["threads"]:
threads[tid] = json.loads(_get(f"https://oauth.reddit.com/comments/{tid}?limit=500&depth=10&raw_json=1",
headers={"Authorization": f"Bearer {token}"}))
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
def delivered(spec, env):
"""A source another repository delivers by pull request (it changes the public file itself): nothing to fetch."""
raise Skip(f"delivered by pull request from {spec.get('by') or 'another repository'}")
def forge_issues(spec, env):
"""Every issue with the label (open and closed), from the repository's host (forge.yaml: Gitea or GitHub)."""
from . import forge
issues = forge.list_issues(spec["label"], env)
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
def local_dir(spec, env):
"""The bot's files, from a local copy of its private repo (DRAGONKIND_BOT_DIR), packed as one JSON."""
root = env.get(spec["env"])
if not root:
raise Skip(f"the bot's repo is private: set {spec['env']} to a local copy to import it")
files = {}
for pattern in spec["files"]: # a pattern may have * for one folder level (one folder per bot account)
hits = sorted(glob.glob(os.path.join(root, pattern)))
for p in hits:
files[os.path.relpath(p, root).replace(os.sep, "/")] = Path(p).read_text(encoding="utf-8")
if not hits and "*" not in pattern:
files[pattern] = None
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
def pack_local(spec, src):
"""The bot's files from a folder (a copy of the bot repo) or a .zip someone sent (the files may sit under any
top folder inside it), packed as one JSON like local_dir."""
src = Path(src).expanduser()
if src.is_dir():
return local_dir(spec, {spec["env"]: str(src)})
files = {rel: None for rel in spec["files"] if "*" not in rel}
with zipfile.ZipFile(src) as z:
names = [n for n in z.namelist() if not n.endswith("/") and "__MACOSX" not in n]
for pattern in spec["files"]:
for n in names:
# the files may sit under a top folder (a zipped repo): match the pattern on the end of the name
parts = n.split("/")
rel = next(("/".join(parts[i:]) for i in range(len(parts)) if fnmatch.fnmatchcase("/".join(parts[i:]), pattern)), None)
if rel and files.get(rel) is None:
files[rel] = z.read(n).decode("utf-8")
if not any(files.values()):
raise RuntimeError(f"{src.name}: none of {', '.join(spec['files'])} found inside")
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir, "delivered": delivered}
# ---------------------------------------------------------------- the raw store
def raw_paths(name, ext):
d = RAW_DIR / name
return d / f"current.{ext}", d / f"previous.{ext}"
def store(name, ext, data, manifest, today=None):
"""Keep `data` as the current raw file. Returns True when it differs from the last one seen."""
current, previous = raw_paths(name, ext)
digest = sha256_bytes(data)
entry = manifest.get(name) or {}
changed = entry.get("sha256") != digest
current.parent.mkdir(parents=True, exist_ok=True)
on_disk = sha256_bytes(current.read_bytes()) if current.exists() else None
if on_disk and on_disk != digest:
shutil.copyfile(current, previous) # keep one previous version; anything older is gone
if on_disk != digest:
fd, tmp = tempfile.mkstemp(dir=current.parent)
with os.fdopen(fd, "wb") as f:
f.write(data)
os.replace(tmp, current)
if changed:
manifest[name] = {"sha256": digest, "bytes": len(data),
"changed": today or datetime.now(timezone.utc).strftime("%Y-%m-%d")}
return changed
def extract(name, spec, manifest, env=None, local_file=None):
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
env = os.environ if env is None else env
ext = spec["extract"].get("ext") or "json"
kind = spec["extract"]["kind"]
if local_file and kind == "local_dir":
data = pack_local(spec["extract"], local_file) # a bot folder or a .zip someone sent
elif local_file:
data = Path(local_file).expanduser().read_bytes()
else:
data = EXTRACTORS[kind](spec["extract"], env)
changed = store(name, ext, data, manifest)
return raw_paths(name, ext)[0], changed