"""Extract: fetch each source as it is now into the raw store. No transformation happens here. The raw store is data/raw// (git-ignored, never published): current. the latest version previous. the version before it (kept only in case it is needed; the daily workflow keeps it in an encrypted, private cache). Nothing older is kept and old versions are not hashed. data/manifest.json records, for each source, the sha256 of the current raw file and the date it last changed. Each extractor returns the raw bytes, or raises Skip when the source cannot be reached from here (a missing secret, the bot's private repo not on this machine). """ import base64 import fnmatch import glob import json import os import shutil import tempfile import urllib.parse import urllib.request import zipfile from datetime import datetime, timezone from pathlib import Path from .common import path, sha256_bytes USER_AGENT = "threshing-atlas-collector/1.0 (+https://github.com/ekcSchiano/threshing-atlas)" RAW_DIR = path("data/raw") class Skip(Exception): """The source cannot be extracted here; not an error.""" def _get(url, headers=None, data=None, timeout=60): req = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT, **(headers or {})}) with urllib.request.urlopen(req, timeout=timeout) as r: return r.read() # ---------------------------------------------------------------- extractors, by `kind` def http(spec, env): data = _get(spec["url"]) head = data[:15].lower() if head.startswith(b"