Compare commits
2
Commits
cd618d37d3
...
d7befc8e18
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d7befc8e18 | ||
|
|
98be27fc1f |
@@ -0,0 +1,31 @@
|
||||
# A review summary on every Reddit scraper pull request (issue #23): the runs added, changed and gone, the ones that
|
||||
# break a game rule, other flags, possible copies of form rows, and result pages already counted. Posted as one
|
||||
# comment on the pull request, refreshed on every push. The reviewer corrects runs in
|
||||
# collection/manual/reddit_comments.yaml (collection/README.md, "Reviewing Reddit runs"), then merges.
|
||||
#
|
||||
# Secrets: ATLAS_BOT_TOKEN (as in update-sources.yml), to post the comment.
|
||||
name: Review Reddit runs
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- data/public/reddit_comments.json
|
||||
- collection/manual/reddit_comments.yaml
|
||||
jobs:
|
||||
summary:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements.txt
|
||||
- name: The file as it is on main
|
||||
run: |
|
||||
git fetch --depth 1 origin "${{ github.base_ref }}"
|
||||
git show FETCH_HEAD:data/public/reddit_comments.json > "$RUNNER_TEMP/base.json" || echo '{"runs": []}' > "$RUNNER_TEMP/base.json"
|
||||
- name: Build the runs and post the summary
|
||||
env:
|
||||
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||
run: |
|
||||
python -m collection build
|
||||
python -m collection review-reddit --base "$RUNNER_TEMP/base.json" --out "$RUNNER_TEMP/review.md" --pr "${{ github.event.pull_request.number }}"
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Is the same attempt reported twice, in the community form and in a Reddit comment? (issue #24)
|
||||
|
||||
python analysis/overlap/overlap.py [--check]
|
||||
|
||||
There are no player ids, so a copy can only be proven by a shared dragonkind.com result-page id. Neither the form
|
||||
nor the register records one, and the comment-to-result-page link is already applied when the runs are built
|
||||
(collection/etl/runs.py). Everything else is measured, never removed:
|
||||
|
||||
- a Reddit comment run is *comparable* when it is dated and gives at least MIN_ANSWERS route answers besides how it
|
||||
ended (fewer answers match dozens of form rows by chance on a busy day);
|
||||
- it *matches* a form row submitted within WINDOW days when every answer it gives is the form row's answer;
|
||||
- the same test with the comment's date shifted by a few weeks (SHIFTS, only where the form has rows that week) is
|
||||
what chance alone produces, so matched - expected is the estimate of runs reported twice.
|
||||
|
||||
Writes data/public/overlap.json (the page's Data & methods tab, and the scraper pull-request summary). Nothing is
|
||||
dropped: identical routes are expected, and the Reddit runs already have their own reporting-bias group in the
|
||||
route model (planner/policy.yaml).
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
|
||||
OUT = ROOT / "data/public/overlap.json"
|
||||
FIELDS = ["company", "left_squadmates", "direction", "first_sighting", "sighting_action", "scream", "resolution",
|
||||
"combat_first", "combat_second", "post_path", "hour", "branch", "outcome", "outcome_color", "tail", "attempt"]
|
||||
ENDING = ("outcome", "outcome_color")
|
||||
MIN_ANSWERS = 4
|
||||
WINDOW = 1
|
||||
SHIFTS = (-28, -21, -14, 14, 21, 28)
|
||||
|
||||
|
||||
def given(v):
|
||||
return v not in (None, "") and v != "Unknown" and not (isinstance(v, str) and v.startswith("("))
|
||||
|
||||
|
||||
def answers(r):
|
||||
return [k for k in FIELDS if given(r.get(k))]
|
||||
|
||||
|
||||
def comparable(r):
|
||||
return bool(r.get("date")) and len([k for k in answers(r) if k not in ENDING]) >= MIN_ANSWERS
|
||||
|
||||
|
||||
def days(date, shift=0, window=WINDOW):
|
||||
d = dt.date.fromisoformat(date[:10]) + dt.timedelta(days=shift)
|
||||
return [(d + dt.timedelta(days=i)).isoformat() for i in range(-window, window + 1)]
|
||||
|
||||
|
||||
def form_index(runs):
|
||||
by_day = collections.defaultdict(list)
|
||||
for r in runs:
|
||||
if r["source"] == "reddit spreadsheet" and r.get("date"):
|
||||
by_day[r["date"][:10]].append(r)
|
||||
return by_day
|
||||
|
||||
|
||||
def matches(c, by_day, shift=0):
|
||||
"""Form rows within the window that give every answer the comment gives."""
|
||||
keys = answers(c)
|
||||
return [f for d in days(c["date"], shift) for f in by_day.get(d, []) if all(f.get(k) == c[k] for k in keys)]
|
||||
|
||||
|
||||
def measure(runs):
|
||||
by_day = form_index(runs)
|
||||
comments = [r for r in runs if r["source"] == "reddit comment"]
|
||||
comp = [c for c in comments if comparable(c)]
|
||||
found = {c["run_id"]: [f["run_id"] for f in matches(c, by_day)] for c in comp}
|
||||
matched = sum(1 for v in found.values() if v)
|
||||
# chance: each comment's match rate on shifted dates, over the shifts whose window has form rows at all
|
||||
expected = 0.0
|
||||
for c in comp:
|
||||
valid = [s for s in SHIFTS if any(by_day.get(d) for d in days(c["date"], s))]
|
||||
if valid:
|
||||
expected += sum(1 for s in valid if matches(c, by_day, s)) / len(valid)
|
||||
return {
|
||||
"schema": 1,
|
||||
"rule": {"min_answers": MIN_ANSWERS, "window_days": WINDOW, "baseline_shifts_days": list(SHIFTS)},
|
||||
"comment_runs": len(comments),
|
||||
"comparable": len(comp),
|
||||
"matched": matched,
|
||||
"expected_by_chance": round(expected, 1),
|
||||
"likely_double": round(max(0.0, matched - expected), 1),
|
||||
"matches": [{"comment": k, "form": v} for k, v in sorted(found.items()) if v],
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--check", action="store_true", help="fail if data/public/overlap.json is out of date")
|
||||
a = ap.parse_args(argv)
|
||||
text = json.dumps(measure(runs_io.read(drop=())), indent=1, ensure_ascii=False) + "\n"
|
||||
if a.check:
|
||||
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
||||
print(f"out of date (run `python {Path(__file__).relative_to(ROOT)}`): {OUT.relative_to(ROOT)}")
|
||||
return 1
|
||||
return 0
|
||||
OUT.write_text(text, encoding="utf-8")
|
||||
m = json.loads(text)
|
||||
print(f"{OUT.relative_to(ROOT)}: {m['matched']} of {m['comparable']} comparable Reddit runs match a form row "
|
||||
f"(~{m['expected_by_chance']} by chance; ~{m['likely_double']} likely reported twice)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -41,6 +41,25 @@ The route PDF is a flowchart of the choices, not a log of runs, so it adds no ro
|
||||
fingerprint tells us when the routes it documents change. The register lists bonded dragons without the route that
|
||||
led there, so it adds no rows either.
|
||||
|
||||
## Reviewing Reddit runs
|
||||
|
||||
Ken's scraper opens (or updates) a pull request from `data/reddit-comments` that changes
|
||||
`data/public/reddit_comments.json`. The **Review Reddit runs** workflow (`.gitea/workflows/reddit-review.yml`) posts a
|
||||
summary on it, refreshed on every push (`python -m collection review-reddit --base <main's file>` locally):
|
||||
|
||||
- runs added, changed and gone since main;
|
||||
- **runs that break a game rule** (`core/spec/form.yaml` `impossible_flags`): left out of the route model and the
|
||||
statistics until corrected. Open the comment and decide;
|
||||
- other flags on the new runs (kept);
|
||||
- runs that may also be in the community form: the same answers within a day, with how many chance alone gives
|
||||
(`analysis/overlap/overlap.py`, `data/public/overlap.json`). Measured only, never removed;
|
||||
- runs whose result page is already a `reddit thread` row (counted once automatically).
|
||||
|
||||
Corrections go in `manual/reddit_comments.yaml`, applied at build time, so they never wait for the threads:
|
||||
`reject: [id]` (not a run), `fix: {id: {question: answer}}` (misread; `null` removes an answer) and
|
||||
`add: [{id, thread, posted, answers}]` (missed). A corrected run loses the "not checked by hand" note. No usernames:
|
||||
the comment id is enough to find the comment again. Then merge, once CI passes.
|
||||
|
||||
## Raw files and privacy
|
||||
|
||||
`data/raw/` is git-ignored and never published. It holds the current raw file of each source and the one before
|
||||
|
||||
@@ -131,6 +131,18 @@ def open_issue(title, body, env=None, f=None):
|
||||
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
|
||||
|
||||
|
||||
def upsert_comment(number, marker, body, env=None, f=None):
|
||||
"""Post `body` on issue or pull request `number`, replacing an earlier comment that starts with `marker`
|
||||
(so a summary refreshed on every push stays one comment)."""
|
||||
f = f or config()
|
||||
tok = token(env)
|
||||
comments = _call("GET", f"{api_base(f)}/issues/{number}/comments?limit=50&per_page=100", tok=tok, f=f) or []
|
||||
mine = [c for c in comments if (c.get("body") or "").startswith(marker)]
|
||||
if mine:
|
||||
return _call("PATCH", f"{api_base(f)}/issues/comments/{mine[-1]['id']}", {"body": body}, tok, f)["id"]
|
||||
return _call("POST", f"{api_base(f)}/issues/{number}/comments", {"body": body}, tok, f)["id"]
|
||||
|
||||
|
||||
def ensure_label(name, color, env=None, f=None):
|
||||
f = f or config()
|
||||
tok = token(env)
|
||||
|
||||
@@ -88,6 +88,7 @@ def rebuild():
|
||||
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
|
||||
_step("screen tree", [py, "analysis/tree/build_tree.py"])
|
||||
_step("known answers for the bot (guide + screen tree)", [py, "analysis/known/build_known.py"])
|
||||
_step("runs reported twice (form + Reddit)", [py, "analysis/overlap/overlap.py"])
|
||||
try:
|
||||
import scipy # noqa: F401
|
||||
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""A review summary for a Reddit scraper pull request (issue #23).
|
||||
|
||||
python -m collection review-reddit --base BASE.json [--out summary.md] [--pr N]
|
||||
|
||||
Compares the pull request's data/public/reddit_comments.json with `BASE.json` (the same file on main) and reads the
|
||||
runs as built from it (data/public/runs_clean.csv, so run `python -m collection build` first). The summary lists
|
||||
what a reviewer needs to decide on, before merging:
|
||||
|
||||
- runs added, changed (answers differ) and gone;
|
||||
- runs that break a game rule: left out of the route model and the statistics until corrected;
|
||||
- other flags on the new or changed runs (kept, for information);
|
||||
- runs that may also be in the community form (analysis/overlap/overlap.py: same answers within a day);
|
||||
- runs whose result page is already a "reddit thread" row (counted once, automatically).
|
||||
|
||||
Corrections go in collection/manual/reddit_comments.yaml (reject / fix / add; see collection/README.md). With
|
||||
`--pr N` the summary is posted on the pull request, replacing the previous one.
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
|
||||
from .common import ROOT, path, pipeline, read_yaml
|
||||
|
||||
MARKER = "<!-- atlas:reddit-review -->"
|
||||
PROVENANCE = "read from a Reddit comment"
|
||||
|
||||
|
||||
def _overlap():
|
||||
spec = importlib.util.spec_from_file_location("overlap", ROOT / "analysis/overlap/overlap.py")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def link(run_id, thread):
|
||||
cid = str(run_id).split("-")[0]
|
||||
return f"[{run_id}](https://www.reddit.com/r/fourthwing/comments/{thread}/_/{cid}/)"
|
||||
|
||||
|
||||
def diff(base, head):
|
||||
"""(added, changed, gone) comment-run ids between two reddit_comments.json documents."""
|
||||
b = {str(r["id"]): r for r in base.get("runs", [])}
|
||||
h = {str(r["id"]): r for r in head.get("runs", [])}
|
||||
added = sorted(set(h) - set(b))
|
||||
gone = sorted(set(b) - set(h))
|
||||
changed = sorted(k for k in set(h) & set(b) if h[k].get("answers") != b[k].get("answers"))
|
||||
return added, changed, gone
|
||||
|
||||
|
||||
def summary(base, head, runs, known_results=(), overlap=None):
|
||||
"""The Markdown summary. `runs`: typed records of runs_clean.csv (with run_id)."""
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from atlas import runs as runs_io
|
||||
|
||||
added, changed, gone = diff(base, head)
|
||||
focus = set(added) | set(changed)
|
||||
thread = {str(r["id"]): r.get("thread") for r in head.get("runs", [])}
|
||||
result = {str(r["id"]): r.get("result") for r in head.get("runs", [])}
|
||||
rows = {r["run_id"][len("comment-"):]: r for r in runs if r["source"] == "reddit comment"}
|
||||
flags = {k: [f for f in (r.get("data_flags") or "").split("; ") if f and not f.startswith(PROVENANCE)]
|
||||
for k, r in rows.items()}
|
||||
impossible = sorted(k for k in rows if runs_io.impossible(rows[k]))
|
||||
|
||||
L = [MARKER, "## Reddit comment runs: review summary", ""]
|
||||
L.append(f"**{len(head.get('runs', []))} runs** in this file: **{len(added)} new**, {len(changed)} changed, "
|
||||
f"{len(gone)} gone since main. {head.get('unread', 0)} comments mention a bond or a burn but could not "
|
||||
"be read as a run.")
|
||||
L.append("")
|
||||
|
||||
L += ["### Runs that break a game rule", "",
|
||||
"Left out of the route model and the statistics until corrected. Check the comment: `fix` a misread "
|
||||
"answer, or `reject` a run that is not one.", ""]
|
||||
bad = [k for k in impossible if k in focus] + [k for k in impossible if k not in focus]
|
||||
if bad:
|
||||
L += ["| Run | New? | Rule |", "|---|---|---|"]
|
||||
L += [f"| {link(k, thread.get(k) or '')} | {'yes' if k in focus else ''} | "
|
||||
f"{'; '.join(f for f in flags[k] if runs_io.impossible({'data_flags': f}))} |" for k in bad]
|
||||
else:
|
||||
L.append("None.")
|
||||
L.append("")
|
||||
|
||||
other = [k for k in sorted(focus) if k in flags and flags[k] and k not in impossible]
|
||||
L += ["### Other flags on new or changed runs (kept)", ""]
|
||||
L += [f"- {link(k, thread.get(k) or '')}: {'; '.join(flags[k])}" for k in other] or ["None."]
|
||||
L.append("")
|
||||
|
||||
if overlap is not None:
|
||||
twice = {m["comment"][len("comment-"):]: m["form"] for m in overlap.get("matches", [])}
|
||||
hits = [k for k in sorted(focus) if k in twice]
|
||||
L += ["### May also be in the community form", "",
|
||||
f"Same answers as a form row submitted within {overlap['rule']['window_days']} day(s). Over all Reddit "
|
||||
f"runs: {overlap['matched']} of {overlap['comparable']} comparable runs match, about "
|
||||
f"{overlap['expected_by_chance']} by chance, so about {overlap['likely_double']} look reported twice. "
|
||||
"Nothing is removed: identical routes are expected.", ""]
|
||||
L += [f"- {link(k, thread.get(k) or '')}: {', '.join(twice[k][:5])}" for k in hits] or ["None among the new runs."]
|
||||
L.append("")
|
||||
|
||||
counted = [k for k in sorted(focus) if result.get(k) and str(result[k]).upper() in known_results]
|
||||
L += ["### Result pages already counted", "",
|
||||
"Their result page is also a \"reddit thread\" row; the run is counted once, with its route.", ""]
|
||||
L += [f"- {link(k, thread.get(k) or '')}: {result[k]}" for k in counted] or ["None."]
|
||||
L.append("")
|
||||
|
||||
L += ["### How to correct", "",
|
||||
"In `collection/manual/reddit_comments.yaml` (applied at build time, on this branch or later):",
|
||||
"- `reject: [comment id]`: not a run (a joke, a friend's run, two runs told as one);",
|
||||
"- `fix: {comment id: {question: answer}}`: answers read wrong (`null` removes one);",
|
||||
"- `add: [{id, thread, posted, answers}]`: a run the reader missed.",
|
||||
"", "A corrected run loses the \"not checked by hand\" note. Then merge."]
|
||||
return "\n".join(L) + "\n"
|
||||
|
||||
|
||||
def main(base_file, out=None, pr=None):
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from atlas import runs as runs_io
|
||||
|
||||
spec = pipeline()["sources"]["reddit_comments"]
|
||||
base = json.loads(open(base_file, encoding="utf-8").read()) if base_file else {"runs": []}
|
||||
head = json.loads(path(spec["load"]).read_text(encoding="utf-8"))
|
||||
runs = runs_io.read(drop=())
|
||||
thread_spec = pipeline()["sources"].get("reddit_thread") or {}
|
||||
manual = (thread_spec.get("transform") or {}).get("manual")
|
||||
known = {str(r["id"]).upper() for r in (read_yaml(manual)["results"] if manual else [])}
|
||||
text = summary(base, head, runs, known, _overlap().measure(runs))
|
||||
if out:
|
||||
open(out, "w", encoding="utf-8").write(text)
|
||||
else:
|
||||
print(text)
|
||||
if pr:
|
||||
from . import forge
|
||||
forge.upsert_comment(int(pr), MARKER, text)
|
||||
print(f"summary posted on #{pr}")
|
||||
return 0
|
||||
@@ -132,6 +132,10 @@ def main(argv=None):
|
||||
k.add_argument("--out", default="exports")
|
||||
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
||||
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
|
||||
rv = sub.add_parser("review-reddit", help="a review summary for a Reddit scraper pull request (run `build` first)")
|
||||
rv.add_argument("--base", help="data/public/reddit_comments.json as it is on main")
|
||||
rv.add_argument("--out", help="write the summary here (default: print it)")
|
||||
rv.add_argument("--pr", help="also post it on this pull request, replacing the previous summary")
|
||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
||||
@@ -155,6 +159,9 @@ def main(argv=None):
|
||||
if a.cmd == "rebuild":
|
||||
from .etl.handoff import rebuild
|
||||
return rebuild()
|
||||
if a.cmd == "review-reddit":
|
||||
from .etl.review_reddit import main as review
|
||||
return review(a.base, a.out, a.pr)
|
||||
if a.cmd == "send-pdf":
|
||||
from .etl.send_pdf import send
|
||||
return send()
|
||||
|
||||
+12
-2
@@ -126,6 +126,16 @@
|
||||
"sha256": "b169d157e731db23c99f2f8317a9718d31801b7d64336b3d414511ac13f32493",
|
||||
"contentSize": "23174 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "overlap",
|
||||
"name": "overlap",
|
||||
"description": "How many Reddit comment runs may also be in the community form (the same answers within a day), against what chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.",
|
||||
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/overlap.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "04280a52f028334451c505ddc35004d98cc1f4595b2294c3746a09e34454e8b0",
|
||||
"contentSize": "1976 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "reports",
|
||||
@@ -153,8 +163,8 @@
|
||||
"description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).",
|
||||
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/stats.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "89a6246da2495685733e20f2de9897299bd19d469aaeadbd8883972856787ce5",
|
||||
"contentSize": "5151910 B"
|
||||
"sha256": "7431173d9b667d4914d17bbc299d994d14c3104e3e89a1eb75d690c1b21ade6a",
|
||||
"contentSize": "5110214 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
|
||||
@@ -41,6 +41,12 @@ files:
|
||||
description: >-
|
||||
Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day
|
||||
and answers only, no usernames or comment text.
|
||||
overlap:
|
||||
path: data/public/overlap.json
|
||||
format: application/json
|
||||
description: >-
|
||||
How many Reddit comment runs may also be in the community form (the same answers within a day), against what
|
||||
chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.
|
||||
reports:
|
||||
path: data/public/reports.json
|
||||
format: application/json
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"rule": {
|
||||
"min_answers": 4,
|
||||
"window_days": 1,
|
||||
"baseline_shifts_days": [
|
||||
-28,
|
||||
-21,
|
||||
-14,
|
||||
14,
|
||||
21,
|
||||
28
|
||||
]
|
||||
},
|
||||
"comment_runs": 82,
|
||||
"comparable": 36,
|
||||
"matched": 9,
|
||||
"expected_by_chance": 7.3,
|
||||
"likely_double": 1.7,
|
||||
"matches": [
|
||||
{
|
||||
"comment": "comment-pdjr7gm",
|
||||
"form": [
|
||||
"sheet-8",
|
||||
"sheet-38"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pdq6tsw",
|
||||
"form": [
|
||||
"sheet-168",
|
||||
"sheet-190",
|
||||
"sheet-398"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pe0kwol",
|
||||
"form": [
|
||||
"sheet-224"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pe2l6mz",
|
||||
"form": [
|
||||
"sheet-5",
|
||||
"sheet-25",
|
||||
"sheet-107",
|
||||
"sheet-292",
|
||||
"sheet-393",
|
||||
"sheet-471",
|
||||
"sheet-483",
|
||||
"sheet-514",
|
||||
"sheet-520"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pe2n3j1",
|
||||
"form": [
|
||||
"sheet-195",
|
||||
"sheet-276"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pev3uge",
|
||||
"form": [
|
||||
"sheet-870",
|
||||
"sheet-916",
|
||||
"sheet-934",
|
||||
"sheet-960",
|
||||
"sheet-961",
|
||||
"sheet-962",
|
||||
"sheet-975",
|
||||
"sheet-984",
|
||||
"sheet-999",
|
||||
"sheet-1008",
|
||||
"sheet-1009",
|
||||
"sheet-1019",
|
||||
"sheet-1023",
|
||||
"sheet-1068",
|
||||
"sheet-1069",
|
||||
"sheet-1077",
|
||||
"sheet-1081",
|
||||
"sheet-1089",
|
||||
"sheet-1097",
|
||||
"sheet-1101",
|
||||
"sheet-1110",
|
||||
"sheet-1129",
|
||||
"sheet-1194",
|
||||
"sheet-1200",
|
||||
"sheet-1201",
|
||||
"sheet-1235",
|
||||
"sheet-1238",
|
||||
"sheet-1257",
|
||||
"sheet-1259",
|
||||
"sheet-1264",
|
||||
"sheet-1271",
|
||||
"sheet-1278",
|
||||
"sheet-1283",
|
||||
"sheet-1290",
|
||||
"sheet-1292"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pf22tpj",
|
||||
"form": [
|
||||
"sheet-1031",
|
||||
"sheet-1042",
|
||||
"sheet-1048",
|
||||
"sheet-1140",
|
||||
"sheet-1142",
|
||||
"sheet-1143",
|
||||
"sheet-1144",
|
||||
"sheet-1171",
|
||||
"sheet-1261",
|
||||
"sheet-1262",
|
||||
"sheet-1303",
|
||||
"sheet-1304",
|
||||
"sheet-1364"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pfhmw5d",
|
||||
"form": [
|
||||
"sheet-1504"
|
||||
]
|
||||
},
|
||||
{
|
||||
"comment": "comment-pgzz9hc",
|
||||
"form": [
|
||||
"sheet-1714"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+2
-2
@@ -6,11 +6,11 @@
|
||||
"scripts": {
|
||||
"build": "node site/build.js",
|
||||
"dev": "node site/build.js && python3 -m http.server 8000 -d dist",
|
||||
"test:py": "python3 -m collection build --check && python3 planner/build_experiments.py --check && python3 analysis/black/evidence.py --check && python3 analysis/tree/build_tree.py --check && python3 analysis/known/build_known.py --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'",
|
||||
"test:py": "python3 -m collection build --check && python3 planner/build_experiments.py --check && python3 analysis/black/evidence.py --check && python3 analysis/tree/build_tree.py --check && python3 analysis/known/build_known.py --check && python3 analysis/overlap/overlap.py --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'",
|
||||
"test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js tests/core/known.test.js",
|
||||
"test": "npm run test:py && npm run test:unit",
|
||||
"graph": "node analysis/network/build-graph.js",
|
||||
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py && python3 analysis/tree/build_tree.py && python3 analysis/known/build_known.py",
|
||||
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py && python3 analysis/tree/build_tree.py && python3 analysis/known/build_known.py && python3 analysis/overlap/overlap.py",
|
||||
"cycle": "python3 -m collection cycle",
|
||||
"stats": "python3 analysis/stats/site_stats.py",
|
||||
"collect": "python3 -m collection update",
|
||||
|
||||
@@ -84,6 +84,7 @@
|
||||
// ------------------------------------------------------------ data loading
|
||||
async function loadStats() {
|
||||
state.stats = await (await fetch("data/stats.json")).json();
|
||||
try { state.overlap = await (await fetch("data/overlap.json")).json(); } catch { state.overlap = null; }
|
||||
const sel = $("#f-preset");
|
||||
sel.innerHTML = Object.entries(META().presets).map(([k, l]) => `<option value="${k}">${esc(l)}</option>`).join("");
|
||||
sel.value = state.preset;
|
||||
@@ -554,6 +555,7 @@
|
||||
</tbody></table></div><p class="small">Bulk logs come from players who recorded every run, so they are the least selective rows. Their rare-dragon rates are several times lower than the form's, which is the strongest sign of reporting bias in this data.</p></div>
|
||||
<div class="panel"><h3>Rows flagged for contradictions</h3><p class="small muted">${m.n_flagged} of ${r.n} rows (${pct(m.n_flagged / r.n)}). Flagged rows stay in unless you pick "Rows without data-quality flags" above.</p><div class="tbl-wrap"><table><tbody>${flags.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td></tr>`).join("") || `<tr><td>None</td></tr>`}</tbody></table></div></div>
|
||||
<div class="panel"><h3>How runs ended</h3><div class="tbl-wrap"><table><tbody>${outs.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td><td class="num">${pct(n / r.n)}</td></tr>`).join("")}</tbody></table></div><p class="small muted">"No outcome recorded" rows reached the hour but the form shows no Bonded or Incinerated. They count toward reaching a branch but not toward bond rates.</p></div>
|
||||
${state.overlap && state.overlap.comparable ? `<div class="panel"><h3>The same run reported twice?</h3><p class="small">A rider may log a run in the form and post it in a megathread comment too. With no player IDs, a copy can only be measured, not proven, so nothing is removed. Of ${state.overlap.comparable} Reddit runs detailed enough to compare, ${state.overlap.matched} give the same answers as a form row sent within ${state.overlap.rule.window_days} day; chance alone (the same test a few weeks off) gives about ${state.overlap.expected_by_chance}. So about ${Math.round(state.overlap.likely_double)} of ${state.overlap.comment_runs} Reddit runs look reported twice${state.overlap.likely_double < 0.1 * state.overlap.comment_runs ? ", too few to matter" : ""}.</p></div>` : ""}
|
||||
<div class="panel"><h3>Independence</h3><p class="small">The form has no player ID. One player's 50 runs count as 50 independent runs, so intervals are somewhat narrower than they should be. ${pct(Object.entries(m.sources).filter(([k]) => !["Form", "Reddit comments", "DragonKind bot", "Reported on the Atlas"].includes(k)).reduce((s, [, n]) => s + n, 0) / r.n, 0)} of rows arrived as bulk imports, each probably one player's or one group's log.</p></div>
|
||||
<div class="panel"><h3>What the form does not ask</h3><p class="small">The redesigned form asks how you reacted to the first dragon (answered in ${r.factors.reach_hour.rows.filter((x) => x.factor === "sighting_action" && !["Unknown", "(not asked)"].includes(x.level)).reduce((a, x) => a + x.n, 0)} rows; most riders answer "Unsure") and how the bully fight went (${r.factors.reach_hour.rows.filter((x) => x.factor === "combat_first" && x.level !== "(not recorded)").reduce((a, x) => a + x.n, 0)} rows). It still does not record most choices inside the bonding encounters. Theories that rely on those choices cannot be confirmed or refuted here.</p></div>`;
|
||||
}
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ for (const f of fs.readdirSync(path.join(root, "site/atlas"))) copy(`site/atlas/
|
||||
copy("core/graph.js", "graph.js");
|
||||
copy("core/solver.js", "solver.js");
|
||||
// The numbers the page shows (all computed in Python) and the published data
|
||||
for (const f of ["stats.json", "register.json", "graph.json", "policy.json", "black.json", "tree.json", "experiments.json", "pdf_encounters.json"]) copy(`data/public/${f}`, `data/${f}`);
|
||||
for (const f of ["stats.json", "register.json", "graph.json", "policy.json", "black.json", "tree.json", "experiments.json", "pdf_encounters.json", "overlap.json"]) copy(`data/public/${f}`, `data/${f}`);
|
||||
copy("data/public/runs_clean.csv", "data/runs_clean.csv");
|
||||
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
|
||||
// Source files shown in the page's "view the code" panel
|
||||
|
||||
@@ -335,6 +335,30 @@ class RedditComments(unittest.TestCase):
|
||||
kept = build_policy.drop_flagged(rows)
|
||||
self.assertEqual([r["source"] for r in kept], ["reddit comment", "reddit spreadsheet"])
|
||||
|
||||
def test_review_summary_and_overlap(self):
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from collection.etl import review_reddit
|
||||
ov = review_reddit._overlap()
|
||||
base = {"runs": [{"id": "a1", "thread": "t", "answers": {"outcome": "Bonded"}}]}
|
||||
head = {"runs": [{"id": "a1", "thread": "t", "answers": {"outcome": "Incinerated at a bonding encounter"}},
|
||||
{"id": "b2", "thread": "t", "answers": {}}, {"id": "c3", "thread": "t", "answers": {}, "result": "THRESH-X1"}]}
|
||||
self.assertEqual(review_reddit.diff(base, head), (["b2", "c3"], ["a1"], []))
|
||||
route = dict(company="Alone", direction="East", scream="Ignore the scream", hour="Tree", outcome="Bonded",
|
||||
outcome_color="Red", date="2026-10-05")
|
||||
runs = [{"run_id": "comment-b2", "source": "reddit comment", "data_flags": "read from a Reddit comment; "
|
||||
"Black recorded on a Valley run (the Black is only reached from the tree)", **{k: None for k in ov.FIELDS}, "date": None},
|
||||
{"run_id": "comment-c3", "source": "reddit comment", "data_flags": "", **{k: None for k in ov.FIELDS}, **route},
|
||||
{"run_id": "sheet-1", "source": "reddit spreadsheet", "data_flags": "", **{k: None for k in ov.FIELDS}, **route}]
|
||||
m = ov.measure(runs)
|
||||
self.assertEqual((m["comparable"], m["matched"]), (1, 1))
|
||||
self.assertEqual(m["matches"], [{"comment": "comment-c3", "form": ["sheet-1"]}])
|
||||
text = review_reddit.summary(base, head, runs, {"THRESH-X1"}, m)
|
||||
self.assertIn("**2 new**, 1 changed", text)
|
||||
self.assertIn("Black recorded on a Valley run", text)
|
||||
self.assertIn("sheet-1", text)
|
||||
self.assertIn("THRESH-X1", text)
|
||||
self.assertTrue(text.startswith(review_reddit.MARKER))
|
||||
|
||||
def test_delivered_not_extracted(self):
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
|
||||
with self.assertRaises(extract.Skip):
|
||||
|
||||
Reference in New Issue
Block a user