Moves only; no behaviour changes. Every file now lives under the part of the project it belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow. core/ engine.js, graph.js, solver.js, python/clean.py collection/ sources/ (form, register, PDF check, daily update), bot/ (importer) analysis/ stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md) planner/ build-policy.js, policy-report.js site/ atlas/ (the page), build.js, qa/ tests/ core/ (unit), parity/ (JS vs Python), fixtures/ Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and policy.json are byte-identical; the built site differs only in comments that name file paths. The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged. Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now default to a temp folder) and test.txt, and drops the unused npm workspaces. Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
99 lines
3.5 KiB
Python
99 lines
3.5 KiB
Python
"""Small, dependency-light statistics used by the analysis (mirrored in core/engine.js)."""
|
|
import math
|
|
|
|
import numpy as np
|
|
from scipy.stats import fisher_exact, norm
|
|
|
|
Z95 = 1.959963984540054
|
|
|
|
|
|
def wilson(k, n, z=Z95):
|
|
"""Wilson score 95% interval for a proportion k/n. Returns (p, lo, hi); None when n == 0."""
|
|
if n == 0:
|
|
return (None, None, None)
|
|
p = k / n
|
|
d = 1 + z * z / n
|
|
c = (p + z * z / (2 * n)) / d
|
|
h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
|
|
return (p, max(0.0, c - h), min(1.0, c + h))
|
|
|
|
|
|
def odds_ratio(a, b, c, d):
|
|
"""2x2 table [[a, b], [c, d]] = [[exposed&event, exposed&no-event], [unexposed&event, unexposed&no-event]].
|
|
Returns OR, 95% CI (Woolf log method with Haldane-Anscombe +0.5 correction when any cell is 0),
|
|
and the two-sided Fisher exact p-value (computed on the uncorrected table)."""
|
|
corr = 0.5 if 0 in (a, b, c, d) else 0.0
|
|
A, B, C, D = a + corr, b + corr, c + corr, d + corr
|
|
or_ = (A * D) / (B * C)
|
|
se = math.sqrt(1 / A + 1 / B + 1 / C + 1 / D)
|
|
lo, hi = math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se)
|
|
p = fisher_exact([[a, b], [c, d]], alternative="two-sided")[1]
|
|
return or_, lo, hi, p, corr > 0
|
|
|
|
|
|
def bh(pvals):
|
|
"""Benjamini-Hochberg false-discovery-rate adjusted q-values."""
|
|
m = len(pvals)
|
|
if m == 0:
|
|
return []
|
|
order = sorted(range(m), key=lambda i: pvals[i])
|
|
q = [0.0] * m
|
|
prev = 1.0
|
|
for rank in range(m, 0, -1):
|
|
i = order[rank - 1]
|
|
prev = min(prev, pvals[i] * m / rank)
|
|
q[i] = prev
|
|
return q
|
|
|
|
|
|
def logistic_ridge(X, y, lam=1.0, iters=100, tol=1e-10):
|
|
"""Logistic regression fitted by Newton-Raphson (IRLS) with a small L2 penalty `lam`
|
|
on every coefficient except the intercept (column 0). The penalty keeps estimates finite
|
|
when a category has 0 events (complete separation). Returns beta and Wald standard errors
|
|
from the penalised information matrix."""
|
|
X = np.asarray(X, float)
|
|
y = np.asarray(y, float)
|
|
n, k = X.shape
|
|
beta = np.zeros(k)
|
|
P = np.eye(k) * lam
|
|
P[0, 0] = 0
|
|
for _ in range(iters):
|
|
eta = X @ beta
|
|
mu = 1 / (1 + np.exp(-eta))
|
|
W = mu * (1 - mu)
|
|
grad = X.T @ (y - mu) - P @ beta
|
|
H = (X * W[:, None]).T @ X + P
|
|
step = np.linalg.solve(H, grad)
|
|
beta += step
|
|
if np.max(np.abs(step)) < tol:
|
|
break
|
|
mu = 1 / (1 + np.exp(-(X @ beta)))
|
|
H = (X * (mu * (1 - mu))[:, None]).T @ X + P
|
|
se = np.sqrt(np.diag(np.linalg.inv(H)))
|
|
return beta, se
|
|
|
|
|
|
def p_from_z(z):
|
|
return 2 * (1 - norm.cdf(abs(z)))
|
|
|
|
|
|
def mantel_haenszel(tables):
|
|
"""Mantel-Haenszel common OR over 2x2 tables [a, b, c, d], Robins-Breslow-Greenland 95% CI,
|
|
and the CMH chi-square p-value (1 df, continuity-corrected). Mirrors mantelHaenszel() in engine.js."""
|
|
R = S = PR = PSQR = QS = sA = sE = sV = 0.0
|
|
for a, b, c, d in tables:
|
|
n = a + b + c + d
|
|
if n < 2:
|
|
continue
|
|
r, s_ = a * d / n, b * c / n
|
|
P, Q = (a + d) / n, (b + c) / n
|
|
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_
|
|
m1, m0, n1, n0 = a + b, c + d, a + c, b + d
|
|
sA += a; sE += m1 * n1 / n; sV += m1 * m0 * n1 * n0 / (n * n * (n - 1))
|
|
if not R or not S:
|
|
return None
|
|
or_ = R / S
|
|
se = math.sqrt(PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S))
|
|
chi = max(0.0, abs(sA - sE) - 0.5) ** 2 / sV if sV > 0 else 0.0
|
|
return or_, math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se), p_from_z(math.sqrt(chi))
|