"""The statistics behind every number on the page: Wilson intervals, odds ratios with Fisher's test, Benjamini-Hochberg q-values, Mantel-Haenszel, ridge logistic regression. Tested against known answers in tests/core/test_stats.py.""" import math import numpy as np from scipy.stats import fisher_exact, norm Z95 = 1.959963984540054 def wilson(k, n, z=Z95): """Wilson score 95% interval for a proportion k/n. Returns (p, lo, hi); None when n == 0.""" if n == 0: return (None, None, None) p = k / n d = 1 + z * z / n c = (p + z * z / (2 * n)) / d h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d return (p, max(0.0, c - h), min(1.0, c + h)) def odds_ratio(a, b, c, d): """2x2 table [[a, b], [c, d]] = [[exposed&event, exposed&no-event], [unexposed&event, unexposed&no-event]]. Returns OR, 95% CI (Woolf log method with Haldane-Anscombe +0.5 correction when any cell is 0), and the two-sided Fisher exact p-value (computed on the uncorrected table).""" corr = 0.5 if 0 in (a, b, c, d) else 0.0 A, B, C, D = a + corr, b + corr, c + corr, d + corr or_ = (A * D) / (B * C) se = math.sqrt(1 / A + 1 / B + 1 / C + 1 / D) lo, hi = math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se) p = fisher_exact([[a, b], [c, d]], alternative="two-sided")[1] return or_, lo, hi, p, corr > 0 def bh(pvals): """Benjamini-Hochberg false-discovery-rate adjusted q-values.""" m = len(pvals) if m == 0: return [] order = sorted(range(m), key=lambda i: pvals[i]) q = [0.0] * m prev = 1.0 for rank in range(m, 0, -1): i = order[rank - 1] prev = min(prev, pvals[i] * m / rank) q[i] = prev return q def logistic_ridge(X, y, lam=1.0, iters=100, tol=1e-10): """Logistic regression fitted by Newton-Raphson (IRLS) with a small L2 penalty `lam` on every coefficient except the intercept (column 0). The penalty keeps estimates finite when a category has 0 events (complete separation). Returns beta and Wald standard errors from the penalised information matrix.""" X = np.asarray(X, float) y = np.asarray(y, float) n, k = X.shape beta = np.zeros(k) P = np.eye(k) * lam P[0, 0] = 0 for _ in range(iters): eta = X @ beta mu = 1 / (1 + np.exp(-eta)) W = mu * (1 - mu) grad = X.T @ (y - mu) - P @ beta H = (X * W[:, None]).T @ X + P step = np.linalg.solve(H, grad) beta += step if np.max(np.abs(step)) < tol: break mu = 1 / (1 + np.exp(-(X @ beta))) H = (X * (mu * (1 - mu))[:, None]).T @ X + P se = np.sqrt(np.diag(np.linalg.inv(H))) return beta, se def p_from_z(z): return 2 * (1 - norm.cdf(abs(z))) def mantel_haenszel(tables): """Mantel-Haenszel common OR over 2x2 tables [a, b, c, d], Robins-Breslow-Greenland 95% CI, and the CMH chi-square p-value (1 df, continuity-corrected).""" R = S = PR = PSQR = QS = sA = sE = sV = 0.0 for a, b, c, d in tables: n = a + b + c + d if n < 2: continue r, s_ = a * d / n, b * c / n P, Q = (a + d) / n, (b + c) / n R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_ m1, m0, n1, n0 = a + b, c + d, a + c, b + d sA += a; sE += m1 * n1 / n; sV += m1 * m0 * n1 * n0 / (n * n * (n - 1)) if not R or not S: return None or_ = R / S se = math.sqrt(PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S)) chi = max(0.0, abs(sA - sE) - 0.5) ** 2 / sV if sV > 0 else 0.0 return or_, math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se), p_from_z(math.sqrt(chi))