163 lines
7.8 KiB
Python
163 lines
7.8 KiB
Python
"""`bingham eval`: how well the screen finds what the labeled set says is relevant.
|
|
|
|
Recall comes first, since a screen's costly error is a missed paper. Labels
|
|
from random sampling (bingham label) are weighted by band: each labeled
|
|
paper stands for N_b / n_b papers of its band in the store (post-
|
|
stratification), so estimates cover the whole stream, including papers the
|
|
re-ranker never saw. Intervals are 90% bootstrap intervals, resampling within
|
|
bands. Near-miss labels (bingham label --near-misses) are not random and are
|
|
reported separately.
|
|
|
|
For each profile:
|
|
recall, precision at the current threshold, weighted
|
|
where misses happen never re-ranked (not nominated by cosine or BM25) or
|
|
re-ranked below the threshold; the missed papers listed
|
|
candidate recall share of relevant papers the nomination stage passed on
|
|
MRR per fetch day, reciprocal rank of the first labeled-
|
|
relevant paper among all that day's papers by re-rank
|
|
score (unlabeled papers count as not relevant)
|
|
thresholds weighted recall and precision on a grid, and the highest
|
|
threshold that keeps recall at eval.target_recall
|
|
"""
|
|
|
|
import random
|
|
from collections import defaultdict
|
|
|
|
from art import env, store
|
|
from art.label import BANDS, load_labels, population
|
|
from art.profile import profiles as prof
|
|
|
|
|
|
def _rates(rows, t):
|
|
"""Weighted (recall, precision) at threshold t for rows of (weight, relevant, rerank)."""
|
|
tp = sum(w for w, rel, r in rows if rel and r is not None and r >= t)
|
|
fn = sum(w for w, rel, r in rows if rel and (r is None or r < t))
|
|
fp = sum(w for w, rel, r in rows if not rel and r is not None and r >= t)
|
|
recall = tp / (tp + fn) if tp + fn else None
|
|
precision = tp / (tp + fp) if tp + fp else None
|
|
return recall, precision
|
|
|
|
|
|
def _interval(by_band, t, weights, reps=500, seed=0):
|
|
"""90% bootstrap interval for recall, resampling labeled papers within each band."""
|
|
rng = random.Random(seed)
|
|
vals = []
|
|
for _ in range(reps):
|
|
rows = []
|
|
for b, items in by_band.items():
|
|
rows += [(weights[b],) + rng.choice(items)[1:] for _ in items]
|
|
r, _ = _rates(rows, t)
|
|
if r is not None:
|
|
vals.append(r)
|
|
if len(vals) < 20:
|
|
return None
|
|
vals.sort()
|
|
return vals[int(0.05 * len(vals))], vals[int(0.95 * len(vals)) - 1]
|
|
|
|
|
|
def _fmt(x):
|
|
return " n/a" if x is None else f"{x:5.2f}"
|
|
|
|
|
|
def run(conn, target: float | None = None) -> None:
|
|
cfg = env.config()
|
|
th = cfg["screen"]["thresholds"]
|
|
target = target or cfg.get("eval", {}).get("target_recall", 0.9)
|
|
profiles = prof.load().profiles
|
|
pop = population(conn, profiles)
|
|
labels = {i: r for i, r in load_labels().items() if i in pop}
|
|
rand = {i: r for i, r in labels.items() if not r.get("near_miss")}
|
|
if not labels:
|
|
print("no labels yet; run `bingham label` first")
|
|
return
|
|
|
|
N = defaultdict(int)
|
|
for v in pop.values():
|
|
N[v["band"]] += 1
|
|
n = defaultdict(int)
|
|
for i in rand:
|
|
n[pop[i]["band"]] += 1
|
|
weights = {b: N[b] / n[b] for b in BANDS if n[b]}
|
|
print(f"{len(labels)} labeled papers ({len(rand)} random, {len(labels) - len(rand)} near misses); "
|
|
f"{len(pop)} screened papers in the store.")
|
|
print("band papers labeled weight")
|
|
for b in BANDS:
|
|
print(f" {b:8s} {N[b]:8d} {n[b]:8d} {weights.get(b, 0):7.1f}")
|
|
missing = [b for b in BANDS if N[b] and not n[b]]
|
|
if missing:
|
|
print(f" no labels yet in band(s) {', '.join(missing)}: their papers are not represented")
|
|
|
|
days = defaultdict(list) # fetch day -> item ids, for MRR
|
|
fetched = dict(conn.execute(f"SELECT id, substr(fetched_at, 1, 10) FROM items WHERE {store.SCREENABLE}"))
|
|
for i in pop:
|
|
days[fetched.get(i)].append(i)
|
|
|
|
for p in profiles:
|
|
t = th.get(p.id, th["default"])
|
|
shown = {i: r for i, r in rand.items() if p.id in r.get("profiles", [])}
|
|
rows_by_band = defaultdict(list)
|
|
for i, r in shown.items():
|
|
s = pop[i]["scores"].get(p.id, {})
|
|
rows_by_band[pop[i]["band"]].append((i, p.id in r["relevant"], s.get("rerank")))
|
|
rows = [(weights[b], rel, rr) for b, items in rows_by_band.items() for _, rel, rr in items]
|
|
n_rel = sum(rel for _, rel, _ in rows)
|
|
print(f"\n== {p.id} (threshold {t}; {len(rows)} labeled, {n_rel} relevant)")
|
|
if not n_rel:
|
|
print(" no relevant papers labeled yet")
|
|
continue
|
|
recall, precision = _rates(rows, t)
|
|
ci = _interval(rows_by_band, t, weights) if n_rel >= 3 else None
|
|
print(f" recall {_fmt(recall)}" + (f" (90% interval {ci[0]:.2f}-{ci[1]:.2f})" if ci else "")
|
|
+ f" precision {_fmt(precision)}")
|
|
|
|
rel_items = [(i, pop[i]["scores"].get(p.id, {})) for i, r in shown.items() if p.id in r["relevant"]]
|
|
nominated = sum(weights[pop[i]["band"]] for i, s in rel_items if s.get("rerank") is not None)
|
|
total = sum(weights[pop[i]["band"]] for i, _ in rel_items)
|
|
print(f" candidate recall {nominated / total:.2f} (relevant papers that reached the re-ranker)")
|
|
by = defaultdict(int)
|
|
for _, s in rel_items:
|
|
by[s.get("nominated_by") or "not nominated"] += 1
|
|
print(" relevant papers nominated by: " + ", ".join(f"{k} {v}" for k, v in sorted(by.items())))
|
|
|
|
misses = [(i, s) for i, s in rel_items if s.get("rerank") is None or s["rerank"] < t]
|
|
if misses:
|
|
print(" missed:")
|
|
for i, s in sorted(misses, key=lambda m: -(m[1].get("rerank") or -1)):
|
|
item = conn.execute("SELECT title, source FROM items WHERE id = ?", (i,)).fetchone()
|
|
where = "never re-ranked" if s.get("rerank") is None else f"re-rank {s['rerank']:.2f}"
|
|
print(f" {where:16s} cosine {s.get('cosine', 0):.2f} {item['source']:10s} {item['title'][:80]}")
|
|
|
|
rr = []
|
|
for day, ids in days.items():
|
|
relevant_today = {i for i in ids if i in shown and p.id in shown[i]["relevant"]}
|
|
if not relevant_today:
|
|
continue
|
|
ranked = sorted(ids, key=lambda i: (-(pop[i]["scores"].get(p.id, {}).get("rerank") or -1),
|
|
-pop[i]["scores"].get(p.id, {}).get("cosine", 0)))
|
|
rr.append(1 / (1 + next(k for k, i in enumerate(ranked) if i in relevant_today)))
|
|
if rr:
|
|
print(f" MRR {sum(rr) / len(rr):.2f} over {len(rr)} fetch days with a labeled relevant paper")
|
|
|
|
grid = [round(0.05 * k, 2) for k in range(1, 13)]
|
|
rates = [(g,) + _rates(rows, g) for g in grid]
|
|
print(" threshold " + " ".join(f"{g:5.2f}" for g in grid))
|
|
print(" recall " + " ".join(_fmt(r) for _, r, _ in rates))
|
|
print(" precision " + " ".join(_fmt(pr) for _, _, pr in rates))
|
|
ok = [g for g, r, _ in rates if r is not None and r >= target]
|
|
if n_rel < 5:
|
|
print(f" too few relevant labels ({n_rel}) to suggest a threshold")
|
|
elif ok:
|
|
print(f" highest threshold with recall >= {target}: {max(ok)}")
|
|
else:
|
|
print(f" no threshold on the grid reaches recall {target}")
|
|
|
|
near = {i: r for i, r in labels.items() if r.get("near_miss")}
|
|
if near:
|
|
print("\nnear misses labeled (scored just below a threshold):")
|
|
for p in profiles:
|
|
t = th.get(p.id, th["default"])
|
|
cand = [i for i in near if (s := pop[i]["scores"].get(p.id, {})).get("rerank") is not None
|
|
and s["rerank"] < t and p.id in near[i].get("profiles", [])]
|
|
if cand:
|
|
k = sum(p.id in near[i]["relevant"] for i in cand)
|
|
print(f" {p.id:24s} {k} of {len(cand)} relevant")
|