diff --git a/README.md b/README.md index afff20a..d4c934e 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,12 @@ Blocking→high, Minor→small): `penalty × points[claimed]`. So a Blocking-cla `high(8) × -0.5 = -4`, and a model with the odd good find but many false positives nets *down* — even negative — instead of coasting on its hits. +And an editable **solo-find bonus ×** (default `1.5`). Because findings are content-addressed, the +number of models that reported one is known, so a confirmed finding that **only that model** caught +(no other model reported it) scores `severity × bonus` — rewarding catching what the swarm missed. +The `solo` column counts those. This is derived from the data (reporter count); the grader never has +to flag it. Set the bonus to `1` to disable. + Auth: the `/ui` shell is public (it holds no data); paste the store token into its **connect** box, or open `/ui?token=` once (remembered in `localStorage`). Prefer your own dashboard? Point Grafana/Metabase/etc. at the SQLite file or the same `/export` + `/scoreboard` + `/runs` JSON. diff --git a/ui.html b/ui.html index 43f7d70..e114f46 100644 --- a/ui.html +++ b/ui.html @@ -80,6 +80,7 @@ high critical false-positive penalty × + solo-find bonus × @@ -166,6 +167,7 @@ function curve(){ return c; } function fpMult(){ const v = parseFloat(document.getElementById("fp_mult").value); return isNaN(v) ? 0 : v; } +function soloBonus(){ const v = parseFloat(document.getElementById("solo_bonus").value); return isNaN(v) ? 1 : v; } // A false positive has no graded severity, so penalize it by the severity the // MODEL claimed — its lens verdict (raw_severity) — mapped onto the curve. The // louder the wrong cry, the bigger the penalty. @@ -214,9 +216,13 @@ function rowMatch(row, f){ function aggregate(f){ const c = curve(); + // GLOBAL reporter set per finding (ignores filters) — a finding is "solo" when + // exactly one model ever reported it, so the model filter can't fake solo-ness. + const reporters = new Map(); + for (const r of ROWS){ if(!reporters.has(r.finding_id)) reporters.set(r.finding_id, new Set()); reporters.get(r.finding_id).add(r.model); } const M = new Map(); const get = m => { if(!M.has(m)) M.set(m, {model:m, provider:"", runs:0, minutes:0, inTok:0, outTok:0, - findings:new Set(), confirmed:new Set(), fp:new Map(), ungraded:new Set(), sev:Object.fromEntries(SEVS.map(s=>[s,new Set()]))}); return M.get(m); }; + findings:new Set(), confirmed:new Map(), fp:new Map(), ungraded:new Set()}); return M.get(m); }; for (const r of RUNS){ if(!runMatch(r,f)) continue; const m=get(r.model); m.runs++; m.minutes += (r.duration_secs||0)/60; m.inTok += r.input_tokens||0; m.outTok += r.output_tokens||0; if(r.provider) m.provider=r.provider; } @@ -224,20 +230,26 @@ function aggregate(f){ const rows = ROWS.filter(r => rowMatch(r, f)); for (const r of rows){ const m=get(r.model); if(r.provider) m.provider=m.provider||r.provider; m.findings.add(r.finding_id); - if (r.graded && r.is_real === true){ m.confirmed.add(r.finding_id); if (r.severity) m.sev[r.severity].add(r.finding_id); } + if (r.graded && r.is_real === true){ m.confirmed.set(r.finding_id, r.severity || ""); } else if (r.graded && r.is_real === false){ m.fp.set(r.finding_id, rawToSevKey(r.raw_severity)); } else { m.ungraded.add(r.finding_id); } } - const fpm = fpMult(); + const fpm = fpMult(), sb = soloBonus(); const out = [...M.values()].map(m => { - const sevCounts = Object.fromEntries(SEVS.map(s=>[s, m.sev[s].size])); - const confirmedPoints = SEVS.reduce((a,s)=> a + c[s]*sevCounts[s], 0); + const sevCounts = Object.fromEntries(SEVS.map(s=>[s,0])); + let confirmedPoints = 0, solo = 0; + for (const [fid, sevv] of m.confirmed){ + if (sevCounts[sevv] !== undefined) sevCounts[sevv]++; + const isSolo = (reporters.get(fid)?.size || 1) === 1; // only this model ever reported it + if (isSolo) solo++; + confirmedPoints += (c[sevv] || 0) * (isSolo ? sb : 1); + } let fpPen = 0; for (const k of m.fp.values()) fpPen += (c[k]||0) * fpm; // negative when fpm<0 - const points = confirmedPoints + fpPen; // NET of the false-positive penalty + const points = confirmedPoints + fpPen; // NET: solo-boosted confirmed + FP penalty const findings = m.findings.size, confirmed = m.confirmed.size; return { model:m.model, provider:m.provider, runs:m.runs, minutes:m.minutes, - inTok:m.inTok, outTok:m.outTok, findings, confirmed, fp:m.fp.size, ungraded:m.ungraded.size, + inTok:m.inTok, outTok:m.outTok, findings, confirmed, solo, fp:m.fp.size, ungraded:m.ungraded.size, sev:sevCounts, confirmedPoints, fpPen, points, ptsPerMin: m.minutes>0 ? points/m.minutes : null, ptsPerRun: m.runs>0 ? points/m.runs : null, @@ -249,7 +261,7 @@ function aggregate(f){ const COLS = [ {k:"model", t:"model", l:true}, {k:"provider", t:"provider", l:true}, {k:"runs", t:"runs"}, {k:"minutes", t:"min", fmt:v=>v.toFixed(1)}, - {k:"findings", t:"findings"}, {k:"confirmed", t:"real"}, {k:"fp", t:"FP"}, {k:"ungraded", t:"ungr"}, + {k:"findings", t:"findings"}, {k:"confirmed", t:"real"}, {k:"solo", t:"solo"}, {k:"fp", t:"FP"}, {k:"ungraded", t:"ungr"}, {k:"confirmedPct", t:"real%", fmt:v=>v==null?"—":v.toFixed(0)+"%"}, {k:"fpPen", t:"fp pen", fmt:v=>v?v.toFixed(1):"0"}, {k:"points", t:"points (net)", fmt:v=>v.toFixed(0)}, @@ -286,6 +298,7 @@ function render(){ td.innerHTML = col.fmt ? col.fmt(v) : (v==null?"—":v); if ((col.k==="ptsPerMin" || col.k==="ptsPerRun" || col.k==="points") && v!=null) td.classList.add(v<0 ? "bad" : "good"); if (col.k==="fpPen" && v<0) td.classList.add("bad"); + if (col.k==="solo" && v>0) td.classList.add("good"); if (col.k==="fp" && v>0) td.classList.add("warn"); tr.appendChild(td); }