From 0cb6b25f11718892d6a2f8223ea476cc141937ad Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Sat, 27 Jun 2026 09:50:18 -0400 Subject: [PATCH] feat(ui): false-positive penalty (severity-scaled, default -0.5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an editable 'false-positive penalty ×' to the dashboard. A false positive carries no graded severity, so it's penalized by the severity the model CLAIMED (its lens verdict / raw_severity, mapped onto the curve: Blocking->high, Minor->small). points(net) = confirmed points + Σ penalty×points[claimed], so a model with a few good finds but many false positives nets down — even negative — and sorts to the bottom. Adds an 'fp pen' column; net points/pts-min/pts-run shown red when negative. Client-side only; the store stays point-free. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 6 ++++++ ui.html | 31 ++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 259ab1e..afff20a 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,12 @@ points curve (default `trivial=1, small=3, medium=5, high=8, critical=20`) and c `points = Σ weight[severity]·count` and `value/min = points / minutes` on the fly — retune it without touching stored data. +There's also an editable **false-positive penalty ×** (default `-0.5`). A false positive has no +graded severity, so it's penalized by the severity the model **claimed** (its lens verdict — +Blocking→high, Minor→small): `penalty × points[claimed]`. So a Blocking-claimed FP at `-0.5` costs +`high(8) × -0.5 = -4`, and a model with the odd good find but many false positives nets *down* — +even negative — instead of coasting on its hits. + Auth: the `/ui` shell is public (it holds no data); paste the store token into its **connect** box, or open `/ui?token=` once (remembered in `localStorage`). Prefer your own dashboard? Point Grafana/Metabase/etc. at the SQLite file or the same `/export` + `/scoreboard` + `/runs` JSON. diff --git a/ui.html b/ui.html index 08d840c..43f7d70 100644 --- a/ui.html +++ b/ui.html @@ -79,6 +79,7 @@ medium high critical + false-positive penalty × @@ -164,6 +165,17 @@ function curve(){ for (const s of SEVS) c[s] = parseFloat(document.getElementById("p_"+s).value) || 0; return c; } +function fpMult(){ const v = parseFloat(document.getElementById("fp_mult").value); return isNaN(v) ? 0 : v; } +// A false positive has no graded severity, so penalize it by the severity the +// MODEL claimed — its lens verdict (raw_severity) — mapped onto the curve. The +// louder the wrong cry, the bigger the penalty. +function rawToSevKey(raw){ + const s = (raw||"").toLowerCase(); + if (s.includes("blocking")) return "high"; + if (s.includes("minor")) return "small"; + if (s.includes("no material")) return "trivial"; + return "medium"; // unknown / "Reviewed" +} function filters(){ return { from: document.getElementById("from").value, @@ -204,7 +216,7 @@ function aggregate(f){ const c = curve(); const M = new Map(); const get = m => { if(!M.has(m)) M.set(m, {model:m, provider:"", runs:0, minutes:0, inTok:0, outTok:0, - findings:new Set(), confirmed:new Set(), fp:new Set(), ungraded:new Set(), sev:Object.fromEntries(SEVS.map(s=>[s,new Set()]))}); return M.get(m); }; + findings:new Set(), confirmed:new Set(), fp:new Map(), ungraded:new Set(), sev:Object.fromEntries(SEVS.map(s=>[s,new Set()]))}); return M.get(m); }; for (const r of RUNS){ if(!runMatch(r,f)) continue; const m=get(r.model); m.runs++; m.minutes += (r.duration_secs||0)/60; m.inTok += r.input_tokens||0; m.outTok += r.output_tokens||0; if(r.provider) m.provider=r.provider; } @@ -213,17 +225,20 @@ function aggregate(f){ for (const r of rows){ const m=get(r.model); if(r.provider) m.provider=m.provider||r.provider; m.findings.add(r.finding_id); if (r.graded && r.is_real === true){ m.confirmed.add(r.finding_id); if (r.severity) m.sev[r.severity].add(r.finding_id); } - else if (r.graded && r.is_real === false){ m.fp.add(r.finding_id); } + else if (r.graded && r.is_real === false){ m.fp.set(r.finding_id, rawToSevKey(r.raw_severity)); } else { m.ungraded.add(r.finding_id); } } + const fpm = fpMult(); const out = [...M.values()].map(m => { const sevCounts = Object.fromEntries(SEVS.map(s=>[s, m.sev[s].size])); - const points = SEVS.reduce((a,s)=> a + c[s]*sevCounts[s], 0); + const confirmedPoints = SEVS.reduce((a,s)=> a + c[s]*sevCounts[s], 0); + let fpPen = 0; for (const k of m.fp.values()) fpPen += (c[k]||0) * fpm; // negative when fpm<0 + const points = confirmedPoints + fpPen; // NET of the false-positive penalty const findings = m.findings.size, confirmed = m.confirmed.size; return { model:m.model, provider:m.provider, runs:m.runs, minutes:m.minutes, inTok:m.inTok, outTok:m.outTok, findings, confirmed, fp:m.fp.size, ungraded:m.ungraded.size, - sev:sevCounts, points, + sev:sevCounts, confirmedPoints, fpPen, points, ptsPerMin: m.minutes>0 ? points/m.minutes : null, ptsPerRun: m.runs>0 ? points/m.runs : null, confirmedPct: findings>0 ? confirmed/findings*100 : null }; @@ -236,7 +251,8 @@ const COLS = [ {k:"runs", t:"runs"}, {k:"minutes", t:"min", fmt:v=>v.toFixed(1)}, {k:"findings", t:"findings"}, {k:"confirmed", t:"real"}, {k:"fp", t:"FP"}, {k:"ungraded", t:"ungr"}, {k:"confirmedPct", t:"real%", fmt:v=>v==null?"—":v.toFixed(0)+"%"}, - {k:"points", t:"points", fmt:v=>v.toFixed(0)}, + {k:"fpPen", t:"fp pen", fmt:v=>v?v.toFixed(1):"0"}, + {k:"points", t:"points (net)", fmt:v=>v.toFixed(0)}, {k:"ptsPerMin", t:"pts/min", fmt:v=>v==null?"—":v.toFixed(2)}, {k:"ptsPerRun", t:"pts/run", fmt:v=>v==null?"—":v.toFixed(1)}, {k:"sev", t:"by severity", l:true, fmt:sev=>SEVS.filter(s=>sev[s]).map(s=>`${s[0].toUpperCase()}${sev[s]}`).join(" ")||"—"}, @@ -268,8 +284,9 @@ function render(){ const td = document.createElement("td"); if (col.l) td.className="l"; const v = m[col.k]; td.innerHTML = col.fmt ? col.fmt(v) : (v==null?"—":v); - if (col.k==="ptsPerMin" && v!=null) td.classList.add("good"); - if (col.k==="fp" && v>0) td.classList.add("bad"); + if ((col.k==="ptsPerMin" || col.k==="ptsPerRun" || col.k==="points") && v!=null) td.classList.add(v<0 ? "bad" : "good"); + if (col.k==="fpPen" && v<0) td.classList.add("bad"); + if (col.k==="fp" && v>0) td.classList.add("warn"); tr.appendChild(td); } mb.appendChild(tr);