feat: cross-model consensus consolidation (one ranked comment, not N walls)
With >=2 models the swarm now posts ONE consensus comment instead of a per-model comment each. Every model writes its structured findings to a shared dir (GADFLY_FINDINGS_OUT); after the swarm finishes, a consolidation pass (the binary in GADFLY_CONSOLIDATE_DIR mode) clusters findings by location (±3 lines), counts how many models independently flagged each, escalates to the highest reported severity, and renders an agreement-ranked table — cross-model agreement being the strongest real-vs-false-positive signal we have. - consensus.go: modelFindings artifact, collectFindings (shared with emit), writeFindingsOut, runConsolidate, clustering + renderConsensus. Lone low-severity findings fold away; a lone critical still surfaces; each model's full review is preserved folded inside the one comment. - main.go: GADFLY_CONSOLIDATE_DIR dispatches to consolidation; writeFindingsOut after a normal review. - emit.go: reuse collectFindings (one definition of "what a finding is"). - run.sh: when GADFLY_CONSOLIDATE=1, write findings + skip the per-model comment (live progress still on the status board). - entrypoint.sh: auto-enable for >=2 models; run the consolidator after the barrier and upsert ONE consensus comment; FALL BACK to per-model comments if consolidation yields nothing (advisory invariant: never lose output). - README + reusable workflow (`consolidate` input) + entrypoint docs updated. Tests: clustering/agreement/tolerance, severity escalation, nit folding, lone critical, write→consolidate round-trip, empty-dir error. gofmt/vet clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+77
-2
@@ -45,6 +45,8 @@
|
||||
# GADFLY_FINDINGS_URL optional gadfly-reports store base URL; set to POST the run +
|
||||
# findings for model-quality tracking (off when empty)
|
||||
# GADFLY_FINDINGS_TOKEN optional bearer token for the gadfly-reports store
|
||||
# GADFLY_CONSOLIDATE cross-model consensus comment: "auto" (default; on for >=2
|
||||
# models), "1" force on, "0" force off (one comment per model)
|
||||
set -uo pipefail
|
||||
|
||||
# One model by default: the specialist suite already provides breadth, so a
|
||||
@@ -64,6 +66,29 @@ die() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
API() { curl -fsS --connect-timeout 20 --max-time 30 -H "Authorization: token ${GITEA_TOKEN}" "$@"; }
|
||||
|
||||
# upsert_comment_body MARKER BODY — create or update (by leading MARKER) a single
|
||||
# PR comment. Mirrors run.sh's per-model upsert; used for the consensus comment
|
||||
# and the per-model fallback when consolidation is on.
|
||||
upsert_comment_body() {
|
||||
local marker="$1" body="$2" post_body existing_id="" page=1 cmts
|
||||
post_body="$(jq -n --arg b "$body" '{body:$b}')"
|
||||
while [ "$page" -le 10 ]; do
|
||||
cmts="$(API "${GITEA_API}/issues/${PR}/comments?limit=50&page=${page}" || echo '[]')"
|
||||
[ "$(echo "$cmts" | jq 'length')" = "0" ] && break
|
||||
existing_id="$(echo "$cmts" | jq -r --arg m "$marker" \
|
||||
'.[] | select(.body != null and (.body | startswith($m))) | .id' | head -n1)"
|
||||
[ -n "$existing_id" ] && break
|
||||
page=$((page+1))
|
||||
done
|
||||
if [ -n "$existing_id" ]; then
|
||||
curl -sS --connect-timeout 20 --max-time 30 -X PATCH -H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" "${GITEA_API}/issues/comments/${existing_id}" -d "$post_body" >/dev/null
|
||||
else
|
||||
curl -sS --connect-timeout 20 --max-time 30 -X POST -H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" "${GITEA_API}/issues/${PR}/comments" -d "$post_body" >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# --- is the commenter allowed to trigger a re-review? ----------------------
|
||||
actor_allowed() {
|
||||
local actor="$1"
|
||||
@@ -176,10 +201,11 @@ provider_cap() { # provider -> concurrency (override map "p=N,...", else default
|
||||
}
|
||||
|
||||
review_one() {
|
||||
local sf=""
|
||||
local sf="" ff=""
|
||||
[ "${GADFLY_STATUS_BOARD:-1}" != "0" ] && sf="$(status_file_for "$1")"
|
||||
[ "$CONSOLIDATE" = "1" ] && ff="$(findings_file_for "$1")"
|
||||
PROVIDER=ollama MODEL="$1" GADFLY_BIN="/usr/local/bin/gadfly" GADFLY_REPO_DIR="$REPO_DIR" \
|
||||
GADFLY_STATUS_FILE="$sf" \
|
||||
GADFLY_STATUS_FILE="$sf" GADFLY_FINDINGS_OUT="$ff" GADFLY_CONSOLIDATE="$CONSOLIDATE" \
|
||||
bash "${SCRIPTS_DIR}/run.sh" || log "model $1 failed (continuing)"
|
||||
# If the binary never wrote real status (run.sh skipped it: empty diff, no key,
|
||||
# binary missing), the pre-seed stays {started:0, done:false} and the board
|
||||
@@ -196,6 +222,25 @@ IFS=',' read -ra _raw <<< "$MODELS" || true
|
||||
MODEL_LIST=()
|
||||
for raw in "${_raw[@]}"; do m="$(echo "$raw" | tr -d '[:space:]')"; [ -n "$m" ] && MODEL_LIST+=("$m"); done
|
||||
|
||||
# --- cross-model consolidation decision ------------------------------------
|
||||
# With >=2 models, post ONE consensus comment (findings clustered + ranked by
|
||||
# cross-model agreement) instead of N per-model walls of prose. Each model writes
|
||||
# its findings to FINDINGS_DIR; a final pass (the binary in GADFLY_CONSOLIDATE_DIR
|
||||
# mode) renders the consensus comment. GADFLY_CONSOLIDATE: "auto" (default; on for
|
||||
# >=2 models), "1" force on, "0" force off (keep per-model comments).
|
||||
FINDINGS_DIR="${WORKDIR}/findings"
|
||||
CONSOLIDATE=0
|
||||
case "${GADFLY_CONSOLIDATE:-auto}" in
|
||||
1) CONSOLIDATE=1 ;;
|
||||
0) CONSOLIDATE=0 ;;
|
||||
*) [ "${#MODEL_LIST[@]}" -ge 2 ] && CONSOLIDATE=1 ;;
|
||||
esac
|
||||
findings_file_for() { echo "${FINDINGS_DIR}/$(echo "$1" | tr -c '[:alnum:]._-' '_').json"; }
|
||||
if [ "$CONSOLIDATE" = "1" ]; then
|
||||
rm -rf "$FINDINGS_DIR"; mkdir -p "$FINDINGS_DIR"
|
||||
log "consolidation ON: ${#MODEL_LIST[@]} models -> one consensus comment"
|
||||
fi
|
||||
|
||||
# Distinct providers, in first-seen order (no associative arrays — portable).
|
||||
PROVIDERS=""
|
||||
for m in "${MODEL_LIST[@]}"; do
|
||||
@@ -257,4 +302,34 @@ if [ -n "$BOARD_PID" ]; then
|
||||
touch "${STATUS_DIR}/.done" 2>/dev/null || true
|
||||
wait "$BOARD_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# --- cross-model consensus comment -----------------------------------------
|
||||
# Render ONE consensus comment from the per-model findings the swarm wrote. This
|
||||
# is advisory and best-effort: if the consolidation pass produces nothing, fall
|
||||
# back to posting each model's review as its own comment (the per-model comments
|
||||
# were suppressed during the run), so a consolidation hiccup never loses output.
|
||||
if [ "$CONSOLIDATE" = "1" ]; then
|
||||
n_files="$(ls -1 "${FINDINGS_DIR}"/*.json 2>/dev/null | wc -l | tr -d '[:space:]')"
|
||||
log "consolidating findings from ${n_files} model(s)"
|
||||
CONSENSUS="$(GADFLY_CONSOLIDATE_DIR="$FINDINGS_DIR" /usr/local/bin/gadfly 2>"${WORKDIR}/consolidate.err" || true)"
|
||||
if [ -n "$CONSENSUS" ]; then
|
||||
BODY="$(printf '%s\n\n<sub>Automated adversarial review by Gadfly — consensus across the model swarm. Advisory only — does not block merge.</sub>' "$CONSENSUS")"
|
||||
upsert_comment_body "<!-- gadfly-consensus -->" "$BODY"
|
||||
log "consensus comment posted"
|
||||
else
|
||||
log "consolidation produced no output; falling back to per-model comments"
|
||||
log "$(tail -c 500 "${WORKDIR}/consolidate.err" 2>/dev/null)"
|
||||
for f in "${FINDINGS_DIR}"/*.json; do
|
||||
[ -f "$f" ] || continue
|
||||
m="$(jq -r '.model // ""' "$f" 2>/dev/null)"
|
||||
[ -z "$m" ] && continue
|
||||
prov="$(jq -r '.provider // ""' "$f" 2>/dev/null)"
|
||||
md="$(jq -r '.markdown // ""' "$f" 2>/dev/null)"
|
||||
marker="<!-- gadfly-review:ollama:${m} -->"
|
||||
body="$(printf '%s\n### 🪰 Gadfly review — `%s` (%s)\n\n%s\n\n<sub>Automated adversarial review by Gadfly. Advisory only — does not block merge.</sub>' \
|
||||
"$marker" "$m" "$prov" "$md")"
|
||||
upsert_comment_body "$marker" "$body"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
log "done"
|
||||
|
||||
Reference in New Issue
Block a user