steveandClaude Opus 5
3af0f09387
Fourteen findings, and seven of them from all four models are the same one: endpointProvider was missing the no-cross-vendor-fallback guard I had just added to resolveModel. I fixed a credential leak on one path and left its sibling leaking, in the commit whose own message argued those two paths must move together. That is the third time in this PR. So it is no longer a rule written twice. openAICompatOptions owns it and both paths call it; builtinCompatProviders names the vendors that must never inherit OPENAI_API_KEY, replacing a `provider == "kimi" || provider == "qwen"` literal that was a fourth uncounted copy of the list. The test drives a real request at a local server and demands two things: that no request arrives carrying the OpenAI key, AND that the call fails closed naming the variable to set — the second half because my first draft pointed the provider at vendor.example, so the server saw nothing and the assertion held for a reason unrelated to the fix. Break-checked: removing the guard puts "Bearer sk-openai-must-not-travel" on the wire to the other vendor. The scrub check failed open. As a bare condition, a grep ERROR (exit >= 2) reads as "not found" and skips the guard — a credential check that passes precisely when it cannot see the filesystem it is searching. It now distinguishes 0/1/>=2 and refuses to continue on error. A bare "claude-code" spec has no "/", so the provider fell back to ollama-cloud and the pre-flight would skip a reviewer that authenticates with CLAUDE_CODE_OAUTH_TOKEN and needs no Ollama key. Engine specs are now exempt. preflight.sh's provider list duplicated its own case arms; both now read one table. And its comment claimed the Go cross-check fails if either list misses an entry from the other, when only one direction is checked — the reverse is not even desirable, since ollama-cloud and anthropic belong in that table and not in the Go one. The comment now says what is enforced. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
102 lines
4.8 KiB
Bash
Executable File
102 lines
4.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Table test for the credential pre-flight in preflight.sh.
|
|
#
|
|
# It SOURCES the real implementation rather than copying it, so there is no
|
|
# second definition that can pass while production fails.
|
|
#
|
|
# Run: scripts/preflight_test.sh (exit 0 = all cases pass)
|
|
set -u
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
# shellcheck source=preflight.sh
|
|
. "$SCRIPT_DIR/preflight.sh"
|
|
|
|
fail=0
|
|
check() { # description, want, got
|
|
if [ "$2" = "$3" ]; then
|
|
echo "ok $1"
|
|
else
|
|
echo "FAIL $1 — want '$2', got '$3'"
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# probe <provider> [VAR=VAL ...] — run the real function in a clean environment
|
|
# under the same shell options production uses (set -u), so an unset-variable
|
|
# bug surfaces here instead of in a live review.
|
|
probe() {
|
|
local provider="$1" model="${GADFLY_TEST_MODEL:-}"; shift
|
|
env -i PATH="$PATH" HOME="$HOME" "$@" bash -c "
|
|
set -u
|
|
. '$SCRIPT_DIR/preflight.sh'
|
|
gadfly_preflight_key '$provider' '$model'
|
|
"
|
|
}
|
|
|
|
echo "== registry path: keyed providers with no key must name their variable =="
|
|
check "qwen, no key" "QWEN_API_KEY" "$(probe qwen)"
|
|
check "kimi, no key" "KIMI_API_KEY" "$(probe kimi)"
|
|
check "ollama-cloud, no key" "OLLAMA_CLOUD_API_KEY" "$(probe ollama-cloud)"
|
|
check "openai, no key" "OPENAI_API_KEY" "$(probe openai)"
|
|
check "openai-compatible, none" "OPENAI_API_KEY" "$(probe openai-compatible)"
|
|
check "anthropic, no key" "ANTHROPIC_API_KEY" "$(probe anthropic)"
|
|
|
|
echo "== registry path: the provider's own key lets it run =="
|
|
check "qwen, keyed" "" "$(probe qwen QWEN_API_KEY=k)"
|
|
check "kimi, keyed" "" "$(probe kimi KIMI_API_KEY=k)"
|
|
check "ollama-cloud, keyed" "" "$(probe ollama-cloud OLLAMA_API_KEY=k)"
|
|
check "openai-compatible, keyed" "" "$(probe openai-compatible OPENAI_API_KEY=k)"
|
|
|
|
echo "== a wrong-provider key never satisfies a provider (no cross-fallback) =="
|
|
check "qwen w/ only OPENAI key" "QWEN_API_KEY" "$(probe qwen OPENAI_API_KEY=k)"
|
|
check "kimi w/ only QWEN key" "KIMI_API_KEY" "$(probe kimi QWEN_API_KEY=k)"
|
|
|
|
echo "== an empty-string key counts as missing, not present =="
|
|
check "qwen, empty key" "QWEN_API_KEY" "$(probe qwen QWEN_API_KEY=)"
|
|
|
|
echo "== GADFLY_API_KEY does NOT substitute on the registry path =="
|
|
# resolveModel reads GADFLY_API_KEY only after its `baseURL == ""` early
|
|
# return, so on this path the built-in reads its own variable and a set
|
|
# GADFLY_API_KEY changes nothing. Treating it as sufficient was a false pass.
|
|
check "qwen w/ GADFLY_API_KEY only" "QWEN_API_KEY" "$(probe qwen GADFLY_API_KEY=k)"
|
|
|
|
echo "== override path (GADFLY_BASE_URL set) is deliberately not pre-flighted =="
|
|
# The credential there is GADFLY_API_KEY with a client-specific fallback, and
|
|
# the built-ins' own variables are never read. Checking one path's rules
|
|
# against the other produced a false pass in BOTH directions, so this path is
|
|
# left alone rather than guessed at.
|
|
check "qwen + BASE_URL, no keys" "" "$(probe qwen GADFLY_BASE_URL=https://x)"
|
|
check "qwen + BASE_URL + own key" "" "$(probe qwen GADFLY_BASE_URL=https://x QWEN_API_KEY=k)"
|
|
check "qwen + BASE_URL + GADFLY key" "" "$(probe qwen GADFLY_BASE_URL=https://x GADFLY_API_KEY=k)"
|
|
check "openai + BASE_URL, no keys" "" "$(probe openai GADFLY_BASE_URL=https://x)"
|
|
|
|
echo "== providers needing no key are never blocked, with nothing set =="
|
|
for p in ollama llama-swap llama-swaps llamaswap llamaswaps foreman google gemini some-dsn-name; do
|
|
check "unkeyed $p" "" "$(probe "$p")"
|
|
done
|
|
|
|
# google is absent from the table on purpose: it accepts GOOGLE_API_KEY *or*
|
|
# GEMINI_API_KEY, so a one-name arm would skip a correctly-configured run.
|
|
check "google w/ only GEMINI_API_KEY" "" "$(probe google GEMINI_API_KEY=k)"
|
|
|
|
echo "== a whitespace-only GADFLY_BASE_URL counts as unset, as it does in Go =="
|
|
# resolveModel TrimSpaces it and takes the registry path; if this check
|
|
# disagreed, the missing key would arrive as a bare 401 with no skip notice.
|
|
check "qwen + blank BASE_URL" "QWEN_API_KEY" "$(probe qwen GADFLY_BASE_URL=" ")"
|
|
|
|
echo "== engine specs carry their own auth and are never pre-flighted =="
|
|
# A bare "claude-code" has no "/", so the caller's provider falls back to
|
|
# ollama-cloud; judging it by that would skip a reviewer using
|
|
# CLAUDE_CODE_OAUTH_TOKEN, which needs no Ollama key.
|
|
check "bare claude-code, no ollama key" "" "$(GADFLY_TEST_MODEL=claude-code probe ollama-cloud)"
|
|
check "claude-code/opus, no ollama key" "" "$(GADFLY_TEST_MODEL=claude-code/opus probe ollama-cloud)"
|
|
check "opencode/x, no ollama key" "" "$(GADFLY_TEST_MODEL=opencode/x probe ollama-cloud)"
|
|
# ...but a genuine ollama-cloud model still is.
|
|
check "ollama-cloud model, no key" "OLLAMA_CLOUD_API_KEY" "$(GADFLY_TEST_MODEL=glm-5.2:cloud probe ollama-cloud)"
|
|
|
|
if [ "$fail" -ne 0 ]; then
|
|
echo "RESULT: preflight table FAILED"
|
|
exit 1
|
|
fi
|
|
echo "RESULT: all pre-flight cases pass"
|