mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(runner): accept chosen_action echo that carries the weight suffix
real runs showed the model copies the whole numbered line including the trailing (w34) weight annotation, so strict-skip rejected ~91% of picks and the llm was paralyzed. strip the weight suffix before comparing. also nudge the prompt to stress-test repeated submissions (idempotency).
This commit is contained in:
1 parent
938b2cb714
commit
c9e0f6a366
1 file changed
+16
-2
@@ -11,6 +11,7 @@ import (
|
||||
"image/color"
|
||||
"image/png"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/llmclient"
|
||||
@@ -33,7 +34,7 @@ const (
|
||||
// screenshot, picks ONE number, and echoes that action so a mismatch can be
|
||||
// caught. The app-specific description (spec instructions) is appended.
|
||||
const llmSystemPrompt = "You are exercising a UI to find bugs. Each turn you get a screenshot and a numbered list of concrete actions, each with a weight hinting how much the test author wants it exercised (higher = more). " +
|
||||
"Pick the ONE action most likely to make progress or expose a defect, and feel free to repeat an action when repetition is what would trip a bug. " +
|
||||
"Pick the ONE action most likely to make progress or expose a defect. Bugs often hide in repeated or rapid actions, so once a screen works, deliberately stress it — for example submitting the same form twice in a row to check it is not applied more than once — rather than only advancing. " +
|
||||
"Respond with your reasoning, the chosen number, and chosen_action copied verbatim from that line. For a typing action, also provide the text to enter."
|
||||
|
||||
// llmSource selects each step's action with an OpenAI-compatible vision model
|
||||
@@ -137,7 +138,9 @@ func (s *llmSource) selectViaLLM(ctx context.Context) (llmSelection, bool) {
|
||||
candidate := candidates[output.Choice-1]
|
||||
// Strict skip: the echoed action must match the numbered entry, so a model
|
||||
// that reasoned about one target but named a number for another cannot act.
|
||||
if strings.TrimSpace(output.ChosenAction) != candidate.Description {
|
||||
// Models copy the whole rendered line including its trailing "(w34)" weight
|
||||
// annotation, so strip that before comparing to the (weight-free) description.
|
||||
if stripWeightSuffix(output.ChosenAction) != candidate.Description {
|
||||
s.logger.Warn("llm chosen_action mismatch; skipping",
|
||||
"choice", output.Choice, "echoed", output.ChosenAction, "candidate", candidate.Description)
|
||||
return llmSelection{}, false
|
||||
@@ -266,6 +269,17 @@ type choiceOutput struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// weightSuffix matches the trailing " (w34)" annotation appended to each
|
||||
// numbered line, which models copy verbatim into chosen_action.
|
||||
var weightSuffix = regexp.MustCompile(`\s*\(w\d+\)$`)
|
||||
|
||||
// stripWeightSuffix trims surrounding whitespace and a trailing weight
|
||||
// annotation from the model's echoed action so it can be compared to the
|
||||
// weight-free candidate description.
|
||||
func stripWeightSuffix(echo string) string {
|
||||
return strings.TrimSpace(weightSuffix.ReplaceAllString(strings.TrimSpace(echo), ""))
|
||||
}
|
||||
|
||||
// parseChoice decodes the model's JSON content into the structured choice.
|
||||
func parseChoice(content string) (choiceOutput, error) {
|
||||
content = strings.TrimSpace(content)
|
||||
|
||||
Reference in new issue
Block a user