mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
test(runner): the echo guard admits a repeated description
Descriptions can now repeat after candidates dedup by what they execute. The guard is index-anchored, so this pins that a repeated string cannot make it misfire in either direction. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
bc4b9fb25b
commit
78a56c44f5
1 file changed
+45
@@ -680,6 +680,51 @@ func TestLLMSourceStrictSkipsOnEchoMismatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// llmSharedLabelTreeJSON has two rows a user reads as the same word, so the
|
||||
// numbered list holds two entries rendering identically.
|
||||
const llmSharedLabelTreeJSON = `{
|
||||
"attributes": {"bounds": "[0,0,400,800]"},
|
||||
"children": [
|
||||
{"attributes": {"resource-id": "delete_alpha", "text": "Delete", "bounds": "[0,0,400,100]"}, "clickable": true, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "delete_beta", "text": "Delete", "bounds": "[0,100,400,200]"}, "clickable": true, "enabled": true, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
// TestLLMSourceEchoGuardAdmitsARepeatedDescription is the other half of the
|
||||
// strict skip: it compares the echo against the entry the model NUMBERED, so a
|
||||
// description shared by two entries still selects the one whose number came
|
||||
// back. A guard that looked the echo up by description instead would run the
|
||||
// first row for both numbers.
|
||||
func TestLLMSourceEchoGuardAdmitsARepeatedDescription(t *testing.T) {
|
||||
fake := newFakeOpenRouter(t)
|
||||
source, verifierInstance := newLLMSource(t, fake)
|
||||
pushSnapshotTree(t, verifierInstance, llmSharedLabelTreeJSON)
|
||||
|
||||
var repeated []verifier.ActionCandidate
|
||||
for _, candidate := range verifierInstance.Candidates(verifier.LabelSourceVisibleText) {
|
||||
if candidate.Description == `Tap "Delete"` {
|
||||
repeated = append(repeated, candidate)
|
||||
}
|
||||
}
|
||||
if len(repeated) != 2 {
|
||||
t.Fatalf("want two entries sharing one description, got %d", len(repeated))
|
||||
}
|
||||
|
||||
second := repeated[1]
|
||||
fake.choice = second.Index
|
||||
fake.chosenAction = second.Description
|
||||
action, err := source.NextAction(context.Background(), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("NextAction err = %v, want the second row's tap", err)
|
||||
}
|
||||
if action.On != "id:delete_beta" {
|
||||
t.Errorf("action targets %q, want id:delete_beta", action.On)
|
||||
}
|
||||
if source.lastSource != "llm" {
|
||||
t.Errorf("lastSource = %q, want llm; a repeated description must not strict-skip", source.lastSource)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMSourceSkipsOnHTTPError(t *testing.T) {
|
||||
fake := newFakeOpenRouter(t)
|
||||
// Replace the handler with one that always errors.
|
||||
|
||||
Reference in new issue
Block a user