mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 20:17:09 +00:00
test(runner): the echo guard admits a repeated description
Descriptions can now repeat after candidates dedup by what they execute. The guard is index-anchored, so this pins that a repeated string cannot make it misfire in either direction. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
bc4b9fb25b
commit
78a56c44f5
1 file changed
+45
@@ -680,6 +680,51 @@ func TestLLMSourceStrictSkipsOnEchoMismatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// llmSharedLabelTreeJSON has two rows a user reads as the same word, so the
|
||||||
|
// numbered list holds two entries rendering identically.
|
||||||
|
const llmSharedLabelTreeJSON = `{
|
||||||
|
"attributes": {"bounds": "[0,0,400,800]"},
|
||||||
|
"children": [
|
||||||
|
{"attributes": {"resource-id": "delete_alpha", "text": "Delete", "bounds": "[0,0,400,100]"}, "clickable": true, "enabled": true, "children": []},
|
||||||
|
{"attributes": {"resource-id": "delete_beta", "text": "Delete", "bounds": "[0,100,400,200]"}, "clickable": true, "enabled": true, "children": []}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
// TestLLMSourceEchoGuardAdmitsARepeatedDescription is the other half of the
|
||||||
|
// strict skip: it compares the echo against the entry the model NUMBERED, so a
|
||||||
|
// description shared by two entries still selects the one whose number came
|
||||||
|
// back. A guard that looked the echo up by description instead would run the
|
||||||
|
// first row for both numbers.
|
||||||
|
func TestLLMSourceEchoGuardAdmitsARepeatedDescription(t *testing.T) {
|
||||||
|
fake := newFakeOpenRouter(t)
|
||||||
|
source, verifierInstance := newLLMSource(t, fake)
|
||||||
|
pushSnapshotTree(t, verifierInstance, llmSharedLabelTreeJSON)
|
||||||
|
|
||||||
|
var repeated []verifier.ActionCandidate
|
||||||
|
for _, candidate := range verifierInstance.Candidates(verifier.LabelSourceVisibleText) {
|
||||||
|
if candidate.Description == `Tap "Delete"` {
|
||||||
|
repeated = append(repeated, candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(repeated) != 2 {
|
||||||
|
t.Fatalf("want two entries sharing one description, got %d", len(repeated))
|
||||||
|
}
|
||||||
|
|
||||||
|
second := repeated[1]
|
||||||
|
fake.choice = second.Index
|
||||||
|
fake.chosenAction = second.Description
|
||||||
|
action, err := source.NextAction(context.Background(), 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NextAction err = %v, want the second row's tap", err)
|
||||||
|
}
|
||||||
|
if action.On != "id:delete_beta" {
|
||||||
|
t.Errorf("action targets %q, want id:delete_beta", action.On)
|
||||||
|
}
|
||||||
|
if source.lastSource != "llm" {
|
||||||
|
t.Errorf("lastSource = %q, want llm; a repeated description must not strict-skip", source.lastSource)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLLMSourceSkipsOnHTTPError(t *testing.T) {
|
func TestLLMSourceSkipsOnHTTPError(t *testing.T) {
|
||||||
fake := newFakeOpenRouter(t)
|
fake := newFakeOpenRouter(t)
|
||||||
// Replace the handler with one that always errors.
|
// Replace the handler with one that always errors.
|
||||||
|
|||||||
Reference in new issue
Block a user