mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(verifier): a secure field's typed value never reaches the record
A folio login run wrote the account email and password in cleartext into llm-calls.jsonl, 166 times in one run, beside screenshots of the same screens. Three sites rendered it: the recent-action memory, the candidate list, and the trace. One helper now covers all three so a fourth cannot bypass it, and the driver still receives the real text. Android redacts every typed value because it cannot tell a secure field from any other. That asymmetry is deliberate and documented: safe by default on the target that cannot tell.
This commit is contained in:
1 parent
b1e95739ad
commit
38d328df90
12 files changed
+418
-16
No files matched your search
@@ -0,0 +1,180 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/verifier"
|
||||
)
|
||||
|
||||
// typedCredential stands in for what a login setup types. Nothing the run
|
||||
// records or sends may carry it.
|
||||
const typedCredential = "fixture-passphrase-9f21"
|
||||
|
||||
// iosLoginTreeJSON is the shape internal/driver/ioscompanion produces for a
|
||||
// login form: every editable field reports `secure`, so a field carrying
|
||||
// `secure:false` is positively known not to be a credential entry.
|
||||
const iosLoginTreeJSON = `{
|
||||
"attributes": {"bounds": "[0,0,390,844]", "class": "Window"},
|
||||
"children": [
|
||||
{"attributes": {"identifier": "LoginEmail", "class": "TextField", "hintText": "Email", "bounds": "[20,200,370,244]"},
|
||||
"editable": true, "enabled": true, "secure": false, "children": []},
|
||||
{"attributes": {"identifier": "LoginPassword", "class": "SecureTextField", "hintText": "Password", "bounds": "[20,260,370,304]"},
|
||||
"editable": true, "enabled": true, "secure": true, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
// webLoginTreeJSON is the same form as the chrome dump reports it.
|
||||
const webLoginTreeJSON = `{
|
||||
"attributes": {"bounds": "[0,0,1280,800]", "tag": "html"},
|
||||
"children": [
|
||||
{"attributes": {"resource-id": "login-email", "tag": "input", "hintText": "Email", "bounds": "[20,200,400,240]"},
|
||||
"editable": true, "enabled": true, "secure": false, "children": []},
|
||||
{"attributes": {"resource-id": "login-password", "tag": "input", "hintText": "Password", "bounds": "[20,260,400,300]"},
|
||||
"editable": true, "enabled": true, "secure": true, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
// androidLoginTreeJSON is the same form on Android, where the native tree
|
||||
// mapper drops uiautomator's password attribute and neither field carries the
|
||||
// fact.
|
||||
const androidLoginTreeJSON = `{
|
||||
"attributes": {"bounds": "[0,0,1080,2340]"},
|
||||
"children": [
|
||||
{"attributes": {"resource-id": "login_email", "class": "android.widget.EditText", "hintText": "Email", "bounds": "[20,200,1060,300]"},
|
||||
"enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "login_password", "class": "android.widget.EditText", "hintText": "Password", "bounds": "[20,320,1060,420]"},
|
||||
"enabled": true, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
func typeInto(selector string) verifier.Action {
|
||||
return verifier.Action{Kind: verifier.ActionKindInputText, On: selector, Text: typedCredential}
|
||||
}
|
||||
|
||||
func TestTraceActionForRedactsTypedValuesTheTargetCannotClear(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
treeJSON string
|
||||
selector string
|
||||
}{
|
||||
{"ios secure field", iosLoginTreeJSON, "id:LoginPassword"},
|
||||
{"web secure field", webLoginTreeJSON, "id:login-password"},
|
||||
{"android field reported as neither", androidLoginTreeJSON, "id:login_email"},
|
||||
{"android password field", androidLoginTreeJSON, "id:login_password"},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
traceAction := traceActionFor(typeInto(testCase.selector), mustParseTree(t, testCase.treeJSON))
|
||||
if traceAction.Text != verifier.RedactedInputText {
|
||||
t.Errorf("trace action text = %q, want it redacted", traceAction.Text)
|
||||
}
|
||||
if traceAction.Selector != testCase.selector {
|
||||
t.Errorf("trace selector = %q, want %q so the record still shows which field was typed into",
|
||||
traceAction.Selector, testCase.selector)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTraceActionForKeepsTypedValuesForAFieldReportedNotSecure(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
treeJSON string
|
||||
selector string
|
||||
}{
|
||||
{"ios", iosLoginTreeJSON, "id:LoginEmail"},
|
||||
{"web", webLoginTreeJSON, "id:login-email"},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
const address = "[email protected]"
|
||||
action := verifier.Action{Kind: verifier.ActionKindInputText, On: testCase.selector, Text: address}
|
||||
traceAction := traceActionFor(action, mustParseTree(t, testCase.treeJSON))
|
||||
if traceAction.Text != address {
|
||||
t.Errorf("trace action text = %q, want the typed value on a field reported not secure", traceAction.Text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The redaction is a rendering rule, never a change to what is dispatched: the
|
||||
// app has to receive the keystrokes a user would have produced.
|
||||
func TestApplyActionTypesTheRealValueIntoEveryField(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
treeJSON string
|
||||
selector string
|
||||
}{
|
||||
{"ios secure field", iosLoginTreeJSON, "id:LoginPassword"},
|
||||
{"web secure field", webLoginTreeJSON, "id:login-password"},
|
||||
{"android field", androidLoginTreeJSON, "id:login_password"},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
fastFocusSettle(t)
|
||||
driverMock := mockdriver.New()
|
||||
mustDispatch(t, driverMock, typeInto(testCase.selector), mustParseTree(t, testCase.treeJSON))
|
||||
|
||||
typed := ""
|
||||
for _, recorded := range driverMock.Actions() {
|
||||
if recorded.Kind == mockdriver.ActionInputText {
|
||||
typed = recorded.Text
|
||||
}
|
||||
}
|
||||
if typed != typedCredential {
|
||||
t.Errorf("driver received %q, want the real typed value", typed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const secureLoginSpec = `
|
||||
import { llm, always, actions, InputText, taps, typing, weighted } from "@sanderling/spec";
|
||||
globalThis.properties = { ok: always(() => true) };
|
||||
globalThis.setup = actions(() => {
|
||||
if (globalThis.__typed) return [];
|
||||
globalThis.__typed = true;
|
||||
return [InputText({ into: "id:LoginPassword", text: "` + typedCredential + `" })];
|
||||
});
|
||||
globalThis.actions = weighted([1, taps], [1, typing]);
|
||||
globalThis.generator = llm({ model: "test/model" });
|
||||
`
|
||||
|
||||
// The recorded call is llm-calls.jsonl: its user prompt carries the
|
||||
// recent-action memory and its candidates carry the numbered list, which is
|
||||
// where a login run put the account password in cleartext beside a screenshot
|
||||
// of the same screen.
|
||||
func TestLLMCallRecordKeepsTypedSecretsOutOfThePromptAndCandidates(t *testing.T) {
|
||||
fake := newFakeOpenRouter(t)
|
||||
source, verifierInstance := newLLMSourceWithSpec(t, fake, secureLoginSpec)
|
||||
|
||||
pushSnapshotTree(t, verifierInstance, iosLoginTreeJSON)
|
||||
action, err := source.NextAction(context.Background(), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("setup NextAction: %v", err)
|
||||
}
|
||||
if action.Text != typedCredential {
|
||||
t.Fatalf("setup action text = %q, want the real value dispatched to the app", action.Text)
|
||||
}
|
||||
|
||||
pushSnapshotTree(t, verifierInstance, iosLoginTreeJSON)
|
||||
fake.choice = 1
|
||||
fake.chosenAction = mustCandidates(t, verifierInstance, verifier.LabelSourceVisibleText)[0].Description
|
||||
if _, err := source.NextAction(context.Background(), 1); err != nil {
|
||||
t.Fatalf("llm NextAction: %v", err)
|
||||
}
|
||||
|
||||
call := lastCall(t, source)
|
||||
if strings.Contains(call.UserPrompt, typedCredential) {
|
||||
t.Error("the recorded user prompt carries the typed value")
|
||||
}
|
||||
if !strings.Contains(call.UserPrompt, "InputText") {
|
||||
t.Errorf("the recorded user prompt lost the recent-action memory entirely: %q", call.UserPrompt)
|
||||
}
|
||||
for _, candidate := range call.Candidates {
|
||||
if strings.Contains(candidate.Description, typedCredential) {
|
||||
t.Errorf("recorded candidate %d carries the typed value: %q", candidate.Index, candidate.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user