fix(runner): a secure field's value does not reach state.lastAction either

folio extracts lastAction, and extractor values are persisted as
extractor_changes, so the password still reached the run directory
through the spec after the three render sites were closed.

The wrap sits in the runner rather than in lastActionFields because the
hosts hold the next step's tree, not the one the action was chosen
against: a field that stops being secure between the two would publish
what the trace withheld. Live and replay now agree byte for byte.

docs/manual/runs.md

@@ -31,7 +31,7 @@ The trace is written incrementally. An interrupted run is complete up to the ste
 
 ## Typed values in the record
 
-A run types into whatever the app puts on screen, login forms included, and the trace and the model call record are both shared. So a typed value is written down as `[redacted]` whenever the target may be a credential entry: the trace action, the recent-action memory the prompt carries, and the numbered candidate list all render it that way. The app still receives the real keystrokes; only the record is redacted, and the record still names the field that was typed into.
+A run types into whatever the app puts on screen, login forms included, and the trace and the model call record are both shared. So a typed value is written down as `[redacted]` whenever the target may be a credential entry: the trace action, the recent-action memory the prompt carries, the numbered candidate list, and the `state.lastAction` a spec reads (and can extract into the trace) all render it that way. The app still receives the real keystrokes; only the record is redacted, and the record still names the field that was typed into.
 
 Which values that covers differs by platform, because the platforms differ in what they report. iOS and web state on every editable field whether it masks its input, so only the fields that do are redacted and the rest of the memory keeps its values. Android reports nothing: uiautomator's password attribute is dropped by the native tree mapper before the driver sees it, a password field is indistinguishable from a search box there, and so every typed value on Android is redacted.
 

docs/manual/spec-language.md

@@ -47,6 +47,8 @@ interface State {
 
 `lastAction.applied` is `true` when the runner saw the dispatch succeed and `null` when the apply call failed with the action possibly already delivered: an RPC deadline can fire after the tap reached the app, and nothing can find out afterwards. So there are three states, not two. `state.lastAction === null` means no action ran; `applied === null` means one ran whose fate is unknown. A property that attributes an effect to the action ("this submit must move the balance by the typed amount") has to decline unless `applied` is `true`, or a timeout convicts a healthy app. A property that counts what the app COULD have done should include it: an unconfirmed submit belongs in an upper bound on how many submits a window holds.
 
+`lastAction.text` carries the value as the record renders it, not the value the app received. A typed value reads `[redacted]` wherever the target may be a credential entry, which on iOS and web means the fields that report themselves secure and on Android means all of them, since Android reports nothing either way. A spec extracting `state.lastAction` writes it to the trace, so it is redacted for the same reason the trace action is; see [typed values in the record](../runs/#typed-values-in-the-record). A property that has to reason about what a field holds should read the field off `ax`.
+
 ## Selectors
 
 Selectors are passed to `ax.find()`, `ax.findAll()`, and element-scoped `.find()` / `.findAll()`. A tree-level lookup scans the whole hierarchy, the root element included; an element-scoped one scans that element's descendants. Both selector forms scan the same set, so `ax.find("id:page")` and `ax.find({ id: "page" })` return the same element.

internal/runner/redaction_test.go

@@ -2,8 +2,10 @@ package runner
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"testing"
+	"time"
 
 	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
 
@@ -129,6 +131,67 @@ func TestApplyActionTypesTheRealValueIntoEveryField(t *testing.T) {
 	}
 }
 
+// lastActionExtractorSpec is what examples/folio/sanderling/spec.ts does with
+// the previous step's action: it extracts state.lastAction whole. Extractor
+// values are written to the trace as extractor_changes, so a typed value that
+// reaches state.lastAction.text lands in the run directory through the spec
+// rather than through the runner.
+const lastActionExtractorSpec = `
+import { actions, always, extract, InputText } from "@sanderling/spec";
+const reported = extract("lastAction", state => state.lastAction);
+globalThis.properties = {
+  theActionReachesTheSpec: always(() => reported.current !== undefined),
+};
+globalThis.actions = actions(() => [InputText({ into: "%s", text: "` + typedCredential + `" })]);
+`
+
+func TestTheTraceNeverCarriesATypedSecretThroughALastActionExtractor(t *testing.T) {
+	for _, testCase := range []struct {
+		name     string
+		treeJSON string
+		selector string
+	}{
+		{"ios secure field", iosLoginTreeJSON, "id:LoginPassword"},
+		{"web secure field", webLoginTreeJSON, "id:login-password"},
+		{"android field reported as neither", androidLoginTreeJSON, "id:login_email"},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			fastFocusSettle(t)
+			state := newHarnessWithSpec(t, fmt.Sprintf(lastActionExtractorSpec, testCase.selector))
+			state.mock.HierarchyJSON = testCase.treeJSON
+
+			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			defer cancel()
+			if _, err := Run(ctx, Options{
+				Duration:    time.Hour,
+				IdleTimeout: 20 * time.Millisecond,
+				MaxSteps:    2,
+				Driver:      state.mock,
+				Verifier:    state.verifier,
+				TraceWriter: state.writer,
+			}); err != nil {
+				t.Fatalf("Run: %v", err)
+			}
+
+			steps := traceSteps(t, state.writer.Directory())
+			if len(steps) < 2 {
+				t.Fatalf("trace holds %d step(s), want the step that reports the action back", len(steps))
+			}
+			change, ok := steps[1].ExtractorChanges["lastAction"]
+			if !ok {
+				t.Fatalf("step 2 recorded no reading of state.lastAction: %+v", steps[1])
+			}
+			if strings.Contains(string(change.Curr), typedCredential) {
+				t.Errorf("the trace carries the typed value through state.lastAction: %s", change.Curr)
+			}
+			if !strings.Contains(string(change.Curr), verifier.RedactedInputText) {
+				t.Errorf("state.lastAction reported %s, want the typed value redacted in place",
+					change.Curr)
+			}
+		})
+	}
+}
+
 const secureLoginSpec = `
 import { llm, always, actions, InputText, taps, typing, weighted } from "@sanderling/spec";
 globalThis.properties = { ok: always(() => true) };

internal/runner/runner.go

@@ -451,7 +451,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 				// the effect committed. Reporting no action here would let a
 				// property convict the app for an effect with no cause, so the
 				// action is reported with its fate unknown instead.
-				unconfirmed := nextAction
+				unconfirmed := verifier.RecordedAction(nextAction, tree)
 				lastAction = &unconfirmed
 			} else if notDispatched != "" {
 				// The action was chosen but nothing reached the driver, so the
@@ -466,7 +466,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 				lastAction = nil
 			} else {
 				consecutiveApplyFailures = 0
-				applied := nextAction
+				applied := verifier.RecordedAction(nextAction, tree)
 				applied.Applied = true
 				lastAction = &applied
 			}

internal/verifier/redaction.go

@@ -50,12 +50,30 @@ func RecordedActionText(action Action, tree *hierarchy.Tree) string {
 	return recordedInputText(action.Text, secureFactOf(target))
 }
 
+// RecordedAction is the action as everything downstream of the dispatch sees
+// it. The runner reports the previous step's action to the spec as
+// state.lastAction, and a spec extracting it (examples/folio/sanderling/spec.ts)
+// writes it to the trace, so the copy the runner keeps carries the recorded
+// text rather than the typed one.
+//
+// The decision is made here rather than where the two hosts render
+// state.lastAction because only the caller holds the tree that can answer it.
+// The hosts hold the NEXT step's tree, where the selector may name a different
+// element (a revealed password field reports secure:false) or none at all, and
+// cmd/internal-tools/oracle-reduction replays state.lastAction from the trace's
+// already-recorded text, which redacting here matches exactly.
+func RecordedAction(action Action, tree *hierarchy.Tree) Action {
+	action.Text = RecordedActionText(action, tree)
+	return action
+}
+
 // recordedInputText is the one place a typed value is rendered for a record.
-// The prompt's recent-action memory, the numbered candidate list and the trace
-// all go through it, so a fourth record added later cannot publish a value the
-// other three withhold. The driver dispatch reads Action.Text directly and is
-// the only reader of the real value, which is what keeps the app receiving the
-// keystrokes a user would have produced.
+// The prompt's recent-action memory, the numbered candidate list, the trace and
+// the action the runner reports back as state.lastAction all go through it, so
+// a fifth record added later cannot publish a value the other four withhold.
+// The driver dispatch reads Action.Text directly and is the only reader of the
+// real value, which is what keeps the app receiving the keystrokes a user would
+// have produced.
 //
 // A target the platform reports as a secure entry is redacted, and so is a
 // target carrying no report at all. iOS and web state the fact on every

internal/verifier/redaction_test.go

@@ -3,6 +3,8 @@ package verifier
 import (
 	"strings"
 	"testing"
+
+	"github.com/priyanshujain/sanderling/internal/hierarchy"
 )
 
 // typedCredential stands in for what a login setup types. Nothing rendered for
@@ -109,3 +111,86 @@ func TestCandidatesRedactTextTypedIntoAnAndroidElementHandle(t *testing.T) {
 			candidates[0].Action.Text)
 	}
 }
+
+// lastActionOnBothHosts reports what a spec reading state.lastAction sees on
+// each host for an action the runner has recorded: the goja host stringifies
+// its own object, the web host receives EncodeLastAction's JSON and installs
+// the parsed value in the page.
+func lastActionOnBothHosts(t *testing.T, action Action, treeJSON string) (string, string) {
+	t.Helper()
+	tree, err := hierarchy.Parse(treeJSON)
+	if err != nil {
+		t.Fatalf("parse tree: %v", err)
+	}
+	recorded := RecordedAction(action, tree)
+	verifier := newVerifier(t)
+	mustLoad(t, verifier, `
+		globalThis.last = __sanderling__.extract(state => JSON.stringify(state.lastAction));
+	`)
+	if err := verifier.PushSnapshot(SnapshotInput{
+		Snapshots:  Snapshots{},
+		Tree:       tree,
+		LastAction: &recorded,
+	}); err != nil {
+		t.Fatalf("PushSnapshot: %v", err)
+	}
+	handle := verifier.runtime.GlobalObject().Get("last").ToObject(verifier.runtime)
+	return handle.Get("current").String(), string(EncodeLastAction(&recorded))
+}
+
+func typedInto(selector, text string) Action {
+	return Action{Kind: ActionKindInputText, On: selector, Text: text}
+}
+
+// A spec extracting state.lastAction (examples/folio/sanderling/spec.ts) writes
+// what it reads into the trace, so state.lastAction is a record like the other
+// three and carries the same rule on both hosts.
+func TestStateLastActionRedactsATypedValueTheTargetCannotClear(t *testing.T) {
+	for _, testCase := range []struct {
+		name     string
+		treeJSON string
+		selector string
+	}{
+		{"secure field", secureLoginTreeJSON, "id:LoginPassword"},
+		{"android field reported as neither", androidLoginTreeJSON, "id:login_email"},
+		{"android password field", androidLoginTreeJSON, "id:login_password"},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			goja, web := lastActionOnBothHosts(
+				t, typedInto(testCase.selector, typedCredential), testCase.treeJSON)
+			for _, host := range []struct{ name, reported string }{
+				{"goja", goja},
+				{"web", web},
+			} {
+				if strings.Contains(host.reported, typedCredential) {
+					t.Errorf("the %s host publishes the typed value in state.lastAction: %s",
+						host.name, host.reported)
+				}
+				if !strings.Contains(host.reported, RedactedInputText) {
+					t.Errorf("the %s host reports state.lastAction as %s, want the typed value "+
+						"redacted in place", host.name, host.reported)
+				}
+			}
+			if goja != web {
+				t.Errorf("the two hosts disagree on state.lastAction\n goja: %s\n  web: %s", goja, web)
+			}
+		})
+	}
+}
+
+func TestStateLastActionKeepsATypedValueForAFieldReportedNotSecure(t *testing.T) {
+	const address = "[email protected]"
+	goja, web := lastActionOnBothHosts(t, typedInto("id:LoginEmail", address), secureLoginTreeJSON)
+	for _, host := range []struct{ name, reported string }{
+		{"goja", goja},
+		{"web", web},
+	} {
+		if !strings.Contains(host.reported, address) {
+			t.Errorf("the %s host reports state.lastAction as %s, want the typed value on a "+
+				"field reported not secure", host.name, host.reported)
+		}
+	}
+	if goja != web {
+		t.Errorf("the two hosts disagree on state.lastAction\n goja: %s\n  web: %s", goja, web)
+	}
+}

pkg/spec/src/types.ts

@@ -134,6 +134,11 @@ export interface ExceptionRecord {
  * reported", which is weaker than "the app never restarted": a target whose
  * foreground the runner cannot read never relaunches the app and cannot promise
  * that either.
+ *
+ * `text` is the value as the record renders it: `[redacted]` wherever the
+ * target may be a credential entry, which on Android is every target, since it
+ * reports nothing either way. Read the field off `ax` to reason about what it
+ * holds.
  */
 export type LastAction = Action & { applied: true | null; relaunched: true | null };