diff --git a/internal/runner/testdata/trace.jsonl b/internal/runner/testdata/trace.jsonl index 365792e..7ed3324 100644 --- a/internal/runner/testdata/trace.jsonl +++ b/internal/runner/testdata/trace.jsonl @@ -1,4 +1,4 @@ -{"extractor_changes":{"extractor_0":{"prev":null,"curr":0}},"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":1,"timestamp":"0001-01-01T00:00:00Z"} -{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":2,"timestamp":"0001-01-01T00:00:00Z"} -{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":3,"timestamp":"0001-01-01T00:00:00Z"} -{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":4,"timestamp":"0001-01-01T00:00:00Z"} +{"extractor_changes":{"extractor_0":{"prev":null,"curr":0}},"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":1,"timestamp":"0001-01-01T00:00:00Z","trace_version":1} +{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":2,"timestamp":"0001-01-01T00:00:00Z","trace_version":1} +{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":3,"timestamp":"0001-01-01T00:00:00Z","trace_version":1} +{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":4,"timestamp":"0001-01-01T00:00:00Z","trace_version":1} diff --git a/internal/trace/writer.go b/internal/trace/writer.go index d2f6b3f..3a7c514 100644 --- a/internal/trace/writer.go +++ b/internal/trace/writer.go @@ -13,14 +13,31 @@ import ( "github.com/priyanshujain/sanderling/internal/hierarchy" ) +// TraceVersion is stamped on every step this build writes. A step decoding to +// version 0 predates the fields introduced with version 1 (the tree's stored +// depths, per-step logs, per-step exceptions), which is what separates "this +// trace cannot answer the question" from "this step had nothing to report". +const TraceVersion = 1 + type Step struct { - Index int `json:"step"` - Timestamp time.Time `json:"timestamp"` - Screen string `json:"screen,omitempty"` - Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"` + Index int `json:"step"` + TraceVersion int `json:"trace_version,omitempty"` + Timestamp time.Time `json:"timestamp"` + Screen string `json:"screen,omitempty"` + Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"` // NextAction is the action chosen for the next iteration based on observing this step. - NextAction *Action `json:"next_action,omitempty"` - Exceptions []Exception `json:"exceptions,omitempty"` + NextAction *Action `json:"next_action,omitempty"` + // Logs are the platform log lines collected for this step, Exceptions the + // uncaught errors read at verification time: the error surface behind + // state.logs and state.exceptions, which the default properties read and + // an offline oracle has no other source for. + Logs []LogEntry `json:"logs,omitempty"` + Exceptions []Exception `json:"exceptions,omitempty"` + // Navigations are the document-replacing navigations seen since the + // previous step: the app reloaded, submitted a form, or changed route. + // Each one restarts the app's own runtime, so without them a reload and a + // generator repeating itself read the same way in a trace. + Navigations []Navigation `json:"navigations,omitempty"` Violations []string `json:"violations,omitempty"` Hierarchy *hierarchy.Tree `json:"hierarchy,omitempty"` Residuals map[string]json.RawMessage `json:"residuals,omitempty"` @@ -35,6 +52,11 @@ type Step struct { // count of executed actions cannot be inflated by steps whose action the // runner threw away. Empty when the action ran (or when none was chosen). ActionSkipped string `json:"action_skipped,omitempty"` + // ObservationError names why this step's device read produced no tree, + // empty when a tree was read. A screen with no elements on it is a tree, + // so without this a step that observed nothing at all is indistinguishable + // from a step that observed an app showing nothing. + ObservationError string `json:"observation_error,omitempty"` // SkippedVerification is set true exactly when the verifier was skipped // for this step, so downstream tooling can tell a deliberately-skipped // step from one that was verified and came back clean. @@ -44,6 +66,12 @@ type Step struct { Witnesses map[string]Witness `json:"witnesses,omitempty"` } +// Navigation is one document-replacing navigation the run observed. +type Navigation struct { + URL string `json:"url"` + UnixMillis int64 `json:"unix_millis,omitempty"` +} + // Witness is the trace-side record of a property violation: why it fired, the // two steps a deferred obligation spans, and the extractor values behind it. type Witness struct { @@ -114,6 +142,15 @@ type PointRecord struct { Y int `json:"y"` } +// LogEntry mirrors one platform log line the runner collected for this step, +// in the shape state.logs exposes to a spec. +type LogEntry struct { + UnixMillis int64 `json:"unix_millis,omitempty"` + Level string `json:"level,omitempty"` + Tag string `json:"tag,omitempty"` + Message string `json:"message,omitempty"` +} + type Exception struct { Class string `json:"class"` Message string `json:"message,omitempty"` @@ -195,12 +232,15 @@ func (w *Writer) WriteMeta(meta Meta) error { return os.WriteFile(filepath.Join(w.directory, "meta.json"), body, 0o644) } +// WriteStep stamps the format version itself so no caller can write a step +// that cannot be told apart from one written before the format changed. func (w *Writer) WriteStep(step Step) error { w.mutex.Lock() defer w.mutex.Unlock() if w.file == nil { return fmt.Errorf("trace: writer is closed") } + step.TraceVersion = TraceVersion return w.encoder.Encode(step) } diff --git a/internal/trace/writer_test.go b/internal/trace/writer_test.go index 119c339..cc5e6dd 100644 --- a/internal/trace/writer_test.go +++ b/internal/trace/writer_test.go @@ -143,11 +143,6 @@ func TestWriteStep_HierarchyAndResidualsRoundTrip(t *testing.T) { t.Errorf("residuals round-trip wrong: %s", got.Residuals["prop1"]) } - // Intentionally-lossy contract: Tree marshals only Elements (Root and - // Node.Children are json:"-"). The flat element list survives; tree - // structure does not. Lock both halves so a regression that drops the - // element list, or one that silently starts persisting structure the - // replay UI would then depend on, is caught. if got.Hierarchy == nil { t.Fatal("hierarchy dropped from trace") } @@ -157,8 +152,22 @@ func TestWriteStep_HierarchyAndResidualsRoundTrip(t *testing.T) { if got.Hierarchy.Elements[1].Text != "hi" { t.Errorf("element field lost: %+v", got.Hierarchy.Elements[1]) } - if got.Hierarchy.Root != nil { - t.Errorf("Root is json:\"-\" and must decode nil, got %+v", got.Hierarchy.Root) + if got.Hierarchy.Root == nil { + t.Fatal("tree structure not reconstructed from the stored form") + } + if got.Hierarchy.Root.ResourceID != "root" || + len(got.Hierarchy.Root.Children) != 1 { + t.Fatalf("root rebuilt wrong: %+v", got.Hierarchy.Root.Element) + } + if resolved := got.Hierarchy.Find("id:child"); resolved == nil || + resolved.Text != "hi" { + t.Errorf( + "selector resolves online but not against the decoded tree: %+v", + resolved, + ) + } + if resolved := got.Hierarchy.Find("id:child"); resolved != got.Hierarchy.Elements[1] { + t.Error("decoded elements and the rebuilt nodes are different pointers") } } @@ -493,3 +502,48 @@ func TestWriteMeta_OmitsArmMembershipWhenUnset(t *testing.T) { } } } + +// TestStepPredatingTheFormatIsDistinguishable is the backward-compatibility +// contract: the existing corpus must still load, and a step from it must be +// separable from one this build wrote with nothing to report, or "this trace +// predates the format" reads as "this step had no logs". +func TestStepPredatingTheFormatIsDistinguishable(t *testing.T) { + const stored = `{"step":3,"timestamp":"2026-06-10T21:22:07Z",` + + `"hierarchy":{"elements":[{"resourceId":"root"},{"resourceId":"child"}]}}` + var old Step + if err := json.Unmarshal([]byte(stored), &old); err != nil { + t.Fatalf( + "a trace written before the format change no longer loads: %v", + err, + ) + } + if old.TraceVersion != 0 { + t.Errorf( + "trace_version = %d, want 0 for a step that predates the field", + old.TraceVersion, + ) + } + if len(old.Hierarchy.Elements) != 2 || old.Hierarchy.Root != nil { + t.Errorf("old hierarchy reinterpreted: elements=%d root=%v", + len(old.Hierarchy.Elements), old.Hierarchy.Root) + } + + directory := t.TempDir() + writer, _ := NewWriter(directory) + defer writer.Close() + if err := writer.WriteStep(Step{Index: 3}); err != nil { + t.Fatal(err) + } + body, _ := os.ReadFile(filepath.Join(directory, "trace.jsonl")) + var fresh Step + if err := json.Unmarshal(body, &fresh); err != nil { + t.Fatal(err) + } + if fresh.TraceVersion != TraceVersion { + t.Errorf("a step with nothing to report stamped version %d, want %d", + fresh.TraceVersion, TraceVersion) + } + if len(fresh.Logs) != 0 { + t.Errorf("logs = %v, want none", fresh.Logs) + } +}