mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(trace): version each step and record its logs, exceptions and navigations
a step now carries trace_version, the platform log lines and uncaught errors behind state.logs and state.exceptions, the document-replacing navigations seen since the previous step, and observation_error naming why a device read produced no tree. version 0 is a step written before those fields existed, which is what separates a trace that cannot answer the question from a step that had nothing to report.
This commit is contained in:
pj committed
1 parent
944d1e9ad9
commit
1ea5ab0c59
3 files changed
+111
-17
No files matched your search
Vendored
+4
-4
@@ -1,4 +1,4 @@
|
||||
{"extractor_changes":{"extractor_0":{"prev":null,"curr":0}},"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":1,"timestamp":"0001-01-01T00:00:00Z"}
|
||||
{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":2,"timestamp":"0001-01-01T00:00:00Z"}
|
||||
{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":3,"timestamp":"0001-01-01T00:00:00Z"}
|
||||
{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":4,"timestamp":"0001-01-01T00:00:00Z"}
|
||||
{"extractor_changes":{"extractor_0":{"prev":null,"curr":0}},"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":1,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
|
||||
{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":2,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
|
||||
{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":3,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
|
||||
{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":4,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
|
||||
@@ -13,14 +13,31 @@ import (
|
||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||
)
|
||||
|
||||
// TraceVersion is stamped on every step this build writes. A step decoding to
|
||||
// version 0 predates the fields introduced with version 1 (the tree's stored
|
||||
// depths, per-step logs, per-step exceptions), which is what separates "this
|
||||
// trace cannot answer the question" from "this step had nothing to report".
|
||||
const TraceVersion = 1
|
||||
|
||||
type Step struct {
|
||||
Index int `json:"step"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Screen string `json:"screen,omitempty"`
|
||||
Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"`
|
||||
Index int `json:"step"`
|
||||
TraceVersion int `json:"trace_version,omitempty"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Screen string `json:"screen,omitempty"`
|
||||
Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"`
|
||||
// NextAction is the action chosen for the next iteration based on observing this step.
|
||||
NextAction *Action `json:"next_action,omitempty"`
|
||||
Exceptions []Exception `json:"exceptions,omitempty"`
|
||||
NextAction *Action `json:"next_action,omitempty"`
|
||||
// Logs are the platform log lines collected for this step, Exceptions the
|
||||
// uncaught errors read at verification time: the error surface behind
|
||||
// state.logs and state.exceptions, which the default properties read and
|
||||
// an offline oracle has no other source for.
|
||||
Logs []LogEntry `json:"logs,omitempty"`
|
||||
Exceptions []Exception `json:"exceptions,omitempty"`
|
||||
// Navigations are the document-replacing navigations seen since the
|
||||
// previous step: the app reloaded, submitted a form, or changed route.
|
||||
// Each one restarts the app's own runtime, so without them a reload and a
|
||||
// generator repeating itself read the same way in a trace.
|
||||
Navigations []Navigation `json:"navigations,omitempty"`
|
||||
Violations []string `json:"violations,omitempty"`
|
||||
Hierarchy *hierarchy.Tree `json:"hierarchy,omitempty"`
|
||||
Residuals map[string]json.RawMessage `json:"residuals,omitempty"`
|
||||
@@ -35,6 +52,11 @@ type Step struct {
|
||||
// count of executed actions cannot be inflated by steps whose action the
|
||||
// runner threw away. Empty when the action ran (or when none was chosen).
|
||||
ActionSkipped string `json:"action_skipped,omitempty"`
|
||||
// ObservationError names why this step's device read produced no tree,
|
||||
// empty when a tree was read. A screen with no elements on it is a tree,
|
||||
// so without this a step that observed nothing at all is indistinguishable
|
||||
// from a step that observed an app showing nothing.
|
||||
ObservationError string `json:"observation_error,omitempty"`
|
||||
// SkippedVerification is set true exactly when the verifier was skipped
|
||||
// for this step, so downstream tooling can tell a deliberately-skipped
|
||||
// step from one that was verified and came back clean.
|
||||
@@ -44,6 +66,12 @@ type Step struct {
|
||||
Witnesses map[string]Witness `json:"witnesses,omitempty"`
|
||||
}
|
||||
|
||||
// Navigation is one document-replacing navigation the run observed.
|
||||
type Navigation struct {
|
||||
URL string `json:"url"`
|
||||
UnixMillis int64 `json:"unix_millis,omitempty"`
|
||||
}
|
||||
|
||||
// Witness is the trace-side record of a property violation: why it fired, the
|
||||
// two steps a deferred obligation spans, and the extractor values behind it.
|
||||
type Witness struct {
|
||||
@@ -114,6 +142,15 @@ type PointRecord struct {
|
||||
Y int `json:"y"`
|
||||
}
|
||||
|
||||
// LogEntry mirrors one platform log line the runner collected for this step,
|
||||
// in the shape state.logs exposes to a spec.
|
||||
type LogEntry struct {
|
||||
UnixMillis int64 `json:"unix_millis,omitempty"`
|
||||
Level string `json:"level,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type Exception struct {
|
||||
Class string `json:"class"`
|
||||
Message string `json:"message,omitempty"`
|
||||
@@ -195,12 +232,15 @@ func (w *Writer) WriteMeta(meta Meta) error {
|
||||
return os.WriteFile(filepath.Join(w.directory, "meta.json"), body, 0o644)
|
||||
}
|
||||
|
||||
// WriteStep stamps the format version itself so no caller can write a step
|
||||
// that cannot be told apart from one written before the format changed.
|
||||
func (w *Writer) WriteStep(step Step) error {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
if w.file == nil {
|
||||
return fmt.Errorf("trace: writer is closed")
|
||||
}
|
||||
step.TraceVersion = TraceVersion
|
||||
return w.encoder.Encode(step)
|
||||
}
|
||||
|
||||
|
||||
@@ -143,11 +143,6 @@ func TestWriteStep_HierarchyAndResidualsRoundTrip(t *testing.T) {
|
||||
t.Errorf("residuals round-trip wrong: %s", got.Residuals["prop1"])
|
||||
}
|
||||
|
||||
// Intentionally-lossy contract: Tree marshals only Elements (Root and
|
||||
// Node.Children are json:"-"). The flat element list survives; tree
|
||||
// structure does not. Lock both halves so a regression that drops the
|
||||
// element list, or one that silently starts persisting structure the
|
||||
// replay UI would then depend on, is caught.
|
||||
if got.Hierarchy == nil {
|
||||
t.Fatal("hierarchy dropped from trace")
|
||||
}
|
||||
@@ -157,8 +152,22 @@ func TestWriteStep_HierarchyAndResidualsRoundTrip(t *testing.T) {
|
||||
if got.Hierarchy.Elements[1].Text != "hi" {
|
||||
t.Errorf("element field lost: %+v", got.Hierarchy.Elements[1])
|
||||
}
|
||||
if got.Hierarchy.Root != nil {
|
||||
t.Errorf("Root is json:\"-\" and must decode nil, got %+v", got.Hierarchy.Root)
|
||||
if got.Hierarchy.Root == nil {
|
||||
t.Fatal("tree structure not reconstructed from the stored form")
|
||||
}
|
||||
if got.Hierarchy.Root.ResourceID != "root" ||
|
||||
len(got.Hierarchy.Root.Children) != 1 {
|
||||
t.Fatalf("root rebuilt wrong: %+v", got.Hierarchy.Root.Element)
|
||||
}
|
||||
if resolved := got.Hierarchy.Find("id:child"); resolved == nil ||
|
||||
resolved.Text != "hi" {
|
||||
t.Errorf(
|
||||
"selector resolves online but not against the decoded tree: %+v",
|
||||
resolved,
|
||||
)
|
||||
}
|
||||
if resolved := got.Hierarchy.Find("id:child"); resolved != got.Hierarchy.Elements[1] {
|
||||
t.Error("decoded elements and the rebuilt nodes are different pointers")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -493,3 +502,48 @@ func TestWriteMeta_OmitsArmMembershipWhenUnset(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStepPredatingTheFormatIsDistinguishable is the backward-compatibility
|
||||
// contract: the existing corpus must still load, and a step from it must be
|
||||
// separable from one this build wrote with nothing to report, or "this trace
|
||||
// predates the format" reads as "this step had no logs".
|
||||
func TestStepPredatingTheFormatIsDistinguishable(t *testing.T) {
|
||||
const stored = `{"step":3,"timestamp":"2026-06-10T21:22:07Z",` +
|
||||
`"hierarchy":{"elements":[{"resourceId":"root"},{"resourceId":"child"}]}}`
|
||||
var old Step
|
||||
if err := json.Unmarshal([]byte(stored), &old); err != nil {
|
||||
t.Fatalf(
|
||||
"a trace written before the format change no longer loads: %v",
|
||||
err,
|
||||
)
|
||||
}
|
||||
if old.TraceVersion != 0 {
|
||||
t.Errorf(
|
||||
"trace_version = %d, want 0 for a step that predates the field",
|
||||
old.TraceVersion,
|
||||
)
|
||||
}
|
||||
if len(old.Hierarchy.Elements) != 2 || old.Hierarchy.Root != nil {
|
||||
t.Errorf("old hierarchy reinterpreted: elements=%d root=%v",
|
||||
len(old.Hierarchy.Elements), old.Hierarchy.Root)
|
||||
}
|
||||
|
||||
directory := t.TempDir()
|
||||
writer, _ := NewWriter(directory)
|
||||
defer writer.Close()
|
||||
if err := writer.WriteStep(Step{Index: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := os.ReadFile(filepath.Join(directory, "trace.jsonl"))
|
||||
var fresh Step
|
||||
if err := json.Unmarshal(body, &fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fresh.TraceVersion != TraceVersion {
|
||||
t.Errorf("a step with nothing to report stamped version %d, want %d",
|
||||
fresh.TraceVersion, TraceVersion)
|
||||
}
|
||||
if len(fresh.Logs) != 0 {
|
||||
t.Errorf("logs = %v, want none", fresh.Logs)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user