mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
* ci(folio): run gradle on jdk 21 for the metro plugin the metro gradle plugin folio builds with publishes org.gradle.jvm.version 21 and java 21 class files, so every leg failed at the folio build on a 17 runtime. local builds pass on jdk 25, which is why only ci saw it. * fix(build): clean pkg/spec/dist, not the dead spec-api path * chore: point stale spec-api comments at pkg/spec * fix(spec): publish src so an installed package carries the runtime entries * fix(testrun): alias the installed spec package so one module graph loads * fix(spec): export Direction, ScrollAction and LongPressAction from the entry * docs(spec): cut the package readme to a description and doc links * docs: say how the cli and spec package versions relate * fix(verifier): report whether the last action was confirmed applied Both hosts get applied: true when the runner saw the dispatch succeed and applied: null when it could not, so an unconfirmed action stops arriving at the spec as no action at all. * fix(runner): an apply error leaves the action's fate unknown, not undone A deadline that fires after the tap was dispatched leaves the effect committed. Reporting nil made the spec see an effect with no action to cause it, which is how the counting property convicts a healthy app. * fix(release): stage the sidecar jar at the renamed embed path * test(replay-ui): trace fixtures for the vacuity counts one real green run, one run that rendered nothing, one that judges every property at least once. * ci(replay-ui): count the steps each property judged the exit code says no property returned false; it does not say any property was ever evaluated. this reads the trace and reports judged vs declined per property, and fails when the step page never rendered. * test(replay-ui): cover the summary script from make test * ci(replay-ui): summarise through the vacuity script * docs(ci): explain the replay-ui judged/declined counts * fix(verifier): encode element-valued extractors into the trace An ax element exports with its find/findAll host functions attached, and json.Marshal refuses the whole value over them: json: unsupported type: func(goja.FunctionCall) goja.Value. The encoding failed, curr stayed nil, and the goja hosts (ios, android) recorded null for every element-valued extractor in both the per-step diff and the violation witness. Apply the web host's sanitize rule before marshaling, so one rule encodes an element on both hosts. * test(verifier): pin element encoding to one rule on both hosts * test(runner): assert an element reaches trace.jsonl and its witness * feat(spec): give state.lastAction an applied field Three states, not two: no action is a null lastAction, applied: true is an action the runner confirmed, applied: null is one it dispatched and never learned the fate of. * fix(folio): do not attribute an effect to an unconfirmed action submitChangesBalanceByTypedAmount and createdAccountHasNonZeroBalance both convict by pinning an effect on the last action, so both decline unless the runner saw it applied. The fixtures now say which fate they mean. * test(folio): an unconfirmed submit belongs in the window The count is an upper bound on the submits a window holds, so the tap that may have landed counts and committedTransactionsExceedSubmits has nothing to convict on. * test(runner): a tap that lands under a failed apply is not a double submit Drives the real folio counting predicates through the runner against a device that commits the tap and then times out. The double-submit case is the control: without it a green proves only that the property never fired. * test(verifier): pin the three lastAction states on both hosts The web page is handed the same applied field the goja object exposes, so a property cannot read one thing on native and another on web. * docs(spec-language): document the three lastAction states
This commit is contained in:
34 files changed
+1410
-158
No files matched your search
@@ -0,0 +1,121 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/trace"
|
||||
)
|
||||
|
||||
// elementExtractorSpec reads a live ax element, the shape every field and
|
||||
// button in examples/folio/sanderling/spec.ts is extracted with. The property
|
||||
// is false the moment the field is on screen, so the run records a witness
|
||||
// whose only interesting content is that element.
|
||||
const elementExtractorSpec = `
|
||||
import { actions, always, extract } from "@sanderling/spec";
|
||||
const amountField = extract("amountField", s => s.ax.find({ "resource-id": "TxnAmountField" }));
|
||||
globalThis.properties = {
|
||||
noAmountField: always(() => amountField.current === undefined),
|
||||
};
|
||||
globalThis.actions = actions(() => []);
|
||||
`
|
||||
|
||||
const amountFieldTreeJSON = `{
|
||||
"attributes": {"resource-id": "root", "bounds": "[0,0,400,800]"},
|
||||
"enabled": true,
|
||||
"children": [
|
||||
{"attributes": {"resource-id": "TxnAmountField", "text": "199", "bounds": "[0,100,400,160]"},
|
||||
"editable": true, "enabled": true, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
// TestRunner_TraceRecordsElementValuedExtractors is the guard on the artifact a
|
||||
// person opens to decide whether a conviction is real. An element-valued
|
||||
// extractor used to reach the trace as null on the goja hosts (ios, android):
|
||||
// its exported value carries the element's find/findAll host functions, which
|
||||
// json.Marshal refuses, so the encoding failed and both the per-step diff and
|
||||
// the witness recorded nothing. A witness that reads null for the field the
|
||||
// property fired on describes a state the property could not have fired in,
|
||||
// which is worse than a blank.
|
||||
func TestRunner_TraceRecordsElementValuedExtractors(t *testing.T) {
|
||||
state := newHarnessWithSpec(t, elementExtractorSpec)
|
||||
state.mock.HierarchyJSON = amountFieldTreeJSON
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 2,
|
||||
Driver: state.mock,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if !containsProperty(summary.Violations, "noAmountField") {
|
||||
t.Fatalf("noAmountField did not violate, so the element never reached a predicate: %v",
|
||||
summary.Violations)
|
||||
}
|
||||
|
||||
file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
type traceLine struct {
|
||||
Step int `json:"step"`
|
||||
ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"`
|
||||
Witnesses map[string]trace.Witness `json:"witnesses"`
|
||||
}
|
||||
changes, witnesses := 0, 0
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||
for scanner.Scan() {
|
||||
var line traceLine
|
||||
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||
t.Fatalf("trace line decode: %v", err)
|
||||
}
|
||||
if change, ok := line.ExtractorChanges["amountField"]; ok {
|
||||
changes++
|
||||
assertAmountField(t, fmt.Sprintf("step %d extractor_changes", line.Step), change.Curr)
|
||||
}
|
||||
for name, witness := range line.Witnesses {
|
||||
witnesses++
|
||||
assertAmountField(t, fmt.Sprintf("step %d %s witness", line.Step, name),
|
||||
witness.Extractors["amountField"])
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Fatalf("scan trace: %v", err)
|
||||
}
|
||||
if changes == 0 {
|
||||
t.Error("amountField never appears in extractor_changes; the element the run read is not in the trace")
|
||||
}
|
||||
if witnesses == 0 {
|
||||
t.Error("no witness reached the trace; nothing was compared")
|
||||
}
|
||||
}
|
||||
|
||||
// assertAmountField reads the recorded element the way a person opening the
|
||||
// trace would: the field's text is the number the property was judged on.
|
||||
func assertAmountField(t *testing.T, where string, recorded json.RawMessage) {
|
||||
t.Helper()
|
||||
var element struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.Unmarshal(recorded, &element); err != nil {
|
||||
t.Fatalf("%s: decode %s: %v", where, recorded, err)
|
||||
}
|
||||
if element.Text != "199" {
|
||||
t.Errorf("%s: recorded element is %s, want its text to read 199", where, recorded)
|
||||
}
|
||||
}
|
||||
@@ -298,11 +298,18 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
logger.Warn("apply error; marking step transitional", "step", stepIndex, "err", err)
|
||||
transitional = true
|
||||
applySkipped = true
|
||||
lastAction = nil
|
||||
// The error says the call failed, not that the gesture never
|
||||
// reached the app: a deadline that fires after dispatch leaves
|
||||
// the effect committed. Reporting no action here would let a
|
||||
// property convict the app for an effect with no cause, so the
|
||||
// action is reported with its fate unknown instead.
|
||||
unconfirmed := nextAction
|
||||
lastAction = &unconfirmed
|
||||
} else {
|
||||
consecutiveApplyFailures = 0
|
||||
actionCopy := nextAction
|
||||
lastAction = &actionCopy
|
||||
applied := nextAction
|
||||
applied.Applied = true
|
||||
lastAction = &applied
|
||||
}
|
||||
} else {
|
||||
lastAction = nil
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/driver"
|
||||
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
|
||||
)
|
||||
|
||||
// An apply error is not proof that nothing landed. An RPC deadline that fires
|
||||
// after the tap was dispatched leaves the transaction committed, and a runner
|
||||
// that reports "no action" for it hands
|
||||
// submitCommitsOneTransactionPerAction a rise of one transaction against a
|
||||
// window of zero submits: a conviction manufactured out of the runner's own
|
||||
// uncertainty, on the property carrying most of the detection on android.
|
||||
//
|
||||
// The spec below is the real folio predicate pair, imported from the example,
|
||||
// so what this asserts is the verdict the shipped property reaches.
|
||||
const uncertainApplySpecTemplate = `
|
||||
import { actions, always, extract, next, Tap } from "@sanderling/spec";
|
||||
import {
|
||||
committedTransactionsExceedSubmits,
|
||||
countSubmitsInWindow,
|
||||
} from "%s";
|
||||
|
||||
let submits = 0;
|
||||
const submitsInWindow = extract("submitsInWindow", state => {
|
||||
const window = countSubmitsInWindow({
|
||||
previousCount: submits,
|
||||
lastAction: state.lastAction,
|
||||
fresh: true,
|
||||
});
|
||||
submits = window.next;
|
||||
return window.reported;
|
||||
});
|
||||
|
||||
const counts = extract("counts", state => {
|
||||
const text = state.ax.find("id:TxnCount")?.text;
|
||||
return text ? { Travel: parseInt(text, 10) } : null;
|
||||
});
|
||||
|
||||
globalThis.properties = {
|
||||
submitCommitsOneTransactionPerAction: always(
|
||||
next(() =>
|
||||
!committedTransactionsExceedSubmits({
|
||||
countsBefore: counts.previous ?? null,
|
||||
countsAfter: counts.current,
|
||||
submitsInWindow: submitsInWindow.current,
|
||||
}),
|
||||
),
|
||||
),
|
||||
};
|
||||
globalThis.actions = actions(() => [Tap({ on: "id:TxnSubmit" })]);
|
||||
`
|
||||
|
||||
const homeWithTxnCount = `{"attributes":{"resource-id":"HomeScreen"},"children":[
|
||||
{"attributes":{"resource-id":"TxnCount","text":"%d"},"children":[]},
|
||||
{"attributes":{"resource-id":"TxnSubmit","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true}
|
||||
]}`
|
||||
|
||||
// dispatchThenFailDriver is the device condition the runner cannot see through:
|
||||
// the tap reaches the app and commits, then the call the runner is waiting on
|
||||
// times out. Every later hierarchy read shows the committed transactions.
|
||||
type dispatchThenFailDriver struct {
|
||||
*mockdriver.Driver
|
||||
commitsPerTap int64
|
||||
committed atomic.Int64
|
||||
}
|
||||
|
||||
func (d *dispatchThenFailDriver) Tap(context.Context, int, int) error {
|
||||
return d.dispatchThenFail()
|
||||
}
|
||||
|
||||
func (d *dispatchThenFailDriver) TapSelector(context.Context, string) error {
|
||||
return d.dispatchThenFail()
|
||||
}
|
||||
|
||||
func (d *dispatchThenFailDriver) dispatchThenFail() error {
|
||||
d.committed.Add(d.commitsPerTap)
|
||||
return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded")
|
||||
}
|
||||
|
||||
func (d *dispatchThenFailDriver) Snapshot(context.Context) (string, driver.Image, error) {
|
||||
return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil
|
||||
}
|
||||
|
||||
func TestRunner_ApplyErrorAfterDispatchDoesNotConvictTheSubmitCountingProperty(t *testing.T) {
|
||||
predicates, err := filepath.Abs("../../examples/folio/sanderling/predicates.ts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spec := fmt.Sprintf(uncertainApplySpecTemplate, predicates)
|
||||
|
||||
run := func(t *testing.T, commitsPerTap int64) []ViolationRecord {
|
||||
t.Helper()
|
||||
state := newHarnessWithSpec(t, spec)
|
||||
device := &dispatchThenFailDriver{Driver: state.mock, commitsPerTap: commitsPerTap}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 2,
|
||||
Driver: device,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if summary.Steps != 2 {
|
||||
t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps)
|
||||
}
|
||||
if got := device.committed.Load(); got != commitsPerTap*2 {
|
||||
t.Fatalf("the device committed %d transaction(s), want %d; the taps never reached it",
|
||||
got, commitsPerTap*2)
|
||||
}
|
||||
return summary.Violations
|
||||
}
|
||||
|
||||
t.Run("one transaction per tap is not a double submit", func(t *testing.T) {
|
||||
if violations := run(t, 1); len(violations) != 0 {
|
||||
t.Errorf("the counting property convicted a healthy app: %v\n"+
|
||||
"one transaction rose against a submit the runner dispatched but "+
|
||||
"could not confirm, and the spec was told no action happened",
|
||||
violations)
|
||||
}
|
||||
})
|
||||
|
||||
// The control. Without it a green above proves nothing: a property that
|
||||
// never sees a comparable pair is silently vacuous and reports the same
|
||||
// empty violation list.
|
||||
t.Run("two transactions per tap still convicts", func(t *testing.T) {
|
||||
violations := run(t, 2)
|
||||
if len(violations) == 0 {
|
||||
t.Fatal("the counting property missed a double submit; the harness never " +
|
||||
"put the property in a position to fire, so the case above proves nothing")
|
||||
}
|
||||
if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" {
|
||||
t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package runner
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -71,7 +72,47 @@ func TestRunner_WebInstallsLastActionInThePage(t *testing.T) {
|
||||
// Every later step carries what the runner actually applied. The shape is
|
||||
// the goja host's (internal/verifier/marshal.go lastActionFields), pinned
|
||||
// against it by TestLastAction_WebJSONMatchesTheGojaObject.
|
||||
const want = `{"kind":"Tap","on":"id:TxnSubmit"}`
|
||||
const want = `{"kind":"Tap","applied":true,"on":"id:TxnSubmit"}`
|
||||
if web.installed[1] != want {
|
||||
t.Errorf("step 2 installed %s, want %s", web.installed[1], want)
|
||||
}
|
||||
}
|
||||
|
||||
// failingTapWebDriver dispatches the tap and then fails the call, the shape an
|
||||
// RPC deadline takes: the page has the click, the runner has an error.
|
||||
type failingTapWebDriver struct {
|
||||
*tappingWebDriver
|
||||
}
|
||||
|
||||
func (d *failingTapWebDriver) Tap(context.Context, int, int) error {
|
||||
return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded")
|
||||
}
|
||||
|
||||
// The web leg of the same three states the goja host reports. "applied":null is
|
||||
// not "no action": a property gated on the last action still sees the tap and
|
||||
// decides for itself, which it cannot do if the page is handed a bare null.
|
||||
func TestRunner_WebInstallsAnUnconfirmedActionWithItsFateUnknown(t *testing.T) {
|
||||
state := newHarnessWithSpec(t, lastActionSpec)
|
||||
web := &failingTapWebDriver{tappingWebDriver: &tappingWebDriver{Driver: state.mock}}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 2,
|
||||
Driver: web,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
}); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
if len(web.installed) < 2 {
|
||||
t.Fatalf("the page was handed lastAction %d time(s); the web path never installed it",
|
||||
len(web.installed))
|
||||
}
|
||||
const want = `{"kind":"Tap","applied":null,"on":"id:TxnSubmit"}`
|
||||
if web.installed[1] != want {
|
||||
t.Errorf("step 2 installed %s, want %s", web.installed[1], want)
|
||||
}
|
||||
|
||||
@@ -350,16 +350,20 @@ func resolveRuntimeSibling(specAPIPath, userSpecPath, filename string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// resolveSpecAPIPath returns the path to pkg/spec/src/index.ts inside
|
||||
// a sanderling source checkout, searched upward from the spec file and the cwd.
|
||||
// Returns "" when not found, in which case esbuild resolves @sanderling/spec via
|
||||
// node_modules the way a downstream user's project would.
|
||||
// resolveSpecAPIPath returns the path to the spec API's index.ts: a sanderling
|
||||
// source checkout first, searched upward from the spec file and the cwd, then
|
||||
// an installed node_modules/@sanderling/spec. Aliasing the installed copy is
|
||||
// what keeps the spec and the runtime entry on one module graph; resolving the
|
||||
// bare specifier through package.json "exports" would load dist/ alongside the
|
||||
// runtime's src/ and give sampler-rng.ts two instances.
|
||||
func resolveSpecAPIPath(specPath string) string {
|
||||
var candidates []string
|
||||
var checkout, installed []string
|
||||
if absoluteSpec, err := filepath.Abs(specPath); err == nil {
|
||||
directory := filepath.Dir(absoluteSpec)
|
||||
for {
|
||||
candidates = append(candidates, filepath.Join(directory, "pkg/spec/src/index.ts"))
|
||||
checkout = append(checkout, filepath.Join(directory, "pkg/spec/src/index.ts"))
|
||||
installed = append(installed,
|
||||
filepath.Join(directory, "node_modules/@sanderling/spec/src/index.ts"))
|
||||
parent := filepath.Dir(directory)
|
||||
if parent == directory {
|
||||
break
|
||||
@@ -368,9 +372,9 @@ func resolveSpecAPIPath(specPath string) string {
|
||||
}
|
||||
}
|
||||
if cwd, err := os.Getwd(); err == nil {
|
||||
candidates = append(candidates, filepath.Join(cwd, "pkg/spec/src/index.ts"))
|
||||
checkout = append(checkout, filepath.Join(cwd, "pkg/spec/src/index.ts"))
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
for _, candidate := range append(checkout, installed...) {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ package testrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -304,3 +306,153 @@ func TestLaunchAppBoundsWedgedDriver(t *testing.T) {
|
||||
t.Fatal("launchApp never returned: the pre-run launch is unbounded, so a wedged driver hangs the run forever")
|
||||
}
|
||||
}
|
||||
|
||||
// repoFile walks up from the test's working directory and returns the absolute
|
||||
// path of rel inside the sanderling checkout.
|
||||
func repoFile(t *testing.T, rel string) string {
|
||||
t.Helper()
|
||||
directory, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
candidate := filepath.Join(directory, rel)
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate
|
||||
}
|
||||
parent := filepath.Dir(directory)
|
||||
if parent == directory {
|
||||
t.Fatalf("%s not found above the test directory", rel)
|
||||
}
|
||||
directory = parent
|
||||
}
|
||||
}
|
||||
|
||||
// publishedFiles returns the "files" entries of pkg/spec/package.json, the
|
||||
// exact set npm ships in the @sanderling/spec tarball.
|
||||
func publishedFiles(t *testing.T) []string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(repoFile(t, "pkg/spec/package.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var manifest struct {
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return manifest.Files
|
||||
}
|
||||
|
||||
// installPublishedPackage reproduces what `npm install @sanderling/spec`
|
||||
// unpacks into node_modules: only the paths package.json publishes.
|
||||
func installPublishedPackage(t *testing.T, dest string) {
|
||||
t.Helper()
|
||||
specDir := filepath.Dir(repoFile(t, "pkg/spec/package.json"))
|
||||
for _, entry := range publishedFiles(t) {
|
||||
source := filepath.Join(specDir, entry)
|
||||
if _, err := os.Stat(source); err != nil {
|
||||
continue
|
||||
}
|
||||
copyTree(t, source, filepath.Join(dest, entry))
|
||||
}
|
||||
}
|
||||
|
||||
func copyTree(t *testing.T, source, dest string) {
|
||||
t.Helper()
|
||||
err := filepath.WalkDir(source, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relative, err := filepath.Rel(source, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := filepath.Join(dest, relative)
|
||||
if entry.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(target, data, 0o644)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRuntimeSibling_PublishedPackageShipsTheRuntimes pins npm's "files"
|
||||
// list against the resolver that consumes it. The tarball shipped dist/ alone
|
||||
// while the node_modules fallback looks for src/goja-runtime.ts, so every
|
||||
// `npm install @sanderling/spec` user hit "goja-runtime.ts not found".
|
||||
func TestResolveRuntimeSibling_PublishedPackageShipsTheRuntimes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
installPublishedPackage(t, filepath.Join(root, "node_modules", "@sanderling", "spec"))
|
||||
specPath := filepath.Join(root, "spec.ts")
|
||||
if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, filename := range []string{"goja-runtime.ts", "web-runtime.ts"} {
|
||||
if resolveRuntimeSibling("", specPath, filename) == "" {
|
||||
t.Errorf("%s unreachable from a published install; package.json publishes %v",
|
||||
filename, publishedFiles(t))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrepareBundleInputs_InstalledPackageSharesOneModuleGraph pins the
|
||||
// downstream case: with no sanderling checkout above the spec, the aliases and
|
||||
// the runtime entry must name the SAME installed copy. An unset alias let
|
||||
// esbuild resolve @sanderling/spec to dist/ while the runtime came from src/,
|
||||
// which loads sampler-rng.ts twice; from(), strings(), integers() and emails()
|
||||
// then read an rng the picker never set and collapse to a fixed default.
|
||||
func TestPrepareBundleInputs_InstalledPackageSharesOneModuleGraph(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
installed := filepath.Join(root, "node_modules", "@sanderling", "spec")
|
||||
installPublishedPackage(t, installed)
|
||||
specPath := filepath.Join(root, "sanderling", "spec.ts")
|
||||
if err := os.MkdirAll(filepath.Dir(specPath), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chdir(cwd) })
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
prep, err := prepareBundleInputs(Options{Spec: specPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source := filepath.Join(installed, "src")
|
||||
want := map[string]string{
|
||||
"@sanderling/spec": filepath.Join(source, "index.ts"),
|
||||
"@sanderling/spec/defaults": filepath.Join(source, "defaults/index.ts"),
|
||||
"@sanderling/spec/defaults/properties": filepath.Join(source, "defaults/properties.ts"),
|
||||
}
|
||||
for key, wantValue := range want {
|
||||
if prep.aliases[key] != wantValue {
|
||||
t.Errorf("alias %q = %q, want %q", key, prep.aliases[key], wantValue)
|
||||
}
|
||||
}
|
||||
if got := prep.gojaRuntimePath; got != filepath.Join(source, "goja-runtime.ts") {
|
||||
t.Errorf("gojaRuntimePath = %q, want it beside the aliased index.ts", got)
|
||||
}
|
||||
if got := resolveWebRuntimePath(prep.specAPIPath, specPath); got != filepath.Join(source, "web-runtime.ts") {
|
||||
t.Errorf("webRuntimePath = %q, want it beside the aliased index.ts", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package verifier
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const elementTreeJSON = `{
|
||||
"attributes": {"resource-id": "root", "bounds": "[0,0,400,800]"},
|
||||
"enabled": true,
|
||||
"children": [
|
||||
{"attributes": {"resource-id": "TxnAmountField", "text": "199", "bounds": "[0,100,400,160]"},
|
||||
"editable": true, "enabled": true, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
const elementExtractorSpec = `
|
||||
const field = __sanderling__.extract(state => state.ax.find({ "resource-id": "TxnAmountField" }), "field");
|
||||
globalThis.properties = {};
|
||||
`
|
||||
|
||||
// canonicalElement is the trace's record of one ax element, written out in the
|
||||
// key order encoding/json emits. It is the contract both hosts owe the replay
|
||||
// UI: an element the reader can read, with no host-function members and nothing
|
||||
// dropped. Keys the two hosts disagree on (a DOM has no `checked`, a native
|
||||
// tree has no `dataset`) are each host's own business; the ENCODING is not.
|
||||
const canonicalElement = `{
|
||||
"__sanderlingSelector": "resource-id:TxnAmountField",
|
||||
"attrs": {
|
||||
"bounds": "[0,100,400,160]",
|
||||
"editable": "true",
|
||||
"enabled": "true",
|
||||
"resource-id": "TxnAmountField",
|
||||
"text": "199"
|
||||
},
|
||||
"bounds": {"bottom": 160, "left": 0, "right": 400, "top": 100},
|
||||
"checked": false,
|
||||
"class": "",
|
||||
"clickable": false,
|
||||
"desc": "",
|
||||
"editable": true,
|
||||
"enabled": true,
|
||||
"focused": false,
|
||||
"id": "TxnAmountField",
|
||||
"selected": false,
|
||||
"text": "199",
|
||||
"x": 200,
|
||||
"y": 130
|
||||
}`
|
||||
|
||||
// TestExtractorEncoding_ElementIsIdenticalOnBothHosts holds the two extractor
|
||||
// paths to one encoding of one element. The goja hosts (ios, android) run the
|
||||
// getter in-process and encode the value it returned; the web host runs it in
|
||||
// V8 and injects the page's reading through OverrideExtractorValues. A reader
|
||||
// opening a trace does not know which host wrote it, so the same element has to
|
||||
// land as the same bytes either way.
|
||||
//
|
||||
// The goja side used to write null here: an ax element carries find/findAll as
|
||||
// host functions and json.Marshal refuses the whole object over them.
|
||||
func TestExtractorEncoding_ElementIsIdenticalOnBothHosts(t *testing.T) {
|
||||
want := compactJSON(t, canonicalElement)
|
||||
|
||||
native := newVerifier(t)
|
||||
mustLoad(t, native, elementExtractorSpec)
|
||||
pushTree(t, native, elementTreeJSON)
|
||||
fromGoja := string(native.extractors[0].curr)
|
||||
if fromGoja != want {
|
||||
t.Errorf("goja host encoded the element as\n %s\nwant\n %s", fromGoja, want)
|
||||
}
|
||||
|
||||
web := newVerifier(t)
|
||||
mustLoad(t, web, elementExtractorSpec)
|
||||
if err := web.PushSnapshot(SnapshotInput{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := web.OverrideExtractorValues(map[int]json.RawMessage{0: json.RawMessage(want)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fromWeb := string(web.extractors[0].curr)
|
||||
if fromWeb != fromGoja {
|
||||
t.Errorf("the same element reaches the trace as\n %s\non the web host and\n %s\non goja",
|
||||
fromWeb, fromGoja)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractorEncoding_MirrorsTheWebSanitizeRule pins the goja host to the
|
||||
// rule the web host applies before a reading leaves the page (sanitize in
|
||||
// pkg/spec/src/web-runtime.ts, asserted there by the "sanitize ..." tests in
|
||||
// pkg/spec/test/web-runtime.test.ts). Two hosts encoding one value two ways is
|
||||
// the same defect as encoding it not at all: the reader cannot line the traces
|
||||
// up.
|
||||
func TestExtractorEncoding_MirrorsTheWebSanitizeRule(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
expression string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "function-valued properties are dropped",
|
||||
expression: `({ keep: 1, fn: () => 7 })`,
|
||||
want: `{"keep":1}`,
|
||||
},
|
||||
{
|
||||
name: "a top-level function is not a value",
|
||||
expression: `(() => 7)`,
|
||||
want: `null`,
|
||||
},
|
||||
{
|
||||
name: "a self-referential cycle breaks instead of overflowing",
|
||||
expression: `(() => { const a = { name: "root" }; a.self = a; return a; })()`,
|
||||
want: `{"name":"root","self":null}`,
|
||||
},
|
||||
{
|
||||
name: "arrays and nested plain values are preserved",
|
||||
expression: `({ items: [1, "two", { ok: true }] })`,
|
||||
want: `{"items":[1,"two",{"ok":true}]}`,
|
||||
},
|
||||
{
|
||||
name: "a non-finite number is not a value",
|
||||
expression: `Number("nope")`,
|
||||
want: `null`,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := encodeSpecValue(t, test.expression); got != test.want {
|
||||
t.Errorf("encoded as %s, want %s", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractorEncoding_BoundsRecursionPastTheDepthLimit mirrors the web host's
|
||||
// depth cap. state.ax hands out no cyclic element, but a spec returning a value
|
||||
// it built itself can nest without end, and a walk with no bound takes the run
|
||||
// down with a stack overflow.
|
||||
func TestExtractorEncoding_BoundsRecursionPastTheDepthLimit(t *testing.T) {
|
||||
encoded := encodeSpecValue(t, `(() => {
|
||||
let deep = { leaf: true };
|
||||
for (let i = 0; i < 40; i++) deep = { next: deep };
|
||||
return deep;
|
||||
})()`)
|
||||
|
||||
var node any
|
||||
if err := json.Unmarshal([]byte(encoded), &node); err != nil {
|
||||
t.Fatalf("decode %s: %v", encoded, err)
|
||||
}
|
||||
for depth := 0; depth < recordableMaxDepth; depth++ {
|
||||
object, ok := node.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("depth %d: recursion stopped early at %v", depth, node)
|
||||
}
|
||||
node = object["next"]
|
||||
}
|
||||
if node != nil {
|
||||
t.Errorf("depth %d is %v, want null", recordableMaxDepth, node)
|
||||
}
|
||||
}
|
||||
|
||||
// encodeSpecValue returns what the trace records for an extractor whose getter
|
||||
// returned the given expression.
|
||||
func encodeSpecValue(t *testing.T, expression string) string {
|
||||
t.Helper()
|
||||
verifier := newVerifier(t)
|
||||
mustLoad(t, verifier, "__sanderling__.extract(state => "+expression+", \"value\");\nglobalThis.properties = {};")
|
||||
if err := verifier.PushSnapshot(SnapshotInput{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(verifier.extractors[0].curr)
|
||||
}
|
||||
|
||||
func compactJSON(t *testing.T, source string) string {
|
||||
t.Helper()
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, []byte(source)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return compact.String()
|
||||
}
|
||||
@@ -26,7 +26,7 @@ type stateInput struct {
|
||||
}
|
||||
|
||||
// stateObject builds the JS-side `state` object matching the State type from
|
||||
// pkg/spec-api. Fields beyond snapshots/ax are included when the caller
|
||||
// pkg/spec. Fields beyond snapshots/ax are included when the caller
|
||||
// populated them on stateInput.
|
||||
func stateObject(runtime *goja.Runtime, input stateInput) (*goja.Object, error) {
|
||||
state := runtime.NewObject()
|
||||
@@ -344,7 +344,19 @@ func lastActionFields(action *Action) []actionField {
|
||||
point := func(x, y int) []actionField {
|
||||
return []actionField{{key: "x", value: x}, {key: "y", value: y}}
|
||||
}
|
||||
fields := []actionField{{key: "kind", value: string(action.Kind)}}
|
||||
// An action whose apply call failed is not an action that did not happen:
|
||||
// the dispatch may have landed before the error. That is unknown, and
|
||||
// unknown is null here for the same reason every other absence in the spec
|
||||
// surface is, so a property decides for itself instead of being handed a
|
||||
// "nothing happened" the runner cannot vouch for.
|
||||
var applied any
|
||||
if action.Applied {
|
||||
applied = true
|
||||
}
|
||||
fields := []actionField{
|
||||
{key: "kind", value: string(action.Kind)},
|
||||
{key: "applied", value: applied},
|
||||
}
|
||||
if action.On != "" {
|
||||
fields = append(fields, actionField{key: "on", value: action.On})
|
||||
}
|
||||
@@ -397,7 +409,7 @@ func objectFromFields(runtime *goja.Runtime, fields []actionField) *goja.Object
|
||||
// has no Go-side state object to read: the runner pushes this JSON into the
|
||||
// page before each extractor evaluation. A nil action encodes as JSON null,
|
||||
// the same value the goja host reports on the first step of a run and after a
|
||||
// step whose action was never applied.
|
||||
// step whose action was never dispatched.
|
||||
func EncodeLastAction(action *Action) json.RawMessage {
|
||||
if action == nil {
|
||||
return json.RawMessage("null")
|
||||
|
||||
@@ -168,6 +168,7 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) {
|
||||
}{
|
||||
{"nil", nil},
|
||||
{"Tap", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", X: 12, Y: 34}},
|
||||
{"TapApplied", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}},
|
||||
{"TapWithoutSelector", &Action{Kind: ActionKindTap, X: 12, Y: 34}},
|
||||
{"DoubleTap", &Action{Kind: ActionKindDoubleTap, On: `desc:say "hi" <b>`}},
|
||||
{"InputText", &Action{Kind: ActionKindInputText, On: "id:field", Text: "50"}},
|
||||
@@ -193,3 +194,42 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A spec has to be able to tell three things apart: no action ran, an action
|
||||
// ran, and an action was dispatched whose fate the runner cannot vouch for.
|
||||
// The third used to be reported as the first, which is how a property that
|
||||
// reasons "an effect landed with no action to cause it" convicts an app over
|
||||
// an RPC deadline.
|
||||
func TestLastAction_SeparatesNoActionFromAnActionOfUnknownFate(t *testing.T) {
|
||||
verifier := newVerifier(t)
|
||||
mustLoad(t, verifier, `
|
||||
globalThis.fate = __sanderling__.extract(state =>
|
||||
state.lastAction === null ? "no action"
|
||||
: state.lastAction.applied === true ? "applied"
|
||||
: state.lastAction.applied === null ? "unknown"
|
||||
: "unreadable");
|
||||
`)
|
||||
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
action *Action
|
||||
want string
|
||||
}{
|
||||
{"nothing ran", nil, "no action"},
|
||||
{"dispatch confirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}, "applied"},
|
||||
{"dispatch unconfirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit"}, "unknown"},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
if err := verifier.PushSnapshot(SnapshotInput{
|
||||
Snapshots: Snapshots{},
|
||||
LastAction: testCase.action,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handle := verifier.runtime.GlobalObject().Get("fate").ToObject(verifier.runtime)
|
||||
if got := handle.Get("current").String(); got != testCase.want {
|
||||
t.Errorf("the spec read %q, want %q", got, testCase.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,11 @@ type Action struct {
|
||||
// Direction is the scroll direction for ActionKindScroll: one of "up",
|
||||
// "down", "left", "right". Empty for every other kind.
|
||||
Direction string
|
||||
// Applied is meaningful only on the action a step reports to the spec as
|
||||
// state.lastAction: true when the runner saw the dispatch succeed, false
|
||||
// when the apply call failed and nothing can say whether the action
|
||||
// reached the app. The spec is told which of the two it is.
|
||||
Applied bool
|
||||
}
|
||||
|
||||
// LogEntry mirrors a logcat line captured between steps.
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
@@ -356,21 +358,76 @@ func (v *Verifier) runExtractor(extractor *extractorState, state goja.Value) (go
|
||||
}
|
||||
|
||||
// encodeExtractorValue produces a stable JSON encoding of an extractor's
|
||||
// current value for diff comparison. goja values that don't survive Export
|
||||
// (e.g. wrapped host functions) yield nil; callers treat nil as "unknown" and
|
||||
// emit no diff entry.
|
||||
// current value for diff comparison. Values that still don't survive encoding
|
||||
// yield nil; callers treat nil as "unknown" and emit no diff entry.
|
||||
func encodeExtractorValue(value goja.Value) []byte {
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return []byte("null")
|
||||
}
|
||||
exported := value.Export()
|
||||
body, err := json.Marshal(exported)
|
||||
body, err := json.Marshal(recordableValue(value.Export(), 0, map[uintptr]bool{}))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// recordableMaxDepth mirrors SANITIZE_MAX_DEPTH in pkg/spec/src/web-runtime.ts.
|
||||
const recordableMaxDepth = 32
|
||||
|
||||
// recordableValue applies the web host's sanitize rule (web-runtime.ts) to an
|
||||
// exported goja value: function members are dropped, a cycle or a branch past
|
||||
// the depth cap becomes null, and a non-finite number becomes null. One rule on
|
||||
// both hosts is what lets the replay UI render a trace without the reader
|
||||
// having to know which host produced it. An ax element carries its find and
|
||||
// findAll host functions, and json.Marshal rejects the whole element over them,
|
||||
// so without this an element-valued extractor reached the trace as null.
|
||||
func recordableValue(value any, depth int, seen map[uintptr]bool) any {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
address := reflect.ValueOf(typed).Pointer()
|
||||
if depth >= recordableMaxDepth || seen[address] {
|
||||
return nil
|
||||
}
|
||||
seen[address] = true
|
||||
members := make(map[string]any, len(typed))
|
||||
for key, member := range typed {
|
||||
if reflect.ValueOf(member).Kind() == reflect.Func {
|
||||
continue
|
||||
}
|
||||
members[key] = recordableValue(member, depth+1, seen)
|
||||
}
|
||||
return members
|
||||
case []any:
|
||||
if depth >= recordableMaxDepth {
|
||||
return nil
|
||||
}
|
||||
// Every zero-length allocation shares one address, so tracking an empty
|
||||
// array would identify it as every other empty array. It cannot close a
|
||||
// cycle either way.
|
||||
if len(typed) > 0 {
|
||||
address := reflect.ValueOf(typed).Pointer()
|
||||
if seen[address] {
|
||||
return nil
|
||||
}
|
||||
seen[address] = true
|
||||
}
|
||||
members := make([]any, len(typed))
|
||||
for index, member := range typed {
|
||||
members[index] = recordableValue(member, depth+1, seen)
|
||||
}
|
||||
return members
|
||||
case float64:
|
||||
if math.IsNaN(typed) || math.IsInf(typed, 0) {
|
||||
return nil
|
||||
}
|
||||
return typed
|
||||
}
|
||||
if reflect.ValueOf(value).Kind() == reflect.Func {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// ChangedExtractors returns the named extractors whose value changed between
|
||||
// the prior PushSnapshot and the current one. The map is keyed by extractor
|
||||
// name; unnamed extractors (extractor_N fallback) are included so the replay
|
||||
|
||||
Reference in new issue
Block a user