mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
test(runner): an overlay dismissal must not convict the counting property
This commit is contained in:
pj committed
1 parent
578ead6554
commit
94a1d59d60
2 files changed
+277
-141
No files matched your search
@@ -0,0 +1,277 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/driver"
|
||||
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
|
||||
)
|
||||
|
||||
const guardedBundleID = "app.folio"
|
||||
|
||||
// committingDevice is a device whose submit taps commit transactions the next
|
||||
// hierarchy read shows, and which can say how many it has committed so a test
|
||||
// can prove the taps landed before reading anything into a verdict.
|
||||
type committingDevice interface {
|
||||
driver.DeviceDriver
|
||||
commits() int64
|
||||
}
|
||||
|
||||
// leavesForegroundAfterSubmitDriver is the condition the app-scope guard exists
|
||||
// for: the submit tap lands and commits, and the app is no longer the
|
||||
// foreground app by the time the next step looks. Folio's transactions are in
|
||||
// sqlite, so the commit survives the relaunch and the next reading shows it.
|
||||
type leavesForegroundAfterSubmitDriver struct {
|
||||
*mockdriver.Driver
|
||||
commitsPerTap int64
|
||||
committed atomic.Int64
|
||||
away atomic.Bool
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) Tap(context.Context, int, int) error {
|
||||
return d.commitThenLeave()
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) TapSelector(context.Context, string) error {
|
||||
return d.commitThenLeave()
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) commitThenLeave() error {
|
||||
d.committed.Add(d.commitsPerTap)
|
||||
d.away.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) commits() int64 { return d.committed.Load() }
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) Launch(
|
||||
ctx context.Context,
|
||||
bundleID string,
|
||||
clearState bool,
|
||||
env map[string]string,
|
||||
) error {
|
||||
d.away.Store(false)
|
||||
return d.Driver.Launch(ctx, bundleID, clearState, env)
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) ForegroundApp(context.Context) (string, error) {
|
||||
if d.away.Load() {
|
||||
return "com.android.launcher", nil
|
||||
}
|
||||
return guardedBundleID, nil
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) FocusedWindowApp(ctx context.Context) (string, error) {
|
||||
return d.ForegroundApp(ctx)
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) Snapshot(context.Context) (string, driver.Image, error) {
|
||||
return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil
|
||||
}
|
||||
|
||||
// obscuredAfterSubmitDriver is the other half of the same guard: the app stays
|
||||
// the resumed activity, but a system window (the notification shade) owns the
|
||||
// focused window when the next step looks, and the guard presses back to
|
||||
// collapse it.
|
||||
type obscuredAfterSubmitDriver struct {
|
||||
*mockdriver.Driver
|
||||
commitsPerTap int64
|
||||
committed atomic.Int64
|
||||
obscured atomic.Bool
|
||||
}
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) Tap(context.Context, int, int) error {
|
||||
return d.commitThenObscure()
|
||||
}
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) TapSelector(context.Context, string) error {
|
||||
return d.commitThenObscure()
|
||||
}
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) commitThenObscure() error {
|
||||
d.committed.Add(d.commitsPerTap)
|
||||
d.obscured.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) commits() int64 { return d.committed.Load() }
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) PressKey(ctx context.Context, key string) error {
|
||||
if key == "back" {
|
||||
d.obscured.Store(false)
|
||||
}
|
||||
return d.Driver.PressKey(ctx, key)
|
||||
}
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) ForegroundApp(context.Context) (string, error) {
|
||||
return guardedBundleID, nil
|
||||
}
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) FocusedWindowApp(context.Context) (string, error) {
|
||||
if d.obscured.Load() {
|
||||
return "com.android.systemui", nil
|
||||
}
|
||||
return guardedBundleID, nil
|
||||
}
|
||||
|
||||
func (d *obscuredAfterSubmitDriver) Snapshot(context.Context) (string, driver.Image, error) {
|
||||
return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil
|
||||
}
|
||||
|
||||
// runTwoSubmitSteps drives two steps of the shipped folio counting property
|
||||
// against a device that commits on every tap, and hands back what the property
|
||||
// decided. Both steps have to run: the first arms the comparison, the second is
|
||||
// where the guard fires and the pair is judged.
|
||||
func runTwoSubmitSteps(
|
||||
t *testing.T,
|
||||
state *harness,
|
||||
device committingDevice,
|
||||
commitsPerTap int64,
|
||||
) []ViolationRecord {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 2,
|
||||
BundleID: guardedBundleID,
|
||||
Driver: device,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if summary.Steps != 2 {
|
||||
t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair",
|
||||
summary.Steps)
|
||||
}
|
||||
if got := device.commits(); got != commitsPerTap*2 {
|
||||
t.Fatalf("the device committed %d transaction(s), want %d; the taps never reached it",
|
||||
got, commitsPerTap*2)
|
||||
}
|
||||
return summary.Violations
|
||||
}
|
||||
|
||||
func countMockActions(state *harness, kind mockdriver.ActionKind, key string) int {
|
||||
count := 0
|
||||
for _, action := range state.mock.Actions() {
|
||||
if action.Kind != kind {
|
||||
continue
|
||||
}
|
||||
if key != "" && action.Key != key {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func specWithFolioPredicates(t *testing.T) string {
|
||||
t.Helper()
|
||||
predicates, err := filepath.Abs("../../examples/folio/sanderling/predicates.ts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fmt.Sprintf(submitCountingSpecTemplate, predicates)
|
||||
}
|
||||
|
||||
// A relaunch is not proof that nothing ran before it. The submit was dispatched
|
||||
// and confirmed; what the relaunch changed is that the app restarted between
|
||||
// the two readings the property compares. Reporting "no action" for it hands
|
||||
// submitCommitsOneTransactionPerAction a transaction rise of one against a
|
||||
// window of zero submits, which is the conviction #77 fixed for the apply-error
|
||||
// path, manufactured here out of the scope guard instead.
|
||||
func TestRunner_ARelaunchDoesNotConvictTheSubmitCountingProperty(t *testing.T) {
|
||||
spec := specWithFolioPredicates(t)
|
||||
|
||||
run := func(t *testing.T, commitsPerTap int64) []ViolationRecord {
|
||||
t.Helper()
|
||||
state := newHarnessWithSpec(t, spec)
|
||||
device := &leavesForegroundAfterSubmitDriver{
|
||||
Driver: state.mock,
|
||||
commitsPerTap: commitsPerTap,
|
||||
}
|
||||
violations := runTwoSubmitSteps(t, state, device, commitsPerTap)
|
||||
if countMockActions(state, mockdriver.ActionLaunch, "") == 0 {
|
||||
t.Fatal("the app was never relaunched, so the guard this test is about never ran")
|
||||
}
|
||||
return violations
|
||||
}
|
||||
|
||||
t.Run("one transaction per tap is not a double submit", func(t *testing.T) {
|
||||
if violations := run(t, 1); len(violations) != 0 {
|
||||
t.Errorf("the counting property convicted a healthy app: %v\n"+
|
||||
"one transaction rose against a submit the runner confirmed, and the "+
|
||||
"spec was told no action happened because the app was relaunched",
|
||||
violations)
|
||||
}
|
||||
})
|
||||
|
||||
// The control. Without it a green above proves nothing: a property that
|
||||
// never sees a comparable pair is silently vacuous and reports the same
|
||||
// empty violation list.
|
||||
t.Run("two transactions per tap still convicts", func(t *testing.T) {
|
||||
violations := run(t, 2)
|
||||
if len(violations) == 0 {
|
||||
t.Fatal("the counting property missed a double submit; the harness never " +
|
||||
"put the property in a position to fire, so the case above proves nothing")
|
||||
}
|
||||
if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" {
|
||||
t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The same hole through the guard's other branch. A system window holding the
|
||||
// focus says nothing about whether the tap under it ran: it was dispatched, and
|
||||
// what nobody can say afterwards is whether the app received it. That is the
|
||||
// unknown `applied` already carries, and it counts toward the submits a window
|
||||
// could hold. Reporting no action instead convicts the app of a transaction
|
||||
// with no cause.
|
||||
func TestRunner_AnOverlayDoesNotConvictTheSubmitCountingProperty(t *testing.T) {
|
||||
spec := specWithFolioPredicates(t)
|
||||
|
||||
run := func(t *testing.T, commitsPerTap int64) []ViolationRecord {
|
||||
t.Helper()
|
||||
state := newHarnessWithSpec(t, spec)
|
||||
device := &obscuredAfterSubmitDriver{
|
||||
Driver: state.mock,
|
||||
commitsPerTap: commitsPerTap,
|
||||
}
|
||||
violations := runTwoSubmitSteps(t, state, device, commitsPerTap)
|
||||
if countMockActions(state, mockdriver.ActionPressKey, "back") == 0 {
|
||||
t.Fatal("the overlay was never dismissed, so the guard this test is about never ran")
|
||||
}
|
||||
if countMockActions(state, mockdriver.ActionLaunch, "") != 0 {
|
||||
t.Fatal("a resumed-but-obscured app must not be relaunched")
|
||||
}
|
||||
return violations
|
||||
}
|
||||
|
||||
t.Run("one transaction per tap is not a double submit", func(t *testing.T) {
|
||||
if violations := run(t, 1); len(violations) != 0 {
|
||||
t.Errorf("the counting property convicted a healthy app: %v\n"+
|
||||
"one transaction rose against a submit the runner dispatched, and the "+
|
||||
"spec was told no action happened because a system window took the focus",
|
||||
violations)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("two transactions per tap still convicts", func(t *testing.T) {
|
||||
violations := run(t, 2)
|
||||
if len(violations) == 0 {
|
||||
t.Fatal("the counting property missed a double submit; the harness never " +
|
||||
"put the property in a position to fire, so the case above proves nothing")
|
||||
}
|
||||
if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" {
|
||||
t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/driver"
|
||||
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
|
||||
)
|
||||
|
||||
const relaunchBundleID = "app.folio"
|
||||
|
||||
// leavesForegroundAfterSubmitDriver is the device condition the app-scope guard
|
||||
// exists for: the submit tap lands and commits, and the app is no longer the
|
||||
// foreground app by the time the next step looks. Folio's transactions are in
|
||||
// sqlite, so the commit survives the relaunch and the next hierarchy read shows
|
||||
// it.
|
||||
type leavesForegroundAfterSubmitDriver struct {
|
||||
*mockdriver.Driver
|
||||
commitsPerTap int64
|
||||
committed atomic.Int64
|
||||
away atomic.Bool
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) Tap(context.Context, int, int) error {
|
||||
return d.commitThenLeave()
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) TapSelector(context.Context, string) error {
|
||||
return d.commitThenLeave()
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) commitThenLeave() error {
|
||||
d.committed.Add(d.commitsPerTap)
|
||||
d.away.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) Launch(
|
||||
ctx context.Context,
|
||||
bundleID string,
|
||||
clearState bool,
|
||||
env map[string]string,
|
||||
) error {
|
||||
d.away.Store(false)
|
||||
return d.Driver.Launch(ctx, bundleID, clearState, env)
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) ForegroundApp(context.Context) (string, error) {
|
||||
if d.away.Load() {
|
||||
return "com.android.launcher", nil
|
||||
}
|
||||
return relaunchBundleID, nil
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) FocusedWindowApp(ctx context.Context) (string, error) {
|
||||
return d.ForegroundApp(ctx)
|
||||
}
|
||||
|
||||
func (d *leavesForegroundAfterSubmitDriver) Snapshot(context.Context) (string, driver.Image, error) {
|
||||
return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil
|
||||
}
|
||||
|
||||
// A relaunch is not proof that nothing ran before it. The submit was dispatched
|
||||
// and confirmed; what the relaunch changed is that the app restarted between
|
||||
// the two readings the property compares. Reporting "no action" for it hands
|
||||
// submitCommitsOneTransactionPerAction a transaction rise of one against a
|
||||
// window of zero submits, which is the conviction #77 fixed for the apply-error
|
||||
// path, manufactured here out of the scope guard instead.
|
||||
func TestRunner_ARelaunchDoesNotConvictTheSubmitCountingProperty(t *testing.T) {
|
||||
predicates, err := filepath.Abs("../../examples/folio/sanderling/predicates.ts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spec := fmt.Sprintf(submitCountingSpecTemplate, predicates)
|
||||
|
||||
run := func(t *testing.T, commitsPerTap int64) []ViolationRecord {
|
||||
t.Helper()
|
||||
state := newHarnessWithSpec(t, spec)
|
||||
device := &leavesForegroundAfterSubmitDriver{Driver: state.mock, commitsPerTap: commitsPerTap}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 2,
|
||||
BundleID: relaunchBundleID,
|
||||
Driver: device,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if summary.Steps != 2 {
|
||||
t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps)
|
||||
}
|
||||
if got := device.committed.Load(); got != commitsPerTap*2 {
|
||||
t.Fatalf("the device committed %d transaction(s), want %d; the taps never reached it",
|
||||
got, commitsPerTap*2)
|
||||
}
|
||||
relaunches := 0
|
||||
for _, action := range state.mock.Actions() {
|
||||
if action.Kind == mockdriver.ActionLaunch {
|
||||
relaunches++
|
||||
}
|
||||
}
|
||||
if relaunches == 0 {
|
||||
t.Fatal("the app was never relaunched, so the guard this test is about never ran")
|
||||
}
|
||||
return summary.Violations
|
||||
}
|
||||
|
||||
t.Run("one transaction per tap is not a double submit", func(t *testing.T) {
|
||||
if violations := run(t, 1); len(violations) != 0 {
|
||||
t.Errorf("the counting property convicted a healthy app: %v\n"+
|
||||
"one transaction rose against a submit the runner confirmed, and the "+
|
||||
"spec was told no action happened because the app was relaunched",
|
||||
violations)
|
||||
}
|
||||
})
|
||||
|
||||
// The control. Without it a green above proves nothing: a property that
|
||||
// never sees a comparable pair is silently vacuous and reports the same
|
||||
// empty violation list.
|
||||
t.Run("two transactions per tap still convicts", func(t *testing.T) {
|
||||
violations := run(t, 2)
|
||||
if len(violations) == 0 {
|
||||
t.Fatal("the counting property missed a double submit; the harness never " +
|
||||
"put the property in a position to fire, so the case above proves nothing")
|
||||
}
|
||||
if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" {
|
||||
t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user