sanderling/internal/runner/uncertain_last_action_test.go

package runner

import (
	"context"
	"errors"
	"fmt"
	"path/filepath"
	"sync/atomic"
	"testing"
	"time"

	"github.com/priyanshujain/sanderling/internal/driver"
	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
)

// An apply error is not proof that nothing landed. An RPC deadline that fires
// after the tap was dispatched leaves the transaction committed, and a runner
// that reports "no action" for it hands
// submitCommitsOneTransactionPerAction a rise of one transaction against a
// window of zero submits: a conviction manufactured out of the runner's own
// uncertainty, on the property carrying most of the detection on android.
//
// The spec below is the real folio predicate pair, imported from the example,
// so what this asserts is the verdict the shipped property reaches.
const submitCountingSpecTemplate = `
import { actions, always, extract, next, Tap } from "@sanderling/spec";
import {
  committedTransactionsExceedSubmits,
  countSubmitsInWindow,
} from "%s";

let submits = 0;
const submitsInWindow = extract("submitsInWindow", state => {
  const window = countSubmitsInWindow({
    previousCount: submits,
    lastAction: state.lastAction,
    fresh: true,
  });
  submits = window.next;
  return window.reported;
});

const counts = extract("counts", state => {
  const text = state.ax.find("id:TxnCount")?.text;
  return text ? { Travel: parseInt(text, 10) } : null;
});

globalThis.properties = {
  submitCommitsOneTransactionPerAction: always(
    next(() =>
      !committedTransactionsExceedSubmits({
        countsBefore: counts.previous ?? null,
        countsAfter: counts.current,
        submitsInWindow: submitsInWindow.current,
      }),
    ),
  ),
};
globalThis.actions = actions(() => [Tap({ on: "id:TxnSubmit" })]);
`

const homeWithTxnCount = `{"attributes":{"resource-id":"HomeScreen"},"children":[
	{"attributes":{"resource-id":"TxnCount","text":"%d"},"children":[]},
	{"attributes":{"resource-id":"TxnSubmit","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true}
]}`

// dispatchThenFailDriver is the device condition the runner cannot see through:
// the tap reaches the app and commits, then the call the runner is waiting on
// times out. Every later hierarchy read shows the committed transactions.
type dispatchThenFailDriver struct {
	*mockdriver.Driver
	commitsPerTap int64
	committed     atomic.Int64
}

func (d *dispatchThenFailDriver) Tap(context.Context, int, int) error {
	return d.dispatchThenFail()
}

func (d *dispatchThenFailDriver) TapSelector(context.Context, string) error {
	return d.dispatchThenFail()
}

func (d *dispatchThenFailDriver) dispatchThenFail() error {
	d.committed.Add(d.commitsPerTap)
	return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded")
}

func (d *dispatchThenFailDriver) Snapshot(context.Context) (string, driver.Image, error) {
	return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil
}

func TestRunner_ApplyErrorAfterDispatchDoesNotConvictTheSubmitCountingProperty(t *testing.T) {
	predicates, err := filepath.Abs("../../examples/folio/sanderling/predicates.ts")
	if err != nil {
		t.Fatal(err)
	}
	spec := fmt.Sprintf(submitCountingSpecTemplate, predicates)

	run := func(t *testing.T, commitsPerTap int64) []ViolationRecord {
		t.Helper()
		state := newHarnessWithSpec(t, spec)
		device := &dispatchThenFailDriver{Driver: state.mock, commitsPerTap: commitsPerTap}

		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		summary, err := Run(ctx, Options{
			Duration:    time.Hour,
			IdleTimeout: 20 * time.Millisecond,
			MaxSteps:    2,
			Driver:      device,
			Verifier:    state.verifier,
			TraceWriter: state.writer,
		})
		if err != nil {
			t.Fatalf("Run: %v", err)
		}
		if summary.Steps != 2 {
			t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps)
		}
		if got := device.committed.Load(); got != commitsPerTap*2 {
			t.Fatalf("the device committed %d transaction(s), want %d; the taps never reached it",
				got, commitsPerTap*2)
		}
		return summary.Violations
	}

	t.Run("one transaction per tap is not a double submit", func(t *testing.T) {
		if violations := run(t, 1); len(violations) != 0 {
			t.Errorf("the counting property convicted a healthy app: %v\n"+
				"one transaction rose against a submit the runner dispatched but "+
				"could not confirm, and the spec was told no action happened",
				violations)
		}
	})

	// The control. Without it a green above proves nothing: a property that
	// never sees a comparable pair is silently vacuous and reports the same
	// empty violation list.
	t.Run("two transactions per tap still convicts", func(t *testing.T) {
		violations := run(t, 2)
		if len(violations) == 0 {
			t.Fatal("the counting property missed a double submit; the harness never " +
				"put the property in a position to fire, so the case above proves nothing")
		}
		if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" {
			t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties)
		}
	})
}