diff --git a/.gitignore b/.gitignore index 5bcbfd2..a42af9c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,14 @@ node_modules/ # `go build -tags withsidecar`. Never commit: it's ~130 MB. internal/sidecarassets/assets/sidecar-all.jar +# Simulator companion bundle. Build artifact produced by prepare.sh before +# `go build -tags withcompanion`. Never commit: it's ~16 MB. +internal/driver/ioscompanion/companionassets/assets/companion-*.tar.gz + +# In-simulator runner bundle. Build artifact produced by companion/prepare.sh +# before `go build -tags withcompanion`. Never commit: it's ~5 MB. +internal/driver/ioscompanion/runnerassets/assets/runner-*.tar.gz + # spec-api compiled output pkg/spec/dist/ @@ -48,6 +56,10 @@ replay-ui/node_modules/ replay-ui/dist/ replay-ui/.vite/ +# Simulator runner generated project and build output (xcodegen artifacts) +companion/CompanionRunner.xcodeproj/ +companion/build/ + # folio-web frontend examples/folio-web/node_modules/ examples/folio-web/dist/ @@ -60,4 +72,5 @@ examples/folio-web/.vite/ # Personal research notes /research/ -/talk/ \ No newline at end of file +/talk/ +keys/* diff --git a/Makefile b/Makefile index df0bee4..a474602 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,12 @@ BUF := buf GO_PACKAGES := ./... SIDECAR_JAR := sidecar/build/libs/sidecar-all.jar SIDECAR_EMBED := internal/sidecarassets/assets/sidecar-all.jar -SIDECAR_SRC := $(shell find sidecar/src -type f \( -name '*.kt' -o -name '*.kts' \) 2>/dev/null) build.gradle.kts settings.gradle.kts +COMPANION_EMBED := internal/driver/ioscompanion/companionassets/assets/companion-1.1.8.tar.gz +COMPANION_PREPARE := internal/driver/ioscompanion/companionassets/prepare.sh +RUNNER_EMBED := internal/driver/ioscompanion/runnerassets/assets/runner-1.0.0.tar.gz +RUNNER_PREPARE := companion/prepare.sh +RUNNER_SRC := $(shell find companion/Sources -type f -name '*.swift' 2>/dev/null) companion/project.yml +SIDECAR_SRC := $(shell find sidecar/src -type f \( -name '*.kt' -o -name '*.kts' \) 2>/dev/null) sidecar/build.gradle.kts build.gradle.kts settings.gradle.kts SANDERLING_BIN := bin/sanderling DOCS_SRC := $(shell find docs -type f -name '*.md' -not -path 'docs/_*') @@ -22,7 +27,7 @@ DOCS_TEMPLATE := docs/_template/page.html REPLAY_DIST := internal/replay/dist WEB_DIST := replay-ui/dist -.PHONY: bootstrap proto sidecar sanderling install test test-go test-browser test-kotlin test-spec-api web-test web-typecheck web-build web-dev replay-dev docs clean release-cli release-npm-dry +.PHONY: bootstrap proto sidecar sanderling install test test-go test-browser test-companion test-kotlin test-spec-api web-test web-typecheck web-build web-dev replay-dev docs clean release-cli release-npm-dry bootstrap: $(GO) mod download @@ -37,14 +42,14 @@ sidecar: $(SIDECAR_JAR) sanderling: $(SANDERLING_BIN) -$(SANDERLING_BIN): $(SIDECAR_EMBED) web-build +$(SANDERLING_BIN): $(SIDECAR_EMBED) $(COMPANION_EMBED) $(RUNNER_EMBED) web-build mkdir -p bin - $(GO) build -tags withsidecar -o $(SANDERLING_BIN) ./cmd/sanderling + $(GO) build -tags "withsidecar withcompanion" -o $(SANDERLING_BIN) ./cmd/sanderling # Installs `sanderling` into $GOBIN (or $GOPATH/bin) so it's directly on PATH for # anyone with a standard Go toolchain setup. -install: $(SIDECAR_EMBED) web-build - $(GO) install -tags withsidecar ./cmd/sanderling +install: $(SIDECAR_EMBED) $(COMPANION_EMBED) $(RUNNER_EMBED) web-build + $(GO) install -tags "withsidecar withcompanion" ./cmd/sanderling @dest="$$($(GO) env GOBIN)"; [ -n "$$dest" ] || dest="$$($(GO) env GOPATH)/bin"; echo "installed sanderling to $$dest" web-build: @@ -69,6 +74,12 @@ $(SIDECAR_EMBED): $(SIDECAR_JAR) mkdir -p $(dir $@) cp $< $@ +$(COMPANION_EMBED): $(COMPANION_PREPARE) + $(COMPANION_PREPARE) + +$(RUNNER_EMBED): $(RUNNER_SRC) $(RUNNER_PREPARE) + $(RUNNER_PREPARE) + test: test-go test-spec-api web-typecheck web-test test-go: @@ -82,6 +93,12 @@ web-test: test-browser: $(GO) test -tags browser ./test/browser/... ./internal/driver/chrome/... +# Runs the withcompanion-tagged tests (asset embedding, extraction, checksum +# reuse) against the real companion and runner bundles. Kept out of `test` +# because preparing the companion bundle needs the darwin toolchain. +test-companion: $(COMPANION_EMBED) $(RUNNER_EMBED) + $(GO) test -tags withcompanion ./internal/driver/ioscompanion/... + test-kotlin: ANDROID_HOME=$(ANDROID_HOME) $(GRADLE) :sidecar:test diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..b8de086 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,36 @@ +# Third Party Notices + +This project distributes third party software. The components below are +included under their respective licenses. + +## idb-companion + +- Project: idb (https://github.com/facebook/idb) +- Version: 1.1.8 +- Copyright: Copyright (c) Meta Platforms, Inc. and affiliates. +- License: MIT + +The simulator companion is vendored as a prebuilt binary artifact (the +`idb_companion` executable together with its runtime frameworks). It is +embedded into the sanderling binary at build time and extracted to a local +directory at runtime. + +### MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/buf.yaml b/buf.yaml index bc92f10..c58a455 100644 --- a/buf.yaml +++ b/buf.yaml @@ -1,6 +1,13 @@ version: v2 +# The pinned simulator companion proto lives under proto/idbcompanion and is +# deliberately excluded here. It is its own standalone buf module (see +# proto/idbcompanion/buf.yaml) so the STANDARD lint rules below never apply to +# it and a normal `make proto` never generates from it. Its stubs are committed +# and regenerated only via proto/idbcompanion/buf.gen.yaml. modules: - path: proto + excludes: + - proto/idbcompanion lint: use: - STANDARD diff --git a/cmd/sanderling/doctor.go b/cmd/sanderling/doctor.go index dafe612..d642c3e 100644 --- a/cmd/sanderling/doctor.go +++ b/cmd/sanderling/doctor.go @@ -14,6 +14,8 @@ import ( "github.com/chromedp/chromedp" + "github.com/priyanshujain/sanderling/internal/driver/ioscompanion" + "github.com/priyanshujain/sanderling/internal/ios" "github.com/priyanshujain/sanderling/internal/sidecarassets" ) @@ -33,6 +35,8 @@ func doctorChecksFor(platform string) []doctorCheck { return androidChecks() case "ios": return iosChecks() + case "ios-device": + return append(iosChecks(), iosDeviceChecks()...) case "all": return allChecks() default: @@ -55,19 +59,37 @@ func androidChecks() []doctorCheck { } } +// iosChecks covers the simulator path, which the native companion drives with +// no JVM. A simulator host with no Java still passes. Physical-device runs +// additionally need devicectl, the usbmuxd socket, a connected device, and +// signing credentials, covered by iosDeviceChecks and surfaced through the +// "all" union. func iosChecks() []doctorCheck { return []doctorCheck{ - {Name: "xcrun on PATH", Run: checkExecutableOnPath("xcrun")}, - {Name: "simctl on PATH", Run: checkExecutableOnPath("simctl")}, - {Name: "java 17+ on PATH", Run: checkJavaVersion}, - {Name: "sidecar JAR is real (not placeholder)", Run: checkSidecarJAR}, + {Name: "xcrun on PATH (ios simulator)", Run: checkExecutableOnPath("xcrun")}, + {Name: "simctl available (ios simulator)", Run: checkSimctl}, + } +} + +// iosDeviceChecks covers the prerequisites a physical iOS device needs: the +// runner is built and driven over a native usbmux tunnel, so devicectl installs +// the app, the macOS usbmuxd socket carries the tunnel, a device must be +// connected and paired, and App Store Connect signing credentials must be +// present for the no-UI build. Everything here is part of macOS + Xcode; nothing +// is installed. +func iosDeviceChecks() []doctorCheck { + return []doctorCheck{ + {Name: "devicectl available (ios physical device)", Run: checkDevicectl}, + {Name: "usbmuxd socket present (ios physical device)", Run: checkUsbmuxd}, + {Name: "an iOS device is connected and paired", Run: checkDeviceConnected}, + {Name: "App Store Connect signing credentials present", Run: checkDeviceSigning}, } } func allChecks() []doctorCheck { seen := map[string]bool{} var combined []doctorCheck - for _, group := range [][]doctorCheck{webChecks(), androidChecks(), iosChecks()} { + for _, group := range [][]doctorCheck{webChecks(), androidChecks(), iosChecks(), iosDeviceChecks()} { for _, c := range group { if seen[c.Name] { continue @@ -98,6 +120,57 @@ func checkChromiumLaunch(ctx context.Context) error { return nil } +// checkSimctl exercises `xcrun simctl help`: simctl is an xcrun subcommand, +// not a standalone binary, so a PATH lookup can never find it. +func checkSimctl(ctx context.Context) error { + if err := exec.CommandContext(ctx, "xcrun", "simctl", "help").Run(); err != nil { + return fmt.Errorf("xcrun simctl help: %w", err) + } + return nil +} + +// Device-check seams: package-level so the doctor's device checks run against +// canned results instead of a real device. +var ( + doctorConnectedDevices = ios.ConnectedDevices + doctorVerifySigning = ioscompanion.VerifyDeviceSigning + doctorVerifyUsbmuxd = ioscompanion.VerifyUsbmuxdSocket +) + +// checkDevicectl exercises `xcrun devicectl --version`: devicectl is an xcrun +// subcommand, so a PATH lookup cannot find it. +func checkDevicectl(ctx context.Context) error { + if err := exec.CommandContext(ctx, "xcrun", "devicectl", "--version").Run(); err != nil { + return fmt.Errorf("xcrun devicectl --version: %w", err) + } + return nil +} + +// checkUsbmuxd confirms the macOS usbmuxd socket is present: the native device +// tunnel speaks to it directly instead of shelling out to a third-party client. +func checkUsbmuxd(_ context.Context) error { + return doctorVerifyUsbmuxd() +} + +// checkDeviceConnected confirms at least one physical iOS device is connected +// and paired, the prerequisite for the tunnel and the install. +func checkDeviceConnected(ctx context.Context) error { + devices, err := doctorConnectedDevices(ctx) + if err != nil { + return err + } + if len(devices) == 0 { + return fmt.Errorf("no connected iOS device; connect and pair an iPhone") + } + return nil +} + +// checkDeviceSigning confirms the App Store Connect signing environment is +// complete and the key file exists, so the no-UI device build can sign. +func checkDeviceSigning(_ context.Context) error { + return doctorVerifySigning() +} + func checkSidecarJAR(_ context.Context) error { if sidecarassets.IsPlaceholder() { return fmt.Errorf("placeholder JAR embedded; run `make sidecar && make sanderling` to embed the real fat JAR") @@ -116,15 +189,15 @@ func parseDoctorArgs(args []string, stderr io.Writer) (doctorOptions, error) { flagSet := flag.NewFlagSet("doctor", flag.ContinueOnError) flagSet.SetOutput(stderr) var options doctorOptions - flagSet.StringVar(&options.platform, "platform", "all", "target platform: web, android, ios, all") + flagSet.StringVar(&options.platform, "platform", "all", "target platform: web, android, ios, ios-device, all") if err := flagSet.Parse(args); err != nil { return doctorOptions{}, err } switch options.platform { - case "web", "android", "ios", "all": + case "web", "android", "ios", "ios-device", "all": return options, nil default: - return doctorOptions{}, fmt.Errorf("unsupported platform: %q (web, android, ios, all)", options.platform) + return doctorOptions{}, fmt.Errorf("unsupported platform: %q (web, android, ios, ios-device, all)", options.platform) } } diff --git a/cmd/sanderling/doctor_test.go b/cmd/sanderling/doctor_test.go index d33a34c..3155180 100644 --- a/cmd/sanderling/doctor_test.go +++ b/cmd/sanderling/doctor_test.go @@ -8,6 +8,8 @@ import ( "io" "strings" "testing" + + "github.com/priyanshujain/sanderling/internal/ios" ) func TestRunDoctorChecks_AllPass(t *testing.T) { @@ -127,13 +129,76 @@ func TestDoctorChecksFor_All_IsUnion(t *testing.T) { for _, c := range all { names[c.Name]++ } - for _, name := range []string{"adb on PATH", "xcrun on PATH", "headless chromium can launch"} { + for _, name := range []string{"adb on PATH", "xcrun on PATH (ios simulator)", "headless chromium can launch"} { if names[name] != 1 { t.Errorf("expected %q in 'all' exactly once, got %d", name, names[name]) } } } +func TestDoctorChecksFor_iOSSimulator_OmitsJava(t *testing.T) { + for _, c := range doctorChecksFor("ios") { + if strings.Contains(c.Name, "java") || strings.Contains(c.Name, "sidecar") { + t.Errorf("ios simulator checks must omit %q; simulator runs need no JVM", c.Name) + } + } +} + +func TestDoctorChecksFor_iOSDevice_CoversDevicePrereqs(t *testing.T) { + checks := doctorChecksFor("ios-device") + for _, c := range checks { + if strings.Contains(c.Name, "java") || strings.Contains(c.Name, "sidecar") { + t.Errorf("device checks must not include the retired %q", c.Name) + } + } + for _, want := range []string{"devicectl", "usbmuxd", "connected and paired", "signing credentials"} { + found := false + for _, c := range checks { + if strings.Contains(c.Name, want) { + found = true + } + } + if !found { + t.Errorf("ios-device checks missing %q: %+v", want, checks) + } + } +} + +func TestCheckDeviceConnected(t *testing.T) { + original := doctorConnectedDevices + t.Cleanup(func() { doctorConnectedDevices = original }) + + doctorConnectedDevices = func(context.Context) ([]ios.Device, error) { + return []ios.Device{{Name: "iPhone"}}, nil + } + if err := checkDeviceConnected(context.Background()); err != nil { + t.Fatalf("a connected device must pass: %v", err) + } + + doctorConnectedDevices = func(context.Context) ([]ios.Device, error) { return nil, nil } + if err := checkDeviceConnected(context.Background()); err == nil { + t.Fatal("no device must fail") + } +} + +func TestCheckDeviceSigning_SurfacesSeamResult(t *testing.T) { + // checkDeviceSigning is a passthrough to the driver's credential check; the + // credential logic itself is covered by TestReadSigningCredentials* in the + // ioscompanion package. Here we only confirm the wiring through the seam. + original := doctorVerifySigning + t.Cleanup(func() { doctorVerifySigning = original }) + + doctorVerifySigning = func() error { return nil } + if err := checkDeviceSigning(context.Background()); err != nil { + t.Fatalf("a passing signing check must surface nil: %v", err) + } + + doctorVerifySigning = func() error { return errors.New("missing credentials") } + if err := checkDeviceSigning(context.Background()); err == nil { + t.Fatal("a failing signing check must surface the error") + } +} + func TestDoctorChecksFor_UnknownPlatform(t *testing.T) { if got := doctorChecksFor("fuchsia"); got != nil { t.Errorf("expected nil for unknown platform, got %+v", got) diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index 8273656..33d8589 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -8,6 +8,8 @@ import ( "fmt" "io" "os" + "os/signal" + "syscall" "time" ) @@ -16,15 +18,16 @@ import ( var Version = "dev" type testOptions struct { - spec string - bundleID string - platform string - avd string - iosDevice string - duration time.Duration - seed int64 - output string - clearData bool + spec string + bundleID string + platform string + avd string + iosDevice string + iosAppPath string + duration time.Duration + seed int64 + output string + clearData bool } const topUsage = `sanderling is a property-based UI fuzzer for mobile apps. @@ -49,7 +52,8 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { flagSet.StringVar(&options.bundleID, "bundle-id", "", "target app bundle ID (required)") flagSet.StringVar(&options.platform, "platform", "android", "target platform: android, ios, web") flagSet.StringVar(&options.avd, "avd", "", "Android AVD name to boot if no device is connected") - flagSet.StringVar(&options.iosDevice, "ios-device", "", "iOS simulator name or UDID to boot if none is running") + flagSet.StringVar(&options.iosDevice, "ios-device", "", "iOS target: a simulator name/UDID to boot, or a connected device's name, UDID, or CoreDevice id") + flagSet.StringVar(&options.iosAppPath, "ios-app-path", "", "path to the .app bundle for iOS clear-state reinstall (simulator: simctl; device: devicectl)") flagSet.DurationVar(&options.duration, "duration", 5*time.Minute, "total test duration") flagSet.Int64Var(&options.seed, "seed", 0, "RNG seed (0 = random)") flagSet.StringVar(&options.output, "output", "./runs", "output directory for traces") @@ -72,7 +76,11 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { } func runTest(options testOptions, stdout io.Writer) error { - ctx, cancel := context.WithCancel(context.Background()) + // A signal-aware root context: on Ctrl-C the cancellation propagates into + // the drivers' process contexts, so spawned children (the iOS companion, + // the xcodebuild runner session) get their SIGTERM instead of outliving + // the run as orphans. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() return runTestPipeline(ctx, options, stdout) } diff --git a/cmd/sanderling/main_test.go b/cmd/sanderling/main_test.go index 2cb99a5..fa9ebc0 100644 --- a/cmd/sanderling/main_test.go +++ b/cmd/sanderling/main_test.go @@ -181,6 +181,31 @@ func TestParseTestArgs_IosDeviceFlag(t *testing.T) { } } +func TestParseTestArgs_IosAppPathFlag(t *testing.T) { + options, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example.app", + "--platform", "ios", + "--ios-app-path", "/tmp/build/iosApp.app", + }, io.Discard) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if options.iosAppPath != "/tmp/build/iosApp.app" { + t.Errorf("expected iosAppPath=/tmp/build/iosApp.app, got %q", options.iosAppPath) + } +} + +func TestParseTestArgs_IosAppPathOptional(t *testing.T) { + options, err := parseTestArgs([]string{"--spec", "s.ts", "--bundle-id", "com.example.app"}, io.Discard) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if options.iosAppPath != "" { + t.Errorf("iosAppPath default: got %q, want empty", options.iosAppPath) + } +} + func TestRun_TestSubcommand_PipelineErrors(t *testing.T) { // Web skips host-dependent device boot and bundles first, so a missing spec // deterministically surfaces a bundle-resolution error. This proves the diff --git a/cmd/sanderling/test_run.go b/cmd/sanderling/test_run.go index 96d32e1..f8407d1 100644 --- a/cmd/sanderling/test_run.go +++ b/cmd/sanderling/test_run.go @@ -9,14 +9,15 @@ import ( func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) error { return testrun.Execute(ctx, testrun.Options{ - Spec: options.spec, - BundleID: options.bundleID, - Platform: options.platform, - AVD: options.avd, - IosDevice: options.iosDevice, - Duration: options.duration, - Seed: options.seed, - Output: options.output, - ClearData: options.clearData, + Spec: options.spec, + BundleID: options.bundleID, + Platform: options.platform, + AVD: options.avd, + IosDevice: options.iosDevice, + IosAppPath: options.iosAppPath, + Duration: options.duration, + Seed: options.seed, + Output: options.output, + ClearData: options.clearData, }, stdout) } diff --git a/companion/Sources/AppLifecycle.swift b/companion/Sources/AppLifecycle.swift new file mode 100644 index 0000000..065abc5 --- /dev/null +++ b/companion/Sources/AppLifecycle.swift @@ -0,0 +1,74 @@ +import XCTest + +// Launches and terminates apps through the automation session. Lifecycle +// performed outside the session leaves its cached app proxies bound to dead +// processes, after which snapshots hang and typing asserts. +enum AppLifecycle { + enum LifecycleError: Error { + case failed(String) + } + + static func launch(bundleIdentifier: String, foregroundIfRunning: Bool) throws { + try onMainCatching { + let application = XCUIApplication(bundleIdentifier: bundleIdentifier) + if foregroundIfRunning { + // activate starts the app when needed and foregrounds it when + // already running, without a forced relaunch. + application.activate() + } else { + application.launch() + } + } + } + + static func terminate(bundleIdentifier: String) throws { + try onMainCatching { + let application = XCUIApplication(bundleIdentifier: bundleIdentifier) + if application.state == .notRunning { + return + } + application.terminate() + } + } + + // state reports the app's run state in the vocabulary the Go transport maps + // to a process state: foreground, background, or notRunning. On the + // simulator the hybrid never calls it; on device it backs ForegroundApp. + static func state(bundleIdentifier: String) -> String { + var result = "notRunning" + let collect = { + switch XCUIApplication(bundleIdentifier: bundleIdentifier).state { + case .runningForeground: + result = "foreground" + case .runningBackground, .runningBackgroundSuspended: + result = "background" + default: + result = "notRunning" + } + } + if Thread.isMainThread { + collect() + } else { + DispatchQueue.main.sync(execute: collect) + } + return result + } + + // onMainCatching runs automation work on the main thread and converts a + // framework assertion into a thrown error so the server survives it. + private static func onMainCatching(_ work: @escaping () -> Void) throws { + var caughtError: NSError? + var completed = false + let run = { + completed = CompanionRunCatching(work, &caughtError) + } + if Thread.isMainThread { + run() + } else { + DispatchQueue.main.sync(execute: run) + } + if !completed { + throw LifecycleError.failed(caughtError?.localizedDescription ?? "unknown") + } + } +} diff --git a/companion/Sources/ElementTypeName.swift b/companion/Sources/ElementTypeName.swift new file mode 100644 index 0000000..d5f517d --- /dev/null +++ b/companion/Sources/ElementTypeName.swift @@ -0,0 +1,93 @@ +import XCTest + +// Maps XCUIElement.ElementType to the capitalized conventional type names the +// Go mapper consumes. textView is deliberately "TextArea": the Go side treats +// "TextArea" and "TextField" as the editable types. +func elementTypeName(_ elementType: XCUIElement.ElementType) -> String { + switch elementType { + case .application: return "Application" + case .group: return "Group" + case .window: return "Window" + case .sheet: return "Sheet" + case .drawer: return "Drawer" + case .alert: return "Alert" + case .dialog: return "Dialog" + case .button: return "Button" + case .radioButton: return "RadioButton" + case .radioGroup: return "RadioGroup" + case .checkBox: return "CheckBox" + case .disclosureTriangle: return "DisclosureTriangle" + case .popUpButton: return "PopUpButton" + case .comboBox: return "ComboBox" + case .menuButton: return "MenuButton" + case .toolbarButton: return "ToolbarButton" + case .popover: return "Popover" + case .keyboard: return "Keyboard" + case .key: return "Key" + case .navigationBar: return "NavigationBar" + case .tabBar: return "TabBar" + case .tabGroup: return "TabGroup" + case .toolbar: return "Toolbar" + case .statusBar: return "StatusBar" + case .table: return "Table" + case .tableRow: return "TableRow" + case .tableColumn: return "TableColumn" + case .outline: return "Outline" + case .outlineRow: return "OutlineRow" + case .browser: return "Browser" + case .collectionView: return "CollectionView" + case .slider: return "Slider" + case .pageIndicator: return "PageIndicator" + case .progressIndicator: return "ProgressIndicator" + case .activityIndicator: return "ActivityIndicator" + case .segmentedControl: return "SegmentedControl" + case .picker: return "Picker" + case .pickerWheel: return "PickerWheel" + case .switch: return "Switch" + case .toggle: return "Toggle" + case .link: return "Link" + case .image: return "Image" + case .icon: return "Icon" + case .searchField: return "SearchField" + case .scrollView: return "ScrollView" + case .scrollBar: return "ScrollBar" + case .staticText: return "StaticText" + case .textField: return "TextField" + case .secureTextField: return "SecureTextField" + case .datePicker: return "DatePicker" + case .textView: return "TextArea" + case .menu: return "Menu" + case .menuItem: return "MenuItem" + case .menuBar: return "MenuBar" + case .menuBarItem: return "MenuBarItem" + case .map: return "Map" + case .webView: return "WebView" + case .incrementArrow: return "IncrementArrow" + case .decrementArrow: return "DecrementArrow" + case .timeline: return "Timeline" + case .ratingIndicator: return "RatingIndicator" + case .valueIndicator: return "ValueIndicator" + case .splitGroup: return "SplitGroup" + case .splitter: return "Splitter" + case .relevanceIndicator: return "RelevanceIndicator" + case .colorWell: return "ColorWell" + case .helpTag: return "HelpTag" + case .matte: return "Matte" + case .dockItem: return "DockItem" + case .ruler: return "Ruler" + case .rulerMarker: return "RulerMarker" + case .grid: return "Grid" + case .levelIndicator: return "LevelIndicator" + case .cell: return "Cell" + case .layoutArea: return "LayoutArea" + case .layoutItem: return "LayoutItem" + case .handle: return "Handle" + case .stepper: return "Stepper" + case .tab: return "Tab" + case .touchBar: return "TouchBar" + case .statusItem: return "StatusItem" + case .any: return "Any" + case .other: return "Other" + @unknown default: return "Other" + } +} diff --git a/companion/Sources/Gesture.swift b/companion/Sources/Gesture.swift new file mode 100644 index 0000000..59148f2 --- /dev/null +++ b/companion/Sources/Gesture.swift @@ -0,0 +1,206 @@ +import XCTest +import CoreGraphics + +// Synthesizes one timestamped event record from a list of primitive events. +// +// The synthesizer constrains how precise timing can be expressed: +// - Within one pointer path, event offsets are honored to the millisecond, +// but nothing after the path's first lift is delivered: a second press is +// silently dropped, whether added through the press helper or as a raw +// pointer event. +// - Across paths, every event is delivered, but each path's timeline is +// normalized to its own first event, so a later start offset collapses. +// +// Sequential taps therefore become one path per tap, each anchored by a raw +// zero-offset move event at the tap point: the anchor pins the path's +// timeline origin to the record's, turning the tap's absolute press offset +// into an in-path delta, which is honored. This preserves the inter-tap gap +// to the millisecond. +enum Gesture { + enum GestureError: Error { + case missingField(String) + case unknownKind(String) + case touchUpWithoutTouchDown + case touchDownWhileTouchActive + case synthesisFailed(String) + } + + // minimumHoldSeconds keeps a tap's press observable when its down and up + // arrive at the same offset; far below any long-press threshold. + private static let minimumHoldSeconds = 0.03 + + // pathGapSeconds orders a press strictly after the previous lift when a + // requested gap is shorter than the minimum hold. + private static let pathGapSeconds = 0.005 + + // The raw shape of a touch move, read once from a probe path built with + // the path helpers so no private enum values are hardcoded. + private struct MoveEventShape { + let type: Int + let button: Int + let clicks: UInt + } + + private static let moveShape: MoveEventShape = { + let probe = XCPointerEventPath(forTouchAt: CGPoint(x: 1, y: 1), offset: 0) + probe.move(to: CGPoint(x: 2, y: 2), atOffset: minimumHoldSeconds) + probe.liftUp(atOffset: minimumHoldSeconds * 2) + let events = probe.pointerEvents + let move = events.count > 1 ? events[1] : nil + return MoveEventShape( + type: move?.eventType ?? 2, + button: move?.buttonType ?? 0, + clicks: move.map { UInt($0.clickCount) } ?? 0 + ) + }() + + // probe reports the raw events the path helpers produce, for diagnostics. + static func probe() -> [String: Any] { + let path = XCPointerEventPath(forTouchAt: CGPoint(x: 1, y: 2), offset: 0) + path.move(to: CGPoint(x: 3, y: 4), atOffset: 0.05) + path.liftUp(atOffset: 0.1) + return [ + "descriptions": path.pointerEvents.map { $0.description }, + "types": path.pointerEvents.map { $0.eventType }, + "buttons": path.pointerEvents.map { $0.buttonType }, + "offsets": path.pointerEvents.map { $0.offset }, + ] + } + + // touchSegment is one press-to-lift span at absolute record offsets. + private struct touchSegment { + var downPoint: CGPoint + var downOffset: Double + var upPoint: CGPoint + var upOffset: Double + var movePoint: CGPoint? + var moveOffset: Double = 0 + } + + static func perform(events: [[String: Any]]) throws { + var offset = 0.0 + var lastLiftOffset = -1.0 + var segments: [touchSegment] = [] + var active: touchSegment? + + func point(_ event: [String: Any], _ xKey: String, _ yKey: String) throws -> CGPoint { + guard let x = event[xKey] as? Double, let y = event[yKey] as? Double else { + throw GestureError.missingField("\(xKey)/\(yKey)") + } + return CGPoint(x: x, y: y) + } + + for event in events { + guard let kind = event["kind"] as? String else { + throw GestureError.missingField("kind") + } + switch kind { + case "touchDown": + guard active == nil else { + throw GestureError.touchDownWhileTouchActive + } + let location = try point(event, "x", "y") + let downOffset = max(offset, lastLiftOffset + Gesture.pathGapSeconds) + active = touchSegment( + downPoint: location, downOffset: downOffset, + upPoint: location, upOffset: downOffset + Gesture.minimumHoldSeconds) + case "touchUp": + guard var segment = active else { + throw GestureError.touchUpWithoutTouchDown + } + segment.upPoint = try point(event, "x", "y") + segment.upOffset = max(offset, segment.downOffset + Gesture.minimumHoldSeconds) + lastLiftOffset = segment.upOffset + segments.append(segment) + active = nil + case "delay": + guard let milliseconds = event["milliseconds"] as? Double else { + throw GestureError.missingField("milliseconds") + } + offset += milliseconds / 1000.0 + case "swipe": + guard active == nil else { + throw GestureError.touchDownWhileTouchActive + } + let from = try point(event, "fromX", "fromY") + let to = try point(event, "toX", "toY") + guard let seconds = event["seconds"] as? Double else { + throw GestureError.missingField("seconds") + } + let downOffset = max(offset, lastLiftOffset + Gesture.pathGapSeconds) + segments.append(touchSegment( + downPoint: from, downOffset: downOffset, + upPoint: to, upOffset: downOffset + seconds, + movePoint: to, moveOffset: downOffset + seconds)) + lastLiftOffset = downOffset + seconds + offset = downOffset + seconds + default: + throw GestureError.unknownKind(kind) + } + } + if let segment = active { + // A down without an up would leave a stuck touch on screen. + segments.append(segment) + } + guard !segments.isEmpty else { + throw GestureError.missingField("events") + } + + // The synthesis stack raises ObjC exceptions for invalid paths; bridge + // them into a thrown error so the server survives a bad record. + var caughtException: NSError? + var synthesisError: Error? + let completed = CompanionRunCatching({ + do { + try Gesture.synthesize(segments: segments) + } catch { + synthesisError = error + } + }, &caughtException) + if !completed { + throw GestureError.synthesisFailed(caughtException?.localizedDescription ?? "unknown exception") + } + if let synthesisError = synthesisError { + throw synthesisError + } + } + + private static func synthesize(segments: [touchSegment]) throws { + let record = XCSynthesizedEventRecord(name: "companion", interfaceOrientation: 0) + for (index, segment) in segments.enumerated() { + // Paths play back one after another with their start offsets + // normalized away, so each path carries its events relative to its + // own press. A trailing hover move then stretches the path out to + // the next press's absolute offset, which preserves the requested + // inter-tap gap under sequential playback. + let base = segment.downOffset + let path = XCPointerEventPath(forTouchAt: segment.downPoint, offset: 0) + // Distinct pointer identities: without this, sequential taps at + // the same point merge into one multi-tap gesture and the app + // sees a single click no matter the gap. + path.index = UInt64(index) + if let movePoint = segment.movePoint { + path.move(to: movePoint, atOffset: segment.moveOffset - base) + } + let upDelta = segment.upOffset - base + path.liftUp(atOffset: upDelta) + if index + 1 < segments.count { + let tailDelta = segments[index + 1].downOffset - base + if tailDelta > upDelta { + path._addPointerEvent(XCPointerEvent( + type: Gesture.moveShape.type, + buttonType: Gesture.moveShape.button, + coordinate: segment.upPoint, + offset: tailDelta, + clickCount: Gesture.moveShape.clicks)) + } + } + record.add(path) + } + + // Synchronous delivery: returns whether the event was synthesized and + // populates the error out-pointer on failure. This avoids the async + // completion-block path. + try record.synthesize() + } +} diff --git a/companion/Sources/PrivateAutomation/PrivateAutomation.h b/companion/Sources/PrivateAutomation/PrivateAutomation.h new file mode 100644 index 0000000..ac66745 --- /dev/null +++ b/companion/Sources/PrivateAutomation/PrivateAutomation.h @@ -0,0 +1,74 @@ +#import +#import + +// Private XCTest event-synthesis interfaces. These ship inside the testing +// frameworks bundled with Xcode and are the only path to timestamped touch +// events with millisecond-precision offsets. The selectors below were verified +// present in Xcode 26's XCUIAutomation framework binary. + +NS_ASSUME_NONNULL_BEGIN + +// One raw pointer event inside a path: type, screen coordinate, and offset in +// seconds. Constructed directly so a single path can carry several +// press-and-lift pairs at exact offsets; the press-and-lift helper methods on +// the path silently drop a second press after a lift. +@interface XCPointerEvent : NSObject ++ (instancetype)eventWithType:(NSInteger)eventType + buttonType:(NSInteger)buttonType + coordinate:(CGPoint)coordinate + offset:(double)offset + clickCount:(NSUInteger)clickCount; +@property (nonatomic, readonly) NSInteger eventType; +@property (nonatomic, readonly) NSInteger buttonType; +@property (nonatomic, readonly) NSUInteger clickCount; +@property (nonatomic, readonly) CGPoint coordinate; +@property (nonatomic, readonly) double offset; +@end + +// One pointer's path through a synthesized event. Offsets are seconds from the +// start of the event record. +@interface XCPointerEventPath : NSObject +- (instancetype)initForTouchAtPoint:(CGPoint)point offset:(double)offset; +- (void)moveToPoint:(CGPoint)point atOffset:(double)offset; +- (void)pressDownAtOffset:(double)offset; +- (void)liftUpAtOffset:(double)offset; +- (void)_addPointerEvent:(XCPointerEvent *)pointerEvent; +@property (nonatomic, readonly) NSArray *pointerEvents; +@property (nonatomic) unsigned long long index; +@end + +// A complete synthesized event composed of one or more pointer paths. +// synthesizeWithError: delivers the event synchronously and returns whether it +// succeeded, which avoids the asynchronous completion-block path. +@interface XCSynthesizedEventRecord : NSObject +- (instancetype)initWithName:(NSString *)name + interfaceOrientation:(NSInteger)interfaceOrientation; +- (void)addPointerEventPath:(XCPointerEventPath *)pointerEventPath; +- (BOOL)synthesizeWithError:(NSError *_Nullable *_Nullable)error; +@end + +// The runner-side session that delivers synthesized events to the system under +// test. +@interface XCTRunnerDaemonSession : NSObject ++ (instancetype)sharedSession; +- (void)synthesizeEvent:(XCSynthesizedEventRecord *)eventRecord + completion:(void (^)(NSError *_Nullable error))completion; +@end + +// Runs the block and converts any raised NSException into an NSError so a +// testing-framework assertion does not terminate the runner process. +NS_INLINE BOOL CompanionRunCatching(void (^block)(void), NSError *_Nullable *_Nullable error) { + @try { + block(); + return YES; + } @catch (NSException *exception) { + if (error) { + *error = [NSError errorWithDomain:@"dev.sanderling.companion" + code:1 + userInfo:@{NSLocalizedDescriptionKey: exception.reason ?: exception.name}]; + } + return NO; + } +} + +NS_ASSUME_NONNULL_END diff --git a/companion/Sources/RunnerTestCase.swift b/companion/Sources/RunnerTestCase.swift new file mode 100644 index 0000000..43af2e5 --- /dev/null +++ b/companion/Sources/RunnerTestCase.swift @@ -0,0 +1,14 @@ +import XCTest + +// The single long-lived test method: it starts the server and parks the runner +// in a run loop so the simulator process stays alive serving the protocol. +final class RunnerTestCase: XCTestCase { + func testServeForever() throws { + // A failed automation call must never end the test: the server has to + // keep serving so the host can recover and retry. + continueAfterFailure = true + let server = try Server() + server.start() + RunLoop.current.run() + } +} diff --git a/companion/Sources/Server.swift b/companion/Sources/Server.swift new file mode 100644 index 0000000..0f84cc4 --- /dev/null +++ b/companion/Sources/Server.swift @@ -0,0 +1,187 @@ +import Foundation +import Network +import UIKit +import XCTest + +// Newline-delimited JSON-over-TCP server. Listens on 127.0.0.1 on the port from +// the COMPANION_PORT environment variable (default 27753). Each line is one +// request; each reply is one line. +final class Server { + private let port: NWEndpoint.Port + private let listener: NWListener + private let queue = DispatchQueue(label: "dev.sanderling.companion.server") + // Requests are handled off the network queue so blocking automation work + // (snapshot, gesture synthesis) never stalls accept and receive. + private let work = DispatchQueue(label: "dev.sanderling.companion.work") + + // The bundle identifier of the most recently snapshotted app, used as the + // default target for typeText when no explicit bundleId is supplied. + private var currentBundleIdentifier = "com.apple.springboard" + + init() throws { + let resolvedPort = ProcessInfo.processInfo.environment["COMPANION_PORT"] + .flatMap { UInt16($0) } ?? 27753 + self.port = NWEndpoint.Port(rawValue: resolvedPort)! + let parameters = NWParameters.tcp + parameters.allowLocalEndpointReuse = true + // The simulator shares the host network stack, so an unbound listener + // would be reachable from the host's LAN interfaces. Pin it to + // loopback so only local processes can drive the automation surface. + parameters.requiredLocalEndpoint = NWEndpoint.hostPort(host: "127.0.0.1", port: self.port) + self.listener = try NWListener(using: parameters) + } + + func start() { + listener.newConnectionHandler = { [weak self] connection in + self?.accept(connection) + } + listener.start(queue: queue) + } + + private func accept(_ connection: NWConnection) { + connection.start(queue: queue) + receive(on: connection, buffer: Data()) + } + + private func receive(on connection: NWConnection, buffer: Data) { + connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { [weak self] data, _, isComplete, error in + guard let self = self else { return } + var working = buffer + if let data = data { + working.append(data) + } + while let newlineIndex = working.firstIndex(of: 0x0A) { + let lineData = working.subdata(in: working.startIndex.. [String: Any] { + guard let request = try JSONSerialization.jsonObject(with: lineData) as? [String: Any] else { + throw ServerError.malformedRequest + } + let requestId = request["id"] as? Int ?? 0 + guard let method = request["method"] as? String else { + throw ServerError.malformedRequest + } + let params = request["params"] as? [String: Any] ?? [:] + let result = try handle(method: method, params: params) + return ["id": requestId, "result": result] + } + + private func handle(method: String, params: [String: Any]) throws -> [String: Any] { + switch method { + case "health": + return ["ok": true] + case "describe": + return describeScreen() + case "snapshot": + let bundleIdentifier = params["bundleId"] as? String ?? currentBundleIdentifier + currentBundleIdentifier = bundleIdentifier + return ["elements": Snapshot.elements(bundleIdentifier: bundleIdentifier)] + case "gesture": + let events = params["events"] as? [[String: Any]] ?? [] + try Gesture.perform(events: events) + return ["ok": true] + case "gestureProbe": + return Gesture.probe() + case "launch": + let bundleIdentifier = params["bundleId"] as? String ?? currentBundleIdentifier + let foregroundIfRunning = params["foregroundIfRunning"] as? Bool ?? false + try AppLifecycle.launch(bundleIdentifier: bundleIdentifier, foregroundIfRunning: foregroundIfRunning) + return ["ok": true] + case "terminate": + let bundleIdentifier = params["bundleId"] as? String ?? currentBundleIdentifier + try AppLifecycle.terminate(bundleIdentifier: bundleIdentifier) + return ["ok": true] + case "typeText": + let text = params["text"] as? String ?? "" + let replace = params["replace"] as? Bool ?? false + let bundleIdentifier = params["bundleId"] as? String ?? currentBundleIdentifier + try TextInput.type(text: text, replace: replace, bundleIdentifier: bundleIdentifier) + return ["ok": true] + case "eraseText": + let count = params["count"] as? Int ?? 0 + let bundleIdentifier = params["bundleId"] as? String ?? currentBundleIdentifier + try TextInput.erase(count: count, bundleIdentifier: bundleIdentifier) + return ["ok": true] + case "pressKey": + let key = params["key"] as? String ?? "" + let bundleIdentifier = params["bundleId"] as? String ?? currentBundleIdentifier + try TextInput.pressKey(key: key, bundleIdentifier: bundleIdentifier) + return ["ok": true] + case "appState": + let bundleIdentifier = params["bundleId"] as? String ?? currentBundleIdentifier + return ["state": AppLifecycle.state(bundleIdentifier: bundleIdentifier)] + case "screenshot": + return try screenshot() + default: + throw ServerError.unknownMethod(method) + } + } + + private func describeScreen() -> [String: Any] { + // The runner process has no foreground scene, so UIScreen.main.bounds + // returns a fallback size. The springboard application snapshot frame is + // the device size in points; the screen scale comes from UIScreen. + var scale: CGFloat = 1 + let collect = { scale = UIScreen.main.scale } + if Thread.isMainThread { + collect() + } else { + DispatchQueue.main.sync(execute: collect) + } + var width = 0 + var height = 0 + if let root = Snapshot.elements(bundleIdentifier: "com.apple.springboard").first, + let frame = root["frame"] as? [String: Any] { + width = Int(frame["width"] as? Double ?? 0) + height = Int(frame["height"] as? Double ?? 0) + } + return [ + "widthPoints": width, + "heightPoints": height, + "scale": Double(scale), + ] + } + + private func screenshot() throws -> [String: Any] { + let image = XCUIScreen.main.screenshot() + return ["pngBase64": image.pngRepresentation.base64EncodedString()] + } + + private func send(_ object: [String: Any], on connection: NWConnection) { + guard var data = try? JSONSerialization.data(withJSONObject: object) else { + return + } + data.append(0x0A) + connection.send(content: data, completion: .contentProcessed { _ in }) + } + + enum ServerError: Error { + case malformedRequest + case unknownMethod(String) + } +} diff --git a/companion/Sources/Snapshot.swift b/companion/Sources/Snapshot.swift new file mode 100644 index 0000000..63783ce --- /dev/null +++ b/companion/Sources/Snapshot.swift @@ -0,0 +1,80 @@ +import XCTest + +// Flattens an accessibility snapshot tree into the legacy flat element array. +enum Snapshot { + // Produces the flat element array for the given bundle identifier. Falls back + // to springboard if the requested app is not running or its snapshot throws. + // Snapshot resolution can demand the test's execution context, which lives + // on the main thread, and a framework assertion raises an exception that + // would kill the server; so the capture runs on main inside the catch + // bridge, and an assertion yields an empty dump, which upstream treats as + // collapsed and retries. + static func elements(bundleIdentifier: String) -> [[String: Any]] { + var result: [[String: Any]] = [] + var caughtError: NSError? + let work = { + _ = CompanionRunCatching({ + if let captured = try? capture(bundleIdentifier: bundleIdentifier) { + result = captured + } else if let fallback = try? capture(bundleIdentifier: "com.apple.springboard") { + result = fallback + } + }, &caughtError) + } + if Thread.isMainThread { + work() + } else { + DispatchQueue.main.sync(execute: work) + } + return result + } + + private static func capture(bundleIdentifier: String) throws -> [[String: Any]] { + let application = XCUIApplication(bundleIdentifier: bundleIdentifier) + let root = try application.snapshot() + var result: [[String: Any]] = [] + walk(root, into: &result) + return result + } + + private static func walk(_ node: XCUIElementSnapshot, into result: inout [[String: Any]]) { + // The keyboard subtree is pruned: the legacy accessibility bridge + // never exposed it, its key frames are unreliable as tap targets, and + // its shift-state churn destabilizes settle hashing. + if node.elementType == .keyboard { + return + } + result.append(serialize(node)) + for child in node.children { + walk(child, into: &result) + } + } + + private static func serialize(_ node: XCUIElementSnapshot) -> [String: Any] { + let frame = node.frame + return [ + "type": elementTypeName(node.elementType), + "frame": [ + "x": Double(frame.origin.x), + "y": Double(frame.origin.y), + "width": Double(frame.size.width), + "height": Double(frame.size.height), + ], + "enabled": node.isEnabled, + "AXLabel": nullableString(node.label), + "AXValue": nullableString(stringifyValue(node.value)), + "AXUniqueId": nullableString(node.identifier), + ] + } + + private static func nullableString(_ value: String?) -> Any { + guard let value = value, !value.isEmpty else { return NSNull() } + return value + } + + private static func stringifyValue(_ value: Any?) -> String? { + guard let value = value else { return nil } + if let text = value as? String { return text } + return String(describing: value) + } +} diff --git a/companion/Sources/TextInput.swift b/companion/Sources/TextInput.swift new file mode 100644 index 0000000..2641fae --- /dev/null +++ b/companion/Sources/TextInput.swift @@ -0,0 +1,101 @@ +import XCTest + +// Types unicode text into the focused element through the public typeText API, +// avoiding the pasteboard entirely. With replace enabled, the current value +// length is consumed first by prepending that many delete keys so the whole +// replacement happens in one typeText call. +enum TextInput { + enum TextInputError: Error { + case typingFailed(String) + } + + static func type(text: String, replace: Bool, bundleIdentifier: String) throws { + let application = XCUIApplication(bundleIdentifier: bundleIdentifier) + + var payload = text + if replace { + let deleteCount = deletePrefixLength(bundleIdentifier: bundleIdentifier) + if deleteCount > 0 { + let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: deleteCount) + payload = deletes + text + } + } + + // Type at the application level so it lands in whatever holds keyboard + // focus, without resolving a specific element. Compose Multiplatform + // text fields do not reliably expose hasKeyboardFocus to the query layer. + // typeText must run on the main thread. + var caughtError: NSError? + var completed = false + runOnMain { + completed = CompanionRunCatching({ + application.typeText(payload) + }, &caughtError) + } + if !completed { + throw TextInputError.typingFailed(caughtError?.localizedDescription ?? "unknown") + } + } + + // erase deletes count characters from the focused field by typing that many + // delete keys. Used on device, where EraseText routes to the runner instead + // of the legacy companion's HID stream. + static func erase(count: Int, bundleIdentifier: String) throws { + guard count > 0 else { return } + let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count) + try typeOnFocus(deletes, bundleIdentifier: bundleIdentifier) + } + + // pressKey types a single logical key into the focused field. Only return is + // supported, matching the simulator companion's key surface. + static func pressKey(key: String, bundleIdentifier: String) throws { + switch key { + case "return", "enter", "Return", "Enter": + try typeOnFocus(XCUIKeyboardKey.return.rawValue, bundleIdentifier: bundleIdentifier) + default: + throw TextInputError.typingFailed("unsupported key \(key)") + } + } + + private static func typeOnFocus(_ payload: String, bundleIdentifier: String) throws { + let application = XCUIApplication(bundleIdentifier: bundleIdentifier) + var caughtError: NSError? + var completed = false + runOnMain { + completed = CompanionRunCatching({ + application.typeText(payload) + }, &caughtError) + } + if !completed { + throw TextInputError.typingFailed(caughtError?.localizedDescription ?? "unknown") + } + } + + private static func runOnMain(_ work: () -> Void) { + if Thread.isMainThread { + work() + } else { + DispatchQueue.main.sync(execute: work) + } + } + + // Sizes the delete prefix for replace. The snapshot does not say which + // editable field holds keyboard focus (Compose fields never expose + // hasKeyboardFocus), so the prefix covers the longest editable value on + // screen: deletes beyond the focused field's content are no-ops at the + // start of the field, while undersizing would leave residue and silently + // turn replace into append. + private static func deletePrefixLength(bundleIdentifier: String) -> Int { + let elements = Snapshot.elements(bundleIdentifier: bundleIdentifier) + var longest = 0 + for element in elements { + guard let type = element["type"] as? String, + type == "TextArea" || type == "TextField", + let value = element["AXValue"] as? String else { + continue + } + longest = max(longest, value.count) + } + return longest + } +} diff --git a/companion/prepare.sh b/companion/prepare.sh new file mode 100755 index 0000000..b1d8b8e --- /dev/null +++ b/companion/prepare.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# Builds the embeddable simulator runner tarball. +# +# Regenerates the Xcode project, builds the UI test bundle for a generic +# simulator destination (so no booted simulator is needed), then stages a +# relocatable test root: the runner app alongside its xctestrun. A port +# placeholder is injected into the xctestrun so the driver can substitute the +# real port before launching. The staged tree is packaged as a tarball. +# +# Output: ../internal/driver/ioscompanion/runnerassets/assets/runner-1.0.0.tar.gz + +set -euo pipefail + +# Keep the archiver from emitting AppleDouble (._name) sidecar entries that +# would otherwise duplicate extended attributes into the payload. +export COPYFILE_DISABLE=1 + +VERSION="1.0.0" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ASSETS_DIR="${SCRIPT_DIR}/../internal/driver/ioscompanion/runnerassets/assets" +OUTPUT="${ASSETS_DIR}/runner-${VERSION}.tar.gz" + +cd "${SCRIPT_DIR}" + +# Regenerate the project from its spec. The .xcodeproj is gitignored, so a +# fresh checkout must produce it before building. +/opt/homebrew/bin/xcodegen --spec project.yml + +# Build the UI test bundle for a generic simulator destination so no booted +# simulator is required; products land under build/Build/Products/. +xcodebuild build-for-testing \ + -project CompanionRunner.xcodeproj \ + -scheme CompanionRunner \ + -destination 'generic/platform=iOS Simulator' \ + -derivedDataPath build \ + CODE_SIGNING_ALLOWED=NO + +PRODUCTS="${SCRIPT_DIR}/build/Build/Products" + +STAGE="$(mktemp -d)" +trap 'rm -rf "${STAGE}"' EXIT + +# The xctestrun name embeds the simulator SDK version, so resolve it rather +# than hardcoding it. +XCTESTRUN="$(find "${PRODUCTS}" -maxdepth 1 -name '*.xctestrun' -print -quit)" +if [ -z "${XCTESTRUN}" ]; then + echo "no xctestrun under ${PRODUCTS}" >&2 + exit 1 +fi +cp "${XCTESTRUN}" "${STAGE}/runner.xctestrun" + +# Copy the runner app into the staged products directory. The .swiftmodule +# sibling is build-time only and is not copied. +mkdir -p "${STAGE}/Debug-iphonesimulator" +cp -R \ + "${PRODUCTS}/Debug-iphonesimulator/CompanionRunnerUITests-Runner.app" \ + "${STAGE}/Debug-iphonesimulator/CompanionRunnerUITests-Runner.app" + +# Inject the port placeholder the driver substitutes at launch time. +/usr/libexec/PlistBuddy \ + -c "Add :CompanionRunnerUITests:EnvironmentVariables:COMPANION_PORT string __COMPANION_PORT__" \ + "${STAGE}/runner.xctestrun" + +STAGED_BYTES="$(find "${STAGE}" -type f -exec stat -f%z {} + | awk '{sum += $1} END {print sum}')" + +mkdir -p "${ASSETS_DIR}" +tar -czf "${OUTPUT}" -C "${STAGE}" . + +SHA="$(shasum -a 256 "${OUTPUT}" | awk '{print $1}')" + +echo "wrote ${OUTPUT}" +echo "staged uncompressed size: ${STAGED_BYTES} bytes" +echo "sha256: ${SHA}" diff --git a/companion/project.yml b/companion/project.yml new file mode 100644 index 0000000..4ebd6a8 --- /dev/null +++ b/companion/project.yml @@ -0,0 +1,36 @@ +name: CompanionRunner +options: + bundleIdPrefix: dev.sanderling + deploymentTarget: + iOS: "16.0" + createIntermediateGroups: true + +settings: + base: + CODE_SIGNING_ALLOWED: "NO" + CODE_SIGNING_REQUIRED: "NO" + SWIFT_VERSION: "5.0" + +targets: + CompanionRunnerUITests: + type: bundle.ui-testing + platform: iOS + productName: CompanionRunnerUITests + sources: + - Sources + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: dev.sanderling.companion + SWIFT_OBJC_BRIDGING_HEADER: Sources/PrivateAutomation/PrivateAutomation.h + OTHER_LDFLAGS: + - "-framework" + - "XCTest" + +schemes: + CompanionRunner: + build: + targets: + CompanionRunnerUITests: all + test: + targets: + - CompanionRunnerUITests diff --git a/conformance/.gitignore b/conformance/.gitignore new file mode 100644 index 0000000..a1e0396 --- /dev/null +++ b/conformance/.gitignore @@ -0,0 +1 @@ +runs/ diff --git a/conformance/gates.sh b/conformance/gates.sh new file mode 100755 index 0000000..947b82d --- /dev/null +++ b/conformance/gates.sh @@ -0,0 +1,412 @@ +#!/usr/bin/env bash +# Scripted conformance gate for the iOS simulator driver. Runs five serial, +# non-overlapping 3-minute fuzz runs against the folio example app +# (examples/folio) and scores five gates (G1..G5) over the captured traces and +# output. Exits non-zero if any gate fails. +# +# Backends: +# BACKEND=simulator (default) drive the booted iOS simulator +# BACKEND=device drive an attached physical iPhone via the +# driver's runner-only device path; select it +# with IOS_DEVICE="" (passed as --ios-device) +# +# Usage: +# ./gates.sh run the simulator gates +# BACKEND=device IOS_DEVICE="iPhone" ./gates.sh +# ./gates.sh --self-test run the offline analyzer tests only +# +# Tunables (environment): +# RUNS=5 number of serial runs +# DURATION=3m per-run fuzz duration +# SEED=0 fuzz seed +# P95_LIMIT_MS=2500 G5 p95 step-latency ceiling in milliseconds +# SANDERLING=sanderling binary to invoke + +set -euo pipefail + +script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +folio_directory="$(cd "${script_directory}/../examples/folio" && pwd)" + +BACKEND="${BACKEND:-simulator}" +RUNS="${RUNS:-5}" +DURATION="${DURATION:-3m}" +SEED="${SEED:-0}" +P95_LIMIT_MS="${P95_LIMIT_MS:-2500}" +SANDERLING="${SANDERLING:-sanderling}" +IOS_DEVICE="${IOS_DEVICE:-iPhone 17 Pro}" + +bundle_id="app.folio" +spec_path="${folio_directory}/sanderling/spec.ts" +# The built app bundle differs by SDK: the simulator build lands under +# Debug-iphonesimulator, the device build under Debug-iphoneos. +if [[ "$BACKEND" == "device" ]]; then + ios_app="${folio_directory}/app/iosApp/build/Build/Products/Debug-iphoneos/iosApp.app" +else + ios_app="${folio_directory}/app/iosApp/build/Build/Products/Debug-iphonesimulator/iosApp.app" +fi + +# The companion binary, embedded for simulator runs. Referenced by file name +# only for the orphan-process check; prose elsewhere says "the companion". +companion_process_name="idb_companion" + +# Known-benign stderr lines the companion always prints. These are matched as +# fixed substrings and excluded from the G2 ERROR scan. Keep this list tight: +# only lines that are provably harmless and emitted on every healthy run. +benign_stderr_substrings=( + # The dynamic linker reports the same Objective-C class registered by two + # loaded images. The companion runs fine; this is cosmetic. The real line + # reads "objc[]: Class ...", so the fixed substring is "]: Class". + "]: Class" + "is implemented in both" + "One of the two will be used. Which one is undefined." + # gRPC and absl emit informational banner lines on startup. + "WARNING: All log messages before absl::InitializeLog()" +) + +# ---- gate analyzers (pure, operate on a single run directory) -------------- + +# G2 helper: strip benign companion noise, then report sanderling ERROR lines. +# sanderling's progress logger renders error-level records as lines beginning +# "error:" (see internal/testrun/progress.go). We also catch the upper-case +# ERROR token (word-bounded, so ERRORS/ERRORLESS and path fragments do not +# trip the gate) for safety against future handlers. +error_lines_in() { + local output_file="$1" + local filtered + filtered="$(cat "$output_file")" + local pattern + for pattern in "${benign_stderr_substrings[@]}"; do + filtered="$(printf '%s\n' "$filtered" | grep -vF "$pattern" || true)" + done + printf '%s\n' "$filtered" | grep -E '(^error:|\bERROR\b)' || true +} + +# G1: process exit status recorded by the runner loop. +gate_exit_zero() { + local run_directory="$1" + [[ "$(cat "${run_directory}/exit_status")" == "0" ]] +} + +# G2: no sanderling ERROR lines after filtering benign companion noise. +gate_no_error_lines() { + local run_directory="$1" + local found + found="$(error_lines_in "${run_directory}/output.log")" + [[ -z "$found" ]] +} + +# G3: the first hierarchy snapshot shows the login screen with empty email and +# password fields (clear-state proof). Reads the first trace line carrying a +# hierarchy and asserts LoginEmail/LoginPassword carry no text. +gate_clear_state() { + local run_directory="$1" + local trace_file="${run_directory}/trace.jsonl" + [[ -f "$trace_file" ]] || return 1 + local verdict + verdict="$(jq -s -r ' + [ .[] | select(.hierarchy != null) ] as $withHierarchy + | if ($withHierarchy | length) == 0 then "fail:no-hierarchy" + else ($withHierarchy[0].hierarchy.elements // []) as $elements + | ($elements | map(select(.resourceId == "LoginEmail")) | first) as $email + | ($elements | map(select(.resourceId == "LoginPassword")) | first) as $password + | if $email == null or $password == null then "fail:no-login-fields" + elif (($email.attrs.text // $email.text // "") != "") then "fail:email-not-empty" + elif (($password.attrs.text // $password.text // "") != "") then "fail:password-not-empty" + else "pass" end + end + ' "$trace_file")" + [[ "$verdict" == "pass" ]] +} + +# G4: no doubled text after InputText. For each InputText action targeting a +# field, the field's value in the NEXT hierarchy must not contain the input +# concatenated with itself (catches append-vs-replace and double-paste bugs). +# The action chosen at step N is applied before step N+1 is observed, so the +# effect lands in the following snapshot. +gate_no_doubled_text() { + local run_directory="$1" + local trace_file="${run_directory}/trace.jsonl" + [[ -f "$trace_file" ]] || return 1 + local verdict + verdict="$(jq -s -r ' + # Map a selector like "testTag:LoginScreen > testTag:LoginEmail" to its + # target field id: the token after the final ":" of the last segment. + def target_field(selector): + (selector | split(">") | last | gsub("^\\s+|\\s+$";"")) as $last + | ($last | split(":") | last); + + [ .[] | select(.hierarchy != null) ] as $steps + | reduce range(0; ($steps | length)) as $i ([]; + ($steps[$i]) as $current + | (if $i + 1 < ($steps | length) then $steps[$i + 1] else null end) as $next + | if ($current.next_action.kind == "InputText") and ($next != null) + then + (target_field($current.next_action.selector // "")) as $field + | ($current.next_action.text // "") as $typed + | (($next.hierarchy.elements // []) + | map(select(.resourceId == $field)) | first) as $element + | if $element != null and $typed != "" + then + (($element.attrs.text // $element.text // "")) as $value + | if ($value | contains($typed + $typed)) + then . + [{field: $field, typed: $typed, value: $value}] + else . end + else . end + else . end) + | if length == 0 then "pass" + else "fail:" + (.[0].field) + ":" + (.[0].value) end + ' "$trace_file")" + [[ "$verdict" == "pass" ]] +} + +# Emit one step-latency sample per consecutive trace-step pair, in +# milliseconds, on stdout. The trace records a wall-clock timestamp per step; +# the latency of a step is the gap to the next step's observation. Used by G5, +# which aggregates samples across all runs before computing the p95. Timestamps +# are RFC3339 with fractional seconds and a numeric offset, so they are parsed +# with python3 rather than jq's UTC-only fromdateiso8601. +emit_step_latencies() { + local trace_file="$1" + [[ -f "$trace_file" ]] || return 0 + jq -r 'select(.timestamp != null) | .timestamp' "$trace_file" \ + | python3 -c ' +import sys +from datetime import datetime +stamps = [datetime.fromisoformat(line.strip()) for line in sys.stdin if line.strip()] +for earlier, later in zip(stamps, stamps[1:]): + print(int((later - earlier).total_seconds() * 1000)) +' +} + +# Compute the p95 (nearest-rank) of the latency samples on stdin, in ms. +p95_of() { + python3 -c ' +import sys, math +samples = sorted(int(float(line)) for line in sys.stdin if line.strip()) +if not samples: + print(0) + sys.exit(0) +rank = max(1, math.ceil(0.95 * len(samples))) +print(samples[rank - 1]) +' +} + +# G5 orphan check: report any lingering companion, runner session (the hybrid +# simulator driver hosts an in-simulator runner), and, on the device backend, +# the device runner session. The usbmux tunnel is an in-process forwarder that +# dies with sanderling, so it leaves no process to check. Empty output is clean. +orphan_processes() { + local found="" + if pgrep -f "$companion_process_name" >/dev/null 2>&1; then + found+="companion " + fi + if pgrep -f "sanderling-runner.*xctestrun" >/dev/null 2>&1; then + found+="runner-session " + fi + if pgrep -f "CompanionRunnerUITests-Runner" >/dev/null 2>&1; then + found+="runner-app " + fi + if [[ "$BACKEND" == "device" ]]; then + # The device test session that hosts the runner. Its destination carries + # platform=iOS,id=. + if pgrep -f "xctestrun.*platform=iOS,id=" >/dev/null 2>&1; then + found+="device-session " + fi + fi + printf '%s' "$found" +} + +# ---- run orchestration ----------------------------------------------------- + +invoke_sanderling() { + local output_directory="$1" + local output_log="$2" + local exit_status_file="$3" + + # Both backends pass --ios-app-path so each run reinstalls the current build + # for a clean clear-state start (device install via devicectl, simulator via + # simctl). The device backend selects the connected iPhone by name; the + # simulator backend boots IOS_DEVICE if nothing is booted. + local target_flags=(--ios-device "$IOS_DEVICE" --ios-app-path "$ios_app") + + local status=0 + "$SANDERLING" test \ + --platform ios \ + --spec "$spec_path" \ + --bundle-id "$bundle_id" \ + "${target_flags[@]}" \ + --duration "$DURATION" \ + --seed "$SEED" \ + --output "$output_directory" \ + >"$output_log" 2>&1 || status=$? + printf '%s' "$status" >"$exit_status_file" +} + +# Locate the run directory sanderling created under output_directory (it nests +# a timestamped subdirectory) and normalise its artifacts up one level. +collect_run_artifacts() { + local output_directory="$1" + [[ -f "${output_directory}/trace.jsonl" ]] && return 0 + local produced + produced="$(find "$output_directory" -mindepth 2 -name 'trace.jsonl' 2>/dev/null | head -1 || true)" + if [[ -n "$produced" ]]; then + cp "$produced" "${output_directory}/trace.jsonl" 2>/dev/null || true + fi +} + +run_gates() { + if [[ "$BACKEND" == "simulator" ]]; then + echo "preparing folio build for the simulator backend" + ( cd "$folio_directory" && just ios >/dev/null ) + else + echo "preparing folio device build" + ( cd "$folio_directory" && just ios-device >/dev/null ) + fi + + local timestamp + timestamp="$(date +%Y%m%d-%H%M%S)" + local gate_root="${script_directory}/runs/${timestamp}" + mkdir -p "$gate_root" + echo "gate root: ${gate_root}" + echo "backend=${BACKEND} runs=${RUNS} duration=${DURATION} seed=${SEED} p95_limit_ms=${P95_LIMIT_MS}" + + local all_latencies="${gate_root}/all-latencies.txt" + : >"$all_latencies" + + local -a g1 g2 g3 g4 g5 + local run_index + for ((run_index = 1; run_index <= RUNS; run_index++)); do + local run_directory="${gate_root}/run-${run_index}" + mkdir -p "$run_directory" + echo "run ${run_index}/${RUNS} -> ${run_directory}" + + invoke_sanderling "$run_directory" "${run_directory}/output.log" "${run_directory}/exit_status" + collect_run_artifacts "$run_directory" + + g1[run_index]=$(gate_exit_zero "$run_directory" && echo PASS || echo FAIL) + g2[run_index]=$(gate_no_error_lines "$run_directory" && echo PASS || echo FAIL) + g3[run_index]=$(gate_clear_state "$run_directory" && echo PASS || echo FAIL) + g4[run_index]=$(gate_no_doubled_text "$run_directory" && echo PASS || echo FAIL) + + emit_step_latencies "${run_directory}/trace.jsonl" >>"$all_latencies" + + local orphans + orphans="$(orphan_processes)" + if [[ -n "$orphans" ]]; then + g5[run_index]="FAIL" + echo " orphaned processes after run ${run_index}: ${orphans}" + else + g5[run_index]="PENDING" + fi + done + + local p95 + p95="$(p95_of <"$all_latencies")" + local final_orphans + final_orphans="$(orphan_processes)" + + # G5 is global: p95 is computed over every run's samples and an orphan after + # any single run fails the whole gate. Decide once, then stamp every row. + local g5_global="PASS" + if [[ "$p95" -ge "$P95_LIMIT_MS" ]]; then + g5_global="FAIL" + fi + if [[ -n "$final_orphans" ]]; then + g5_global="FAIL" + echo "orphaned processes at end: ${final_orphans}" + fi + for ((run_index = 1; run_index <= RUNS; run_index++)); do + if [[ "${g5[run_index]}" == "FAIL" ]]; then + g5_global="FAIL" + fi + done + for ((run_index = 1; run_index <= RUNS; run_index++)); do + g5[run_index]="$g5_global" + done + + echo + printf 'run G1 G2 G3 G4 G5\n' + local verdict="PASS" + for ((run_index = 1; run_index <= RUNS; run_index++)); do + printf '%-4s %-5s %-5s %-5s %-5s %-5s\n' \ + "$run_index" "${g1[run_index]}" "${g2[run_index]}" "${g3[run_index]}" "${g4[run_index]}" "${g5[run_index]}" + for cell in "${g1[run_index]}" "${g2[run_index]}" "${g3[run_index]}" "${g4[run_index]}" "${g5[run_index]}"; do + [[ "$cell" == "PASS" ]] || verdict="FAIL" + done + done + echo + echo "p95 step latency: ${p95}ms (limit ${P95_LIMIT_MS}ms)" + if [[ "$verdict" == "PASS" ]]; then + echo "GATES PASS" + return 0 + fi + echo "GATES FAIL" + return 1 +} + +# ---- offline self-test ----------------------------------------------------- +# Exercises every analyzer against canned passing/failing run directories under +# testdata/ so the parsing logic can be checked without a device. + +self_test() { + local testdata="${script_directory}/testdata" + local failures=0 + + assert() { + local label="$1" expected="$2" actual="$3" + if [[ "$expected" == "$actual" ]]; then + printf 'ok %s\n' "$label" + else + printf 'FAIL %s (expected %s, got %s)\n' "$label" "$expected" "$actual" + failures=$((failures + 1)) + fi + } + + assert "G1 pass run exits zero" PASS \ + "$(gate_exit_zero "${testdata}/pass" && echo PASS || echo FAIL)" + assert "G1 fail run nonzero exit" FAIL \ + "$(gate_exit_zero "${testdata}/g1-nonzero-exit" && echo PASS || echo FAIL)" + + assert "G2 pass run no errors" PASS \ + "$(gate_no_error_lines "${testdata}/pass" && echo PASS || echo FAIL)" + assert "G2 benign noise tolerated" PASS \ + "$(gate_no_error_lines "${testdata}/g2-benign-only" && echo PASS || echo FAIL)" + assert "G2 real error caught" FAIL \ + "$(gate_no_error_lines "${testdata}/g2-real-error" && echo PASS || echo FAIL)" + + assert "G3 pass clear state" PASS \ + "$(gate_clear_state "${testdata}/pass" && echo PASS || echo FAIL)" + assert "G3 dirty fields caught" FAIL \ + "$(gate_clear_state "${testdata}/g3-dirty-field" && echo PASS || echo FAIL)" + + assert "G4 pass no doubling" PASS \ + "$(gate_no_doubled_text "${testdata}/pass" && echo PASS || echo FAIL)" + assert "G4 doubled text caught" FAIL \ + "$(gate_no_doubled_text "${testdata}/g4-doubled-text" && echo PASS || echo FAIL)" + + local pass_p95 slow_p95 + pass_p95="$(emit_step_latencies "${testdata}/pass/trace.jsonl" | p95_of)" + slow_p95="$(emit_step_latencies "${testdata}/g5-slow-p95/trace.jsonl" | p95_of)" + assert "G5 pass p95 under limit" PASS \ + "$([[ "$pass_p95" -lt "$P95_LIMIT_MS" ]] && echo PASS || echo FAIL)" + assert "G5 slow p95 over limit" FAIL \ + "$([[ "$slow_p95" -lt "$P95_LIMIT_MS" ]] && echo PASS || echo FAIL)" + + echo + if [[ "$failures" -eq 0 ]]; then + echo "SELF-TEST PASS" + return 0 + fi + echo "SELF-TEST FAIL (${failures} failed)" + return 1 +} + +main() { + case "${1:-}" in + --self-test) self_test ;; + "") run_gates ;; + *) echo "usage: $0 [--self-test]" >&2; exit 2 ;; + esac +} + +main "$@" diff --git a/conformance/testdata/g1-nonzero-exit/exit_status b/conformance/testdata/g1-nonzero-exit/exit_status new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/conformance/testdata/g1-nonzero-exit/exit_status @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/conformance/testdata/g1-nonzero-exit/output.log b/conformance/testdata/g1-nonzero-exit/output.log new file mode 100644 index 0000000..ca891a9 --- /dev/null +++ b/conformance/testdata/g1-nonzero-exit/output.log @@ -0,0 +1 @@ +run complete diff --git a/conformance/testdata/g1-nonzero-exit/trace.jsonl b/conformance/testdata/g1-nonzero-exit/trace.jsonl new file mode 100644 index 0000000..f40d3b2 --- /dev/null +++ b/conformance/testdata/g1-nonzero-exit/trace.jsonl @@ -0,0 +1,4 @@ +{"step": 1, "timestamp": "2026-06-06T12:00:00.600000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "demo@folio.app", "selector": "testTag:LoginScreen > testTag:LoginEmail"}} +{"step": 2, "timestamp": "2026-06-06T12:00:01.200000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "ledger123", "selector": "testTag:LoginScreen > testTag:LoginPassword"}} +{"step": 3, "timestamp": "2026-06-06T12:00:01.800000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} +{"step": 4, "timestamp": "2026-06-06T12:00:02.400000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} diff --git a/conformance/testdata/g2-benign-only/exit_status b/conformance/testdata/g2-benign-only/exit_status new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/conformance/testdata/g2-benign-only/exit_status @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/conformance/testdata/g2-benign-only/output.log b/conformance/testdata/g2-benign-only/output.log new file mode 100644 index 0000000..9403538 --- /dev/null +++ b/conformance/testdata/g2-benign-only/output.log @@ -0,0 +1,5 @@ +companion pid=4242 listening on 127.0.0.1:51000 +objc[4242]: Class FBSDKError is implemented in both /a (0x1) and /b (0x2). One of the two will be used. Which one is undefined. +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +archived previous runs under runs/ERRORS_archive +run complete diff --git a/conformance/testdata/g2-benign-only/trace.jsonl b/conformance/testdata/g2-benign-only/trace.jsonl new file mode 100644 index 0000000..f40d3b2 --- /dev/null +++ b/conformance/testdata/g2-benign-only/trace.jsonl @@ -0,0 +1,4 @@ +{"step": 1, "timestamp": "2026-06-06T12:00:00.600000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "demo@folio.app", "selector": "testTag:LoginScreen > testTag:LoginEmail"}} +{"step": 2, "timestamp": "2026-06-06T12:00:01.200000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "ledger123", "selector": "testTag:LoginScreen > testTag:LoginPassword"}} +{"step": 3, "timestamp": "2026-06-06T12:00:01.800000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} +{"step": 4, "timestamp": "2026-06-06T12:00:02.400000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} diff --git a/conformance/testdata/g2-real-error/exit_status b/conformance/testdata/g2-real-error/exit_status new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/conformance/testdata/g2-real-error/exit_status @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/conformance/testdata/g2-real-error/output.log b/conformance/testdata/g2-real-error/output.log new file mode 100644 index 0000000..6fb91f1 --- /dev/null +++ b/conformance/testdata/g2-real-error/output.log @@ -0,0 +1,5 @@ +companion pid=4242 listening on 127.0.0.1:51000 +objc[4242]: Class FBSDKError is implemented in both /a (0x1) and /b (0x2). One of the two will be used. Which one is undefined. +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +archived previous runs under runs/ERRORS_archive +error: hierarchy fetch failed: companion connection lost diff --git a/conformance/testdata/g2-real-error/trace.jsonl b/conformance/testdata/g2-real-error/trace.jsonl new file mode 100644 index 0000000..f40d3b2 --- /dev/null +++ b/conformance/testdata/g2-real-error/trace.jsonl @@ -0,0 +1,4 @@ +{"step": 1, "timestamp": "2026-06-06T12:00:00.600000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "demo@folio.app", "selector": "testTag:LoginScreen > testTag:LoginEmail"}} +{"step": 2, "timestamp": "2026-06-06T12:00:01.200000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "ledger123", "selector": "testTag:LoginScreen > testTag:LoginPassword"}} +{"step": 3, "timestamp": "2026-06-06T12:00:01.800000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} +{"step": 4, "timestamp": "2026-06-06T12:00:02.400000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} diff --git a/conformance/testdata/g3-dirty-field/exit_status b/conformance/testdata/g3-dirty-field/exit_status new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/conformance/testdata/g3-dirty-field/exit_status @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/conformance/testdata/g3-dirty-field/output.log b/conformance/testdata/g3-dirty-field/output.log new file mode 100644 index 0000000..ca891a9 --- /dev/null +++ b/conformance/testdata/g3-dirty-field/output.log @@ -0,0 +1 @@ +run complete diff --git a/conformance/testdata/g3-dirty-field/trace.jsonl b/conformance/testdata/g3-dirty-field/trace.jsonl new file mode 100644 index 0000000..5319d91 --- /dev/null +++ b/conformance/testdata/g3-dirty-field/trace.jsonl @@ -0,0 +1,2 @@ +{"step": 1, "timestamp": "2026-06-06T12:00:00.600000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "stale@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "leftover"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "demo@folio.app", "selector": "testTag:LoginScreen > testTag:LoginEmail"}} +{"step": 2, "timestamp": "2026-06-06T12:00:01.200000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "leftover"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} diff --git a/conformance/testdata/g4-doubled-text/exit_status b/conformance/testdata/g4-doubled-text/exit_status new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/conformance/testdata/g4-doubled-text/exit_status @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/conformance/testdata/g4-doubled-text/output.log b/conformance/testdata/g4-doubled-text/output.log new file mode 100644 index 0000000..ca891a9 --- /dev/null +++ b/conformance/testdata/g4-doubled-text/output.log @@ -0,0 +1 @@ +run complete diff --git a/conformance/testdata/g4-doubled-text/trace.jsonl b/conformance/testdata/g4-doubled-text/trace.jsonl new file mode 100644 index 0000000..7108273 --- /dev/null +++ b/conformance/testdata/g4-doubled-text/trace.jsonl @@ -0,0 +1,2 @@ +{"step": 1, "timestamp": "2026-06-06T12:00:00.600000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "demo@folio.app", "selector": "testTag:LoginScreen > testTag:LoginEmail"}} +{"step": 2, "timestamp": "2026-06-06T12:00:01.200000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.appdemo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} diff --git a/conformance/testdata/g5-slow-p95/exit_status b/conformance/testdata/g5-slow-p95/exit_status new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/conformance/testdata/g5-slow-p95/exit_status @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/conformance/testdata/g5-slow-p95/output.log b/conformance/testdata/g5-slow-p95/output.log new file mode 100644 index 0000000..ca891a9 --- /dev/null +++ b/conformance/testdata/g5-slow-p95/output.log @@ -0,0 +1 @@ +run complete diff --git a/conformance/testdata/g5-slow-p95/trace.jsonl b/conformance/testdata/g5-slow-p95/trace.jsonl new file mode 100644 index 0000000..ae4d996 --- /dev/null +++ b/conformance/testdata/g5-slow-p95/trace.jsonl @@ -0,0 +1,4 @@ +{"step": 1, "timestamp": "2026-06-06T12:00:04+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "demo@folio.app", "selector": "testTag:LoginScreen > testTag:LoginEmail"}} +{"step": 2, "timestamp": "2026-06-06T12:00:08+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "ledger123", "selector": "testTag:LoginScreen > testTag:LoginPassword"}} +{"step": 3, "timestamp": "2026-06-06T12:00:12+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} +{"step": 4, "timestamp": "2026-06-06T12:00:16+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} diff --git a/conformance/testdata/generate.py b/conformance/testdata/generate.py new file mode 100644 index 0000000..2132ce3 --- /dev/null +++ b/conformance/testdata/generate.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# Regenerates the canned gate-analyzer fixtures. Each fixture is a minimal but +# format-accurate run directory (trace.jsonl + output.log + exit_status) that +# isolates one gate's pass or fail condition. Run from this directory: +# python3 generate.py +import json +import os +from datetime import datetime, timedelta, timezone + +HERE = os.path.dirname(os.path.abspath(__file__)) +BASE = datetime(2026, 6, 6, 12, 0, 0, tzinfo=timezone(timedelta(hours=5, minutes=30))) + + +def element(resource_id, text="", element_class="android.widget.EditText"): + return { + "resourceId": resource_id, + "class": element_class, + "editable": True, + "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, + "attrs": {"resource-id": resource_id, "class": element_class, "text": text}, + } + + +def login_screen(email_text="", password_text=""): + return { + "elements": [ + element("LoginScreen", element_class="android.view.View"), + element("LoginEmail", email_text), + element("LoginPassword", password_text), + element("LoginSubmit", element_class="android.view.View"), + ] + } + + +def input_action(field, text): + return { + "kind": "InputText", + "text": text, + "selector": f"testTag:LoginScreen > testTag:{field}", + } + + +def step(index, gap_ms, hierarchy, next_action=None): + timestamp = (BASE + timedelta(milliseconds=index * gap_ms)).isoformat() + record = {"step": index, "timestamp": timestamp, "hierarchy": hierarchy} + if next_action is not None: + record["next_action"] = next_action + return record + + +def write_run(name, steps, exit_status="0", output_lines=None): + directory = os.path.join(HERE, name) + os.makedirs(directory, exist_ok=True) + with open(os.path.join(directory, "trace.jsonl"), "w") as handle: + for record in steps: + handle.write(json.dumps(record) + "\n") + with open(os.path.join(directory, "exit_status"), "w") as handle: + handle.write(exit_status) + lines = output_lines if output_lines is not None else ["run complete\n"] + with open(os.path.join(directory, "output.log"), "w") as handle: + handle.writelines(lines) + + +# A clean run: empty login fields first, single (not doubled) typed values in +# the following snapshots, sub-second step gaps. +def healthy_steps(gap_ms=600): + return [ + step(1, gap_ms, login_screen("", ""), input_action("LoginEmail", "demo@folio.app")), + step(2, gap_ms, login_screen("demo@folio.app", ""), input_action("LoginPassword", "ledger123")), + step(3, gap_ms, login_screen("demo@folio.app", "ledger123")), + step(4, gap_ms, login_screen("demo@folio.app", "ledger123")), + ] + + +BENIGN_NOISE = [ + "companion pid=4242 listening on 127.0.0.1:51000\n", + "objc[4242]: Class FBSDKError is implemented in both /a (0x1) and /b (0x2). " + "One of the two will be used. Which one is undefined.\n", + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + # An incidental ERROR substring inside a longer word: the G2 scan is + # word-bounded, so this must not trip the gate. + "archived previous runs under runs/ERRORS_archive\n", +] + +write_run("pass", healthy_steps()) + +write_run("g1-nonzero-exit", healthy_steps(), exit_status="1") + +write_run("g2-benign-only", healthy_steps(), output_lines=BENIGN_NOISE + ["run complete\n"]) + +write_run( + "g2-real-error", + healthy_steps(), + output_lines=BENIGN_NOISE + + ["error: hierarchy fetch failed: companion connection lost\n"], +) + +# G3: first snapshot already carries text in the login fields (clear-state +# failed to wipe the previous session). +write_run( + "g3-dirty-field", + [ + step(1, 600, login_screen("stale@folio.app", "leftover"), input_action("LoginEmail", "demo@folio.app")), + step(2, 600, login_screen("demo@folio.app", "leftover")), + ], +) + +# G4: after typing into LoginEmail the next snapshot shows the value appended to +# itself (append-vs-replace / double-paste regression). +write_run( + "g4-doubled-text", + [ + step(1, 600, login_screen("", ""), input_action("LoginEmail", "demo@folio.app")), + step(2, 600, login_screen("demo@folio.appdemo@folio.app", "")), + ], +) + +# G5: step gaps far exceed the 2.5s ceiling, driving p95 over the limit. +write_run("g5-slow-p95", healthy_steps(gap_ms=4000)) + +print("fixtures written") diff --git a/conformance/testdata/pass/exit_status b/conformance/testdata/pass/exit_status new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/conformance/testdata/pass/exit_status @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/conformance/testdata/pass/output.log b/conformance/testdata/pass/output.log new file mode 100644 index 0000000..ca891a9 --- /dev/null +++ b/conformance/testdata/pass/output.log @@ -0,0 +1 @@ +run complete diff --git a/conformance/testdata/pass/trace.jsonl b/conformance/testdata/pass/trace.jsonl new file mode 100644 index 0000000..f40d3b2 --- /dev/null +++ b/conformance/testdata/pass/trace.jsonl @@ -0,0 +1,4 @@ +{"step": 1, "timestamp": "2026-06-06T12:00:00.600000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "demo@folio.app", "selector": "testTag:LoginScreen > testTag:LoginEmail"}} +{"step": 2, "timestamp": "2026-06-06T12:00:01.200000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": ""}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}, "next_action": {"kind": "InputText", "text": "ledger123", "selector": "testTag:LoginScreen > testTag:LoginPassword"}} +{"step": 3, "timestamp": "2026-06-06T12:00:01.800000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} +{"step": 4, "timestamp": "2026-06-06T12:00:02.400000+05:30", "hierarchy": {"elements": [{"resourceId": "LoginScreen", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginScreen", "class": "android.view.View", "text": ""}}, {"resourceId": "LoginEmail", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginEmail", "class": "android.widget.EditText", "text": "demo@folio.app"}}, {"resourceId": "LoginPassword", "class": "android.widget.EditText", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginPassword", "class": "android.widget.EditText", "text": "ledger123"}}, {"resourceId": "LoginSubmit", "class": "android.view.View", "editable": true, "bounds": {"left": 34, "top": 87, "right": 286, "bottom": 135}, "attrs": {"resource-id": "LoginSubmit", "class": "android.view.View", "text": ""}}]}} diff --git a/docs/manual/cli.md b/docs/manual/cli.md index f715614..6f22de2 100644 --- a/docs/manual/cli.md +++ b/docs/manual/cli.md @@ -19,11 +19,12 @@ Run a spec against an app for a fixed duration. | `--launcher-activity` | resolved | Optional `/` to launch. Overrides default resolution. | | `--platform` | `android` | Target platform: `android`, `ios`, or `web`. | | `--avd` | optional (android) | Android AVD name to boot if no device is connected. Required only when no device is connected and multiple AVDs exist. | -| `--ios-device` | optional (ios) | iOS simulator name or UDID to boot if none is running. | +| `--ios-device` | optional (ios) | iOS target: a simulator name/UDID to boot, or a connected device's name, UDID, or CoreDevice id. | +| `--ios-app-path` | optional (ios) | Path to the `.app` bundle for clear-state reinstall (simulator via `simctl`, device via `devicectl`). | | `--duration` | `5m` | Total test duration (`30s`, `5m`, `2h`, `1d`). | | `--seed` | `0` | PRNG seed. `0` uses a random seed and records it in `meta.json`. | | `--output` | `./runs` | Output directory for traces. | -| `--clear-data` | `false` | Clear app data before launching so the run starts from a fresh install. | +| `--clear-data` | `true` | Clear app data before launching so the run starts from a fresh install. Pass `--clear-data=false` to resume prior state. | ## `sanderling replay [run-or-runs-dir]` @@ -42,7 +43,7 @@ See [the replay UI page](../replay/) for the panel reference and keyboard shortc Check the host environment for a working sanderling setup. ``` -sanderling doctor [--platform web|android|ios|all] +sanderling doctor [--platform web|android|ios|ios-device|all] ``` `--platform` defaults to `all`, which runs every platform's checks (deduped). Pass a specific platform to scope the output. @@ -51,7 +52,8 @@ sanderling doctor [--platform web|android|ios|all] |---|---| | `web` | headless Chromium can launch (the bundled CDP surface boots a real browser). | | `android` | `adb` on PATH; `emulator` on PATH or under `ANDROID_HOME`; Java 17+; embedded native sidecar JAR is real. | -| `ios` | `xcrun` on PATH; `simctl` on PATH; Java 17+; embedded native sidecar JAR is real. | +| `ios` | `xcrun` on PATH; `simctl` on PATH. The simulator path drives the native companion with no JVM. | +| `ios-device` | the `ios` checks plus `devicectl`; the macOS `usbmuxd` socket; a connected, paired device; App Store Connect signing credentials present. | ## `sanderling version` diff --git a/docs/manual/getting-started.md b/docs/manual/getting-started.md index 04517a7..d59898c 100644 --- a/docs/manual/getting-started.md +++ b/docs/manual/getting-started.md @@ -87,6 +87,21 @@ IOS_DEVICE="iPhone 15" just test-ios # pick a different simulator `just test-ios` boots the simulator if needed, builds and installs the app, then runs `sanderling test --platform ios`. +#### Physical device + +A connected iPhone is driven over a usbmux tunnel by a runner the driver builds and signs at run time. The tunnel talks to macOS's own `usbmuxd`, so nothing extra is installed beyond Xcode. It needs App Store Connect signing credentials in the environment (a gitignored `.env` is loaded by `just`): + +```sh +SANDERLING_IOS_TEAM=<10-char team id> +ASC_API_KEY_ID= +ASC_API_ISSUER_ID= +ASC_API_KEY_PATH= + +IOS_DEVICE="iPhone" just test-ios-device # name, UDID, or CoreDevice id +``` + +Run `sanderling doctor --platform ios-device` to check `devicectl`, the `usbmuxd` socket, a connected and paired device, and the signing credentials before a run. + ## What you get When the run ends, the trace lands in `sanderling/runs//`: diff --git a/examples/folio/app/androidApp/src/main/AndroidManifest.xml b/examples/folio/app/androidApp/src/main/AndroidManifest.xml index c94068c..0001cc9 100644 --- a/examples/folio/app/androidApp/src/main/AndroidManifest.xml +++ b/examples/folio/app/androidApp/src/main/AndroidManifest.xml @@ -2,6 +2,7 @@ Folio +