mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
Merge origin/master into llm-recording-and-analysis
Brings in #73, which landed web fact-parity work overlapping this branch: selector-tagged ax handles, aria-disabled in `enabled`, shadow-DOM traversal and a `scrollable`/`editable` dump, plus a third folio property and two new testTags on HomeScreen. Six files conflicted. The option-carrying ones took the union of both sides' fields, so `--label-source` and `--exit-on-violation` both reach the pipeline. In web-runtime.ts both sides changed how an element is described: master gave `elementHandle` a selector to tag handles with, this branch gave it raw attribute names and a field hint. Both survive, and `enabled` now answers through master's isEnabled while `editable` stays. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
commit
d987526e47
73 files changed
+6639
-457
No files matched your search
+60
-12
@@ -20,6 +20,26 @@ import (
|
||||
|
||||
const sidecarStartupTimeout = 30 * time.Second
|
||||
|
||||
// launchTimeout bounds the pre-run app launch. It happens before the runner
|
||||
// starts, so --duration does not cover it, and Execute's context is the bare
|
||||
// signal-aware root with no deadline of its own: a driver wedged here would
|
||||
// hang the run forever having printed nothing and written no trace. Generous
|
||||
// enough to sit above every driver's own launch bound (the iOS clear-state path
|
||||
// reinstalls the app first) so a driver-level error is what a user usually
|
||||
// sees, and this stays the backstop. A variable so the timeout test can shrink
|
||||
// it.
|
||||
var launchTimeout = 3 * time.Minute
|
||||
|
||||
// launchApp starts the app under test under a bounded context.
|
||||
func launchApp(ctx context.Context, activeDriver driver.DeviceDriver, options Options) error {
|
||||
launchCtx, cancel := context.WithTimeout(ctx, launchTimeout)
|
||||
defer cancel()
|
||||
if err := activeDriver.Launch(launchCtx, options.BundleID, options.ClearData, nil); err != nil {
|
||||
return fmt.Errorf("launch app: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Options are the parameters for a single test pipeline run.
|
||||
type Options struct {
|
||||
Spec string
|
||||
@@ -38,6 +58,10 @@ type Options struct {
|
||||
// Arm labels the experiment cell this run belongs to and is recorded in
|
||||
// meta.json so a directory of runs can be attributed to a cell.
|
||||
Arm string
|
||||
// ExitOnViolation stops the run at the first violation and reports the
|
||||
// recorded violations as a ViolationsError, so a caller (CI) can tell
|
||||
// "the run found the bug" from "the run finished clean".
|
||||
ExitOnViolation bool
|
||||
// Generator selects the action picker: "llm" or the default seeded picker.
|
||||
Generator string
|
||||
// LabelSource selects how candidates are named to the model picker, and is
|
||||
@@ -145,8 +169,8 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error {
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
if err := activeDriver.Launch(ctx, options.BundleID, options.ClearData, nil); err != nil {
|
||||
return fmt.Errorf("launch app: %w", err)
|
||||
if err := launchApp(ctx, activeDriver, options); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if web, ok := activeDriver.(driver.WebDriver); ok && len(webBundle.JavaScript) > 0 {
|
||||
@@ -193,16 +217,17 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error {
|
||||
fmt.Fprintf(stdout, "running for %s (seed=%d)\n", options.Duration, seed)
|
||||
}
|
||||
summary, err := runner.Run(ctx, runner.Options{
|
||||
Duration: options.Duration,
|
||||
MaxSteps: options.MaxSteps,
|
||||
IdleTimeout: 1 * time.Second,
|
||||
BundleID: options.BundleID,
|
||||
Driver: activeDriver,
|
||||
Verifier: verifierInstance,
|
||||
TraceWriter: traceWriter,
|
||||
Logger: newProgressLogger(stdout),
|
||||
Generator: options.Generator,
|
||||
LabelSource: options.LabelSource,
|
||||
Duration: options.Duration,
|
||||
MaxSteps: options.MaxSteps,
|
||||
IdleTimeout: 1 * time.Second,
|
||||
BundleID: options.BundleID,
|
||||
Driver: activeDriver,
|
||||
Verifier: verifierInstance,
|
||||
TraceWriter: traceWriter,
|
||||
Logger: newProgressLogger(stdout),
|
||||
Generator: options.Generator,
|
||||
LabelSource: options.LabelSource,
|
||||
StopOnViolation: options.ExitOnViolation,
|
||||
})
|
||||
|
||||
terminateCtx, terminateCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
@@ -215,9 +240,32 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error {
|
||||
|
||||
fmt.Fprintf(stdout, "\nelapsed: %s\n", summary.EndTime.Sub(summary.StartTime).Round(time.Millisecond))
|
||||
runner.RenderSummary(stdout, summary, options.Platform)
|
||||
return runOutcome(options, summary)
|
||||
}
|
||||
|
||||
// runOutcome turns a finished run into the pipeline's result. Without
|
||||
// --exit-on-violation a run that found violations is still a successful run
|
||||
// (the summary reports them), which is the behaviour every existing caller
|
||||
// depends on.
|
||||
func runOutcome(options Options, summary runner.Summary) error {
|
||||
if options.ExitOnViolation && len(summary.Violations) > 0 {
|
||||
return ViolationsError{Count: len(summary.Violations)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ViolationsError reports a run that recorded violations under
|
||||
// --exit-on-violation. It is deliberately distinct from every other error the
|
||||
// pipeline returns: those mean the harness broke, this one means the run did
|
||||
// its job and found something.
|
||||
type ViolationsError struct {
|
||||
Count int
|
||||
}
|
||||
|
||||
func (e ViolationsError) Error() string {
|
||||
return fmt.Sprintf("%d violation record(s)", e.Count)
|
||||
}
|
||||
|
||||
// bundleInputs holds the pre-driver assembly: alias map, seed, esbuild defines,
|
||||
// and the resolved spec-API/goja-runtime paths the bundler consumes.
|
||||
type bundleInputs struct {
|
||||
|
||||
Reference in new issue
Block a user