feat(cli): add --max-steps for step-bounded runs

runner.Options.MaxSteps already worked but was unreachable from the command
line. A step budget is what makes two generators comparable: one making a
model call per step and one drawing from a PRNG are not comparable per second.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 18:11:24 +05:30
1 parent 26b49b379a
commit 8388ac5d2a
4 files changed
+51 -1

No files matched your search

+5
View File
@@ -27,6 +27,7 @@ type testOptions struct {
iosAppPath string
androidAppPath string
duration time.Duration
maxSteps int
seed int64
output string
clearData bool
@@ -60,6 +61,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
flagSet.StringVar(&options.iosAppPath, "ios-app-path", "", "path to the .app bundle for iOS clear-state reinstall (simulator: simctl; device: devicectl)")
flagSet.StringVar(&options.androidAppPath, "android-app-path", "", "path to the .apk for Android clear-state reinstall; required to reset apps on OEM builds that deny `pm clear`")
flagSet.DurationVar(&options.duration, "duration", 5*time.Minute, "total test duration")
flagSet.IntVar(&options.maxSteps, "max-steps", 0, "stop after this many steps (0 = no cap; the duration deadline governs). A step budget is what makes two generators comparable, since one making a model call per step and one drawing from a PRNG are not comparable per second")
flagSet.Int64Var(&options.seed, "seed", 0, "RNG seed (0 = random)")
flagSet.StringVar(&options.output, "output", "./runs", "output directory for traces")
flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state")
@@ -78,6 +80,9 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
default:
return testOptions{}, fmt.Errorf("unsupported platform: %q (android, ios, web)", options.platform)
}
if options.maxSteps < 0 {
return testOptions{}, fmt.Errorf("--max-steps must not be negative: %d", options.maxSteps)
}
switch options.generator {
case "seeded", "llm":
default:
+38
View File
@@ -259,3 +259,41 @@ func TestRun_TestSubcommand_PipelineErrors(t *testing.T) {
t.Errorf("expected bundle-resolution error for the missing spec, got %v", err)
}
}
func TestParseTestArgs_MaxStepsDefaultsToUncapped(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
}, io.Discard)
if err != nil {
t.Fatal(err)
}
if options.maxSteps != 0 {
t.Errorf("maxSteps default: got %d, want 0", options.maxSteps)
}
}
func TestParseTestArgs_MaxSteps(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--max-steps", "300",
}, io.Discard)
if err != nil {
t.Fatal(err)
}
if options.maxSteps != 300 {
t.Errorf("maxSteps: got %d, want 300", options.maxSteps)
}
}
func TestParseTestArgs_RejectsNegativeMaxSteps(t *testing.T) {
_, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--max-steps", "-1",
}, io.Discard)
if err == nil {
t.Fatal("expected an error for a negative --max-steps")
}
}
+1
View File
@@ -18,6 +18,7 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer)
IosAppPath: options.iosAppPath,
AndroidAppPath: options.androidAppPath,
Duration: options.duration,
MaxSteps: options.maxSteps,
Seed: options.seed,
Output: options.output,
ClearData: options.clearData,