mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(runner): treat transient apply errors as transitional steps
Sidecar input RPCs occasionally hang with DEADLINE_EXCEEDED or UNAVAILABLE on long fuzzing runs. The per-step loop previously propagated any applyAction error and killed the run after a single flake. Detect transient gRPC failures via status.FromError, mark the step transitional, skip the post-action idle poll, and continue to the next step. Fatal errors (outer ctx cancellation, non-transient codes, verifier crashes) still propagate.
This commit is contained in:
1 parent
7beefdc5c9
commit
113282ffb2
1 file changed
+54
-16
+54
-16
@@ -10,6 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/driver"
|
||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||
@@ -215,6 +217,28 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
logger.Warn("residual encode failed", "step", stepIndex, "err", residualErr)
|
||||
}
|
||||
|
||||
applySkipped := false
|
||||
if nextErr == nil {
|
||||
if err := applyAction(ctx, options.Driver, nextAction, tree); err != nil {
|
||||
if isWDADrop(err) {
|
||||
return summary, fmt.Errorf("step %d: iOS XCTest runner lost connection - known WDA startup flake, re-run the test: %w", stepIndex, err)
|
||||
}
|
||||
if isTransientApplyError(ctx, err) {
|
||||
logger.Warn("transient apply error; marking step transitional", "step", stepIndex, "err", err)
|
||||
transitional = true
|
||||
applySkipped = true
|
||||
lastAction = nil
|
||||
} else {
|
||||
return summary, fmt.Errorf("step %d apply: %w", stepIndex, err)
|
||||
}
|
||||
} else {
|
||||
actionCopy := nextAction
|
||||
lastAction = &actionCopy
|
||||
}
|
||||
} else {
|
||||
lastAction = nil
|
||||
}
|
||||
|
||||
step := trace.Step{
|
||||
Index: stepIndex,
|
||||
Timestamp: stepStart,
|
||||
@@ -237,24 +261,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
Properties: violations,
|
||||
})
|
||||
}
|
||||
|
||||
if nextErr == nil {
|
||||
if err := applyAction(ctx, options.Driver, nextAction, tree); err != nil {
|
||||
if isWDADrop(err) {
|
||||
return summary, fmt.Errorf("step %d: iOS XCTest runner lost connection - known WDA startup flake, re-run the test: %w", stepIndex, err)
|
||||
}
|
||||
return summary, fmt.Errorf("step %d apply: %w", stepIndex, err)
|
||||
}
|
||||
actionCopy := nextAction
|
||||
lastAction = &actionCopy
|
||||
} else {
|
||||
lastAction = nil
|
||||
}
|
||||
|
||||
// Wait actions are themselves a settling: skip the idle poll. Actions
|
||||
// that mutate the UI fall through to WaitForIdle so the next step's
|
||||
// concurrent fetches observe a stable post-action state.
|
||||
if nextErr == nil && nextAction.Kind != verifier.ActionKindWait {
|
||||
// concurrent fetches observe a stable post-action state. A transient
|
||||
// apply error means nothing landed, so the idle poll has nothing to
|
||||
// settle and may itself hang on the same device condition.
|
||||
if nextErr == nil && !applySkipped && nextAction.Kind != verifier.ActionKindWait {
|
||||
idleCtx, idleCancel := context.WithTimeout(ctx, options.IdleTimeout)
|
||||
idleErr := options.Driver.WaitForIdle(idleCtx, options.IdleTimeout)
|
||||
if idleErr != nil && idleCtx.Err() == nil {
|
||||
@@ -829,3 +841,29 @@ func isWDADrop(err error) bool {
|
||||
return strings.Contains(msg, "ConnectException") ||
|
||||
(strings.Contains(msg, "code = Internal") && strings.Contains(msg, "SocketException"))
|
||||
}
|
||||
|
||||
// isTransientApplyError reports whether an applyAction failure is a transient
|
||||
// device-side hang (sidecar RPC deadline, momentary unavailability) rather than
|
||||
// a fatal condition. Such steps are recorded as transitional and the loop
|
||||
// continues. The run context being cancelled is never transient: it means the
|
||||
// caller wants to stop.
|
||||
func isTransientApplyError(runCtx context.Context, err error) bool {
|
||||
if err == nil || runCtx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
if s, ok := status.FromError(err); ok {
|
||||
switch s.Code() {
|
||||
case codes.DeadlineExceeded, codes.Unavailable:
|
||||
return true
|
||||
case codes.Internal:
|
||||
message := s.Message()
|
||||
if strings.Contains(message, "DEADLINE_EXCEEDED") || strings.Contains(message, "UNAVAILABLE") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in new issue
Block a user