mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 20:17:09 +00:00
feat(runner): treat transient apply errors as transitional steps
Sidecar input RPCs occasionally hang with DEADLINE_EXCEEDED or UNAVAILABLE on long fuzzing runs. The per-step loop previously propagated any applyAction error and killed the run after a single flake. Detect transient gRPC failures via status.FromError, mark the step transitional, skip the post-action idle poll, and continue to the next step. Fatal errors (outer ctx cancellation, non-transient codes, verifier crashes) still propagate.
This commit is contained in:
1 parent
7beefdc5c9
commit
113282ffb2
1 file changed
+54
-16
+54
-16
@@ -10,6 +10,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
"google.golang.org/grpc/status"
|
||||||
|
|
||||||
"github.com/priyanshujain/sanderling/internal/driver"
|
"github.com/priyanshujain/sanderling/internal/driver"
|
||||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||||
@@ -215,6 +217,28 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
logger.Warn("residual encode failed", "step", stepIndex, "err", residualErr)
|
logger.Warn("residual encode failed", "step", stepIndex, "err", residualErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
applySkipped := false
|
||||||
|
if nextErr == nil {
|
||||||
|
if err := applyAction(ctx, options.Driver, nextAction, tree); err != nil {
|
||||||
|
if isWDADrop(err) {
|
||||||
|
return summary, fmt.Errorf("step %d: iOS XCTest runner lost connection - known WDA startup flake, re-run the test: %w", stepIndex, err)
|
||||||
|
}
|
||||||
|
if isTransientApplyError(ctx, err) {
|
||||||
|
logger.Warn("transient apply error; marking step transitional", "step", stepIndex, "err", err)
|
||||||
|
transitional = true
|
||||||
|
applySkipped = true
|
||||||
|
lastAction = nil
|
||||||
|
} else {
|
||||||
|
return summary, fmt.Errorf("step %d apply: %w", stepIndex, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
actionCopy := nextAction
|
||||||
|
lastAction = &actionCopy
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
lastAction = nil
|
||||||
|
}
|
||||||
|
|
||||||
step := trace.Step{
|
step := trace.Step{
|
||||||
Index: stepIndex,
|
Index: stepIndex,
|
||||||
Timestamp: stepStart,
|
Timestamp: stepStart,
|
||||||
@@ -237,24 +261,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
Properties: violations,
|
Properties: violations,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if nextErr == nil {
|
|
||||||
if err := applyAction(ctx, options.Driver, nextAction, tree); err != nil {
|
|
||||||
if isWDADrop(err) {
|
|
||||||
return summary, fmt.Errorf("step %d: iOS XCTest runner lost connection - known WDA startup flake, re-run the test: %w", stepIndex, err)
|
|
||||||
}
|
|
||||||
return summary, fmt.Errorf("step %d apply: %w", stepIndex, err)
|
|
||||||
}
|
|
||||||
actionCopy := nextAction
|
|
||||||
lastAction = &actionCopy
|
|
||||||
} else {
|
|
||||||
lastAction = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait actions are themselves a settling: skip the idle poll. Actions
|
// Wait actions are themselves a settling: skip the idle poll. Actions
|
||||||
// that mutate the UI fall through to WaitForIdle so the next step's
|
// that mutate the UI fall through to WaitForIdle so the next step's
|
||||||
// concurrent fetches observe a stable post-action state.
|
// concurrent fetches observe a stable post-action state. A transient
|
||||||
if nextErr == nil && nextAction.Kind != verifier.ActionKindWait {
|
// apply error means nothing landed, so the idle poll has nothing to
|
||||||
|
// settle and may itself hang on the same device condition.
|
||||||
|
if nextErr == nil && !applySkipped && nextAction.Kind != verifier.ActionKindWait {
|
||||||
idleCtx, idleCancel := context.WithTimeout(ctx, options.IdleTimeout)
|
idleCtx, idleCancel := context.WithTimeout(ctx, options.IdleTimeout)
|
||||||
idleErr := options.Driver.WaitForIdle(idleCtx, options.IdleTimeout)
|
idleErr := options.Driver.WaitForIdle(idleCtx, options.IdleTimeout)
|
||||||
if idleErr != nil && idleCtx.Err() == nil {
|
if idleErr != nil && idleCtx.Err() == nil {
|
||||||
@@ -829,3 +841,29 @@ func isWDADrop(err error) bool {
|
|||||||
return strings.Contains(msg, "ConnectException") ||
|
return strings.Contains(msg, "ConnectException") ||
|
||||||
(strings.Contains(msg, "code = Internal") && strings.Contains(msg, "SocketException"))
|
(strings.Contains(msg, "code = Internal") && strings.Contains(msg, "SocketException"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isTransientApplyError reports whether an applyAction failure is a transient
|
||||||
|
// device-side hang (sidecar RPC deadline, momentary unavailability) rather than
|
||||||
|
// a fatal condition. Such steps are recorded as transitional and the loop
|
||||||
|
// continues. The run context being cancelled is never transient: it means the
|
||||||
|
// caller wants to stop.
|
||||||
|
func isTransientApplyError(runCtx context.Context, err error) bool {
|
||||||
|
if err == nil || runCtx.Err() != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if s, ok := status.FromError(err); ok {
|
||||||
|
switch s.Code() {
|
||||||
|
case codes.DeadlineExceeded, codes.Unavailable:
|
||||||
|
return true
|
||||||
|
case codes.Internal:
|
||||||
|
message := s.Message()
|
||||||
|
if strings.Contains(message, "DEADLINE_EXCEEDED") || strings.Contains(message, "UNAVAILABLE") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
Reference in new issue
Block a user