fix: harden apple runner recovery

apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -181,6 +181,38 @@ extension RunnerTests {
     )
     XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse))
   }
+
+  func testExecuteDispatchedReturnsBusyBeforeMainThreadFastPath() throws {
+    let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#)
+    abandonedMainThreadWorkCount = 1
+    abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -2)
+    defer {
+      abandonedMainThreadWorkCount = 0
+      abandonedMainThreadWorkSince = nil
+    }
+
+    let response = try executeDispatched(command: command)
+
+    XCTAssertFalse(response.ok)
+    XCTAssertEqual(response.error?.code, "RUNNER_BUSY")
+    XCTAssertTrue(response.error?.message.contains("previous command") == true)
+  }
+
+  func testExecuteDispatchedReturnsWedgedBeforeMainThreadFastPath() throws {
+    let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-wedged"}"#)
+    abandonedMainThreadWorkCount = 1
+    abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -(mainThreadWedgeThreshold + 1))
+    defer {
+      abandonedMainThreadWorkCount = 0
+      abandonedMainThreadWorkSince = nil
+    }
+
+    let response = try executeDispatched(command: command)
+
+    XCTAssertFalse(response.ok)
+    XCTAssertEqual(response.error?.code, "RUNNER_WEDGED")
+    XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true)
+  }
 #endif
 
   func execute(command: Command) throws -> Response {
@@ -296,9 +328,6 @@ extension RunnerTests {
   }
 
   private func executeDispatched(command: Command) throws -> Response {
-    if Thread.isMainThread {
-      return try executeOnMainSafely(command: command)
-    }
     // XCTest work cannot be cancelled mid-flight: once the watchdog abandons a main-queue
     // block, queueing more main-thread commands behind it only buries the runner deeper.
     // Refuse fast instead so the daemon backs off while the abandoned work drains; past the
@@ -311,6 +340,9 @@ extension RunnerTests {
     case .wedged(let abandonedForSeconds):
       return runnerWedgedResponse(command: command, abandonedForSeconds: abandonedForSeconds)
     }
+    if Thread.isMainThread {
+      return try executeOnMainSafely(command: command)
+    }
     var result: Result<Response, Error>?
     let semaphore = DispatchSemaphore(value: 0)
     let workState = MainThreadWorkState()

apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -114,6 +114,17 @@ extension RunnerTests {
 
   func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget {
     guard let x, let y else {
+      // Bare `type` targets the current first responder. On iOS we intentionally do not trust
+      // `hasKeyboardFocus`, but an already-visible software keyboard is sufficient evidence that
+      // app.typeText has a receiver; waiting the full readiness timeout cannot prove a stronger
+      // target because there is no selector/coordinate focus move to validate.
+      if isKeyboardVisible(app: app) {
+        return TextEntryTarget(
+          element: focusedTextInput(app: app),
+          refreshPoint: nil,
+          prefersFocusedElement: true
+        )
+      }
       let focused = waitForTextEntryReadiness(
         app: app,
         target: TextEntryTarget(

apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift

@@ -234,6 +234,45 @@ extension RunnerTests {
     return trimmed
   }
 
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS
+  func testDuplicateCommandIdCoalescesOntoInFlightExecution() throws {
+    let command = try JSONDecoder().decode(
+      Command.self,
+      from: Data(#"{"command":"snapshot","commandId":"snapshot-coalesce"}"#.utf8)
+    )
+    var primaryData: Data?
+    var waiterData: Data?
+    defer {
+      inFlightCommandIds.removeAll()
+      inFlightCommandWaiters.removeAll()
+    }
+
+    XCTAssertFalse(
+      attachToInFlightCommandIfNeeded(command: command) { result in
+        primaryData = result.data
+      }
+    )
+    XCTAssertTrue(
+      attachToInFlightCommandIfNeeded(command: command) { result in
+        waiterData = result.data
+      }
+    )
+
+    let delivered = Data("single-result".utf8)
+    deliverCommandResult(
+      command: command,
+      result: (delivered, false)
+    ) { result in
+      primaryData = result.data
+    }
+
+    XCTAssertEqual(primaryData, delivered)
+    XCTAssertEqual(waiterData, delivered)
+    XCTAssertFalse(inFlightCommandIds.contains("snapshot-coalesce"))
+    XCTAssertNil(inFlightCommandWaiters["snapshot-coalesce"])
+  }
+#endif
+
   // MARK: - Response Encoding
 
   private func jsonResponse(status: Int, response: Response) -> Data {

src/platforms/apple/core/__tests__/runner-client.test.ts

@@ -700,6 +700,59 @@ test('parseRunnerResponse preserves XCTest recorded failure code and hint', asyn
   );
 });
 
+test('parseRunnerResponse maps RUNNER_BUSY to retriable command failure', async () => {
+  const hint = 'Wait a few seconds and retry.';
+  const response = new Response(
+    JSON.stringify({
+      ok: false,
+      error: {
+        code: 'RUNNER_BUSY',
+        message: 'The runner is still finishing abandoned work.',
+        hint,
+      },
+    }),
+  );
+  const session = { ready: true };
+
+  await assert.rejects(
+    () => parseRunnerResponse(response, session, '/tmp/runner.log'),
+    (error: unknown) => {
+      assert.ok(error instanceof AppError);
+      assert.equal(error.code, 'COMMAND_FAILED');
+      assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
+      assert.equal(error.details?.retriable, true);
+      assert.equal(error.details?.hint, hint);
+      assert.equal(isRetryableRunnerError(error), true);
+      return true;
+    },
+  );
+});
+
+test('parseRunnerResponse preserves RUNNER_WEDGED as a fatal runner code', async () => {
+  const response = new Response(
+    JSON.stringify({
+      ok: false,
+      error: {
+        code: 'RUNNER_WEDGED',
+        message: 'The runner main thread is wedged.',
+        hint: 'The runner session will be restarted.',
+      },
+    }),
+  );
+  const session = { ready: true };
+
+  await assert.rejects(
+    () => parseRunnerResponse(response, session, '/tmp/runner.log'),
+    (error: unknown) => {
+      assert.ok(error instanceof AppError);
+      assert.equal(error.code, 'RUNNER_WEDGED');
+      assert.equal(error.details?.runnerErrorCode, 'RUNNER_WEDGED');
+      assert.equal(isRetryableRunnerError(error), false);
+      return true;
+    },
+  );
+});
+
 test('parseRunnerResponse classifies target app AXRuntime CoreText font crashes from runner log tail', async () => {
   const logPath = writeRunnerLogTail(`
 Thread 0 Crashed::  Dispatch queue: com.apple.main-thread

src/platforms/apple/core/__tests__/runner-command-retry.test.ts

@@ -10,12 +10,14 @@ const {
   mockEnsureRunnerSession,
   mockExecuteRunnerCommandWithSession,
   mockEmitDiagnostic,
+  mockGetRunnerSessionSnapshot,
   mockInvalidateRunnerSession,
   mockMarkRunnerXctestrunArtifactBadForRun,
 } = vi.hoisted(() => ({
   mockEnsureRunnerSession: vi.fn(),
   mockExecuteRunnerCommandWithSession: vi.fn(),
   mockEmitDiagnostic: vi.fn(),
+  mockGetRunnerSessionSnapshot: vi.fn(),
   mockInvalidateRunnerSession: vi.fn(),
   mockMarkRunnerXctestrunArtifactBadForRun: vi.fn(),
 }));
@@ -38,6 +40,7 @@ vi.mock('../runner/runner-session.ts', async () => {
     ...actual,
     ensureRunnerSession: mockEnsureRunnerSession,
     executeRunnerCommandWithSession: mockExecuteRunnerCommandWithSession,
+    getRunnerSessionSnapshot: mockGetRunnerSessionSnapshot,
     invalidateRunnerSession: mockInvalidateRunnerSession,
   };
 });
@@ -63,6 +66,7 @@ import type { RunnerXctestrunArtifact } from '../runner/runner-xctestrun.ts';
 beforeEach(() => {
   vi.resetAllMocks();
   resetRunnerRecycleLedgerForTests();
+  mockGetRunnerSessionSnapshot.mockReturnValue(null);
   mockMarkRunnerXctestrunArtifactBadForRun.mockResolvedValue(undefined);
 });
 
@@ -1190,6 +1194,54 @@ test('a request pays for at most one runner recycle, then fails fast with a pres
   assert.equal(mockEnsureRunnerSession.mock.calls.length, 2);
 });
 
+test('a failed replacement boot does not consume the request recycle budget', async () => {
+  const requestId = 'req-recycle-transient-boot-failure';
+  mockEnsureRunnerSession
+    .mockResolvedValueOnce(makeRunnerSession({ port: 8100, ready: true }))
+    .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner did not accept connection'))
+    .mockResolvedValueOnce(makeRunnerSession({ port: 8101, ready: true }));
+  mockExecuteRunnerCommandWithSession
+    .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
+    .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
+    .mockResolvedValueOnce({ nodes: [], truncated: false });
+
+  const result = await runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' }, { requestId });
+
+  assert.deepEqual(result, { nodes: [], truncated: false });
+  assert.equal(mockEnsureRunnerSession.mock.calls.length, 3);
+});
+
+test('alive session reuse in the same request does not consume recycle budget', async () => {
+  const requestId = 'req-alive-reuse-before-recycle';
+  mockGetRunnerSessionSnapshot
+    .mockReturnValueOnce(null)
+    .mockReturnValueOnce({ alive: true })
+    .mockReturnValueOnce(null);
+  mockEnsureRunnerSession
+    .mockResolvedValueOnce(makeRunnerSession({ port: 8100, ready: true }))
+    .mockResolvedValueOnce(makeRunnerSession({ port: 8100, ready: true }))
+    .mockResolvedValueOnce(makeRunnerSession({ port: 8101, ready: true }));
+  mockExecuteRunnerCommandWithSession
+    .mockResolvedValueOnce({ message: 'first tap' })
+    .mockResolvedValueOnce({ message: 'second tap' })
+    .mockResolvedValueOnce({ message: 'recovered tap' });
+
+  assert.deepEqual(
+    await runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }, { requestId }),
+    { message: 'first tap' },
+  );
+  assert.deepEqual(
+    await runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }, { requestId }),
+    { message: 'second tap' },
+  );
+  assert.deepEqual(
+    await runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }, { requestId }),
+    { message: 'recovered tap' },
+  );
+
+  assert.equal(mockEnsureRunnerSession.mock.calls.length, 3);
+});
+
 test('a later command in the same request cannot pay for a second recycle boot', async () => {
   const requestId = 'req-restart-cap';
   const staleSession = makeRunnerSession({ port: 8100, ready: true });

src/platforms/apple/core/__tests__/runner-recycle-ledger.test.ts

@@ -3,11 +3,12 @@ import assert from 'node:assert/strict';
 import { AppError } from '../../../../kernel/errors.ts';
 import {
   buildRunnerRecycleBudgetExhaustedError,
+  commitRunnerRecycle,
   hasRunnerRequestTouchedSession,
   markRunnerRequestTouchedSession,
   resetRunnerRecycleLedgerForTests,
   runnerRecycleLedgerKey,
-  tryConsumeRunnerRecycle,
+  tryBeginRunnerRecycle,
 } from '../runner/runner-recycle-ledger.ts';
 
 beforeEach(() => {
@@ -23,18 +24,28 @@ test('ledger key prefers the request id and falls back to the command id', () =>
   assert.equal(runnerRecycleLedgerKey({ requestId: '  ' }, { commandId: ' ' }), undefined);
 });
 
-test('a request gets exactly one runner recycle, then must fail fast', () => {
+test('a request gets exactly one successful runner recycle, then must fail fast', () => {
   const key = 'request:req-wedge';
-  assert.equal(tryConsumeRunnerRecycle(key), true);
-  assert.equal(tryConsumeRunnerRecycle(key), false);
-  assert.equal(tryConsumeRunnerRecycle(key), false);
+  assert.equal(tryBeginRunnerRecycle(key), true);
+  commitRunnerRecycle(key);
+  assert.equal(tryBeginRunnerRecycle(key), false);
+  assert.equal(tryBeginRunnerRecycle(key), false);
   // Other requests keep their own budget.
-  assert.equal(tryConsumeRunnerRecycle('request:req-other'), true);
+  assert.equal(tryBeginRunnerRecycle('request:req-other'), true);
+});
+
+test('failed replacement boots do not consume the recycle budget', () => {
+  const key = 'request:req-transient-boot-failure';
+  assert.equal(tryBeginRunnerRecycle(key), true);
+  assert.equal(tryBeginRunnerRecycle(key), true);
+  commitRunnerRecycle(key);
+  assert.equal(tryBeginRunnerRecycle(key), false);
 });
 
 test('untracked keys never block (no scope to account against)', () => {
-  assert.equal(tryConsumeRunnerRecycle(undefined), true);
-  assert.equal(tryConsumeRunnerRecycle(undefined), true);
+  assert.equal(tryBeginRunnerRecycle(undefined), true);
+  assert.equal(tryBeginRunnerRecycle(undefined), true);
+  commitRunnerRecycle(undefined); // no-op, must not throw
   assert.equal(hasRunnerRequestTouchedSession(undefined), false);
   markRunnerRequestTouchedSession(undefined); // no-op, must not throw
 });

src/platforms/apple/core/__tests__/runner-session.test.ts

@@ -1650,6 +1650,38 @@ test('runner session clears recency when an allowlisted command returns XCTest r
   assert.equal(getRunnerSessionSnapshot(device.id), null);
 });
 
+test('runner session invalidates when the runner reports abandoned main-thread work is wedged', async () => {
+  const device = { ...IOS_SIMULATOR, id: 'runner-session-wedged-sim' };
+  const session = await ensureRunnerSession(device, {});
+  session.ready = true;
+  mockWaitForRunner.mockClear();
+  mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ uptimeMs: 42 }));
+  mockSendRunnerCommandOnce.mockResolvedValueOnce(
+    runnerError({
+      code: 'RUNNER_WEDGED',
+      message: 'The runner main thread is wedged.',
+    }),
+  );
+
+  await assert.rejects(
+    () =>
+      executeRunnerCommandWithSession(
+        device,
+        session,
+        { command: 'tap', x: 1, y: 2, appBundleId: 'com.example.demo' },
+        '/tmp/runner.log',
+        30_000,
+      ),
+    (error: unknown) => {
+      assert.ok(error instanceof AppError);
+      assert.equal(error.code, 'RUNNER_WEDGED');
+      return true;
+    },
+  );
+
+  assert.equal(getRunnerSessionSnapshot(device.id), null);
+});
+
 function makeRunnerSession(overrides: Partial<RunnerSession> = {}): RunnerSession {
   return {
     sessionId: `session-${overrides.port ?? 8100}`,

src/platforms/apple/core/runner/runner-contract.ts

@@ -118,6 +118,7 @@ export type RunnerSequenceStep = {
 export function isRetryableRunnerError(err: unknown): boolean {
   if (!(err instanceof AppError)) return false;
   if (err.code !== 'COMMAND_FAILED') return false;
+  if (err.details?.retriable === true) return true;
   const message = `${err.message ?? ''}`.toLowerCase();
   if (message.includes('xcodebuild exited early')) return false;
   if (message.includes('device is busy') && message.includes('connecting')) return false;