true]; } } } namespace app\common\service { class DifyWorkflowRunStatus { public static function lookup(array $identity, int $timeoutMs): array { ++$GLOBALS['closedRecoveryAdapterCalls']; return ($GLOBALS['closedRecoveryAdapter'])($identity, $timeoutMs); } } } namespace think\facade { class Log { public static function write(string $message, string $level = ''): void { $GLOBALS['closedRecoveryLogs'][] = [$message, $level]; if (!empty($GLOBALS['closedRecoveryLogThrows'])) throw new \RuntimeException('synthetic private logging failure'); } } } namespace { require dirname(__DIR__) . '/app/adminapi/service/AssistantRequestControl.php'; require dirname(__DIR__) . '/app/adminapi/controller/tcm/DiagnosisController.php'; use app\adminapi\service\AssistantRequestControl; use app\adminapi\controller\tcm\DiagnosisController; $GLOBALS['closedRecoveryNowNs'] = 100000000000; $GLOBALS['closedRecoveryAdapterCalls'] = 0; $GLOBALS['closedRecoveryLogs'] = []; $assertions = 0; function closedExpect(bool $ok, string $message): void { ++$GLOBALS['assertions']; if (!$ok) throw new \RuntimeException($message); } function advanceClosedMs(int $milliseconds): void { $GLOBALS['closedRecoveryNowNs'] += $milliseconds * 1000000; } function safeClosedResult(array $result): void { $expected = ['status', 'transport_closed', 'eligible_task_count', 'queried_task_count', 'lookup_attempt_count', 'confirmed_task_count', 'upstream_pending_task_count', 'budget_exhausted', 'elapsed_ms']; closedExpect(array_keys($result) === $expected, 'closed caller exposes only its exact aggregate schema'); closedExpect(in_array($result['status'], ['transport_open', 'complete', 'pending', 'budget_exhausted'], true), 'aggregate status is allowlisted'); foreach ($result as $key => $value) { if ($key === 'status') continue; closedExpect(in_array($key, ['transport_closed', 'budget_exhausted'], true) ? is_bool($value) : is_int($value) && $value >= 0, 'aggregate ' . $key . ' has a safe bounded type'); } } $directory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'zyt-closed-recovery-' . bin2hex(random_bytes(8)); $number = 0; $make = static function (string $run = 'synthetic-run', bool $closed = true, bool $bind = true) use ($directory, &$number): array { advanceClosedMs(10000); $uuid = sprintf('29345678-1234-4234-8234-%012d', ++$number); $control = new AssistantRequestControl(17, 23, $uuid, $directory); closedExpect($control->begin(), 'isolated request begins'); $task = 'synthetic-task-' . $number; if ($bind) $control->bindTask('qwen', 'synthetic-user', $task, $run); if ($closed) $control->finish('error'); return [$control, $directory . DIRECTORY_SEPARATOR . hash('sha256', '17:23:' . $uuid) . '.json', $task]; }; $read = static fn (string $path): array => json_decode((string) file_get_contents($path), true, 512, JSON_THROW_ON_ERROR); $write = static function (string $path, array $state): void { file_put_contents($path, json_encode($state, JSON_THROW_ON_ERROR)); }; $handoff = static function (AssistantRequestControl $control, string $task, string $run): void { try { $control->bindTask('qwen', 'synthetic-user', $task, $run); closedExpect(false, 'handoff must stay blocked'); } catch (\RuntimeException $e) { closedExpect($e->getMessage() === 'UPSTREAM_TASK_HANDOFF_BLOCKED', 'unexpected own pair is retained as a blocked handoff'); } }; $never = static function (): array { closedExpect(false, 'lookup must not run'); return []; }; try { [$open, $openPath] = $make('synthetic-open-run', false); $result = $open->reconcileClosedTasks($never); safeClosedResult($result); closedExpect($result['status'] === 'transport_open' && $result['lookup_attempt_count'] === 0 && $result['upstream_pending_task_count'] === 1 && !isset($read($openPath)['closed_verification_started_at']), 'open cancel-worker race neither queries nor starts the global recovery budget'); advanceClosedMs(10000); $open->finish('error'); $result = $open->reconcileClosedTasks(static fn (array $task): array => $task + ['status' => 'stopped']); closedExpect($result['status'] === 'complete' && $result['confirmed_task_count'] === 1, 'stream owner recovers only after finish even if a prior caller observed open transport'); [$empty] = $make('synthetic-empty-run', true, false); closedExpect($empty->reconcileClosedTasks($never)['status'] === 'complete', 'empty closed transport has no GET'); [$legacy, $legacyPath] = $make(''); $legacyResult = $legacy->reconcileClosedTasks($never); closedExpect($legacyResult['status'] === 'pending' && $legacyResult['eligible_task_count'] === 0 && $legacyResult['upstream_pending_task_count'] === 1 && !isset($read($legacyPath)['closed_verification_started_at']), 'legacy missing run remains pending without guessed identity or budget consumption'); foreach (['stopped', 'succeeded', 'failed'] as $status) { [$exact, $exactPath] = $make('synthetic-exact-' . $status); $exact->signalTimeout(); $before = $read($exactPath); $result = $exact->reconcileClosedTasks(static function (array $task, int $timeoutMs) use ($exact, $status): array { closedExpect(count($task) === 5 && $timeoutMs > 0 && $timeoutMs <= 1000, 'lookup receives only exact private pair and bounded I/O deadline'); closedExpect($exact->publicState()['transport_closed'], 'lookup runs outside the file and GC locks'); closedExpect($exact->reconcileClosedTasks(static function (): array { closedExpect(false, 'nested caller cannot duplicate claimed I/O'); return []; })['lookup_attempt_count'] === 0, 'same request verifier lease also covers a nested caller'); return $task + ['status' => $status, 'body' => 'synthetic-private-body']; }); safeClosedResult($result); $public = $exact->publicState(); closedExpect($result['status'] === 'complete' && $result['queried_task_count'] === 1 && $result['lookup_attempt_count'] === 1 && $public['status'] === 'error' && $public['timeout_reached'] && !$public['cancel_requested'] && !$exact->shouldAbort() && $public['upstream_terminal_status'] === $status, 'terminal recovery never turns ordinary timeout into user cancel or report success'); closedExpect(!isset($read($exactPath)['upstream']) && !str_contains((string) file_get_contents($exactPath), 'synthetic-private-body'), 'only verified identity is released; lookup body is discarded'); closedExpect($read($exactPath)['verification_attempts'] === 1 && ($read($exactPath)['stop_attempts'] ?? null) === ($before['stop_attempts'] ?? null), 'recovery does not issue or reset stop attempts'); closedExpect(!$exact->canStartAttempt() && !$exact->begin(), 'recovery cannot restart or replay a closed request generation'); closedExpect($exact->reconcileClosedTasks($never)['lookup_attempt_count'] === 0, 'confirmed pair never causes another GET'); } [$known, , $knownTask] = $make('synthetic-known', false); $handoff($known, 'synthetic-known-handoff', 'synthetic-known-handoff-run'); $known->markTerminal($knownTask, 'succeeded', 'synthetic-known'); $known->markTerminal('synthetic-known-handoff', 'failed', 'synthetic-known-handoff-run'); $known->finish('done'); closedExpect($known->reconcileClosedTasks($never)['status'] === 'complete', 'all stream-confirmed targets skip independent lookup'); [$multi, $multiPath, $multiTask] = $make('synthetic-multi-primary'); $handoff($multi, 'synthetic-multi-second', 'synthetic-multi-second-run'); $handoff($multi, 'synthetic-multi-legacy', ''); $seen = []; $result = $multi->reconcileClosedTasks(static function (array $task) use (&$seen): array { $seen[] = $task['task_id']; return $task + ['status' => 'failed']; }); closedExpect($seen === [$multiTask, 'synthetic-multi-second'] && $result['confirmed_task_count'] === 2 && $result['upstream_pending_task_count'] === 1 && $result['status'] === 'pending', 'each own valid pair is verified separately and one missing-run handoff stays pending'); closedExpect(!isset($read($multiPath)['upstream']) && !isset($read($multiPath)['handoff_cleanup']['synthetic-multi-second']['upstream']) && isset($read($multiPath)['handoff_cleanup']['synthetic-multi-legacy']['upstream']), 'verification releases only each exact matched target'); foreach ([['status' => 'running'], ['status' => 'complete'], ['status' => 'cancelled'], ['ok' => false, 'status' => 'succeeded', 'error_code' => 'RUN_STATUS_HTTP_ERROR'], ['workflow_run_id' => 'synthetic-foreign-run', 'status' => 'stopped'], ['status' => null]] as $bad) { [$pending, $pendingPath] = $make('synthetic-pending-run'); $times = []; $result = $pending->reconcileClosedTasks(static function (array $task) use ($bad, &$times): array { $times[] = $GLOBALS['closedRecoveryNowNs']; return $bad + ['workflow_run_id' => $task['workflow_run_id']]; }); closedExpect(count($times) === 2 && $times[1] - $times[0] >= 250000000 && $result['status'] === 'pending' && $result['upstream_pending_task_count'] === 1, 'nonterminal/unknown/HTTP failure/mismatched result stays pending under two spaced attempts'); $before = $read($pendingPath); $pending->reconcileClosedTasks($never); closedExpect($read($pendingPath)['verification_attempts'] === 2 && $read($pendingPath)['verification_started_at'] === $before['verification_started_at'], 'repeat caller does not renew per-generation attempts/start'); } [$global, $globalPath] = $make('synthetic-global-primary'); $handoff($global, 'synthetic-global-second', 'synthetic-global-second-run'); $handoff($global, 'synthetic-global-third', 'synthetic-global-third-run'); $timeouts = []; $result = $global->reconcileClosedTasks(static function (array $task, int $timeoutMs) use (&$timeouts): array { $timeouts[] = [$task['task_id'], $timeoutMs]; if (count($timeouts) <= 2) { advanceClosedMs(700); return $task + ['status' => 'running']; } advanceClosedMs($timeoutMs); return $task + ['status' => 'succeeded']; }); closedExpect($result['status'] === 'budget_exhausted' && $result['budget_exhausted'] && $result['elapsed_ms'] === 2000 && $result['queried_task_count'] === 2 && $result['lookup_attempt_count'] === 3 && $result['upstream_pending_task_count'] === 3, 'one global two-second budget includes both primary attempts, spacing and the second target; third target gets no read'); closedExpect($timeouts[0][1] === 1000 && $timeouts[1][1] === 1000 && $timeouts[2][1] <= 350, 'later target inherits only remaining global I/O time'); $globalStart = $read($globalPath)['closed_verification_started_at']; advanceClosedMs(10000); closedExpect($global->reconcileClosedTasks($never)['budget_exhausted'] && $read($globalPath)['closed_verification_started_at'] === $globalStart, 'subsequent invocation never renews the global budget'); [$existing, $existingPath] = $make('synthetic-existing-budget'); $state = $read($existingPath); $state['verification_started_at'] = $GLOBALS['closedRecoveryNowNs'] / 1e9 - 1.8; $state['verification_attempts'] = 1; $write($existingPath, $state); $result = $existing->reconcileClosedTasks(static function (array $task, int $timeoutMs): array { closedExpect($timeoutMs > 0 && $timeoutMs <= 200, 'new caller can only narrow an existing generation budget'); return $task + ['status' => 'stopped']; }); closedExpect($result['status'] === 'complete' && $read($existingPath)['verification_attempts'] === 2 && $read($existingPath)['verification_started_at'] === $state['verification_started_at'], 'existing generation budget start is never extended'); foreach (['generation', 'workflow_run_id', 'task_id', 'profile', 'user', 'token'] as $changed) { [$stale, $stalePath] = $make('synthetic-stale-run'); $result = $stale->reconcileClosedTasks(static function (array $task) use ($stalePath, $read, $write, $changed): array { $state = $read($stalePath); if ($changed === 'generation') ++$state['upstream']['generation']; elseif ($changed === 'token') $state['verification_claim']['token'] = 'synthetic-replacement'; else $state['upstream'][$changed] = 'synthetic-replacement'; $write($stalePath, $state); return $task + ['status' => 'succeeded']; }); closedExpect($result['status'] === 'pending' && $result['lookup_attempt_count'] === 1 && !$stale->publicState()['upstream_terminal_confirmed'], 'stale ' . $changed . ' result cannot confirm a changed target'); } [$snapshot, $snapshotPath] = $make('synthetic-snapshot-primary'); $handoff($snapshot, 'synthetic-snapshot-second', 'synthetic-snapshot-second-run'); $result = $snapshot->reconcileClosedTasks(static function (array $task) use ($snapshotPath, $read, $write): array { closedExpect($task['workflow_run_id'] === 'synthetic-snapshot-primary', 'only captured original primary is queried'); $state = $read($snapshotPath); $state['handoff_cleanup']['synthetic-snapshot-second']['upstream']['workflow_run_id'] = 'synthetic-replacement-run'; $write($snapshotPath, $state); return $task + ['status' => 'stopped']; }); closedExpect($result['lookup_attempt_count'] === 1 && $result['upstream_pending_task_count'] === 1, 'a handoff changed after the snapshot is rejected before any query for the replacement'); [$late] = $make('synthetic-late-run'); $result = $late->reconcileClosedTasks(static function (array $task, int $timeoutMs): array { advanceClosedMs($timeoutMs + 1); return $task + ['status' => 'stopped']; }); closedExpect($result['status'] === 'pending' && !$late->publicState()['upstream_terminal_confirmed'], 'late callback cannot confirm despite exact identity'); [$throws, $throwsPath] = $make('synthetic-throw-run'); $result = $throws->reconcileClosedTasks(static function (): array { throw new \RuntimeException('synthetic-secret-error'); }); closedExpect($result['lookup_attempt_count'] === 2 && $result['status'] === 'pending' && !str_contains((string) file_get_contents($throwsPath), 'synthetic-secret-error'), 'throwing callbacks stay pending and discard exception prose'); [$reboot, $rebootPath] = $make('synthetic-reboot-run'); $state = $read($rebootPath); $state['closed_verification_started_at'] = $GLOBALS['closedRecoveryNowNs'] / 1e9 + 100; $write($rebootPath, $state); closedExpect($reboot->reconcileClosedTasks($never)['budget_exhausted'], 'global monotonic clock reset never renews budget'); $controller = new DiagnosisController(); $finishError = new \ReflectionMethod($controller, 'finishAssistantError'); foreach ([false, true] as $cancelled) { [$controlled] = $make('synthetic-controller-run', false); if ($cancelled) $controlled->signalCancel(); else $controlled->signalTimeout(); $GLOBALS['closedRecoveryAdapter'] = static function (array $task) use ($controlled): array { closedExpect($controlled->publicState()['transport_closed'], 'controller adapter runs after finish'); return $task + ['status' => 'stopped']; }; $events = []; $emit = static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; }; $callsBefore = $GLOBALS['closedRecoveryAdapterCalls']; $finishError->invoke($controller, $emit, $controlled, $cancelled); $last = $events[count($events) - 1]; closedExpect($GLOBALS['closedRecoveryAdapterCalls'] === $callsBefore + 1 && $last[0] === 'error' && $last[1]['code'] === ($cancelled ? 'AI_ASSISTANT_CANCELLED' : 'UPSTREAM_TIMEOUT') && $last[1]['upstream_pending_task_count'] === 0 && $last[1]['cancel_requested'] === $cancelled, 'controller keeps original error/cancel outcome and emits recovered safe state'); closedExpect($cancelled ? count($events) === 2 : count($events) === 1 && $last[1]['message'] === 'synthetic original error' && $last[1]['retryable'], 'ordinary deadline never emits cancellation progress or replaces original error message/meta'); } [$logFailure] = $make('synthetic-log-failure', false); $GLOBALS['closedRecoveryAdapter'] = static fn (array $task): array => $task + ['status' => 'failed']; $GLOBALS['closedRecoveryLogThrows'] = true; $events = []; $finishError->invoke($controller, static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; }, $logFailure, false); closedExpect($events[0][1]['code'] === 'UPSTREAM_TIMEOUT' && $events[0][1]['upstream_terminal_status'] === 'failed', 'recovery telemetry failure cannot mask original error or verified safe state'); $GLOBALS['closedRecoveryLogThrows'] = false; [$adapterFailure] = $make('synthetic-controller-adapter-error', false); $GLOBALS['closedRecoveryAdapter'] = static function (): array { throw new \RuntimeException('synthetic-private-adapter-error'); }; $events = []; $finishError->invoke($controller, static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; }, $adapterFailure, false); closedExpect($events[0][1]['code'] === 'UPSTREAM_TIMEOUT' && $events[0][1]['message'] === 'synthetic original error' && $events[0][1]['upstream_pending_task_count'] === 1, 'adapter exceptions preserve original final error and pending state'); [$controlFailure, $controlFailurePath] = $make('synthetic-controller-control-error', false); $GLOBALS['closedRecoveryAdapter'] = static function (array $task) use ($controlFailurePath): array { file_put_contents($controlFailurePath, 'synthetic private corrupt control'); return $task + ['status' => 'stopped']; }; $events = []; $finishError->invoke($controller, static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; }, $controlFailure, false); closedExpect($events[0][1] === ['code' => 'UPSTREAM_TIMEOUT', 'message' => 'synthetic original error', 'retryable' => true], 'failed recovery/state reads preserve exactly the original error payload without exception metadata'); $source = (string) file_get_contents(dirname(__DIR__) . '/app/adminapi/controller/tcm/DiagnosisController.php'); $catchStart = strpos($source, "Log::warning('diagnosis ai assistant sse failed '"); $catchEnd = strpos($source, 'public function cancelAiAssistant()', $catchStart); $catchSource = substr($source, $catchStart, $catchEnd - $catchStart); closedExpect(strpos($catchSource, "->finish('error')") < strpos($catchSource, '->reconcileClosedAssistant($control)') && substr_count($catchSource, '->reconcileClosedAssistant($control)') === 1 && strpos($catchSource, '->reconcileClosedAssistant($control)') < strpos($catchSource, "\$emit('error'"), 'unexpected exception finishes then invokes the same helper once before safe final error'); $runStart = strpos($source, 'private function runAssistantSse('); $runEnd = strpos($source, 'private function flushSseOutput()', $runStart); $run = substr($source, $runStart, $runEnd - $runStart); closedExpect(strpos($run, 'logAssistantGenerationDrain(') < strpos($run, "->finish('done')") && strpos($run, "->finish('done')") < strpos($run, '->reconcileClosedAssistant($control)') && strpos($run, '->reconcileClosedAssistant($control)') < strpos($run, "\$emit('done'") && str_contains($run, "\$timing->log('sse_done', ['duration_ms' => AssistantSseTiming::elapsedMs(\$sseStartedAt)"), 'normal final emits follow finish/recovery; prerecovery generation/drain and full SSE duration remain separate'); $callbackStart = strpos($source, '$options = [', strpos($source, 'public function aiAssistantStream()')); $callbackEnd = strpos($source, ' try {', $callbackStart); closedExpect(!str_contains(substr($source, $callbackStart, $callbackEnd - $callbackStart), 'reconcileClosedTasks') && !str_contains(substr($source, $callbackStart, $callbackEnd - $callbackStart), 'DifyWorkflowRunStatus'), 'cURL callbacks and new-attempt gate never perform status lookup'); foreach ($GLOBALS['closedRecoveryLogs'] as [$log]) { closedExpect(!str_contains($log, 'synthetic-') && !str_contains($log, 'workflow_run_id') && !str_contains($log, 'task_id'), 'recovery logs contain only safe aggregate fields'); } echo 'Assistant closed transport recovery: ' . $assertions . " assertions passed\n"; } finally { // Remove only this suite's explicitly created isolated directory and files. foreach (glob($directory . DIRECTORY_SEPARATOR . '*.json') ?: [] as $path) @unlink($path); @unlink($directory . DIRECTORY_SEPARATOR . '.gc.lock'); if (is_dir($directory)) @rmdir($directory); } }