334 lines
23 KiB
PHP
334 lines
23 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// Actual control/controller, isolated synthetic controls and injected lookups only.
|
|
// No framework bootstrap, runtime config, network, model, database or real control.
|
|
namespace app\adminapi\service {
|
|
function hrtime(bool $asNumber = false) {
|
|
$now = $GLOBALS['closedRecoveryNowNs'];
|
|
return $asNumber ? $now : [intdiv($now, 1000000000), $now % 1000000000];
|
|
}
|
|
function usleep(int $microseconds): void { $GLOBALS['closedRecoveryNowNs'] += $microseconds * 1000; }
|
|
}
|
|
|
|
namespace app\adminapi\controller {
|
|
class BaseAdminController {}
|
|
}
|
|
|
|
namespace app\adminapi\logic\tcm {
|
|
class DiagnosisAiLogic {
|
|
public static function getAssistantErrorCode(): string { return 'UPSTREAM_TIMEOUT'; }
|
|
public static function getError(): string { return 'synthetic original error'; }
|
|
public static function getAssistantErrorMeta(): array { return ['retryable' => true]; }
|
|
}
|
|
}
|
|
|
|
namespace app\common\service {
|
|
class DifyWorkflowRunStatus {
|
|
public static function lookup(array $identity, int $timeoutMs): array {
|
|
++$GLOBALS['closedRecoveryAdapterCalls'];
|
|
return ($GLOBALS['closedRecoveryAdapter'])($identity, $timeoutMs);
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace think\facade {
|
|
class Log {
|
|
public static function write(string $message, string $level = ''): void {
|
|
$GLOBALS['closedRecoveryLogs'][] = [$message, $level];
|
|
if (!empty($GLOBALS['closedRecoveryLogThrows'])) throw new \RuntimeException('synthetic private logging failure');
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/app/adminapi/service/AssistantRequestControl.php';
|
|
require dirname(__DIR__) . '/app/adminapi/controller/tcm/DiagnosisController.php';
|
|
|
|
use app\adminapi\service\AssistantRequestControl;
|
|
use app\adminapi\controller\tcm\DiagnosisController;
|
|
|
|
$GLOBALS['closedRecoveryNowNs'] = 100000000000;
|
|
$GLOBALS['closedRecoveryAdapterCalls'] = 0;
|
|
$GLOBALS['closedRecoveryLogs'] = [];
|
|
$assertions = 0;
|
|
function closedExpect(bool $ok, string $message): void {
|
|
++$GLOBALS['assertions'];
|
|
if (!$ok) throw new \RuntimeException($message);
|
|
}
|
|
function advanceClosedMs(int $milliseconds): void { $GLOBALS['closedRecoveryNowNs'] += $milliseconds * 1000000; }
|
|
function safeClosedResult(array $result): void {
|
|
$expected = ['status', 'transport_closed', 'eligible_task_count', 'queried_task_count',
|
|
'lookup_attempt_count', 'confirmed_task_count', 'upstream_pending_task_count', 'budget_exhausted', 'elapsed_ms'];
|
|
closedExpect(array_keys($result) === $expected, 'closed caller exposes only its exact aggregate schema');
|
|
closedExpect(in_array($result['status'], ['transport_open', 'complete', 'pending', 'budget_exhausted'], true), 'aggregate status is allowlisted');
|
|
foreach ($result as $key => $value) {
|
|
if ($key === 'status') continue;
|
|
closedExpect(in_array($key, ['transport_closed', 'budget_exhausted'], true)
|
|
? is_bool($value) : is_int($value) && $value >= 0, 'aggregate ' . $key . ' has a safe bounded type');
|
|
}
|
|
}
|
|
|
|
$directory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'zyt-closed-recovery-' . bin2hex(random_bytes(8));
|
|
$number = 0;
|
|
$make = static function (string $run = 'synthetic-run', bool $closed = true, bool $bind = true) use ($directory, &$number): array {
|
|
advanceClosedMs(10000);
|
|
$uuid = sprintf('29345678-1234-4234-8234-%012d', ++$number);
|
|
$control = new AssistantRequestControl(17, 23, $uuid, $directory);
|
|
closedExpect($control->begin(), 'isolated request begins');
|
|
$task = 'synthetic-task-' . $number;
|
|
if ($bind) $control->bindTask('qwen', 'synthetic-user', $task, $run);
|
|
if ($closed) $control->finish('error');
|
|
return [$control, $directory . DIRECTORY_SEPARATOR . hash('sha256', '17:23:' . $uuid) . '.json', $task];
|
|
};
|
|
$read = static fn (string $path): array => json_decode((string) file_get_contents($path), true, 512, JSON_THROW_ON_ERROR);
|
|
$write = static function (string $path, array $state): void { file_put_contents($path, json_encode($state, JSON_THROW_ON_ERROR)); };
|
|
$handoff = static function (AssistantRequestControl $control, string $task, string $run): void {
|
|
try {
|
|
$control->bindTask('qwen', 'synthetic-user', $task, $run);
|
|
closedExpect(false, 'handoff must stay blocked');
|
|
} catch (\RuntimeException $e) {
|
|
closedExpect($e->getMessage() === 'UPSTREAM_TASK_HANDOFF_BLOCKED', 'unexpected own pair is retained as a blocked handoff');
|
|
}
|
|
};
|
|
$never = static function (): array { closedExpect(false, 'lookup must not run'); return []; };
|
|
|
|
try {
|
|
[$open, $openPath] = $make('synthetic-open-run', false);
|
|
$result = $open->reconcileClosedTasks($never);
|
|
safeClosedResult($result);
|
|
closedExpect($result['status'] === 'transport_open' && $result['lookup_attempt_count'] === 0
|
|
&& $result['upstream_pending_task_count'] === 1 && !isset($read($openPath)['closed_verification_started_at']),
|
|
'open cancel-worker race neither queries nor starts the global recovery budget');
|
|
advanceClosedMs(10000);
|
|
$open->finish('error');
|
|
$result = $open->reconcileClosedTasks(static fn (array $task): array => $task + ['status' => 'stopped']);
|
|
closedExpect($result['status'] === 'complete' && $result['confirmed_task_count'] === 1,
|
|
'stream owner recovers only after finish even if a prior caller observed open transport');
|
|
|
|
[$empty] = $make('synthetic-empty-run', true, false);
|
|
closedExpect($empty->reconcileClosedTasks($never)['status'] === 'complete', 'empty closed transport has no GET');
|
|
[$legacy, $legacyPath] = $make('');
|
|
$legacyResult = $legacy->reconcileClosedTasks($never);
|
|
closedExpect($legacyResult['status'] === 'pending' && $legacyResult['eligible_task_count'] === 0
|
|
&& $legacyResult['upstream_pending_task_count'] === 1 && !isset($read($legacyPath)['closed_verification_started_at']),
|
|
'legacy missing run remains pending without guessed identity or budget consumption');
|
|
|
|
foreach (['stopped', 'succeeded', 'failed'] as $status) {
|
|
[$exact, $exactPath] = $make('synthetic-exact-' . $status);
|
|
$exact->signalTimeout();
|
|
$before = $read($exactPath);
|
|
$result = $exact->reconcileClosedTasks(static function (array $task, int $timeoutMs) use ($exact, $status): array {
|
|
closedExpect(count($task) === 5 && $timeoutMs > 0 && $timeoutMs <= 1000, 'lookup receives only exact private pair and bounded I/O deadline');
|
|
closedExpect($exact->publicState()['transport_closed'], 'lookup runs outside the file and GC locks');
|
|
closedExpect($exact->reconcileClosedTasks(static function (): array {
|
|
closedExpect(false, 'nested caller cannot duplicate claimed I/O'); return [];
|
|
})['lookup_attempt_count'] === 0, 'same request verifier lease also covers a nested caller');
|
|
return $task + ['status' => $status, 'body' => 'synthetic-private-body'];
|
|
});
|
|
safeClosedResult($result);
|
|
$public = $exact->publicState();
|
|
closedExpect($result['status'] === 'complete' && $result['queried_task_count'] === 1 && $result['lookup_attempt_count'] === 1
|
|
&& $public['status'] === 'error' && $public['timeout_reached'] && !$public['cancel_requested'] && !$exact->shouldAbort()
|
|
&& $public['upstream_terminal_status'] === $status, 'terminal recovery never turns ordinary timeout into user cancel or report success');
|
|
closedExpect(!isset($read($exactPath)['upstream']) && !str_contains((string) file_get_contents($exactPath), 'synthetic-private-body'),
|
|
'only verified identity is released; lookup body is discarded');
|
|
closedExpect($read($exactPath)['verification_attempts'] === 1
|
|
&& ($read($exactPath)['stop_attempts'] ?? null) === ($before['stop_attempts'] ?? null), 'recovery does not issue or reset stop attempts');
|
|
closedExpect(!$exact->canStartAttempt() && !$exact->begin(), 'recovery cannot restart or replay a closed request generation');
|
|
closedExpect($exact->reconcileClosedTasks($never)['lookup_attempt_count'] === 0, 'confirmed pair never causes another GET');
|
|
}
|
|
|
|
[$known, , $knownTask] = $make('synthetic-known', false);
|
|
$handoff($known, 'synthetic-known-handoff', 'synthetic-known-handoff-run');
|
|
$known->markTerminal($knownTask, 'succeeded', 'synthetic-known');
|
|
$known->markTerminal('synthetic-known-handoff', 'failed', 'synthetic-known-handoff-run');
|
|
$known->finish('done');
|
|
closedExpect($known->reconcileClosedTasks($never)['status'] === 'complete', 'all stream-confirmed targets skip independent lookup');
|
|
|
|
[$multi, $multiPath, $multiTask] = $make('synthetic-multi-primary');
|
|
$handoff($multi, 'synthetic-multi-second', 'synthetic-multi-second-run');
|
|
$handoff($multi, 'synthetic-multi-legacy', '');
|
|
$seen = [];
|
|
$result = $multi->reconcileClosedTasks(static function (array $task) use (&$seen): array {
|
|
$seen[] = $task['task_id'];
|
|
return $task + ['status' => 'failed'];
|
|
});
|
|
closedExpect($seen === [$multiTask, 'synthetic-multi-second'] && $result['confirmed_task_count'] === 2
|
|
&& $result['upstream_pending_task_count'] === 1 && $result['status'] === 'pending',
|
|
'each own valid pair is verified separately and one missing-run handoff stays pending');
|
|
closedExpect(!isset($read($multiPath)['upstream']) && !isset($read($multiPath)['handoff_cleanup']['synthetic-multi-second']['upstream'])
|
|
&& isset($read($multiPath)['handoff_cleanup']['synthetic-multi-legacy']['upstream']), 'verification releases only each exact matched target');
|
|
|
|
foreach ([['status' => 'running'], ['status' => 'complete'], ['status' => 'cancelled'],
|
|
['ok' => false, 'status' => 'succeeded', 'error_code' => 'RUN_STATUS_HTTP_ERROR'],
|
|
['workflow_run_id' => 'synthetic-foreign-run', 'status' => 'stopped'], ['status' => null]] as $bad) {
|
|
[$pending, $pendingPath] = $make('synthetic-pending-run');
|
|
$times = [];
|
|
$result = $pending->reconcileClosedTasks(static function (array $task) use ($bad, &$times): array {
|
|
$times[] = $GLOBALS['closedRecoveryNowNs'];
|
|
return $bad + ['workflow_run_id' => $task['workflow_run_id']];
|
|
});
|
|
closedExpect(count($times) === 2 && $times[1] - $times[0] >= 250000000
|
|
&& $result['status'] === 'pending' && $result['upstream_pending_task_count'] === 1,
|
|
'nonterminal/unknown/HTTP failure/mismatched result stays pending under two spaced attempts');
|
|
$before = $read($pendingPath);
|
|
$pending->reconcileClosedTasks($never);
|
|
closedExpect($read($pendingPath)['verification_attempts'] === 2
|
|
&& $read($pendingPath)['verification_started_at'] === $before['verification_started_at'], 'repeat caller does not renew per-generation attempts/start');
|
|
}
|
|
|
|
[$global, $globalPath] = $make('synthetic-global-primary');
|
|
$handoff($global, 'synthetic-global-second', 'synthetic-global-second-run');
|
|
$handoff($global, 'synthetic-global-third', 'synthetic-global-third-run');
|
|
$timeouts = [];
|
|
$result = $global->reconcileClosedTasks(static function (array $task, int $timeoutMs) use (&$timeouts): array {
|
|
$timeouts[] = [$task['task_id'], $timeoutMs];
|
|
if (count($timeouts) <= 2) { advanceClosedMs(700); return $task + ['status' => 'running']; }
|
|
advanceClosedMs($timeoutMs);
|
|
return $task + ['status' => 'succeeded'];
|
|
});
|
|
closedExpect($result['status'] === 'budget_exhausted' && $result['budget_exhausted'] && $result['elapsed_ms'] === 2000
|
|
&& $result['queried_task_count'] === 2 && $result['lookup_attempt_count'] === 3 && $result['upstream_pending_task_count'] === 3,
|
|
'one global two-second budget includes both primary attempts, spacing and the second target; third target gets no read');
|
|
closedExpect($timeouts[0][1] === 1000 && $timeouts[1][1] === 1000 && $timeouts[2][1] <= 350,
|
|
'later target inherits only remaining global I/O time');
|
|
$globalStart = $read($globalPath)['closed_verification_started_at'];
|
|
advanceClosedMs(10000);
|
|
closedExpect($global->reconcileClosedTasks($never)['budget_exhausted']
|
|
&& $read($globalPath)['closed_verification_started_at'] === $globalStart, 'subsequent invocation never renews the global budget');
|
|
|
|
[$existing, $existingPath] = $make('synthetic-existing-budget');
|
|
$state = $read($existingPath);
|
|
$state['verification_started_at'] = $GLOBALS['closedRecoveryNowNs'] / 1e9 - 1.8;
|
|
$state['verification_attempts'] = 1;
|
|
$write($existingPath, $state);
|
|
$result = $existing->reconcileClosedTasks(static function (array $task, int $timeoutMs): array {
|
|
closedExpect($timeoutMs > 0 && $timeoutMs <= 200, 'new caller can only narrow an existing generation budget');
|
|
return $task + ['status' => 'stopped'];
|
|
});
|
|
closedExpect($result['status'] === 'complete' && $read($existingPath)['verification_attempts'] === 2
|
|
&& $read($existingPath)['verification_started_at'] === $state['verification_started_at'], 'existing generation budget start is never extended');
|
|
|
|
foreach (['generation', 'workflow_run_id', 'task_id', 'profile', 'user', 'token'] as $changed) {
|
|
[$stale, $stalePath] = $make('synthetic-stale-run');
|
|
$result = $stale->reconcileClosedTasks(static function (array $task) use ($stalePath, $read, $write, $changed): array {
|
|
$state = $read($stalePath);
|
|
if ($changed === 'generation') ++$state['upstream']['generation'];
|
|
elseif ($changed === 'token') $state['verification_claim']['token'] = 'synthetic-replacement';
|
|
else $state['upstream'][$changed] = 'synthetic-replacement';
|
|
$write($stalePath, $state);
|
|
return $task + ['status' => 'succeeded'];
|
|
});
|
|
closedExpect($result['status'] === 'pending' && $result['lookup_attempt_count'] === 1
|
|
&& !$stale->publicState()['upstream_terminal_confirmed'], 'stale ' . $changed . ' result cannot confirm a changed target');
|
|
}
|
|
|
|
[$snapshot, $snapshotPath] = $make('synthetic-snapshot-primary');
|
|
$handoff($snapshot, 'synthetic-snapshot-second', 'synthetic-snapshot-second-run');
|
|
$result = $snapshot->reconcileClosedTasks(static function (array $task) use ($snapshotPath, $read, $write): array {
|
|
closedExpect($task['workflow_run_id'] === 'synthetic-snapshot-primary', 'only captured original primary is queried');
|
|
$state = $read($snapshotPath);
|
|
$state['handoff_cleanup']['synthetic-snapshot-second']['upstream']['workflow_run_id'] = 'synthetic-replacement-run';
|
|
$write($snapshotPath, $state);
|
|
return $task + ['status' => 'stopped'];
|
|
});
|
|
closedExpect($result['lookup_attempt_count'] === 1 && $result['upstream_pending_task_count'] === 1,
|
|
'a handoff changed after the snapshot is rejected before any query for the replacement');
|
|
|
|
[$late] = $make('synthetic-late-run');
|
|
$result = $late->reconcileClosedTasks(static function (array $task, int $timeoutMs): array {
|
|
advanceClosedMs($timeoutMs + 1); return $task + ['status' => 'stopped'];
|
|
});
|
|
closedExpect($result['status'] === 'pending' && !$late->publicState()['upstream_terminal_confirmed'], 'late callback cannot confirm despite exact identity');
|
|
[$throws, $throwsPath] = $make('synthetic-throw-run');
|
|
$result = $throws->reconcileClosedTasks(static function (): array { throw new \RuntimeException('synthetic-secret-error'); });
|
|
closedExpect($result['lookup_attempt_count'] === 2 && $result['status'] === 'pending'
|
|
&& !str_contains((string) file_get_contents($throwsPath), 'synthetic-secret-error'), 'throwing callbacks stay pending and discard exception prose');
|
|
|
|
[$reboot, $rebootPath] = $make('synthetic-reboot-run');
|
|
$state = $read($rebootPath); $state['closed_verification_started_at'] = $GLOBALS['closedRecoveryNowNs'] / 1e9 + 100;
|
|
$write($rebootPath, $state);
|
|
closedExpect($reboot->reconcileClosedTasks($never)['budget_exhausted'], 'global monotonic clock reset never renews budget');
|
|
|
|
$controller = new DiagnosisController();
|
|
$finishError = new \ReflectionMethod($controller, 'finishAssistantError');
|
|
foreach ([false, true] as $cancelled) {
|
|
[$controlled] = $make('synthetic-controller-run', false);
|
|
if ($cancelled) $controlled->signalCancel(); else $controlled->signalTimeout();
|
|
$GLOBALS['closedRecoveryAdapter'] = static function (array $task) use ($controlled): array {
|
|
closedExpect($controlled->publicState()['transport_closed'], 'controller adapter runs after finish');
|
|
return $task + ['status' => 'stopped'];
|
|
};
|
|
$events = [];
|
|
$emit = static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; };
|
|
$callsBefore = $GLOBALS['closedRecoveryAdapterCalls'];
|
|
$finishError->invoke($controller, $emit, $controlled, $cancelled);
|
|
$last = $events[count($events) - 1];
|
|
closedExpect($GLOBALS['closedRecoveryAdapterCalls'] === $callsBefore + 1 && $last[0] === 'error'
|
|
&& $last[1]['code'] === ($cancelled ? 'AI_ASSISTANT_CANCELLED' : 'UPSTREAM_TIMEOUT')
|
|
&& $last[1]['upstream_pending_task_count'] === 0 && $last[1]['cancel_requested'] === $cancelled,
|
|
'controller keeps original error/cancel outcome and emits recovered safe state');
|
|
closedExpect($cancelled ? count($events) === 2 : count($events) === 1
|
|
&& $last[1]['message'] === 'synthetic original error' && $last[1]['retryable'],
|
|
'ordinary deadline never emits cancellation progress or replaces original error message/meta');
|
|
}
|
|
[$logFailure] = $make('synthetic-log-failure', false);
|
|
$GLOBALS['closedRecoveryAdapter'] = static fn (array $task): array => $task + ['status' => 'failed'];
|
|
$GLOBALS['closedRecoveryLogThrows'] = true;
|
|
$events = [];
|
|
$finishError->invoke($controller, static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; }, $logFailure, false);
|
|
closedExpect($events[0][1]['code'] === 'UPSTREAM_TIMEOUT' && $events[0][1]['upstream_terminal_status'] === 'failed', 'recovery telemetry failure cannot mask original error or verified safe state');
|
|
$GLOBALS['closedRecoveryLogThrows'] = false;
|
|
|
|
[$adapterFailure] = $make('synthetic-controller-adapter-error', false);
|
|
$GLOBALS['closedRecoveryAdapter'] = static function (): array { throw new \RuntimeException('synthetic-private-adapter-error'); };
|
|
$events = [];
|
|
$finishError->invoke($controller, static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; }, $adapterFailure, false);
|
|
closedExpect($events[0][1]['code'] === 'UPSTREAM_TIMEOUT' && $events[0][1]['message'] === 'synthetic original error'
|
|
&& $events[0][1]['upstream_pending_task_count'] === 1, 'adapter exceptions preserve original final error and pending state');
|
|
|
|
[$controlFailure, $controlFailurePath] = $make('synthetic-controller-control-error', false);
|
|
$GLOBALS['closedRecoveryAdapter'] = static function (array $task) use ($controlFailurePath): array {
|
|
file_put_contents($controlFailurePath, 'synthetic private corrupt control');
|
|
return $task + ['status' => 'stopped'];
|
|
};
|
|
$events = [];
|
|
$finishError->invoke($controller, static function (string $event, array $payload) use (&$events): bool { $events[] = [$event, $payload]; return true; }, $controlFailure, false);
|
|
closedExpect($events[0][1] === ['code' => 'UPSTREAM_TIMEOUT', 'message' => 'synthetic original error', 'retryable' => true],
|
|
'failed recovery/state reads preserve exactly the original error payload without exception metadata');
|
|
|
|
$source = (string) file_get_contents(dirname(__DIR__) . '/app/adminapi/controller/tcm/DiagnosisController.php');
|
|
$catchStart = strpos($source, "Log::warning('diagnosis ai assistant sse failed '");
|
|
$catchEnd = strpos($source, 'public function cancelAiAssistant()', $catchStart);
|
|
$catchSource = substr($source, $catchStart, $catchEnd - $catchStart);
|
|
closedExpect(strpos($catchSource, "->finish('error')") < strpos($catchSource, '->reconcileClosedAssistant($control)')
|
|
&& substr_count($catchSource, '->reconcileClosedAssistant($control)') === 1
|
|
&& strpos($catchSource, '->reconcileClosedAssistant($control)') < strpos($catchSource, "\$emit('error'"),
|
|
'unexpected exception finishes then invokes the same helper once before safe final error');
|
|
$runStart = strpos($source, 'private function runAssistantSse(');
|
|
$runEnd = strpos($source, 'private function flushSseOutput()', $runStart);
|
|
$run = substr($source, $runStart, $runEnd - $runStart);
|
|
closedExpect(strpos($run, 'logAssistantGenerationDrain(') < strpos($run, "->finish('done')")
|
|
&& strpos($run, "->finish('done')") < strpos($run, '->reconcileClosedAssistant($control)')
|
|
&& strpos($run, '->reconcileClosedAssistant($control)') < strpos($run, "\$emit('done'")
|
|
&& str_contains($run, "\$timing->log('sse_done', ['duration_ms' => AssistantSseTiming::elapsedMs(\$sseStartedAt)"),
|
|
'normal final emits follow finish/recovery; prerecovery generation/drain and full SSE duration remain separate');
|
|
$callbackStart = strpos($source, '$options = [', strpos($source, 'public function aiAssistantStream()'));
|
|
$callbackEnd = strpos($source, ' try {', $callbackStart);
|
|
closedExpect(!str_contains(substr($source, $callbackStart, $callbackEnd - $callbackStart), 'reconcileClosedTasks')
|
|
&& !str_contains(substr($source, $callbackStart, $callbackEnd - $callbackStart), 'DifyWorkflowRunStatus'), 'cURL callbacks and new-attempt gate never perform status lookup');
|
|
foreach ($GLOBALS['closedRecoveryLogs'] as [$log]) {
|
|
closedExpect(!str_contains($log, 'synthetic-') && !str_contains($log, 'workflow_run_id') && !str_contains($log, 'task_id'), 'recovery logs contain only safe aggregate fields');
|
|
}
|
|
|
|
echo 'Assistant closed transport recovery: ' . $assertions . " assertions passed\n";
|
|
} finally {
|
|
// Remove only this suite's explicitly created isolated directory and files.
|
|
foreach (glob($directory . DIRECTORY_SEPARATOR . '*.json') ?: [] as $path) @unlink($path);
|
|
@unlink($directory . DIRECTORY_SEPARATOR . '.gc.lock');
|
|
if (is_dir($directory)) @rmdir($directory);
|
|
}
|
|
}
|