ZYT/zyt
3
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

修复

Browse Source
This commit is contained in:
Your Name
2026-10-27 14:03:35 +08:00
parent 7986e4baf5
commit f9cbd42113
417 changed files with 71876 additions and 5344 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/tests/DifyUploadedFileReuseTest.php
+275
View File
@@ -0,0 +1,275 @@
<?php
declare(strict_types=1);
// Real uploader/cache, local synthetic files, namespaced fake cURL and fake OCR.
// Child PHP processes exercise real OS flock. No sockets, DNS, model or database.
namespace app\common\service {
require __DIR__ . '/fixtures/CancellableCurlMultiStub.php';
function is_link(string $path): bool { return ($GLOBALS['reuseSyntheticLink'] ?? null) === $path || \is_link($path); }
function curl_init(): \stdClass { return (object) ['errno' => CURLE_OK, 'code' => 201]; }
function curl_setopt_array(\stdClass $handle, array $options): bool { $handle->options = $options; return true; }
function curl_exec(\stdClass $handle): bool {
$options = $handle->options;
if (!str_contains($options[CURLOPT_URL], '.synthetic.test/') || !str_ends_with($options[CURLOPT_URL], '/files/upload')
|| !is_array($options[CURLOPT_POSTFIELDS])) throw new \RuntimeException('external I/O forbidden');
$fixture = $GLOBALS['reuseFixture'];
$counter = fopen($fixture . '/counter', 'c+b'); flock($counter, LOCK_EX);
$count = (int) stream_get_contents($counter) + 1;
rewind($counter); ftruncate($counter, 0); fwrite($counter, (string) $count); fflush($counter);
flock($counter, LOCK_UN); fclose($counter);
$bytes = file_get_contents($options[CURLOPT_POSTFIELDS]['file']->getFilename());
$GLOBALS['reuseUploadDigests'][] = hash('sha256', $bytes);
if (($GLOBALS['reuseMode'] ?? '') === 'crash') {
file_put_contents($fixture . '/owner-ready', 'ready');
$stop = microtime(true) + 4;
while (!is_file($fixture . '/release') && microtime(true) < $stop) usleep(10000);
exit(17);
}
if (($GLOBALS['reuseMode'] ?? '') === 'fail') file_put_contents($fixture . '/owner-ready', 'ready');
usleep(($GLOBALS['reuseDelay'] ?? 0) * 1000);
if (($GLOBALS['reuseMode'] ?? '') === 'fail') { $handle->errno = CURLE_RECV_ERROR; return false; }
($options[CURLOPT_WRITEFUNCTION])($handle, json_encode(['id' => 'synthetic-upload-' . $count]));
if (($GLOBALS['reuseMode'] ?? '') === 'cancel-after-upload') $GLOBALS['reuseCancelled'] = true;
return true;
}
function curl_errno(\stdClass $handle): int { return $handle->errno; }
function curl_getinfo(\stdClass $handle, int $option): int { return $handle->code; }
function curl_close(\stdClass $handle): void {}
class ClinicalImageTextService {
public static function read(array $images, string $root, float $deadline, ?callable $abort, array $settings): array {
$rows = [];
foreach ($images as $image) {
$path = realpath($image['path']);
if ($path === false || !str_starts_with($path, realpath($root) . DIRECTORY_SEPARATOR)) throw new \RuntimeException('nonprivate OCR source');
$GLOBALS['reuseOcrDigests'][] = hash_file('sha256', $path);
$rows[] = ['index' => $image['index'], 'status' => 'processed', 'lines' => [
['text' => 'SYNTHETIC 6.70 %; -0.12 mmol/L', 'confidence' => 0.91, 'box' => [[0, 0], [1, 0], [1, 1], [0, 1]]]]];
}
return ['status' => 'completed', 'images' => $rows];
}
}
}
namespace {
require dirname(__DIR__) . '/app/common/service/DifyRequiredAttachmentUpload.php';
use app\common\service\DifyRequiredAttachmentUpload as Uploader;
use app\common\service\DifyUploadedFileReuse as Reuse;
function reuseExpect(bool $condition, string $message): void {
if (!$condition) throw new RuntimeException($message);
$GLOBALS['reuseAssertions'] = ($GLOBALS['reuseAssertions'] ?? 0) + 1;
}
function reuseOptions(string $root): array {
return ['public_root' => $root . '/public', 'temp_root' => $root . '/private',
'local_origin' => 'https://app.synthetic.test', 'reuse_uploaded_files' => true, 'reuse_ttl_seconds' => 120,
'ocr' => ['enabled' => true, 'timeout_seconds' => 10]];
}
function reuseFiles(int $count = 1): array {
return array_map(static fn (int $index): array => ['type' => 'image',
'url' => 'https://app.synthetic.test/uploads/image-' . $index . '.png'], range(1, $count));
}
function reuseRun(string $root, array $changes = [], array $files = [], string $user = 'synthetic-user',
string $key = 'SYNTHETIC_KEY_ONLY', string $endpoint = 'https://dify.synthetic.test/v1/chat-messages', ?callable $abort = null): array {
$GLOBALS['reuseFixture'] = $root;
$GLOBALS['reuseOcrDigests'] = []; $GLOBALS['reuseUploadDigests'] = [];
return Uploader::prepare($endpoint, $key, $user, $files ?: reuseFiles(), 20, $abort,
array_replace(reuseOptions($root), $changes));
}
function reuseCount(string $root): int { return (int) @file_get_contents($root . '/counter'); }
function reuseRoot(string $base, string $name): string {
$root = $base . '/' . $name;
mkdir($root . '/public/uploads', 0700, true); mkdir($root . '/private', 0700);
$png = base64_decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=');
foreach (range(1, 10) as $index) file_put_contents($root . '/public/uploads/image-' . $index . '.png', $png);
return $root;
}
function reuseSlot(string $root, string $token): string {
return realpath($root . '/private') . DIRECTORY_SEPARATOR . '.dify-upload-reuse-v1' . DIRECTORY_SEPARATOR
. sprintf('%03d.cache', hexdec(substr($token, 0, 4)) % 256);
}
function reuseWorker(string $root, string $name, string $mode = '', int $count = 1, int $delay = 100): array {
$out = $root . '/' . $name . '.out'; $error = $root . '/' . $name . '.err';
$process = proc_open([PHP_BINARY, __FILE__, '--worker', $root, $mode, (string) $count, (string) $delay],
[0 => ['file', 'NUL', 'r'], 1 => ['file', $out, 'w'], 2 => ['file', $error, 'w']], $pipes,
null, null, ['bypass_shell' => true]);
if (!is_resource($process)) throw new RuntimeException('worker spawn failed');
return [$process, $out, $error];
}
function reuseJoin(array $worker, int $expectedExit = 0): ?array {
[$process, $out, $error] = $worker;
$stop = microtime(true) + 8;
do { $state = proc_get_status($process); if (!$state['running']) break; usleep(10000); } while (microtime(true) < $stop);
if ($state['running']) { proc_terminate($process, 9); throw new RuntimeException('worker exceeded finite fixture deadline'); }
$code = $state['exitcode']; proc_close($process);
reuseExpect($code === $expectedExit && trim((string) file_get_contents($error)) === '', 'child exited cleanly without private diagnostics');
return $expectedExit === 0 ? json_decode((string) file_get_contents($out), true, 32, JSON_THROW_ON_ERROR) : null;
}
function reuseWaitFile(string $file): void {
$stop = microtime(true) + 4;
while (!is_file($file) && microtime(true) < $stop) usleep(10000);
reuseExpect(is_file($file), 'owner reached deterministic lock-held fixture boundary');
}
if (($argv[1] ?? '') === '--worker') {
$root = $argv[2]; $GLOBALS['reuseMode'] = $argv[3]; $GLOBALS['reuseDelay'] = (int) $argv[5];
$result = reuseRun($root, [], reuseFiles((int) $argv[4]));
echo json_encode(['result' => $result, 'ocr_count' => count($GLOBALS['reuseOcrDigests']),
'ocr_digests' => $GLOBALS['reuseOcrDigests'], 'upload_digests' => $GLOBALS['reuseUploadDigests']]);
exit;
}
$base = sys_get_temp_dir() . '/dify-reuse-offline-' . bin2hex(random_bytes(8)); mkdir($base, 0700);
$GLOBALS['reuseMode'] = ''; $GLOBALS['reuseDelay'] = 0;
try {
$root = reuseRoot($base, 'defaults');
foreach ([1, 2] as $_) $legacy = reuseRun($root, ['reuse_uploaded_files' => false]);
reuseExpect(reuseCount($root) === 2 && !isset($legacy['reuse_tokens'])
&& !is_dir($root . '/private/.dify-upload-reuse-v1'), 'disabled direct callers retain fresh uploads and no cache');
$root = reuseRoot($base, 'hit'); $timings = []; $progress = [];
$changes = ['on_timing' => static function (array $event) use (&$timings): void { $timings[] = $event; },
'on_upload' => static function (array $event) use (&$progress): void { $progress[] = $event; }];
$first = reuseRun($root, $changes); $second = reuseRun($root, $changes);
reuseExpect($first['ok'] && $second['ok'] && reuseCount($root) === 1
&& $first['reuse_miss_count'] === 1 && $second['reuse_hit_count'] === 1, 'fresh authorized snapshot reuses completed upload descriptor');
reuseExpect($first['files'] === $second['files'] && $second['uploaded_indices'] === [1]
&& count($GLOBALS['reuseOcrDigests']) === 1 && $GLOBALS['reuseUploadDigests'] === [], 'hit preserves original attachment and reruns OCR from private original snapshot');
reuseExpect($GLOBALS['reuseOcrDigests'][0] === hash_file('sha256', $root . '/public/uploads/image-1.png')
&& str_contains($second['image_text_check']['images'][0]['lines'][0]['text'], '6.70'), 'hit numeric evidence stays tied to exact current source bytes');
foreach ($timings as $event) reuseExpect(array_keys($event) === ['source_ms', 'upload_ms', 'ocr_ms', 'reuse_wait_ms']
&& count(array_filter($event, static fn ($value): bool => is_int($value) && $value >= 0)) === 4, 'timing has only bounded numeric categories');
reuseExpect(!str_contains(json_encode($progress), $first['reuse_tokens'][0])
&& !str_contains(json_encode($progress), 'synthetic-upload-'), 'progress contains no opaque cache keys or transport IDs');
$recordPath = reuseSlot($root, $first['reuse_tokens'][0]); $raw = file_get_contents($recordPath);
foreach (['SYNTHETIC_KEY_ONLY', 'synthetic-user', 'image-1', 'synthetic.test', '6.70', 'public', 'iVBOR'] as $secret) {
reuseExpect(!str_contains($raw, $secret), 'cache stores only opaque descriptor metadata');
}
reuseExpect(filesize($recordPath) <= 1024 && count(glob(dirname($recordPath) . '/*')) <= 256, 'private fixed slot storage is bounded');
if (PHP_OS_FAMILY !== 'Windows') reuseExpect((fileperms($recordPath) & 0777) === 0600
&& (fileperms(dirname($recordPath)) & 0777) === 0700, 'POSIX cache storage is private');
Uploader::invalidateReuse($first['reuse_tokens'], reuseOptions($root));
$invalidated = reuseRun($root);
reuseExpect(reuseCount($root) === 2 && $invalidated['reuse_miss_count'] === 1, 'public internal invalidation forces one new upload');
$record = json_decode(file_get_contents($recordPath), true); $record['created'] = microtime(true) - 121; $record['expires'] = microtime(true) - 1;
file_put_contents($recordPath, json_encode($record)); reuseRun($root);
reuseExpect(reuseCount($root) === 3, 'expired record is a fresh-upload miss');
foreach (['{broken', json_encode(['v' => 1, 'id' => 'BAD ID']), str_repeat('x', 1025)] as $corrupt) {
$before = reuseCount($root); file_put_contents($recordPath, $corrupt); $result = reuseRun($root);
reuseExpect($result['ok'] && reuseCount($root) === $before + 1, 'malformed, invalid-ID, oversized records fail safely to fresh upload');
}
$record = json_decode(file_get_contents($recordPath), true);
reuseExpect($record['expires'] - $record['created'] <= 120.001, 'retention never exceeds 120 seconds');
foreach (['wrong-token', 'invalid-id', 'extended-expiry'] as $bad) {
$record = json_decode(file_get_contents($recordPath), true);
if ($bad === 'wrong-token') $record['token'] = str_repeat('f', 64);
elseif ($bad === 'invalid-id') $record['id'] = 'INVALID ID';
else $record['expires'] = $record['created'] + 121;
file_put_contents($recordPath, json_encode($record)); $before = reuseCount($root); reuseRun($root);
reuseExpect(reuseCount($root) === $before + 1, 'wrong scope, invalid ID and excessive expiry never become hits');
}
$telemetryFailure = reuseRun($root, ['on_timing' => static function (): void { throw new RuntimeException('synthetic timing observer'); }]);
reuseExpect($telemetryFailure['ok'] && $telemetryFailure['reuse_hit_count'] === 1, 'numeric timing observer failure cannot alter transport');
$basis = ['key', 'https://a.synthetic.test/files/upload', 'user', 'image', 'image/png', 'attachment-1.png', str_repeat('a', 64)];
$token = Reuse::token(...$basis);
foreach (array_keys($basis) as $position) {
$variant = $basis; $variant[$position] .= '-different';
reuseExpect(Reuse::token(...$variant) !== $token, 'credential, endpoint, user, type, MIME, filename, bytes each isolate a key');
}
reuseExpect(preg_match('/^[a-f0-9]{64}$/D', $token) === 1, 'only opaque HMAC tokens are returned');
foreach ([['user' => 'other-user'], ['key' => 'OTHER_SYNTHETIC_KEY'], ['endpoint' => 'https://other.synthetic.test/v1/chat-messages']] as $separation) {
$before = reuseCount($root);
$result = reuseRun($root, [], [], $separation['user'] ?? 'synthetic-user', $separation['key'] ?? 'SYNTHETIC_KEY_ONLY',
$separation['endpoint'] ?? 'https://dify.synthetic.test/v1/chat-messages');
reuseExpect($result['ok'] && reuseCount($root) === $before + 1, 'uploader does not reuse another user/app/endpoint descriptor');
}
$before = reuseCount($root); file_put_contents($root . '/public/uploads/image-1.png', 'different exact synthetic bytes'); reuseRun($root);
reuseExpect(reuseCount($root) === $before + 1, 'same URL with changed bytes misses current snapshot identity');
unlink($root . '/public/uploads/image-1.png'); $before = reuseCount($root); $missing = reuseRun($root);
reuseExpect($missing['ok'] && $missing['files'] === [] && count($missing['skipped']) === 1
&& reuseCount($root) === $before && $missing['reuse_tokens'] === [], 'now-missing source cannot consult prior cache');
$forbidden = reuseRun($root, [], [['type' => 'image', 'url' => 'https://forbidden.synthetic.test/uploads/image-1.png']]);
reuseExpect(!$forbidden['ok'] && $forbidden['error_code'] === 'REQUIRED_FILES_SOURCE_UNTRUSTED', 'fresh whole-manifest authorization precedes cache lookup');
$root = reuseRoot($base, 'outage'); file_put_contents($root . '/private/.dify-upload-reuse-v1', 'synthetic obstruction');
$a = reuseRun($root); $b = reuseRun($root);
reuseExpect($a['ok'] && $b['ok'] && reuseCount($root) === 2, 'cache directory outage never blocks fresh original uploads');
$root = reuseRoot($base, 'symlink'); $seed = reuseRun($root); $path = reuseSlot($root, $seed['reuse_tokens'][0]);
$GLOBALS['reuseSyntheticLink'] = $path; $before = reuseCount($root); $linked = reuseRun($root);
reuseExpect($linked['ok'] && reuseCount($root) === $before + 1, 'symlink slot is rejected before reading or writing cached metadata');
$GLOBALS['reuseSyntheticLink'] = dirname($path); $before = reuseCount($root); reuseRun($root);
reuseExpect(reuseCount($root) === $before + 1, 'symlink cache directory falls back without following it');
unset($GLOBALS['reuseSyntheticLink']);
$root = reuseRoot($base, 'rollback'); $cancelled = false;
$failed = reuseRun($root, ['on_upload' => static function (array $event): void {
if ($event['status'] === 'uploaded') throw new RuntimeException('synthetic observer failure');
}]);
$next = reuseRun($root);
reuseExpect(!$failed['ok'] && $next['ok'] && reuseCount($root) === 2, 'later preparation failure rolls back its published positive entry');
Uploader::invalidateReuse($next['reuse_tokens'], reuseOptions($root));
$GLOBALS['reuseMode'] = 'cancel-after-upload'; $GLOBALS['reuseCancelled'] = false;
$cancel = reuseRun($root, [], [], 'synthetic-user', 'SYNTHETIC_KEY_ONLY', 'https://dify.synthetic.test/v1/chat-messages',
static fn (): bool => $GLOBALS['reuseCancelled']);
$GLOBALS['reuseMode'] = ''; $recovery = reuseRun($root);
reuseExpect(!$cancel['ok'] && $cancel['error_code'] === 'AI_ASSISTANT_CANCELLED'
&& $recovery['ok'] && $recovery['reuse_miss_count'] === 1, 'cancelled upload owner leaves no positive entry and recovery uploads afresh');
$root = reuseRoot($base, 'concurrent-one');
$one = reuseWorker($root, 'one', '', 1, 350); $two = reuseWorker($root, 'two', '', 1, 350);
$a = reuseJoin($one); $b = reuseJoin($two);
reuseExpect($a['result']['ok'] && $b['result']['ok'] && reuseCount($root) === 1
&& $a['result']['files'] === $b['result']['files'], 'two PHP processes single-flight exactly one successful remote upload');
reuseExpect($a['ocr_count'] === 1 && $b['ocr_count'] === 1 && $a['ocr_digests'] === $b['ocr_digests']
&& max($a['result']['timings']['reuse_wait_ms'], $b['result']['timings']['reuse_wait_ms']) >= 200,
'waiter measures bounded wait and both requests rerun OCR on fresh identical bytes');
$root = reuseRoot($base, 'concurrent-ten');
$one = reuseWorker($root, 'one', '', 10, 25); $two = reuseWorker($root, 'two', '', 10, 25);
$a = reuseJoin($one); $b = reuseJoin($two);
reuseExpect($a['result']['ok'] && $b['result']['ok'] && reuseCount($root) === 10
&& $a['result']['files'] === $b['result']['files'] && $a['result']['uploaded_indices'] === range(1, 10)
&& $b['result']['uploaded_indices'] === range(1, 10), 'C2 ten-source manifests preserve all original positions with ten uploads total');
reuseExpect($a['ocr_count'] === 10 && $b['ocr_count'] === 10
&& count($a['result']['image_text_check']['images']) === 10 && count($b['result']['image_text_check']['images']) === 10,
'all twenty logical image sources still receive independent fresh original OCR');
$root = reuseRoot($base, 'owner-fail');
$one = reuseWorker($root, 'failed', 'fail', 1, 250); reuseWaitFile($root . '/owner-ready');
$two = reuseWorker($root, 'next', '', 1, 250);
$a = reuseJoin($one); $b = reuseJoin($two);
reuseExpect(!$a['result']['ok'] && $b['result']['ok'] && reuseCount($root) === 2, 'failed owner releases lock; waiter performs its own validated upload');
$root = reuseRoot($base, 'owner-crash');
$crashed = reuseWorker($root, 'crashed', 'crash'); reuseWaitFile($root . '/owner-ready');
$GLOBALS['reuseFixture'] = $root;
$start = microtime(true);
$cancel = reuseRun($root, [], [], 'synthetic-user', 'SYNTHETIC_KEY_ONLY', 'https://dify.synthetic.test/v1/chat-messages',
static fn (): bool => microtime(true) - $start > 0.08);
reuseExpect(!$cancel['ok'] && $cancel['error_code'] === 'AI_ASSISTANT_CANCELLED'
&& microtime(true) - $start < 0.5 && reuseCount($root) === 1, 'cancelled waiter leaves live owner untouched without duplicate upload');
$digest = hash_file('sha256', $root . '/public/uploads/image-1.png');
$token = Reuse::token('SYNTHETIC_KEY_ONLY', 'https://dify.synthetic.test/v1/files/upload', 'synthetic-user', 'image', 'image/png', 'attachment-1.png', $digest);
$start = microtime(true); $deadlineCaught = false;
try { Reuse::resolve(realpath($root . '/private'), $token, 120, microtime(true) + 0.08, null,
static function (): string { throw new RuntimeException('waiter must not upload'); }); }
catch (RuntimeException $error) { $deadlineCaught = $error->getMessage() === 'REQUIRED_FILES_TIMEOUT'; }
reuseExpect($deadlineCaught && microtime(true) - $start < 0.5, 'nonblocking flock wait respects absolute caller deadline');
$next = reuseWorker($root, 'recovered'); file_put_contents($root . '/release', 'release');
reuseJoin($crashed, 17); $recovered = reuseJoin($next);
reuseExpect($recovered['result']['ok'] && reuseCount($root) === 2, 'OS releases crashed owner lock; next process uploads and commits safely');
echo 'Dify uploaded-file reuse: ' . $GLOBALS['reuseAssertions'] . " offline assertions OK; fresh snapshots, scope, TTL/corruption, privacy, invalidation, rollback, C2 single-flight, owner failure/crash, cancellation/deadline, original OCR. Network/model/DB calls=0.\n";
} finally {
// Remove only this freshly generated, verified private fixture root.
$resolved = realpath($base); $temporary = realpath(sys_get_temp_dir());
if ($resolved !== false && $temporary !== false && str_starts_with(strtolower($resolved), strtolower($temporary . DIRECTORY_SEPARATOR))
&& str_starts_with(basename($resolved), 'dify-reuse-offline-')) {
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($resolved, FilesystemIterator::SKIP_DOTS), RecursiveIteratorIterator::CHILD_FIRST);
foreach ($iterator as $item) { if ($item->isDir() && !$item->isLink()) @rmdir($item->getPathname()); else @unlink($item->getPathname()); }
@rmdir($resolved);
}
}
}