Files
zyt/server/tests/DifyRequiredAttachmentUploadTest.php
T
2026-10-23 10:40:43 +08:00

310 lines
23 KiB
PHP

<?php
declare(strict_types=1);
namespace app\common\service {
/** Route synthetic public storage to loopback AFTER recording real security options. */
function curl_setopt_array($handle, array $options): bool
{
$original = $options;
$url = (string) ($options[CURLOPT_URL] ?? '');
if (str_starts_with($url, 'https://storage.synthetic.test/')) {
$options[CURLOPT_URL] = 'http://127.0.0.1:' . $GLOBALS['attachmentFixturePort'] . substr($url, strlen('https://storage.synthetic.test'));
unset($options[CURLOPT_RESOLVE]);
} elseif (!str_starts_with($url, 'http://127.0.0.1:' . $GLOBALS['attachmentFixturePort'] . '/')) {
throw new \RuntimeException('The synthetic test forbids external connections');
}
$GLOBALS['attachmentCurlOptions'][] = $original;
return \curl_setopt_array($handle, $options);
}
}
namespace {
// This same file is a loopback-only fixture router. It never loads application config.
if (PHP_SAPI === 'cli-server') {
$root = (string) getenv('DIFY_ATTACHMENT_FIXTURE_ROOT');
$path = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH);
$append = static function (array $entry) use ($root): void {
file_put_contents($root . '/requests.jsonl', json_encode($entry) . "\n", FILE_APPEND | LOCK_EX);
};
if ($_SERVER['REQUEST_METHOD'] === 'POST' && str_ends_with($path, '/files/upload')) {
$upload = $_FILES['file'] ?? [];
$temporary = $upload['tmp_name'] ?? '';
$id = bin2hex(random_bytes(16));
$append(['kind' => 'upload', 'user' => $_POST['user'] ?? '', 'name' => $upload['name'] ?? '',
'mime' => $upload['type'] ?? '', 'size' => $upload['size'] ?? 0,
'hash' => is_file($temporary) ? hash_file('sha256', $temporary) : '', 'id' => $id,
'authorization' => $_SERVER['HTTP_AUTHORIZATION'] ?? '', 'path' => $path]);
if (str_starts_with($path, '/slow-headers/')) usleep(1200000);
if (str_starts_with($path, '/budget/')) usleep(650000);
header('Content-Type: application/json');
if (str_starts_with($path, '/fail/')) { http_response_code(422); echo '{"error":"fixture-private-path-url-key-body"}'; return; }
if (str_starts_with($path, '/fail-second/')) {
$entries = array_map(static fn (string $line): array => json_decode($line, true), file($root . '/requests.jsonl', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES));
$uploads = array_filter($entries, static fn (array $entry): bool => ($entry['kind'] ?? '') === 'upload');
if (count($uploads) === 2) { http_response_code(422); echo '{"error":"fixture-private-path-url-key-body"}'; return; }
}
if (str_starts_with($path, '/bad-id/')) { http_response_code(201); echo '{"id":"invalid/id"}'; return; }
if (str_starts_with($path, '/large-response/')) { http_response_code(201); echo str_repeat('x', 300000); return; }
http_response_code(201);
if (str_starts_with($path, '/slow-body/')) { echo '{"id":"'; flush(); usleep(1200000); echo $id . '"}'; return; }
echo json_encode(['id' => $id]);
return;
}
$append(['kind' => 'download', 'path' => $path, 'authorization' => $_SERVER['HTTP_AUTHORIZATION'] ?? '']);
if ($path === '/redirect') { header('Location: http://169.254.169.254/latest/meta-data/'); http_response_code(302); echo 'redirect'; return; }
if ($path === '/oversize-header') { header('Content-Length: 2000000'); echo 'x'; return; }
if ($path === '/oversize-stream') { echo str_repeat('x', 30000); return; }
if ($path === '/slow-download') usleep(1200000);
if ($path === '/budget-download') usleep(650000);
header('Content-Type: image/png');
readfile($root . '/public/uploads/first.png');
return;
}
require dirname(__DIR__) . '/app/common/service/DifyRequiredAttachmentUpload.php';
use app\common\service\DifyRequiredAttachmentUpload;
function attachmentExpect(bool $ok, string $message): void
{
if (!$ok) throw new RuntimeException($message);
}
function attachmentError(array $result, string $code, string $message): void
{
attachmentExpect(empty($result['ok']) && ($result['error_code'] ?? '') === $code && !isset($result['files']),
$message . ' (' . ($result['error_code'] ?? 'no error') . ')');
attachmentExpect(!str_contains(json_encode($result), 'fixture-private-path-url-key-body'), 'upstream body never escapes');
}
function attachmentRemoveTree(string $path, string $workspace): void
{
$resolved = realpath($path); $root = realpath($workspace);
if ($resolved === false || $root === false || ($resolved !== $root && !str_starts_with(str_replace('\\', '/', $resolved), str_replace('\\', '/', $root) . '/'))) {
throw new RuntimeException('Refusing to clean outside synthetic test workspace');
}
foreach (new DirectoryIterator($resolved) as $item) {
if ($item->isDot()) continue;
if ($item->isDir() && !$item->isLink()) attachmentRemoveTree($item->getPathname(), $workspace);
else unlink($item->getPathname());
}
rmdir($resolved);
}
$root = sys_get_temp_dir() . '/dify-required-synthetic-' . bin2hex(random_bytes(8));
mkdir($root . '/public/uploads/nested', 0700, true);
mkdir($root . '/private', 0700);
mkdir($root . '/public/uploads-evil', 0700);
$image = imagecreatetruecolor(64, 64);
imagefill($image, 0, 0, imagecolorallocate($image, 50, 130, 180));
imagepng($image, $root . '/public/uploads/first.png');
imagefill($image, 0, 0, imagecolorallocate($image, 180, 130, 50));
imagepng($image, $root . '/public/uploads/nested/second.png');
imagedestroy($image);
file_put_contents($root . '/public/uploads/report.pdf', "%PDF-1.4\n1 0 obj<</Type/Catalog>>endobj\n%%EOF\n");
file_put_contents($root . '/public/uploads/audio.wav', 'RIFF' . pack('V', 38) . 'WAVEfmt ' . pack('VvvVVvv', 16, 1, 1, 8000, 16000, 2, 16) . 'data' . pack('V', 2) . "\0\0");
file_put_contents($root . '/public/uploads/large.txt', str_repeat('x', 30000));
file_put_contents($root . '/private/do-not-read.txt', 'synthetic-private-file');
file_put_contents($root . '/public/uploads-evil/escape.txt', 'synthetic-private-file');
$socket = stream_socket_server('tcp://127.0.0.1:0', $errno, $error);
attachmentExpect(is_resource($socket), 'allocate loopback fixture port');
$address = stream_socket_get_name($socket, false);
fclose($socket);
$port = (int) substr($address, strrpos($address, ':') + 1);
$attachmentFixturePort = $port;
$attachmentCurlOptions = [];
$process = proc_open([PHP_BINARY, '-S', '127.0.0.1:' . $port, __FILE__],
[['pipe', 'r'], ['file', $root . '/fixture-output.log', 'ab'], ['file', $root . '/fixture-error.log', 'ab']],
$pipes, $root, ['DIFY_ATTACHMENT_FIXTURE_ROOT' => $root,
'SystemRoot' => (string) getenv('SystemRoot'), 'TEMP' => (string) getenv('TEMP'), 'TMP' => (string) getenv('TMP')], ['bypass_shell' => true]);
try {
attachmentExpect(is_resource($process), 'start loopback fixture');
if (isset($pipes[0]) && is_resource($pipes[0])) fclose($pipes[0]);
$ready = false;
for ($attempt = 0; $attempt < 150; ++$attempt) {
$probe = @fsockopen('127.0.0.1', $port, $errno, $error, 0.02);
if (is_resource($probe)) { fclose($probe); $ready = true; break; }
usleep(20000);
}
attachmentExpect($ready, 'loopback fixture becomes ready');
$endpoint = 'http://127.0.0.1:' . $port . '/v1/chat-messages';
$options = ['local_origins' => ['https://site.synthetic.test', 'https://admin.synthetic.test'],
'storage_origins' => ['https://storage.synthetic.test'], 'public_root' => $root . '/public',
'temp_root' => $root . '/private/uploads', 'max_file_bytes' => 20000,
'resolve_host' => static fn (string $host): array => ['8.8.8.8', '2606:4700:4700::1111']];
$files = [
['type' => 'image', 'transfer_method' => 'remote_url', 'url' => 'https://site.synthetic.test/uploads/first.png'],
['type' => 'image', 'url' => 'https://admin.synthetic.test/uploads/nested/second.png'],
['type' => 'document', 'url' => 'https://site.synthetic.test/uploads/report.pdf'],
];
$requests = static function () use ($root): array {
if (!is_file($root . '/requests.jsonl')) return [];
return array_map(static fn (string $line): array => json_decode($line, true), file($root . '/requests.jsonl', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES));
};
$clear = static function () use ($root, &$attachmentCurlOptions): void {
file_put_contents($root . '/requests.jsonl', ''); $attachmentCurlOptions = [];
};
$clean = static function () use ($root): void {
attachmentExpect((glob($root . '/private/uploads/required-*') ?: []) === [], 'temporary files are removed after every terminal outcome');
};
$run = static function (array $input, array $extra = [], ?callable $abort = null, string $user = 'synthetic-user', ?string $target = null, int $timeout = 5) use ($endpoint, $options): array {
return DifyRequiredAttachmentUpload::prepare($target ?? $endpoint, 'fixture-only-key', $user, $input, $timeout, $abort, array_replace($options, $extra));
};
$clear(); $events = [];
$result = $run(array_merge($files, [
['type' => 'video', 'url' => 'https://untrusted.invalid/record.mp4'],
['type' => 'document', 'url' => 'https://untrusted.invalid/record.MOV'],
['type' => 'image', 'url' => 'https://untrusted.invalid/opaque', 'mime_type' => 'video/webm'],
]), ['on_upload' => static function (array $event) use (&$events): void { $events[] = $event; }]);
attachmentExpect(!empty($result['ok']) && count($result['files']) === 3 && $result['attachment_transport'] === 'uploaded_files', 'two images and PDF all become local_file references');
attachmentExpect(array_column($result['files'], 'type') === ['image', 'image', 'document']
&& array_unique(array_column($result['files'], 'transfer_method')) === ['local_file'], 'types and attachment order preserved');
attachmentExpect(count($requests()) === 3 && array_unique(array_column($requests(), 'user')) === ['synthetic-user'], 'every multipart upload has the exact chat user');
attachmentExpect(array_column($requests(), 'hash') === [hash_file('sha256', $root . '/public/uploads/first.png'),
hash_file('sha256', $root . '/public/uploads/nested/second.png'), hash_file('sha256', $root . '/public/uploads/report.pdf')], 'multipart contains complete binary bytes');
attachmentExpect(count($events) === 6 && array_column($events, 'status') === ['uploading', 'uploaded', 'uploading', 'uploaded', 'uploading', 'uploaded'], 'safe one-based progress follows each actual upload');
foreach ($events as $event) attachmentExpect(array_keys($event) === ['index', 'total', 'status'] && $event['total'] === 3, 'progress contains safe counts/status only');
foreach ($attachmentCurlOptions as $curlOptions) {
$temporary = $curlOptions[CURLOPT_POSTFIELDS]['file']->getFilename();
attachmentExpect(str_starts_with(str_replace('\\', '/', $temporary), str_replace('\\', '/', $root) . '/private/uploads/') && !is_file($temporary), 'CURLFile uses cleaned nonpublic snapshot');
attachmentExpect($curlOptions[CURLOPT_SSL_VERIFYPEER] === true && $curlOptions[CURLOPT_SSL_VERIFYHOST] === 2 && $curlOptions[CURLOPT_PROXY] === '', 'transport verifies TLS and ignores environment proxies');
}
$clean();
$clear();
$duplicates = $run([$files[0], $files[0]]);
attachmentExpect(!empty($duplicates['ok']) && count($duplicates['files']) === 2 && count($requests()) === 1
&& $duplicates['files'][0]['upload_file_id'] === $duplicates['files'][1]['upload_file_id'], 'same request reuses ID while preserving both references');
$otherUser = $run([$files[0]], [], null, 'another-synthetic-user');
attachmentExpect(!empty($otherUser['ok']) && count($requests()) === 2
&& $otherUser['files'][0]['upload_file_id'] !== $duplicates['files'][0]['upload_file_id']
&& $requests()[1]['user'] === 'another-synthetic-user', 'no upload IDs or cache cross user boundaries');
$sameUserNextRequest = $run([$files[0]]);
attachmentExpect(!empty($sameUserNextRequest['ok']) && count($requests()) === 3
&& $sameUserNextRequest['files'][0]['upload_file_id'] !== $duplicates['files'][0]['upload_file_id'], 'even same-user requests have no global ID cache');
$clean();
$clear();
$audio = $run([['type' => 'audio', 'url' => 'https://site.synthetic.test/uploads/audio.wav']]);
attachmentExpect(!empty($audio['ok']) && $audio['files'][0]['type'] === 'audio' && $requests()[0]['mime'] === 'audio/x-wav', 'audio bytes retain audio type');
$clean();
$clear();
$source = static fn (string $url): array => [['type' => 'document', 'url' => $url]];
foreach ([
'https://evil.synthetic.test/uploads/first.png',
'https://site.synthetic.test.evil.invalid/uploads/first.png',
'http://site.synthetic.test/uploads/first.png',
'https://site.synthetic.test:444/uploads/first.png',
] as $url) attachmentError($run($source($url)), 'REQUIRED_FILES_SOURCE_UNTRUSTED', 'exact origin prevents alias/scheme/port bypass');
foreach ([
'https://site.synthetic.test/uploads/../private/do-not-read.txt',
'https://site.synthetic.test/uploads/%2e%2e/private/do-not-read.txt',
'https://site.synthetic.test/uploads/%252e%252e/private/do-not-read.txt',
'https://site.synthetic.test/uploads/nested%2fsecond.png',
'https://site.synthetic.test/uploads/nested%5csecond.png',
'https://site.synthetic.test/uploads//first.png',
'https://site.synthetic.test/uploads/first.png::$DATA',
'https://site.synthetic.test/uploads/first.png.',
'https://site.synthetic.test/uploads-evil/escape.txt',
'https://site.synthetic.test/private/do-not-read.txt',
'https://site.synthetic.test@evil.invalid/uploads/first.png',
'https://site.synthetic.test/uploads/first.png#fragment',
'file:///private/do-not-read.txt',
'/uploads/first.png',
] as $url) attachmentError($run($source($url)), 'REQUIRED_FILES_INVALID', 'local path/URL bypass rejected before network');
attachmentError($run($source('https://site.synthetic.test/uploads/missing.png')), 'REQUIRED_FILES_SOURCE_UNAVAILABLE', 'missing local files do not trigger network fallback');
attachmentExpect($requests() === [], 'origin/path failures never upload or download');
if (@symlink($root . '/private/do-not-read.txt', $root . '/public/uploads/link.txt')) {
attachmentError($run($source('https://site.synthetic.test/uploads/link.txt')), 'REQUIRED_FILES_SOURCE_UNAVAILABLE', 'realpath rejects symlink escape');
}
$clean();
$clear();
$remote = [['type' => 'image', 'url' => 'https://storage.synthetic.test/download.png']];
$downloaded = $run($remote);
attachmentExpect(!empty($downloaded['ok']) && count($requests()) === 2 && $requests()[0]['kind'] === 'download'
&& $requests()[0]['authorization'] === '' && $requests()[1]['authorization'] === 'Bearer fixture-only-key', 'remote bytes upload without sharing Dify key with storage');
attachmentExpect($attachmentCurlOptions[0][CURLOPT_RESOLVE] === ['storage.synthetic.test:443:8.8.8.8']
&& $attachmentCurlOptions[0][CURLOPT_FOLLOWLOCATION] === false, 'validated DNS address is pinned and redirect following disabled');
$clear(); $dnsCalls = 0;
$rebound = $run($remote, ['resolve_host' => static function () use (&$dnsCalls): array { return [++$dnsCalls === 1 ? '8.8.8.8' : '127.0.0.1']; }]);
attachmentExpect(!empty($rebound['ok']) && $dnsCalls === 1
&& $attachmentCurlOptions[0][CURLOPT_RESOLVE] === ['storage.synthetic.test:443:8.8.8.8'], 'a rebound DNS answer cannot replace the pinned validated address');
$clean();
foreach (['127.0.0.1', '10.0.0.1', '172.16.0.1', '192.168.1.1', '169.254.169.254', '100.100.100.200',
'0.0.0.0', '224.0.0.1', '::1', 'fc00::1', 'fe80::1', '::ffff:127.0.0.1', '64:ff9b::7f00:1', '2002:7f00:1::', '2001:db8::1'] as $ip) {
$clear();
attachmentError($run($remote, ['resolve_host' => static fn (): array => [$ip]]), 'REQUIRED_FILES_SOURCE_UNTRUSTED', 'private/metadata/transition DNS addresses rejected');
attachmentExpect($requests() === [] && $attachmentCurlOptions === [], 'unsafe DNS never opens a connection');
}
$clear();
attachmentError($run($remote, ['resolve_host' => static fn (): array => ['8.8.8.8', '127.0.0.1']]), 'REQUIRED_FILES_SOURCE_UNTRUSTED', 'mixed public/private DNS is rejected in full');
attachmentError($run($remote, ['resolve_host' => static fn (): array => []]), 'REQUIRED_FILES_DOWNLOAD_FAILED', 'unresolvable storage fails explicitly');
foreach (['https://storage.synthetic.test/redirect', 'https://storage.synthetic.test/oversize-header', 'https://storage.synthetic.test/oversize-stream'] as $url) {
$clear();
attachmentError($run($source($url)), str_ends_with($url, '/redirect') ? 'REQUIRED_FILES_DOWNLOAD_FAILED' : 'REQUIRED_FILES_TOO_LARGE', 'redirect/oversize downloads fail without dropping mandatory file');
attachmentExpect(count($requests()) === 1 && $requests()[0]['kind'] === 'download', 'redirect is never followed and bad download is never uploaded');
$clean();
}
$clear();
attachmentError($run($source('https://site.synthetic.test/uploads/large.txt')), 'REQUIRED_FILES_TOO_LARGE', 'local file copy has the same byte limit');
attachmentExpect($requests() === [], 'oversize local file never uploads');
$clean();
foreach (['fail', 'bad-id', 'large-response'] as $mode) {
$clear(); $failedEvents = [];
$failed = $run($files, ['on_upload' => static function (array $event) use (&$failedEvents): void { $failedEvents[] = $event; }], null, 'synthetic-user',
'http://127.0.0.1:' . $port . '/' . $mode . '/v1/chat-messages');
attachmentError($failed, 'REQUIRED_FILES_UPLOAD_FAILED', 'provider upload error never returns a partial attachment list');
attachmentExpect(count($requests()) === 1 && $failedEvents[count($failedEvents) - 1]['status'] === 'failed', 'upload failure stops before a model request and marks safe progress failed');
$clean();
}
$clear();
$midway = $run($files, [], null, 'synthetic-user', 'http://127.0.0.1:' . $port . '/fail-second/v1/chat-messages');
attachmentError($midway, 'REQUIRED_FILES_UPLOAD_FAILED', 'failure after a successful upload still returns no partial files');
attachmentExpect(count($requests()) === 2 && $midway['failed_index'] === 2 && $midway['attachment_count'] === 3, 'failure identifies safe index and never skips ahead to remaining attachment');
$clean();
$clear();
attachmentError($run($files, [], static fn (): bool => true), 'AI_ASSISTANT_CANCELLED', 'cancellation before preparation opens no transport');
attachmentExpect($requests() === [], 'early cancellation has no network side effects');
$cancelDuringCopy = false;
attachmentError($run([$files[0]], ['on_upload' => static function () use (&$cancelDuringCopy): void { $cancelDuringCopy = true; }],
static function () use (&$cancelDuringCopy): bool { return $cancelDuringCopy; }), 'AI_ASSISTANT_CANCELLED', 'cancellation during local copy cleans its new temporary file');
attachmentExpect($requests() === [], 'cancelled local preparation never uploads');
$clean();
foreach (['slow-headers', 'slow-body'] as $mode) {
$clear(); $start = microtime(true);
$cancelled = $run([$files[0]], [], static fn (): bool => microtime(true) - $start > 0.15, 'synthetic-user',
'http://127.0.0.1:' . $port . '/' . $mode . '/v1/chat-messages');
attachmentError($cancelled, 'AI_ASSISTANT_CANCELLED', 'cancellation interrupts upload response ' . $mode);
attachmentExpect(microtime(true) - $start < 0.7, 'curl_multi observes cancellation while waiting for headers/body');
$clean(); usleep(1250000);
}
$clear(); $start = microtime(true);
attachmentError($run($source('https://storage.synthetic.test/slow-download'), [], static fn (): bool => microtime(true) - $start > 0.15),
'AI_ASSISTANT_CANCELLED', 'cancellation interrupts remote download header wait');
attachmentExpect(microtime(true) - $start < 0.7, 'download cancellation does not wait for its full timeout');
$clean(); usleep(1250000);
$clear(); $start = microtime(true);
$budget = $run(array_slice($files, 0, 2), [], null, 'synthetic-user', 'http://127.0.0.1:' . $port . '/budget/v1/chat-messages', 1);
attachmentError($budget, 'REQUIRED_FILES_TIMEOUT', 'multiple uploads share one deadline');
attachmentExpect(microtime(true) - $start < 1.4 && count($requests()) === 2, 'second upload receives only remaining request budget');
$clean(); usleep(700000);
$clear(); $start = microtime(true);
$combinedBudget = $run($source('https://storage.synthetic.test/budget-download'), [], null, 'synthetic-user',
'http://127.0.0.1:' . $port . '/budget/v1/chat-messages', 1);
attachmentError($combinedBudget, 'REQUIRED_FILES_TIMEOUT', 'download and subsequent upload consume the same deadline');
attachmentExpect(microtime(true) - $start < 1.4 && count($requests()) === 2, 'upload does not receive a fresh budget after downloading');
$clean(); usleep(700000);
$clear();
attachmentError($run([$files[0]], ['temp_root' => $root . '/public/unsafe-temp']), 'CONFIG_INVALID', 'temporary files cannot be created in public');
attachmentExpect($requests() === [], 'unsafe temp configuration opens no connection');
attachmentError($run([['type' => 'image', 'transfer_method' => 'local_file', 'url' => $files[0]['url']]]), 'REQUIRED_FILES_INVALID', 'callers cannot inject upload IDs from other requests');
$clean();
echo "Dify required attachment upload: binary multipart, all sources, origins/paths/DNS/pinning/redirects/limits, isolation, cancellation/deadline, cleanup: OK\n";
} finally {
if (is_resource($process)) { proc_terminate($process); proc_close($process); }
attachmentRemoveTree($root, $root);
}
}