This commit is contained in:
Your Name
2026-10-27 14:03:35 +08:00
parent 7986e4baf5
commit f9cbd42113
417 changed files with 71876 additions and 5344 deletions
@@ -4,6 +4,8 @@ declare(strict_types=1);
namespace app\common\service;
require_once __DIR__ . '/DifyUploadedFileReuse.php';
/**
* Turn an already-authorized attachment list into Dify local_file references.
*
@@ -27,7 +29,11 @@ final class DifyRequiredAttachmentUpload
* on_upload(array{index:int,total:int,status:string,reason?:string}), resolved_hosts
* (exact hostname => nonempty public IP list). Pins take precedence over the optional
* server-owned resolve_host callable, then native DNS for unpinned hosts only.
* No file ID cache survives this call, including between different Dify users.
* File IDs remain request-local unless the server explicitly enables
* reuse_uploaded_files (default false); reuse_ttl_seconds is capped at 120.
* Reuse always follows a fresh authorized source read and is scoped by HMAC
* to credential, endpoint, user, type, MIME, upload filename and exact bytes.
* on_timing receives numeric totals only, never transport IDs or cache tokens.
* skip_missing defaults to true: only confirmed absent local files or completed
* HTTP 404/410 downloads are skipped. Other source/transport errors still fail.
* Successful files and uploaded_indices have matching order; skipped metadata
@@ -51,6 +57,9 @@ final class DifyRequiredAttachmentUpload
$skipped = [];
$index = 0;
$total = 0;
$reuseTokens = []; $published = []; $reuseHits = 0; $reuseMisses = 0;
$timing = ['source_ms' => 0, 'upload_ms' => 0, 'ocr_ms' => 0, 'reuse_wait_ms' => 0];
$tempRoot = null; $succeeded = false;
$deadline = microtime(true) + max(0, $timeout);
try {
self::check($deadline, $shouldAbort);
@@ -103,6 +112,7 @@ final class DifyRequiredAttachmentUpload
self::notify($options, $index, $total, 'uploading');
}
if (!array_key_exists($cacheKey, $cache)) {
$sourceStarted = microtime(true);
$path = @tempnam($tempRoot, 'required-');
if ($path === false) self::fail('REQUIRED_FILES_PREPARE_FAILED');
$temporary[] = $path;
@@ -125,10 +135,33 @@ final class DifyRequiredAttachmentUpload
$extension = ['image/png' => 'png', 'image/jpeg' => 'jpg', 'application/pdf' => 'pdf',
'audio/wav' => 'wav', 'audio/mpeg' => 'mp3', 'text/plain' => 'txt'][$mime] ?? 'bin';
}
$cache[$cacheKey] = self::upload($endpoint, $apiKey, $user, $path, $mime,
'attachment-' . $index . '.' . $extension, $deadline, $shouldAbort, $options['ca_file'], $multi);
$filename = 'attachment-' . $index . '.' . $extension;
$token = null;
if (!empty($options['reuse_uploaded_files'])) {
$digest = @hash_file('sha256', $path);
if (!is_string($digest)) self::fail('REQUIRED_FILES_PREPARE_FAILED');
$token = DifyUploadedFileReuse::token($apiKey, $endpoint, $user, $item['type'], $mime, $filename, $digest);
}
$timing['source_ms'] += max(0, (int) round((microtime(true) - $sourceStarted) * 1000));
$upload = static function () use ($endpoint, $apiKey, $user, $path, $mime, $filename,
$deadline, $shouldAbort, $options, $multi, &$timing): string {
$uploadStarted = microtime(true);
try { return self::upload($endpoint, $apiKey, $user, $path, $mime, $filename,
$deadline, $shouldAbort, $options['ca_file'], $multi); }
finally { $timing['upload_ms'] += max(0, (int) round((microtime(true) - $uploadStarted) * 1000)); }
};
if ($token !== null) {
$reused = DifyUploadedFileReuse::resolve($tempRoot, $token, $options['reuse_ttl_seconds'],
$deadline, $shouldAbort, $upload);
$reuseTokens[$token] = true;
$timing['reuse_wait_ms'] += $reused['wait_ms'];
if ($reused['hit']) $reuseHits++; else $reuseMisses++;
if ($reused['generation'] !== null) $published[$token] = $reused['generation'];
$cache[$cacheKey] = $reused['id'];
} else $cache[$cacheKey] = $upload();
if ($item['type'] === 'image') $imagePaths[$cacheKey] = $path;
}
if (!$exists) $timing['source_ms'] += max(0, (int) round((microtime(true) - $sourceStarted) * 1000));
}
if ($cache[$cacheKey] === null) {
if (!$options['skip_missing']) self::fail(array_key_exists('path', $item)
@@ -146,11 +179,17 @@ final class DifyRequiredAttachmentUpload
self::check($deadline, $shouldAbort);
$textCheck = ['status' => 'disabled', 'images' => []];
if ($imageSources !== [] && !empty($options['ocr']['enabled'])) {
$textCheck = self::optionalImageText($imageSources, $tempRoot, $deadline, $shouldAbort, $options);
$ocrStarted = microtime(true);
try { $textCheck = self::optionalImageText($imageSources, $tempRoot, $deadline, $shouldAbort, $options); }
finally { $timing['ocr_ms'] += max(0, (int) round((microtime(true) - $ocrStarted) * 1000)); }
}
self::check($deadline, $shouldAbort);
return ['ok' => true, 'files' => $prepared, 'uploaded_indices' => $uploadedIndices,
$result = ['ok' => true, 'files' => $prepared, 'uploaded_indices' => $uploadedIndices,
'skipped' => $skipped, 'attachment_transport' => 'uploaded_files', 'image_text_check' => $textCheck];
if (!empty($options['reuse_uploaded_files'])) $result += ['reuse_tokens' => array_keys($reuseTokens),
'reuse_hit_count' => $reuseHits, 'reuse_miss_count' => $reuseMisses, 'timings' => $timing];
$succeeded = true;
return $result;
} catch (\Throwable $error) {
// Never expose cURL response bodies, URLs, local paths, credentials or exception text.
$code = $error instanceof \RuntimeException ? $error->getMessage() : 'REQUIRED_FILES_PREPARE_FAILED';
@@ -174,11 +213,29 @@ final class DifyRequiredAttachmentUpload
return ['ok' => false, 'error_code' => $code, 'error' => $messages[$code],
'failed_index' => $index, 'attachment_count' => $total, 'skipped' => $skipped];
} finally {
if (!$succeeded && $published !== [] && is_string($tempRoot)) {
DifyUploadedFileReuse::drop($tempRoot, array_keys($published), $published);
}
foreach ($temporary as $path) if (is_file($path)) @unlink($path);
if ($multi !== null && $multi !== false) curl_multi_close($multi);
if (is_callable($options['on_timing'] ?? null)) {
try { ($options['on_timing'])($timing); } catch (\Throwable $ignored) { /* Telemetry cannot change transport. */ }
}
}
}
/** Internal stale-file recovery. Tokens must never be included in public receipts. */
public static function invalidateReuse(array $tokens, array $options = []): void
{
try {
$options = self::options($options);
$publicRoot = realpath((string) ($options['public_root'] ?? dirname(__DIR__, 3) . '/public'));
$root = realpath((string) ($options['temp_root'] ?? dirname(__DIR__, 3) . '/runtime/dify-required-attachments'));
if ($publicRoot === false || $root === false || $root === $publicRoot || self::inside($publicRoot, $root)) return;
DifyUploadedFileReuse::drop($root, $tokens);
} catch (\Throwable $ignored) { /* A cache outage does not prevent a fresh upload. */ }
}
/** Spend only optional OCR time; the original image/model deadline is unchanged. */
private static function optionalImageText(array $images, string $root, float $deadline, ?callable $shouldAbort, array $options): array
{
@@ -221,6 +278,9 @@ final class DifyRequiredAttachmentUpload
catch (\Throwable $ignored) { $configured = []; }
}
$options = array_replace(is_array($configured) ? $configured : [], $options);
if (!array_key_exists('reuse_uploaded_files', $options)) $options['reuse_uploaded_files'] = false;
if (!is_bool($options['reuse_uploaded_files'])) self::fail('CONFIG_INVALID');
$options['reuse_ttl_seconds'] = max(1, min(120, (int) ($options['reuse_ttl_seconds'] ?? 120)));
if (!array_key_exists('skip_missing', $options)) $options['skip_missing'] = true;
if (!is_bool($options['skip_missing'])) self::fail('CONFIG_INVALID');
$caFile = $options['ca_file'] ?? '';
@@ -408,7 +468,7 @@ final class DifyRequiredAttachmentUpload
} elseif (is_callable($options['resolve_host'] ?? null)) {
$addresses = ($options['resolve_host'])($host);
} else {
// Validate every returned address, then pin the actual transfer to one.
// Validate every returned address, then pin the transfer to that set.
// Native PHP DNS is synchronous; its elapsed time still consumes this deadline.
$records = @dns_get_record($host, DNS_A | DNS_AAAA);
$addresses = [];
@@ -420,7 +480,9 @@ final class DifyRequiredAttachmentUpload
self::check($deadline, $shouldAbort);
if (!is_array($addresses) || $addresses === []) self::fail('REQUIRED_FILES_DOWNLOAD_FAILED');
foreach ($addresses as $address) if (!is_string($address) || !self::publicIp($address)) self::fail('REQUIRED_FILES_SOURCE_UNTRUSTED');
$ip = $addresses[0];
// Keep every validated address available to libcurl. A stale first DNS
// pin must not hide a reachable alternate for the same verified host.
$pinnedAddresses = array_map(static fn (string $ip): string => str_contains($ip, ':') ? '[' . $ip . ']' : $ip, $addresses);
$port = (int) ($parts['port'] ?? ($parts['scheme'] === 'https' ? 443 : 80));
$output = @fopen($path, 'wb');
if ($output === false) self::fail('REQUIRED_FILES_PREPARE_FAILED');
@@ -430,7 +492,7 @@ final class DifyRequiredAttachmentUpload
$writeError = '';
try {
$curlOptions = self::curlOptions($url, $deadline, $shouldAbort, $options['ca_file']);
$curlOptions[CURLOPT_RESOLVE] = [$host . ':' . $port . ':' . (str_contains($ip, ':') ? '[' . $ip . ']' : $ip)];
$curlOptions[CURLOPT_RESOLVE] = [$host . ':' . $port . ':' . implode(',', $pinnedAddresses)];
$curlOptions[CURLOPT_HEADERFUNCTION] = static function ($handle, string $line) use ($maxBytes, &$writeError): int {
if (preg_match('/^Content-Length:\s*(\d+)/i', $line, $match) && (float) $match[1] > $maxBytes) {
$writeError = 'REQUIRED_FILES_TOO_LARGE'; return 0;