ZYT/zyt
2
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

feat: configure audio providers and add local sample acceptance

Browse Source
This commit is contained in:
long
2026-10-07 18:15:32 +08:00
parent 70be2fc70f
commit fea33285e8
21 changed files with 1103 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/common/service/followupaudio/FollowupAudioDify.php
+81 -73
View File
@@ -4,7 +4,7 @@ declare(strict_types=1);
namespace app\common\service\followupaudio;
/** Dedicated, fail-closed local_file audio transport; intentionally does not use DifyChatService. */
/** Fail-closed audio transport; explicit Dify/OpenAI driver, never text or provider fallback. */
final class FollowupAudioDify
{
private array $settings;
@@ -24,7 +24,7 @@ final class FollowupAudioDify
if (empty($this->settings['enabled'])) {
throw new FollowupAudioException('FEATURE_DISABLED');
}
if (empty($this->settings['audio_verified']) || !in_array((string) ($task['model_key'] ?? ''), (array) ($this->settings['verified_profiles'] ?? []), true)) {
if (!FollowupAudioProviderConfig::verified((string) ($task['model_key'] ?? ''), $this->settings, $this->provider)) {
throw new FollowupAudioException('AUDIO_NOT_VERIFIED');
}
if ((int) ($task['upstream_started_at'] ?? 0) > 0) {
@@ -53,25 +53,28 @@ final class FollowupAudioDify
'upload_id' => 'synthetic', 'sha256' => $fixture['sha256'],
'recorded_at' => '2026-09-29 10:00:00', 'model_key' => $profile,
'duration_seconds' => (float) ($fixture['duration_seconds'] ?? 0),
], $heartbeat);
], $heartbeat, true);
}
/** Returns status and an irreversible app-identity fingerprint, never configuration values/credentials. */
public function configurationStatus(string $profile): array
{
try {
[$base, $key] = $this->resolveProfile($profile);
return ['configured' => true, 'profile' => $profile, 'code' => 'OK',
'application_fingerprint' => hash('sha256', $base . "\0" . $key)];
} catch (FollowupAudioException $e) {
return ['configured' => false, 'profile' => in_array($profile, ['qwen', 'openai'], true) ? $profile : 'invalid',
'code' => $e->errorCode];
}
return FollowupAudioProviderConfig::status($profile, $this->settings, $this->provider);
}
private function analyzeFile(string $path, array $task, callable $heartbeat): array
private function analyzeFile(string $path, array $task, callable $heartbeat, bool $synthetic = false): array
{
[$base, $key, $timeout] = $this->resolveProfile((string) ($task['model_key'] ?? ''));
[$base, $key, $timeout, $provider] = $this->resolveProfile((string) ($task['model_key'] ?? ''));
if (!str_starts_with($base, 'https://') && (!$synthetic || empty($this->settings['allow_insecure_synthetic']))) {
throw new FollowupAudioException('HTTPS_REQUIRED');
}
if (!$synthetic) {
$snapshot = json_decode((string) ($task['upstream_ids_json'] ?? ''), true);
if (!is_array($snapshot) || !is_string($snapshot['provider_fingerprint'] ?? null)
|| !hash_equals($provider['fingerprint'], $snapshot['provider_fingerprint'])) {
throw new FollowupAudioException('PROVIDER_CONFIGURATION_CHANGED');
}
}
$audio = $this->inspectAudio($path, (string) ($task['sha256'] ?? ''));
if (isset($task['duration_seconds']) && abs((float) $task['duration_seconds'] - $audio['duration']) > 1.0) {
throw new FollowupAudioException('AUDIO_INVALID');
@@ -81,36 +84,52 @@ final class FollowupAudioDify
if (!$date || $date->format('Y-m-d H:i:s') !== $recordedAt) {
throw new FollowupAudioException('RECORDED_AT_INVALID');
}
$processing = $this->prepareAudio($audio, $heartbeat);
$processing = $this->prepareAudio($audio, $heartbeat, $provider['driver'] === 'openai_audio');
try {
$query = $this->prompt($recordedAt, $audio['duration']);
$ids = ['request_id' => 'fa-' . bin2hex(random_bytes(16))];
$ids = ['request_id' => 'fa-' . bin2hex(random_bytes(16)), 'provider_fingerprint' => $provider['fingerprint']];
$user = 'followup-audio-' . substr(hash('sha256', (string) ($task['id'] ?? '') . ':' . $ids['request_id']), 0, 32);
// Persist intent BEFORE any network side effect, including upload. A crashed worker cannot resend.
$this->checkpoint($heartbeat, ['stage' => 'uploading', 'upstream_started_at' => time(),
'upstream_ids_json' => json_encode($ids, JSON_THROW_ON_ERROR)], false);
$uploaded = $this->request([
'url' => $base . '/files/upload', 'api_key' => $key, 'timeout' => $timeout, 'request_ids' => $ids,
'multipart' => ['user' => $user, 'file' => new \CURLFile($processing['path'], $processing['mime'], 'followup-audio.' . $processing['extension'])],
], $heartbeat);
$fileId = $this->identifier($uploaded['id'] ?? null);
if ($fileId === '') {
throw new FollowupAudioException('UPSTREAM_UPLOAD_INVALID', true);
if ($provider['driver'] === 'openai_audio') {
// Build and validate the complete request before persisting the single billable intent.
$payload = $this->openAiPayload($processing, $query, $provider['model']);
$this->checkpoint($heartbeat, ['stage' => 'analyzing', 'upstream_started_at' => time(),
'upstream_ids_json' => json_encode($ids, JSON_THROW_ON_ERROR)], false);
$response = $this->request(['url' => $base . '/chat/completions', 'api_key' => $key,
'timeout' => $timeout, 'json' => $payload, 'request_ids' => $ids], $heartbeat);
$response['message_id'] = $this->identifier($response['id'] ?? null);
$choices = is_array($response['choices'] ?? null) ? $response['choices'] : [];
$choice = is_array($choices[0] ?? null) ? $choices[0] : [];
$choice['message'] = is_array($choice['message'] ?? null) ? $choice['message'] : [];
$response['answer'] = count($choices) === 1
&& ($choice['finish_reason'] ?? '') === 'stop' && empty($choice['message']['refusal'])
&& empty($choice['message']['tool_calls']) ? ($choice['message']['content'] ?? null) : null;
} else {
// Persist intent BEFORE any network side effect, including upload. A crashed worker cannot resend.
$this->checkpoint($heartbeat, ['stage' => 'uploading', 'upstream_started_at' => time(),
'upstream_ids_json' => json_encode($ids, JSON_THROW_ON_ERROR)], false);
$uploaded = $this->request([
'url' => $base . '/files/upload', 'api_key' => $key, 'timeout' => $timeout, 'request_ids' => $ids,
'multipart' => ['user' => $user, 'file' => new \CURLFile($processing['path'], $processing['mime'], 'followup-audio.' . $processing['extension'])],
], $heartbeat);
$fileId = $this->identifier($uploaded['id'] ?? null);
if ($fileId === '') {
throw new FollowupAudioException('UPSTREAM_UPLOAD_INVALID', true);
}
// Preserve the observed file ID even if the upstream mislabeled/rejected its media type.
$this->checkpoint($heartbeat, ['stage' => 'analyzing', 'upstream_file_id' => $fileId], true);
if (isset($uploaded['mime_type']) && (!is_string($uploaded['mime_type']) || !str_starts_with($uploaded['mime_type'], 'audio/'))) {
throw new FollowupAudioException('UPSTREAM_AUDIO_REJECTED');
}
$payload = [
'inputs' => new \stdClass(),
'query' => $query,
'response_mode' => 'blocking', 'user' => $user, 'auto_generate_name' => false,
'files' => [['type' => 'audio', 'transfer_method' => 'local_file', 'upload_file_id' => $fileId]],
];
self::assertAudioPayload($payload, $fileId);
$response = $this->request(['url' => $base . '/chat-messages', 'api_key' => $key,
'timeout' => $timeout, 'json' => $payload, 'request_ids' => $ids], $heartbeat);
}
// Preserve the observed file ID even if the upstream mislabeled/rejected its media type.
$this->checkpoint($heartbeat, ['stage' => 'analyzing', 'upstream_file_id' => $fileId], true);
if (isset($uploaded['mime_type']) && (!is_string($uploaded['mime_type']) || !str_starts_with($uploaded['mime_type'], 'audio/'))) {
throw new FollowupAudioException('UPSTREAM_AUDIO_REJECTED');
}
$payload = [
'inputs' => new \stdClass(),
'query' => $query,
'response_mode' => 'blocking', 'user' => $user, 'auto_generate_name' => false,
'files' => [['type' => 'audio', 'transfer_method' => 'local_file', 'upload_file_id' => $fileId]],
];
self::assertAudioPayload($payload, $fileId);
$response = $this->request(['url' => $base . '/chat-messages', 'api_key' => $key,
'timeout' => $timeout, 'json' => $payload, 'request_ids' => $ids], $heartbeat);
foreach (['task_id', 'message_id', 'conversation_id', 'upstream_request_id'] as $name) {
$id = $this->identifier($response[$name] ?? null);
if ($id !== '') { $ids[$name] = $id; }
@@ -156,38 +175,25 @@ final class FollowupAudioDify
private function resolveProfile(string $profile): array
{
if (!in_array($profile, ['qwen', 'openai'], true)) {
throw new FollowupAudioException('INVALID_PROFILE');
}
$base = rtrim((string) ($this->provider['base_url'] ?? ''), '/');
$key = (string) ($this->provider['models'][$profile]['api_key'] ?? '');
if ($base === '' || trim($key) === '') {
throw new FollowupAudioException('CONFIG_MISSING');
}
$parts = parse_url($base);
if (!is_array($parts) || !in_array($parts['scheme'] ?? '', ['https', 'http'], true)
|| empty($parts['host']) || isset($parts['user']) || isset($parts['pass']) || isset($parts['query'])
|| isset($parts['fragment']) || preg_match('/[\x00-\x20\x7f]/', $base)
|| preg_match('/[\x00-\x20\x7f]/', $key)) {
throw new FollowupAudioException('CONFIG_INVALID');
}
$path = (string) ($parts['path'] ?? '');
if (str_ends_with($path, '/chat/completions')) {
throw new FollowupAudioException('DIFY_APPLICATION_REQUIRED');
}
if (str_ends_with($path, '/chat-messages')) {
$base = substr($base, 0, -strlen('/chat-messages'));
} elseif (!str_ends_with($path, '/v1')) {
$base .= '/v1';
}
$provider = FollowupAudioProviderConfig::resolve($profile, $this->settings, $this->provider);
$timeout = (int) ($this->settings['request_timeout'] ?? 240);
if ($timeout < 1 || $timeout > 300) {
throw new FollowupAudioException('CONFIG_INVALID');
if ($timeout < 1 || $timeout > 300) { throw new FollowupAudioException('CONFIG_INVALID'); }
if (!function_exists('curl_init')) { throw new FollowupAudioException('CURL_UNAVAILABLE'); }
return [$provider['base_url'], $provider['api_key'], $timeout, $provider];
}
/** Only MP3/WAV input_audio is supported, with bytes from the verified complete processing copy. */
private function openAiPayload(array $audio, string $query, string $model): array
{
if (!in_array($audio['extension'], ['mp3', 'wav'], true)) { throw new FollowupAudioException('AUDIO_INVALID'); }
$bytes = file_get_contents($audio['path']);
if (!is_string($bytes) || $bytes === '' || strlen($bytes) > (int) ($this->settings['upstream_max_bytes'] ?? 20971520)) {
throw new FollowupAudioException('UPSTREAM_AUDIO_LIMIT');
}
if (!function_exists('curl_init')) {
throw new FollowupAudioException('CURL_UNAVAILABLE');
}
return [$base, $key, $timeout];
return ['model' => $model, 'stream' => false, 'messages' => [['role' => 'user', 'content' => [
['type' => 'text', 'text' => $query],
['type' => 'input_audio', 'input_audio' => ['data' => base64_encode($bytes), 'format' => $audio['extension']]],
]]]];
}
private function inspectAudio(string $path, string $sha256, bool $processing = false): array
@@ -240,7 +246,7 @@ final class FollowupAudioDify
}
/** Preserve the exact original. Only a private, bounded audio copy may be sent to the same Dify app. */
private function prepareAudio(array $audio, callable $heartbeat): array
private function prepareAudio(array $audio, callable $heartbeat, bool $openAi = false): array
{
// Dify has a separate audio-upload ceiling (default 50 MiB); use a conservative 20 MiB local budget.
// Deployment must verify its own app/model limits via the synthetic probe before enabling a profile.
@@ -249,7 +255,8 @@ final class FollowupAudioDify
if ($limit < 1 || $limit > 52428800 || $timeout < 1 || $timeout > 600) {
throw new FollowupAudioException('CONFIG_INVALID');
}
if (filesize($audio['path']) <= $limit && $audio['extension'] !== 'amr') { return $audio; }
if (filesize($audio['path']) <= $limit && $audio['extension'] !== 'amr'
&& (!$openAi || in_array($audio['extension'], ['mp3', 'wav'], true))) { return $audio; }
// 32 kbit/s mono speech preserves the entire hour within ~14.5 MB, without splitting model requests.
if ($audio['duration'] * 4000 + 2048 > $limit) {
throw new FollowupAudioException('UPSTREAM_AUDIO_LIMIT');
@@ -416,8 +423,9 @@ final class FollowupAudioDify
$http = (int) ($response['http_code'] ?? 0);
// Even failed/uncertain replies can contain a billable task ID. Persist it before raising a sanitized error.
$candidate = json_decode((string) ($response['body'] ?? ''), true);
if (is_array($candidate) && isset($candidate['id'], $spec['json']['messages'])) { $candidate['message_id'] = $candidate['id']; }
if ((int) ($response['errno'] ?? 0) !== 0 || $http < 200 || $http >= 300
|| !is_array($candidate) || !empty($candidate['code']) || ($candidate['event'] ?? '') === 'error') {
|| !is_array($candidate) || !empty($candidate['code']) || !empty($candidate['error']) || ($candidate['event'] ?? '') === 'error') {
$observed = $spec['request_ids'] ?? [];
foreach (['task_id', 'message_id', 'conversation_id'] as $name) {
$id = $this->identifier($candidate[$name] ?? null);
@@ -441,7 +449,7 @@ final class FollowupAudioDify
try { $body = json_decode((string) ($response['body'] ?? ''), true, 64, JSON_THROW_ON_ERROR); }
catch (\Throwable $e) { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
if (!is_array($body)) { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
if (!empty($body['code']) || ($body['event'] ?? '') === 'error') {
if (!empty($body['code']) || !empty($body['error']) || ($body['event'] ?? '') === 'error') {
throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true);
}
$requestId = $this->identifier($response['request_id'] ?? null);