ZYT/zyt
2
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add scoped read-only audio preview and isolated Dify transport

Browse Source
This commit is contained in:
long
2026-10-09 16:10:59 +08:00
parent 27772bec61
commit 4d0af7f3f4
36 changed files with 1517 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/common/service/followupaudio/FollowupAudioPipeline.php
+67 -14
View File
@@ -71,13 +71,14 @@ final class FollowupAudioPipeline
continue;
}
$next = $state;
unset($next['chunks'][$index]['upstream_ids']);
$next['chunks'][$index]['state'] = 'intent';
$next['chunks'][$index]['request_id'] = 'fa-' . bin2hex(random_bytes(16));
$this->save($state, $next, $task, $heartbeat, 'transcribing', true);
$response = $this->request('asr', ['multipart' => ['model' => $this->provider['asr']['model'], 'response_format' => 'json',
'file' => new \CURLFile($chunkPath, 'audio/wav', 'chunk.wav')]], $heartbeat);
$response = $this->request('asr', $this->transcriptionRequest($chunkPath, $state['chunks'][$index]['request_id']), $heartbeat);
if ($response['rejected']) {
$next = $state; $next['chunks'][$index]['state'] = 'rejected';
if ($response['upstream_ids'] !== []) { $next['chunks'][$index]['upstream_ids'] = $response['upstream_ids']; }
$this->save($state, $next, $task, $heartbeat, 'transcribing');
throw new FollowupAudioException('ASR_REJECTED');
}
@@ -86,6 +87,7 @@ final class FollowupAudioPipeline
throw new FollowupAudioException('ASR_RESPONSE_INVALID', true);
}
$next = $state; $next['chunks'][$index]['state'] = 'complete'; $next['chunks'][$index]['text'] = $body['text'];
if ($response['upstream_ids'] !== []) { $next['chunks'][$index]['upstream_ids'] = $response['upstream_ids']; }
$this->save($state, $next, $task, $heartbeat, 'transcribing');
unlink($chunkPath);
}
@@ -93,23 +95,33 @@ final class FollowupAudioPipeline
$transcript = implode("\n", array_column($segments, 'text'));
self::assertTranscriptQuality($transcript, $segments);
if ($state['extraction']['state'] !== 'complete') {
$payload = $this->extractRequest($transcript, $segments, $task['recorded_at'], FollowupAudioFields::catalog());
$requestId = 'fa-' . bin2hex(random_bytes(16));
$payload = $this->extractRequest($transcript, $segments, $task['recorded_at'], FollowupAudioFields::catalog(), $requestId);
self::beat($heartbeat, []);
$this->assertSource($audio['path'], $task['sha256']);
$next = $state; $next['extraction'] = ['state' => 'intent', 'request_id' => 'fa-' . bin2hex(random_bytes(16))];
$next = $state; $next['extraction'] = ['state' => 'intent', 'request_id' => $requestId];
$this->save($state, $next, $task, $heartbeat, 'extracting', true);
$response = $this->request('extraction', ['json' => $payload], $heartbeat);
if ($response['rejected']) {
$next = $state; $next['extraction']['state'] = 'rejected';
if ($response['upstream_ids'] !== []) { $next['extraction']['upstream_ids'] = $response['upstream_ids']; }
$this->save($state, $next, $task, $heartbeat, 'extracting');
throw new FollowupAudioException('EXTRACTION_REJECTED');
}
$choices = $response['body']['choices'] ?? [];
$choice = is_array($choices) && count($choices) === 1 ? ($choices[0] ?? []) : [];
$answer = ($choice['finish_reason'] ?? '') === 'stop' && empty($choice['message']['refusal']) && empty($choice['message']['tool_calls'])
&& is_string($choice['message']['content'] ?? null) ? $choice['message']['content'] : '';
if (($this->provider['extraction']['protocol'] ?? 'openai') === 'dify_chat') {
$body = $response['body'];
$finish = $body['finish_reason'] ?? $body['metadata']['finish_reason'] ?? $body['metadata']['usage']['finish_reason'] ?? null;
$answer = ($finish === null || $finish === 'stop') && empty($body['refusal']) && empty($body['tool_calls'])
&& is_string($body['answer'] ?? null) ? $body['answer'] : '';
} else {
$choices = $response['body']['choices'] ?? [];
$choice = is_array($choices) && count($choices) === 1 ? ($choices[0] ?? []) : [];
$answer = ($choice['finish_reason'] ?? '') === 'stop' && empty($choice['message']['refusal']) && empty($choice['message']['tool_calls'])
&& is_string($choice['message']['content'] ?? null) ? $choice['message']['content'] : '';
}
// A received invalid answer is still a known completed request; retain it encrypted, never silently reissue it.
$next = $state; $next['extraction']['state'] = 'complete'; $next['extraction']['answer'] = $answer;
if ($response['upstream_ids'] !== []) { $next['extraction']['upstream_ids'] = $response['upstream_ids']; }
$this->save($state, $next, $task, $heartbeat, 'validating');
}
$this->assertSource($audio['path'], $task['sha256']);
@@ -127,7 +139,7 @@ final class FollowupAudioPipeline
}
/** Pure production request builder for authorized cached-ASR acceptance; performs no network or state mutation. */
public function extractRequest(string $transcript, array $segments, string $recordedAt, array $catalog): array
public function extractRequest(string $transcript, array $segments, string $recordedAt, array $catalog, ?string $requestId = null): array
{
$part = $this->provider['extraction'];
$mode = $part['response_format'] ?? 'json_schema';
@@ -138,6 +150,11 @@ final class FollowupAudioPipeline
|| ($thinking !== null && !is_bool($thinking))) { throw new FollowupAudioException('CONFIG_INVALID'); }
$prompt = FollowupAudioTranscriptPrompt::build($transcript, $segments, $recordedAt, $catalog);
self::assertTranscriptQuality($transcript, $segments);
if (($part['protocol'] ?? 'openai') === 'dify_chat') {
if ($mode !== 'prompt_json' || empty($part['binding_revision']) || $thinking !== null) { throw new FollowupAudioException('CONFIG_INVALID'); }
return ['inputs' => new \stdClass(), 'query' => $prompt, 'response_mode' => 'blocking',
'user' => $this->opaqueUser($requestId ?? hash('sha256', $transcript)), 'auto_generate_name' => false];
}
$payload = ['model' => $part['model'], 'stream' => false, 'temperature' => 0, 'max_tokens' => $maxTokens,
'messages' => [['role' => 'user', 'content' => $prompt]]];
if ($mode === 'json_schema') { $payload['response_format'] = self::buildResponseFormat($catalog, FollowupAudioTranscriptPrompt::citations($segments)); }
@@ -147,6 +164,20 @@ final class FollowupAudioPipeline
return $payload;
}
public function transcriptionRequest(string $chunkPath, string $requestId): array
{
$file = new \CURLFile($chunkPath, 'audio/wav', 'chunk.wav');
if (($this->provider['asr']['protocol'] ?? 'openai') === 'dify') {
return ['multipart' => ['file' => $file, 'user' => $this->opaqueUser($requestId)]];
}
return ['multipart' => ['model' => $this->provider['asr']['model'], 'response_format' => 'json', 'file' => $file]];
}
private function opaqueUser(string $requestId): string
{
return 'fa-opaque-' . substr(hash('sha256', $this->provider['fingerprint'] . ':' . $requestId), 0, 48);
}
public static function buildResponseFormat(array $catalog, array $citations): array
{
return FollowupAudioExtractionSchema::responseFormat($catalog, $citations);
@@ -265,19 +296,41 @@ final class FollowupAudioPipeline
$limit = (int) ($this->settings['max_response_bytes'] ?? 8388608);
if ($timeout < 1 || $timeout > 300 || $limit < 1024 || $limit > 8388608) { throw new FollowupAudioException('CONFIG_INVALID'); }
$part = $this->provider[$stage];
$spec = ['url' => $part['base_url'] . ($stage === 'asr' ? '/audio/transcriptions' : '/chat/completions'),
$protocol = $part['protocol'] ?? 'openai';
$endpoint = $stage === 'asr' ? ($protocol === 'dify' ? '/audio-to-text' : '/audio/transcriptions')
: ($protocol === 'dify_chat' ? '/chat-messages' : '/chat/completions');
$spec = ['url' => $part['base_url'] . $endpoint,
'api_key' => $part['api_key'], 'timeout' => $timeout, 'stage' => $stage] + $payload;
try { $response = $this->transport ? ($this->transport)($spec, $heartbeat) : $this->curl($spec, $heartbeat, $limit); }
try {
$response = $this->transport ? ($this->transport)($spec, $heartbeat)
: (($this->provider['http_transport'] ?? 'curl') === 'openssl_stream'
? FollowupAudioStreamTransport::request($spec, $heartbeat, $limit, $this->provider['allow_loopback_tunnel'])
: $this->curl($spec, $heartbeat, $limit));
}
catch (\Throwable $e) { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
$http = (int) ($response['http_code'] ?? 0);
$candidate = is_string($response['body'] ?? null) && strlen($response['body']) <= $limit ? json_decode($response['body'], true) : null;
$ids = [];
if (is_array($candidate)) {
foreach (['task_id', 'message_id', 'conversation_id'] as $key) {
if (is_string($candidate[$key] ?? null) && preg_match('/^[A-Za-z0-9_.:-]{1,190}$/D', $candidate[$key])) { $ids[$key] = $candidate[$key]; }
}
if (!isset($ids['message_id']) && $stage === 'extraction' && is_string($candidate['id'] ?? null)
&& preg_match('/^[A-Za-z0-9_.:-]{1,190}$/D', $candidate['id'])) { $ids['message_id'] = $candidate['id']; }
}
if ($ids !== []) {
$fields = ['upstream_ids_json' => json_encode($ids, JSON_THROW_ON_ERROR)];
if (isset($ids['message_id']) || isset($ids['task_id'])) { $fields['upstream_run_id'] = $ids['message_id'] ?? $ids['task_id']; }
self::beat($heartbeat, $fields); // Preserve observed opaque IDs even when the reply is uncertain/rejected.
}
if (($response['errno'] ?? 0) !== 0 || $http === 0 || $http >= 500 || $http === 408
|| !is_string($response['body'] ?? null) || strlen($response['body']) > $limit) { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
if (in_array($http, [400, 401, 403, 404, 413, 415, 422, 429], true)) { return ['rejected' => true, 'body' => []]; }
if (in_array($http, [400, 401, 403, 404, 413, 415, 422, 429], true)) { return ['rejected' => true, 'body' => [], 'upstream_ids' => $ids]; }
if ($http < 200 || $http >= 300) { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
try { $body = json_decode($response['body'], true, 64, JSON_THROW_ON_ERROR); }
catch (\Throwable $e) { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
if (!is_array($body) || !empty($body['error'])) { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
return ['rejected' => false, 'body' => $body];
if (!is_array($body) || !empty($body['error']) || !empty($body['code']) || ($body['event'] ?? '') === 'error') { throw new FollowupAudioException('UPSTREAM_UNCERTAIN', true); }
return ['rejected' => false, 'body' => $body, 'upstream_ids' => $ids];
}
private function curl(array $spec, callable $heartbeat, int $limit): array