Files
2026-08-05 15:56:08 +08:00

3226 lines
131 KiB
PHP
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
namespace app\controller\api;
use app\model\AiModel;
use app\model\Conversation as ConversationModel;
use app\model\Message;
use app\model\UploadFile;
use app\service\AgentCatalog;
use app\service\ComfyJobDeferredException;
use app\service\ComfyUIService;
use app\service\CosyVoiceService;
use app\service\DifyService;
use app\service\DocumentTextService;
use app\service\GuestAccessService;
use app\service\OpenAIService;
use app\service\PermissionService;
use app\service\SettingsService;
use think\facade\Log;
class Chat extends BaseApi
{
public function speech()
{
$user = $this->authUser();
GuestAccessService::assertAccountRequired($user, '语音对话');
$input = $this->request->post();
$text = trim((string) ($input['text'] ?? ''));
if ($text === '') {
return $this->error('语音内容不能为空', 422);
}
if (mb_strlen($text) > 600) {
return $this->error('单次语音内容不能超过 600 个字符', 422);
}
$modelId = isset($input['model_id']) && $input['model_id'] !== ''
? (int) $input['model_id']
: null;
$persona = CosyVoiceService::getPersona();
$voice = trim((string) ($persona['fallback_voice'] ?? $input['voice'] ?? 'marin'));
$allowedVoices = [
'alloy', 'ash', 'ballad', 'coral', 'echo', 'fable', 'nova',
'onyx', 'sage', 'shimmer', 'verse', 'marin', 'cedar',
];
if (!in_array($voice, $allowedVoices, true)) {
$voice = 'marin';
}
$speech = null;
if (CosyVoiceService::canAttempt()) {
try {
$speech = CosyVoiceService::speech($text);
} catch (\Throwable $error) {
Log::warning('CosyVoice speech fallback: ' . $error->getMessage());
}
}
if (!$speech) {
$model = OpenAIService::getSpeechModel($modelId);
$speech = OpenAIService::speech($model, $text, $voice);
$speech['provider'] = 'openai';
}
return response($speech['audio'], 200, [
'Content-Type' => $speech['content_type'],
'Content-Length' => (string) strlen($speech['audio']),
'Cache-Control' => 'no-store',
'X-Content-Type-Options' => 'nosniff',
'X-TTS-Provider' => $speech['provider'] ?? 'unknown',
]);
}
public function speechStream(): never
{
$user = $this->authUser();
GuestAccessService::assertAccountRequired($user, '语音对话');
$input = $this->request->post();
$text = trim((string) ($input['text'] ?? ''));
$requestId = trim((string) ($input['request_id'] ?? ''));
if (!preg_match('/^[A-Za-z0-9._:-]{8,128}$/', $requestId)) {
$requestId = bin2hex(random_bytes(16));
}
if ($text === '' || mb_strlen($text) > 600 || !CosyVoiceService::canAttempt()) {
http_response_code($text === '' || mb_strlen($text) > 600 ? 422 : 503);
header('Content-Type: application/json; charset=utf-8');
echo json_encode([
'code' => 1,
'message' => $text === ''
? '语音内容不能为空'
: (mb_strlen($text) > 600 ? '单次语音内容不能超过 600 个字符' : 'CosyVoice 暂时不可用'),
'data' => null,
], JSON_UNESCAPED_UNICODE);
exit;
}
while (ob_get_level() > 0) {
ob_end_clean();
}
@ini_set('zlib.output_compression', '0');
ignore_user_abort(false);
OpenAIService::sseHeaders();
$persona = CosyVoiceService::getPersona();
OpenAIService::sseEvent('meta', [
'provider' => 'cosyvoice',
'request_id' => $requestId,
'cancel_url' => rtrim((string) ($persona['base_url'] ?? ''), '/')
. '/cancel/' . rawurlencode($requestId),
'format' => 'pcm_s16le',
'sample_rate' => (int) ($persona['sample_rate'] ?? 24000),
'channels' => 1,
]);
try {
$result = CosyVoiceService::streamSpeech($text, static function (string $pcm): void {
OpenAIService::sseEvent('audio', [
'audio' => base64_encode($pcm),
]);
}, $requestId);
if (empty($result['aborted'])) {
OpenAIService::sseEvent('done', [
'bytes' => (int) ($result['bytes'] ?? 0),
]);
}
} catch (\Throwable $error) {
Log::warning('CosyVoice stream failed: ' . $error->getMessage());
OpenAIService::sseEvent('error', [
'message' => $error->getMessage(),
]);
}
exit;
}
public function speechCancel()
{
$user = $this->authUser();
GuestAccessService::assertAccountRequired($user, '语音对话');
$requestId = trim((string) $this->request->post('request_id', ''));
if (!preg_match('/^[A-Za-z0-9._:-]{8,128}$/', $requestId)) {
return $this->error('语音请求标识无效', 422);
}
return $this->success([
'cancelled' => CosyVoiceService::cancelSpeech($requestId),
'request_id' => $requestId,
]);
}
public function completions()
{
$user = $this->authUser();
PermissionService::checkDailyLimit($user);
$input = $this->request->post();
$conversationId = (int) ($input['conversation_id'] ?? 0);
$content = trim($input['content'] ?? '');
$attachments = $input['attachments'] ?? [];
$agentId = trim((string) ($input['agent_id'] ?? ''));
$imageTool = trim((string) ($input['image_tool'] ?? ''));
$voiceMode = !empty($input['voice_mode']);
$stream = ($input['stream'] ?? true) !== false;
$allowedImageTools = ['enhance', 'erase', 'watermark', 'cutout', 'outpaint', 'replace', 'text', 'restore', 'creative', 'commit'];
if ($imageTool !== '' && !in_array($imageTool, $allowedImageTools, true)) {
return $this->error('未知的图片处理工具', 422);
}
if (!is_array($attachments)) {
return $this->error('附件格式无效', 422);
}
GuestAccessService::assertTextChatOnly($user, $attachments, $agentId, $imageTool, $voiceMode);
if (!$conversationId) {
return $this->error('缺少 conversation_id');
}
if (!$content && empty($attachments)) {
return $this->error('消息内容不能为空');
}
if ($imageTool !== '' && !$this->hasImageAttachments($attachments)) {
return $this->error('图片处理工具需要一张原图', 422);
}
$agent = AgentCatalog::find($agentId);
if ($agentId !== '' && !$agent) {
return $this->error('所选 Agent 不存在或已停用', 422);
}
$conversation = ConversationModel::where('id', $conversationId)
->where('user_id', $user['id'])
->whereNull('deleted_at')
->find();
if (!$conversation) {
return $this->error('会话不存在', 404);
}
$imageGenerationContent = $content;
if (GuestAccessService::isGuest($user)) {
$model = GuestAccessService::model();
if ((int) $conversation->model_id !== (int) $model->id) {
$conversation->save([
'model_id' => (int) $model->id,
'external_conversation_id' => null,
]);
}
} else {
$model = OpenAIService::getModel($conversation->model_id ? (int) $conversation->model_id : null);
}
if ($imageTool !== '' && $imageTool !== 'commit') {
$preferredImageModelId = ($model->provider ?? '') === 'comfy' ? (int) ($model->id ?? 0) : null;
$model = OpenAIService::getImageModel($preferredImageModelId ?: null);
$agent = null;
} elseif ($imageTool === 'commit') {
$agent = null;
} elseif ($agent) {
$preferredModelId = (int) ($model->id ?? 0);
// Agent always resolves the intent and produces a clean visual prompt before ComfyUI.
// Sending raw Chinese commands directly to the image workflow can render command text.
$model = OpenAIService::getLanguageModel(
($model->provider ?? '') !== 'comfy' ? $preferredModelId : null
);
}
if (($model->provider ?? 'openai') === 'comfy') {
if ($agent && !AgentCatalog::supportsImage($agent)) {
return $this->error('所选 Agent 不支持图片生成,请更换 Agent 后重试', 422);
}
if ($content === '') {
return $this->error('图片生成或处理模式需要输入文字描述', 422);
}
if (!empty($attachments)) {
$imageCount = count(array_filter(
$attachments,
fn ($attachment) => is_array($attachment) && ($attachment['type'] ?? '') === 'image'
));
if ($imageCount !== count($attachments)) {
return $this->error('ComfyUI 图片处理模式只接受图片附件', 422);
}
if ($imageCount > 2) {
return $this->error('图片处理最多上传两张图片:第一张原图,第二张黑白遮罩', 422);
}
}
}
$contentType = SettingsService::isFeatureEnabled('markdown') ? 'markdown' : 'text';
if (!empty($attachments)) {
$contentType = 'mixed';
}
Message::create([
'conversation_id' => $conversationId,
'role' => 'user',
'content' => $content,
'content_type' => $contentType,
'attachments' => $attachments,
]);
PermissionService::incrementDailyCount((int) $user['id']);
if ((int) $conversation->message_count === 0 && $content) {
ConversationModel::where('id', $conversationId)->update(['title' => mb_substr($content, 0, 30)]);
}
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
if ($imageTool === 'commit') {
return $this->handleCommittedImage($conversationId, $attachments, $content, $stream, $user);
}
if (($model->provider ?? 'openai') === 'dify') {
return $this->handleDifyCompletion($conversation, $model, $content, $attachments, $stream, $user, $agent);
}
if (($model->provider ?? 'openai') === 'comfy') {
return $this->handleComfyCompletion(
$conversation,
$model,
$imageGenerationContent,
$stream,
$user,
$agent,
$attachments,
$imageTool
);
}
if ($model->support_context ?? true) {
$history = Message::where('conversation_id', $conversationId)
->field('role,content,attachments')
->order('id', 'desc')
->limit($voiceMode ? 16 : 50)
->select()
->toArray();
$history = array_reverse($history);
} else {
// 模型未开启上下文支持,仅发送当前这一条消息,不携带历史记录
$history = [[
'role' => 'user',
'content' => $content,
'attachments' => $attachments,
]];
}
$apiMessages = $this->buildApiMessages($history, $model);
if ($voiceMode) {
$voicePersona = CosyVoiceService::getPersona();
$personaName = trim((string) ($voicePersona['name'] ?? 'AI 客服')) ?: 'AI 客服';
$personaPrompt = trim((string) ($voicePersona['role_prompt'] ?? ''));
array_unshift($apiMessages, [
'role' => 'system',
'content' => '你是名为“' . $personaName . '”的 AI 客服。人物设定:' . $personaPrompt
. ' 当前正在进行低延迟实时语音对话。像真人客服一样先回应用户的真实诉求,语气口语化、有耐心、有适度共情,不复述问题,不使用 Markdown 列表,不说“作为 AI”。先给结论,通常控制在 1 到 3 句;信息不足时每轮只追问一个最关键的问题,除非用户明确要求详细说明。',
]);
}
$agentImageActionAllowed = false;
if ($agent) {
$agentSystemPrompt = $agent['system_prompt'];
$imageTurn = $this->resolveAgentImageTurn($content, $history);
$explicitImageRequest = $imageTurn['explicit'];
$contextualImageRequest = $imageTurn['contextual'];
$contextualImageSubject = $imageTurn['subject'];
$revisionImageRequest = $imageTurn['revision'];
$uploadedImageEdit = $imageTurn['uploaded_edit'];
$plannedImageRequest = $imageTurn['planned'];
$independentImageRequest = $imageTurn['independent'];
$agentImageActionAllowed = $imageTurn['allowed'];
if ($agentImageActionAllowed) {
$agentSystemPrompt .= "\n\n【系统路由判定】本轮是"
. ($contextualImageRequest
? '结合最近对话识别出的隐含视觉展示请求。必须从前文提取用户真正想看的主体和视觉特征,直接生成图片,不得继续文字讲解或声称无法展示。'
: ($revisionImageRequest
? ($uploadedImageEdit
? '对本轮上传原图的编辑请求。必须严格保留未要求修改的区域,只执行用户指定的像素级调整。'
: '对当前生成图片的修改请求。必须基于上一张有效图片状态执行调整。')
: ($plannedImageRequest
? '对最近图片方案的明确确认。必须按已确认方案直接生成图片。'
: '明确的图片生成请求。')))
. ($contextualImageSubject !== ''
? '本轮代词或省略表达所指的核心主体已确定为“' . $contextualImageSubject . '”。prompt 必须明确描绘该主体,禁止把代词本身解释成普通男性、女性或其他类别。'
: '')
. '只输出 generate_image 动作 JSON,不要解释或复述用户命令;'
. ($uploadedImageEdit
? 'prompt 必须是纯英文、完整、可直接编辑图片的视觉描述;必须保留原图中未被用户指定删除的现有文字、标志和排版,只删除用户明确点名的水印或对象,禁止新增文字。'
: 'prompt 的视觉描述必须以英文为主、完整且可直接生图。用户未明确要求画面文字时默认禁字;若用户明确给出标题、书名或画面文字,必须把原文逐字写成 EXACT_VISIBLE_TEXT: <<<原文>>>,禁止翻译、改写,并且不得再追加 no text/no title 等冲突限制。')
. '只保留用户明确要求在画面中可见的内容;生成之前的流程、环节、衔接和操作措辞不得变成画面元素。'
. ($independentImageRequest
? '这是独立的新图,不得继承或混入上一张图片的主体、文字和场景。'
: '这是对当前图片的修改,必须合并当前有效画面状态。');
} else {
$agentSystemPrompt .= "\n\n【系统路由判定】本轮不是图片生成或图片修改请求。"
. '必须针对用户当前问题正常输出文本回答,禁止输出 generate_image 动作,'
. '禁止继承、复用或改写此前任何图片的主体、提示词和场景。'
. '如果当前消息是对上一轮的简短质疑或情绪反馈,应回应质疑并重新核对,不能解释该短语的字面含义。';
}
if ($revisionImageRequest) {
$generatedImages = $this->latestAssistantImageAttachments(
$conversationId,
4,
false
);
$generationState = $this->generationStateFromAttachments($generatedImages);
if ($generationState === '') {
$generationState = $this->latestAssistantGenerationState($conversationId, false);
}
if ($generatedImages || $generationState !== '') {
$iterationContext = $this->buildAgentImageIterationContext(
$conversationId,
$content,
$generationState
);
if ($iterationContext !== '') {
$agentSystemPrompt .= "\n\n" . $iterationContext;
}
}
}
array_unshift($apiMessages, [
'role' => 'system',
'content' => $agentSystemPrompt,
]);
}
if ($stream) {
$this->streamResponse(
$conversationId,
$model,
$apiMessages,
$user,
$agent,
$agentImageActionAllowed
);
}
return $this->syncResponse(
$conversationId,
$model,
$apiMessages,
$user,
$agent,
$agentImageActionAllowed
);
}
/**
* 保存浏览器端完成的确定性图片编辑(例如精确文字排版),不再经过扩散模型二次重绘。
*/
private function handleCommittedImage(
int $conversationId,
array $attachments,
string $content,
bool $stream,
array $user
) {
$image = null;
foreach ($attachments as $attachment) {
if (is_array($attachment)
&& ($attachment['type'] ?? '') === 'image'
&& !empty($attachment['url'])) {
$image = $attachment;
break;
}
}
if ($image === null
|| self::resolveStoredPath((string) $image['url'], (int) $user['id']) === null) {
return $this->error('编辑后的图片不存在或不属于当前用户,请重新处理', 422);
}
unset($image['hidden'], $image['_localPreview'], $image['preview']);
$image['type'] = 'image';
$assistantContent = $content !== '' ? $content . '' : '图片编辑完成:';
$visibleAttachments = [$image];
Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $assistantContent,
'content_type' => 'mixed',
'attachments' => $visibleAttachments,
]);
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
$payload = [
'content' => $assistantContent,
'attachments' => $visibleAttachments,
];
if (!$stream) {
return $this->success($payload);
}
while (ob_get_level() > 0) {
ob_end_clean();
}
OpenAIService::sseHeaders();
OpenAIService::sseEvent('image', ['attachments' => $visibleAttachments]);
OpenAIService::sseEvent('message', ['content' => $assistantContent]);
OpenAIService::sseEvent('done', $payload);
exit;
}
/**
* ComfyUI 文生图:用户文本作为 prompt,生成图片后作为附件写入助手消息。
*/
private function handleComfyCompletion(
ConversationModel $conversation,
AiModel $model,
string $content,
bool $stream,
array $user,
?array $agent = null,
array $attachments = [],
string $imageTool = ''
) {
$conversationId = (int) $conversation->id;
$prompt = AgentCatalog::buildImagePrompt($agent, $content);
$prompt = $this->applyRequestedVisibleText($prompt, $content);
if ($prompt === '') {
return $this->error('请输入图片描述');
}
$editContext = $attachments
? $this->buildComfyEditContext($attachments, $content, (int) $user['id'], $imageTool)
: null;
$imageCount = $editContext === null
? $this->requestedImageCount($content)
: 1;
if ($stream) {
$this->streamComfyResponse(
$conversationId,
$model,
$prompt,
$user,
$content,
$editContext,
$imageCount
);
}
return $this->syncComfyResponse(
$conversationId,
$model,
$prompt,
$user,
$content,
$editContext,
$imageCount
);
}
private function streamComfyResponse(
int $conversationId,
AiModel $model,
string $prompt,
array $user,
string $generationState = '',
?array $editContext = null,
int $imageCount = 1
): void
{
while (ob_get_level() > 0) {
ob_end_clean();
}
@set_time_limit(0);
ignore_user_abort(true);
OpenAIService::sseHeaders();
$assistantId = null;
$promptId = '';
$jobCreatedAt = time();
$prompt = $this->prepareComfyEditPrompt($prompt, $editContext);
$imageCount = $editContext === null ? max(1, min(4, $imageCount)) : 1;
$generationProgress = $this->imageGenerationProgress($imageCount);
if (in_array(($editContext['operation'] ?? ''), ['remove_watermark', 'remove_text'], true)) {
$generationState = $this->localizedRemovalGenerationState($editContext);
}
try {
$localAttachments = $this->processAutomaticLocalizedRemoval($editContext, (int) $user['id']);
if ($localAttachments !== null) {
$attachments = $this->attachGenerationState(
$localAttachments,
$generationState ?: $prompt
);
$fullContent = $this->localizedRemovalSuccessContent($editContext);
Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $fullContent,
'content_type' => 'mixed',
'attachments' => $attachments,
]);
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
OpenAIService::sseEvent('image', ['attachments' => $attachments]);
OpenAIService::sseEvent('message', ['content' => $fullContent]);
OpenAIService::sseEvent('done', [
'content' => $fullContent,
'attachments' => $attachments,
]);
exit;
}
OpenAIService::sseEvent('progress', [
'content' => $editContext ? '正在提交图片处理任务…' : $generationProgress,
]);
$promptId = $this->submitComfyTask($model, $prompt, $editContext, $imageCount);
$jobCreatedAt = time();
$pendingAttachment = ComfyUIService::buildJobAttachment(
$promptId,
(int) $model->id,
$editContext ? '图片处理中,请稍候…可关闭页面,完成后自动显示' : $generationProgress . '可关闭页面,完成后自动显示',
$jobCreatedAt,
$imageCount
);
$pendingAttachment['generation_prompt'] = mb_substr(trim($generationState ?: $prompt), 0, 8000);
$assistant = Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $editContext ? '图片处理中,请稍候…可关闭页面,完成后自动显示' : $generationProgress . '可关闭页面,完成后自动显示',
'content_type' => 'mixed',
'attachments' => [$pendingAttachment],
]);
$assistantId = (int) $assistant->id;
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
OpenAIService::sseEvent('progress', [
'content' => $editContext ? '图片处理中,请稍候…可关闭页面,完成后自动显示' : $generationProgress . '可关闭页面,完成后自动显示',
]);
$attachments = ComfyUIService::waitAndCollect(
$model,
$promptId,
(int) $user['id'],
function (string $message) use ($assistantId, $promptId, $model, $jobCreatedAt, $generationState, $prompt, $imageCount) {
OpenAIService::sseEvent('progress', ['content' => $message]);
if ($assistantId) {
$pendingAttachment = ComfyUIService::buildJobAttachment(
$promptId,
(int) $model->id,
$message,
$jobCreatedAt,
$imageCount
);
$pendingAttachment['generation_prompt'] = mb_substr(trim($generationState ?: $prompt), 0, 8000);
Message::where('id', $assistantId)->update([
'content' => $message,
'attachments' => json_encode([$pendingAttachment], JSON_UNESCAPED_UNICODE),
]);
}
},
25
);
$attachments = $this->attachGenerationState($attachments, $generationState ?: $prompt);
} catch (ComfyJobDeferredException $e) {
// 任务仍在 ComfyUI:保持 pending,前端轮询 / 刷新后可取回
if ($assistantId && $promptId !== '') {
$msg = $e->getMessage();
$pendingAttachment = ComfyUIService::buildJobAttachment(
$promptId,
(int) $model->id,
$msg,
$jobCreatedAt,
$imageCount
);
$pendingAttachment['generation_prompt'] = mb_substr(trim($generationState ?: $prompt), 0, 8000);
Message::where('id', $assistantId)->update([
'content' => $msg,
'attachments' => json_encode([$pendingAttachment], JSON_UNESCAPED_UNICODE),
]);
}
OpenAIService::sseEvent('progress', ['content' => $e->getMessage()]);
OpenAIService::sseEvent('done', [
'content' => $e->getMessage(),
'pending' => true,
]);
exit;
} catch (\Throwable $e) {
if ($assistantId) {
Message::where('id', $assistantId)->update([
'content' => '图片生成失败:' . $e->getMessage(),
'attachments' => json_encode([], JSON_UNESCAPED_UNICODE),
]);
}
OpenAIService::sseEvent('error', ['message' => $e->getMessage()]);
exit;
}
// 若刷新后的恢复逻辑已经写好图片,避免覆盖
$fresh = Message::find($assistantId);
if ($fresh && ComfyUIService::hasImageAttachments($fresh->attachments)) {
$attachments = is_array($fresh->attachments) ? $fresh->attachments : [];
$fullContent = (string) $fresh->content;
} else {
$fullContent = $editContext ? '已根据要求处理图片:' : '已根据描述生成图片:';
Message::where('id', $assistantId)->update([
'content' => $fullContent,
'attachments' => json_encode($attachments, JSON_UNESCAPED_UNICODE),
]);
}
OpenAIService::sseEvent('image', ['attachments' => $attachments]);
OpenAIService::sseEvent('message', ['content' => $fullContent]);
OpenAIService::sseEvent('done', [
'content' => $fullContent,
'attachments' => $attachments,
]);
exit;
}
private function syncComfyResponse(
int $conversationId,
AiModel $model,
string $prompt,
array $user,
string $generationState = '',
?array $editContext = null,
int $imageCount = 1
)
{
@set_time_limit(0);
ignore_user_abort(true);
$assistantId = null;
$promptId = '';
$jobCreatedAt = time();
$prompt = $this->prepareComfyEditPrompt($prompt, $editContext);
$imageCount = $editContext === null ? max(1, min(4, $imageCount)) : 1;
$generationProgress = $this->imageGenerationProgress($imageCount);
if (in_array(($editContext['operation'] ?? ''), ['remove_watermark', 'remove_text'], true)) {
$generationState = $this->localizedRemovalGenerationState($editContext);
}
try {
$localAttachments = $this->processAutomaticLocalizedRemoval($editContext, (int) $user['id']);
if ($localAttachments !== null) {
$attachments = $this->attachGenerationState(
$localAttachments,
$generationState ?: $prompt
);
$fullContent = $this->localizedRemovalSuccessContent($editContext);
Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $fullContent,
'content_type' => 'mixed',
'attachments' => $attachments,
]);
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
return $this->success([
'content' => $fullContent,
'attachments' => $attachments,
]);
}
$promptId = $this->submitComfyTask($model, $prompt, $editContext, $imageCount);
$jobCreatedAt = time();
$pendingAttachment = ComfyUIService::buildJobAttachment(
$promptId,
(int) $model->id,
$editContext ? '图片处理中,请稍候…可关闭页面,完成后自动显示' : $generationProgress . '可关闭页面,完成后自动显示',
$jobCreatedAt,
$imageCount
);
$pendingAttachment['generation_prompt'] = mb_substr(trim($generationState ?: $prompt), 0, 8000);
$assistant = Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $editContext ? '图片处理中,请稍候…可关闭页面,完成后自动显示' : $generationProgress . '可关闭页面,完成后自动显示',
'content_type' => 'mixed',
'attachments' => [$pendingAttachment],
]);
$assistantId = (int) $assistant->id;
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
$attachments = ComfyUIService::waitAndCollect($model, $promptId, (int) $user['id'], null, 25);
$attachments = $this->attachGenerationState($attachments, $generationState ?: $prompt);
} catch (ComfyJobDeferredException $e) {
if ($assistantId && $promptId !== '') {
$pendingAttachment = ComfyUIService::buildJobAttachment(
$promptId,
(int) $model->id,
$e->getMessage(),
$jobCreatedAt,
$imageCount
);
$pendingAttachment['generation_prompt'] = mb_substr(trim($generationState ?: $prompt), 0, 8000);
Message::where('id', $assistantId)->update([
'content' => $e->getMessage(),
'attachments' => json_encode([$pendingAttachment], JSON_UNESCAPED_UNICODE),
]);
}
return $this->success([
'content' => $e->getMessage(),
'pending' => true,
], '任务仍在后台生成');
} catch (\Throwable $e) {
if ($assistantId) {
Message::where('id', $assistantId)->update([
'content' => '图片生成失败:' . $e->getMessage(),
'attachments' => json_encode([], JSON_UNESCAPED_UNICODE),
]);
}
return $this->error($e->getMessage(), 502);
}
$fresh = Message::find($assistantId);
if ($fresh && ComfyUIService::hasImageAttachments($fresh->attachments)) {
$attachments = is_array($fresh->attachments) ? $fresh->attachments : [];
$fullContent = (string) $fresh->content;
} else {
$fullContent = $editContext ? '已根据要求处理图片:' : '已根据描述生成图片:';
Message::where('id', $assistantId)->update([
'content' => $fullContent,
'attachments' => json_encode($attachments, JSON_UNESCAPED_UNICODE),
]);
}
return $this->success([
'content' => $fullContent,
'attachments' => $attachments,
]);
}
private function submitComfyTask(
AiModel $model,
string $prompt,
?array $editContext,
int $imageCount = 1
): string {
if ($editContext === null) {
return ComfyUIService::submit($model, $prompt, max(1, min(4, $imageCount)));
}
if (($editContext['operation'] ?? '') === 'remove_background') {
return ComfyUIService::submitBackgroundRemoval(
$model,
$editContext['source_path']
);
}
try {
return ComfyUIService::submitEdit(
$model,
$prompt,
$editContext['source_path'],
$editContext['mask_path'],
$editContext['mode'],
$editContext['denoise'] ?? null,
(string) ($editContext['operation'] ?? 'edit')
);
} finally {
if (($editContext['temporary_mask'] ?? false)
&& is_string($editContext['mask_path'] ?? null)
&& is_file($editContext['mask_path'])) {
@unlink($editContext['mask_path']);
}
}
}
/**
* Generate one image by default. Batch generation is only enabled when the
* user explicitly asks for a number of images/variants, capped at four.
*/
private function requestedImageCount(string $content): int
{
$content = mb_strtolower(trim($content));
if ($content === '') {
return 1;
}
$unit = '(?:张(?:图片)?|幅(?:图片)?|个\s*(?:不同|独立|全新)?\s*(?:图片|图像|方案|版本|构图|封面))';
if (preg_match('/([1-9]\d*)\s*' . $unit . '/u', $content, $matches)) {
return max(1, min(4, (int) $matches[1]));
}
if (preg_match('/([一二两三四五六七八九十]{1,3})\s*' . $unit . '/u', $content, $matches)) {
$digits = [
'一' => 1, '二' => 2, '两' => 2, '三' => 3, '四' => 4,
'五' => 5, '六' => 6, '七' => 7, '八' => 8, '九' => 9,
];
$numberText = $matches[1];
if ($numberText === '十') {
$count = 10;
} elseif (str_contains($numberText, '十')) {
[$tens, $ones] = array_pad(explode('十', $numberText, 2), 2, '');
$count = ($tens === '' ? 1 : ($digits[$tens] ?? 1)) * 10
+ ($ones === '' ? 0 : ($digits[$ones] ?? 0));
} else {
$count = $digits[$numberText] ?? 1;
}
return max(1, min(4, $count));
}
return 1;
}
private function imageGenerationProgress(int $imageCount): string
{
return '正在生成 ' . max(1, min(4, $imageCount)) . ' 张图片…';
}
private function processAutomaticLocalizedRemoval(?array $editContext, int $userId): ?array
{
if (!in_array(($editContext['operation'] ?? ''), ['remove_watermark', 'remove_text'], true)) {
return null;
}
try {
$displayName = match ($editContext['removal_target'] ?? '') {
'author' => 'author_removed.png',
'title' => 'title_removed.png',
'all_text' => 'text_removed.png',
default => 'watermark_removed.png',
};
return ComfyUIService::removeMaskedContentWithContentAwareFill(
$editContext['source_path'],
$editContext['mask_path'],
$userId,
$displayName,
($editContext['removal_target'] ?? '') === 'author'
? 'horizontal_text_band'
: 'boundary'
);
} finally {
if (($editContext['temporary_mask'] ?? false)
&& is_string($editContext['mask_path'] ?? null)
&& is_file($editContext['mask_path'])) {
@unlink($editContext['mask_path']);
}
}
}
private function localizedRemovalSuccessContent(?array $editContext): string
{
return match ($editContext['removal_target'] ?? '') {
'author' => '已去除作者文字:',
'title' => '已去除标题文字:',
'all_text' => '已去除画面文字:',
default => '已去除水印:',
};
}
private function localizedRemovalGenerationState(?array $editContext): string
{
return match ($editContext['removal_target'] ?? '') {
'author' => 'The author credit text has been removed while preserving all other image content.',
'title' => 'The title text has been removed while preserving all other image content.',
'all_text' => 'All requested text has been removed while preserving the underlying image.',
default => 'The watermark has been removed while preserving all other image content.',
};
}
private function buildComfyEditContext(
array $attachments,
string $content,
int $userId,
string $imageTool = ''
): array {
$images = array_values(array_filter(
$attachments,
fn ($attachment) => is_array($attachment)
&& ($attachment['type'] ?? '') === 'image'
&& !empty($attachment['url'])
));
if ($images === []) {
throw new \InvalidArgumentException('图片处理需要上传一张原图');
}
$sourcePath = self::resolveStoredPath((string) $images[0]['url'], $userId);
if ($sourcePath === null) {
throw new \InvalidArgumentException('原图不存在或不属于当前用户,请重新上传');
}
$requestedMode = in_array($imageTool, ['erase', 'watermark', 'outpaint', 'replace', 'text'], true)
? 'inpaint'
: AgentCatalog::imageEditMode($content);
$editFlags = AgentCatalog::resolveImageEditFlags($content, $imageTool);
$isBackgroundRemoval = $editFlags['is_background_removal'];
$isWatermarkRemoval = $editFlags['is_watermark_removal'];
$textRemovalTarget = $isBackgroundRemoval || $isWatermarkRemoval
? null
: AgentCatalog::imageTextRemovalTarget($content);
$isImageEnhancement = AgentCatalog::requestsImageEnhancement($content);
$maskPath = null;
$temporaryMask = false;
if ($requestedMode === 'inpaint' && isset($images[1])) {
$maskPath = self::resolveStoredPath((string) $images[1]['url'], $userId);
if ($maskPath === null) {
throw new \InvalidArgumentException('遮罩图片不存在或不属于当前用户,请重新上传');
}
$sourceSize = @getimagesize($sourcePath);
$maskSize = @getimagesize($maskPath);
if (!is_array($sourceSize)
|| !is_array($maskSize)
|| $sourceSize[0] !== $maskSize[0]
|| $sourceSize[1] !== $maskSize[1]) {
throw new \InvalidArgumentException('黑白遮罩必须与原图尺寸完全一致');
}
} elseif ($requestedMode === 'inpaint' && $isWatermarkRemoval) {
$maskPath = ComfyUIService::createAutomaticWatermarkMask($sourcePath, $content);
$temporaryMask = true;
} elseif ($requestedMode === 'inpaint' && $textRemovalTarget !== null) {
$maskPath = ComfyUIService::createAutomaticTextRemovalMask($sourcePath, $textRemovalTarget);
$temporaryMask = true;
}
$operation = $isBackgroundRemoval
? 'remove_background'
: ($isWatermarkRemoval
? 'remove_watermark'
: ($textRemovalTarget !== null
? 'remove_text'
: ($imageTool === 'outpaint' ? 'outpaint' : 'edit')));
$context = [
'source_path' => $sourcePath,
'mask_path' => $maskPath,
'mode' => $requestedMode === 'inpaint' && $maskPath !== null ? 'inpaint' : 'img2img',
'requested_mode' => $requestedMode,
'operation' => $operation,
'removal_target' => $textRemovalTarget,
'denoise' => $isWatermarkRemoval || $textRemovalTarget !== null
? 0.58
: ($imageTool === 'outpaint'
// VAEEncodeForInpaint already protects unmasked pixels; the masked
// border must be fully repainted or mirrored padding ghosts survive.
? 1.0
: ($isImageEnhancement ? 0.28 : null)),
'temporary_mask' => $temporaryMask,
];
// An inpaint checkpoint cannot infer the full semantic world from a thin
// boundary alone. Caption the source first so the generated border stays
// in the same scene instead of drifting into an unrelated environment.
if ($operation === 'outpaint') {
$context['outpaint_scene_prompt'] = $this->describeOutpaintScene($sourcePath, $userId);
}
return $context;
}
private function prepareComfyEditPrompt(string $prompt, ?array $editContext): string
{
$operation = (string) ($editContext['operation'] ?? '');
if ($operation === 'outpaint') {
$scenePrompt = trim((string) ($editContext['outpaint_scene_prompt'] ?? ''));
$scene = $scenePrompt !== ''
? rtrim($scenePrompt, ".。 \t\n\r\0\x0B")
: 'adjacent scenery, matching illumination, perspective, colors, textures, depth, and atmosphere';
return 'OUTPAINT_FULL_BLEED: Generate genuinely new full-bleed surroundings beyond every original edge, '
. 'seamlessly continue the ' . $scene . '. Give immediate priority to local edge continuation: '
. 'matching boundary colors, illumination direction, perspective scale, and every structure crossing an edge; '
. 'keep new border content subordinate to the existing composition. Preserve the protected center exactly.';
}
if ($operation !== 'remove_watermark') {
return $prompt;
}
return 'Remove every visible watermark inside the white masked region. Fill it only with a seamless '
. 'continuation of the immediately adjacent background texture, colors, lighting, and edges. '
. 'The masked region must contain no new object, panel, interface, symbol, character, number, or letter. '
. 'Do not recreate, replace, imitate, or add any watermark, logo, signature, label, or corner text. Preserve the existing title, '
. 'author name, typography, characters, composition, and every area outside the mask exactly.';
}
/**
* Build a concise SDXL scene description from the actual source image.
* Failure is deliberately non-fatal: outpainting still works with the
* structural fallback prompt when no vision model is available.
*/
private function describeOutpaintScene(string $sourcePath, int $userId): string
{
try {
$visionModel = AiModel::where('enabled', 1)
->where('provider', '<>', 'comfy')
->where('support_image', 1)
->order('is_default', 'desc')
->order('sort_order')
->find();
if (!$visionModel || !is_file($sourcePath)) {
return '';
}
$instruction = 'Analyze only the attached image. Return exactly six short comma-separated English fragments, each two to five words, describing the environment and its actual boundary content for SDXL outpainting. '
. 'Use this order: broad style and environment, left-edge colors and scenery, right-edge colors and scenery, top-edge colors and scenery, bottom-edge colors and scenery, existing illumination direction and perspective. Preserve the images actual dominant colors and describe only features already present at those edges. '
. 'Strictly omit people, characters, text, letters, numbers, typography, books, posters, panels, screens, interfaces, holograms, data, logos, watermarks, and blank padding. '
. 'Use simple concrete visual terms. No sentence, instructions, labels, markdown, or explanation.';
$provider = (string) ($visionModel->provider ?? 'openai');
$answer = '';
if ($provider === 'dify') {
$mime = @mime_content_type($sourcePath) ?: 'image/png';
$difyUserId = 'outpaint-caption-' . $userId . '-' . bin2hex(random_bytes(4));
[$fileId, $uploadError] = DifyService::uploadFileWithDetail(
$visionModel,
$sourcePath,
$mime,
basename($sourcePath),
$difyUserId
);
if (!$fileId) {
throw new \RuntimeException('视觉描述图片上传失败' . ($uploadError ? '' . $uploadError : ''));
}
$result = DifyService::chat(
$visionModel,
$instruction,
[[
'type' => 'image',
'transfer_method' => 'local_file',
'upload_file_id' => $fileId,
]],
null,
$difyUserId
);
$answer = (string) ($result['answer'] ?? '');
} else {
$data = @file_get_contents($sourcePath);
if ($data === false) {
return '';
}
$mime = @mime_content_type($sourcePath) ?: 'image/png';
$result = OpenAIService::chat($visionModel, [[
'role' => 'user',
'content' => [
['type' => 'text', 'text' => $instruction],
[
'type' => 'image_url',
'image_url' => ['url' => 'data:' . $mime . ';base64,' . base64_encode($data)],
],
],
]]);
$answer = OpenAIService::extractMessageContent($result);
}
$answer = strip_tags($answer);
$answer = preg_replace('/[`*_#>]+/u', ' ', $answer) ?? $answer;
$answer = preg_replace('/\s+/u', ' ', $answer) ?? $answer;
$answer = trim($answer, " \t\n\r\0\x0B\"'");
return mb_substr($answer, 0, 500);
} catch (\Throwable $exception) {
Log::warning('Outpaint scene caption failed: ' . $exception->getMessage());
return '';
}
}
private function latestAgentImageEditContext(int $conversationId, int $userId): ?array
{
$message = Message::where('conversation_id', $conversationId)
->where('role', 'user')
->field('content,attachments')
->order('id', 'desc')
->find();
if (!$message) {
return null;
}
$content = trim((string) $message->content);
if (!AgentCatalog::requestsImageEditing($content)
&& !AgentCatalog::requestsImageRevision($content)) {
return null;
}
$attachments = $message->attachments ?? [];
if (is_string($attachments)) {
$attachments = json_decode($attachments, true) ?: [];
}
if (!ComfyUIService::hasImageAttachments($attachments)) {
if (AgentCatalog::requestsWatermarkRemoval($content)) {
$attachments = $this->latestUserImageAttachments($conversationId, 1);
}
}
if (!ComfyUIService::hasImageAttachments($attachments)) {
$attachments = array_slice(
$this->latestAssistantImageAttachments($conversationId, 1, false),
0,
1
);
}
if (!$attachments) {
return null;
}
return $this->buildComfyEditContext($attachments, $content, $userId);
}
/**
* Dify 使用完全不同的接口协议(POST {base}/chat-messages,而不是 OpenAI 的
* /chat/completions),且由 Dify 自己维护会话上下文,因此单独走一条处理流程。
*/
private function handleDifyCompletion(
ConversationModel $conversation,
AiModel $model,
string $content,
array $attachments,
bool $stream,
array $user,
?array $agent = null
) {
$conversationId = (int) $conversation->id;
$difyUserId = 'user-' . $user['id'];
$usesGeneratedImageContext = false;
$generatedImageState = '';
$hasCurrentImages = $this->hasImageAttachments($attachments);
$canUseImages = SettingsService::isFeatureEnabled('image') && (bool) ($model->support_image ?? true);
$agentHistory = $agent ? $this->recentAgentHistory($conversationId) : [];
$imageTurn = $agent
? $this->resolveAgentImageTurn($content, $agentHistory)
: $this->emptyAgentImageTurn();
$explicitImageRequest = $imageTurn['explicit'];
$contextualImageRequest = $imageTurn['contextual'];
$contextualImageSubject = $imageTurn['subject'];
$revisionImageRequest = $imageTurn['revision'];
$uploadedImageEdit = $imageTurn['uploaded_edit'];
$plannedImageRequest = $imageTurn['planned'];
$independentImageRequest = $imageTurn['independent'];
$agentImageActionAllowed = $imageTurn['allowed'];
$referencesImage = !$independentImageRequest
&& ($this->referencesConversationImage($content) || $revisionImageRequest);
if (!$hasCurrentImages && $referencesImage) {
$generatedImages = $canUseImages
? $this->latestAssistantImageAttachments($conversationId, 4, false)
: [];
if ($generatedImages) {
$attachments = array_merge($attachments, $generatedImages);
$generatedImageState = $this->generationStateFromAttachments($generatedImages);
$usesGeneratedImageContext = true;
}
if ($revisionImageRequest && $generatedImageState === '') {
$generatedImageState = $this->latestAssistantGenerationState($conversationId, false);
$usesGeneratedImageContext = $generatedImageState !== '';
}
}
$noteSuffix = '';
$files = $this->buildDifyFiles($attachments, $model, $difyUserId, $noteSuffix);
// 文档优先走 Dify files;仅当未能成功传给 Dify 时,才本地提取文字兜底
$docContext = '';
$documentSent = !empty(array_filter($files, fn ($f) => ($f['type'] ?? '') === 'document'));
$hasDocuments = !empty(array_filter($attachments, fn ($a) => ($a['type'] ?? '') === 'document'));
if ($hasDocuments && !$documentSent) {
$docContext = $this->buildDifyDocumentContext($attachments);
}
$hasImages = !empty(array_filter($attachments, fn ($a) => ($a['type'] ?? '') === 'image'));
$query = $this->buildDifyQuery(
$content,
$noteSuffix,
$hasImages,
$hasDocuments,
!empty($files),
$docContext,
$documentSent,
$usesGeneratedImageContext
);
if ($agent) {
$agentSystemPrompt = $agent['system_prompt'];
if ($agentImageActionAllowed) {
$agentSystemPrompt .= "\n\n【系统路由判定】本轮是"
. ($contextualImageRequest
? '结合最近对话识别出的隐含视觉展示请求。必须从前文提取用户真正想看的主体和视觉特征,直接生成图片,不得继续文字讲解或声称无法展示。'
: ($revisionImageRequest
? '对当前生成图片的修改请求。必须基于上一张有效图片状态执行调整。'
: ($plannedImageRequest
? '对最近图片方案的明确确认。必须按已确认方案直接生成图片。'
: '明确的图片生成请求。')))
. ($contextualImageSubject !== ''
? '本轮代词或省略表达所指的核心主体已确定为“' . $contextualImageSubject . '”。prompt 必须明确描绘该主体,禁止把代词本身解释成普通男性、女性或其他类别。'
: '')
. '只输出 generate_image 动作 JSON,不要解释或复述用户命令;'
. ($uploadedImageEdit
? 'prompt 必须是纯英文、完整、可直接编辑图片的视觉描述;必须保留原图中未被用户指定删除的现有文字、标志和排版,只删除用户明确点名的水印或对象,禁止新增文字。'
: 'prompt 的视觉描述必须以英文为主、完整且可直接生图。用户未明确要求画面文字时默认禁字;若用户明确给出标题、书名或画面文字,必须把原文逐字写成 EXACT_VISIBLE_TEXT: <<<原文>>>,禁止翻译、改写,并且不得再追加 no text/no title 等冲突限制。')
. '只保留用户明确要求在画面中可见的内容;生成之前的流程、环节、衔接和操作措辞不得变成画面元素。'
. ($independentImageRequest
? '这是独立的新图,不得继承或混入上一张图片的主体、文字和场景。'
: '这是对当前图片的修改,必须合并当前有效画面状态。');
} else {
$agentSystemPrompt .= "\n\n【系统路由判定】本轮不是图片生成或图片修改请求。"
. '必须针对用户当前问题正常输出文本回答,禁止输出 generate_image 动作,'
. '禁止继承、复用或改写此前任何图片的主体、提示词和场景。'
. '如果当前消息是对上一轮的简短质疑或情绪反馈,应回应质疑并重新核对,不能解释该短语的字面含义。';
}
if ($revisionImageRequest && $usesGeneratedImageContext) {
$iterationContext = $this->buildAgentImageIterationContext(
$conversationId,
$content,
$generatedImageState
);
if ($iterationContext !== '') {
$query = $iterationContext . "\n\n" . $query;
}
}
if ($contextualImageRequest || $plannedImageRequest) {
$recentContext = $this->recentAgentTextContext($conversationId);
if ($recentContext !== '') {
$query = $recentContext . "\n\n" . $query;
}
}
$query = "【Agent 工作方式:{$agent['name']}\n{$agentSystemPrompt}\n\n【用户任务】\n{$query}";
}
$externalConversationId = ($model->support_context ?? true)
? ($conversation->external_conversation_id ?: null)
: null;
if ($stream) {
$this->streamDifyResponse($conversationId, $model, $query, $files, $externalConversationId, $difyUserId, $user, $hasImages, $attachments, $docContext, $agent, $agentImageActionAllowed);
}
return $this->syncDifyResponse($conversationId, $model, $query, $files, $externalConversationId, $difyUserId, $user, $hasImages, $attachments, $docContext, $agent, $agentImageActionAllowed);
}
private function buildDifyQuery(
string $content,
string $noteSuffix,
bool $hasImages,
bool $hasDocuments,
bool $filesSent,
string $docContext = '',
bool $documentSent = false,
bool $usesGeneratedImageContext = false
): string {
$content = trim($content);
$parts = [];
if ($docContext !== '') {
if (str_starts_with($docContext, '【文档:')) {
$parts[] = "【系统说明】以下是用户上传文档的提取正文,请直接基于正文内容回答,不要回复「请联系人工客服」等兜底话术。\n\n" . $docContext;
} else {
$parts[] = $docContext;
}
}
if ($hasImages && $filesSent && !$hasDocuments) {
if ($usesGeneratedImageContext) {
$parts[] = $content === ''
? '以下附件是本会话中图片生成模型最近生成的图片,请直接查看并描述图片内容。'
: "以下附件是本会话中图片生成模型最近生成的图片。请结合图片回答用户的问题:{$content}";
} elseif ($content === '') {
$parts[] = '请详细描述用户上传的图片内容,并回答用户可能关心的问题。';
} else {
$parts[] = "用户上传了图片,请结合图片内容回答以下问题:{$content}";
}
} elseif ($hasDocuments && ($documentSent || $docContext !== '')) {
if ($content === '') {
$parts[] = $documentSent
? '请阅读用户上传的文档,概括要点并回答用户可能关心的问题。'
: '请根据以上文档内容概括要点,并回答用户可能关心的问题。';
} else {
$parts[] = $documentSent
? "请结合用户上传的文档回答:{$content}"
: "请根据以上文档内容回答:{$content}";
}
} elseif ($hasImages && $hasDocuments && $filesSent) {
if ($content === '') {
$parts[] = '请结合用户上传的图片和文档内容进行分析并回答。';
} else {
$parts[] = "请结合用户上传的图片和文档回答:{$content}";
}
} else {
if ($content !== '') {
$parts[] = $content;
}
if ($noteSuffix !== '') {
$parts[] = $noteSuffix;
}
if (empty($parts)) {
$parts[] = '(用户发送了文件,请查看附件并回答)';
}
}
return trim(implode("\n\n", $parts));
}
private function hasImageAttachments(array $attachments): bool
{
foreach ($attachments as $attachment) {
if (!is_array($attachment)) {
continue;
}
$type = strtolower((string) ($attachment['type'] ?? ''));
$mime = strtolower((string) ($attachment['mime'] ?? ''));
$source = (string) ($attachment['name'] ?? $attachment['url'] ?? '');
if ($type === 'image' || str_starts_with($mime, 'image/') || preg_match('/\.(?:jpe?g|png|gif|webp|bmp)(?:\?.*)?$/i', $source)) {
return true;
}
}
return false;
}
private function referencesConversationImage(string $content): bool
{
$content = trim($content);
if ($content === '') {
return true;
}
return preg_match(
'/图片|图像|照片|相片|画面|上图|这张|这个|那个|这是|它|上面的|刚才|生成的图|识别|改图|重画|image|picture|photo/iu',
$content
) === 1;
}
private function latestAssistantImageAttachments(
int $conversationId,
int $maxImages = 4,
bool $onlyLatestAssistantMessage = false
): array
{
$rows = Message::where('conversation_id', $conversationId)
->where('role', 'assistant')
->field('attachments')
->order('id', 'desc')
->limit(50)
->select();
foreach ($rows as $row) {
$attachments = $row->attachments ?? [];
if (is_string($attachments)) {
$attachments = json_decode($attachments, true) ?: [];
}
if (!is_array($attachments)) {
continue;
}
$images = array_values(array_filter(
$attachments,
fn ($attachment) => is_array($attachment)
&& ($attachment['type'] ?? '') === 'image'
&& !empty($attachment['url'])
));
if ($images) {
return array_slice($images, 0, max(1, $maxImages));
}
if ($onlyLatestAssistantMessage) {
return [];
}
}
return [];
}
private function latestUserImageAttachments(int $conversationId, int $maxImages = 1): array
{
$rows = Message::where('conversation_id', $conversationId)
->where('role', 'user')
->field('attachments')
->order('id', 'desc')
->limit(50)
->select();
foreach ($rows as $row) {
$attachments = $row->attachments ?? [];
if (is_string($attachments)) {
$attachments = json_decode($attachments, true) ?: [];
}
if (!is_array($attachments)) {
continue;
}
$images = array_values(array_filter(
$attachments,
fn ($attachment) => is_array($attachment)
&& ($attachment['type'] ?? '') === 'image'
&& !empty($attachment['url'])
));
if ($images) {
return array_slice($images, 0, max(1, $maxImages));
}
}
return [];
}
private function latestAssistantGenerationState(
int $conversationId,
bool $onlyLatestAssistantMessage = false
): string {
$rows = Message::where('conversation_id', $conversationId)
->where('role', 'assistant')
->field('attachments')
->order('id', 'desc')
->limit(50)
->select();
foreach ($rows as $row) {
$state = $this->generationStateFromAttachments($row->attachments ?? []);
if ($state !== '') {
return $state;
}
if ($onlyLatestAssistantMessage) {
return '';
}
}
return '';
}
private function hasGeneratedImageState($attachments): bool
{
return ComfyUIService::hasImageAttachments($attachments)
|| $this->generationStateFromAttachments($attachments) !== '';
}
private function recentGeneratedImageConversation(int $conversationId): ?array
{
$rows = Message::where('conversation_id', $conversationId)
->field('id,role,content,attachments')
->order('id', 'desc')
->limit(40)
->select()
->toArray();
$generatedImageId = null;
$originalRequest = '';
$currentGenerationPrompt = '';
$fallbackImageId = null;
$fallbackRequest = '';
foreach ($rows as $row) {
if (($row['role'] ?? '') !== 'assistant'
|| !$this->hasGeneratedImageState($row['attachments'] ?? [])) {
continue;
}
if ($currentGenerationPrompt === '') {
$currentGenerationPrompt = $this->generationStateFromAttachments($row['attachments'] ?? []);
}
$imageId = (int) $row['id'];
$candidateRequest = '';
foreach ($rows as $candidate) {
if ((int) ($candidate['id'] ?? 0) < $imageId && ($candidate['role'] ?? '') === 'user') {
$candidateRequest = trim((string) ($candidate['content'] ?? ''));
break;
}
}
if ($candidateRequest === '') {
continue;
}
// Keep the oldest image as a fallback for prompts without an explicit "make an image" phrase.
$fallbackImageId = $imageId;
$fallbackRequest = $candidateRequest;
$isIndependentImageRequest = AgentCatalog::requestsImageGeneration($candidateRequest)
&& !AgentCatalog::requestsImageRevision($candidateRequest)
&& !AgentCatalog::referencesPriorImagePlan($candidateRequest);
if ($isIndependentImageRequest) {
$generatedImageId = $imageId;
$originalRequest = $candidateRequest;
break;
}
}
if (!$generatedImageId && $fallbackImageId && $fallbackRequest !== '') {
$generatedImageId = $fallbackImageId;
$originalRequest = $fallbackRequest;
}
if (!$generatedImageId || $originalRequest === '') {
return null;
}
$messagesAfterImage = array_values(array_filter(
array_reverse($rows),
fn (array $row) => (int) ($row['id'] ?? 0) > $generatedImageId
));
return [
'original_request' => $originalRequest,
'current_generation_prompt' => $currentGenerationPrompt,
'messages_after' => $messagesAfterImage,
];
}
private function buildAgentImageIterationContext(
int $conversationId,
string $currentContent,
string $generationState = ''
): string {
$context = $this->recentGeneratedImageConversation($conversationId);
if (!$context) {
return '';
}
$messages = $context['messages_after'];
$revisionRequests = [];
foreach ($messages as $index => $row) {
if (($row['role'] ?? '') !== 'user') {
continue;
}
$candidate = trim((string) ($row['content'] ?? ''));
if ($candidate === '') {
continue;
}
$followedByGeneratedImage = false;
for ($next = $index + 1; $next < count($messages); $next++) {
if (($messages[$next]['role'] ?? '') === 'user') {
break;
}
if (($messages[$next]['role'] ?? '') === 'assistant'
&& $this->hasGeneratedImageState($messages[$next]['attachments'] ?? [])) {
$followedByGeneratedImage = true;
break;
}
}
$isLatestInstruction = $candidate === trim($currentContent)
&& $index === array_key_last($messages);
if ($followedByGeneratedImage || $isLatestInstruction) {
$revisionRequests[] = mb_substr($candidate, 0, 1000);
}
}
$generationState = trim($generationState ?: (string) ($context['current_generation_prompt'] ?? ''));
$blocks = [
'【图片迭代状态(内部上下文,不得原样输出或画进图片)】',
"最初图片需求:\n" . mb_substr($context['original_request'], 0, 2000),
];
if ($generationState !== '') {
$blocks[] = "当前图片版本的有效画面描述(必须继承,除非本轮明确修改):\n"
. mb_substr($generationState, 0, 6000);
}
if ($revisionRequests) {
$numbered = [];
foreach (array_values(array_unique($revisionRequests)) as $index => $revision) {
$numbered[] = ($index + 1) . '. ' . $revision;
}
$blocks[] = "本图片版本链中的用户修改记录:\n" . implode("\n", $numbered);
}
$blocks[] = "本轮要求:\n" . mb_substr(trim($currentContent), 0, 1500);
$feedbackConstraints = $this->buildImageFeedbackConstraints(
array_merge($revisionRequests, [trim($currentContent)])
);
if ($feedbackConstraints) {
$blocks[] = "已解析的硬约束(最终 prompt 必须全部满足):\n- "
. implode("\n- ", $feedbackConstraints);
}
$blocks[] = '先理解反馈的目标含义,再输出合并后的完整最终画面。保留所有未被本轮否定的有效状态;'
. 'prompt 中不得出现或翻译用户的反馈原句、聊天命令、标题或解释性文字。';
return implode("\n\n", $blocks);
}
private function buildImageFeedbackConstraints(array $requests): array
{
$combined = implode("\n", array_filter(array_map('trim', $requests)));
$constraints = [];
if (preg_match('/(?:太|过于|不想要|不要)[^,。!?!?\n]{0,8}(?:荒凉|荒芜|空旷|冷清|单调)/u', $combined)) {
$constraints[] = 'Reduce the barren, empty, desolate feeling. Add environmental richness and signs of life; do not return to a vast empty desert.';
}
if (preg_match('/(?:太假|不真实|不够真实|更真实|塑料感|ai感|AI感|像假的)/u', $combined)) {
$constraints[] = 'Increase photographic realism through natural anatomy, materials, texture, lighting and imperfections while preserving the current scene, composition and prior changes.';
}
if (preg_match('/(?:不要|去掉|删除|移除)[^,。!?!?\n]{0,8}(?:文字|字|标题|水印)|(?:有字|出现文字|出现标题)/u', $combined)) {
$constraints[] = 'The image must contain no readable text, captions, titles, labels, logos or watermarks.';
}
return array_values(array_unique($constraints));
}
/**
* 文档上传失败时的兜底:本地提取 PDF/Word 文字注入 query。
*/
private function buildDifyDocumentContext(array $attachments): string
{
$blocks = [];
foreach ($attachments as $att) {
if (($att['type'] ?? '') !== 'document') {
continue;
}
$name = $att['name'] ?? '文档';
$path = $this->resolveUploadPath($att['url'] ?? '');
if (!$path) {
$blocks[] = "[文档「{$name}」在服务器上找不到,无法读取]";
continue;
}
$text = DocumentTextService::extract($path, $att['mime'] ?? '', $name);
if ($text === null || $text === '') {
$reason = DocumentTextService::unsupportedReason($name, $att['mime'] ?? '');
$blocks[] = "[文档「{$name}{$reason}]";
continue;
}
$blocks[] = "【文档:{$name}\n{$text}";
}
return implode("\n\n", $blocks);
}
private function streamDifyResponse(
int $conversationId,
AiModel $model,
string $query,
array $files,
?string $externalConversationId,
string $difyUserId,
array $user,
bool $hasImages = false,
array $attachments = [],
string $docContext = '',
?array $agent = null,
bool $allowAgentImageAction = false
): void {
while (ob_get_level() > 0) {
ob_end_clean();
}
OpenAIService::sseHeaders();
$fullContent = '';
$streamError = null;
$finalTokens = 0;
$resolvedExternalConversationId = $externalConversationId;
$agentStreamBuffer = '';
$agentTextStreaming = $agent !== null;
$replaceAgentStream = false;
DifyService::streamChat(
$model,
$query,
$files,
$externalConversationId,
$difyUserId,
function ($delta) use (
&$fullContent,
$agent,
$agentTextStreaming,
&$agentStreamBuffer
) {
$fullContent .= $delta;
if (!$agent) {
OpenAIService::sseEvent('message', ['content' => $delta]);
} elseif ($agentTextStreaming) {
$agentStreamBuffer .= $delta;
$safeContent = $this->takeSafeAgentStreamPrefix($agentStreamBuffer);
if ($safeContent !== '') {
OpenAIService::sseEvent('message', ['content' => $safeContent]);
}
}
},
function (?string $newConversationId, int $tokens) use ($conversationId, &$finalTokens, &$resolvedExternalConversationId) {
$finalTokens = $tokens;
if ($newConversationId) {
$resolvedExternalConversationId = $newConversationId;
ConversationModel::where('id', $conversationId)->update([
'external_conversation_id' => $newConversationId,
]);
}
},
function (string $message) use (&$streamError) {
$streamError = $message;
},
function () use ($conversationId, &$resolvedExternalConversationId) {
$resolvedExternalConversationId = null;
ConversationModel::where('id', $conversationId)->update([
'external_conversation_id' => null,
]);
}
);
if ($streamError) {
OpenAIService::sseEvent('error', ['message' => $streamError]);
exit;
}
if ($fullContent === '') {
OpenAIService::sseEvent('error', ['message' => 'AI 未返回内容,请检查 Dify 应用配置和 API Key']);
exit;
}
$imageAction = $agent
? $this->parseAgentImageActionForTurn($fullContent, $allowAgentImageAction)
: null;
if ($imageAction) {
if ($allowAgentImageAction) {
$this->streamAgentImageAction($conversationId, $imageAction['prompt'], $user, $agent);
}
$repair = $this->recoverDifyAgentTextResponse(
$model,
$query,
$files,
$resolvedExternalConversationId,
$difyUserId
);
$fullContent = $repair['content'];
$replaceAgentStream = $agentTextStreaming;
$finalTokens += $repair['tokens'];
if ($repair['conversation_id'] !== '') {
ConversationModel::where('id', $conversationId)->update([
'external_conversation_id' => $repair['conversation_id'],
]);
}
} elseif ($agent && $this->shouldRecoverAllowedAgentImageAction($allowAgentImageAction, $imageAction)) {
// The deterministic router already approved an image action. Recover malformed
// model output instead of leaking explanations or raw action JSON to the user.
$repair = $this->recoverDifyAgentImageAction(
$model,
$query,
$files,
$resolvedExternalConversationId,
$difyUserId
);
$finalTokens += $repair['tokens'];
if ($repair['action']) {
$this->streamAgentImageAction(
$conversationId,
$repair['action']['prompt'],
$user,
$agent
);
}
$fullContent = '图片动作生成失败,请重新描述需要处理的画面。';
$replaceAgentStream = $agentTextStreaming;
} elseif ($agent && $this->containsFabricatedAgentImage($fullContent)) {
$repair = $this->recoverDifyAgentTextResponse(
$model,
$query,
$files,
$resolvedExternalConversationId,
$difyUserId
);
$fullContent = $repair['content'];
$replaceAgentStream = $agentTextStreaming;
$finalTokens += $repair['tokens'];
}
$hint = $this->buildDifyFallbackHint($fullContent, $files, $hasImages, $attachments, $docContext);
$hint .= $this->buildDocumentExtractHint($docContext);
if ($hint !== '') {
$fullContent .= $hint;
}
if ($agent && $replaceAgentStream) {
$agentStreamBuffer = '';
OpenAIService::sseEvent('replace', ['content' => $fullContent]);
} elseif ($agent) {
$tail = $this->takeSafeAgentStreamPrefix($agentStreamBuffer, true);
if ($tail !== '') {
OpenAIService::sseEvent('message', ['content' => $tail]);
}
if ($hint !== '') {
OpenAIService::sseEvent('message', ['content' => $hint]);
}
} elseif ($hint !== '') {
OpenAIService::sseEvent('message', ['content' => $hint]);
}
Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $fullContent,
'content_type' => 'markdown',
'tokens_used' => $finalTokens,
]);
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
OpenAIService::sseEvent('done', ['content' => $fullContent]);
exit;
}
private function syncDifyResponse(
int $conversationId,
AiModel $model,
string $query,
array $files,
?string $externalConversationId,
string $difyUserId,
array $user,
bool $hasImages = false,
array $attachments = [],
string $docContext = '',
?array $agent = null,
bool $allowAgentImageAction = false
) {
$result = DifyService::chat($model, $query, $files, $externalConversationId, $difyUserId);
$content = trim($result['answer'] ?? '');
if ($content === '') {
return $this->error('AI 未返回内容,请检查 Dify 应用配置', 502);
}
$resolvedExternalConversationId = (string) ($result['conversation_id'] ?? $externalConversationId ?? '');
if ($resolvedExternalConversationId !== '') {
ConversationModel::where('id', $conversationId)->update([
'external_conversation_id' => $resolvedExternalConversationId,
]);
} elseif (!empty($result['conversation_reset'])) {
ConversationModel::where('id', $conversationId)->update([
'external_conversation_id' => null,
]);
}
$imageAction = $agent
? $this->parseAgentImageActionForTurn($content, $allowAgentImageAction)
: null;
if ($imageAction) {
if ($allowAgentImageAction) {
return $this->syncAgentImageAction($conversationId, $imageAction['prompt'], $user, $agent);
}
$repair = $this->recoverDifyAgentTextResponse(
$model,
$query,
$files,
$resolvedExternalConversationId !== '' ? $resolvedExternalConversationId : null,
$difyUserId
);
$content = $repair['content'];
$result['tokens'] = (int) ($result['tokens'] ?? 0) + $repair['tokens'];
if ($repair['conversation_id'] !== '') {
ConversationModel::where('id', $conversationId)->update([
'external_conversation_id' => $repair['conversation_id'],
]);
}
} elseif ($agent && $this->shouldRecoverAllowedAgentImageAction($allowAgentImageAction, $imageAction)) {
$repair = $this->recoverDifyAgentImageAction(
$model,
$query,
$files,
$resolvedExternalConversationId !== '' ? $resolvedExternalConversationId : null,
$difyUserId
);
$result['tokens'] = (int) ($result['tokens'] ?? 0) + $repair['tokens'];
if ($repair['action']) {
return $this->syncAgentImageAction(
$conversationId,
$repair['action']['prompt'],
$user,
$agent
);
}
$content = '图片动作生成失败,请重新描述需要处理的画面。';
} elseif ($agent && $this->containsFabricatedAgentImage($content)) {
$repair = $this->recoverDifyAgentTextResponse(
$model,
$query,
$files,
$resolvedExternalConversationId !== '' ? $resolvedExternalConversationId : null,
$difyUserId
);
$content = $repair['content'];
$result['tokens'] = (int) ($result['tokens'] ?? 0) + $repair['tokens'];
}
$content .= $this->buildDifyFallbackHint($content, $files, $hasImages, $attachments, $docContext);
$content .= $this->buildDocumentExtractHint($docContext);
$tokens = $result['tokens'] ?? 0;
Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $content,
'content_type' => 'markdown',
'tokens_used' => $tokens,
]);
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
return $this->success(['content' => $content, 'tokens' => $tokens]);
}
/**
* 把图片/文档附件转发到 Dify chat-messages 的 files 参数。
* - 图片:优先 remote_url(公网可达时),否则 local_file
* - 文档:走 type=document + local_file(避免 Docker 拉不到本机 URL
*/
private function buildDifyFiles(array $attachments, AiModel $model, string $difyUserId, string &$noteSuffix): array
{
$files = [];
$skippedImageNames = [];
$skippedDocNames = [];
$failedNames = [];
$allowImages = SettingsService::isFeatureEnabled('image') && (bool) ($model->support_image ?? true);
$allowDocuments = SettingsService::isFeatureEnabled('document');
foreach ($attachments as $att) {
$attType = $att['type'] ?? '';
if (!in_array($attType, ['image', 'document'], true)) {
continue;
}
if ($attType === 'image' && !$allowImages) {
$skippedImageNames[] = $att['name'] ?? '图片';
continue;
}
if ($attType === 'document' && !$allowDocuments) {
$skippedDocNames[] = $att['name'] ?? '文档';
continue;
}
$path = $this->resolveUploadPath($att['url'] ?? '');
$name = $att['name'] ?? '文件';
if (!$path) {
$failedNames[] = $name . '(服务器找不到文件)';
continue;
}
$mime = $att['mime'] ?? '';
if (!$mime || $mime === 'application/octet-stream') {
$mime = @mime_content_type($path) ?: 'application/octet-stream';
}
$difyType = $this->mapDifyFileType($attType, $mime, $name);
// 文档统一 local_file;图片仅在公网可达时用 remote_url
if ($difyType === 'image') {
$publicUrl = $this->publicUploadUrl($att['url'] ?? '');
if ($publicUrl !== ''
&& $this->isUrlReachableByDify($publicUrl)
&& $this->urlActuallyServesFile($publicUrl)
) {
$files[] = [
'type' => 'image',
'transfer_method' => 'remote_url',
'url' => $publicUrl,
];
continue;
}
}
[$fileId, $uploadError] = DifyService::uploadFileWithDetail($model, $path, $mime, $name, $difyUserId);
if ($fileId) {
$files[] = [
'type' => $difyType,
'transfer_method' => 'local_file',
'upload_file_id' => $fileId,
];
continue;
}
$failedNames[] = $name . 'Dify 上传失败' . ($uploadError ? '' . $uploadError : '') . '';
}
$notes = [];
if ($skippedImageNames) {
$notes[] = '[用户发送了图片:' . implode('、', $skippedImageNames) . ',但当前模型不支持图片识别]';
}
if ($skippedDocNames) {
$notes[] = '[用户发送了文档:' . implode('、', $skippedDocNames) . ',但文档功能未开启]';
}
if ($failedNames) {
$notes[] = '[用户发送了文件:' . implode('、', $failedNames) . ',但上传到 AI 平台失败]';
}
$noteSuffix = implode("\n", $notes);
return $files;
}
/**
* 映射为本系统附件类型对应的 Dify files.type
*/
private function mapDifyFileType(string $attType, string $mime, string $filename): string
{
if ($attType === 'image' || str_starts_with($mime, 'image/')) {
return 'image';
}
$ext = strtolower(pathinfo($filename, PATHINFO_EXTENSION));
if (in_array($ext, ['mp3', 'wav', 'm4a', 'ogg', 'amr', 'mpga'], true) || str_starts_with($mime, 'audio/')) {
return 'audio';
}
if (in_array($ext, ['mp4', 'mov', 'mpeg', 'webm'], true) || str_starts_with($mime, 'video/')) {
return 'video';
}
// pdf / doc / docx / txt / md 等
return 'document';
}
/**
* Dify 知识库 Chatbot 在无法匹配文档时常返回固定兜底话术;若用户发了图却收到这类回复,追加配置提示。
*/
private function buildDifyFallbackHint(string $answer, array $files, bool $hasImages, array $attachments, string $docContext = ''): string
{
if (empty($attachments)) {
return '';
}
$patterns = ['无法提供确切答案', '联系人工客服', '建议您联系人工'];
$matched = false;
foreach ($patterns as $pattern) {
if (str_contains($answer, $pattern)) {
$matched = true;
break;
}
}
if (!$matched) {
return '';
}
$hasDocuments = !empty(array_filter($attachments, fn ($a) => ($a['type'] ?? '') === 'document'));
$docExtracted = $docContext !== '' && str_contains($docContext, '【文档:');
$documentInFiles = !empty(array_filter($files, fn ($f) => ($f['type'] ?? '') === 'document'));
if ($hasDocuments) {
if ($documentInFiles) {
return "\n\n---\n**系统提示**:文档已传给 Dify,但仍返回知识库兜底回复。"
. " 请在 Dify 应用中开启「文件上传 / 文档理解」,并改用支持文档的 Agent 或 Chatflow。";
}
if (!$docExtracted) {
return "\n\n---\n**系统提示**PDF/Word **未能传给 Dify,本地文字提取也失败**。"
. " 请检查 Dify 文件上传接口,或在 PHP 服务器安装 poppler-utils。";
}
return "\n\n---\n**系统提示**:文档未能通过 Dify files 识别,已改用本地提取文字发送,但 Dify 仍返回兜底回复。"
. " 请在 Dify 后台确认应用支持文档上传,并选择支持长文本的模型。";
}
if (!$hasImages) {
return '';
}
if (empty($files)) {
return "\n\n---\n**系统提示**:图片未能传给 Dify(上传失败)。请检查 Chat 服务器能否访问 Dify(`{$this->request->host()}` → Dify API),以及 `/api/uploads/` 图片 URL 是否公网可访问。";
}
return "\n\n---\n**系统提示**:Dify 返回了默认兜底回复,通常表示应用**未开启视觉识图**或使用的是**纯知识库 Chatbot**(只检索文档、看不懂图片)。请在 Dify 后台:① 改用 **Agent 应用**并启用「文件上传 / Vision」;② 选择支持识图的模型(如 GPT-4o、Qwen-VL);③ 在 Dify 编排页用同样图片测试是否正常。";
}
/**
* 文档提取失败时,无论 Dify 说什么都追加明确提示。
*/
private function buildDocumentExtractHint(string $docContext): string
{
if ($docContext === '' || str_contains($docContext, '【文档:')) {
return '';
}
if (!preg_match('/\[文档「[^」]+」(.+?)\]/u', $docContext, $m)) {
return '';
}
return "\n\n---\n**系统提示**" . trim($m[1]);
}
private function streamResponse(
int $conversationId,
$model,
array $apiMessages,
array $user,
?array $agent = null,
bool $allowAgentImageAction = false
): void
{
while (ob_get_level() > 0) {
ob_end_clean();
}
OpenAIService::sseHeaders();
$fullContent = '';
$streamError = null;
$agentStreamBuffer = '';
$agentTextStreaming = $agent !== null;
$replaceAgentStream = false;
OpenAIService::streamChat($model, $apiMessages, function ($chunk) use (
&$fullContent,
$agent,
$agentTextStreaming,
&$agentStreamBuffer
) {
$delta = OpenAIService::extractStreamDelta($chunk);
if ($delta) {
$fullContent .= $delta;
if (!$agent) {
OpenAIService::sseEvent('message', ['content' => $delta]);
} elseif ($agentTextStreaming) {
$agentStreamBuffer .= $delta;
$safeContent = $this->takeSafeAgentStreamPrefix($agentStreamBuffer);
if ($safeContent !== '') {
OpenAIService::sseEvent('message', ['content' => $safeContent]);
}
}
}
}, function (string $message) use (&$streamError) {
$streamError = $message;
});
if ($streamError) {
OpenAIService::sseEvent('error', ['message' => $streamError]);
exit;
}
if ($fullContent === '') {
OpenAIService::sseEvent('error', ['message' => 'AI 未返回内容,请检查模型 API Key 和接口配置']);
exit;
}
$imageAction = $agent
? $this->parseAgentImageActionForTurn($fullContent, $allowAgentImageAction)
: null;
if ($imageAction) {
if ($allowAgentImageAction) {
$this->streamAgentImageAction($conversationId, $imageAction['prompt'], $user, $agent);
}
$fullContent = $this->recoverOpenAiAgentTextResponse($model, $apiMessages);
$replaceAgentStream = $agentTextStreaming;
} elseif ($agent && $this->shouldRecoverAllowedAgentImageAction($allowAgentImageAction, $imageAction)) {
$recoveredAction = $this->recoverOpenAiAgentImageAction($model, $apiMessages);
if ($recoveredAction) {
$this->streamAgentImageAction($conversationId, $recoveredAction['prompt'], $user, $agent);
}
$fullContent = '图片动作生成失败,请重新描述需要处理的画面。';
$replaceAgentStream = $agentTextStreaming;
} elseif ($agent && $this->containsFabricatedAgentImage($fullContent)) {
$fullContent = $this->recoverOpenAiAgentTextResponse($model, $apiMessages);
$replaceAgentStream = $agentTextStreaming;
}
if ($agent && $replaceAgentStream) {
$agentStreamBuffer = '';
OpenAIService::sseEvent('replace', ['content' => $fullContent]);
} elseif ($agent) {
$tail = $this->takeSafeAgentStreamPrefix($agentStreamBuffer, true);
if ($tail !== '') {
OpenAIService::sseEvent('message', ['content' => $tail]);
}
}
Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $fullContent,
'content_type' => 'markdown',
]);
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
OpenAIService::sseEvent('done', ['content' => $fullContent]);
exit;
}
private function syncResponse(
int $conversationId,
$model,
array $apiMessages,
array $user,
?array $agent = null,
bool $allowAgentImageAction = false
)
{
$result = OpenAIService::chat($model, $apiMessages);
$content = OpenAIService::extractMessageContent($result);
if ($content === '') {
return $this->error('AI 未返回内容,请检查模型配置', 502);
}
$imageAction = $agent
? $this->parseAgentImageActionForTurn($content, $allowAgentImageAction)
: null;
if ($imageAction) {
if ($allowAgentImageAction) {
return $this->syncAgentImageAction($conversationId, $imageAction['prompt'], $user, $agent);
}
$content = $this->recoverOpenAiAgentTextResponse($model, $apiMessages);
} elseif ($agent && $this->shouldRecoverAllowedAgentImageAction($allowAgentImageAction, $imageAction)) {
$recoveredAction = $this->recoverOpenAiAgentImageAction($model, $apiMessages);
if ($recoveredAction) {
return $this->syncAgentImageAction($conversationId, $recoveredAction['prompt'], $user, $agent);
}
$content = '图片动作生成失败,请重新描述需要处理的画面。';
} elseif ($agent && $this->containsFabricatedAgentImage($content)) {
$content = $this->recoverOpenAiAgentTextResponse($model, $apiMessages);
}
$tokens = $result['usage']['total_tokens'] ?? 0;
Message::create([
'conversation_id' => $conversationId,
'role' => 'assistant',
'content' => $content,
'content_type' => 'markdown',
'tokens_used' => $tokens,
]);
ConversationModel::where('id', $conversationId)->inc('message_count')->update([
'updated_at' => date('Y-m-d H:i:s'),
]);
PermissionService::incrementDailyCount((int) $user['id']);
return $this->success(['content' => $content, 'tokens' => $tokens]);
}
private function streamAgentImageAction(
int $conversationId,
string $actionPrompt,
array $user,
array $agent
): void {
$model = OpenAIService::getImageModel();
$editContext = $this->latestAgentImageEditContext($conversationId, (int) $user['id']);
$prompt = $this->buildAgentActionImagePrompt($conversationId, $actionPrompt, $editContext !== null);
$renderPrompt = $editContext ? $prompt : AgentCatalog::buildImagePrompt($agent, $prompt);
$this->streamComfyResponse(
$conversationId,
$model,
$renderPrompt,
$user,
$renderPrompt,
$editContext
);
}
private function syncAgentImageAction(
int $conversationId,
string $actionPrompt,
array $user,
array $agent
) {
$model = OpenAIService::getImageModel();
$editContext = $this->latestAgentImageEditContext($conversationId, (int) $user['id']);
$prompt = $this->buildAgentActionImagePrompt($conversationId, $actionPrompt, $editContext !== null);
$renderPrompt = $editContext ? $prompt : AgentCatalog::buildImagePrompt($agent, $prompt);
return $this->syncComfyResponse(
$conversationId,
$model,
$renderPrompt,
$user,
$renderPrompt,
$editContext
);
}
private function buildAgentActionImagePrompt(
int $conversationId,
string $actionPrompt,
bool $editingSource = false
): string
{
$actionPrompt = trim($actionPrompt);
$latestUserContent = (string) (Message::where('conversation_id', $conversationId)
->where('role', 'user')
->order('id', 'desc')
->value('content') ?? '');
$actionPrompt = $this->applyRequestedVisibleText($actionPrompt, $latestUserContent);
$exactVisibleText = str_contains($actionPrompt, 'EXACT_VISIBLE_TEXT: <<<');
$guardrail = $editingSource
? 'Preserve all existing text, typography, logos, and design elements unless the user explicitly asked to remove them. Remove only the specified watermark or masked object. Do not add any new text or watermark.'
: ($exactVisibleText
? 'Render only the explicitly marked exact visible text. Keep every character verbatim, in the same order, without translation, substitution, duplication, or additional wording.'
: 'No readable text, no letters, no Chinese characters, no title, no caption, no watermark.');
return mb_substr($actionPrompt, 0, 8000) . "\n\n" . $guardrail;
}
private function applyRequestedVisibleText(string $prompt, string $userContent): string
{
if (str_contains($prompt, 'EXACT_VISIBLE_TEXT: <<<')) {
return $prompt;
}
$visibleText = $this->extractRequestedVisibleText($userContent);
if ($visibleText === null) {
return $prompt;
}
$prompt = preg_replace(
'/\s+(?:titled|with\s+(?:the\s+)?title)\s+[\'\"“][^\'\"”]+[\'\"”]/iu',
'',
$prompt
) ?? $prompt;
$prompt = preg_replace(
'/(?:^|[,.]\s*)no\s+(?:readable\s+)?text\b(?:\s*,\s*no\s+[^,.\n]+)*[.]?/iu',
'',
$prompt
) ?? $prompt;
$prompt = preg_replace('/\s{2,}/u', ' ', trim($prompt)) ?? trim($prompt);
return $prompt . "\n\nEXACT_VISIBLE_TEXT: <<<{$visibleText}>>>. "
. 'Render these exact UTF-8 characters once as the main title. Do not translate, rewrite, substitute, omit, reorder, or duplicate any character.';
}
private function extractRequestedVisibleText(string $content): ?string
{
$content = trim($content);
if ($content === '') {
return null;
}
$candidate = '';
if (preg_match('/(?:标题|书名|主标题|封面文字|画面文字)\s*(?:是|为|用|写|写上||:)\s*[《「“\"]?(.{1,40}?)[》」”\"]?(?=$|[。;;\n])/u', $content, $matches)) {
$candidate = (string) $matches[1];
} elseif (preg_match('/《([^》\n]{1,40})》/u', $content, $matches)
&& preg_match('/(?:封面|海报|画面|图片|图中)/u', $content)) {
$candidate = (string) $matches[1];
} elseif (preg_match('/(?:小说|网文|书籍|图书)?封面\s*[:]\s*(.{1,40})$/u', $content, $matches)) {
$candidate = (string) $matches[1];
}
$candidate = trim($candidate, " \t\n\r\0\x0B《》「」『』“”\"'");
if ($candidate === '' || mb_strlen($candidate) > 32) {
return null;
}
if (preg_match('/(?:人物|角色|站在|坐在|背景|场景|风格|配色|构图|镜头|光线|不要|不能|需要|画面)/u', $candidate)) {
return null;
}
return str_replace(['<<<', '>>>'], '', $candidate);
}
/**
* Stream Agent prose immediately while retaining only a possible image-action
* envelope. This avoids delaying every short answer just to protect JSON routes.
*/
private function takeSafeAgentStreamPrefix(string &$buffer, bool $flush = false): string
{
if ($buffer === '') {
return '';
}
$safe = '';
$actionPattern = '/(?:```(?:json)?\s*)?\{\s*"(?:action|task|tool)"\s*:/iu';
while ($buffer !== '') {
if (preg_match($actionPattern, $buffer, $matches, PREG_OFFSET_CAPTURE)) {
$offset = (int) ($matches[0][1] ?? 0);
$safe .= substr($buffer, 0, $offset);
$buffer = substr($buffer, $offset);
return $safe;
}
if ($flush) {
$safe .= $buffer;
$buffer = '';
return $safe;
}
if (!preg_match('/[{`]/', $buffer, $candidate, PREG_OFFSET_CAPTURE)) {
$safe .= $buffer;
$buffer = '';
return $safe;
}
$offset = (int) ($candidate[0][1] ?? 0);
if ($offset > 0) {
$safe .= substr($buffer, 0, $offset);
$buffer = substr($buffer, $offset);
}
if ($this->isPossibleAgentImageActionPrefix($buffer)) {
return $safe;
}
// The candidate is ordinary prose/JSON/code. Release one character and
// continue scanning so a later action envelope can still be intercepted.
$safe .= substr($buffer, 0, 1);
$buffer = substr($buffer, 1);
}
return $safe;
}
private function isPossibleAgentImageActionPrefix(string $candidate): bool
{
if ($candidate === '') {
return false;
}
if ($candidate[0] === '{') {
$compact = mb_strtolower((string) preg_replace('/\s+/u', '', $candidate));
foreach (['{"action":', '{"task":', '{"tool":'] as $marker) {
if (str_starts_with($marker, $compact)) {
return true;
}
}
return false;
}
$leadingBackticks = strspn($candidate, '`');
if ($leadingBackticks < 3) {
return $leadingBackticks === strlen($candidate);
}
if (!str_starts_with($candidate, '```')) {
return false;
}
$rest = substr($candidate, 3);
if ($rest === '') {
return true;
}
$lowerRest = mb_strtolower($rest);
if (strlen($rest) < 4 && str_starts_with('json', $lowerRest)) {
return true;
}
if (str_starts_with($lowerRest, 'json')) {
$rest = substr($rest, 4);
} elseif (!preg_match('/^\s/u', $rest) && !str_starts_with($rest, '{')) {
return false;
}
$rest = ltrim($rest);
return $rest === '' || $this->isPossibleAgentImageActionPrefix($rest);
}
private function parseAgentImageActionForTurn(
string $content,
bool $allowAgentImageAction
): ?array {
$action = AgentCatalog::parseImageAction($content);
if ($action !== null || !$allowAgentImageAction) {
return $action;
}
return AgentCatalog::parseRoutedImageAction($content);
}
private function shouldRecoverAllowedAgentImageAction(
bool $allowAgentImageAction,
?array $parsedAction
): bool {
return $allowAgentImageAction && $parsedAction === null;
}
private function isIndependentAgentImageRequest(string $content): bool
{
return AgentCatalog::requestsImageGeneration($content)
&& !AgentCatalog::requestsImageRevision($content)
&& !AgentCatalog::referencesPriorImagePlan($content);
}
private function emptyAgentImageTurn(): array
{
return [
'explicit' => false,
'contextual' => false,
'subject' => '',
'revision' => false,
'uploaded_edit' => false,
'planned' => false,
'independent' => false,
'allowed' => false,
];
}
/**
* Resolve the current turn once and reuse the result for prompt routing,
* context selection, and the final action gate.
*/
private function resolveAgentImageTurn(string $content, array $history): array
{
$explicit = AgentCatalog::requestsImageGeneration($content);
$contextual = !$explicit
&& AgentCatalog::requestsContextualImageGeneration($content, $history);
$imageEditing = AgentCatalog::requestsImageEditing($content);
$uploadedEdit = $this->currentUserTurnHasImages($content, $history)
&& ($imageEditing
|| AgentCatalog::requestsImageRevision($content));
$activeGeneratedImage = $this->latestConversationTurnIsGeneratedImage($history);
$recentGeneratedImage = $activeGeneratedImage
|| ($imageEditing && $this->recentConversationHasGeneratedImage($history));
$planned = AgentCatalog::referencesPriorImagePlan($content)
|| AgentCatalog::confirmsContextualImagePlan($content, $history);
$plannedRevision = $planned && $this->recentAssistantTurnContinuesGeneratedImage($history);
$revision = $uploadedEdit
|| ($recentGeneratedImage && (AgentCatalog::requestsImageRevision($content)
|| $imageEditing))
|| $plannedRevision;
$allowed = $explicit || $contextual || $revision || $planned;
return [
'explicit' => $explicit,
'contextual' => $contextual,
'subject' => $contextual
? AgentCatalog::resolveContextualImageSubject($content, $history)
: '',
'revision' => $revision,
'uploaded_edit' => $uploadedEdit,
'planned' => $planned,
'independent' => $contextual
|| ($planned && !$plannedRevision)
|| ($explicit && $this->isIndependentAgentImageRequest($content)),
'allowed' => $allowed,
];
}
private function currentUserTurnHasImages(string $content, array $history): bool
{
$content = mb_strtolower(trim($content));
for ($index = count($history) - 1; $index >= 0; $index--) {
$message = $history[$index];
if (($message['role'] ?? '') !== 'user') {
continue;
}
if (mb_strtolower(trim((string) ($message['content'] ?? ''))) !== $content) {
return false;
}
return ComfyUIService::hasImageAttachments($message['attachments'] ?? []);
}
return false;
}
private function latestConversationTurnIsGeneratedImage(array $history): bool
{
$skippedCurrentUser = false;
for ($index = count($history) - 1; $index >= 0; $index--) {
$message = $history[$index];
if (!$skippedCurrentUser && ($message['role'] ?? '') === 'user') {
$skippedCurrentUser = true;
continue;
}
$content = trim((string) ($message['content'] ?? ''));
$attachments = $message['attachments'] ?? [];
if ($content === '' && empty($attachments)) {
continue;
}
return ($message['role'] ?? '') === 'assistant'
&& $this->hasGeneratedImageState($attachments);
}
return false;
}
/**
* Keep the last valid image alive across a few failed explanatory turns. A
* confirmed edit intent is still required before this state can be used.
*/
private function recentConversationHasGeneratedImage(array $history, int $limit = 8): bool
{
$skippedCurrentUser = false;
$inspected = 0;
for ($index = count($history) - 1; $index >= 0; $index--) {
$message = $history[$index];
if (!$skippedCurrentUser && ($message['role'] ?? '') === 'user') {
$skippedCurrentUser = true;
continue;
}
$content = trim((string) ($message['content'] ?? ''));
$attachments = $message['attachments'] ?? [];
if ($content === '' && empty($attachments)) {
continue;
}
if (($message['role'] ?? '') === 'assistant'
&& $this->hasGeneratedImageState($attachments)) {
return true;
}
$inspected++;
if ($inspected >= $limit) {
break;
}
}
return false;
}
private function recentAssistantTurnContinuesGeneratedImage(array $history): bool
{
$skippedCurrentUser = false;
$assistantIndex = null;
$assistantContent = '';
for ($index = count($history) - 1; $index >= 0; $index--) {
$message = $history[$index];
if (!$skippedCurrentUser && ($message['role'] ?? '') === 'user') {
$skippedCurrentUser = true;
continue;
}
if (($message['role'] ?? '') !== 'assistant') {
continue;
}
$assistantContent = trim((string) ($message['content'] ?? ''));
if ($assistantContent === '') {
continue;
}
$assistantIndex = $index;
break;
}
if ($assistantIndex === null
|| !preg_match(
'/(?:这张图|这幅图|当前图片|当前画面|基于[^。!?!?\n]{0,30}(?:建议|调整)|重新生成一张|调整方向)/u',
$assistantContent
)) {
return false;
}
$minimumIndex = max(0, $assistantIndex - 6);
for ($index = $assistantIndex - 1; $index >= $minimumIndex; $index--) {
$message = $history[$index];
if (($message['role'] ?? '') === 'assistant'
&& $this->hasGeneratedImageState($message['attachments'] ?? [])) {
return true;
}
}
return false;
}
private function recentAgentHistory(int $conversationId): array
{
$history = Message::where('conversation_id', $conversationId)
->field('role,content,attachments')
->order('id', 'desc')
->limit(12)
->select()
->toArray();
return array_reverse($history);
}
private function recoverOpenAiAgentTextResponse($model, array $apiMessages): string
{
array_unshift($apiMessages, [
'role' => 'system',
'content' => '系统已判定当前用户消息不是图片生成或图片修改请求。请忽略此前所有生图动作和图片提示词,只针对最后一条用户消息给出正常、准确的文本回答。禁止输出 JSON 或 generate_image。',
]);
try {
$result = OpenAIService::chat($model, $apiMessages);
$content = trim(OpenAIService::extractMessageContent($result));
if ($content !== '' && AgentCatalog::parseImageAction($content) === null) {
return $content;
}
} catch (\Throwable $exception) {
Log::warning('Agent text recovery failed: ' . $exception->getMessage());
}
return '这条消息应作为普通问答处理,但语言模型连续返回了无效的图片动作。请稍后重试,我不会执行错误的生图操作。';
}
private function recoverOpenAiAgentImageAction($model, array $apiMessages): ?array
{
array_unshift($apiMessages, [
'role' => 'system',
'content' => '系统已确认当前用户消息需要生成或修改图片。禁止提供外部图片链接,禁止声称图片已经生成。只输出一行 generate_image 动作 JSONprompt 必须是完整英文视觉描述。',
]);
try {
$result = OpenAIService::chat($model, $apiMessages);
return AgentCatalog::parseImageAction(OpenAIService::extractMessageContent($result));
} catch (\Throwable $exception) {
Log::warning('Agent image action recovery failed: ' . $exception->getMessage());
return null;
}
}
private function recoverDifyAgentTextResponse(
AiModel $model,
string $query,
array $files,
?string $externalConversationId,
string $difyUserId
): array {
$repairQuery = $query
. "\n\n【系统纠错】当前用户消息不是图片生成或图片修改请求。"
. '忽略此前所有生图动作和图片提示词,只针对当前问题正常输出文本答案。'
. '禁止输出 JSON 或 generate_image。';
try {
$result = DifyService::chat(
$model,
$repairQuery,
$files,
$externalConversationId,
$difyUserId
);
$content = trim((string) ($result['answer'] ?? ''));
if ($content !== '' && AgentCatalog::parseImageAction($content) === null) {
return [
'content' => $content,
'tokens' => (int) ($result['tokens'] ?? 0),
'conversation_id' => (string) ($result['conversation_id'] ?? ''),
];
}
} catch (\Throwable $exception) {
Log::warning('Dify Agent text recovery failed: ' . $exception->getMessage());
}
return [
'content' => '这条消息应作为普通问答处理,但语言模型连续返回了无效的图片动作。请稍后重试,我不会执行错误的生图操作。',
'tokens' => 0,
'conversation_id' => '',
];
}
private function recoverDifyAgentImageAction(
AiModel $model,
string $query,
array $files,
?string $externalConversationId,
string $difyUserId
): array {
$repairQuery = $query
. "\n\n【系统纠错】当前用户消息必须调用图片生成。"
. '禁止提供外部图片链接,禁止声称图片已经生成。'
. '只输出一行 generate_image 动作 JSONprompt 必须是完整英文视觉描述。';
try {
$result = DifyService::chat(
$model,
$repairQuery,
$files,
$externalConversationId,
$difyUserId
);
return [
'action' => AgentCatalog::parseImageAction((string) ($result['answer'] ?? '')),
'tokens' => (int) ($result['tokens'] ?? 0),
'conversation_id' => (string) ($result['conversation_id'] ?? ''),
];
} catch (\Throwable $exception) {
Log::warning('Dify Agent image action recovery failed: ' . $exception->getMessage());
return ['action' => null, 'tokens' => 0, 'conversation_id' => ''];
}
}
private function containsFabricatedAgentImage(string $content): bool
{
$hasRemoteImage = preg_match(
'/!\[[^\]]*\]\(\s*https?:\/\/[^)]+\)|<img\b[^>]*\bsrc\s*=\s*["\']https?:\/\//iu',
$content
) === 1;
if (!$hasRemoteImage) {
return false;
}
return preg_match(
'/(?:根据描述|为你|为您|已经|已|这是一张|以下是|下面是)[^。!?!?\n]{0,30}(?:生成|创作|制作|绘制|图片|图像)/u',
$content
) === 1;
}
private function recentAgentTextContext(int $conversationId): string
{
$history = Message::where('conversation_id', $conversationId)
->field('role,content')
->order('id', 'desc')
->limit(8)
->select()
->toArray();
$lines = [];
foreach (array_reverse($history) as $message) {
$content = trim((string) ($message['content'] ?? ''));
if ($content === '') {
continue;
}
$role = ($message['role'] ?? '') === 'assistant' ? '助手' : '用户';
$lines[] = $role . '' . mb_substr($content, 0, 1200);
}
return $lines
? "【最近对话语境(用于解析省略和指代,不得原样输出)】\n" . implode("\n", $lines)
: '';
}
private function attachGenerationState(array $attachments, string $generationState): array
{
$generationState = mb_substr(trim($generationState), 0, 8000);
if ($generationState === '') {
return $attachments;
}
foreach ($attachments as &$attachment) {
if (is_array($attachment) && ($attachment['type'] ?? '') === 'image') {
$attachment['generation_prompt'] = $generationState;
}
}
unset($attachment);
return $attachments;
}
private function generationStateFromAttachments($attachments): string
{
if (is_string($attachments)) {
$attachments = json_decode($attachments, true) ?: [];
}
if (!is_array($attachments)) {
return '';
}
foreach ($attachments as $attachment) {
if (is_array($attachment)
&& in_array($attachment['type'] ?? '', ['image', ComfyUIService::JOB_TYPE], true)
&& !empty($attachment['generation_prompt'])) {
return trim((string) $attachment['generation_prompt']);
}
}
return '';
}
private function buildApiMessages(array $history, AiModel $model): array
{
$allowImages = SettingsService::isFeatureEnabled('image') && (bool) ($model->support_image ?? true);
$messages = [];
$normalizedHistory = [];
$latestAssistantImageIndex = null;
foreach ($history as $index => $msg) {
$attachments = $msg['attachments'] ?? [];
if (is_string($attachments)) {
$attachments = json_decode($attachments, true) ?: [];
}
if (!is_array($attachments)) {
$attachments = [];
}
$imageAttachments = array_values(array_filter(
$attachments,
fn ($att) => is_array($att) && ($att['type'] ?? '') === 'image'
));
$documentAttachments = array_values(array_filter(
$attachments,
fn ($att) => is_array($att) && ($att['type'] ?? '') === 'document'
));
$normalizedHistory[$index] = [
'role' => $msg['role'],
'content' => (string) ($msg['content'] ?? ''),
'images' => $imageAttachments,
'documents' => $documentAttachments,
];
if (($msg['role'] ?? '') === 'assistant' && $imageAttachments) {
$latestAssistantImageIndex = $index;
}
}
$assistantImageContextIndex = null;
if ($latestAssistantImageIndex !== null && $normalizedHistory) {
$historyKeys = array_keys($normalizedHistory);
$lastHistoryKey = $historyKeys[count($historyKeys) - 1];
$lastMessage = $normalizedHistory[$lastHistoryKey];
$imagePosition = array_search($latestAssistantImageIndex, $historyKeys, true);
$isImmediateFollowUp = $lastMessage['role'] === 'user'
&& $imagePosition !== false
&& $imagePosition === count($historyKeys) - 2;
$referencesImage = $lastMessage['role'] === 'user'
&& $this->referencesConversationImage($lastMessage['content']);
$revisesImmediateImage = $isImmediateFollowUp
&& AgentCatalog::requestsImageRevision($lastMessage['content']);
if (!$lastMessage['images'] && ($referencesImage || $revisesImmediateImage)) {
$assistantImageContextIndex = $latestAssistantImageIndex;
}
}
foreach ($normalizedHistory as $index => $msg) {
$content = $msg['content'];
$imageAttachments = $msg['images'];
$documentAttachments = $msg['documents'];
// Attach only for a direct follow-up or an explicit reference to the latest generated image.
if ($msg['role'] === 'assistant' && $index !== $assistantImageContextIndex) {
$imageAttachments = [];
}
if ($documentAttachments) {
// 暂不支持解析文档内容,先告知模型用户发送了文件,避免模型对附件只字不提
$names = array_map(fn ($att) => $att['name'] ?? '文件', $documentAttachments);
$source = $msg['role'] === 'assistant' ? '助手生成了文件' : '用户发送了文件';
$note = '[' . $source . '' . implode('、', $names) . ',当前暂不支持自动解析文件内容]';
$content = trim($content . "\n" . $note);
}
if ($imageAttachments && !$allowImages) {
// 当前模型不支持图片/多模态输入,仅告知模型用户发送了图片,避免直接把图片传给不支持的接口导致报错
$names = array_map(fn ($att) => $att['name'] ?? '图片', $imageAttachments);
$source = $msg['role'] === 'assistant' ? '图片生成模型生成了图片' : '用户发送了图片';
$note = '[' . $source . '' . implode('、', $names) . ',但当前模型不支持图片识别,无法查看图片内容]';
$content = trim($content . "\n" . $note);
}
if ($imageAttachments && $allowImages) {
$parts = [];
if ($msg['role'] === 'assistant') {
if ($content !== '') {
$messages[] = ['role' => 'assistant', 'content' => $content];
}
$parts[] = [
'type' => 'text',
'text' => '[会话图片上下文:以下图片由图片生成模型在本会话中生成,请在后续问题中直接查看并引用图片内容。]',
];
} elseif ($content !== '') {
$parts[] = ['type' => 'text', 'text' => $content];
}
foreach ($imageAttachments as $att) {
$parts[] = [
'type' => 'image_url',
'image_url' => ['url' => $this->resolveImageUrl($att)],
];
}
$messages[] = [
'role' => $msg['role'] === 'assistant' ? 'user' : $msg['role'],
'content' => $parts,
];
continue;
}
$messages[] = ['role' => $msg['role'], 'content' => $content];
}
return $this->mergeConsecutiveRoles($messages);
}
/**
* 很多严格遵循 OpenAI 协议的模型(尤其是自部署/国产模型)要求消息角色必须
* user/assistant 严格交替出现。历史记录中如果因为之前请求失败等原因导致连续
* 出现同角色消息(例如连续两条 user),直接发给这类模型会报错
* "Conversation roles must alternate user/assistant/...",因此这里做合并兜底。
*/
private function mergeConsecutiveRoles(array $messages): array
{
$merged = [];
foreach ($messages as $msg) {
$lastIndex = count($merged) - 1;
if ($lastIndex >= 0 && $merged[$lastIndex]['role'] === $msg['role']) {
$merged[$lastIndex]['content'] = $this->mergeMessageContent(
$merged[$lastIndex]['content'],
$msg['content']
);
continue;
}
$merged[] = $msg;
}
return $merged;
}
private function mergeMessageContent($a, $b)
{
if (!is_array($a) && !is_array($b)) {
$a = (string) $a;
$b = (string) $b;
if ($a === '') {
return $b;
}
if ($b === '') {
return $a;
}
return $a . "\n\n" . $b;
}
$toParts = function ($val) {
if (is_array($val)) {
return $val;
}
$val = (string) $val;
return $val === '' ? [] : [['type' => 'text', 'text' => $val]];
};
return array_merge($toParts($a), $toParts($b));
}
/**
* 优先把本地上传的图片转成 base64 内嵌到请求中,避免 AI 模型(尤其是部署在其他机器上的模型)
* 反过来去请求一个只有本机才能访问的 URL(如 127.0.0.1)而连接失败。
*/
private function resolveImageUrl(array $att): string
{
$url = $att['url'] ?? '';
$dataUri = $this->attachmentToDataUri($url, $att['mime'] ?? null);
if ($dataUri) {
return $dataUri;
}
return $this->absoluteUrl($url);
}
private function attachmentToDataUri(string $url, ?string $mime): ?string
{
$path = $this->resolveUploadPath($url);
if (!$path || !is_file($path)) {
return null;
}
$data = @file_get_contents($path);
if ($data === false) {
return null;
}
if (!$mime) {
$mime = @mime_content_type($path) ?: 'application/octet-stream';
}
return 'data:' . $mime . ';base64,' . base64_encode($data);
}
private function resolveUploadPath(string $url): ?string
{
return self::resolveStoredPath($url);
}
/**
* 生成供外部系统访问的上传文件绝对地址。
* 优先使用 .env 的 APP_PUBLIC_URLDify 容器需能访问到该地址)。
*/
private function publicUploadUrl(string $url): string
{
if ($url === '') {
return '';
}
if (str_starts_with($url, 'http://') || str_starts_with($url, 'https://')) {
return $url;
}
$base = rtrim((string) env('APP_PUBLIC_URL', ''), '/');
if ($base !== '') {
return $base . (str_starts_with($url, '/') ? $url : '/' . $url);
}
return $this->absoluteUrl($url);
}
private function absoluteUrl(string $url): string
{
if (str_starts_with($url, 'http://') || str_starts_with($url, 'https://')) {
return $url;
}
$scheme = $this->request->scheme();
$host = $this->request->host();
return "{$scheme}://{$host}{$url}";
}
/**
* Dify(常在 Docker)无法访问本机回环地址,此类 URL 应改走 local_file。
*/
private function isUrlReachableByDify(string $url): bool
{
$host = parse_url($url, PHP_URL_HOST);
if (!$host) {
return false;
}
$host = strtolower($host);
if (in_array($host, ['127.0.0.1', 'localhost', '::1', '0.0.0.0'], true)) {
return false;
}
// 常见 Docker 内网主机名仍可能不可达,若配置了 APP_PUBLIC_URL 则信任该域名
$publicBase = rtrim((string) env('APP_PUBLIC_URL', ''), '/');
if ($publicBase !== '') {
$publicHost = strtolower((string) parse_url($publicBase, PHP_URL_HOST));
if ($publicHost !== '' && $host === $publicHost) {
return true;
}
}
// 纯内网 IP:默认仍尝试 remote_url(同网可达);回环已在上面拦截
return true;
}
/**
* 实际请求一次公网 URL,确认能取到文件(200/206)才交给 Dify remote_url。
* 线上常见 404 场景:APP_PUBLIC_URL 指向了别的域名/旧服务器,或 Nginx 静态规则
* 把 /api/uploads/*.png 当静态文件处理绕过了 PHP 路由。校验失败则降级 local_file。
*/
private function urlActuallyServesFile(string $url): bool
{
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 3,
CURLOPT_RANGE => '0-0',
CURLOPT_TIMEOUT => 5,
CURLOPT_CONNECTTIMEOUT => 3,
CURLOPT_SSL_VERIFYPEER => false,
]);
curl_exec($ch);
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($httpCode === 200 || $httpCode === 206) {
return true;
}
Log::warning("Dify remote_url 预检失败(HTTP {$httpCode}):{$url},将改用 local_file 上传");
return false;
}
/**
* 解析 /api/uploads/{filename} 对应的本地磁盘路径
*/
public static function resolveStoredPath(string $url, ?int $userId = null): ?string
{
if ($url === '') {
return null;
}
$filename = basename(parse_url($url, PHP_URL_PATH) ?: $url);
$filename = urldecode($filename);
$storedQuery = UploadFile::where('stored_name', $filename);
if ($userId !== null) {
$storedQuery->where('user_id', $userId);
}
$upload = $storedQuery->find();
if (!$upload) {
$pathQuery = UploadFile::where('file_path', 'like', '%/' . $filename);
if ($userId !== null) {
$pathQuery->where('user_id', $userId);
}
$upload = $pathQuery->find();
}
if (!$upload) {
return null;
}
$path = rtrim(config('upload.path'), '/\\') . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, $upload->file_path);
return is_file($path) ? $path : null;
}
}