gengx
This commit is contained in:
@@ -0,0 +1,308 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use app\model\UploadFile;
|
||||
|
||||
/**
|
||||
* 短剧确定性配音：不再依赖 H3 原生音频猜测语言。
|
||||
*/
|
||||
class VideoDubService
|
||||
{
|
||||
public const LANG_MANDARIN = 'zh-CN';
|
||||
public const LANG_NONE = 'none';
|
||||
public const LANG_NATIVE = 'native';
|
||||
public const AUDIO_CHARACTER = 'character_dialogue';
|
||||
public const AUDIO_NARRATION = 'narration';
|
||||
public const AUDIO_SCENE = 'scene_sound';
|
||||
public const AUDIO_AMBIENT = 'ambient_only';
|
||||
public const AUDIO_POST_TTS = 'post_tts_narration';
|
||||
public const POLICY_VERSION = 'h3-joint-av-v5';
|
||||
|
||||
public static function normalizeLanguage(?string $language): string
|
||||
{
|
||||
return in_array($language, [self::LANG_MANDARIN, self::LANG_NONE, self::LANG_NATIVE], true)
|
||||
? (string) $language
|
||||
: self::LANG_MANDARIN;
|
||||
}
|
||||
|
||||
public static function label(string $language): string
|
||||
{
|
||||
return match (self::normalizeLanguage($language)) {
|
||||
self::LANG_NONE => '无配音',
|
||||
self::LANG_NATIVE => 'H3 原生音轨',
|
||||
default => '普通话（H3 原生口型同步）',
|
||||
};
|
||||
}
|
||||
|
||||
public static function normalizeAudioMode(?string $mode): string
|
||||
{
|
||||
return in_array($mode, [
|
||||
self::AUDIO_CHARACTER,
|
||||
self::AUDIO_NARRATION,
|
||||
self::AUDIO_SCENE,
|
||||
self::AUDIO_AMBIENT,
|
||||
self::AUDIO_POST_TTS,
|
||||
], true) ? (string) $mode : self::AUDIO_AMBIENT;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{id:int,processed:bool,language:string}
|
||||
*/
|
||||
public static function replaceVoice(
|
||||
int $videoUploadId,
|
||||
int $userId,
|
||||
string $dialogue,
|
||||
string $language,
|
||||
int $durationSeconds = 5,
|
||||
string $placement = 'start',
|
||||
array $audioPolicy = []
|
||||
): array {
|
||||
$language = self::normalizeLanguage($language);
|
||||
$audioMode = self::normalizeAudioMode((string) ($audioPolicy['audio_mode'] ?? self::AUDIO_AMBIENT));
|
||||
if ($language === self::LANG_NATIVE) {
|
||||
return [
|
||||
'id' => $videoUploadId,
|
||||
'processed' => false,
|
||||
'language' => $language,
|
||||
'source' => 'h3_native',
|
||||
'policy_version' => self::POLICY_VERSION,
|
||||
];
|
||||
}
|
||||
if ($language === self::LANG_MANDARIN && $audioMode === self::AUDIO_CHARACTER) {
|
||||
// 只有画面角色对白保留 H3 联合采样原声，确保人物性别、声音和口型来自同一次生成。
|
||||
return [
|
||||
'id' => $videoUploadId,
|
||||
'processed' => false,
|
||||
'language' => $language,
|
||||
'source' => 'h3_native',
|
||||
'policy_version' => self::POLICY_VERSION,
|
||||
];
|
||||
}
|
||||
if ($language === self::LANG_MANDARIN && $audioMode === self::AUDIO_SCENE) {
|
||||
// 场景音由 H3 与画面同次联合生成，才能让撞击、脚步、发动机等声音
|
||||
// 精确跟随动作；规划器已明确禁止该类镜头产生任何人物声音。
|
||||
return [
|
||||
'id' => $videoUploadId,
|
||||
'processed' => false,
|
||||
'language' => $language,
|
||||
'source' => 'h3_scene_sound',
|
||||
'policy_version' => self::POLICY_VERSION,
|
||||
];
|
||||
}
|
||||
if ($language === self::LANG_MANDARIN && $audioMode === self::AUDIO_AMBIENT) {
|
||||
// H3 已经在同一次联合采样里生成与画面同步的空间底噪、动作声和
|
||||
// 环境声。旧版用极低音量粉红噪声覆盖它，最终成片约 -71dB，
|
||||
// 听感接近静音；保留原声既有声音，也不会破坏动作同步。
|
||||
return [
|
||||
'id' => $videoUploadId,
|
||||
'processed' => false,
|
||||
'language' => $language,
|
||||
'source' => 'h3_ambient',
|
||||
'policy_version' => self::POLICY_VERSION,
|
||||
];
|
||||
}
|
||||
|
||||
$video = UploadFile::where('id', $videoUploadId)
|
||||
->where('user_id', $userId)
|
||||
->where('file_type', 'video')
|
||||
->find();
|
||||
if (!$video) {
|
||||
throw new \RuntimeException('无法读取待配音视频');
|
||||
}
|
||||
$videoPath = self::uploadPath($video);
|
||||
if (!is_file($videoPath)) {
|
||||
throw new \RuntimeException('待配音视频文件不存在');
|
||||
}
|
||||
|
||||
$durationSeconds = max(1, min(30, $durationSeconds));
|
||||
$pcmPath = null;
|
||||
$speechTempoFilter = '';
|
||||
$voiceDelayMs = 200;
|
||||
if ($language === self::LANG_MANDARIN && $audioMode === self::AUDIO_NARRATION) {
|
||||
$spokenText = self::spokenText($dialogue);
|
||||
if ($spokenText !== '') {
|
||||
$pcmPath = tempnam(sys_get_temp_dir(), 'short_drama_tts_');
|
||||
if ($pcmPath === false) {
|
||||
throw new \RuntimeException('无法创建普通话配音任务');
|
||||
}
|
||||
$pcm = self::synthesizeMandarin($spokenText);
|
||||
if (file_put_contents($pcmPath, $pcm) === false) {
|
||||
@unlink($pcmPath);
|
||||
throw new \RuntimeException('无法保存普通话配音数据');
|
||||
}
|
||||
// 24 kHz、16 bit、单声道 PCM：48000 bytes/s。超过镜头时长时加速，避免截断台词。
|
||||
$pcmDuration = strlen($pcm) / 48000;
|
||||
$targetDuration = max(0.8, $durationSeconds - 0.4);
|
||||
$tempo = max(1.0, $pcmDuration / $targetDuration);
|
||||
$speechTempoFilter = self::tempoFilter($tempo);
|
||||
$speechDuration = $pcmDuration / $tempo;
|
||||
if ($placement === 'end') {
|
||||
$voiceDelayMs = max(200, (int) round(
|
||||
max(0.0, $durationSeconds - $speechDuration - 0.3) * 1000
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$subdir = date('Y/m/d');
|
||||
$storedBase = 'dub_' . str_replace('-', '_', $language) . '_' . uniqid('', true) . '.mp4';
|
||||
$relativePath = $subdir . '/' . $storedBase;
|
||||
$fullDir = rtrim(config('upload.path'), '/\\') . DIRECTORY_SEPARATOR
|
||||
. str_replace('/', DIRECTORY_SEPARATOR, $subdir);
|
||||
if (!is_dir($fullDir) && !mkdir($fullDir, 0755, true) && !is_dir($fullDir)) {
|
||||
if ($pcmPath) {
|
||||
@unlink($pcmPath);
|
||||
}
|
||||
throw new \RuntimeException('无法创建配音视频目录');
|
||||
}
|
||||
$outputPath = $fullDir . DIRECTORY_SEPARATOR . $storedBase;
|
||||
|
||||
try {
|
||||
$command = $audioMode === self::AUDIO_AMBIENT && $language !== self::LANG_NONE
|
||||
? [
|
||||
'ffmpeg', '-y', '-i', $videoPath,
|
||||
'-f', 'lavfi', '-i', 'anoisesrc=color=pink:amplitude=0.006:r=48000',
|
||||
'-filter_complex', '[1:a]highpass=f=80,lowpass=f=4800,volume=0.35,apad,atrim=0:' . $durationSeconds . '[amb]',
|
||||
'-map', '0:v:0', '-map', '[amb]',
|
||||
'-t', (string) $durationSeconds,
|
||||
'-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k',
|
||||
'-movflags', '+faststart', $outputPath,
|
||||
]
|
||||
: ($language === self::LANG_NONE || $pcmPath === null
|
||||
? [
|
||||
'ffmpeg', '-y', '-i', $videoPath,
|
||||
'-f', 'lavfi', '-i', 'anullsrc=r=48000:cl=mono',
|
||||
'-map', '0:v:0', '-map', '1:a:0',
|
||||
'-t', (string) $durationSeconds,
|
||||
'-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k',
|
||||
'-movflags', '+faststart', $outputPath,
|
||||
]
|
||||
: [
|
||||
'ffmpeg', '-y', '-i', $videoPath,
|
||||
'-f', 's16le', '-ar', '24000', '-ac', '1', '-i', (string) $pcmPath,
|
||||
'-filter_complex', "[1:a]aresample=48000{$speechTempoFilter},volume=2.0,alimiter=limit=0.90:level=false,adelay={$voiceDelayMs},apad,atrim=0:{$durationSeconds}[dub]",
|
||||
'-map', '0:v:0', '-map', '[dub]',
|
||||
'-c:v', 'copy', '-c:a', 'aac', '-b:a', '160k',
|
||||
'-movflags', '+faststart', '-shortest', $outputPath,
|
||||
]);
|
||||
[$exitCode, $error] = self::run($command);
|
||||
if ($exitCode !== 0 || !is_file($outputPath) || filesize($outputPath) === 0) {
|
||||
@unlink($outputPath);
|
||||
throw new \RuntimeException('普通话音轨合成失败: ' . mb_substr(trim($error), -500));
|
||||
}
|
||||
} finally {
|
||||
if ($pcmPath) {
|
||||
@unlink($pcmPath);
|
||||
}
|
||||
}
|
||||
|
||||
$upload = UploadFile::create([
|
||||
'user_id' => $userId,
|
||||
'original_name' => 'short_drama_' . $language . '.mp4',
|
||||
'stored_name' => $storedBase,
|
||||
'file_path' => $relativePath,
|
||||
'mime_type' => 'video/mp4',
|
||||
'file_size' => (int) filesize($outputPath),
|
||||
'file_type' => 'video',
|
||||
]);
|
||||
|
||||
return [
|
||||
'id' => (int) $upload->id,
|
||||
'processed' => true,
|
||||
'language' => $language,
|
||||
'source' => match (true) {
|
||||
$language === self::LANG_NONE => 'silence',
|
||||
$audioMode === self::AUDIO_AMBIENT => 'clean_ambient',
|
||||
default => 'cosyvoice',
|
||||
},
|
||||
'policy_version' => self::POLICY_VERSION,
|
||||
];
|
||||
}
|
||||
|
||||
private static function tempoFilter(float $tempo): string
|
||||
{
|
||||
if ($tempo <= 1.0) {
|
||||
return '';
|
||||
}
|
||||
$filters = [];
|
||||
while ($tempo > 2.0) {
|
||||
$filters[] = 'atempo=2.0';
|
||||
$tempo /= 2.0;
|
||||
}
|
||||
$filters[] = 'atempo=' . number_format(max(1.0, $tempo), 4, '.', '');
|
||||
return ',' . implode(',', $filters);
|
||||
}
|
||||
|
||||
private static function synthesizeMandarin(string $text): string
|
||||
{
|
||||
$baseUrl = rtrim((string) config('short_drama.tts_base_url'), '/');
|
||||
$promptWav = (string) config('short_drama.tts_prompt_wav');
|
||||
$promptText = (string) config('short_drama.tts_prompt_text');
|
||||
if ($baseUrl === '' || !is_file($promptWav)) {
|
||||
throw new \RuntimeException('普通话配音服务尚未配置');
|
||||
}
|
||||
|
||||
$ch = curl_init($baseUrl . '/inference_zero_shot');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => [
|
||||
'tts_text' => $text,
|
||||
'prompt_text' => $promptText,
|
||||
'prompt_wav' => new \CURLFile($promptWav, 'audio/wav', 'mandarin_reference.wav'),
|
||||
],
|
||||
CURLOPT_HTTPHEADER => ['Accept: application/octet-stream'],
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 90,
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_SSL_VERIFYPEER => false,
|
||||
]);
|
||||
$pcm = curl_exec($ch);
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$error = curl_error($ch);
|
||||
curl_close($ch);
|
||||
if ($pcm === false || $httpCode < 200 || $httpCode >= 300) {
|
||||
throw new \RuntimeException('普通话配音服务调用失败: ' . ($error ?: 'HTTP ' . $httpCode));
|
||||
}
|
||||
if (strlen((string) $pcm) < 4800) {
|
||||
throw new \RuntimeException('普通话配音服务返回空音频');
|
||||
}
|
||||
if (strlen($pcm) % 2 !== 0) {
|
||||
$pcm = substr($pcm, 0, -1);
|
||||
}
|
||||
return $pcm;
|
||||
}
|
||||
|
||||
private static function spokenText(string $dialogue): string
|
||||
{
|
||||
$dialogue = trim($dialogue);
|
||||
$dialogue = preg_replace('/^[^：:]{1,16}[：:]/u', '', $dialogue) ?? $dialogue;
|
||||
return mb_substr(trim($dialogue), 0, 80);
|
||||
}
|
||||
|
||||
private static function uploadPath(UploadFile $upload): string
|
||||
{
|
||||
return rtrim(config('upload.path'), '/\\') . DIRECTORY_SEPARATOR
|
||||
. str_replace('/', DIRECTORY_SEPARATOR, (string) $upload->file_path);
|
||||
}
|
||||
|
||||
/** @return array{int,string} */
|
||||
private static function run(array $command): array
|
||||
{
|
||||
$pipes = [];
|
||||
$process = @proc_open($command, [
|
||||
0 => ['pipe', 'r'],
|
||||
1 => ['pipe', 'w'],
|
||||
2 => ['pipe', 'w'],
|
||||
], $pipes);
|
||||
if (!is_resource($process)) {
|
||||
throw new \RuntimeException('服务器无法启动 FFmpeg');
|
||||
}
|
||||
fclose($pipes[0]);
|
||||
stream_get_contents($pipes[1]);
|
||||
fclose($pipes[1]);
|
||||
$error = (string) stream_get_contents($pipes[2]);
|
||||
fclose($pipes[2]);
|
||||
return [proc_close($process), $error];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user