diff --git a/video-ai-system/.env.example b/video-ai-system/.env.example index b3e6741..8677503 100644 --- a/video-ai-system/.env.example +++ b/video-ai-system/.env.example @@ -13,10 +13,12 @@ JIMENG_MODEL=doubao-seedance-2-0-260128 # === 云雾AI备用通道（可选）=== YUNWU_API_KEY= -# === 阿里百炼 / 万相（Preview-002 待接入）=== +# === 阿里百炼 / 万相 === ALIYUN_BAILIAN_API_KEY= -ALIYUN_BAILIAN_WORKSPACE_ID= -ALIYUN_BAILIAN_BASE_URL= +ALIYUN_BAILIAN_WORKSPACE_ID= # 可选·新加坡地域时需要 +ALIYUN_BAILIAN_BASE_URL=https://dashscope.aliyuncs.com/api/v1 +WAN_POLL_INTERVAL_MS=15000 # 万相建议15秒轮询 +WAN_MAX_POLL_ATTEMPTS=80 # 最多约20分钟 # === 轮询参数 === POLL_INTERVAL_MS=5000 diff --git a/video-ai-system/MODEL-ROUTER.hdlp b/video-ai-system/MODEL-ROUTER.hdlp index c58a2bc..50d2ed8 100644 --- a/video-ai-system/MODEL-ROUTER.hdlp +++ b/video-ai-system/MODEL-ROUTER.hdlp @@ -42,13 +42,27 @@ preview-001 暴露出一个问题: |-----------|----------|----------|----------| | 火山/Seedance | 已接入 | 中文语义、短镜头、图生视频、3D漫剧镜头 | 仍需首帧/参考图控制一致性 | | 可灵/Kling | 已接入 | 参考图/视频、动作控制、部分镜头动态 | 需要按镜头验证，不能盲抽 | +| 阿里百炼/万相2.7 | ✅ D140已接入 | 文生视频(t2v)、图生视频(i2v·首帧/首尾帧/续写/音频驱动) | 北京地域默认，新加坡需workspaceId | | 本地 FFmpeg/video-editor | 已接入 | 拼接、字幕、基础音频、裁剪 | 缺平面追踪、复杂剪辑语法 | +### 万相2.7 适配器详情 + +``` +适配器: engines/wan-api-adapter.js +密钥: ALIYUN_BAILIAN_API_KEY (.env·不进仓库) +模型T2V: wan2.7-t2v (文生视频) +模型I2V: wan2.7-i2v-2026-04-25 (图生视频) +API: https://dashscope.aliyuncs.com/api/v1 +支持: 首帧/首尾帧/视频续写/音频驱动 + 本地图片自动base64 +测试脚本: tools/test-wan-api.js +D140验证: API连通✅ 任务提交成功 task_id返回正常 +``` + --- ## 本阶段新增接入 -> 只记录官方能力方向。真实接入前必须再查官方文档、价格和密钥条件。 +> 万相2.7 已在 D140 接入完成。以下为其他待评估方向。 | 模型/平台 | 官方能力线索 | 适合评估的任务 | |-----------|--------------|----------------| diff --git a/video-ai-system/engines/audio-stem-splitter.js b/video-ai-system/engines/audio-stem-splitter.js new file mode 100644 index 0000000..99c1d7d --- /dev/null +++ b/video-ai-system/engines/audio-stem-splitter.js @@ -0,0 +1,347 @@ +/** + * 光湖视频AI系统 · 音频分轨引擎 + * D140 · 铸渊 ICE-GL-ZY001 · 2026-06-23 + * + * 将混合音频分离为三轨: 对白(voice) / 背景音乐(bgm) / 音效(sfx) + * 基于 FFmpeg 音频滤波链 + 频率/动态范围分析 + * + * 工作原理: + * 1. 对白轨: 人声频率范围(80Hz-8kHz)带通 + 动态范围压缩 + * 2. BGM轨: 低频段(20-250Hz) + 高频段(8kHz+)的中低能量部分 + * 3. SFX轨: 高频瞬态(打击类) + 非人声中频 + * + * 输出三轨独立WAV，供 video-editor.js 混音使用 + * + * 使用方式: + * const { splitAudioStems, analyzeAudioLevels } = require('./audio-stem-splitter'); + * const stems = await splitAudioStems('input.mp4', './stems/'); + * // stems.voice, stems.bgm, stems.sfx → 独立WAV文件路径 + * + * 铸渊 ICE-GL-ZY001 · D140 + */ + +const { execSync, spawnSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const PYTHON = '/Users/bingshuolingdianyuanhe/.workbuddy/binaries/python/envs/default/bin/python3'; + +// ==================== 核心分轨 ==================== + +/** + * 将音视频文件分离为三轨: 对白 / BGM / 音效 + * + * @param {string} inputPath - 输入文件（视频或音频） + * @param {string} outputDir - 输出目录 + * @param {object} [opts] + * @param {number} [opts.voiceLow=80] - 人声低频截止 Hz + * @param {number} [opts.voiceHigh=8000] - 人声高频截止 Hz + * @param {number} [opts.bgmLow=20] - BGM低频 Hz + * @param {number} [opts.bgmHigh=250] - BGM低频截止 Hz + * @param {number} [opts.sfxThreshold] - SFX瞬态检测阈值(dB) + * @returns {Promise<{voice: string, bgm: string, sfx: string, raw: string, meta: object}>} + */ +async function splitAudioStems(inputPath, outputDir, opts = {}) { + if (!fs.existsSync(inputPath)) { + throw new Error(`输入文件不存在: ${inputPath}`); + } + + // 确保输出目录 + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + const baseName = path.basename(inputPath, path.extname(inputPath)); + const rawWav = path.join(outputDir, `${baseName}_raw.wav`); + const voiceWav = path.join(outputDir, `${baseName}_voice.wav`); + const bgmWav = path.join(outputDir, `${baseName}_bgm.wav`); + const sfxWav = path.join(outputDir, `${baseName}_sfx.wav`); + const metaJson = path.join(outputDir, `${baseName}_stems_meta.json`); + + console.log(`[StemSplitter] 输入: ${path.basename(inputPath)}`); + console.log(`[StemSplitter] 输出: ${outputDir}`); + + // Step 1: 提取原始音频 + console.log('[StemSplitter] 1/4 提取原始音频...'); + execSync( + `ffmpeg -y -i "${inputPath}" -vn -acodec pcm_s16le -ar 48000 -ac 2 "${rawWav}" 2>/dev/null`, + { timeout: 60000 } + ); + + if (!fs.existsSync(rawWav)) { + throw new Error('原始音频提取失败'); + } + + const duration = probeDuration(rawWav); + console.log(` 原始音频: ${duration.toFixed(1)}s`); + + // Step 2: 分离对白轨（人声频段带通 + 动态增强） + console.log('[StemSplitter] 2/4 分离对白轨...'); + extractVoiceTrack(rawWav, voiceWav, opts); + + // Step 3: 分离BGM轨（低频 + 宽频带中低能量） + console.log('[StemSplitter] 3/4 分离BGM轨...'); + extractBgmTrack(rawWav, bgmWav, opts); + + // Step 4: 分离SFX轨（高频瞬态 + 残余中频） + console.log('[StemSplitter] 4/4 分离音效轨...'); + extractSfxTrack(rawWav, sfxWav, voiceWav, bgmWav, opts); + + // 生成元数据 + const meta = { + input: inputPath, + duration: duration, + tracks: { + voice: { file: voiceWav, duration: probeDuration(voiceWav) }, + bgm: { file: bgmWav, duration: probeDuration(bgmWav) }, + sfx: { file: sfxWav, duration: probeDuration(sfxWav) }, + }, + createdAt: new Date().toISOString(), + }; + + fs.writeFileSync(metaJson, JSON.stringify(meta, null, 2)); + + console.log(`[StemSplitter] ✅ 分轨完成`); + console.log(` 对白: ${path.basename(voiceWav)} (${meta.tracks.voice.duration.toFixed(1)}s)`); + console.log(` BGM: ${path.basename(bgmWav)} (${meta.tracks.bgm.duration.toFixed(1)}s)`); + console.log(` 音效: ${path.basename(sfxWav)} (${meta.tracks.sfx.duration.toFixed(1)}s)`); + + return { + voice: voiceWav, + bgm: bgmWav, + sfx: sfxWav, + raw: rawWav, + meta, + }; +} + +// ==================== 对白轨提取 ==================== + +function extractVoiceTrack(inputWav, outputWav, opts = {}) { + const voiceLow = opts.voiceLow || 80; + const voiceHigh = opts.voiceHigh || 8000; + + // 滤波链: + // 1. highpass=f=voiceLow — 去除低频噪声 + // 2. lowpass=f=voiceHigh — 限制人声高频范围 + // 3. compand — 动态范围压缩，增强人声 + // 4. afftdn=nr=10 — 降噪 + const filter = [ + `highpass=f=${voiceLow}`, + `lowpass=f=${voiceHigh}`, + `compand=attacks=0:points=-80/-90|-45/-15|-27/-9|0/-7|20/-7`, + `afftdn=nr=10:nf=-25`, + ].join(','); + + execSync( + `ffmpeg -y -i "${inputWav}" -af "${filter}" -acodec pcm_s16le -ar 48000 -ac 2 "${outputWav}" 2>/dev/null`, + { timeout: 120000 } + ); + + if (!fs.existsSync(outputWav)) { + throw new Error('对白轨提取失败'); + } +} + +// ==================== BGM轨提取 ==================== + +function extractBgmTrack(inputWav, outputWav, opts = {}) { + const bgmLow = opts.bgmLow || 20; + const bgmHigh = opts.bgmHigh || 250; + + // BGM特征: 低频持续 + 宽频带中低能量 + // 滤波链: + // 1. lowpass=f=bgmHigh — 截取低频段（鼓/贝斯/低频氛围） + // 2. bass=g=3 — 增强低频 + // 3. compand — 压缩动态范围，使BGM更平稳 + // 4. volume=0.7 — 稍微降低（混音时通常BGM低于人声） + const filter = [ + `lowpass=f=${bgmHigh}`, + `bass=g=3:f=${bgmLow}:w=80`, + `compand=attacks=1:decays=1:points=-80/-90|-45/-20|0/-12|20/-12`, + `volume=0.7`, + ].join(','); + + execSync( + `ffmpeg -y -i "${inputWav}" -af "${filter}" -acodec pcm_s16le -ar 48000 -ac 2 "${outputWav}" 2>/dev/null`, + { timeout: 120000 } + ); + + if (!fs.existsSync(outputWav)) { + throw new Error('BGM轨提取失败'); + } +} + +// ==================== SFX轨提取 ==================== + +function extractSfxTrack(inputWav, outputWav, voiceWav, bgmWav, opts = {}) { + // SFX = 原始音频 - 对白 - BGM + // 方法: 用 sidechaincompress 或直接用频段提取 + // + // 滤波链: + // 1. highpass=f=2000 — 取高频段（打击/碰撞/环境音效多在高频） + // 2. 用原始音频减去voice和bgm的频段 + // 实际实现: 取高频 + 中频瞬态部分 + + const filter = [ + `highpass=f=1500`, // 高频段 + `treble=g=2:f=4000`, // 高频增强 + `compand=attacks=0:decays=0.3:points=-80/-90|-30/-30|0/-5|20/-5`, // 快速响应瞬态 + `volume=0.8`, + `silenceremove=start_periods=0:start_threshold=-50dB:start_silence=0.1:stop_threshold=-50dB:stop_silence=0.05`, + ].join(','); + + execSync( + `ffmpeg -y -i "${inputWav}" -af "${filter}" -acodec pcm_s16le -ar 48000 -ac 2 "${outputWav}" 2>/dev/null`, + { timeout: 120000 } + ); + + if (!fs.existsSync(outputWav)) { + throw new Error('音效轨提取失败'); + } +} + +// ==================== 音频电平分析 ==================== + +/** + * 分析音频文件的电平信息（RMS、峰值、LUFS近似值） + * 用于自动混音决策 + * + * @param {string} audioPath + * @returns {{rms: number, peak: number, duration: number, channels: number}} + */ +function analyzeAudioLevels(audioPath) { + if (!fs.existsSync(audioPath)) { + throw new Error(`音频文件不存在: ${audioPath}`); + } + + // 使用 ffprobe 获取音频信息 + const probeOut = execSync( + `ffprobe -v quiet -show_entries format=duration:stream=channels,sample_rate -of json "${audioPath}"`, + { encoding: 'utf8', timeout: 5000 } + ); + const probeData = JSON.parse(probeOut); + const duration = parseFloat(probeData.format?.duration || 0); + const channels = probeData.streams?.[0]?.channels || 2; + + // 使用 ffmpeg volumedetect 获取电平 + const volOut = execSync( + `ffmpeg -i "${audioPath}" -af volumedetect -f null /dev/null 2>&1 | grep -E "mean_volume|max_volume"`, + { encoding: 'utf8', timeout: 30000 } + ); + + const meanMatch = volOut.match(/mean_volume:\s*(-?\d+\.?\d*)\s*dB/); + const maxMatch = volOut.match(/max_volume:\s*(-?\d+\.?\d*)\s*dB/); + + const rms = meanMatch ? parseFloat(meanMatch[1]) : -20; + const peak = maxMatch ? parseFloat(maxMatch[1]) : -3; + + return { + rms, // RMS电平 (dB) + peak, // 峰值电平 (dB) + duration, // 时长 (秒) + channels, // 声道数 + sampleRate: probeData.streams?.[0]?.sample_rate || 48000, + }; +} + +// ==================== 自动混音建议 ==================== + +/** + * 根据三轨电平分析，生成混音参数建议 + * + * @param {string} voiceWav + * @param {string} bgmWav + * @param {string} sfxWav + * @returns {{voiceVolume: number, bgmVolume: number, sfxVolume: number, reasoning: string[]}} + */ +function suggestMixLevels(voiceWav, bgmWav, sfxWav) { + const reasoning = []; + + // 分析各轨电平 + const voiceLevel = analyzeAudioLevels(voiceWav); + const bgmLevel = bgmWav && fs.existsSync(bgmWav) ? analyzeAudioLevels(bgmWav) : null; + const sfxLevel = sfxWav && fs.existsSync(sfxWav) ? analyzeAudioLevels(sfxWav) : null; + + // 目标: 对白 > BGM > SFX（一般情况） + // 对白 RMS 目标: -16 ~ -12 dB + // BGM RMS 目标: -24 ~ -20 dB (比对白低8dB左右) + // SFX RMS 目标: -20 ~ -15 dB + + let voiceVolume = 1.0; + let bgmVolume = 0.4; + let sfxVolume = 0.6; + + // 根据对白RMS调整 + const voiceTarget = -14; + if (voiceLevel.rms < voiceTarget - 3) { + voiceVolume = Math.min(1.5, (voiceTarget - voiceLevel.rms) / 6 + 1.0); + reasoning.push(`对白RMS偏低(${voiceLevel.rms}dB)，增益×${voiceVolume.toFixed(2)}`); + } else if (voiceLevel.rms > voiceTarget + 3) { + voiceVolume = Math.max(0.5, 1.0 - (voiceLevel.rms - voiceTarget) / 10); + reasoning.push(`对白RMS偏高(${voiceLevel.rms}dB)，衰减×${voiceVolume.toFixed(2)}`); + } + + // 根据BGM电平调整 + if (bgmLevel) { + const bgmTarget = -22; + const bgmDiff = bgmLevel.rms - bgmTarget; + if (Math.abs(bgmDiff) > 3) { + bgmVolume = Math.max(0.15, Math.min(0.8, 0.4 * Math.pow(10, -bgmDiff / 20))); + reasoning.push(`BGM电平调整: RMS=${bgmLevel.rms}dB → 音量×${bgmVolume.toFixed(2)}`); + } + } + + // 根据SFX电平调整 + if (sfxLevel) { + const sfxTarget = -18; + const sfxDiff = sfxLevel.rms - sfxTarget; + if (Math.abs(sfxDiff) > 3) { + sfxVolume = Math.max(0.2, Math.min(1.0, 0.6 * Math.pow(10, -sfxDiff / 20))); + reasoning.push(`SFX电平调整: RMS=${sfxLevel.rms}dB → 音量×${sfxVolume.toFixed(2)}`); + } + } + + // 确保对白比BGM高至少6dB + if (bgmLevel && voiceLevel.rms + 20 * Math.log10(voiceVolume) < + bgmLevel.rms + 20 * Math.log10(bgmVolume) + 6) { + bgmVolume *= 0.7; + reasoning.push(`对白-BGM间距不足6dB，BGM再降×0.7`); + } + + return { + voiceVolume: Math.round(voiceVolume * 100) / 100, + bgmVolume: Math.round(bgmVolume * 100) / 100, + sfxVolume: Math.round(sfxVolume * 100) / 100, + voiceLevel, + bgmLevel, + sfxLevel, + reasoning, + }; +} + +// ==================== 工具函数 ==================== + +function probeDuration(filePath) { + try { + const out = execSync( + `ffprobe -v quiet -show_entries format=duration -of csv=p=0 "${filePath}"`, + { encoding: 'utf8', timeout: 5000 } + ); + return parseFloat(out.trim()) || 0; + } catch (_) { + return 0; + } +} + +// ==================== 导出 ==================== + +module.exports = { + splitAudioStems, + extractVoiceTrack, + extractBgmTrack, + extractSfxTrack, + analyzeAudioLevels, + suggestMixLevels, + probeDuration, +}; diff --git a/video-ai-system/engines/auto-cut-director.js b/video-ai-system/engines/auto-cut-director.js new file mode 100644 index 0000000..ddb9ba4 --- /dev/null +++ b/video-ai-system/engines/auto-cut-director.js @@ -0,0 +1,405 @@ +/** + * 光湖视频AI系统 · 自动剪辑控制层 + * D140 · 铸渊 ICE-GL-ZY001 · 2026-06-23 + * + * 职责: 解析分镜脚本 → 选镜头 → 生成时间线 → 调度追踪/分轨/剪辑 → 成片 + * + * 这是视频AI系统的"导演"——不直接处理像素和音频， + * 而是协调 planar-tracker / audio-stem-splitter / video-editor 三个引擎。 + * + * 流水线: + * 1. 解析分镜脚本 (HLDP分镜格式 / JSON) + * 2. 对每镜执行 OpenCV 追踪 → 导出运动数据 + * 3. 对源音频执行分轨 → 对白/BGM/SFX三轨 + * 4. 基于脚本+运动数据生成剪辑时间线 + * 5. 调用 video-editor 完成最终合成 + * + * 使用方式: + * const { autoCut } = require('./auto-cut-director'); + * const result = await autoCut({ + * script: './project/storyboard.json', + * shots: './project/shots/', // 镜头素材目录 + * audio: './project/audio.wav', // 可选·源音频 + * output: './output/final.mp4', + * }); + * + * 铸渊 ICE-GL-ZY001 · D140 + */ + +const { execSync, spawnSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const { exec } = require('child_process'); + +// 引入现有引擎 +const videoEditor = require('./video-editor'); +const { splitAudioStems, suggestMixLevels } = require('./audio-stem-splitter'); + +const PYTHON = '/Users/bingshuolingdianyuanhe/.workbuddy/binaries/python/envs/default/bin/python3'; +const TRACKER_SCRIPT = path.resolve(__dirname, 'planar-tracker.py'); + +// 输出路径 +const JZAO_VIDEO_ROOT = '/Volumes/JZAO/铸渊-ICE-GL-ZY001/OUT-输出/视频'; +const DEFAULT_OUTPUT = fs.existsSync(JZAO_VIDEO_ROOT) ? JZAO_VIDEO_ROOT : path.resolve(__dirname, '../outputs'); + +// ==================== 主入口 ==================== + +/** + * 自动剪辑 — 从分镜脚本到成片的完整流水线 + * + * @param {object} opts + * @param {string|object} opts.script - 分镜脚本路径(HLDP/JSON) 或已解析的脚本对象 + * @param {string} opts.shots - 镜头素材目录 + * @param {string} [opts.audio] - 源音频文件（视频的原声） + * @param {string} [opts.voiceover] - 配音文件（外部录制的人声） + * @param {string} [opts.bgm] - BGM文件 + * @param {string} [opts.subtitle] - SRT字幕文件 + * @param {string} [opts.output] - 输出路径 + * @param {object} [opts.resolution] - {w, h} 默认1920x1080 + * @param {number} [opts.fps=24] - 帧率 + * @param {boolean} [opts.stabilize=false] - 是否对每镜做稳定化 + * @param {boolean} [opts.autoMix=true] - 是否自动混音 + * @returns {Promise<{outputPath: string, duration: number, timeline: array, stemsMeta: object}>} + */ +async function autoCut(opts) { + const { + script: scriptInput, + shots: shotsDir, + audio: sourceAudio, + voiceover, + bgm: externalBgm, + subtitle, + output: outputArg, + resolution = { w: 1920, h: 1080 }, + fps = 24, + stabilize = false, + autoMix = true, + } = opts; + + console.log('[Director] ═══ 自动剪辑流水线启动 ═══'); + + // ── Step 0: 解析分镜脚本 ── + console.log('[Director] 0/6 解析分镜脚本...'); + const script = typeof scriptInput === 'string' ? loadScript(scriptInput) : scriptInput; + const timeline_spec = buildTimelineSpec(script); + console.log(`[Director] 分镜数: ${timeline_spec.length}`); + + // ── Step 1: 匹配镜头文件 ── + console.log('[Director] 1/6 匹配镜头素材...'); + const matched = matchShots(timeline_spec, shotsDir); + const missing = matched.filter(m => !m.file); + if (missing.length > 0) { + console.warn(`[Director] ⚠️ ${missing.length}个镜未找到素材:`); + missing.forEach(m => console.warn(` 镜${m.index}: ${m.shotId || m.description}`)); + } + const available = matched.filter(m => m.file); + console.log(`[Director] 可用: ${available.length}/${matched.length}`); + + // ── Step 2: OpenCV追踪（可选·生成运动数据）── + console.log('[Director] 2/6 平面追踪分析...'); + const workDir = path.join(DEFAULT_OUTPUT, `.director-${Date.now()}`); + fs.mkdirSync(workDir, { recursive: true }); + + for (const shot of available) { + if (stabilize) { + console.log(` [Tracker] 稳定化: ${path.basename(shot.file)}`); + const stabilizedPath = path.join(workDir, `stab_${path.basename(shot.file)}`); + await runPython(TRACKER_SCRIPT, ['stabilize', shot.file, '-o', stabilizedPath]); + if (fs.existsSync(stabilizedPath)) { + shot.originalFile = shot.file; + shot.file = stabilizedPath; + } + } + + // 追踪运动数据（用于生成keyframes） + const trackJson = path.join(workDir, `track_${shot.index}.json`); + await runPython(TRACKER_SCRIPT, ['track', shot.file, '-o', trackJson]); + + if (fs.existsSync(trackJson)) { + const motionJson = path.join(workDir, `motion_${shot.index}.json`); + await runPython(TRACKER_SCRIPT, ['motion_data', shot.file, '--track-json', trackJson, '-o', motionJson]); + + if (fs.existsSync(motionJson)) { + const motionData = JSON.parse(fs.readFileSync(motionJson, 'utf8')); + shot.motionData = motionData; + shot.keyframes = motionData.keyframes || {}; + console.log(` [Tracker] 镜${shot.index}: 稳定性=${motionData.stability?.toFixed(2) || '?'} 强度=${motionData.motion_intensity || '?'}`); + } + } + } + + // ── Step 3: 音频分轨（如果有源音频）── + console.log('[Director] 3/6 音频分轨...'); + let stemsMeta = null; + let mixParams = { voiceVolume: 1.0, bgmVolume: 0.3, sfxVolume: 0.5 }; + + if (sourceAudio && fs.existsSync(sourceAudio)) { + const stemsDir = path.join(workDir, 'stems'); + const stems = await splitAudioStems(sourceAudio, stemsDir); + + if (autoMix) { + const suggestion = suggestMixLevels(stems.voice, stems.bgm, stems.sfx); + mixParams = { + voiceVolume: suggestion.voiceVolume, + bgmVolume: suggestion.bgmVolume, + sfxVolume: suggestion.sfxVolume, + }; + console.log(` [Audio] 自动混音: voice=${mixParams.voiceVolume} bgm=${mixParams.bgmVolume} sfx=${mixParams.sfxVolume}`); + suggestion.reasoning.forEach(r => console.log(` → ${r}`)); + } + + stemsMeta = stems; + } + + // ── Step 4: 构建剪辑时间线 ── + console.log('[Director] 4/6 构建剪辑时间线...'); + const timeline = buildEditorTimeline(available, { fps, resolution }); + + // ── Step 5: 构建音频参数 ── + console.log('[Director] 5/6 构建音频参数...'); + const audioConfig = buildAudioConfig({ + voiceover, + sourceVoice: stemsMeta?.voice, + externalBgm, + sourceBgm: stemsMeta?.bgm, + sourceSfx: stemsMeta?.sfx, + mixParams, + }); + + // ── Step 6: 调用 video-editor 合成 ── + console.log('[Director] 6/6 调用剪辑引擎合成...'); + const outputPath = outputArg || path.join(DEFAULT_OUTPUT, `autocut-${Date.now()}.mp4`); + + const editorResult = await videoEditor.edit({ + timeline, + audio: audioConfig, + subtitle, + resolution, + fps, + output: outputPath, + }); + + // 清理工作目录 + try { + fs.rmSync(workDir, { recursive: true }); + } catch (_) {} + + console.log(`[Director] ═══ 自动剪辑完成 ═══`); + console.log(`[Director] 成片: ${editorResult.outputPath}`); + console.log(`[Director] 时长: ${editorResult.duration?.toFixed(1)}s`); + + return { + outputPath: editorResult.outputPath, + duration: editorResult.duration, + size: editorResult.size, + timeline, + stemsMeta, + mixParams, + }; +} + +// ==================== 分镜脚本解析 ==================== + +/** + * 加载分镜脚本 + * 支持: JSON文件 / HLDP格式（简化的key:value解析） + */ +function loadScript(scriptPath) { + if (!fs.existsSync(scriptPath)) { + throw new Error(`分镜脚本不存在: ${scriptPath}`); + } + + const content = fs.readFileSync(scriptPath, 'utf8'); + const ext = path.extname(scriptPath).toLowerCase(); + + if (ext === '.json') { + return JSON.parse(content); + } + + // HLDP/文本格式解析: 每行 "key: value" 或 "field = value" + const lines = content.split('\n'); + const result = { shots: [] }; + let currentShot = null; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#') || trimmed.startsWith('>') || trimmed.startsWith('⊢')) continue; + + // 镜头分隔 + if (trimmed.match(/^(shot|镜|分镜)\s*[:：]/i) || trimmed.match(/^---\s*shot/i)) { + if (currentShot) result.shots.push(currentShot); + currentShot = { index: result.shots.length + 1 }; + continue; + } + + // key: value 格式 + const match = trimmed.match(/^(\w+)\s*[:：=]\s*(.+)$/); + if (match && currentShot) { + const key = match[1].toLowerCase(); + const value = match[2].trim(); + currentShot[key] = value; + } else if (match) { + result[match[1].toLowerCase()] = match[2].trim(); + } + } + + if (currentShot) result.shots.push(currentShot); + return result; +} + +/** + * 从脚本中提取时间线规格 + */ +function buildTimelineSpec(script) { + const shots = script.shots || script.timeline || []; + return shots.map((s, i) => ({ + index: i + 1, + shotId: s.id || s.shotId || s.shot_id || `shot${String(i + 1).padStart(2, '0')}`, + description: s.description || s.desc || s.prompt || '', + duration: parseFloat(s.duration || s.trim || s.dur || 5), + transition: s.transition || 'fade', + file: s.file || s.path || s.video || null, + })); +} + +// ==================== 镜头匹配 ==================== + +/** + * 将时间线规格与素材目录中的文件匹配 + */ +function matchShots(timelineSpec, shotsDir) { + if (!fs.existsSync(shotsDir)) { + console.warn(`[Director] 素材目录不存在: ${shotsDir}`); + return timelineSpec.map(t => ({ ...t, file: null })); + } + + // 列出目录中所有视频文件 + const videoExts = ['.mp4', '.mov', '.avi', '.mkv', '.webm']; + const files = fs.readdirSync(shotsDir) + .filter(f => videoExts.includes(path.extname(f).toLowerCase())) + .sort(); + + return timelineSpec.map(spec => { + let matchedFile = null; + + // 1. 直接指定文件 + if (spec.file && fs.existsSync(path.resolve(shotsDir, spec.file))) { + matchedFile = path.resolve(shotsDir, spec.file); + } + + // 2. 按shotId匹配 + if (!matchedFile && spec.shotId) { + const match = files.find(f => + f.toLowerCase().includes(spec.shotId.toLowerCase()) || + f.toLowerCase().includes(spec.shotId.toLowerCase().replace('shot', '')) + ); + if (match) matchedFile = path.resolve(shotsDir, match); + } + + // 3. 按序号匹配 + if (!matchedFile) { + const numMatch = files.find(f => { + const nums = f.match(/\d+/g); + return nums && nums.some(n => parseInt(n) === spec.index); + }); + if (numMatch) matchedFile = path.resolve(shotsDir, numMatch); + } + + // 4. 按顺序匹配（如果文件数与镜数一致） + if (!matchedFile && spec.index <= files.length) { + matchedFile = path.resolve(shotsDir, files[spec.index - 1]); + } + + return { ...spec, file: matchedFile }; + }); +} + +// ==================== 时间线构建 ==================== + +function buildEditorTimeline(matchedShots, { fps, resolution }) { + return matchedShots + .filter(s => s.file) + .map(s => { + const entry = { + shot: s.file, + trim: s.duration || 5, + transition: s.transition || 'fade', + }; + + // 注入运动数据生成的keyframes + if (s.keyframes && Object.keys(s.keyframes).length > 0) { + entry.keyframes = s.keyframes; + } + + // 如果运动强度高但稳定化未开启，加入轻微缩放补偿 + if (s.motionData && s.motionData.stability < 0.5 && !entry.keyframes) { + entry.keyframes = { + zoom: { from: 1.05, to: 1.05 }, // 固定5%放大，补偿抖动 + }; + } + + return entry; + }); +} + +// ==================== 音频配置构建 ==================== + +function buildAudioConfig({ + voiceover, sourceVoice, externalBgm, sourceBgm, sourceSfx, mixParams, +}) { + const config = { + voice: voiceover || sourceVoice || null, + bgm: externalBgm || sourceBgm || null, + sfx: [], + bgmVolume: mixParams.bgmVolume ?? 0.3, + voiceVolume: mixParams.voiceVolume ?? 1.0, + }; + + // SFX: 如果有分轨的音效轨，作为单个sfx加入 + if (sourceSfx && fs.existsSync(sourceSfx)) { + config.sfx.push({ at: 0, file: sourceSfx }); + } + + // 如果没有任何音频，返回null + if (!config.voice && !config.bgm && config.sfx.length === 0) { + return null; + } + + return config; +} + +// ==================== Python调用工具 ==================== + +function runPython(script, args, timeout = 120000) { + return new Promise((resolve, reject) => { + const cmd = [PYTHON, script, ...args]; + console.log(` [Python] ${path.basename(script)} ${args.join(' ')}`); + + const proc = spawnSync(PYTHON, [script, ...args], { + encoding: 'utf8', + timeout, + cwd: path.dirname(script), + }); + + if (proc.status === 0) { + if (proc.stdout) console.log(proc.stdout.trim()); + resolve(proc.stdout); + } else { + const errMsg = proc.stderr || proc.stdout || `退出码 ${proc.status}`; + console.error(` [Python] ❌ ${errMsg.substring(0, 300)}`); + // 不reject——追踪失败不应该中断整个流水线 + resolve(null); + } + }); +} + +// ==================== 导出 ==================== + +module.exports = { + autoCut, + loadScript, + buildTimelineSpec, + matchShots, + buildEditorTimeline, + buildAudioConfig, +}; diff --git a/video-ai-system/engines/planar-tracker.py b/video-ai-system/engines/planar-tracker.py new file mode 100644 index 0000000..8152300 --- /dev/null +++ b/video-ai-system/engines/planar-tracker.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 +""" +光湖视频AI系统 · OpenCV 平面追踪引擎 +D140 · 铸渊 ICE-GL-ZY001 · 2026-06-23 + +用途: + 1. 特征点追踪 — 追踪画面中物体运动轨迹 + 2. 画面稳定化 — 去除抖动 + 3. 文字/图形贴合 — 把文字/水印/PIN到追踪的平面上 + 4. 运动数据导出 — 供 video-editor.js 的 keyframes 使用 + +调用方式: + python3 planar-tracker.py