feat(video-ai): D140 万相2.7 API + OpenCV追踪 + 音频分轨 + 自动剪辑控制层
Some checks failed
自动更新代码和重启 / update-and-restart (push) Has been cancelled
CI检查 + 自动部署 / check (push) Has been cancelled
CI检查 + 自动部署 / deploy (push) Has been cancelled

新增引擎:
- engines/wan-api-adapter.js — 阿里百炼万相2.7视频生成(文生+图生)
- engines/planar-tracker.py — OpenCV平面追踪(追踪/稳定/文字贴合/运动数据)
- engines/audio-stem-splitter.js — 音频三轨分离(对白/BGM/SFX)+自动混音建议
- engines/auto-cut-director.js — 自动剪辑控制层(脚本解析→追踪→分轨→合成)
- tools/test-wan-api.js — 万相API测试脚本

更新:
- .env.example — 万相配置模板
- MODEL-ROUTER.hdlp — 万相状态: 待接入→已接入

铸渊 ICE-GL-ZY001 · D140 · 2026-06-23
This commit is contained in:
冰朔 2026-06-23 15:22:22 +08:00
parent 2897eab944
commit c077d7d82e
7 changed files with 1944 additions and 4 deletions

View File

@ -13,10 +13,12 @@ JIMENG_MODEL=doubao-seedance-2-0-260128
# === 云雾AI备用通道可选===
YUNWU_API_KEY=
# === 阿里百炼 / 万相Preview-002 待接入)===
# === 阿里百炼 / 万相 ===
ALIYUN_BAILIAN_API_KEY=
ALIYUN_BAILIAN_WORKSPACE_ID=
ALIYUN_BAILIAN_BASE_URL=
ALIYUN_BAILIAN_WORKSPACE_ID= # 可选·新加坡地域时需要
ALIYUN_BAILIAN_BASE_URL=https://dashscope.aliyuncs.com/api/v1
WAN_POLL_INTERVAL_MS=15000 # 万相建议15秒轮询
WAN_MAX_POLL_ATTEMPTS=80 # 最多约20分钟
# === 轮询参数 ===
POLL_INTERVAL_MS=5000

View File

@ -42,13 +42,27 @@ preview-001 暴露出一个问题:
|-----------|----------|----------|----------|
| 火山/Seedance | 已接入 | 中文语义、短镜头、图生视频、3D漫剧镜头 | 仍需首帧/参考图控制一致性 |
| 可灵/Kling | 已接入 | 参考图/视频、动作控制、部分镜头动态 | 需要按镜头验证,不能盲抽 |
| 阿里百炼/万相2.7 | ✅ D140已接入 | 文生视频(t2v)、图生视频(i2v·首帧/首尾帧/续写/音频驱动) | 北京地域默认新加坡需workspaceId |
| 本地 FFmpeg/video-editor | 已接入 | 拼接、字幕、基础音频、裁剪 | 缺平面追踪、复杂剪辑语法 |
### 万相2.7 适配器详情
```
适配器: engines/wan-api-adapter.js
密钥: ALIYUN_BAILIAN_API_KEY (.env·不进仓库)
模型T2V: wan2.7-t2v (文生视频)
模型I2V: wan2.7-i2v-2026-04-25 (图生视频)
API: https://dashscope.aliyuncs.com/api/v1
支持: 首帧/首尾帧/视频续写/音频驱动 + 本地图片自动base64
测试脚本: tools/test-wan-api.js
D140验证: API连通✅ 任务提交成功 task_id返回正常
```
---
## 本阶段新增接入
> 只记录官方能力方向。真实接入前必须再查官方文档、价格和密钥条件。
> 万相2.7 已在 D140 接入完成。以下为其他待评估方向
| 模型/平台 | 官方能力线索 | 适合评估的任务 |
|-----------|--------------|----------------|

View File

@ -0,0 +1,347 @@
/**
* 光湖视频AI系统 · 音频分轨引擎
* D140 · 铸渊 ICE-GL-ZY001 · 2026-06-23
*
* 将混合音频分离为三轨: 对白(voice) / 背景音乐(bgm) / 音效(sfx)
* 基于 FFmpeg 音频滤波链 + 频率/动态范围分析
*
* 工作原理:
* 1. 对白轨: 人声频率范围(80Hz-8kHz)带通 + 动态范围压缩
* 2. BGM轨: 低频段(20-250Hz) + 高频段(8kHz+)的中低能量部分
* 3. SFX轨: 高频瞬态(打击类) + 非人声中频
*
* 输出三轨独立WAV video-editor.js 混音使用
*
* 使用方式:
* const { splitAudioStems, analyzeAudioLevels } = require('./audio-stem-splitter');
* const stems = await splitAudioStems('input.mp4', './stems/');
* // stems.voice, stems.bgm, stems.sfx → 独立WAV文件路径
*
* 铸渊 ICE-GL-ZY001 · D140
*/
const { execSync, spawnSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const PYTHON = '/Users/bingshuolingdianyuanhe/.workbuddy/binaries/python/envs/default/bin/python3';
// ==================== 核心分轨 ====================
/**
* 将音视频文件分离为三轨: 对白 / BGM / 音效
*
* @param {string} inputPath - 输入文件视频或音频
* @param {string} outputDir - 输出目录
* @param {object} [opts]
* @param {number} [opts.voiceLow=80] - 人声低频截止 Hz
* @param {number} [opts.voiceHigh=8000] - 人声高频截止 Hz
* @param {number} [opts.bgmLow=20] - BGM低频 Hz
* @param {number} [opts.bgmHigh=250] - BGM低频截止 Hz
* @param {number} [opts.sfxThreshold] - SFX瞬态检测阈值(dB)
* @returns {Promise<{voice: string, bgm: string, sfx: string, raw: string, meta: object}>}
*/
async function splitAudioStems(inputPath, outputDir, opts = {}) {
if (!fs.existsSync(inputPath)) {
throw new Error(`输入文件不存在: ${inputPath}`);
}
// 确保输出目录
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
const baseName = path.basename(inputPath, path.extname(inputPath));
const rawWav = path.join(outputDir, `${baseName}_raw.wav`);
const voiceWav = path.join(outputDir, `${baseName}_voice.wav`);
const bgmWav = path.join(outputDir, `${baseName}_bgm.wav`);
const sfxWav = path.join(outputDir, `${baseName}_sfx.wav`);
const metaJson = path.join(outputDir, `${baseName}_stems_meta.json`);
console.log(`[StemSplitter] 输入: ${path.basename(inputPath)}`);
console.log(`[StemSplitter] 输出: ${outputDir}`);
// Step 1: 提取原始音频
console.log('[StemSplitter] 1/4 提取原始音频...');
execSync(
`ffmpeg -y -i "${inputPath}" -vn -acodec pcm_s16le -ar 48000 -ac 2 "${rawWav}" 2>/dev/null`,
{ timeout: 60000 }
);
if (!fs.existsSync(rawWav)) {
throw new Error('原始音频提取失败');
}
const duration = probeDuration(rawWav);
console.log(` 原始音频: ${duration.toFixed(1)}s`);
// Step 2: 分离对白轨(人声频段带通 + 动态增强)
console.log('[StemSplitter] 2/4 分离对白轨...');
extractVoiceTrack(rawWav, voiceWav, opts);
// Step 3: 分离BGM轨低频 + 宽频带中低能量)
console.log('[StemSplitter] 3/4 分离BGM轨...');
extractBgmTrack(rawWav, bgmWav, opts);
// Step 4: 分离SFX轨高频瞬态 + 残余中频)
console.log('[StemSplitter] 4/4 分离音效轨...');
extractSfxTrack(rawWav, sfxWav, voiceWav, bgmWav, opts);
// 生成元数据
const meta = {
input: inputPath,
duration: duration,
tracks: {
voice: { file: voiceWav, duration: probeDuration(voiceWav) },
bgm: { file: bgmWav, duration: probeDuration(bgmWav) },
sfx: { file: sfxWav, duration: probeDuration(sfxWav) },
},
createdAt: new Date().toISOString(),
};
fs.writeFileSync(metaJson, JSON.stringify(meta, null, 2));
console.log(`[StemSplitter] ✅ 分轨完成`);
console.log(` 对白: ${path.basename(voiceWav)} (${meta.tracks.voice.duration.toFixed(1)}s)`);
console.log(` BGM: ${path.basename(bgmWav)} (${meta.tracks.bgm.duration.toFixed(1)}s)`);
console.log(` 音效: ${path.basename(sfxWav)} (${meta.tracks.sfx.duration.toFixed(1)}s)`);
return {
voice: voiceWav,
bgm: bgmWav,
sfx: sfxWav,
raw: rawWav,
meta,
};
}
// ==================== 对白轨提取 ====================
function extractVoiceTrack(inputWav, outputWav, opts = {}) {
const voiceLow = opts.voiceLow || 80;
const voiceHigh = opts.voiceHigh || 8000;
// 滤波链:
// 1. highpass=f=voiceLow — 去除低频噪声
// 2. lowpass=f=voiceHigh — 限制人声高频范围
// 3. compand — 动态范围压缩,增强人声
// 4. afftdn=nr=10 — 降噪
const filter = [
`highpass=f=${voiceLow}`,
`lowpass=f=${voiceHigh}`,
`compand=attacks=0:points=-80/-90|-45/-15|-27/-9|0/-7|20/-7`,
`afftdn=nr=10:nf=-25`,
].join(',');
execSync(
`ffmpeg -y -i "${inputWav}" -af "${filter}" -acodec pcm_s16le -ar 48000 -ac 2 "${outputWav}" 2>/dev/null`,
{ timeout: 120000 }
);
if (!fs.existsSync(outputWav)) {
throw new Error('对白轨提取失败');
}
}
// ==================== BGM轨提取 ====================
function extractBgmTrack(inputWav, outputWav, opts = {}) {
const bgmLow = opts.bgmLow || 20;
const bgmHigh = opts.bgmHigh || 250;
// BGM特征: 低频持续 + 宽频带中低能量
// 滤波链:
// 1. lowpass=f=bgmHigh — 截取低频段(鼓/贝斯/低频氛围)
// 2. bass=g=3 — 增强低频
// 3. compand — 压缩动态范围使BGM更平稳
// 4. volume=0.7 — 稍微降低混音时通常BGM低于人声
const filter = [
`lowpass=f=${bgmHigh}`,
`bass=g=3:f=${bgmLow}:w=80`,
`compand=attacks=1:decays=1:points=-80/-90|-45/-20|0/-12|20/-12`,
`volume=0.7`,
].join(',');
execSync(
`ffmpeg -y -i "${inputWav}" -af "${filter}" -acodec pcm_s16le -ar 48000 -ac 2 "${outputWav}" 2>/dev/null`,
{ timeout: 120000 }
);
if (!fs.existsSync(outputWav)) {
throw new Error('BGM轨提取失败');
}
}
// ==================== SFX轨提取 ====================
function extractSfxTrack(inputWav, outputWav, voiceWav, bgmWav, opts = {}) {
// SFX = 原始音频 - 对白 - BGM
// 方法: 用 sidechaincompress 或直接用频段提取
//
// 滤波链:
// 1. highpass=f=2000 — 取高频段(打击/碰撞/环境音效多在高频)
// 2. 用原始音频减去voice和bgm的频段
// 实际实现: 取高频 + 中频瞬态部分
const filter = [
`highpass=f=1500`, // 高频段
`treble=g=2:f=4000`, // 高频增强
`compand=attacks=0:decays=0.3:points=-80/-90|-30/-30|0/-5|20/-5`, // 快速响应瞬态
`volume=0.8`,
`silenceremove=start_periods=0:start_threshold=-50dB:start_silence=0.1:stop_threshold=-50dB:stop_silence=0.05`,
].join(',');
execSync(
`ffmpeg -y -i "${inputWav}" -af "${filter}" -acodec pcm_s16le -ar 48000 -ac 2 "${outputWav}" 2>/dev/null`,
{ timeout: 120000 }
);
if (!fs.existsSync(outputWav)) {
throw new Error('音效轨提取失败');
}
}
// ==================== 音频电平分析 ====================
/**
* 分析音频文件的电平信息RMS峰值LUFS近似值
* 用于自动混音决策
*
* @param {string} audioPath
* @returns {{rms: number, peak: number, duration: number, channels: number}}
*/
function analyzeAudioLevels(audioPath) {
if (!fs.existsSync(audioPath)) {
throw new Error(`音频文件不存在: ${audioPath}`);
}
// 使用 ffprobe 获取音频信息
const probeOut = execSync(
`ffprobe -v quiet -show_entries format=duration:stream=channels,sample_rate -of json "${audioPath}"`,
{ encoding: 'utf8', timeout: 5000 }
);
const probeData = JSON.parse(probeOut);
const duration = parseFloat(probeData.format?.duration || 0);
const channels = probeData.streams?.[0]?.channels || 2;
// 使用 ffmpeg volumedetect 获取电平
const volOut = execSync(
`ffmpeg -i "${audioPath}" -af volumedetect -f null /dev/null 2>&1 | grep -E "mean_volume|max_volume"`,
{ encoding: 'utf8', timeout: 30000 }
);
const meanMatch = volOut.match(/mean_volume:\s*(-?\d+\.?\d*)\s*dB/);
const maxMatch = volOut.match(/max_volume:\s*(-?\d+\.?\d*)\s*dB/);
const rms = meanMatch ? parseFloat(meanMatch[1]) : -20;
const peak = maxMatch ? parseFloat(maxMatch[1]) : -3;
return {
rms, // RMS电平 (dB)
peak, // 峰值电平 (dB)
duration, // 时长 (秒)
channels, // 声道数
sampleRate: probeData.streams?.[0]?.sample_rate || 48000,
};
}
// ==================== 自动混音建议 ====================
/**
* 根据三轨电平分析生成混音参数建议
*
* @param {string} voiceWav
* @param {string} bgmWav
* @param {string} sfxWav
* @returns {{voiceVolume: number, bgmVolume: number, sfxVolume: number, reasoning: string[]}}
*/
function suggestMixLevels(voiceWav, bgmWav, sfxWav) {
const reasoning = [];
// 分析各轨电平
const voiceLevel = analyzeAudioLevels(voiceWav);
const bgmLevel = bgmWav && fs.existsSync(bgmWav) ? analyzeAudioLevels(bgmWav) : null;
const sfxLevel = sfxWav && fs.existsSync(sfxWav) ? analyzeAudioLevels(sfxWav) : null;
// 目标: 对白 > BGM > SFX一般情况
// 对白 RMS 目标: -16 ~ -12 dB
// BGM RMS 目标: -24 ~ -20 dB (比对白低8dB左右)
// SFX RMS 目标: -20 ~ -15 dB
let voiceVolume = 1.0;
let bgmVolume = 0.4;
let sfxVolume = 0.6;
// 根据对白RMS调整
const voiceTarget = -14;
if (voiceLevel.rms < voiceTarget - 3) {
voiceVolume = Math.min(1.5, (voiceTarget - voiceLevel.rms) / 6 + 1.0);
reasoning.push(`对白RMS偏低(${voiceLevel.rms}dB),增益×${voiceVolume.toFixed(2)}`);
} else if (voiceLevel.rms > voiceTarget + 3) {
voiceVolume = Math.max(0.5, 1.0 - (voiceLevel.rms - voiceTarget) / 10);
reasoning.push(`对白RMS偏高(${voiceLevel.rms}dB),衰减×${voiceVolume.toFixed(2)}`);
}
// 根据BGM电平调整
if (bgmLevel) {
const bgmTarget = -22;
const bgmDiff = bgmLevel.rms - bgmTarget;
if (Math.abs(bgmDiff) > 3) {
bgmVolume = Math.max(0.15, Math.min(0.8, 0.4 * Math.pow(10, -bgmDiff / 20)));
reasoning.push(`BGM电平调整: RMS=${bgmLevel.rms}dB → 音量×${bgmVolume.toFixed(2)}`);
}
}
// 根据SFX电平调整
if (sfxLevel) {
const sfxTarget = -18;
const sfxDiff = sfxLevel.rms - sfxTarget;
if (Math.abs(sfxDiff) > 3) {
sfxVolume = Math.max(0.2, Math.min(1.0, 0.6 * Math.pow(10, -sfxDiff / 20)));
reasoning.push(`SFX电平调整: RMS=${sfxLevel.rms}dB → 音量×${sfxVolume.toFixed(2)}`);
}
}
// 确保对白比BGM高至少6dB
if (bgmLevel && voiceLevel.rms + 20 * Math.log10(voiceVolume) <
bgmLevel.rms + 20 * Math.log10(bgmVolume) + 6) {
bgmVolume *= 0.7;
reasoning.push(`对白-BGM间距不足6dBBGM再降×0.7`);
}
return {
voiceVolume: Math.round(voiceVolume * 100) / 100,
bgmVolume: Math.round(bgmVolume * 100) / 100,
sfxVolume: Math.round(sfxVolume * 100) / 100,
voiceLevel,
bgmLevel,
sfxLevel,
reasoning,
};
}
// ==================== 工具函数 ====================
function probeDuration(filePath) {
try {
const out = execSync(
`ffprobe -v quiet -show_entries format=duration -of csv=p=0 "${filePath}"`,
{ encoding: 'utf8', timeout: 5000 }
);
return parseFloat(out.trim()) || 0;
} catch (_) {
return 0;
}
}
// ==================== 导出 ====================
module.exports = {
splitAudioStems,
extractVoiceTrack,
extractBgmTrack,
extractSfxTrack,
analyzeAudioLevels,
suggestMixLevels,
probeDuration,
};

View File

@ -0,0 +1,405 @@
/**
* 光湖视频AI系统 · 自动剪辑控制层
* D140 · 铸渊 ICE-GL-ZY001 · 2026-06-23
*
* 职责: 解析分镜脚本 选镜头 生成时间线 调度追踪/分轨/剪辑 成片
*
* 这是视频AI系统的"导演"不直接处理像素和音频
* 而是协调 planar-tracker / audio-stem-splitter / video-editor 三个引擎
*
* 流水线:
* 1. 解析分镜脚本 (HLDP分镜格式 / JSON)
* 2. 对每镜执行 OpenCV 追踪 导出运动数据
* 3. 对源音频执行分轨 对白/BGM/SFX三轨
* 4. 基于脚本+运动数据生成剪辑时间线
* 5. 调用 video-editor 完成最终合成
*
* 使用方式:
* const { autoCut } = require('./auto-cut-director');
* const result = await autoCut({
* script: './project/storyboard.json',
* shots: './project/shots/', // 镜头素材目录
* audio: './project/audio.wav', // 可选·源音频
* output: './output/final.mp4',
* });
*
* 铸渊 ICE-GL-ZY001 · D140
*/
const { execSync, spawnSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const { exec } = require('child_process');
// 引入现有引擎
const videoEditor = require('./video-editor');
const { splitAudioStems, suggestMixLevels } = require('./audio-stem-splitter');
const PYTHON = '/Users/bingshuolingdianyuanhe/.workbuddy/binaries/python/envs/default/bin/python3';
const TRACKER_SCRIPT = path.resolve(__dirname, 'planar-tracker.py');
// 输出路径
const JZAO_VIDEO_ROOT = '/Volumes/JZAO/铸渊-ICE-GL-ZY001/OUT-输出/视频';
const DEFAULT_OUTPUT = fs.existsSync(JZAO_VIDEO_ROOT) ? JZAO_VIDEO_ROOT : path.resolve(__dirname, '../outputs');
// ==================== 主入口 ====================
/**
* 自动剪辑 从分镜脚本到成片的完整流水线
*
* @param {object} opts
* @param {string|object} opts.script - 分镜脚本路径(HLDP/JSON) 或已解析的脚本对象
* @param {string} opts.shots - 镜头素材目录
* @param {string} [opts.audio] - 源音频文件视频的原声
* @param {string} [opts.voiceover] - 配音文件外部录制的人声
* @param {string} [opts.bgm] - BGM文件
* @param {string} [opts.subtitle] - SRT字幕文件
* @param {string} [opts.output] - 输出路径
* @param {object} [opts.resolution] - {w, h} 默认1920x1080
* @param {number} [opts.fps=24] - 帧率
* @param {boolean} [opts.stabilize=false] - 是否对每镜做稳定化
* @param {boolean} [opts.autoMix=true] - 是否自动混音
* @returns {Promise<{outputPath: string, duration: number, timeline: array, stemsMeta: object}>}
*/
async function autoCut(opts) {
const {
script: scriptInput,
shots: shotsDir,
audio: sourceAudio,
voiceover,
bgm: externalBgm,
subtitle,
output: outputArg,
resolution = { w: 1920, h: 1080 },
fps = 24,
stabilize = false,
autoMix = true,
} = opts;
console.log('[Director] ═══ 自动剪辑流水线启动 ═══');
// ── Step 0: 解析分镜脚本 ──
console.log('[Director] 0/6 解析分镜脚本...');
const script = typeof scriptInput === 'string' ? loadScript(scriptInput) : scriptInput;
const timeline_spec = buildTimelineSpec(script);
console.log(`[Director] 分镜数: ${timeline_spec.length}`);
// ── Step 1: 匹配镜头文件 ──
console.log('[Director] 1/6 匹配镜头素材...');
const matched = matchShots(timeline_spec, shotsDir);
const missing = matched.filter(m => !m.file);
if (missing.length > 0) {
console.warn(`[Director] ⚠️ ${missing.length}个镜未找到素材:`);
missing.forEach(m => console.warn(`${m.index}: ${m.shotId || m.description}`));
}
const available = matched.filter(m => m.file);
console.log(`[Director] 可用: ${available.length}/${matched.length}`);
// ── Step 2: OpenCV追踪可选·生成运动数据──
console.log('[Director] 2/6 平面追踪分析...');
const workDir = path.join(DEFAULT_OUTPUT, `.director-${Date.now()}`);
fs.mkdirSync(workDir, { recursive: true });
for (const shot of available) {
if (stabilize) {
console.log(` [Tracker] 稳定化: ${path.basename(shot.file)}`);
const stabilizedPath = path.join(workDir, `stab_${path.basename(shot.file)}`);
await runPython(TRACKER_SCRIPT, ['stabilize', shot.file, '-o', stabilizedPath]);
if (fs.existsSync(stabilizedPath)) {
shot.originalFile = shot.file;
shot.file = stabilizedPath;
}
}
// 追踪运动数据用于生成keyframes
const trackJson = path.join(workDir, `track_${shot.index}.json`);
await runPython(TRACKER_SCRIPT, ['track', shot.file, '-o', trackJson]);
if (fs.existsSync(trackJson)) {
const motionJson = path.join(workDir, `motion_${shot.index}.json`);
await runPython(TRACKER_SCRIPT, ['motion_data', shot.file, '--track-json', trackJson, '-o', motionJson]);
if (fs.existsSync(motionJson)) {
const motionData = JSON.parse(fs.readFileSync(motionJson, 'utf8'));
shot.motionData = motionData;
shot.keyframes = motionData.keyframes || {};
console.log(` [Tracker] 镜${shot.index}: 稳定性=${motionData.stability?.toFixed(2) || '?'} 强度=${motionData.motion_intensity || '?'}`);
}
}
}
// ── Step 3: 音频分轨(如果有源音频)──
console.log('[Director] 3/6 音频分轨...');
let stemsMeta = null;
let mixParams = { voiceVolume: 1.0, bgmVolume: 0.3, sfxVolume: 0.5 };
if (sourceAudio && fs.existsSync(sourceAudio)) {
const stemsDir = path.join(workDir, 'stems');
const stems = await splitAudioStems(sourceAudio, stemsDir);
if (autoMix) {
const suggestion = suggestMixLevels(stems.voice, stems.bgm, stems.sfx);
mixParams = {
voiceVolume: suggestion.voiceVolume,
bgmVolume: suggestion.bgmVolume,
sfxVolume: suggestion.sfxVolume,
};
console.log(` [Audio] 自动混音: voice=${mixParams.voiceVolume} bgm=${mixParams.bgmVolume} sfx=${mixParams.sfxVolume}`);
suggestion.reasoning.forEach(r => console.log(`${r}`));
}
stemsMeta = stems;
}
// ── Step 4: 构建剪辑时间线 ──
console.log('[Director] 4/6 构建剪辑时间线...');
const timeline = buildEditorTimeline(available, { fps, resolution });
// ── Step 5: 构建音频参数 ──
console.log('[Director] 5/6 构建音频参数...');
const audioConfig = buildAudioConfig({
voiceover,
sourceVoice: stemsMeta?.voice,
externalBgm,
sourceBgm: stemsMeta?.bgm,
sourceSfx: stemsMeta?.sfx,
mixParams,
});
// ── Step 6: 调用 video-editor 合成 ──
console.log('[Director] 6/6 调用剪辑引擎合成...');
const outputPath = outputArg || path.join(DEFAULT_OUTPUT, `autocut-${Date.now()}.mp4`);
const editorResult = await videoEditor.edit({
timeline,
audio: audioConfig,
subtitle,
resolution,
fps,
output: outputPath,
});
// 清理工作目录
try {
fs.rmSync(workDir, { recursive: true });
} catch (_) {}
console.log(`[Director] ═══ 自动剪辑完成 ═══`);
console.log(`[Director] 成片: ${editorResult.outputPath}`);
console.log(`[Director] 时长: ${editorResult.duration?.toFixed(1)}s`);
return {
outputPath: editorResult.outputPath,
duration: editorResult.duration,
size: editorResult.size,
timeline,
stemsMeta,
mixParams,
};
}
// ==================== 分镜脚本解析 ====================
/**
* 加载分镜脚本
* 支持: JSON文件 / HLDP格式简化的key:value解析
*/
function loadScript(scriptPath) {
if (!fs.existsSync(scriptPath)) {
throw new Error(`分镜脚本不存在: ${scriptPath}`);
}
const content = fs.readFileSync(scriptPath, 'utf8');
const ext = path.extname(scriptPath).toLowerCase();
if (ext === '.json') {
return JSON.parse(content);
}
// HLDP/文本格式解析: 每行 "key: value" 或 "field = value"
const lines = content.split('\n');
const result = { shots: [] };
let currentShot = null;
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#') || trimmed.startsWith('>') || trimmed.startsWith('⊢')) continue;
// 镜头分隔
if (trimmed.match(/^(shot|镜|分镜)\s*[:]/i) || trimmed.match(/^---\s*shot/i)) {
if (currentShot) result.shots.push(currentShot);
currentShot = { index: result.shots.length + 1 };
continue;
}
// key: value 格式
const match = trimmed.match(/^(\w+)\s*[:=]\s*(.+)$/);
if (match && currentShot) {
const key = match[1].toLowerCase();
const value = match[2].trim();
currentShot[key] = value;
} else if (match) {
result[match[1].toLowerCase()] = match[2].trim();
}
}
if (currentShot) result.shots.push(currentShot);
return result;
}
/**
* 从脚本中提取时间线规格
*/
function buildTimelineSpec(script) {
const shots = script.shots || script.timeline || [];
return shots.map((s, i) => ({
index: i + 1,
shotId: s.id || s.shotId || s.shot_id || `shot${String(i + 1).padStart(2, '0')}`,
description: s.description || s.desc || s.prompt || '',
duration: parseFloat(s.duration || s.trim || s.dur || 5),
transition: s.transition || 'fade',
file: s.file || s.path || s.video || null,
}));
}
// ==================== 镜头匹配 ====================
/**
* 将时间线规格与素材目录中的文件匹配
*/
function matchShots(timelineSpec, shotsDir) {
if (!fs.existsSync(shotsDir)) {
console.warn(`[Director] 素材目录不存在: ${shotsDir}`);
return timelineSpec.map(t => ({ ...t, file: null }));
}
// 列出目录中所有视频文件
const videoExts = ['.mp4', '.mov', '.avi', '.mkv', '.webm'];
const files = fs.readdirSync(shotsDir)
.filter(f => videoExts.includes(path.extname(f).toLowerCase()))
.sort();
return timelineSpec.map(spec => {
let matchedFile = null;
// 1. 直接指定文件
if (spec.file && fs.existsSync(path.resolve(shotsDir, spec.file))) {
matchedFile = path.resolve(shotsDir, spec.file);
}
// 2. 按shotId匹配
if (!matchedFile && spec.shotId) {
const match = files.find(f =>
f.toLowerCase().includes(spec.shotId.toLowerCase()) ||
f.toLowerCase().includes(spec.shotId.toLowerCase().replace('shot', ''))
);
if (match) matchedFile = path.resolve(shotsDir, match);
}
// 3. 按序号匹配
if (!matchedFile) {
const numMatch = files.find(f => {
const nums = f.match(/\d+/g);
return nums && nums.some(n => parseInt(n) === spec.index);
});
if (numMatch) matchedFile = path.resolve(shotsDir, numMatch);
}
// 4. 按顺序匹配(如果文件数与镜数一致)
if (!matchedFile && spec.index <= files.length) {
matchedFile = path.resolve(shotsDir, files[spec.index - 1]);
}
return { ...spec, file: matchedFile };
});
}
// ==================== 时间线构建 ====================
function buildEditorTimeline(matchedShots, { fps, resolution }) {
return matchedShots
.filter(s => s.file)
.map(s => {
const entry = {
shot: s.file,
trim: s.duration || 5,
transition: s.transition || 'fade',
};
// 注入运动数据生成的keyframes
if (s.keyframes && Object.keys(s.keyframes).length > 0) {
entry.keyframes = s.keyframes;
}
// 如果运动强度高但稳定化未开启,加入轻微缩放补偿
if (s.motionData && s.motionData.stability < 0.5 && !entry.keyframes) {
entry.keyframes = {
zoom: { from: 1.05, to: 1.05 }, // 固定5%放大,补偿抖动
};
}
return entry;
});
}
// ==================== 音频配置构建 ====================
function buildAudioConfig({
voiceover, sourceVoice, externalBgm, sourceBgm, sourceSfx, mixParams,
}) {
const config = {
voice: voiceover || sourceVoice || null,
bgm: externalBgm || sourceBgm || null,
sfx: [],
bgmVolume: mixParams.bgmVolume ?? 0.3,
voiceVolume: mixParams.voiceVolume ?? 1.0,
};
// SFX: 如果有分轨的音效轨作为单个sfx加入
if (sourceSfx && fs.existsSync(sourceSfx)) {
config.sfx.push({ at: 0, file: sourceSfx });
}
// 如果没有任何音频返回null
if (!config.voice && !config.bgm && config.sfx.length === 0) {
return null;
}
return config;
}
// ==================== Python调用工具 ====================
function runPython(script, args, timeout = 120000) {
return new Promise((resolve, reject) => {
const cmd = [PYTHON, script, ...args];
console.log(` [Python] ${path.basename(script)} ${args.join(' ')}`);
const proc = spawnSync(PYTHON, [script, ...args], {
encoding: 'utf8',
timeout,
cwd: path.dirname(script),
});
if (proc.status === 0) {
if (proc.stdout) console.log(proc.stdout.trim());
resolve(proc.stdout);
} else {
const errMsg = proc.stderr || proc.stdout || `退出码 ${proc.status}`;
console.error(` [Python] ❌ ${errMsg.substring(0, 300)}`);
// 不reject——追踪失败不应该中断整个流水线
resolve(null);
}
});
}
// ==================== 导出 ====================
module.exports = {
autoCut,
loadScript,
buildTimelineSpec,
matchShots,
buildEditorTimeline,
buildAudioConfig,
};

View File

@ -0,0 +1,527 @@
#!/usr/bin/env python3
"""
光湖视频AI系统 · OpenCV 平面追踪引擎
D140 · 铸渊 ICE-GL-ZY001 · 2026-06-23
用途:
1. 特征点追踪 追踪画面中物体运动轨迹
2. 画面稳定化 去除抖动
3. 文字/图形贴合 把文字/水印/PIN到追踪的平面上
4. 运动数据导出 video-editor.js keyframes 使用
调用方式:
python3 planar-tracker.py <mode> <video> [options]
mode:
track 追踪特征点输出运动轨迹JSON
stabilize 画面稳定化输出稳定后视频
pin_text 追踪+贴合文字到指定区域
motion_data 导出运动数据供FFmpeg使用
输出:
track JSON文件 (轨迹数据)
stabilize MP4文件 (稳定后视频)
pin_text MP4文件 (带贴合文字的视频)
motion_data JSON文件 (zoom/pan关键帧数据)
铸渊 ICE-GL-ZY001 · D140
"""
import cv2
import numpy as np
import json
import sys
import os
import argparse
from pathlib import Path
# ==================== 特征点追踪 ====================
def track_features(video_path, output_json, max_points=200, min_distance=15, quality=0.01):
"""
Shi-Tomasi角点检测 + Lucas-Kanade光流追踪
输出JSON格式:
{
"fps": 24,
"frame_count": 120,
"resolution": [1920, 1080],
"tracks": [
{
"point_id": 0,
"points": [{"frame": 0, "x": 100, "y": 200}, ...],
"start": [100, 200],
"end": [105, 198],
"displacement": [5, -2],
"avg_velocity": [0.04, -0.017]
}
]
}
"""
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise RuntimeError(f"无法打开视频: {video_path}")
fps = cap.get(cv2.CAP_PROP_FPS)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
print(f"[Tracker] 视频: {Path(video_path).name} {width}x{height} @ {fps:.1f}fps {frame_count}")
# 读取第一帧
ret, prev_frame = cap.read()
if not ret:
raise RuntimeError("无法读取视频帧")
prev_gray = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY)
# Shi-Tomasi角点检测
prev_points = cv2.goodFeaturesToTrack(
prev_gray, maxCorners=max_points, qualityLevel=quality,
minDistance=min_distance, blockSize=7
)
if prev_points is None:
raise RuntimeError("第一帧未检测到特征点")
print(f"[Tracker] 检测到 {len(prev_points)} 个特征点")
# 初始化轨迹
tracks = []
for i, pt in enumerate(prev_points):
tracks.append({
"point_id": i,
"points": [{"frame": 0, "x": float(pt[0][0]), "y": float(pt[0][1])}],
"start": [float(pt[0][0]), float(pt[0][1])],
"active": True
})
# Lucas-Kanade参数
lk_params = dict(
winSize=(15, 15),
maxLevel=2,
criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)
)
frame_idx = 1
while True:
ret, frame = cap.read()
if not ret:
break
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# 计算光流
next_points, status, error = cv2.calcOpticalFlowPyrLK(
prev_gray, frame_gray, prev_points, None, **lk_params
)
# 更新轨迹
active_count = 0
for i, (track, st) in enumerate(zip(tracks, status)):
if not track["active"]:
continue
if st == 1 and next_points is not None:
x, y = float(next_points[i][0]), float(next_points[i][1])
track["points"].append({"frame": frame_idx, "x": x, "y": y})
track["active"] = True
active_count += 1
else:
track["active"] = False
# 只保留活跃点供下一帧追踪
if active_count > 0:
mask = status.flatten() == 1
prev_points = next_points[mask].reshape(-1, 1, 2)
# 对应更新tracks索引
new_tracks = [t for t, m in zip(tracks, mask) if m]
# 但要保留丢失的轨迹(标记为非活跃)
lost_tracks = [t for t, m in zip(tracks, mask) if not m]
tracks = new_tracks + lost_tracks
# 重新编号并调整
# 注意prev_points和新tracks顺序需要对齐
# 上面的操作保证了new_tracks和prev_points对齐
prev_gray = frame_gray
frame_idx += 1
if frame_idx % 30 == 0:
print(f" [Tracker] 帧 {frame_idx}/{frame_count} 活跃点: {active_count}")
cap.release()
# 计算位移和速度
for track in tracks:
if len(track["points"]) >= 2:
start = track["start"]
end = track["points"][-1]
track["end"] = [end["x"], end["y"]]
track["displacement"] = [end["x"] - start[0], end["y"] - start[1]]
frames = len(track["points"])
track["avg_velocity"] = [
track["displacement"][0] / frames,
track["displacement"][1] / frames
]
# 只保留有足够数据的轨迹
valid_tracks = [t for t in tracks if len(t["points"]) >= frame_count * 0.3]
print(f"[Tracker] 有效轨迹: {len(valid_tracks)}/{len(tracks)}")
result = {
"fps": fps,
"frame_count": frame_idx - 1,
"resolution": [width, height],
"tracks": valid_tracks
}
with open(output_json, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"[Tracker] ✅ 轨迹数据已保存: {output_json}")
return result
# ==================== 画面稳定化 ====================
def stabilize_video(video_path, output_path, smoothing_radius=25, crop_percent=0.08):
"""
基于特征点轨迹的画面稳定化
原理:
1. 追踪相邻帧的特征点
2. 计算相邻帧之间的仿射变换矩阵
3. 累积变换轨迹
4. 平滑轨迹滑动平均
5. 应用平滑后的变换
smoothing_radius: 平滑窗口大小帧数越大越平滑但延迟越大
crop_percent: 边缘裁剪比例补偿稳定后的黑边
"""
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise RuntimeError(f"无法打开视频: {video_path}")
fps = cap.get(cv2.CAP_PROP_FPS)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
print(f"[Stabilize] {Path(video_path).name} {width}x{height} @ {fps:.1f}fps")
# 读取所有帧
frames = []
while True:
ret, frame = cap.read()
if not ret:
break
frames.append(frame)
cap.release()
if len(frames) < 2:
raise RuntimeError("视频帧数不足")
# 追踪特征点
prev_gray = cv2.cvtColor(frames[0], cv2.COLOR_BGR2GRAY)
transforms = [] # 每帧的变换 [dx, dy, da(角度)]
for i in range(1, len(frames)):
curr_gray = cv2.cvtColor(frames[i], cv2.COLOR_BGR2GRAY)
prev_pts = cv2.goodFeaturesToTrack(prev_gray, maxCorners=200, qualityLevel=0.01,
minDistance=15, blockSize=7)
if prev_pts is None or len(prev_pts) < 10:
transforms.append([0, 0, 0])
prev_gray = curr_gray
continue
curr_pts, status, _ = cv2.calcOpticalFlowPyrLK(prev_gray, curr_gray, prev_pts, None)
# 筛选有效点
valid_prev = prev_pts[status == 1]
valid_curr = curr_pts[status == 1]
if len(valid_prev) < 10:
transforms.append([0, 0, 0])
prev_gray = curr_gray
continue
# 计算仿射变换 (只取平移和旋转)
m = cv2.estimateAffinePartial2D(valid_prev, valid_curr)[0]
if m is None:
transforms.append([0, 0, 0])
prev_gray = curr_gray
continue
dx = m[0, 2]
dy = m[1, 2]
da = np.arctan2(m[1, 0], m[0, 0])
transforms.append([dx, dy, da])
prev_gray = curr_gray
if i % 30 == 0:
print(f" [Stabilize] 帧 {i}/{len(frames)}")
# 累积轨迹
trajectory = np.zeros((len(transforms) + 1, 3))
for i, t in enumerate(transforms):
trajectory[i + 1] = trajectory[i] + np.array(t)
# 平滑轨迹(滑动平均)
smoothed = np.zeros_like(trajectory)
for i in range(len(trajectory)):
start = max(0, i - smoothing_radius)
end = min(len(trajectory), i + smoothing_radius + 1)
smoothed[i] = np.mean(trajectory[start:end], axis=0)
# 计算平滑后的变换
smooth_diff = smoothed - trajectory
# 写出稳定后的视频
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
crop_x = int(width * crop_percent)
crop_y = int(height * crop_percent)
out_w = width - 2 * crop_x
out_h = height - 2 * crop_y
writer = cv2.VideoWriter(output_path, fourcc, fps, (out_w, out_h))
for i, frame in enumerate(frames):
dx = smooth_diff[i, 0]
dy = smooth_diff[i, 1]
da = smooth_diff[i, 2]
# 构造仿射变换矩阵
m = np.array([
[np.cos(da), -np.sin(da), dx],
[np.sin(da), np.cos(da), dy]
], dtype=np.float32)
stabilized = cv2.warpAffine(frame, m, (width, height))
# 裁剪黑边
cropped = stabilized[crop_y:crop_y + out_h, crop_x:crop_x + out_w]
writer.write(cropped)
writer.release()
print(f"[Stabilize] ✅ 稳定后视频: {output_path}")
return output_path
# ==================== 文字贴合 ====================
def pin_text_to_video(video_path, output_path, text,
start_frame=0, end_frame=None,
init_x=None, init_y=None,
font_scale=1.0, color=(255, 255, 255),
thickness=2, font=cv2.FONT_HERSHEY_SIMPLEX,
track_mode='feature'):
"""
追踪画面特征点 + 将文字PIN到追踪位置
track_mode:
'feature' 特征点追踪自动选择画面中心区域的强特征点
'fixed' 固定位置不追踪
文字会跟随画面运动保持相对位置不变
"""
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise RuntimeError(f"无法打开视频: {video_path}")
fps = cap.get(cv2.CAP_PROP_FPS)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
if end_frame is None:
end_frame = frame_count
if init_x is None:
init_x = width // 2
if init_y is None:
init_y = height // 2
print(f"[PinText] 文字: '{text}' 位置: ({init_x},{init_y}) 帧: {start_frame}-{end_frame}")
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
# 跳到起始帧
cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
prev_gray = None
prev_point = np.array([[init_x, init_y]], dtype=np.float32).reshape(-1, 1, 2)
current_offset = np.array([0.0, 0.0])
for frame_idx in range(start_frame, min(end_frame, frame_count)):
ret, frame = cap.read()
if not ret:
break
curr_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
if track_mode == 'feature' and prev_gray is not None:
# 追踪文字所在位置附近的特征点
next_point, status, _ = cv2.calcOpticalFlowPyrLK(
prev_gray, curr_gray, prev_point, None,
winSize=(31, 31), maxLevel=2
)
if status is not None and status[0] == 1 and next_point is not None:
# 计算位移
dx = next_point[0][0] - prev_point[0][0]
dy = next_point[0][1] - prev_point[0][1]
current_offset += np.array([dx, dy])
prev_point = next_point
# 计算文字绘制位置
text_x = int(init_x + current_offset[0])
text_y = int(init_y + current_offset[1])
# 绘制文字(带描边)
# 描边
cv2.putText(frame, text, (text_x + 2, text_y + 2), font, font_scale,
(0, 0, 0), thickness + 2, cv2.LINE_AA)
# 主文字
cv2.putText(frame, text, (text_x, text_y), font, font_scale,
color, thickness, cv2.LINE_AA)
writer.write(frame)
prev_gray = curr_gray
if (frame_idx - start_frame) % 30 == 0:
print(f" [PinText] 帧 {frame_idx}/{end_frame} 偏移: ({current_offset[0]:.1f}, {current_offset[1]:.1f})")
cap.release()
writer.release()
print(f"[PinText] ✅ 文字贴合视频: {output_path}")
return output_path
# ==================== 运动数据导出 ====================
def export_motion_data(track_json, output_json, mode='zoom_pan'):
"""
从追踪数据中提取运动趋势导出为 video-editor.js 可用的 keyframes 格式
mode='zoom_pan':
分析整体运动趋势 输出 zoom(缩放) + pan(平移) 关键帧
输出格式:
{
"keyframes": {
"zoom": { "from": 1.0, "to": 1.05 },
"pan": { "from": [0, 0], "to": [12, -8] }
},
"stability": 0.85, // 稳定性评分 0-1
"motion_intensity": "low" | "medium" | "high"
}
"""
with open(track_json, 'r') as f:
track_data = json.load(f)
tracks = track_data.get('tracks', [])
if not tracks:
return {"keyframes": {}, "stability": 0, "motion_intensity": "unknown"}
# 分析所有轨迹的位移
displacements = []
for t in tracks:
if 'displacement' in t:
displacements.append(t['displacement'])
if not displacements:
return {"keyframes": {}, "stability": 0, "motion_intensity": "unknown"}
disp_arr = np.array(displacements)
# 计算平均位移
mean_dx = float(np.mean(disp_arr[:, 0]))
mean_dy = float(np.mean(disp_arr[:, 1]))
# 计算位移标准差(稳定性指标)
std_dx = float(np.std(disp_arr[:, 0]))
std_dy = float(np.std(disp_arr[:, 1]))
stability = 1.0 / (1.0 + (std_dx + std_dy) / 100.0)
stability = max(0.0, min(1.0, stability))
# 运动强度
total_motion = abs(mean_dx) + abs(mean_dy) + (std_dx + std_dy) * 0.5
if total_motion < 20:
motion_intensity = 'low'
elif total_motion < 80:
motion_intensity = 'medium'
else:
motion_intensity = 'high'
# 缩放趋势(通过分析点之间距离的变化)
# 简化版:用位移方差近似
zoom_trend = 1.0 + min(0.2, max(-0.1, (std_dx + std_dy) / 5000.0))
result = {
"keyframes": {
"zoom": {
"from": 1.0,
"to": round(zoom_trend, 4)
},
"pan": {
"from": [0, 0],
"to": [round(mean_dx, 2), round(mean_dy, 2)]
}
},
"stability": round(stability, 4),
"motion_intensity": motion_intensity,
"avg_displacement": [round(mean_dx, 2), round(mean_dy, 2)],
"track_count": len(tracks)
}
with open(output_json, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"[MotionData] ✅ 运动数据: {output_json}")
print(f" 稳定性: {result['stability']} 强度: {motion_intensity}")
print(f" 平均位移: ({mean_dx:.1f}, {mean_dy:.1f})")
return result
# ==================== CLI ====================
def main():
parser = argparse.ArgumentParser(description='光湖OpenCV平面追踪引擎')
parser.add_argument('mode', choices=['track', 'stabilize', 'pin_text', 'motion_data'],
help='运行模式')
parser.add_argument('video', help='输入视频路径')
parser.add_argument('-o', '--output', help='输出路径')
parser.add_argument('--text', help='pin_text模式的文字内容')
parser.add_argument('--x', type=int, help='文字初始X坐标')
parser.add_argument('--y', type=int, help='文字初始Y坐标')
parser.add_argument('--track-json', help='motion_data模式的追踪数据JSON')
parser.add_argument('--smoothing', type=int, default=25, help='稳定化平滑窗口')
args = parser.parse_args()
if args.mode == 'track':
out = args.output or args.video.rsplit('.', 1)[0] + '_tracks.json'
track_features(args.video, out)
elif args.mode == 'stabilize':
out = args.output or args.video.rsplit('.', 1)[0] + '_stabilized.mp4'
stabilize_video(args.video, out, smoothing_radius=args.smoothing)
elif args.mode == 'pin_text':
if not args.text:
print("错误: pin_text模式需要 --text 参数")
sys.exit(1)
out = args.output or args.video.rsplit('.', 1)[0] + '_pinned.mp4'
pin_text_to_video(args.video, out, args.text, init_x=args.x, init_y=args.y)
elif args.mode == 'motion_data':
if not args.track_json:
print("错误: motion_data模式需要 --track-json 参数")
sys.exit(1)
out = args.output or args.video.rsplit('.', 1)[0] + '_motion.json'
export_motion_data(args.track_json, out)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,566 @@
/**
* 光湖视频AI系统 · 阿里百炼万相 API 适配器
* D140 · 铸渊 ICE-GL-ZY001 · 2026-06-23
*
* 阿里云百炼平台 · 万相2.7视频生成
* 文档: https://help.aliyun.com/zh/model-studio/text-to-video-api-reference
*
* 支持能力:
* - 文生视频 (wan2.7-t2v): 纯文本prompt生成视频
* - 图生视频 (wan2.7-i2v): 首帧/首尾帧/音频驱动 + 文本生成视频
* - 视频续写: 首视频片段续写
*
* 使用方式:
* const { generateWanVideo, generateWanImageToVideo, queryWanTask } = require('./wan-api-adapter');
* const result = await generateWanVideo({ prompt: '一只小猫在月光下奔跑', duration: 5 });
*
* 环境变量放在 video-ai-system/.env:
* ALIYUN_BAILIAN_API_KEY=xxx
* ALIYUN_BAILIAN_BASE_URL=https://dashscope.aliyuncs.com/api/v1 (默认)
* ALIYUN_BAILIAN_WORKSPACE_ID=xxx (可选·新加坡地域时需要)
*/
const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');
// 读取环境变量
const envPath = path.resolve(__dirname, '../.env');
if (fs.existsSync(envPath)) {
const envContent = fs.readFileSync(envPath, 'utf-8');
envContent.split('\n').forEach(line => {
const trimmed = line.trim();
if (trimmed && !trimmed.startsWith('#')) {
const [key, ...vals] = trimmed.split('=');
if (key && vals.length) process.env[key.trim()] = vals.join('=').trim();
}
});
}
const API_KEY = process.env.ALIYUN_BAILIAN_API_KEY || '';
const WORKSPACE_ID = process.env.ALIYUN_BAILIAN_WORKSPACE_ID || '';
// 北京地域默认URL新加坡地域需要 workspaceId
const BASE_URL = WORKSPACE_ID
? `https://${WORKSPACE_ID}.ap-southeast-1.maas.aliyuncs.com/api/v1`
: (process.env.ALIYUN_BAILIAN_BASE_URL || 'https://dashscope.aliyuncs.com/api/v1');
const POLL_INTERVAL_MS = parseInt(process.env.WAN_POLL_INTERVAL_MS, 10) || 15000; // 万相建议15秒
const MAX_POLL_ATTEMPTS = parseInt(process.env.WAN_MAX_POLL_ATTEMPTS, 10) || 80; // 最多约20分钟
// 输出路径
const JZAO_VIDEO_ROOT = '/Volumes/JZAO/铸渊-ICE-GL-ZY001/OUT-输出/视频';
const LOCAL_OUTPUT_ROOT = path.resolve(__dirname, '../outputs');
const VIDEO_OUTPUT_ROOT = (() => {
const env = process.env.VIDEO_OUTPUT_ROOT;
if (env) return env;
if (fs.existsSync(JZAO_VIDEO_ROOT)) return JZAO_VIDEO_ROOT;
return LOCAL_OUTPUT_ROOT;
})();
// ==================== HTTP 工具 ====================
function httpRequest(method, urlStr, body, apiKey, extraHeaders = {}) {
const urlObj = new URL(urlStr);
const isHttps = urlObj.protocol === 'https:';
const transport = isHttps ? https : http;
const payload = body ? JSON.stringify(body) : null;
return new Promise((resolve, reject) => {
const headers = {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json',
'X-DashScope-Async': 'enable', // 万相必须设置异步头
...extraHeaders,
};
if (payload) headers['Content-Length'] = Buffer.byteLength(payload);
const options = {
hostname: urlObj.hostname,
port: urlObj.port || (isHttps ? 443 : 80),
path: urlObj.pathname + urlObj.search,
method,
headers,
timeout: 30000,
};
const req = transport.request(options, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
try {
const json = JSON.parse(data);
// 万相错误码在顶层: { code: "xxx", message: "xxx" }
if (json.code && json.code !== 'Success' && res.statusCode >= 400) {
reject(new Error(`万相API错误(${json.code}): ${json.message}`));
return;
}
resolve(json);
} catch (e) {
reject(new Error(`JSON解析失败: ${data.substring(0, 200)}`));
}
});
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('请求超时')); });
if (payload) req.write(payload);
req.end();
});
}
async function httpPost(url, body, apiKey) {
return httpRequest('POST', url, body, apiKey);
}
async function httpGet(url, apiKey) {
return httpRequest('GET', url, null, apiKey);
}
async function downloadFile(videoUrl, outputPath) {
const urlObj = new URL(videoUrl);
const isHttps = urlObj.protocol === 'https:';
const transport = isHttps ? https : http;
return new Promise((resolve, reject) => {
const dir = path.dirname(outputPath);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
const file = fs.createWriteStream(outputPath);
transport.get(videoUrl, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
const redirectUrl = res.headers.location.startsWith('http')
? res.headers.location
: `${urlObj.protocol}//${urlObj.host}${res.headers.location}`;
downloadFile(redirectUrl, outputPath).then(resolve).catch(reject);
return;
}
res.pipe(file);
file.on('finish', () => { file.close(); resolve(outputPath); });
file.on('error', (err) => { fs.unlinkSync(outputPath); reject(err); });
}).on('error', reject);
});
}
// ==================== API 规范常量 ====================
const WAN_T2V_SPEC = {
model: 'wan2.7-t2v',
duration: { min: 2, max: 15, default: 5 },
resolution: { values: ['720P', '1080P'], default: '1080P' },
ratio: { values: ['16:9', '9:16', '1:1', '4:3', '3:4'], default: '16:9' },
promptMaxLen: 5000,
};
const WAN_I2V_SPEC = {
model: 'wan2.7-i2v-2026-04-25',
duration: { min: 2, max: 15, default: 5 },
resolution: { values: ['720P', '1080P'], default: '1080P' },
promptMaxLen: 5000,
// 图生视频的宽高比由输入素材决定不需要单独传ratio
mediaTypes: ['first_frame', 'last_frame', 'driving_audio', 'first_clip'],
};
// ==================== 预校验 ====================
function preflightCheckT2V({ prompt, duration, resolution, ratio }) {
const warnings = [];
const errors = [];
const corrected = {};
// prompt
if (!prompt || !prompt.trim()) {
errors.push('提示词不能为空');
} else if (prompt.length > WAN_T2V_SPEC.promptMaxLen) {
warnings.push(`提示词超过 ${WAN_T2V_SPEC.promptMaxLen} 字符,将自动截断`);
}
corrected.prompt = prompt;
// duration
const dur = parseInt(duration, 10);
if (duration !== undefined && duration !== null) {
if (isNaN(dur) || dur < WAN_T2V_SPEC.duration.min || dur > WAN_T2V_SPEC.duration.max) {
errors.push(`duration 范围: ${WAN_T2V_SPEC.duration.min}~${WAN_T2V_SPEC.duration.max}秒,收到: ${duration}`);
} else {
corrected.duration = dur;
}
} else {
corrected.duration = WAN_T2V_SPEC.duration.default;
}
// resolution
if (!resolution || !WAN_T2V_SPEC.resolution.values.includes(resolution)) {
if (resolution) warnings.push(`resolution "${resolution}" 不支持,修正为 ${WAN_T2V_SPEC.resolution.default}`);
corrected.resolution = WAN_T2V_SPEC.resolution.default;
} else {
corrected.resolution = resolution;
}
// ratio
if (!ratio || !WAN_T2V_SPEC.ratio.values.includes(ratio)) {
if (ratio) warnings.push(`ratio "${ratio}" 不支持,修正为 ${WAN_T2V_SPEC.ratio.default}`);
corrected.ratio = WAN_T2V_SPEC.ratio.default;
} else {
corrected.ratio = ratio;
}
return { valid: errors.length === 0, warnings, errors, corrected };
}
function preflightCheckI2V({ prompt, media, duration, resolution }) {
const warnings = [];
const errors = [];
const corrected = {};
// media必选
if (!media || !Array.isArray(media) || media.length === 0) {
errors.push('图生视频必须提供 media 素材(至少一个 first_frame');
} else {
// 检查media格式
for (const m of media) {
if (!WAN_I2V_SPEC.mediaTypes.includes(m.type)) {
errors.push(`media type "${m.type}" 不支持,可选: ${WAN_I2V_SPEC.mediaTypes.join(', ')}`);
}
if (!m.url) {
errors.push(`media ${m.type} 缺少 url`);
}
}
corrected.media = media;
}
// prompt可选
corrected.prompt = prompt || '';
// duration
const dur = parseInt(duration, 10);
if (duration !== undefined && duration !== null) {
if (isNaN(dur) || dur < WAN_I2V_SPEC.duration.min || dur > WAN_I2V_SPEC.duration.max) {
errors.push(`duration 范围: ${WAN_I2V_SPEC.duration.min}~${WAN_I2V_SPEC.duration.max}`);
} else {
corrected.duration = dur;
}
} else {
corrected.duration = WAN_I2V_SPEC.duration.default;
}
// resolution
if (!resolution || !WAN_I2V_SPEC.resolution.values.includes(resolution)) {
if (resolution) warnings.push(`resolution "${resolution}" 不支持,修正为 ${WAN_I2V_SPEC.resolution.default}`);
corrected.resolution = WAN_I2V_SPEC.resolution.default;
} else {
corrected.resolution = resolution;
}
return { valid: errors.length === 0, warnings, errors, corrected };
}
// ==================== 提交任务 ====================
const SUBMIT_PATH = '/services/aigc/video-generation/video-synthesis';
const QUERY_PATH = '/tasks/';
/**
* 提交万相文生视频任务
* @param {object} opts
* @param {string} opts.prompt - 视频描述提示词
* @param {number} [opts.duration] - 时长 2-15默认5
* @param {string} [opts.resolution] - '720P' | '1080P'默认1080P
* @param {string} [opts.ratio] - '16:9' 默认16:9
* @param {string} [opts.negativePrompt] - 反向提示词
* @param {boolean} [opts.promptExtend=true] - 是否智能改写prompt
* @param {boolean} [opts.watermark=false] - 是否加水印
* @param {number} [opts.seed] - 随机种子
* @returns {Promise<{taskId: string, preflight: object}>}
*/
async function submitT2VTask({ prompt, duration, resolution, ratio, negativePrompt, promptExtend, watermark, seed }) {
const preflight = preflightCheckT2V({ prompt, duration, resolution, ratio });
if (!preflight.valid) {
console.error('[Wan·预校验] ❌ 参数错误:');
preflight.errors.forEach(e => console.error(`${e}`));
throw new Error(`预校验失败: ${preflight.errors.join('; ')}`);
}
if (preflight.warnings.length > 0) {
console.warn('[Wan·预校验] ⚠️', preflight.warnings.join('; '));
}
const c = preflight.corrected;
const body = {
model: WAN_T2V_SPEC.model,
input: {
prompt: c.prompt,
},
parameters: {
resolution: c.resolution,
ratio: c.ratio,
duration: c.duration,
prompt_extend: promptExtend !== false,
watermark: watermark === true,
},
};
if (negativePrompt) body.input.negative_prompt = negativePrompt;
if (seed !== undefined) body.parameters.seed = seed;
console.log(`[Wan·T2V] 提交: "${c.prompt.substring(0, 60)}..." ${c.duration}s ${c.resolution} ${c.ratio}`);
const url = `${BASE_URL}${SUBMIT_PATH}`;
const data = await httpPost(url, body, API_KEY);
const taskId = data.output?.task_id;
if (!taskId) {
throw new Error(`万相未返回task_id: ${JSON.stringify(data).substring(0, 300)}`);
}
console.log(`[Wan·T2V] 任务ID: ${taskId} 状态: ${data.output?.task_status}`);
return { taskId, preflight };
}
/**
* 提交万相图生视频任务
* @param {object} opts
* @param {string} [opts.prompt] - 辅助文本描述
* @param {Array} opts.media - 媒体素材 [{ type: 'first_frame', url: '...' }, ...]
* type可选: first_frame, last_frame, driving_audio, first_clip
* url支持: 公网URL / oss://临时URL / data:image/png;base64,...
* @param {number} [opts.duration] - 时长 2-15
* @param {string} [opts.resolution] - '720P' | '1080P'
* @param {string} [opts.negativePrompt] - 反向提示词
* @param {boolean} [opts.promptExtend=true] - 智能改写
* @param {boolean} [opts.watermark=false] - 加水印
* @param {number} [opts.seed] - 随机种子
* @returns {Promise<{taskId: string, preflight: object}>}
*/
async function submitI2VTask({ prompt, media, duration, resolution, negativePrompt, promptExtend, watermark, seed }) {
// 支持本地图片路径自动转base64
const processedMedia = media.map(m => {
if (m.url && fs.existsSync(m.url)) {
// 本地文件 → 转base64
const imgBuffer = fs.readFileSync(m.url);
const ext = path.extname(m.url).toLowerCase();
const mimeMap = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp', '.bmp': 'image/bmp' };
const mime = mimeMap[ext] || 'image/png';
const b64 = imgBuffer.toString('base64');
console.log(`[Wan·I2V] 本地素材已转base64: ${path.basename(m.url)} (${(imgBuffer.length/1024).toFixed(0)}KB)`);
return { type: m.type, url: `data:${mime};base64,${b64}` };
}
return m;
});
const preflight = preflightCheckI2V({ prompt, media: processedMedia, duration, resolution });
if (!preflight.valid) {
console.error('[Wan·预校验] ❌ 参数错误:');
preflight.errors.forEach(e => console.error(`${e}`));
throw new Error(`预校验失败: ${preflight.errors.join('; ')}`);
}
if (preflight.warnings.length > 0) {
console.warn('[Wan·预校验] ⚠️', preflight.warnings.join('; '));
}
const c = preflight.corrected;
const body = {
model: WAN_I2V_SPEC.model,
input: {
prompt: c.prompt,
media: c.media,
},
parameters: {
resolution: c.resolution,
duration: c.duration,
prompt_extend: promptExtend !== false,
watermark: watermark === true,
},
};
if (negativePrompt) body.input.negative_prompt = negativePrompt;
if (seed !== undefined) body.parameters.seed = seed;
const mediaTypes = c.media.map(m => m.type).join('+');
console.log(`[Wan·I2V] 提交: ${mediaTypes} + "${c.prompt.substring(0, 40)}..." ${c.duration}s ${c.resolution}`);
const url = `${BASE_URL}${SUBMIT_PATH}`;
const data = await httpPost(url, body, API_KEY);
const taskId = data.output?.task_id;
if (!taskId) {
throw new Error(`万相未返回task_id: ${JSON.stringify(data).substring(0, 300)}`);
}
console.log(`[Wan·I2V] 任务ID: ${taskId} 状态: ${data.output?.task_status}`);
return { taskId, preflight };
}
// ==================== 查询任务 ====================
/**
* 查询万相任务状态
* @param {string} taskId
* @returns {Promise<{status: 'generating'|'completed'|'failed', videoUrl?: string, videoMeta?: object, rawResponse?: object, error?: string}>}
*/
async function queryWanTask(taskId) {
const url = `${BASE_URL}${QUERY_PATH}${taskId}`;
const data = await httpGet(url, API_KEY);
const taskStatus = (data.output?.task_status || '').toUpperCase();
if (taskStatus === 'SUCCEEDED') {
const videoUrl = data.output?.video_url;
if (!videoUrl) {
return { status: 'failed', error: '任务成功但未返回视频URL', rawResponse: data };
}
const videoMeta = {};
if (data.usage) {
if (data.usage.duration !== undefined) videoMeta.duration = data.usage.duration;
if (data.usage.output_video_duration !== undefined) videoMeta.outputDuration = data.usage.output_video_duration;
if (data.usage.SR !== undefined) videoMeta.resolutionTier = data.usage.SR;
if (data.usage.ratio !== undefined) videoMeta.ratio = data.usage.ratio;
if (data.usage.video_count !== undefined) videoMeta.videoCount = data.usage.video_count;
}
if (data.output?.orig_prompt) videoMeta.origPrompt = data.output.orig_prompt;
return { status: 'completed', videoUrl, videoMeta, rawResponse: data };
}
if (taskStatus === 'FAILED') {
const errMsg = data.output?.message || data.message || '生成失败';
const errCode = data.output?.code || data.code || '';
return { status: 'failed', error: `${errCode}: ${errMsg}`, rawResponse: data };
}
if (taskStatus === 'CANCELED') {
return { status: 'failed', error: '任务被取消', rawResponse: data };
}
if (taskStatus === 'UNKNOWN') {
return { status: 'failed', error: 'task_id不存在或已过期(24h)', rawResponse: data };
}
// PENDING / RUNNING
return { status: 'generating', rawResponse: data };
}
// ==================== 生成视频(完整流程)====================
/**
* 万相文生视频 提交 + 轮询 + 下载
* @param {object} opts
* @param {string} opts.prompt - 视频提示词
* @param {number} [opts.duration=5] - 时长 2-15
* @param {string} [opts.resolution='1080P'] - 分辨率
* @param {string} [opts.ratio='16:9'] - 宽高比
* @param {string} [opts.negativePrompt] - 反向提示词
* @param {string} [opts.outputPath] - 输出路径
* @param {string} [opts.shotId] - 镜编号
* @param {string} [opts.projectKey] - 项目标识
* @returns {Promise<{videoPath: string, taskId: string, videoMeta: object, preflight: object}>}
*/
async function generateWanVideo(opts) {
if (!API_KEY) {
throw new Error('未配置 ALIYUN_BAILIAN_API_KEY。请在 video-ai-system/.env 中设置。');
}
const { taskId, preflight } = await submitT2VTask(opts);
// 轮询
for (let i = 1; i <= MAX_POLL_ATTEMPTS; i++) {
await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
const result = await queryWanTask(taskId);
if (result.status === 'completed') {
// 解析输出路径
let finalPath;
if (opts.outputPath) {
finalPath = opts.outputPath;
} else if (opts.shotId && opts.projectKey) {
const dir = path.join(VIDEO_OUTPUT_ROOT, opts.projectKey);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
finalPath = path.join(dir, `${opts.shotId}-wan.mp4`);
} else {
finalPath = path.join(VIDEO_OUTPUT_ROOT, `wan-${taskId}.mp4`);
}
console.log(`[Wan] 生成完成!下载到: ${finalPath}`);
await downloadFile(result.videoUrl, finalPath);
console.log(`[Wan] ✅ 视频已保存: ${finalPath}`);
if (result.videoMeta && Object.keys(result.videoMeta).length > 0) {
console.log(`[Wan] 元数据:`, JSON.stringify(result.videoMeta));
}
return { videoPath: finalPath, taskId, videoMeta: result.videoMeta || {}, preflight };
}
if (result.status === 'failed') {
throw new Error(`万相生成失败: ${result.error}`);
}
if (i % 5 === 0) console.log(`[Wan] 生成中... (${i}/${MAX_POLL_ATTEMPTS}, ${i * POLL_INTERVAL_MS / 1000}s)`);
}
throw new Error(`万相轮询超时(${MAX_POLL_ATTEMPTS * POLL_INTERVAL_MS / 1000}s)`);
}
/**
* 万相图生视频 提交 + 轮询 + 下载
* @param {object} opts - submitI2VTask 参数 + outputPath/shotId/projectKey
* @returns {Promise<{videoPath: string, taskId: string, videoMeta: object, preflight: object}>}
*/
async function generateWanImageToVideo(opts) {
if (!API_KEY) {
throw new Error('未配置 ALIYUN_BAILIAN_API_KEY。请在 video-ai-system/.env 中设置。');
}
const { taskId, preflight } = await submitI2VTask(opts);
// 轮询
for (let i = 1; i <= MAX_POLL_ATTEMPTS; i++) {
await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
const result = await queryWanTask(taskId);
if (result.status === 'completed') {
let finalPath;
if (opts.outputPath) {
finalPath = opts.outputPath;
} else if (opts.shotId && opts.projectKey) {
const dir = path.join(VIDEO_OUTPUT_ROOT, opts.projectKey);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
finalPath = path.join(dir, `${opts.shotId}-wan-i2v.mp4`);
} else {
finalPath = path.join(VIDEO_OUTPUT_ROOT, `wan-i2v-${taskId}.mp4`);
}
console.log(`[Wan·I2V] 生成完成!下载到: ${finalPath}`);
await downloadFile(result.videoUrl, finalPath);
console.log(`[Wan·I2V] ✅ 视频已保存: ${finalPath}`);
return { videoPath: finalPath, taskId, videoMeta: result.videoMeta || {}, preflight };
}
if (result.status === 'failed') {
throw new Error(`万相图生视频失败: ${result.error}`);
}
if (i % 5 === 0) console.log(`[Wan·I2V] 生成中... (${i}/${MAX_POLL_ATTEMPTS})`);
}
throw new Error(`万相I2V轮询超时(${MAX_POLL_ATTEMPTS * POLL_INTERVAL_MS / 1000}s)`);
}
// ==================== 导出 ====================
module.exports = {
submitT2VTask,
submitI2VTask,
queryWanTask,
generateWanVideo,
generateWanImageToVideo,
preflightCheckT2V,
preflightCheckI2V,
downloadFile,
WAN_T2V_SPEC,
WAN_I2V_SPEC,
BASE_URL,
API_KEY, // 用于检测是否已配置
};

View File

@ -0,0 +1,79 @@
/**
* 万相API测试脚本
* D140 · 铸渊 ICE-GL-ZY001
*
* 验证万相2.7文生视频API是否正常工作
* 用法: node tools/test-wan-api.js
*/
const { submitT2VTask, queryWanTask, API_KEY, WAN_T2V_SPEC } = require('../engines/wan-api-adapter');
async function main() {
console.log('=== 万相2.7 API 测试 ===\n');
// 1. 检查密钥
if (!API_KEY) {
console.error('❌ ALIYUN_BAILIAN_API_KEY 未配置');
console.error(' 请在 video-ai-system/.env 中设置');
process.exit(1);
}
console.log('✅ API Key 已配置:', API_KEY.substring(0, 15) + '...');
console.log('✅ 模型:', WAN_T2V_SPEC.model);
// 2. 提交一个简单的测试任务
const testPrompt = '一只小猫在月光下奔跑,毛发随风飘动,背景是星空和湖泊,电影级画面';
console.log('\n📝 测试提示词:', testPrompt);
console.log('⏱ 时长: 5秒 分辨率: 720P 比例: 16:9\n');
try {
const { taskId, preflight } = await submitT2VTask({
prompt: testPrompt,
duration: 5,
resolution: '720P',
ratio: '16:9',
});
console.log('\n✅ 任务提交成功!');
console.log(' Task ID:', taskId);
console.log(' 预校验:', preflight.valid ? '通过' : '失败');
// 3. 轮询查询
console.log('\n⏳ 轮询查询中...\n');
let pollCount = 0;
const maxPolls = 80;
while (pollCount < maxPolls) {
pollCount++;
await new Promise(r => setTimeout(r, 15000)); // 15秒间隔
const result = await queryWanTask(taskId);
if (result.status === 'completed') {
console.log('\n🎉 视频生成成功!');
console.log(' 视频URL:', result.videoUrl);
console.log(' 元数据:', JSON.stringify(result.videoMeta, null, 2));
process.exit(0);
}
if (result.status === 'failed') {
console.error('\n❌ 生成失败:', result.error);
console.error(' 原始响应:', JSON.stringify(result.rawResponse, null, 2).substring(0, 500));
process.exit(1);
}
// 仍在生成
const elapsed = pollCount * 15;
console.log(` [${elapsed}s] 生成中... (task: ${taskId.substring(0, 8)}...)`);
}
console.error('\n⏰ 轮询超时');
process.exit(1);
} catch (err) {
console.error('\n❌ 测试失败:', err.message);
process.exit(1);
}
}
main();