#!/usr/bin/env python3 """ audio_pipeline.py — 声画流水线:配音 / 字幕 / BGM / 混音 剧本有声成片的最后一段:把分镜 dialogue 变成配音+字幕+BGM,混入成片。 用法: python audio_pipeline.py voice <分镜.json> <输出目录> # Edge-TTS 配音 python audio_pipeline.py srt <分镜.json> <输出目录> # 生成 SRT 字幕 python audio_pipeline.py bgm <输出目录> <秒数> # 生成 BGM(stable_audio 失败则 ffmpeg 氛围音) python audio_pipeline.py mix <成片.mp4> <配音目录> <输出.mp4> # 混音 """ import os, sys, json, asyncio, subprocess, math VOICES = { "男": "zh-CN-YunxiNeural", "女": "zh-CN-XiaoxiaoNeural", "旁白": "zh-CN-XiaoyiNeural", "少年": "zh-CN-YunyangNeural", "老人": "zh-CN-YunfengNeural", } def load_shots(sb_json): d = json.load(open(sb_json, encoding="utf-8")) shots = d.get("shots", []) return shots def _dialogue_of(shot): """从分镜镜头提取台词(支持字符串或 dict)。""" dl = shot.get("dialogue", "") if not dl: return [] if isinstance(dl, str): return [{"text": dl, "char": shot.get("characters", [""])[0] if shot.get("characters") else "旁白"}] if isinstance(dl, list): out = [] for it in dl: if isinstance(it, str): out.append({"text": it, "char": shot.get("characters", [""])[0] if shot.get("characters") else "旁白"}) elif isinstance(it, dict): out.append({"text": it.get("text", ""), "char": it.get("character", it.get("char", "旁白"))}) return [x for x in out if x["text"].strip()] return [] def generate_voice(sb_json, out_dir): """Edge-TTS 逐条配音。返回 [(shot_no, 文件路径, 台词, 时长)]""" shots = load_shots(sb_json) vdir = os.path.join(out_dir, "voice") os.makedirs(vdir, exist_ok=True) results = [] for i, sh in enumerate(shots): lines = _dialogue_of(sh) if not lines: continue shot_no = sh.get("shot_number", f"S{i+1:02d}") for j, ln in enumerate(lines): voice = VOICES.get(ln["char"], VOICES.get(ln["char"][:1], "zh-CN-XiaoxiaoNeural")) fname = f"shot_{i+1:04d}_{j:02d}.mp3" fpath = os.path.join(vdir, fname) if not os.path.isfile(fpath): text = ln["text"][:500] try: asyncio.run(_tts(text, voice, fpath)) except Exception as e: print(f" ⚠️ 配音失败 {shot_no}:{ln['char']}: {e}") continue dur = _mp3_duration(fpath) results.append({"shot": shot_no, "char": ln["char"], "text": ln["text"], "file": fpath, "duration": dur}) print(f" 🎙 [{shot_no}] {ln['char']}: {ln['text'][:40]} ({dur:.1f}s)") return results async def _tts(text, voice, fpath): import edge_tts c = edge_tts.Communicate(text, voice) await c.save(fpath) def _mp3_duration(path): try: r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration", "-of", "csv=p=0", path], capture_output=True, text=True) return float(r.stdout.strip() or 0) except Exception: return 0 def generate_srt(sb_json, out_dir, voice_results=None): """生成 SRT 字幕(基于配音时长分配时间轴)。""" shots = load_shots(sb_json) sdir = os.path.join(out_dir, "subs") os.makedirs(sdir, exist_ok=True) srt_path = os.path.join(sdir, "subtitles.srt") idx = 1 t = 0.0 lines = [] for i, sh in enumerate(shots): lines_d = _dialogue_of(sh) if not lines_d: continue shot_no = sh.get("shot_number", f"S{i+1:02d}") for ln in lines_d: dur = 2.5 if voice_results: for vr in voice_results: if vr["shot"] == shot_no and vr["char"] == ln["char"]: dur = max(1.0, vr["duration"] + 0.3) break st = t t += dur lines.append(f"{idx}\n{_fmt_srt(st)} --> {_fmt_srt(t)}\n{ln['char']}: {ln['text']}\n") idx += 1 with open(srt_path, "w", encoding="utf-8") as f: f.write("\n".join(lines)) print(f" 📝 字幕: {srt_path} ({idx-1} 条)") return srt_path def _fmt_srt(sec): h = int(sec // 3600); m = int((sec % 3600) // 60); s = int(sec % 60); ms = int((sec - int(sec)) * 1000) return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" def generate_bgm(out_dir, seconds=20, mood="calm ambient background music, gentle piano", use_stable=False): """生成 BGM:默认 ffmpeg 氛围音(稳定);use_stable=True 时先试 stable_audio_3。""" bgm_path = os.path.join(out_dir, "bgm.mp3") ok = False if use_stable: ok = _try_stable_audio(bgm_path, seconds, mood) if ok: print(f" 🎵 BGM(stable_audio_3): {bgm_path} ({seconds}s)") if not ok: _ffmpeg_pad(bgm_path, seconds) print(f" 🎵 BGM(ffmpeg氛围音): {bgm_path} ({seconds}s)") return bgm_path def _try_stable_audio(path, seconds, mood): """调本地 ComfyUI stable_audio_3 生成。返回 bool。""" try: import json, urllib.request, time COMFY = "http://127.0.0.1:8188" sigmas = ", ".join(str(round(1.0 - i / 24, 4)) for i in range(25)) + ", 0.0" wf = { "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "stable_audio_3_medium_base.safetensors"}}, "2": {"class_type": "CLIPTextEncode", "inputs": {"text": mood, "clip": ["1", 1]}}, "3": {"class_type": "CLIPTextEncode", "inputs": {"text": "vocals, singing, noisy, harsh, distorted, speech", "clip": ["1", 1]}}, "4": {"class_type": "ConditioningStableAudio", "inputs": {"positive": ["2", 0], "negative": ["3", 0], "seconds_start": 0, "seconds_total": seconds}}, "5": {"class_type": "EmptyLatentAudio", "inputs": {"seconds": seconds, "batch_size": 1}}, "6": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}}, "9": {"class_type": "ManualSigmas", "inputs": {"sigmas": sigmas}}, "10": {"class_type": "SamplerCustom", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["3", 0], "cfg": 4.0, "noise_seed": 42, "add_noise": True, "sampler": ["6", 0], "sigmas": ["9", 0], "latent_image": ["5", 0]}}, "11": {"class_type": "VAEDecodeAudio", "inputs": {"samples": ["10", 0], "vae": ["1", 2]}}, "12": {"class_type": "SaveAudio", "inputs": {"audio": ["11", 0], "filename_prefix": "eed_bgm_gen"}}, } data = json.dumps({"prompt": wf}).encode() req = urllib.request.Request(f"{COMFY}/prompt", data=data, headers={"Content-Type": "application/json"}) pid = json.loads(urllib.request.urlopen(req, timeout=15).read())["prompt_id"] for _ in range(15): # 30秒内不成功即放弃,避免卡住 time.sleep(2) h = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=8).read()) if pid in h and h[pid].get("outputs"): aud = h[pid]["outputs"].get("12", {}).get("audio", []) if aud: import shutil shutil.copy(os.path.join(os.path.expanduser("~/comfy/ComfyUI/output"), aud[0]["filename"]), path) return True if pid in h and h[pid].get("status", {}).get("status_str") == "error": return False except Exception: return False return False def _ffmpeg_pad(path, seconds): """ffmpeg 生成柔和氛围音垫底(aevalsrc 单输入多频和声,避免多 lavfi 输入偶发失败)。""" expr = (f"0.04*sin(2*PI*220*t)+0.04*sin(2*PI*277*t)+0.04*sin(2*PI*330*t)" f"+0.03*sin(2*PI*165*t)") cmd = ["ffmpeg", "-y", "-f", "lavfi", "-i", f"aevalsrc={expr}:d={seconds}:s=44100", "-af", "lowpass=f=1500,volume=0.35", path] try: r = subprocess.run(cmd, capture_output=True, timeout=60) if r.returncode != 0: print(f" ⚠️ BGM ffmpeg 退出码 {r.returncode}: {(r.stderr or '')[-200:]}") except Exception as e: print(f" ⚠️ BGM ffmpeg 异常: {e}") def mix_final(video, voice_dir, bgm, out_path, subtitles=None): """ffmpeg 混音:成片(无声) + 配音 + BGM(压低)。""" v = ["-i", video] inputs = [video] filters = [] n = 0 vf_in = {} if voice_dir and os.path.isdir(voice_dir): mps = sorted([os.path.join(voice_dir, f) for f in os.listdir(voice_dir) if f.endswith(".mp3")]) for mp in mps: inputs.append(mp); vf_in[f"v{n+1}"] = mp n += 1 if bgm and os.path.isfile(bgm): inputs.append(bgm); vf_in[f"v{n+1}"] = bgm n += 1 if not inputs[1:]: # 无任何音轨:直接复制视频 subprocess.run(["ffmpeg", "-y", "-i", video, "-c", "copy", out_path], capture_output=True) return out_path # 简单混音:所有输入 amix,BGM 音量压低 labels = [f"[{i}:a]" for i in range(1, len(inputs))] vol = [] for i in range(1, len(inputs)): if inputs[i] == bgm: vol.append(f"[{i}:a]volume=0.15[v{i}]") else: vol.append(f"[{i}:a][v{i}]" if False else f"[{i}:a]volume=1.0[v{i}]") fc = ";".join(vol) + ";" + "".join(f"[v{i}]" for i in range(1, len(inputs))) + f"amix=inputs={n}:normalize=0[aout]" cmd = ["ffmpeg", "-y"] + sum([["-i", i] for i in inputs], []) + \ ["-filter_complex", fc, "-map", "0:v", "-map", "[aout]", "-c:v", "copy", "-c:a", "aac", out_path] try: subprocess.run(cmd, capture_output=True, timeout=120) except Exception as e: print(" 混音失败:", e) return out_path if __name__ == "__main__": if len(sys.argv) < 2: print(__doc__); sys.exit(0) cmd = sys.argv[1] if cmd == "voice" and len(sys.argv) >= 4: generate_voice(sys.argv[2], sys.argv[3]) elif cmd == "srt" and len(sys.argv) >= 4: generate_srt(sys.argv[2], sys.argv[3]) elif cmd == "bgm" and len(sys.argv) >= 4: generate_bgm(sys.argv[2], int(float(sys.argv[3])), sys.argv[4] if len(sys.argv) > 4 else "calm ambient background music, gentle piano") elif cmd == "mix" and len(sys.argv) >= 5: mix_final(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) else: print(__doc__)