""" Dialogue Video Generator Module This module extracts dialogue from video files, segments audio, and generates render props for dialogue-style videos with speech bubbles. Usage: python backend/dialogue_video_generator.py --video path/to/video.mp4 python backend/dialogue_video_generator.py # Auto-detect from assets/0_input/ """ import argparse import json import os import subprocess from pathlib import Path from typing import Dict, Any, List from dotenv import load_dotenv from backend.extractor import VideoExtractor class DialogueVideoGenerator: """Generate dialogue video assets from interview recordings""" def __init__(self, video_path: str = None): """ Initialize the generator Args: video_path: Path to video file. If None, auto-detect from assets/0_input/ """ load_dotenv() self.project_root = Path(__file__).parent.parent self.video_path = self._resolve_video_path(video_path) self.segments_output = self.project_root / "assets/1_processed/video_segments.json" self.script_output = self.project_root / "assets/1_processed/dialogue_script.json" self.props_output = self.project_root / "assets/1_processed/render_props_dialogue.json" self.audio_output_dir = self.project_root / "public/dialogue" def _resolve_video_path(self, video_path: str = None) -> Path: """Resolve video path from input or auto-detect""" if video_path: return Path(video_path) # Auto-detect from assets/0_input/ input_dir = self.project_root / "assets/0_input" video_extensions = ["*.mp4", "*.mov", "*.avi", "*.mkv"] for ext in video_extensions: video_files = list(input_dir.glob(ext)) if video_files: print(f"āœ“ Auto-detected video: {video_files[0].name}") return video_files[0] raise FileNotFoundError("No video file found. Please provide --video path or place video in assets/0_input/") def extract_segments(self) -> Dict[str, Any]: """ Extract audio transcription and timestamps from video Returns: Dictionary with transcription, segments, and metadata """ print(f"šŸ“¹ Extracting segments from: {self.video_path.name}") if not self.video_path.exists(): raise FileNotFoundError(f"Video file not found: {self.video_path}") extractor = VideoExtractor() result = extractor.extract(str(self.video_path)) # Save segments with open(self.segments_output, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) print(f"āœ“ Saved {len(result['segments'])} segments to {self.segments_output.name}") return result def slice_audio(self, input_path: str, start: float, end: float, output_path: str): """Slice audio segment using ffmpeg""" duration = end - start cmd = [ "ffmpeg", "-y", "-ss", str(start), "-t", str(duration), "-i", str(input_path), "-c:a", "libmp3lame", "-b:a", "128k", str(output_path) ] subprocess.run(cmd, check=True, capture_output=True) def identify_speaker(self, text: str, last_speaker: str) -> str: """ Identify speaker based on text content Args: text: Dialogue text last_speaker: Previous segment's speaker (for alternation) Returns: Speaker label: "FP" or "å®¶ę—ä»£č”Øę§˜" """ # FP (Financial Planner) keywords fp_keywords = [ "?", "ćŠé”˜ć„ć„ćŸć—ć¾ć™", "恧恗悇恆恋", "ę•™ćˆć¦ć„ćŸć ć‘ć¾ć™ć‹", "ć‚ć‚ŠćŒćØć†ć”ć–ć„ć¾ć™", "ć„ć‚‰ć£ć—ć‚ƒć„ć¾ć™ć‹", "恝恆恧恙恋" ] is_fp = any(kw in text for kw in fp_keywords) # Short confirmations alternate between speakers if text in ["はい。", "恝恆恧恙恭怂"]: return "FP" if last_speaker == "å®¶ę—ä»£č”Øę§˜" else "å®¶ę—ä»£č”Øę§˜" return "FP" if is_fp else "å®¶ę—ä»£č”Øę§˜" def generate_audio_clips(self, segments: List[Dict]) -> List[Dict]: """ Generate individual audio clips for each dialogue segment Args: segments: List of segment dictionaries with text and timestamps Returns: List of processed segments with speaker labels and audio paths """ print(f"šŸŽ™ļø Generating {len(segments)} audio clips...") # Create output directory self.audio_output_dir.mkdir(parents=True, exist_ok=True) # Find audio file audio_path = self.video_path.parent / f"{self.video_path.stem}_audio.wav" if not audio_path.exists(): raise FileNotFoundError(f"Audio file not found: {audio_path}") processed_segments = [] last_speaker = "å®¶ę—ä»£č”Øę§˜" for i, seg in enumerate(segments): text = seg["text"] speaker = self.identify_speaker(text, last_speaker) last_speaker = speaker # Generate audio clip segment_filename = f"dialogue_seg_{i:03d}.mp3" segment_path = self.audio_output_dir / segment_filename print(f" [{i+1}/{len(segments)}] {text[:30]}...") self.slice_audio(str(audio_path), seg["start"], seg["end"], str(segment_path)) processed_segments.append({ "id": i + 1, "text": text, "duration": seg["end"] - seg["start"], "type": "dialogue", "visualType": "dialogue", "speaker": speaker, "audioPath": f"public/dialogue/{segment_filename}", "brollKeyword": "Japanese interview" }) # Save dialogue script with open(self.script_output, "w", encoding="utf-8") as f: json.dump(processed_segments, f, ensure_ascii=False, indent=2) print(f"āœ“ Generated {len(processed_segments)} dialogue segments") return processed_segments def create_render_props(self, dialogue_script: List[Dict]) -> Dict[str, Any]: """ Create render_props.json for dialogue video Args: dialogue_script: List of dialogue segments Returns: Complete render props dictionary """ print("šŸ“ Creating render props for dialogue video...") # Load original render_props for meta and charts original_props_path = self.project_root / "assets/1_processed/render_props.json" if original_props_path.exists(): with open(original_props_path, "r", encoding="utf-8") as f: original_props = json.load(f) else: # Minimal fallback structure original_props = { "meta": {"title": "対話動画", "currencyUnit": "万円", "duration": 180}, "charts": {}, "assets": {"broll": {}, "videoClips": [], "slides": []} } # Construct new props new_props = { "meta": original_props.get("meta", {}), "charts": original_props.get("charts", {}), "script": dialogue_script, "assets": original_props.get("assets", {}) } # Update title if "title" in new_props["meta"]: new_props["meta"]["title"] = "ć€åÆ¾č©±ē‰ˆć€‘" + new_props["meta"]["title"] # Save render props with open(self.props_output, "w", encoding="utf-8") as f: json.dump(new_props, f, ensure_ascii=False, indent=2) print(f"āœ“ Saved render props to {self.props_output.name}") return new_props def run(self) -> str: """ Execute full dialogue video generation pipeline Returns: Path to generated render_props.json """ print("šŸš€ Starting Dialogue Video Generation Pipeline...") print() # Step 1: Extract segments result = self.extract_segments() segments = result["segments"] print() # Step 2: Generate audio clips processed_segments = self.generate_audio_clips(segments) print() # Step 3: Create render props self.create_render_props(processed_segments) print() print(f"āœ… Pipeline complete! Use the following command to render:") print(f" npx remotion render MainVideo assets/2_output/dialogue_video.mp4 \\") print(f" --props=\"{self.props_output.relative_to(self.project_root)}\"") return str(self.props_output) def main(): """CLI entry point""" parser = argparse.ArgumentParser( description="Dialogue Video Generator - Extract and render interview dialogues", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: python backend/dialogue_video_generator.py python backend/dialogue_video_generator.py --video assets/0_input/interview.mp4 """ ) parser.add_argument( "--video", type=str, help="Path to video file (auto-detect from assets/0_input/ if not provided)", ) args = parser.parse_args() try: generator = DialogueVideoGenerator(video_path=args.video) generator.run() except Exception as e: print(f"\nāŒ Generation failed: {e}") import traceback traceback.print_exc() return 1 return 0 if __name__ == "__main__": exit(main())