""" Audio-First Strategy for Video Analysis Workflow: 1. Extract audio from video (FFmpeg) 2. Transcribe audio with timestamps (OpenAI Whisper) 3. Analyze transcript to identify important segments (Gemini) 4. Generate VideoAnalysisResult with precise timestamps Pros: - Lower token cost (text vs video frames) - Precise timestamps from Whisper - Fast processing Cons: - Misses visual information (charts, expressions) - Only suitable for dialogue-heavy videos """ import os from pathlib import Path from typing import Any from openai import OpenAI from .schemas import VideoAnalysisResult, VideoSegment, KeyMoment from .utils import extract_audio, get_video_duration class AudioFirstStrategy: """Implements audio-first video analysis strategy.""" def __init__(self, config: dict[str, Any] | None = None): """ Initialize strategy with configuration. Args: config: Optional configuration dict - language: Language code (default: "ja") - model: Whisper model (default: "whisper-1") - prompt: Optional prompt to guide transcription """ self.config = config or {} self.language = self.config.get("language", "ja") self.model = self.config.get("model", "whisper-1") self.prompt = self.config.get("prompt", None) # Initialize OpenAI client api_key = os.getenv("OPENAI_API_KEY") if not api_key: raise ValueError("OPENAI_API_KEY not found in environment") self.client = OpenAI(api_key=api_key) def analyze(self, video_path: str) -> VideoAnalysisResult: """ Analyze video using audio-first strategy. Args: video_path: Path to video file Returns: VideoAnalysisResult with segments and key moments """ # Step 1: Extract audio print(f"📹 Extracting audio from {video_path}...") audio_path = extract_audio(video_path) # Step 2: Transcribe with Whisper print("🎤 Transcribing audio with Whisper...") transcript_data = self._transcribe_audio(audio_path) # Step 3: Convert to segments print("📝 Processing transcript segments...") segments = self._create_segments(transcript_data) # Step 4: Identify key moments print("⭐ Identifying key moments...") key_moments = self._identify_key_moments(transcript_data) # Step 5: Get video duration duration = get_video_duration(video_path) # Clean up temporary audio file if os.path.exists(audio_path): os.remove(audio_path) return VideoAnalysisResult( source_video=video_path, duration=duration, strategy="audio-first", segments=segments, key_moments=key_moments, metadata={ "language": self.language, "model": self.model, "total_segments": len(segments), "total_key_moments": len(key_moments), } ) def _transcribe_audio(self, audio_path: str) -> dict: """ Transcribe audio using OpenAI Whisper API. Args: audio_path: Path to audio file Returns: Whisper API response with segments and timestamps """ with open(audio_path, "rb") as audio_file: # Use verbose_json response format to get word-level timestamps transcript = self.client.audio.transcriptions.create( model=self.model, file=audio_file, language=self.language, response_format="verbose_json", timestamp_granularities=["segment"] ) # Convert to dict for easier processing return transcript.model_dump() def _create_segments(self, transcript_data: dict) -> list[VideoSegment]: """ Convert Whisper segments to VideoSegment objects. Args: transcript_data: Whisper API response Returns: List of VideoSegment objects """ segments = [] for seg in transcript_data.get("segments", []): segment = VideoSegment( start_time=seg["start"], end_time=seg["end"], description=f"Dialogue segment {seg['id'] + 1}", transcript=seg["text"].strip(), importance="medium", # Will be updated by analysis tags=["dialogue"], ) segments.append(segment) return segments def _identify_key_moments(self, transcript_data: dict) -> list[KeyMoment]: """ Identify key moments from transcript. For now, this is a simple heuristic. In the future, we can use Gemini to analyze the full transcript and identify: - Important quotes - Topic transitions - Emotional moments Args: transcript_data: Whisper API response Returns: List of KeyMoment objects """ key_moments = [] # Simple heuristic: Mark first segment as key moment segments = transcript_data.get("segments", []) if segments: first_seg = segments[0] key_moments.append( KeyMoment( timestamp=first_seg["start"], type="quote", content=first_seg["text"].strip(), context="Opening statement" ) ) # TODO: Add Gemini-based analysis here # - Analyze full transcript # - Identify topic changes # - Extract important quotes # - Mark emotional moments return key_moments