AI-video-lifeplan-demo/backend/video_processor/strategy_audio.py

"""
Audio-First Strategy for Video Analysis

Workflow:
1. Extract audio from video (FFmpeg)
2. Transcribe audio with timestamps (OpenAI Whisper)
3. Analyze transcript to identify important segments (Gemini)
4. Generate VideoAnalysisResult with precise timestamps

Pros:
- Lower token cost (text vs video frames)
- Precise timestamps from Whisper
- Fast processing

Cons:
- Misses visual information (charts, expressions)
- Only suitable for dialogue-heavy videos
"""

import os
from pathlib import Path
from typing import Any

from openai import OpenAI

from .schemas import VideoAnalysisResult, VideoSegment, KeyMoment
from .utils import extract_audio, get_video_duration


class AudioFirstStrategy:
    """Implements audio-first video analysis strategy."""
    
    def __init__(self, config: dict[str, Any] | None = None):
        """
        Initialize strategy with configuration.
        
        Args:
            config: Optional configuration dict
                - language: Language code (default: "ja")
                - model: Whisper model (default: "whisper-1")
                - prompt: Optional prompt to guide transcription
        """
        self.config = config or {}
        self.language = self.config.get("language", "ja")
        self.model = self.config.get("model", "whisper-1")
        self.prompt = self.config.get("prompt", None)
        
        # Initialize OpenAI client
        api_key = os.getenv("OPENAI_API_KEY")
        if not api_key:
            raise ValueError("OPENAI_API_KEY not found in environment")
        self.client = OpenAI(api_key=api_key)
    
    def analyze(self, video_path: str) -> VideoAnalysisResult:
        """
        Analyze video using audio-first strategy.
        
        Args:
            video_path: Path to video file
            
        Returns:
            VideoAnalysisResult with segments and key moments
        """
        # Step 1: Extract audio
        print(f"📹 Extracting audio from {video_path}...")
        audio_path = extract_audio(video_path)
        
        # Step 2: Transcribe with Whisper
        print("🎤 Transcribing audio with Whisper...")
        transcript_data = self._transcribe_audio(audio_path)
        
        # Step 3: Convert to segments
        print("📝 Processing transcript segments...")
        segments = self._create_segments(transcript_data)
        
        # Step 4: Identify key moments
        print("⭐ Identifying key moments...")
        key_moments = self._identify_key_moments(transcript_data)
        
        # Step 5: Get video duration
        duration = get_video_duration(video_path)
        
        # Clean up temporary audio file
        if os.path.exists(audio_path):
            os.remove(audio_path)
        
        return VideoAnalysisResult(
            source_video=video_path,
            duration=duration,
            strategy="audio-first",
            segments=segments,
            key_moments=key_moments,
            metadata={
                "language": self.language,
                "model": self.model,
                "total_segments": len(segments),
                "total_key_moments": len(key_moments),
            }
        )
    
    def _transcribe_audio(self, audio_path: str) -> dict:
        """
        Transcribe audio using OpenAI Whisper API.
        
        Args:
            audio_path: Path to audio file
            
        Returns:
            Whisper API response with segments and timestamps
        """
        with open(audio_path, "rb") as audio_file:
            # Use verbose_json response format to get word-level timestamps
            transcript = self.client.audio.transcriptions.create(
                model=self.model,
                file=audio_file,
                language=self.language,
                response_format="verbose_json",
                timestamp_granularities=["segment"]
            )
        
        # Convert to dict for easier processing
        return transcript.model_dump()
    
    def _create_segments(self, transcript_data: dict) -> list[VideoSegment]:
        """
        Convert Whisper segments to VideoSegment objects.
        
        Args:
            transcript_data: Whisper API response
            
        Returns:
            List of VideoSegment objects
        """
        segments = []
        
        for seg in transcript_data.get("segments", []):
            segment = VideoSegment(
                start_time=seg["start"],
                end_time=seg["end"],
                description=f"Dialogue segment {seg['id'] + 1}",
                transcript=seg["text"].strip(),
                importance="medium",  # Will be updated by analysis
                tags=["dialogue"],
            )
            segments.append(segment)
        
        return segments
    
    def _identify_key_moments(self, transcript_data: dict) -> list[KeyMoment]:
        """
        Identify key moments from transcript.
        
        For now, this is a simple heuristic. In the future, we can use
        Gemini to analyze the full transcript and identify:
        - Important quotes
        - Topic transitions
        - Emotional moments
        
        Args:
            transcript_data: Whisper API response
            
        Returns:
            List of KeyMoment objects
        """
        key_moments = []
        
        # Simple heuristic: Mark first segment as key moment
        segments = transcript_data.get("segments", [])
        if segments:
            first_seg = segments[0]
            key_moments.append(
                KeyMoment(
                    timestamp=first_seg["start"],
                    type="quote",
                    content=first_seg["text"].strip(),
                    context="Opening statement"
                )
            )
        
        # TODO: Add Gemini-based analysis here
        # - Analyze full transcript
        # - Identify topic changes
        # - Extract important quotes
        # - Mark emotional moments
        
        return key_moments