TCPDashboard/strategies/realtime_execution.py

"""
Real-time Strategy Execution Pipeline

This module provides real-time strategy execution capabilities that integrate
with the existing chart data refresh cycle. It handles incremental strategy
calculations, real-time signal generation, and live chart updates.
"""

import pandas as pd
from datetime import datetime, timezone, timedelta
from typing import List, Dict, Any, Optional, Callable, Set, Tuple
from dataclasses import dataclass, field
from threading import Thread, Event, Lock
from queue import Queue, Empty
import asyncio
from concurrent.futures import ThreadPoolExecutor
import time

from database.operations import get_database_operations, DatabaseOperationError
from data.common.data_types import OHLCVCandle
from components.charts.data_integration import MarketDataIntegrator
from .data_integration import StrategyDataIntegrator, StrategyDataIntegrationConfig
from .factory import StrategyFactory
from .data_types import StrategyResult, StrategySignal
from utils.logger import get_logger

# Initialize logger
logger = get_logger()


@dataclass
class RealTimeConfig:
    """Configuration for real-time strategy execution"""
    refresh_interval_seconds: int = 30  # How often to check for new data
    max_strategies_concurrent: int = 5   # Maximum concurrent strategy calculations
    incremental_calculation: bool = True  # Use incremental vs full recalculation
    signal_batch_size: int = 100         # Batch size for signal storage
    enable_signal_broadcasting: bool = True  # Enable real-time signal broadcasting
    max_signal_queue_size: int = 1000    # Maximum signals in queue before dropping
    chart_update_throttle_ms: int = 1000 # Minimum time between chart updates
    error_retry_attempts: int = 3        # Number of retries on calculation errors
    error_retry_delay_seconds: int = 5   # Delay between retry attempts


@dataclass
class StrategyExecutionContext:
    """Context for strategy execution"""
    strategy_name: str
    strategy_config: Dict[str, Any]
    symbol: str
    timeframe: str
    exchange: str = "okx"
    last_calculation_time: Optional[datetime] = None
    consecutive_errors: int = 0
    is_active: bool = True


@dataclass
class RealTimeSignal:
    """Real-time signal with metadata"""
    strategy_result: StrategyResult
    context: StrategyExecutionContext
    generation_time: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
    chart_update_required: bool = True


class StrategySignalBroadcaster:
    """
    Handles real-time signal broadcasting and distribution.
    
    Manages signal queues, chart updates, and database storage
    for real-time strategy signals.
    """
    
    def __init__(self, config: RealTimeConfig):
        """Initialize signal broadcaster."""
        self.config = config
        self.logger = logger
        self.db_ops = get_database_operations(self.logger)
        
        # Signal queues
        self._signal_queue: Queue[RealTimeSignal] = Queue(maxsize=self.config.max_signal_queue_size)
        self._chart_update_queue: Queue[RealTimeSignal] = Queue()
        
        # Chart update throttling
        self._last_chart_update = {}  # symbol_timeframe -> timestamp
        self._chart_update_lock = Lock()
        
        # Background processing
        self._processing_thread: Optional[Thread] = None
        self._stop_event = Event()
        self._is_running = False
        
        # Callback for chart updates
        self._chart_update_callback: Optional[Callable] = None
        
        if self.logger:
            self.logger.info("StrategySignalBroadcaster: Initialized")
    
    def start(self) -> None:
        """Start the signal broadcasting service."""
        if self._is_running:
            return
        
        self._is_running = True
        self._stop_event.clear()
        
        # Start background processing thread
        self._processing_thread = Thread(
            target=self._process_signals_loop,
            name="StrategySignalProcessor",
            daemon=True
        )
        self._processing_thread.start()
        
        if self.logger:
            self.logger.info("StrategySignalBroadcaster: Started signal processing")
    
    def stop(self) -> None:
        """Stop the signal broadcasting service."""
        if not self._is_running:
            return
        
        self._is_running = False
        self._stop_event.set()
        
        if self._processing_thread and self._processing_thread.is_alive():
            self._processing_thread.join(timeout=5.0)
        
        if self.logger:
            self.logger.info("StrategySignalBroadcaster: Stopped signal processing")
    
    def broadcast_signal(self, signal: RealTimeSignal) -> bool:
        """
        Broadcast a real-time signal.
        
        Args:
            signal: Real-time signal to broadcast
            
        Returns:
            True if signal was queued successfully, False if queue is full
        """
        try:
            self._signal_queue.put_nowait(signal)
            return True
        except:
            # Queue is full, drop the signal
            if self.logger:
                self.logger.warning(f"Signal queue full, dropping signal for {signal.context.symbol}")
            return False
    
    def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None:
        """Set callback for chart updates."""
        self._chart_update_callback = callback
    
    def _process_signals_loop(self) -> None:
        """Main signal processing loop."""
        batch_signals = []
        
        while not self._stop_event.is_set():
            try:
                # Collect signals in batches
                try:
                    signal = self._signal_queue.get(timeout=1.0)
                    batch_signals.append(signal)
                    
                    # Collect more signals if available (up to batch size)
                    while len(batch_signals) < self.config.signal_batch_size:
                        try:
                            signal = self._signal_queue.get_nowait()
                            batch_signals.append(signal)
                        except Empty:
                            break
                    
                    # Process the batch
                    if batch_signals:
                        self._process_signal_batch(batch_signals)
                        batch_signals.clear()
                
                except Empty:
                    # No signals to process, continue
                    continue
                    
            except Exception as e:
                if self.logger:
                    self.logger.error(f"Error in signal processing loop: {e}")
                time.sleep(1.0)  # Brief pause on error
    
    def _process_signal_batch(self, signals: List[RealTimeSignal]) -> None:
        """Process a batch of signals."""
        try:
            # Store signals in database
            self._store_signals_batch(signals)
            
            # Process chart updates
            self._process_chart_updates(signals)
            
        except Exception as e:
            if self.logger:
                self.logger.error(f"Error processing signal batch: {e}")
    
    def _store_signals_batch(self, signals: List[RealTimeSignal]) -> None:
        """Store signals in database."""
        try:
            signal_data = []
            for signal in signals:
                result = signal.strategy_result
                context = signal.context
                
                # Only store BUY or SELL signals, not HOLD
                if result.signal and result.signal.signal_type in [SignalType.BUY, SignalType.SELL]:
                    signal_data.append({
                        'strategy_name': context.strategy_name,
                        'strategy_config': context.strategy_config,
                        'symbol': context.symbol,
                        'timeframe': context.timeframe,
                        'exchange': context.exchange,
                        'timestamp': result.timestamp,
                        'signal_type': result.signal.signal_type.value,
                        'price': float(result.price) if result.price else None,
                        'confidence': result.confidence,
                        'signal_metadata': result.metadata or {},
                        'generation_time': signal.generation_time
                    })
            
            if signal_data: # Only call store_signals_batch if there are signals to store
                # Batch insert into database
                self.db_ops.strategy.store_signals_batch(signal_data)
                
                if self.logger:
                    self.logger.debug(f"Stored batch of {len(signal_data)} real-time signals")
            else:
                if self.logger:
                    self.logger.debug("No BUY/SELL signals to store in this batch.")
                
        except Exception as e:
            if self.logger:
                self.logger.error(f"Error storing signal batch: {e}")
    
    def _process_chart_updates(self, signals: List[RealTimeSignal]) -> None:
        """Process chart updates for signals."""
        if not self._chart_update_callback:
            return
        
        # Group signals by symbol/timeframe for throttling
        signal_groups = {}
        for signal in signals:
            if not signal.chart_update_required:
                continue
                
            key = f"{signal.context.symbol}_{signal.context.timeframe}"
            if key not in signal_groups:
                signal_groups[key] = []
            signal_groups[key].append(signal)
        
        # Process chart updates with throttling
        current_time = time.time() * 1000  # milliseconds
        
        with self._chart_update_lock:
            for key, group_signals in signal_groups.items():
                last_update = self._last_chart_update.get(key, 0)
                
                if current_time - last_update >= self.config.chart_update_throttle_ms:
                    # Update chart with latest signal from group
                    latest_signal = max(group_signals, key=lambda s: s.generation_time)
                    
                    try:
                        self._chart_update_callback(latest_signal)
                        self._last_chart_update[key] = current_time
                    except Exception as e:
                        if self.logger:
                            self.logger.error(f"Error in chart update callback: {e}")
    
    def get_signal_stats(self) -> Dict[str, Any]:
        """Get signal broadcasting statistics."""
        return {
            'queue_size': self._signal_queue.qsize(),
            'chart_queue_size': self._chart_update_queue.qsize(),
            'is_running': self._is_running,
            'last_chart_updates': dict(self._last_chart_update)
        }


class RealTimeStrategyProcessor:
    """
    Real-time strategy execution processor.
    
    Integrates with existing chart data refresh cycle to provide
    real-time strategy signal generation and broadcasting.
    """
    
    def __init__(self, config: RealTimeConfig = None):
        """Initialize real-time strategy processor."""
        self.config = config or RealTimeConfig()
        self.logger = logger
        
        # Core components
        self.data_integrator = StrategyDataIntegrator(
            StrategyDataIntegrationConfig(
                cache_timeout_minutes=1,  # Shorter cache for real-time
                enable_indicator_caching=True
            )
        )
        self.market_integrator = MarketDataIntegrator()
        self.strategy_factory = StrategyFactory(self.logger)
        self.signal_broadcaster = StrategySignalBroadcaster(self.config)
        
        # Strategy execution contexts
        self._execution_contexts: Dict[str, StrategyExecutionContext] = {}
        self._context_lock = Lock()
        
        # Performance tracking
        self._performance_stats = {
            'total_calculations': 0,
            'successful_calculations': 0,
            'failed_calculations': 0,
            'average_calculation_time_ms': 0.0,
            'signals_generated': 0,
            'last_update_time': None
        }
        
        # Thread pool for concurrent strategy execution
        self._executor = ThreadPoolExecutor(max_workers=self.config.max_strategies_concurrent)
        
        if self.logger:
            self.logger.info("RealTimeStrategyProcessor: Initialized")
    
    def start(self) -> None:
        """Start the real-time strategy processor."""
        self.signal_broadcaster.start()
        if self.logger:
            self.logger.info("RealTimeStrategyProcessor: Started")
    
    def stop(self) -> None:
        """Stop the real-time strategy processor."""
        self.signal_broadcaster.stop()
        self._executor.shutdown(wait=True)
        if self.logger:
            self.logger.info("RealTimeStrategyProcessor: Stopped")
    
    def register_strategy(
        self,
        strategy_name: str,
        strategy_config: Dict[str, Any],
        symbol: str,
        timeframe: str,
        exchange: str = "okx"
    ) -> str:
        """
        Register a strategy for real-time execution.
        
        Args:
            strategy_name: Name of the strategy
            strategy_config: Strategy configuration
            symbol: Trading symbol
            timeframe: Timeframe
            exchange: Exchange name
            
        Returns:
            Context ID for the registered strategy
        """
        context_id = f"{strategy_name}_{symbol}_{timeframe}_{exchange}"
        
        context = StrategyExecutionContext(
            strategy_name=strategy_name,
            strategy_config=strategy_config,
            symbol=symbol,
            timeframe=timeframe,
            exchange=exchange
        )
        
        with self._context_lock:
            self._execution_contexts[context_id] = context
        
        if self.logger:
            self.logger.info(f"Registered strategy for real-time execution: {context_id}")
        
        return context_id
    
    def unregister_strategy(self, context_id: str) -> bool:
        """
        Unregister a strategy from real-time execution.
        
        Args:
            context_id: Context ID to unregister
            
        Returns:
            True if strategy was unregistered, False if not found
        """
        with self._context_lock:
            if context_id in self._execution_contexts:
                del self._execution_contexts[context_id]
                if self.logger:
                    self.logger.info(f"Unregistered strategy: {context_id}")
                return True
        return False
    
    def execute_realtime_update(
        self,
        symbol: str,
        timeframe: str,
        exchange: str = "okx"
    ) -> List[RealTimeSignal]:
        """
        Execute real-time strategy update for new market data.
        
        This method should be called when new candle data is available,
        typically triggered by the chart refresh cycle.
        
        Args:
            symbol: Trading symbol that was updated
            timeframe: Timeframe that was updated
            exchange: Exchange name
            
        Returns:
            List of generated real-time signals
        """
        start_time = time.time()
        generated_signals = []
        
        try:
            # Find all strategies for this symbol/timeframe
            matching_contexts = []
            with self._context_lock:
                for context_id, context in self._execution_contexts.items():
                    if (context.symbol == symbol and 
                        context.timeframe == timeframe and 
                        context.exchange == exchange and 
                        context.is_active):
                        matching_contexts.append((context_id, context))
            
            if not matching_contexts:
                return generated_signals
            
            # Execute strategies concurrently
            futures = []
            for context_id, context in matching_contexts:
                future = self._executor.submit(
                    self._execute_strategy_context,
                    context_id,
                    context
                )
                futures.append((context_id, future))
            
            # Collect results
            for context_id, future in futures:
                try:
                    signals = future.result(timeout=10.0)  # 10 second timeout
                    generated_signals.extend(signals)
                except Exception as e:
                    if self.logger:
                        self.logger.error(f"Error executing strategy {context_id}: {e}")
                    self._handle_strategy_error(context_id, e)
            
            # Update performance stats
            calculation_time = (time.time() - start_time) * 1000
            self._update_performance_stats(len(generated_signals), calculation_time, True)
            
            if self.logger and generated_signals:
                self.logger.debug(f"Generated {len(generated_signals)} real-time signals for {symbol} {timeframe}")
            
            return generated_signals
            
        except Exception as e:
            if self.logger:
                self.logger.error(f"Error in real-time strategy execution: {e}")
            calculation_time = (time.time() - start_time) * 1000
            self._update_performance_stats(0, calculation_time, False)
            return generated_signals
    
    def _execute_strategy_context(
        self,
        context_id: str,
        context: StrategyExecutionContext
    ) -> List[RealTimeSignal]:
        """Execute a single strategy context."""
        try:
            # Calculate strategy signals
            if self.config.incremental_calculation and context.last_calculation_time:
                # Use incremental calculation for better performance
                results = self._calculate_incremental_signals(context)
            else:
                # Full recalculation
                results = self._calculate_full_signals(context)
            
            # Convert to real-time signals
            real_time_signals = []
            for result in results:
                signal = RealTimeSignal(
                    strategy_result=result,
                    context=context
                )
                real_time_signals.append(signal)
                
                # Broadcast signal
                self.signal_broadcaster.broadcast_signal(signal)
            
            # Update context
            with self._context_lock:
                context.last_calculation_time = datetime.now(timezone.utc)
                context.consecutive_errors = 0
            
            return real_time_signals
            
        except Exception as e:
            if self.logger:
                self.logger.error(f"Error executing strategy context {context_id}: {e}")
            self._handle_strategy_error(context_id, e)
            return []
    
    def _calculate_incremental_signals(
        self,
        context: StrategyExecutionContext
    ) -> List[StrategyResult]:
        """Calculate signals incrementally (only for new data)."""
        # For this initial implementation, fall back to full calculation
        # Incremental calculation optimization can be added later
        return self._calculate_full_signals(context)
    
    def _calculate_full_signals(
        self,
        context: StrategyExecutionContext
    ) -> List[StrategyResult]:
        """Calculate signals with full recalculation."""
        return self.data_integrator.calculate_strategy_signals(
            strategy_name=context.strategy_name,
            strategy_config=context.strategy_config,
            symbol=context.symbol,
            timeframe=context.timeframe,
            days_back=7,  # Use shorter history for real-time
            exchange=context.exchange,
            enable_caching=True
        )
    
    def _handle_strategy_error(self, context_id: str, error: Exception) -> None:
        """Handle strategy execution error."""
        with self._context_lock:
            if context_id in self._execution_contexts:
                context = self._execution_contexts[context_id]
                context.consecutive_errors += 1
                
                # Disable strategy if too many consecutive errors
                if context.consecutive_errors >= self.config.error_retry_attempts:
                    context.is_active = False
                    if self.logger:
                        self.logger.warning(
                            f"Disabling strategy {context_id} due to consecutive errors: {context.consecutive_errors}"
                        )
    
    def _update_performance_stats(
        self,
        signals_generated: int,
        calculation_time_ms: float,
        success: bool
    ) -> None:
        """Update performance statistics."""
        self._performance_stats['total_calculations'] += 1
        if success:
            self._performance_stats['successful_calculations'] += 1
        else:
            self._performance_stats['failed_calculations'] += 1
        
        self._performance_stats['signals_generated'] += signals_generated
        
        # Update average calculation time
        total_calcs = self._performance_stats['total_calculations']
        current_avg = self._performance_stats['average_calculation_time_ms']
        self._performance_stats['average_calculation_time_ms'] = (
            (current_avg * (total_calcs - 1) + calculation_time_ms) / total_calcs
        )
        
        self._performance_stats['last_update_time'] = datetime.now(timezone.utc)
    
    def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None:
        """Set callback for chart updates."""
        self.signal_broadcaster.set_chart_update_callback(callback)
    
    def get_active_strategies(self) -> Dict[str, StrategyExecutionContext]:
        """Get all active strategy contexts."""
        with self._context_lock:
            return {
                context_id: context 
                for context_id, context in self._execution_contexts.items()
                if context.is_active
            }
    
    def get_performance_stats(self) -> Dict[str, Any]:
        """Get real-time execution performance statistics."""
        stats = dict(self._performance_stats)
        stats.update(self.signal_broadcaster.get_signal_stats())
        return stats
    
    def pause_strategy(self, context_id: str) -> bool:
        """Pause a strategy (set as inactive)."""
        with self._context_lock:
            if context_id in self._execution_contexts:
                self._execution_contexts[context_id].is_active = False
                return True
        return False
    
    def resume_strategy(self, context_id: str) -> bool:
        """Resume a strategy (set as active)."""
        with self._context_lock:
            if context_id in self._execution_contexts:
                context = self._execution_contexts[context_id]
                context.is_active = True
                context.consecutive_errors = 0  # Reset error count
                return True
        return False


# Singleton instance for global access
_realtime_processor: Optional[RealTimeStrategyProcessor] = None


def get_realtime_strategy_processor(config: RealTimeConfig = None) -> RealTimeStrategyProcessor:
    """
    Get the singleton real-time strategy processor instance.
    
    Args:
        config: Configuration for the processor (only used on first call)
        
    Returns:
        RealTimeStrategyProcessor instance
    """
    global _realtime_processor
    
    if _realtime_processor is None:
        _realtime_processor = RealTimeStrategyProcessor(config)
    
    return _realtime_processor


def initialize_realtime_strategy_system(config: RealTimeConfig = None) -> RealTimeStrategyProcessor:
    """
    Initialize the real-time strategy system.
    
    Args:
        config: Configuration for the system
        
    Returns:
        Initialized RealTimeStrategyProcessor
    """
    processor = get_realtime_strategy_processor(config)
    processor.start()
    return processor


def shutdown_realtime_strategy_system() -> None:
    """Shutdown the real-time strategy system."""
    global _realtime_processor
    
    if _realtime_processor is not None:
        _realtime_processor.stop()
        _realtime_processor = None