""" Real-time Strategy Execution Pipeline This module provides real-time strategy execution capabilities that integrate with the existing chart data refresh cycle. It handles incremental strategy calculations, real-time signal generation, and live chart updates. """ import pandas as pd from datetime import datetime, timezone, timedelta from typing import List, Dict, Any, Optional, Callable, Set, Tuple from dataclasses import dataclass, field from threading import Thread, Event, Lock from queue import Queue, Empty import asyncio from concurrent.futures import ThreadPoolExecutor import time from database.operations import get_database_operations, DatabaseOperationError from data.common.data_types import OHLCVCandle from components.charts.data_integration import MarketDataIntegrator from .data_integration import StrategyDataIntegrator, StrategyDataIntegrationConfig from .factory import StrategyFactory from .data_types import StrategyResult, StrategySignal from utils.logger import get_logger # Initialize logger logger = get_logger() @dataclass class RealTimeConfig: """Configuration for real-time strategy execution""" refresh_interval_seconds: int = 30 # How often to check for new data max_strategies_concurrent: int = 5 # Maximum concurrent strategy calculations incremental_calculation: bool = True # Use incremental vs full recalculation signal_batch_size: int = 100 # Batch size for signal storage enable_signal_broadcasting: bool = True # Enable real-time signal broadcasting max_signal_queue_size: int = 1000 # Maximum signals in queue before dropping chart_update_throttle_ms: int = 1000 # Minimum time between chart updates error_retry_attempts: int = 3 # Number of retries on calculation errors error_retry_delay_seconds: int = 5 # Delay between retry attempts @dataclass class StrategyExecutionContext: """Context for strategy execution""" strategy_name: str strategy_config: Dict[str, Any] symbol: str timeframe: str exchange: str = "okx" last_calculation_time: Optional[datetime] = None consecutive_errors: int = 0 is_active: bool = True @dataclass class RealTimeSignal: """Real-time signal with metadata""" strategy_result: StrategyResult context: StrategyExecutionContext generation_time: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) chart_update_required: bool = True class StrategySignalBroadcaster: """ Handles real-time signal broadcasting and distribution. Manages signal queues, chart updates, and database storage for real-time strategy signals. """ def __init__(self, config: RealTimeConfig): """Initialize signal broadcaster.""" self.config = config self.logger = logger self.db_ops = get_database_operations(self.logger) # Signal queues self._signal_queue: Queue[RealTimeSignal] = Queue(maxsize=self.config.max_signal_queue_size) self._chart_update_queue: Queue[RealTimeSignal] = Queue() # Chart update throttling self._last_chart_update = {} # symbol_timeframe -> timestamp self._chart_update_lock = Lock() # Background processing self._processing_thread: Optional[Thread] = None self._stop_event = Event() self._is_running = False # Callback for chart updates self._chart_update_callback: Optional[Callable] = None if self.logger: self.logger.info("StrategySignalBroadcaster: Initialized") def start(self) -> None: """Start the signal broadcasting service.""" if self._is_running: return self._is_running = True self._stop_event.clear() # Start background processing thread self._processing_thread = Thread( target=self._process_signals_loop, name="StrategySignalProcessor", daemon=True ) self._processing_thread.start() if self.logger: self.logger.info("StrategySignalBroadcaster: Started signal processing") def stop(self) -> None: """Stop the signal broadcasting service.""" if not self._is_running: return self._is_running = False self._stop_event.set() if self._processing_thread and self._processing_thread.is_alive(): self._processing_thread.join(timeout=5.0) if self.logger: self.logger.info("StrategySignalBroadcaster: Stopped signal processing") def broadcast_signal(self, signal: RealTimeSignal) -> bool: """ Broadcast a real-time signal. Args: signal: Real-time signal to broadcast Returns: True if signal was queued successfully, False if queue is full """ try: self._signal_queue.put_nowait(signal) return True except: # Queue is full, drop the signal if self.logger: self.logger.warning(f"Signal queue full, dropping signal for {signal.context.symbol}") return False def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None: """Set callback for chart updates.""" self._chart_update_callback = callback def _process_signals_loop(self) -> None: """Main signal processing loop.""" batch_signals = [] while not self._stop_event.is_set(): try: # Collect signals in batches try: signal = self._signal_queue.get(timeout=1.0) batch_signals.append(signal) # Collect more signals if available (up to batch size) while len(batch_signals) < self.config.signal_batch_size: try: signal = self._signal_queue.get_nowait() batch_signals.append(signal) except Empty: break # Process the batch if batch_signals: self._process_signal_batch(batch_signals) batch_signals.clear() except Empty: # No signals to process, continue continue except Exception as e: if self.logger: self.logger.error(f"Error in signal processing loop: {e}") time.sleep(1.0) # Brief pause on error def _process_signal_batch(self, signals: List[RealTimeSignal]) -> None: """Process a batch of signals.""" try: # Store signals in database self._store_signals_batch(signals) # Process chart updates self._process_chart_updates(signals) except Exception as e: if self.logger: self.logger.error(f"Error processing signal batch: {e}") def _store_signals_batch(self, signals: List[RealTimeSignal]) -> None: """Store signals in database.""" try: signal_data = [] for signal in signals: result = signal.strategy_result context = signal.context signal_data.append({ 'strategy_name': context.strategy_name, 'strategy_config': context.strategy_config, 'symbol': context.symbol, 'timeframe': context.timeframe, 'exchange': context.exchange, 'timestamp': result.timestamp, 'signal_type': result.signal.signal_type.value if result.signal else 'HOLD', 'price': float(result.price) if result.price else None, 'confidence': result.confidence, 'signal_metadata': result.metadata or {}, 'generation_time': signal.generation_time }) # Batch insert into database self.db_ops.strategy.store_signals_batch(signal_data) if self.logger: self.logger.debug(f"Stored batch of {len(signals)} real-time signals") except Exception as e: if self.logger: self.logger.error(f"Error storing signal batch: {e}") def _process_chart_updates(self, signals: List[RealTimeSignal]) -> None: """Process chart updates for signals.""" if not self._chart_update_callback: return # Group signals by symbol/timeframe for throttling signal_groups = {} for signal in signals: if not signal.chart_update_required: continue key = f"{signal.context.symbol}_{signal.context.timeframe}" if key not in signal_groups: signal_groups[key] = [] signal_groups[key].append(signal) # Process chart updates with throttling current_time = time.time() * 1000 # milliseconds with self._chart_update_lock: for key, group_signals in signal_groups.items(): last_update = self._last_chart_update.get(key, 0) if current_time - last_update >= self.config.chart_update_throttle_ms: # Update chart with latest signal from group latest_signal = max(group_signals, key=lambda s: s.generation_time) try: self._chart_update_callback(latest_signal) self._last_chart_update[key] = current_time except Exception as e: if self.logger: self.logger.error(f"Error in chart update callback: {e}") def get_signal_stats(self) -> Dict[str, Any]: """Get signal broadcasting statistics.""" return { 'queue_size': self._signal_queue.qsize(), 'chart_queue_size': self._chart_update_queue.qsize(), 'is_running': self._is_running, 'last_chart_updates': dict(self._last_chart_update) } class RealTimeStrategyProcessor: """ Real-time strategy execution processor. Integrates with existing chart data refresh cycle to provide real-time strategy signal generation and broadcasting. """ def __init__(self, config: RealTimeConfig = None): """Initialize real-time strategy processor.""" self.config = config or RealTimeConfig() self.logger = logger # Core components self.data_integrator = StrategyDataIntegrator( StrategyDataIntegrationConfig( cache_timeout_minutes=1, # Shorter cache for real-time enable_indicator_caching=True ) ) self.market_integrator = MarketDataIntegrator() self.strategy_factory = StrategyFactory(self.logger) self.signal_broadcaster = StrategySignalBroadcaster(self.config) # Strategy execution contexts self._execution_contexts: Dict[str, StrategyExecutionContext] = {} self._context_lock = Lock() # Performance tracking self._performance_stats = { 'total_calculations': 0, 'successful_calculations': 0, 'failed_calculations': 0, 'average_calculation_time_ms': 0.0, 'signals_generated': 0, 'last_update_time': None } # Thread pool for concurrent strategy execution self._executor = ThreadPoolExecutor(max_workers=self.config.max_strategies_concurrent) if self.logger: self.logger.info("RealTimeStrategyProcessor: Initialized") def start(self) -> None: """Start the real-time strategy processor.""" self.signal_broadcaster.start() if self.logger: self.logger.info("RealTimeStrategyProcessor: Started") def stop(self) -> None: """Stop the real-time strategy processor.""" self.signal_broadcaster.stop() self._executor.shutdown(wait=True) if self.logger: self.logger.info("RealTimeStrategyProcessor: Stopped") def register_strategy( self, strategy_name: str, strategy_config: Dict[str, Any], symbol: str, timeframe: str, exchange: str = "okx" ) -> str: """ Register a strategy for real-time execution. Args: strategy_name: Name of the strategy strategy_config: Strategy configuration symbol: Trading symbol timeframe: Timeframe exchange: Exchange name Returns: Context ID for the registered strategy """ context_id = f"{strategy_name}_{symbol}_{timeframe}_{exchange}" context = StrategyExecutionContext( strategy_name=strategy_name, strategy_config=strategy_config, symbol=symbol, timeframe=timeframe, exchange=exchange ) with self._context_lock: self._execution_contexts[context_id] = context if self.logger: self.logger.info(f"Registered strategy for real-time execution: {context_id}") return context_id def unregister_strategy(self, context_id: str) -> bool: """ Unregister a strategy from real-time execution. Args: context_id: Context ID to unregister Returns: True if strategy was unregistered, False if not found """ with self._context_lock: if context_id in self._execution_contexts: del self._execution_contexts[context_id] if self.logger: self.logger.info(f"Unregistered strategy: {context_id}") return True return False def execute_realtime_update( self, symbol: str, timeframe: str, exchange: str = "okx" ) -> List[RealTimeSignal]: """ Execute real-time strategy update for new market data. This method should be called when new candle data is available, typically triggered by the chart refresh cycle. Args: symbol: Trading symbol that was updated timeframe: Timeframe that was updated exchange: Exchange name Returns: List of generated real-time signals """ start_time = time.time() generated_signals = [] try: # Find all strategies for this symbol/timeframe matching_contexts = [] with self._context_lock: for context_id, context in self._execution_contexts.items(): if (context.symbol == symbol and context.timeframe == timeframe and context.exchange == exchange and context.is_active): matching_contexts.append((context_id, context)) if not matching_contexts: return generated_signals # Execute strategies concurrently futures = [] for context_id, context in matching_contexts: future = self._executor.submit( self._execute_strategy_context, context_id, context ) futures.append((context_id, future)) # Collect results for context_id, future in futures: try: signals = future.result(timeout=10.0) # 10 second timeout generated_signals.extend(signals) except Exception as e: if self.logger: self.logger.error(f"Error executing strategy {context_id}: {e}") self._handle_strategy_error(context_id, e) # Update performance stats calculation_time = (time.time() - start_time) * 1000 self._update_performance_stats(len(generated_signals), calculation_time, True) if self.logger and generated_signals: self.logger.debug(f"Generated {len(generated_signals)} real-time signals for {symbol} {timeframe}") return generated_signals except Exception as e: if self.logger: self.logger.error(f"Error in real-time strategy execution: {e}") calculation_time = (time.time() - start_time) * 1000 self._update_performance_stats(0, calculation_time, False) return generated_signals def _execute_strategy_context( self, context_id: str, context: StrategyExecutionContext ) -> List[RealTimeSignal]: """Execute a single strategy context.""" try: # Calculate strategy signals if self.config.incremental_calculation and context.last_calculation_time: # Use incremental calculation for better performance results = self._calculate_incremental_signals(context) else: # Full recalculation results = self._calculate_full_signals(context) # Convert to real-time signals real_time_signals = [] for result in results: signal = RealTimeSignal( strategy_result=result, context=context ) real_time_signals.append(signal) # Broadcast signal self.signal_broadcaster.broadcast_signal(signal) # Update context with self._context_lock: context.last_calculation_time = datetime.now(timezone.utc) context.consecutive_errors = 0 return real_time_signals except Exception as e: if self.logger: self.logger.error(f"Error executing strategy context {context_id}: {e}") self._handle_strategy_error(context_id, e) return [] def _calculate_incremental_signals( self, context: StrategyExecutionContext ) -> List[StrategyResult]: """Calculate signals incrementally (only for new data).""" # For this initial implementation, fall back to full calculation # Incremental calculation optimization can be added later return self._calculate_full_signals(context) def _calculate_full_signals( self, context: StrategyExecutionContext ) -> List[StrategyResult]: """Calculate signals with full recalculation.""" return self.data_integrator.calculate_strategy_signals( strategy_name=context.strategy_name, strategy_config=context.strategy_config, symbol=context.symbol, timeframe=context.timeframe, days_back=7, # Use shorter history for real-time exchange=context.exchange, enable_caching=True ) def _handle_strategy_error(self, context_id: str, error: Exception) -> None: """Handle strategy execution error.""" with self._context_lock: if context_id in self._execution_contexts: context = self._execution_contexts[context_id] context.consecutive_errors += 1 # Disable strategy if too many consecutive errors if context.consecutive_errors >= self.config.error_retry_attempts: context.is_active = False if self.logger: self.logger.warning( f"Disabling strategy {context_id} due to consecutive errors: {context.consecutive_errors}" ) def _update_performance_stats( self, signals_generated: int, calculation_time_ms: float, success: bool ) -> None: """Update performance statistics.""" self._performance_stats['total_calculations'] += 1 if success: self._performance_stats['successful_calculations'] += 1 else: self._performance_stats['failed_calculations'] += 1 self._performance_stats['signals_generated'] += signals_generated # Update average calculation time total_calcs = self._performance_stats['total_calculations'] current_avg = self._performance_stats['average_calculation_time_ms'] self._performance_stats['average_calculation_time_ms'] = ( (current_avg * (total_calcs - 1) + calculation_time_ms) / total_calcs ) self._performance_stats['last_update_time'] = datetime.now(timezone.utc) def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None: """Set callback for chart updates.""" self.signal_broadcaster.set_chart_update_callback(callback) def get_active_strategies(self) -> Dict[str, StrategyExecutionContext]: """Get all active strategy contexts.""" with self._context_lock: return { context_id: context for context_id, context in self._execution_contexts.items() if context.is_active } def get_performance_stats(self) -> Dict[str, Any]: """Get real-time execution performance statistics.""" stats = dict(self._performance_stats) stats.update(self.signal_broadcaster.get_signal_stats()) return stats def pause_strategy(self, context_id: str) -> bool: """Pause a strategy (set as inactive).""" with self._context_lock: if context_id in self._execution_contexts: self._execution_contexts[context_id].is_active = False return True return False def resume_strategy(self, context_id: str) -> bool: """Resume a strategy (set as active).""" with self._context_lock: if context_id in self._execution_contexts: context = self._execution_contexts[context_id] context.is_active = True context.consecutive_errors = 0 # Reset error count return True return False # Singleton instance for global access _realtime_processor: Optional[RealTimeStrategyProcessor] = None def get_realtime_strategy_processor(config: RealTimeConfig = None) -> RealTimeStrategyProcessor: """ Get the singleton real-time strategy processor instance. Args: config: Configuration for the processor (only used on first call) Returns: RealTimeStrategyProcessor instance """ global _realtime_processor if _realtime_processor is None: _realtime_processor = RealTimeStrategyProcessor(config) return _realtime_processor def initialize_realtime_strategy_system(config: RealTimeConfig = None) -> RealTimeStrategyProcessor: """ Initialize the real-time strategy system. Args: config: Configuration for the system Returns: Initialized RealTimeStrategyProcessor """ processor = get_realtime_strategy_processor(config) processor.start() return processor def shutdown_realtime_strategy_system() -> None: """Shutdown the real-time strategy system.""" global _realtime_processor if _realtime_processor is not None: _realtime_processor.stop() _realtime_processor = None