061b467d43
- fixing data-type circular import - removing HOLD signal from saving to database to reduce data size
655 lines
24 KiB
Python
655 lines
24 KiB
Python
"""
|
|
Real-time Strategy Execution Pipeline
|
|
|
|
This module provides real-time strategy execution capabilities that integrate
|
|
with the existing chart data refresh cycle. It handles incremental strategy
|
|
calculations, real-time signal generation, and live chart updates.
|
|
"""
|
|
|
|
import pandas as pd
|
|
from datetime import datetime, timezone, timedelta
|
|
from typing import List, Dict, Any, Optional, Callable, Set, Tuple
|
|
from dataclasses import dataclass, field
|
|
from threading import Thread, Event, Lock
|
|
from queue import Queue, Empty
|
|
import asyncio
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import time
|
|
|
|
from database.operations import get_database_operations, DatabaseOperationError
|
|
from data.common.data_types import OHLCVCandle
|
|
from components.charts.data_integration import MarketDataIntegrator
|
|
from .data_integration import StrategyDataIntegrator, StrategyDataIntegrationConfig
|
|
from .factory import StrategyFactory
|
|
from .data_types import StrategyResult, StrategySignal
|
|
from utils.logger import get_logger
|
|
|
|
# Initialize logger
|
|
logger = get_logger()
|
|
|
|
|
|
@dataclass
|
|
class RealTimeConfig:
|
|
"""Configuration for real-time strategy execution"""
|
|
refresh_interval_seconds: int = 30 # How often to check for new data
|
|
max_strategies_concurrent: int = 5 # Maximum concurrent strategy calculations
|
|
incremental_calculation: bool = True # Use incremental vs full recalculation
|
|
signal_batch_size: int = 100 # Batch size for signal storage
|
|
enable_signal_broadcasting: bool = True # Enable real-time signal broadcasting
|
|
max_signal_queue_size: int = 1000 # Maximum signals in queue before dropping
|
|
chart_update_throttle_ms: int = 1000 # Minimum time between chart updates
|
|
error_retry_attempts: int = 3 # Number of retries on calculation errors
|
|
error_retry_delay_seconds: int = 5 # Delay between retry attempts
|
|
|
|
|
|
@dataclass
|
|
class StrategyExecutionContext:
|
|
"""Context for strategy execution"""
|
|
strategy_name: str
|
|
strategy_config: Dict[str, Any]
|
|
symbol: str
|
|
timeframe: str
|
|
exchange: str = "okx"
|
|
last_calculation_time: Optional[datetime] = None
|
|
consecutive_errors: int = 0
|
|
is_active: bool = True
|
|
|
|
|
|
@dataclass
|
|
class RealTimeSignal:
|
|
"""Real-time signal with metadata"""
|
|
strategy_result: StrategyResult
|
|
context: StrategyExecutionContext
|
|
generation_time: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
|
chart_update_required: bool = True
|
|
|
|
|
|
class StrategySignalBroadcaster:
|
|
"""
|
|
Handles real-time signal broadcasting and distribution.
|
|
|
|
Manages signal queues, chart updates, and database storage
|
|
for real-time strategy signals.
|
|
"""
|
|
|
|
def __init__(self, config: RealTimeConfig):
|
|
"""Initialize signal broadcaster."""
|
|
self.config = config
|
|
self.logger = logger
|
|
self.db_ops = get_database_operations(self.logger)
|
|
|
|
# Signal queues
|
|
self._signal_queue: Queue[RealTimeSignal] = Queue(maxsize=self.config.max_signal_queue_size)
|
|
self._chart_update_queue: Queue[RealTimeSignal] = Queue()
|
|
|
|
# Chart update throttling
|
|
self._last_chart_update = {} # symbol_timeframe -> timestamp
|
|
self._chart_update_lock = Lock()
|
|
|
|
# Background processing
|
|
self._processing_thread: Optional[Thread] = None
|
|
self._stop_event = Event()
|
|
self._is_running = False
|
|
|
|
# Callback for chart updates
|
|
self._chart_update_callback: Optional[Callable] = None
|
|
|
|
if self.logger:
|
|
self.logger.info("StrategySignalBroadcaster: Initialized")
|
|
|
|
def start(self) -> None:
|
|
"""Start the signal broadcasting service."""
|
|
if self._is_running:
|
|
return
|
|
|
|
self._is_running = True
|
|
self._stop_event.clear()
|
|
|
|
# Start background processing thread
|
|
self._processing_thread = Thread(
|
|
target=self._process_signals_loop,
|
|
name="StrategySignalProcessor",
|
|
daemon=True
|
|
)
|
|
self._processing_thread.start()
|
|
|
|
if self.logger:
|
|
self.logger.info("StrategySignalBroadcaster: Started signal processing")
|
|
|
|
def stop(self) -> None:
|
|
"""Stop the signal broadcasting service."""
|
|
if not self._is_running:
|
|
return
|
|
|
|
self._is_running = False
|
|
self._stop_event.set()
|
|
|
|
if self._processing_thread and self._processing_thread.is_alive():
|
|
self._processing_thread.join(timeout=5.0)
|
|
|
|
if self.logger:
|
|
self.logger.info("StrategySignalBroadcaster: Stopped signal processing")
|
|
|
|
def broadcast_signal(self, signal: RealTimeSignal) -> bool:
|
|
"""
|
|
Broadcast a real-time signal.
|
|
|
|
Args:
|
|
signal: Real-time signal to broadcast
|
|
|
|
Returns:
|
|
True if signal was queued successfully, False if queue is full
|
|
"""
|
|
try:
|
|
self._signal_queue.put_nowait(signal)
|
|
return True
|
|
except:
|
|
# Queue is full, drop the signal
|
|
if self.logger:
|
|
self.logger.warning(f"Signal queue full, dropping signal for {signal.context.symbol}")
|
|
return False
|
|
|
|
def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None:
|
|
"""Set callback for chart updates."""
|
|
self._chart_update_callback = callback
|
|
|
|
def _process_signals_loop(self) -> None:
|
|
"""Main signal processing loop."""
|
|
batch_signals = []
|
|
|
|
while not self._stop_event.is_set():
|
|
try:
|
|
# Collect signals in batches
|
|
try:
|
|
signal = self._signal_queue.get(timeout=1.0)
|
|
batch_signals.append(signal)
|
|
|
|
# Collect more signals if available (up to batch size)
|
|
while len(batch_signals) < self.config.signal_batch_size:
|
|
try:
|
|
signal = self._signal_queue.get_nowait()
|
|
batch_signals.append(signal)
|
|
except Empty:
|
|
break
|
|
|
|
# Process the batch
|
|
if batch_signals:
|
|
self._process_signal_batch(batch_signals)
|
|
batch_signals.clear()
|
|
|
|
except Empty:
|
|
# No signals to process, continue
|
|
continue
|
|
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"Error in signal processing loop: {e}")
|
|
time.sleep(1.0) # Brief pause on error
|
|
|
|
def _process_signal_batch(self, signals: List[RealTimeSignal]) -> None:
|
|
"""Process a batch of signals."""
|
|
try:
|
|
# Store signals in database
|
|
self._store_signals_batch(signals)
|
|
|
|
# Process chart updates
|
|
self._process_chart_updates(signals)
|
|
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"Error processing signal batch: {e}")
|
|
|
|
def _store_signals_batch(self, signals: List[RealTimeSignal]) -> None:
|
|
"""Store signals in database."""
|
|
try:
|
|
signal_data = []
|
|
for signal in signals:
|
|
result = signal.strategy_result
|
|
context = signal.context
|
|
|
|
# Only store BUY or SELL signals, not HOLD
|
|
if result.signal and result.signal.signal_type in [SignalType.BUY, SignalType.SELL]:
|
|
signal_data.append({
|
|
'strategy_name': context.strategy_name,
|
|
'strategy_config': context.strategy_config,
|
|
'symbol': context.symbol,
|
|
'timeframe': context.timeframe,
|
|
'exchange': context.exchange,
|
|
'timestamp': result.timestamp,
|
|
'signal_type': result.signal.signal_type.value,
|
|
'price': float(result.price) if result.price else None,
|
|
'confidence': result.confidence,
|
|
'signal_metadata': result.metadata or {},
|
|
'generation_time': signal.generation_time
|
|
})
|
|
|
|
if signal_data: # Only call store_signals_batch if there are signals to store
|
|
# Batch insert into database
|
|
self.db_ops.strategy.store_signals_batch(signal_data)
|
|
|
|
if self.logger:
|
|
self.logger.debug(f"Stored batch of {len(signal_data)} real-time signals")
|
|
else:
|
|
if self.logger:
|
|
self.logger.debug("No BUY/SELL signals to store in this batch.")
|
|
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"Error storing signal batch: {e}")
|
|
|
|
def _process_chart_updates(self, signals: List[RealTimeSignal]) -> None:
|
|
"""Process chart updates for signals."""
|
|
if not self._chart_update_callback:
|
|
return
|
|
|
|
# Group signals by symbol/timeframe for throttling
|
|
signal_groups = {}
|
|
for signal in signals:
|
|
if not signal.chart_update_required:
|
|
continue
|
|
|
|
key = f"{signal.context.symbol}_{signal.context.timeframe}"
|
|
if key not in signal_groups:
|
|
signal_groups[key] = []
|
|
signal_groups[key].append(signal)
|
|
|
|
# Process chart updates with throttling
|
|
current_time = time.time() * 1000 # milliseconds
|
|
|
|
with self._chart_update_lock:
|
|
for key, group_signals in signal_groups.items():
|
|
last_update = self._last_chart_update.get(key, 0)
|
|
|
|
if current_time - last_update >= self.config.chart_update_throttle_ms:
|
|
# Update chart with latest signal from group
|
|
latest_signal = max(group_signals, key=lambda s: s.generation_time)
|
|
|
|
try:
|
|
self._chart_update_callback(latest_signal)
|
|
self._last_chart_update[key] = current_time
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"Error in chart update callback: {e}")
|
|
|
|
def get_signal_stats(self) -> Dict[str, Any]:
|
|
"""Get signal broadcasting statistics."""
|
|
return {
|
|
'queue_size': self._signal_queue.qsize(),
|
|
'chart_queue_size': self._chart_update_queue.qsize(),
|
|
'is_running': self._is_running,
|
|
'last_chart_updates': dict(self._last_chart_update)
|
|
}
|
|
|
|
|
|
class RealTimeStrategyProcessor:
|
|
"""
|
|
Real-time strategy execution processor.
|
|
|
|
Integrates with existing chart data refresh cycle to provide
|
|
real-time strategy signal generation and broadcasting.
|
|
"""
|
|
|
|
def __init__(self, config: RealTimeConfig = None):
|
|
"""Initialize real-time strategy processor."""
|
|
self.config = config or RealTimeConfig()
|
|
self.logger = logger
|
|
|
|
# Core components
|
|
self.data_integrator = StrategyDataIntegrator(
|
|
StrategyDataIntegrationConfig(
|
|
cache_timeout_minutes=1, # Shorter cache for real-time
|
|
enable_indicator_caching=True
|
|
)
|
|
)
|
|
self.market_integrator = MarketDataIntegrator()
|
|
self.strategy_factory = StrategyFactory(self.logger)
|
|
self.signal_broadcaster = StrategySignalBroadcaster(self.config)
|
|
|
|
# Strategy execution contexts
|
|
self._execution_contexts: Dict[str, StrategyExecutionContext] = {}
|
|
self._context_lock = Lock()
|
|
|
|
# Performance tracking
|
|
self._performance_stats = {
|
|
'total_calculations': 0,
|
|
'successful_calculations': 0,
|
|
'failed_calculations': 0,
|
|
'average_calculation_time_ms': 0.0,
|
|
'signals_generated': 0,
|
|
'last_update_time': None
|
|
}
|
|
|
|
# Thread pool for concurrent strategy execution
|
|
self._executor = ThreadPoolExecutor(max_workers=self.config.max_strategies_concurrent)
|
|
|
|
if self.logger:
|
|
self.logger.info("RealTimeStrategyProcessor: Initialized")
|
|
|
|
def start(self) -> None:
|
|
"""Start the real-time strategy processor."""
|
|
self.signal_broadcaster.start()
|
|
if self.logger:
|
|
self.logger.info("RealTimeStrategyProcessor: Started")
|
|
|
|
def stop(self) -> None:
|
|
"""Stop the real-time strategy processor."""
|
|
self.signal_broadcaster.stop()
|
|
self._executor.shutdown(wait=True)
|
|
if self.logger:
|
|
self.logger.info("RealTimeStrategyProcessor: Stopped")
|
|
|
|
def register_strategy(
|
|
self,
|
|
strategy_name: str,
|
|
strategy_config: Dict[str, Any],
|
|
symbol: str,
|
|
timeframe: str,
|
|
exchange: str = "okx"
|
|
) -> str:
|
|
"""
|
|
Register a strategy for real-time execution.
|
|
|
|
Args:
|
|
strategy_name: Name of the strategy
|
|
strategy_config: Strategy configuration
|
|
symbol: Trading symbol
|
|
timeframe: Timeframe
|
|
exchange: Exchange name
|
|
|
|
Returns:
|
|
Context ID for the registered strategy
|
|
"""
|
|
context_id = f"{strategy_name}_{symbol}_{timeframe}_{exchange}"
|
|
|
|
context = StrategyExecutionContext(
|
|
strategy_name=strategy_name,
|
|
strategy_config=strategy_config,
|
|
symbol=symbol,
|
|
timeframe=timeframe,
|
|
exchange=exchange
|
|
)
|
|
|
|
with self._context_lock:
|
|
self._execution_contexts[context_id] = context
|
|
|
|
if self.logger:
|
|
self.logger.info(f"Registered strategy for real-time execution: {context_id}")
|
|
|
|
return context_id
|
|
|
|
def unregister_strategy(self, context_id: str) -> bool:
|
|
"""
|
|
Unregister a strategy from real-time execution.
|
|
|
|
Args:
|
|
context_id: Context ID to unregister
|
|
|
|
Returns:
|
|
True if strategy was unregistered, False if not found
|
|
"""
|
|
with self._context_lock:
|
|
if context_id in self._execution_contexts:
|
|
del self._execution_contexts[context_id]
|
|
if self.logger:
|
|
self.logger.info(f"Unregistered strategy: {context_id}")
|
|
return True
|
|
return False
|
|
|
|
def execute_realtime_update(
|
|
self,
|
|
symbol: str,
|
|
timeframe: str,
|
|
exchange: str = "okx"
|
|
) -> List[RealTimeSignal]:
|
|
"""
|
|
Execute real-time strategy update for new market data.
|
|
|
|
This method should be called when new candle data is available,
|
|
typically triggered by the chart refresh cycle.
|
|
|
|
Args:
|
|
symbol: Trading symbol that was updated
|
|
timeframe: Timeframe that was updated
|
|
exchange: Exchange name
|
|
|
|
Returns:
|
|
List of generated real-time signals
|
|
"""
|
|
start_time = time.time()
|
|
generated_signals = []
|
|
|
|
try:
|
|
# Find all strategies for this symbol/timeframe
|
|
matching_contexts = []
|
|
with self._context_lock:
|
|
for context_id, context in self._execution_contexts.items():
|
|
if (context.symbol == symbol and
|
|
context.timeframe == timeframe and
|
|
context.exchange == exchange and
|
|
context.is_active):
|
|
matching_contexts.append((context_id, context))
|
|
|
|
if not matching_contexts:
|
|
return generated_signals
|
|
|
|
# Execute strategies concurrently
|
|
futures = []
|
|
for context_id, context in matching_contexts:
|
|
future = self._executor.submit(
|
|
self._execute_strategy_context,
|
|
context_id,
|
|
context
|
|
)
|
|
futures.append((context_id, future))
|
|
|
|
# Collect results
|
|
for context_id, future in futures:
|
|
try:
|
|
signals = future.result(timeout=10.0) # 10 second timeout
|
|
generated_signals.extend(signals)
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"Error executing strategy {context_id}: {e}")
|
|
self._handle_strategy_error(context_id, e)
|
|
|
|
# Update performance stats
|
|
calculation_time = (time.time() - start_time) * 1000
|
|
self._update_performance_stats(len(generated_signals), calculation_time, True)
|
|
|
|
if self.logger and generated_signals:
|
|
self.logger.debug(f"Generated {len(generated_signals)} real-time signals for {symbol} {timeframe}")
|
|
|
|
return generated_signals
|
|
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"Error in real-time strategy execution: {e}")
|
|
calculation_time = (time.time() - start_time) * 1000
|
|
self._update_performance_stats(0, calculation_time, False)
|
|
return generated_signals
|
|
|
|
def _execute_strategy_context(
|
|
self,
|
|
context_id: str,
|
|
context: StrategyExecutionContext
|
|
) -> List[RealTimeSignal]:
|
|
"""Execute a single strategy context."""
|
|
try:
|
|
# Calculate strategy signals
|
|
if self.config.incremental_calculation and context.last_calculation_time:
|
|
# Use incremental calculation for better performance
|
|
results = self._calculate_incremental_signals(context)
|
|
else:
|
|
# Full recalculation
|
|
results = self._calculate_full_signals(context)
|
|
|
|
# Convert to real-time signals
|
|
real_time_signals = []
|
|
for result in results:
|
|
signal = RealTimeSignal(
|
|
strategy_result=result,
|
|
context=context
|
|
)
|
|
real_time_signals.append(signal)
|
|
|
|
# Broadcast signal
|
|
self.signal_broadcaster.broadcast_signal(signal)
|
|
|
|
# Update context
|
|
with self._context_lock:
|
|
context.last_calculation_time = datetime.now(timezone.utc)
|
|
context.consecutive_errors = 0
|
|
|
|
return real_time_signals
|
|
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"Error executing strategy context {context_id}: {e}")
|
|
self._handle_strategy_error(context_id, e)
|
|
return []
|
|
|
|
def _calculate_incremental_signals(
|
|
self,
|
|
context: StrategyExecutionContext
|
|
) -> List[StrategyResult]:
|
|
"""Calculate signals incrementally (only for new data)."""
|
|
# For this initial implementation, fall back to full calculation
|
|
# Incremental calculation optimization can be added later
|
|
return self._calculate_full_signals(context)
|
|
|
|
def _calculate_full_signals(
|
|
self,
|
|
context: StrategyExecutionContext
|
|
) -> List[StrategyResult]:
|
|
"""Calculate signals with full recalculation."""
|
|
return self.data_integrator.calculate_strategy_signals(
|
|
strategy_name=context.strategy_name,
|
|
strategy_config=context.strategy_config,
|
|
symbol=context.symbol,
|
|
timeframe=context.timeframe,
|
|
days_back=7, # Use shorter history for real-time
|
|
exchange=context.exchange,
|
|
enable_caching=True
|
|
)
|
|
|
|
def _handle_strategy_error(self, context_id: str, error: Exception) -> None:
|
|
"""Handle strategy execution error."""
|
|
with self._context_lock:
|
|
if context_id in self._execution_contexts:
|
|
context = self._execution_contexts[context_id]
|
|
context.consecutive_errors += 1
|
|
|
|
# Disable strategy if too many consecutive errors
|
|
if context.consecutive_errors >= self.config.error_retry_attempts:
|
|
context.is_active = False
|
|
if self.logger:
|
|
self.logger.warning(
|
|
f"Disabling strategy {context_id} due to consecutive errors: {context.consecutive_errors}"
|
|
)
|
|
|
|
def _update_performance_stats(
|
|
self,
|
|
signals_generated: int,
|
|
calculation_time_ms: float,
|
|
success: bool
|
|
) -> None:
|
|
"""Update performance statistics."""
|
|
self._performance_stats['total_calculations'] += 1
|
|
if success:
|
|
self._performance_stats['successful_calculations'] += 1
|
|
else:
|
|
self._performance_stats['failed_calculations'] += 1
|
|
|
|
self._performance_stats['signals_generated'] += signals_generated
|
|
|
|
# Update average calculation time
|
|
total_calcs = self._performance_stats['total_calculations']
|
|
current_avg = self._performance_stats['average_calculation_time_ms']
|
|
self._performance_stats['average_calculation_time_ms'] = (
|
|
(current_avg * (total_calcs - 1) + calculation_time_ms) / total_calcs
|
|
)
|
|
|
|
self._performance_stats['last_update_time'] = datetime.now(timezone.utc)
|
|
|
|
def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None:
|
|
"""Set callback for chart updates."""
|
|
self.signal_broadcaster.set_chart_update_callback(callback)
|
|
|
|
def get_active_strategies(self) -> Dict[str, StrategyExecutionContext]:
|
|
"""Get all active strategy contexts."""
|
|
with self._context_lock:
|
|
return {
|
|
context_id: context
|
|
for context_id, context in self._execution_contexts.items()
|
|
if context.is_active
|
|
}
|
|
|
|
def get_performance_stats(self) -> Dict[str, Any]:
|
|
"""Get real-time execution performance statistics."""
|
|
stats = dict(self._performance_stats)
|
|
stats.update(self.signal_broadcaster.get_signal_stats())
|
|
return stats
|
|
|
|
def pause_strategy(self, context_id: str) -> bool:
|
|
"""Pause a strategy (set as inactive)."""
|
|
with self._context_lock:
|
|
if context_id in self._execution_contexts:
|
|
self._execution_contexts[context_id].is_active = False
|
|
return True
|
|
return False
|
|
|
|
def resume_strategy(self, context_id: str) -> bool:
|
|
"""Resume a strategy (set as active)."""
|
|
with self._context_lock:
|
|
if context_id in self._execution_contexts:
|
|
context = self._execution_contexts[context_id]
|
|
context.is_active = True
|
|
context.consecutive_errors = 0 # Reset error count
|
|
return True
|
|
return False
|
|
|
|
|
|
# Singleton instance for global access
|
|
_realtime_processor: Optional[RealTimeStrategyProcessor] = None
|
|
|
|
|
|
def get_realtime_strategy_processor(config: RealTimeConfig = None) -> RealTimeStrategyProcessor:
|
|
"""
|
|
Get the singleton real-time strategy processor instance.
|
|
|
|
Args:
|
|
config: Configuration for the processor (only used on first call)
|
|
|
|
Returns:
|
|
RealTimeStrategyProcessor instance
|
|
"""
|
|
global _realtime_processor
|
|
|
|
if _realtime_processor is None:
|
|
_realtime_processor = RealTimeStrategyProcessor(config)
|
|
|
|
return _realtime_processor
|
|
|
|
|
|
def initialize_realtime_strategy_system(config: RealTimeConfig = None) -> RealTimeStrategyProcessor:
|
|
"""
|
|
Initialize the real-time strategy system.
|
|
|
|
Args:
|
|
config: Configuration for the system
|
|
|
|
Returns:
|
|
Initialized RealTimeStrategyProcessor
|
|
"""
|
|
processor = get_realtime_strategy_processor(config)
|
|
processor.start()
|
|
return processor
|
|
|
|
|
|
def shutdown_realtime_strategy_system() -> None:
|
|
"""Shutdown the real-time strategy system."""
|
|
global _realtime_processor
|
|
|
|
if _realtime_processor is not None:
|
|
_realtime_processor.stop()
|
|
_realtime_processor = None |