4.0 - 4.0 Implement real-time strategy execution and data integration features
- Added `realtime_execution.py` for real-time strategy execution, enabling live signal generation and integration with the dashboard's chart refresh cycle. - Introduced `data_integration.py` to manage market data orchestration, caching, and technical indicator calculations for strategy signal generation. - Implemented `validation.py` for comprehensive validation and quality assessment of strategy-generated signals, ensuring reliability and consistency. - Developed `batch_processing.py` to facilitate efficient backtesting of multiple strategies across large datasets with memory management and performance optimization. - Updated `__init__.py` files to include new modules and ensure proper exports, enhancing modularity and maintainability. - Enhanced unit tests for the new features, ensuring robust functionality and adherence to project standards. These changes establish a solid foundation for real-time strategy execution and data integration, aligning with project goals for modularity, performance, and maintainability.
This commit is contained in:
Vasily.onl
@@ -16,6 +16,10 @@ from .base import BaseStrategy
|
||||
from .factory import StrategyFactory
|
||||
from .data_types import StrategySignal, SignalType, StrategyResult
|
||||
from .manager import StrategyManager, StrategyConfig, StrategyType, StrategyCategory, get_strategy_manager
|
||||
from .data_integration import StrategyDataIntegrator, StrategyDataIntegrationConfig, get_strategy_data_integrator
|
||||
from .validation import StrategySignalValidator, ValidationConfig
|
||||
from .batch_processing import BacktestingBatchProcessor, BatchProcessingConfig
|
||||
from .realtime_execution import RealTimeStrategyProcessor, RealTimeConfig, get_realtime_strategy_processor
|
||||
|
||||
__all__ = [
|
||||
'BaseStrategy',
|
||||
@@ -27,5 +31,15 @@ __all__ = [
|
||||
'StrategyConfig',
|
||||
'StrategyType',
|
||||
'StrategyCategory',
|
||||
'get_strategy_manager'
|
||||
'get_strategy_manager',
|
||||
'StrategyDataIntegrator',
|
||||
'StrategyDataIntegrationConfig',
|
||||
'get_strategy_data_integrator',
|
||||
'StrategySignalValidator',
|
||||
'ValidationConfig',
|
||||
'BacktestingBatchProcessor',
|
||||
'BatchProcessingConfig',
|
||||
'RealTimeStrategyProcessor',
|
||||
'RealTimeConfig',
|
||||
'get_realtime_strategy_processor'
|
||||
]
|
||||
1059
strategies/batch_processing.py
Normal file
1059
strategies/batch_processing.py
Normal file
File diff suppressed because it is too large
Load Diff
1060
strategies/data_integration.py
Normal file
1060
strategies/data_integration.py
Normal file
File diff suppressed because it is too large
Load Diff
649
strategies/realtime_execution.py
Normal file
649
strategies/realtime_execution.py
Normal file
@@ -0,0 +1,649 @@
|
||||
"""
|
||||
Real-time Strategy Execution Pipeline
|
||||
|
||||
This module provides real-time strategy execution capabilities that integrate
|
||||
with the existing chart data refresh cycle. It handles incremental strategy
|
||||
calculations, real-time signal generation, and live chart updates.
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import List, Dict, Any, Optional, Callable, Set, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
from threading import Thread, Event, Lock
|
||||
from queue import Queue, Empty
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import time
|
||||
|
||||
from database.operations import get_database_operations, DatabaseOperationError
|
||||
from data.common.data_types import OHLCVCandle
|
||||
from components.charts.data_integration import MarketDataIntegrator
|
||||
from .data_integration import StrategyDataIntegrator, StrategyDataIntegrationConfig
|
||||
from .factory import StrategyFactory
|
||||
from .data_types import StrategyResult, StrategySignal
|
||||
from utils.logger import get_logger
|
||||
|
||||
# Initialize logger
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RealTimeConfig:
|
||||
"""Configuration for real-time strategy execution"""
|
||||
refresh_interval_seconds: int = 30 # How often to check for new data
|
||||
max_strategies_concurrent: int = 5 # Maximum concurrent strategy calculations
|
||||
incremental_calculation: bool = True # Use incremental vs full recalculation
|
||||
signal_batch_size: int = 100 # Batch size for signal storage
|
||||
enable_signal_broadcasting: bool = True # Enable real-time signal broadcasting
|
||||
max_signal_queue_size: int = 1000 # Maximum signals in queue before dropping
|
||||
chart_update_throttle_ms: int = 1000 # Minimum time between chart updates
|
||||
error_retry_attempts: int = 3 # Number of retries on calculation errors
|
||||
error_retry_delay_seconds: int = 5 # Delay between retry attempts
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyExecutionContext:
|
||||
"""Context for strategy execution"""
|
||||
strategy_name: str
|
||||
strategy_config: Dict[str, Any]
|
||||
symbol: str
|
||||
timeframe: str
|
||||
exchange: str = "okx"
|
||||
last_calculation_time: Optional[datetime] = None
|
||||
consecutive_errors: int = 0
|
||||
is_active: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class RealTimeSignal:
|
||||
"""Real-time signal with metadata"""
|
||||
strategy_result: StrategyResult
|
||||
context: StrategyExecutionContext
|
||||
generation_time: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
chart_update_required: bool = True
|
||||
|
||||
|
||||
class StrategySignalBroadcaster:
|
||||
"""
|
||||
Handles real-time signal broadcasting and distribution.
|
||||
|
||||
Manages signal queues, chart updates, and database storage
|
||||
for real-time strategy signals.
|
||||
"""
|
||||
|
||||
def __init__(self, config: RealTimeConfig):
|
||||
"""Initialize signal broadcaster."""
|
||||
self.config = config
|
||||
self.logger = logger
|
||||
self.db_ops = get_database_operations(self.logger)
|
||||
|
||||
# Signal queues
|
||||
self._signal_queue: Queue[RealTimeSignal] = Queue(maxsize=self.config.max_signal_queue_size)
|
||||
self._chart_update_queue: Queue[RealTimeSignal] = Queue()
|
||||
|
||||
# Chart update throttling
|
||||
self._last_chart_update = {} # symbol_timeframe -> timestamp
|
||||
self._chart_update_lock = Lock()
|
||||
|
||||
# Background processing
|
||||
self._processing_thread: Optional[Thread] = None
|
||||
self._stop_event = Event()
|
||||
self._is_running = False
|
||||
|
||||
# Callback for chart updates
|
||||
self._chart_update_callback: Optional[Callable] = None
|
||||
|
||||
if self.logger:
|
||||
self.logger.info("StrategySignalBroadcaster: Initialized")
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the signal broadcasting service."""
|
||||
if self._is_running:
|
||||
return
|
||||
|
||||
self._is_running = True
|
||||
self._stop_event.clear()
|
||||
|
||||
# Start background processing thread
|
||||
self._processing_thread = Thread(
|
||||
target=self._process_signals_loop,
|
||||
name="StrategySignalProcessor",
|
||||
daemon=True
|
||||
)
|
||||
self._processing_thread.start()
|
||||
|
||||
if self.logger:
|
||||
self.logger.info("StrategySignalBroadcaster: Started signal processing")
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop the signal broadcasting service."""
|
||||
if not self._is_running:
|
||||
return
|
||||
|
||||
self._is_running = False
|
||||
self._stop_event.set()
|
||||
|
||||
if self._processing_thread and self._processing_thread.is_alive():
|
||||
self._processing_thread.join(timeout=5.0)
|
||||
|
||||
if self.logger:
|
||||
self.logger.info("StrategySignalBroadcaster: Stopped signal processing")
|
||||
|
||||
def broadcast_signal(self, signal: RealTimeSignal) -> bool:
|
||||
"""
|
||||
Broadcast a real-time signal.
|
||||
|
||||
Args:
|
||||
signal: Real-time signal to broadcast
|
||||
|
||||
Returns:
|
||||
True if signal was queued successfully, False if queue is full
|
||||
"""
|
||||
try:
|
||||
self._signal_queue.put_nowait(signal)
|
||||
return True
|
||||
except:
|
||||
# Queue is full, drop the signal
|
||||
if self.logger:
|
||||
self.logger.warning(f"Signal queue full, dropping signal for {signal.context.symbol}")
|
||||
return False
|
||||
|
||||
def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None:
|
||||
"""Set callback for chart updates."""
|
||||
self._chart_update_callback = callback
|
||||
|
||||
def _process_signals_loop(self) -> None:
|
||||
"""Main signal processing loop."""
|
||||
batch_signals = []
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
# Collect signals in batches
|
||||
try:
|
||||
signal = self._signal_queue.get(timeout=1.0)
|
||||
batch_signals.append(signal)
|
||||
|
||||
# Collect more signals if available (up to batch size)
|
||||
while len(batch_signals) < self.config.signal_batch_size:
|
||||
try:
|
||||
signal = self._signal_queue.get_nowait()
|
||||
batch_signals.append(signal)
|
||||
except Empty:
|
||||
break
|
||||
|
||||
# Process the batch
|
||||
if batch_signals:
|
||||
self._process_signal_batch(batch_signals)
|
||||
batch_signals.clear()
|
||||
|
||||
except Empty:
|
||||
# No signals to process, continue
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error in signal processing loop: {e}")
|
||||
time.sleep(1.0) # Brief pause on error
|
||||
|
||||
def _process_signal_batch(self, signals: List[RealTimeSignal]) -> None:
|
||||
"""Process a batch of signals."""
|
||||
try:
|
||||
# Store signals in database
|
||||
self._store_signals_batch(signals)
|
||||
|
||||
# Process chart updates
|
||||
self._process_chart_updates(signals)
|
||||
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error processing signal batch: {e}")
|
||||
|
||||
def _store_signals_batch(self, signals: List[RealTimeSignal]) -> None:
|
||||
"""Store signals in database."""
|
||||
try:
|
||||
signal_data = []
|
||||
for signal in signals:
|
||||
result = signal.strategy_result
|
||||
context = signal.context
|
||||
|
||||
signal_data.append({
|
||||
'strategy_name': context.strategy_name,
|
||||
'strategy_config': context.strategy_config,
|
||||
'symbol': context.symbol,
|
||||
'timeframe': context.timeframe,
|
||||
'exchange': context.exchange,
|
||||
'timestamp': result.timestamp,
|
||||
'signal_type': result.signal.signal_type.value if result.signal else 'HOLD',
|
||||
'price': float(result.price) if result.price else None,
|
||||
'confidence': result.confidence,
|
||||
'signal_metadata': result.metadata or {},
|
||||
'generation_time': signal.generation_time
|
||||
})
|
||||
|
||||
# Batch insert into database
|
||||
self.db_ops.strategy.store_signals_batch(signal_data)
|
||||
|
||||
if self.logger:
|
||||
self.logger.debug(f"Stored batch of {len(signals)} real-time signals")
|
||||
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error storing signal batch: {e}")
|
||||
|
||||
def _process_chart_updates(self, signals: List[RealTimeSignal]) -> None:
|
||||
"""Process chart updates for signals."""
|
||||
if not self._chart_update_callback:
|
||||
return
|
||||
|
||||
# Group signals by symbol/timeframe for throttling
|
||||
signal_groups = {}
|
||||
for signal in signals:
|
||||
if not signal.chart_update_required:
|
||||
continue
|
||||
|
||||
key = f"{signal.context.symbol}_{signal.context.timeframe}"
|
||||
if key not in signal_groups:
|
||||
signal_groups[key] = []
|
||||
signal_groups[key].append(signal)
|
||||
|
||||
# Process chart updates with throttling
|
||||
current_time = time.time() * 1000 # milliseconds
|
||||
|
||||
with self._chart_update_lock:
|
||||
for key, group_signals in signal_groups.items():
|
||||
last_update = self._last_chart_update.get(key, 0)
|
||||
|
||||
if current_time - last_update >= self.config.chart_update_throttle_ms:
|
||||
# Update chart with latest signal from group
|
||||
latest_signal = max(group_signals, key=lambda s: s.generation_time)
|
||||
|
||||
try:
|
||||
self._chart_update_callback(latest_signal)
|
||||
self._last_chart_update[key] = current_time
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error in chart update callback: {e}")
|
||||
|
||||
def get_signal_stats(self) -> Dict[str, Any]:
|
||||
"""Get signal broadcasting statistics."""
|
||||
return {
|
||||
'queue_size': self._signal_queue.qsize(),
|
||||
'chart_queue_size': self._chart_update_queue.qsize(),
|
||||
'is_running': self._is_running,
|
||||
'last_chart_updates': dict(self._last_chart_update)
|
||||
}
|
||||
|
||||
|
||||
class RealTimeStrategyProcessor:
|
||||
"""
|
||||
Real-time strategy execution processor.
|
||||
|
||||
Integrates with existing chart data refresh cycle to provide
|
||||
real-time strategy signal generation and broadcasting.
|
||||
"""
|
||||
|
||||
def __init__(self, config: RealTimeConfig = None):
|
||||
"""Initialize real-time strategy processor."""
|
||||
self.config = config or RealTimeConfig()
|
||||
self.logger = logger
|
||||
|
||||
# Core components
|
||||
self.data_integrator = StrategyDataIntegrator(
|
||||
StrategyDataIntegrationConfig(
|
||||
cache_timeout_minutes=1, # Shorter cache for real-time
|
||||
enable_indicator_caching=True
|
||||
)
|
||||
)
|
||||
self.market_integrator = MarketDataIntegrator()
|
||||
self.strategy_factory = StrategyFactory(self.logger)
|
||||
self.signal_broadcaster = StrategySignalBroadcaster(self.config)
|
||||
|
||||
# Strategy execution contexts
|
||||
self._execution_contexts: Dict[str, StrategyExecutionContext] = {}
|
||||
self._context_lock = Lock()
|
||||
|
||||
# Performance tracking
|
||||
self._performance_stats = {
|
||||
'total_calculations': 0,
|
||||
'successful_calculations': 0,
|
||||
'failed_calculations': 0,
|
||||
'average_calculation_time_ms': 0.0,
|
||||
'signals_generated': 0,
|
||||
'last_update_time': None
|
||||
}
|
||||
|
||||
# Thread pool for concurrent strategy execution
|
||||
self._executor = ThreadPoolExecutor(max_workers=self.config.max_strategies_concurrent)
|
||||
|
||||
if self.logger:
|
||||
self.logger.info("RealTimeStrategyProcessor: Initialized")
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the real-time strategy processor."""
|
||||
self.signal_broadcaster.start()
|
||||
if self.logger:
|
||||
self.logger.info("RealTimeStrategyProcessor: Started")
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop the real-time strategy processor."""
|
||||
self.signal_broadcaster.stop()
|
||||
self._executor.shutdown(wait=True)
|
||||
if self.logger:
|
||||
self.logger.info("RealTimeStrategyProcessor: Stopped")
|
||||
|
||||
def register_strategy(
|
||||
self,
|
||||
strategy_name: str,
|
||||
strategy_config: Dict[str, Any],
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
exchange: str = "okx"
|
||||
) -> str:
|
||||
"""
|
||||
Register a strategy for real-time execution.
|
||||
|
||||
Args:
|
||||
strategy_name: Name of the strategy
|
||||
strategy_config: Strategy configuration
|
||||
symbol: Trading symbol
|
||||
timeframe: Timeframe
|
||||
exchange: Exchange name
|
||||
|
||||
Returns:
|
||||
Context ID for the registered strategy
|
||||
"""
|
||||
context_id = f"{strategy_name}_{symbol}_{timeframe}_{exchange}"
|
||||
|
||||
context = StrategyExecutionContext(
|
||||
strategy_name=strategy_name,
|
||||
strategy_config=strategy_config,
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
exchange=exchange
|
||||
)
|
||||
|
||||
with self._context_lock:
|
||||
self._execution_contexts[context_id] = context
|
||||
|
||||
if self.logger:
|
||||
self.logger.info(f"Registered strategy for real-time execution: {context_id}")
|
||||
|
||||
return context_id
|
||||
|
||||
def unregister_strategy(self, context_id: str) -> bool:
|
||||
"""
|
||||
Unregister a strategy from real-time execution.
|
||||
|
||||
Args:
|
||||
context_id: Context ID to unregister
|
||||
|
||||
Returns:
|
||||
True if strategy was unregistered, False if not found
|
||||
"""
|
||||
with self._context_lock:
|
||||
if context_id in self._execution_contexts:
|
||||
del self._execution_contexts[context_id]
|
||||
if self.logger:
|
||||
self.logger.info(f"Unregistered strategy: {context_id}")
|
||||
return True
|
||||
return False
|
||||
|
||||
def execute_realtime_update(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
exchange: str = "okx"
|
||||
) -> List[RealTimeSignal]:
|
||||
"""
|
||||
Execute real-time strategy update for new market data.
|
||||
|
||||
This method should be called when new candle data is available,
|
||||
typically triggered by the chart refresh cycle.
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol that was updated
|
||||
timeframe: Timeframe that was updated
|
||||
exchange: Exchange name
|
||||
|
||||
Returns:
|
||||
List of generated real-time signals
|
||||
"""
|
||||
start_time = time.time()
|
||||
generated_signals = []
|
||||
|
||||
try:
|
||||
# Find all strategies for this symbol/timeframe
|
||||
matching_contexts = []
|
||||
with self._context_lock:
|
||||
for context_id, context in self._execution_contexts.items():
|
||||
if (context.symbol == symbol and
|
||||
context.timeframe == timeframe and
|
||||
context.exchange == exchange and
|
||||
context.is_active):
|
||||
matching_contexts.append((context_id, context))
|
||||
|
||||
if not matching_contexts:
|
||||
return generated_signals
|
||||
|
||||
# Execute strategies concurrently
|
||||
futures = []
|
||||
for context_id, context in matching_contexts:
|
||||
future = self._executor.submit(
|
||||
self._execute_strategy_context,
|
||||
context_id,
|
||||
context
|
||||
)
|
||||
futures.append((context_id, future))
|
||||
|
||||
# Collect results
|
||||
for context_id, future in futures:
|
||||
try:
|
||||
signals = future.result(timeout=10.0) # 10 second timeout
|
||||
generated_signals.extend(signals)
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error executing strategy {context_id}: {e}")
|
||||
self._handle_strategy_error(context_id, e)
|
||||
|
||||
# Update performance stats
|
||||
calculation_time = (time.time() - start_time) * 1000
|
||||
self._update_performance_stats(len(generated_signals), calculation_time, True)
|
||||
|
||||
if self.logger and generated_signals:
|
||||
self.logger.debug(f"Generated {len(generated_signals)} real-time signals for {symbol} {timeframe}")
|
||||
|
||||
return generated_signals
|
||||
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error in real-time strategy execution: {e}")
|
||||
calculation_time = (time.time() - start_time) * 1000
|
||||
self._update_performance_stats(0, calculation_time, False)
|
||||
return generated_signals
|
||||
|
||||
def _execute_strategy_context(
|
||||
self,
|
||||
context_id: str,
|
||||
context: StrategyExecutionContext
|
||||
) -> List[RealTimeSignal]:
|
||||
"""Execute a single strategy context."""
|
||||
try:
|
||||
# Calculate strategy signals
|
||||
if self.config.incremental_calculation and context.last_calculation_time:
|
||||
# Use incremental calculation for better performance
|
||||
results = self._calculate_incremental_signals(context)
|
||||
else:
|
||||
# Full recalculation
|
||||
results = self._calculate_full_signals(context)
|
||||
|
||||
# Convert to real-time signals
|
||||
real_time_signals = []
|
||||
for result in results:
|
||||
signal = RealTimeSignal(
|
||||
strategy_result=result,
|
||||
context=context
|
||||
)
|
||||
real_time_signals.append(signal)
|
||||
|
||||
# Broadcast signal
|
||||
self.signal_broadcaster.broadcast_signal(signal)
|
||||
|
||||
# Update context
|
||||
with self._context_lock:
|
||||
context.last_calculation_time = datetime.now(timezone.utc)
|
||||
context.consecutive_errors = 0
|
||||
|
||||
return real_time_signals
|
||||
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error executing strategy context {context_id}: {e}")
|
||||
self._handle_strategy_error(context_id, e)
|
||||
return []
|
||||
|
||||
def _calculate_incremental_signals(
|
||||
self,
|
||||
context: StrategyExecutionContext
|
||||
) -> List[StrategyResult]:
|
||||
"""Calculate signals incrementally (only for new data)."""
|
||||
# For this initial implementation, fall back to full calculation
|
||||
# Incremental calculation optimization can be added later
|
||||
return self._calculate_full_signals(context)
|
||||
|
||||
def _calculate_full_signals(
|
||||
self,
|
||||
context: StrategyExecutionContext
|
||||
) -> List[StrategyResult]:
|
||||
"""Calculate signals with full recalculation."""
|
||||
return self.data_integrator.calculate_strategy_signals(
|
||||
strategy_name=context.strategy_name,
|
||||
strategy_config=context.strategy_config,
|
||||
symbol=context.symbol,
|
||||
timeframe=context.timeframe,
|
||||
days_back=7, # Use shorter history for real-time
|
||||
exchange=context.exchange,
|
||||
enable_caching=True
|
||||
)
|
||||
|
||||
def _handle_strategy_error(self, context_id: str, error: Exception) -> None:
|
||||
"""Handle strategy execution error."""
|
||||
with self._context_lock:
|
||||
if context_id in self._execution_contexts:
|
||||
context = self._execution_contexts[context_id]
|
||||
context.consecutive_errors += 1
|
||||
|
||||
# Disable strategy if too many consecutive errors
|
||||
if context.consecutive_errors >= self.config.error_retry_attempts:
|
||||
context.is_active = False
|
||||
if self.logger:
|
||||
self.logger.warning(
|
||||
f"Disabling strategy {context_id} due to consecutive errors: {context.consecutive_errors}"
|
||||
)
|
||||
|
||||
def _update_performance_stats(
|
||||
self,
|
||||
signals_generated: int,
|
||||
calculation_time_ms: float,
|
||||
success: bool
|
||||
) -> None:
|
||||
"""Update performance statistics."""
|
||||
self._performance_stats['total_calculations'] += 1
|
||||
if success:
|
||||
self._performance_stats['successful_calculations'] += 1
|
||||
else:
|
||||
self._performance_stats['failed_calculations'] += 1
|
||||
|
||||
self._performance_stats['signals_generated'] += signals_generated
|
||||
|
||||
# Update average calculation time
|
||||
total_calcs = self._performance_stats['total_calculations']
|
||||
current_avg = self._performance_stats['average_calculation_time_ms']
|
||||
self._performance_stats['average_calculation_time_ms'] = (
|
||||
(current_avg * (total_calcs - 1) + calculation_time_ms) / total_calcs
|
||||
)
|
||||
|
||||
self._performance_stats['last_update_time'] = datetime.now(timezone.utc)
|
||||
|
||||
def set_chart_update_callback(self, callback: Callable[[RealTimeSignal], None]) -> None:
|
||||
"""Set callback for chart updates."""
|
||||
self.signal_broadcaster.set_chart_update_callback(callback)
|
||||
|
||||
def get_active_strategies(self) -> Dict[str, StrategyExecutionContext]:
|
||||
"""Get all active strategy contexts."""
|
||||
with self._context_lock:
|
||||
return {
|
||||
context_id: context
|
||||
for context_id, context in self._execution_contexts.items()
|
||||
if context.is_active
|
||||
}
|
||||
|
||||
def get_performance_stats(self) -> Dict[str, Any]:
|
||||
"""Get real-time execution performance statistics."""
|
||||
stats = dict(self._performance_stats)
|
||||
stats.update(self.signal_broadcaster.get_signal_stats())
|
||||
return stats
|
||||
|
||||
def pause_strategy(self, context_id: str) -> bool:
|
||||
"""Pause a strategy (set as inactive)."""
|
||||
with self._context_lock:
|
||||
if context_id in self._execution_contexts:
|
||||
self._execution_contexts[context_id].is_active = False
|
||||
return True
|
||||
return False
|
||||
|
||||
def resume_strategy(self, context_id: str) -> bool:
|
||||
"""Resume a strategy (set as active)."""
|
||||
with self._context_lock:
|
||||
if context_id in self._execution_contexts:
|
||||
context = self._execution_contexts[context_id]
|
||||
context.is_active = True
|
||||
context.consecutive_errors = 0 # Reset error count
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# Singleton instance for global access
|
||||
_realtime_processor: Optional[RealTimeStrategyProcessor] = None
|
||||
|
||||
|
||||
def get_realtime_strategy_processor(config: RealTimeConfig = None) -> RealTimeStrategyProcessor:
|
||||
"""
|
||||
Get the singleton real-time strategy processor instance.
|
||||
|
||||
Args:
|
||||
config: Configuration for the processor (only used on first call)
|
||||
|
||||
Returns:
|
||||
RealTimeStrategyProcessor instance
|
||||
"""
|
||||
global _realtime_processor
|
||||
|
||||
if _realtime_processor is None:
|
||||
_realtime_processor = RealTimeStrategyProcessor(config)
|
||||
|
||||
return _realtime_processor
|
||||
|
||||
|
||||
def initialize_realtime_strategy_system(config: RealTimeConfig = None) -> RealTimeStrategyProcessor:
|
||||
"""
|
||||
Initialize the real-time strategy system.
|
||||
|
||||
Args:
|
||||
config: Configuration for the system
|
||||
|
||||
Returns:
|
||||
Initialized RealTimeStrategyProcessor
|
||||
"""
|
||||
processor = get_realtime_strategy_processor(config)
|
||||
processor.start()
|
||||
return processor
|
||||
|
||||
|
||||
def shutdown_realtime_strategy_system() -> None:
|
||||
"""Shutdown the real-time strategy system."""
|
||||
global _realtime_processor
|
||||
|
||||
if _realtime_processor is not None:
|
||||
_realtime_processor.stop()
|
||||
_realtime_processor = None
|
||||
375
strategies/validation.py
Normal file
375
strategies/validation.py
Normal file
@@ -0,0 +1,375 @@
|
||||
"""
|
||||
Strategy Signal Validation Pipeline
|
||||
|
||||
This module provides validation, filtering, and quality assessment
|
||||
for strategy-generated signals to ensure reliability and consistency.
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .data_types import StrategySignal, SignalType, StrategyResult
|
||||
from utils.logger import get_logger
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationConfig:
|
||||
"""Configuration for signal validation."""
|
||||
min_confidence: float = 0.0
|
||||
max_confidence: float = 1.0
|
||||
required_metadata_fields: List[str] = None
|
||||
allowed_signal_types: List[SignalType] = None
|
||||
price_tolerance_percent: float = 5.0 # Max price deviation from market
|
||||
|
||||
def __post_init__(self):
|
||||
if self.required_metadata_fields is None:
|
||||
self.required_metadata_fields = []
|
||||
if self.allowed_signal_types is None:
|
||||
self.allowed_signal_types = list(SignalType)
|
||||
|
||||
|
||||
class StrategySignalValidator:
|
||||
"""
|
||||
Validates strategy signals for quality, consistency, and compliance.
|
||||
|
||||
Provides comprehensive validation including confidence checks,
|
||||
signal type validation, price reasonableness, and metadata validation.
|
||||
"""
|
||||
|
||||
def __init__(self, config: ValidationConfig = None):
|
||||
"""
|
||||
Initialize signal validator.
|
||||
|
||||
Args:
|
||||
config: Validation configuration
|
||||
"""
|
||||
self.config = config or ValidationConfig()
|
||||
self.logger = get_logger()
|
||||
|
||||
# Validation statistics
|
||||
self._validation_stats = {
|
||||
'total_signals_validated': 0,
|
||||
'valid_signals': 0,
|
||||
'invalid_signals': 0,
|
||||
'validation_errors': {}
|
||||
}
|
||||
|
||||
def validate_signal(self, signal: StrategySignal) -> Tuple[bool, List[str]]:
|
||||
"""
|
||||
Validate a single strategy signal.
|
||||
|
||||
Args:
|
||||
signal: Signal to validate
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, list_of_errors)
|
||||
"""
|
||||
errors = []
|
||||
self._validation_stats['total_signals_validated'] += 1
|
||||
|
||||
# Validate confidence
|
||||
if not (self.config.min_confidence <= signal.confidence <= self.config.max_confidence):
|
||||
errors.append(f"Invalid confidence {signal.confidence}, must be between {self.config.min_confidence} and {self.config.max_confidence}")
|
||||
|
||||
# Validate signal type
|
||||
if signal.signal_type not in self.config.allowed_signal_types:
|
||||
errors.append(f"Signal type {signal.signal_type} not in allowed types")
|
||||
|
||||
# Validate price
|
||||
if signal.price <= 0:
|
||||
errors.append(f"Invalid price {signal.price}, must be positive")
|
||||
|
||||
# Validate required metadata
|
||||
if self.config.required_metadata_fields:
|
||||
if not signal.metadata:
|
||||
errors.append(f"Missing required metadata fields: {self.config.required_metadata_fields}")
|
||||
else:
|
||||
missing_fields = [field for field in self.config.required_metadata_fields
|
||||
if field not in signal.metadata]
|
||||
if missing_fields:
|
||||
errors.append(f"Missing required metadata fields: {missing_fields}")
|
||||
|
||||
# Update statistics
|
||||
is_valid = len(errors) == 0
|
||||
if is_valid:
|
||||
self._validation_stats['valid_signals'] += 1
|
||||
else:
|
||||
self._validation_stats['invalid_signals'] += 1
|
||||
for error in errors:
|
||||
error_type = error.split(':')[0] if ':' in error else error
|
||||
self._validation_stats['validation_errors'][error_type] = \
|
||||
self._validation_stats['validation_errors'].get(error_type, 0) + 1
|
||||
|
||||
return is_valid, errors
|
||||
|
||||
def validate_signals_batch(self, signals: List[StrategySignal]) -> Tuple[List[StrategySignal], List[StrategySignal]]:
|
||||
"""
|
||||
Validate multiple signals and return valid and invalid lists.
|
||||
|
||||
Args:
|
||||
signals: List of signals to validate
|
||||
|
||||
Returns:
|
||||
Tuple of (valid_signals, invalid_signals)
|
||||
"""
|
||||
valid_signals = []
|
||||
invalid_signals = []
|
||||
|
||||
for signal in signals:
|
||||
is_valid, errors = self.validate_signal(signal)
|
||||
if is_valid:
|
||||
valid_signals.append(signal)
|
||||
else:
|
||||
invalid_signals.append(signal)
|
||||
self.logger.debug(f"Invalid signal filtered out: {errors}")
|
||||
|
||||
return valid_signals, invalid_signals
|
||||
|
||||
def filter_signals_by_confidence(
|
||||
self,
|
||||
signals: List[StrategySignal],
|
||||
min_confidence: float = None
|
||||
) -> List[StrategySignal]:
|
||||
"""
|
||||
Filter signals by minimum confidence threshold.
|
||||
|
||||
Args:
|
||||
signals: List of signals to filter
|
||||
min_confidence: Minimum confidence threshold (uses config if None)
|
||||
|
||||
Returns:
|
||||
Filtered list of signals
|
||||
"""
|
||||
threshold = min_confidence if min_confidence is not None else self.config.min_confidence
|
||||
|
||||
filtered_signals = [signal for signal in signals if signal.confidence >= threshold]
|
||||
|
||||
self.logger.debug(f"Filtered {len(signals) - len(filtered_signals)} signals below confidence {threshold}")
|
||||
|
||||
return filtered_signals
|
||||
|
||||
def filter_signals_by_type(
|
||||
self,
|
||||
signals: List[StrategySignal],
|
||||
allowed_types: List[SignalType] = None
|
||||
) -> List[StrategySignal]:
|
||||
"""
|
||||
Filter signals by allowed signal types.
|
||||
|
||||
Args:
|
||||
signals: List of signals to filter
|
||||
allowed_types: Allowed signal types (uses config if None)
|
||||
|
||||
Returns:
|
||||
Filtered list of signals
|
||||
"""
|
||||
types = allowed_types if allowed_types is not None else self.config.allowed_signal_types
|
||||
|
||||
filtered_signals = [signal for signal in signals if signal.signal_type in types]
|
||||
|
||||
self.logger.debug(f"Filtered {len(signals) - len(filtered_signals)} signals by type")
|
||||
|
||||
return filtered_signals
|
||||
|
||||
def get_validation_statistics(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive validation statistics."""
|
||||
stats = self._validation_stats.copy()
|
||||
|
||||
if stats['total_signals_validated'] > 0:
|
||||
stats['validation_success_rate'] = stats['valid_signals'] / stats['total_signals_validated']
|
||||
stats['validation_failure_rate'] = stats['invalid_signals'] / stats['total_signals_validated']
|
||||
else:
|
||||
stats['validation_success_rate'] = 0.0
|
||||
stats['validation_failure_rate'] = 0.0
|
||||
|
||||
return stats
|
||||
|
||||
def transform_signal_confidence(
|
||||
self,
|
||||
signal: StrategySignal,
|
||||
confidence_multiplier: float = 1.0,
|
||||
max_confidence: float = None
|
||||
) -> StrategySignal:
|
||||
"""
|
||||
Transform signal confidence with multiplier and cap.
|
||||
|
||||
Args:
|
||||
signal: Signal to transform
|
||||
confidence_multiplier: Multiplier for confidence
|
||||
max_confidence: Maximum confidence cap (uses config if None)
|
||||
|
||||
Returns:
|
||||
Transformed signal with updated confidence
|
||||
"""
|
||||
max_conf = max_confidence if max_confidence is not None else self.config.max_confidence
|
||||
|
||||
# Create new signal with transformed confidence
|
||||
new_confidence = min(signal.confidence * confidence_multiplier, max_conf)
|
||||
|
||||
transformed_signal = StrategySignal(
|
||||
timestamp=signal.timestamp,
|
||||
symbol=signal.symbol,
|
||||
timeframe=signal.timeframe,
|
||||
signal_type=signal.signal_type,
|
||||
price=signal.price,
|
||||
confidence=new_confidence,
|
||||
metadata=signal.metadata.copy() if signal.metadata else None
|
||||
)
|
||||
|
||||
return transformed_signal
|
||||
|
||||
def enrich_signal_metadata(
|
||||
self,
|
||||
signal: StrategySignal,
|
||||
additional_metadata: Dict[str, Any]
|
||||
) -> StrategySignal:
|
||||
"""
|
||||
Enrich signal with additional metadata.
|
||||
|
||||
Args:
|
||||
signal: Signal to enrich
|
||||
additional_metadata: Additional metadata to add
|
||||
|
||||
Returns:
|
||||
Signal with enriched metadata
|
||||
"""
|
||||
# Merge metadata
|
||||
enriched_metadata = signal.metadata.copy() if signal.metadata else {}
|
||||
enriched_metadata.update(additional_metadata)
|
||||
|
||||
enriched_signal = StrategySignal(
|
||||
timestamp=signal.timestamp,
|
||||
symbol=signal.symbol,
|
||||
timeframe=signal.timeframe,
|
||||
signal_type=signal.signal_type,
|
||||
price=signal.price,
|
||||
confidence=signal.confidence,
|
||||
metadata=enriched_metadata
|
||||
)
|
||||
|
||||
return enriched_signal
|
||||
|
||||
def transform_signals_batch(
|
||||
self,
|
||||
signals: List[StrategySignal],
|
||||
confidence_multiplier: float = 1.0,
|
||||
additional_metadata: Dict[str, Any] = None
|
||||
) -> List[StrategySignal]:
|
||||
"""
|
||||
Apply transformations to multiple signals.
|
||||
|
||||
Args:
|
||||
signals: List of signals to transform
|
||||
confidence_multiplier: Confidence multiplier
|
||||
additional_metadata: Additional metadata to add
|
||||
|
||||
Returns:
|
||||
List of transformed signals
|
||||
"""
|
||||
transformed_signals = []
|
||||
|
||||
for signal in signals:
|
||||
# Apply confidence transformation
|
||||
transformed_signal = self.transform_signal_confidence(signal, confidence_multiplier)
|
||||
|
||||
# Apply metadata enrichment if provided
|
||||
if additional_metadata:
|
||||
transformed_signal = self.enrich_signal_metadata(transformed_signal, additional_metadata)
|
||||
|
||||
transformed_signals.append(transformed_signal)
|
||||
|
||||
self.logger.debug(f"Transformed {len(signals)} signals")
|
||||
|
||||
return transformed_signals
|
||||
|
||||
def calculate_signal_quality_metrics(self, signals: List[StrategySignal]) -> Dict[str, Any]:
|
||||
"""
|
||||
Calculate comprehensive quality metrics for signals.
|
||||
|
||||
Args:
|
||||
signals: List of signals to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary containing quality metrics
|
||||
"""
|
||||
if not signals:
|
||||
return {'error': 'No signals provided for quality analysis'}
|
||||
|
||||
# Basic metrics
|
||||
total_signals = len(signals)
|
||||
confidence_values = [signal.confidence for signal in signals]
|
||||
|
||||
# Signal type distribution
|
||||
signal_type_counts = {}
|
||||
for signal in signals:
|
||||
signal_type_counts[signal.signal_type.value] = signal_type_counts.get(signal.signal_type.value, 0) + 1
|
||||
|
||||
# Confidence metrics
|
||||
avg_confidence = sum(confidence_values) / total_signals
|
||||
min_confidence = min(confidence_values)
|
||||
max_confidence = max(confidence_values)
|
||||
|
||||
# Quality scoring (0-100)
|
||||
high_confidence_signals = sum(1 for conf in confidence_values if conf >= 0.7)
|
||||
quality_score = (high_confidence_signals / total_signals) * 100
|
||||
|
||||
# Metadata completeness
|
||||
signals_with_metadata = sum(1 for signal in signals if signal.metadata)
|
||||
metadata_completeness = (signals_with_metadata / total_signals) * 100
|
||||
|
||||
return {
|
||||
'total_signals': total_signals,
|
||||
'signal_type_distribution': signal_type_counts,
|
||||
'confidence_metrics': {
|
||||
'average': round(avg_confidence, 3),
|
||||
'minimum': round(min_confidence, 3),
|
||||
'maximum': round(max_confidence, 3),
|
||||
'high_confidence_count': high_confidence_signals,
|
||||
'high_confidence_percentage': round((high_confidence_signals / total_signals) * 100, 1)
|
||||
},
|
||||
'quality_score': round(quality_score, 1),
|
||||
'metadata_completeness_percentage': round(metadata_completeness, 1),
|
||||
'recommendations': self._generate_quality_recommendations(signals)
|
||||
}
|
||||
|
||||
def _generate_quality_recommendations(self, signals: List[StrategySignal]) -> List[str]:
|
||||
"""Generate quality improvement recommendations."""
|
||||
recommendations = []
|
||||
|
||||
confidence_values = [signal.confidence for signal in signals]
|
||||
avg_confidence = sum(confidence_values) / len(confidence_values)
|
||||
|
||||
if avg_confidence < 0.5:
|
||||
recommendations.append("Consider increasing confidence thresholds or improving signal generation logic")
|
||||
|
||||
signals_with_metadata = sum(1 for signal in signals if signal.metadata)
|
||||
if signals_with_metadata / len(signals) < 0.8:
|
||||
recommendations.append("Enhance metadata collection to improve signal traceability")
|
||||
|
||||
signal_types = set(signal.signal_type for signal in signals)
|
||||
if len(signal_types) == 1:
|
||||
recommendations.append("Consider diversifying signal types for better strategy coverage")
|
||||
|
||||
return recommendations if recommendations else ["Signal quality appears good - no specific recommendations"]
|
||||
|
||||
def generate_validation_report(self) -> Dict[str, Any]:
|
||||
"""Generate comprehensive validation report."""
|
||||
stats = self.get_validation_statistics()
|
||||
|
||||
return {
|
||||
'report_timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'validation_summary': {
|
||||
'total_validated': stats['total_signals_validated'],
|
||||
'success_rate': f"{stats.get('validation_success_rate', 0) * 100:.1f}%",
|
||||
'failure_rate': f"{stats.get('validation_failure_rate', 0) * 100:.1f}%"
|
||||
},
|
||||
'error_analysis': stats.get('validation_errors', {}),
|
||||
'configuration': {
|
||||
'min_confidence': self.config.min_confidence,
|
||||
'max_confidence': self.config.max_confidence,
|
||||
'allowed_signal_types': [st.value for st in self.config.allowed_signal_types],
|
||||
'required_metadata_fields': self.config.required_metadata_fields
|
||||
},
|
||||
'health_status': 'good' if stats.get('validation_success_rate', 0) >= 0.8 else 'needs_attention'
|
||||
}
|
||||
Reference in New Issue
Block a user