TCPDashboard/tests/strategies/test_realtime_execution.py

"""
Tests for real-time strategy execution pipeline.
"""

import pytest
import pandas as pd
from datetime import datetime, timezone, timedelta
from unittest.mock import Mock, patch, MagicMock
import time
from queue import Queue, Empty
import threading

from strategies.realtime_execution import (
    RealTimeStrategyProcessor,
    StrategySignalBroadcaster,
    RealTimeConfig,
    StrategyExecutionContext,
    RealTimeSignal,
    get_realtime_strategy_processor,
    initialize_realtime_strategy_system,
    shutdown_realtime_strategy_system
)
from strategies.data_types import StrategyResult, StrategySignal, SignalType
from data.common.data_types import OHLCVCandle


class TestRealTimeConfig:
    """Test RealTimeConfig dataclass."""
    
    def test_default_config(self):
        """Test default configuration values."""
        config = RealTimeConfig()
        
        assert config.refresh_interval_seconds == 30
        assert config.max_strategies_concurrent == 5
        assert config.incremental_calculation == True
        assert config.signal_batch_size == 100
        assert config.enable_signal_broadcasting == True
        assert config.max_signal_queue_size == 1000
        assert config.chart_update_throttle_ms == 1000
        assert config.error_retry_attempts == 3
        assert config.error_retry_delay_seconds == 5
    
    def test_custom_config(self):
        """Test custom configuration values."""
        config = RealTimeConfig(
            refresh_interval_seconds=15,
            max_strategies_concurrent=3,
            incremental_calculation=False,
            signal_batch_size=50
        )
        
        assert config.refresh_interval_seconds == 15
        assert config.max_strategies_concurrent == 3
        assert config.incremental_calculation == False
        assert config.signal_batch_size == 50


class TestStrategyExecutionContext:
    """Test StrategyExecutionContext dataclass."""
    
    def test_context_creation(self):
        """Test strategy execution context creation."""
        context = StrategyExecutionContext(
            strategy_name="ema_crossover",
            strategy_config={"short_period": 12, "long_period": 26},
            symbol="BTC-USDT",
            timeframe="1h"
        )
        
        assert context.strategy_name == "ema_crossover"
        assert context.strategy_config == {"short_period": 12, "long_period": 26}
        assert context.symbol == "BTC-USDT"
        assert context.timeframe == "1h"
        assert context.exchange == "okx"
        assert context.last_calculation_time is None
        assert context.consecutive_errors == 0
        assert context.is_active == True
    
    def test_context_with_custom_exchange(self):
        """Test context with custom exchange."""
        context = StrategyExecutionContext(
            strategy_name="rsi",
            strategy_config={"period": 14},
            symbol="ETH-USDT",
            timeframe="4h",
            exchange="binance"
        )
        
        assert context.exchange == "binance"


class TestRealTimeSignal:
    """Test RealTimeSignal dataclass."""
    
    def test_signal_creation(self):
        """Test real-time signal creation."""
        # Create mock strategy result
        strategy_result = Mock(spec=StrategyResult)
        strategy_result.timestamp = datetime.now(timezone.utc)
        strategy_result.confidence = 0.8
        
        # Create context
        context = StrategyExecutionContext(
            strategy_name="macd",
            strategy_config={"fast_period": 12},
            symbol="BTC-USDT",
            timeframe="1d"
        )
        
        # Create signal
        signal = RealTimeSignal(
            strategy_result=strategy_result,
            context=context
        )
        
        assert signal.strategy_result == strategy_result
        assert signal.context == context
        assert signal.chart_update_required == True
        assert isinstance(signal.generation_time, datetime)


class TestStrategySignalBroadcaster:
    """Test StrategySignalBroadcaster class."""
    
    @pytest.fixture
    def config(self):
        """Test configuration."""
        return RealTimeConfig(
            signal_batch_size=5,
            max_signal_queue_size=10,
            chart_update_throttle_ms=100
        )
    
    @pytest.fixture
    def mock_db_ops(self):
        """Mock database operations."""
        with patch('strategies.realtime_execution.get_database_operations') as mock:
            db_ops = Mock()
            db_ops.strategy = Mock()
            db_ops.strategy.store_signals_batch = Mock(return_value=5)
            mock.return_value = db_ops
            yield db_ops
    
    @pytest.fixture
    def broadcaster(self, config, mock_db_ops):
        """Create broadcaster instance."""
        return StrategySignalBroadcaster(config)
    
    def test_broadcaster_initialization(self, broadcaster, config):
        """Test broadcaster initialization."""
        assert broadcaster.config == config
        assert broadcaster._is_running == False
        assert broadcaster._chart_update_callback is None
    
    def test_start_stop_broadcaster(self, broadcaster):
        """Test starting and stopping broadcaster."""
        assert not broadcaster._is_running
        
        broadcaster.start()
        assert broadcaster._is_running
        assert broadcaster._processing_thread is not None
        
        broadcaster.stop()
        assert not broadcaster._is_running
    
    def test_broadcast_signal(self, broadcaster):
        """Test broadcasting signals."""
        # Create test signal
        strategy_result = Mock(spec=StrategyResult)
        context = StrategyExecutionContext(
            strategy_name="test",
            strategy_config={},
            symbol="BTC-USDT",
            timeframe="1h"
        )
        signal = RealTimeSignal(strategy_result=strategy_result, context=context)
        
        # Broadcast signal
        success = broadcaster.broadcast_signal(signal)
        assert success == True
        
        # Check queue has signal
        assert broadcaster._signal_queue.qsize() == 1
    
    def test_broadcast_signal_queue_full(self, config, mock_db_ops):
        """Test broadcasting when queue is full."""
        # Create broadcaster with very small queue
        small_config = RealTimeConfig(max_signal_queue_size=1)
        broadcaster = StrategySignalBroadcaster(small_config)
        
        # Create test signals
        strategy_result = Mock(spec=StrategyResult)
        context = StrategyExecutionContext(
            strategy_name="test",
            strategy_config={},
            symbol="BTC-USDT",
            timeframe="1h"
        )
        signal1 = RealTimeSignal(strategy_result=strategy_result, context=context)
        signal2 = RealTimeSignal(strategy_result=strategy_result, context=context)
        
        # Fill queue
        success1 = broadcaster.broadcast_signal(signal1)
        assert success1 == True
        
        # Try to overfill queue
        success2 = broadcaster.broadcast_signal(signal2)
        assert success2 == False  # Should fail due to full queue
    
    def test_set_chart_update_callback(self, broadcaster):
        """Test setting chart update callback."""
        callback = Mock()
        broadcaster.set_chart_update_callback(callback)
        assert broadcaster._chart_update_callback == callback
    
    def test_get_signal_stats(self, broadcaster):
        """Test getting signal statistics."""
        stats = broadcaster.get_signal_stats()
        
        assert 'queue_size' in stats
        assert 'chart_queue_size' in stats
        assert 'is_running' in stats
        assert 'last_chart_updates' in stats
        assert stats['is_running'] == False


class TestRealTimeStrategyProcessor:
    """Test RealTimeStrategyProcessor class."""
    
    @pytest.fixture
    def config(self):
        """Test configuration."""
        return RealTimeConfig(
            max_strategies_concurrent=2,
            error_retry_attempts=2
        )
    
    @pytest.fixture
    def mock_dependencies(self):
        """Mock all external dependencies."""
        mocks = {}
        
        with patch('strategies.realtime_execution.StrategyDataIntegrator') as mock_integrator:
            mocks['data_integrator'] = Mock()
            mock_integrator.return_value = mocks['data_integrator']
            
            with patch('strategies.realtime_execution.MarketDataIntegrator') as mock_market:
                mocks['market_integrator'] = Mock()
                mock_market.return_value = mocks['market_integrator']
                
                with patch('strategies.realtime_execution.StrategyFactory') as mock_factory:
                    mocks['strategy_factory'] = Mock()
                    mock_factory.return_value = mocks['strategy_factory']
                    
                    yield mocks
    
    @pytest.fixture
    def processor(self, config, mock_dependencies):
        """Create processor instance."""
        return RealTimeStrategyProcessor(config)
    
    def test_processor_initialization(self, processor, config):
        """Test processor initialization."""
        assert processor.config == config
        assert processor._execution_contexts == {}
        assert processor._performance_stats['total_calculations'] == 0
    
    def test_start_stop_processor(self, processor):
        """Test starting and stopping processor."""
        processor.start()
        assert processor.signal_broadcaster._is_running == True
        
        processor.stop()
        assert processor.signal_broadcaster._is_running == False
    
    def test_register_strategy(self, processor):
        """Test registering strategy for real-time execution."""
        context_id = processor.register_strategy(
            strategy_name="ema_crossover",
            strategy_config={"short_period": 12, "long_period": 26},
            symbol="BTC-USDT",
            timeframe="1h"
        )
        
        expected_id = "ema_crossover_BTC-USDT_1h_okx"
        assert context_id == expected_id
        assert context_id in processor._execution_contexts
        
        context = processor._execution_contexts[context_id]
        assert context.strategy_name == "ema_crossover"
        assert context.symbol == "BTC-USDT"
        assert context.timeframe == "1h"
        assert context.is_active == True
    
    def test_unregister_strategy(self, processor):
        """Test unregistering strategy."""
        # Register first
        context_id = processor.register_strategy(
            strategy_name="rsi",
            strategy_config={"period": 14},
            symbol="ETH-USDT",
            timeframe="4h"
        )
        
        assert context_id in processor._execution_contexts
        
        # Unregister
        success = processor.unregister_strategy(context_id)
        assert success == True
        assert context_id not in processor._execution_contexts
        
        # Try to unregister again
        success2 = processor.unregister_strategy(context_id)
        assert success2 == False
    
    def test_execute_realtime_update_no_strategies(self, processor):
        """Test real-time update with no registered strategies."""
        signals = processor.execute_realtime_update("BTC-USDT", "1h")
        assert signals == []
    
    def test_execute_realtime_update_with_strategies(self, processor, mock_dependencies):
        """Test real-time update with registered strategies."""
        # Mock strategy calculation results
        mock_result = Mock(spec=StrategyResult)
        mock_result.timestamp = datetime.now(timezone.utc)
        mock_result.confidence = 0.8
        
        mock_dependencies['data_integrator'].calculate_strategy_signals.return_value = [mock_result]
        
        # Register strategy
        processor.register_strategy(
            strategy_name="ema_crossover",
            strategy_config={"short_period": 12, "long_period": 26},
            symbol="BTC-USDT",
            timeframe="1h"
        )
        
        # Execute update
        signals = processor.execute_realtime_update("BTC-USDT", "1h")
        
        assert len(signals) == 1
        assert isinstance(signals[0], RealTimeSignal)
        assert signals[0].strategy_result == mock_result
    
    def test_get_active_strategies(self, processor):
        """Test getting active strategies."""
        # Register some strategies
        processor.register_strategy("ema", {}, "BTC-USDT", "1h")
        processor.register_strategy("rsi", {}, "ETH-USDT", "4h")
        
        active = processor.get_active_strategies()
        assert len(active) == 2
        
        # Pause one strategy
        context_id = list(active.keys())[0]
        processor.pause_strategy(context_id)
        
        active_after_pause = processor.get_active_strategies()
        assert len(active_after_pause) == 1
    
    def test_pause_resume_strategy(self, processor):
        """Test pausing and resuming strategies."""
        context_id = processor.register_strategy("macd", {}, "BTC-USDT", "1d")
        
        # Pause strategy
        success = processor.pause_strategy(context_id)
        assert success == True
        assert not processor._execution_contexts[context_id].is_active
        
        # Resume strategy
        success = processor.resume_strategy(context_id)
        assert success == True
        assert processor._execution_contexts[context_id].is_active
        
        # Test with invalid context_id
        invalid_success = processor.pause_strategy("invalid_id")
        assert invalid_success == False
    
    def test_get_performance_stats(self, processor):
        """Test getting performance statistics."""
        stats = processor.get_performance_stats()
        
        assert 'total_calculations' in stats
        assert 'successful_calculations' in stats
        assert 'failed_calculations' in stats
        assert 'average_calculation_time_ms' in stats
        assert 'signals_generated' in stats
        assert 'queue_size' in stats  # From signal broadcaster


class TestSingletonAndInitialization:
    """Test singleton pattern and system initialization."""
    
    def test_get_realtime_strategy_processor_singleton(self):
        """Test that processor is singleton."""
        # Clean up any existing processor
        shutdown_realtime_strategy_system()
        
        processor1 = get_realtime_strategy_processor()
        processor2 = get_realtime_strategy_processor()
        
        assert processor1 is processor2
        
        # Clean up
        shutdown_realtime_strategy_system()
    
    def test_initialize_realtime_strategy_system(self):
        """Test system initialization."""
        # Clean up any existing processor
        shutdown_realtime_strategy_system()
        
        config = RealTimeConfig(max_strategies_concurrent=2)
        processor = initialize_realtime_strategy_system(config)
        
        assert processor is not None
        assert processor.signal_broadcaster._is_running == True
        
        # Clean up
        shutdown_realtime_strategy_system()
    
    def test_shutdown_realtime_strategy_system(self):
        """Test system shutdown."""
        # Initialize system
        processor = initialize_realtime_strategy_system()
        assert processor.signal_broadcaster._is_running == True
        
        # Shutdown
        shutdown_realtime_strategy_system()
        
        # Verify shutdown
        # Note: After shutdown, the global processor is set to None
        # So we can't check the processor state, but we can verify
        # a new processor is created on next call
        new_processor = get_realtime_strategy_processor()
        assert new_processor is not None


class TestIntegration:
    """Integration tests for real-time execution pipeline."""
    
    @pytest.fixture
    def integration_config(self):
        """Configuration for integration tests."""
        return RealTimeConfig(
            signal_batch_size=2,
            max_signal_queue_size=5,
            chart_update_throttle_ms=50
        )
    
    def test_end_to_end_signal_flow(self, integration_config):
        """Test complete signal flow from strategy to storage."""
        with patch('strategies.realtime_execution.get_database_operations') as mock_db:
            # Setup mocks
            db_ops = Mock()
            db_ops.strategy = Mock()
            db_ops.strategy.store_signals_batch = Mock(return_value=2)
            mock_db.return_value = db_ops
            
            # Create processor
            processor = RealTimeStrategyProcessor(integration_config)
            processor.start()
            
            try:
                # Mock strategy calculation
                mock_result = Mock(spec=StrategyResult)
                mock_result.timestamp = datetime.now(timezone.utc)
                mock_result.confidence = 0.8
                mock_result.signal = Mock()
                mock_result.signal.signal_type = SignalType.BUY
                mock_result.price = 50000.0
                mock_result.metadata = {"test": True}
                
                with patch.object(processor.data_integrator, 'calculate_strategy_signals') as mock_calc:
                    mock_calc.return_value = [mock_result]
                    
                    # Register strategy
                    processor.register_strategy(
                        strategy_name="test_strategy",
                        strategy_config={"param": "value"},
                        symbol="BTC-USDT",
                        timeframe="1h"
                    )
                    
                    # Execute real-time update
                    signals = processor.execute_realtime_update("BTC-USDT", "1h")
                    
                    assert len(signals) == 1
                    
                    # Wait for signal processing
                    time.sleep(0.2)  # Allow background processing
                    
                    # Verify calculation was called
                    mock_calc.assert_called_once()
                    
            finally:
                processor.stop()
    
    def test_error_handling_and_retry(self, integration_config):
        """Test error handling and retry mechanisms."""
        processor = RealTimeStrategyProcessor(integration_config)
        processor.start()
        
        try:
            # Mock strategy calculation to raise error
            with patch.object(processor.data_integrator, 'calculate_strategy_signals') as mock_calc:
                mock_calc.side_effect = Exception("Test error")
                
                # Register strategy
                context_id = processor.register_strategy(
                    strategy_name="error_strategy",
                    strategy_config={},
                    symbol="BTC-USDT",
                    timeframe="1h"
                )
                
                # Execute multiple times to trigger error handling
                for _ in range(integration_config.error_retry_attempts + 1):
                    processor.execute_realtime_update("BTC-USDT", "1h")
                
                # Strategy should be disabled after max errors
                context = processor._execution_contexts[context_id]
                assert not context.is_active
                assert context.consecutive_errors >= integration_config.error_retry_attempts
                
        finally:
            processor.stop()
    
    def test_concurrent_strategy_execution(self, integration_config):
        """Test concurrent execution of multiple strategies."""
        processor = RealTimeStrategyProcessor(integration_config)
        processor.start()
        
        try:
            # Mock strategy calculations
            mock_result1 = Mock(spec=StrategyResult)
            mock_result1.timestamp = datetime.now(timezone.utc)
            mock_result1.confidence = 0.7
            
            mock_result2 = Mock(spec=StrategyResult)
            mock_result2.timestamp = datetime.now(timezone.utc)
            mock_result2.confidence = 0.9
            
            with patch.object(processor.data_integrator, 'calculate_strategy_signals') as mock_calc:
                mock_calc.side_effect = [[mock_result1], [mock_result2]]
                
                # Register multiple strategies for same symbol/timeframe
                processor.register_strategy("strategy1", {}, "BTC-USDT", "1h")
                processor.register_strategy("strategy2", {}, "BTC-USDT", "1h")
                
                # Execute update
                signals = processor.execute_realtime_update("BTC-USDT", "1h")
                
                # Should get signals from both strategies
                assert len(signals) == 2
                
        finally:
            processor.stop()