TCPDashboard/tests/test_transformation.py

"""
Tests for the common transformation utilities.

This module provides comprehensive test coverage for the base transformation
utilities used across all exchanges.
"""

import pytest
from datetime import datetime, timezone
from decimal import Decimal
from typing import Dict, Any

from data.common.transformation import (
    BaseDataTransformer,
    UnifiedDataTransformer,
    create_standardized_trade,
    batch_create_standardized_trades
)
from data.common.data_types import StandardizedTrade
from data.exchanges.okx.data_processor import OKXDataTransformer


class MockDataTransformer(BaseDataTransformer):
    """Mock transformer for testing base functionality."""
    
    def __init__(self, component_name: str = "mock_transformer"):
        super().__init__("mock", component_name)
    
    def transform_trade_data(self, raw_data: Dict[str, Any], symbol: str) -> StandardizedTrade:
        return create_standardized_trade(
            symbol=symbol,
            trade_id=raw_data['id'],
            price=raw_data['price'],
            size=raw_data['size'],
            side=raw_data['side'],
            timestamp=raw_data['timestamp'],
            exchange="mock",
            raw_data=raw_data
        )
    
    def transform_orderbook_data(self, raw_data: Dict[str, Any], symbol: str) -> Dict[str, Any]:
        return {
            'symbol': symbol,
            'asks': raw_data.get('asks', []),
            'bids': raw_data.get('bids', []),
            'timestamp': self.timestamp_to_datetime(raw_data['timestamp']),
            'exchange': 'mock',
            'raw_data': raw_data
        }
    
    def transform_ticker_data(self, raw_data: Dict[str, Any], symbol: str) -> Dict[str, Any]:
        return {
            'symbol': symbol,
            'last': self.safe_decimal_conversion(raw_data.get('last')),
            'timestamp': self.timestamp_to_datetime(raw_data['timestamp']),
            'exchange': 'mock',
            'raw_data': raw_data
        }


@pytest.fixture
def mock_transformer():
    """Create mock transformer instance."""
    return MockDataTransformer()


@pytest.fixture
def unified_transformer(mock_transformer):
    """Create unified transformer instance."""
    return UnifiedDataTransformer(mock_transformer)


@pytest.fixture
def okx_transformer():
    """Create OKX transformer instance."""
    return OKXDataTransformer("test_okx_transformer")


@pytest.fixture
def sample_trade_data():
    """Sample trade data for testing."""
    return {
        'id': '123456',
        'price': '50000.50',
        'size': '0.1',
        'side': 'buy',
        'timestamp': 1640995200000  # 2022-01-01 00:00:00 UTC
    }


@pytest.fixture
def sample_okx_trade_data():
    """Sample OKX trade data for testing."""
    return {
        'instId': 'BTC-USDT',
        'tradeId': '123456',
        'px': '50000.50',
        'sz': '0.1',
        'side': 'buy',
        'ts': '1640995200000'
    }


@pytest.fixture
def sample_orderbook_data():
    """Sample orderbook data for testing."""
    return {
        'asks': [['50100.5', '1.5'], ['50200.0', '2.0']],
        'bids': [['49900.5', '1.0'], ['49800.0', '2.5']],
        'timestamp': 1640995200000
    }


@pytest.fixture
def sample_okx_orderbook_data():
    """Sample OKX orderbook data for testing."""
    return {
        'instId': 'BTC-USDT',
        'asks': [['50100.5', '1.5'], ['50200.0', '2.0']],
        'bids': [['49900.5', '1.0'], ['49800.0', '2.5']],
        'ts': '1640995200000'
    }


@pytest.fixture
def sample_ticker_data():
    """Sample ticker data for testing."""
    return {
        'last': '50000.50',
        'timestamp': 1640995200000
    }


@pytest.fixture
def sample_okx_ticker_data():
    """Sample OKX ticker data for testing."""
    return {
        'instId': 'BTC-USDT',
        'last': '50000.50',
        'bidPx': '49999.00',
        'askPx': '50001.00',
        'open24h': '49000.00',
        'high24h': '51000.00',
        'low24h': '48000.00',
        'vol24h': '1000.0',
        'ts': '1640995200000'
    }


class TestBaseDataTransformer:
    """Test base data transformer functionality."""
    
    def test_timestamp_to_datetime(self, mock_transformer):
        """Test timestamp conversion to datetime."""
        # Test millisecond timestamp
        dt = mock_transformer.timestamp_to_datetime(1640995200000)
        assert isinstance(dt, datetime)
        assert dt.tzinfo == timezone.utc
        assert dt.year == 2022
        assert dt.month == 1
        assert dt.day == 1
        
        # Test second timestamp
        dt = mock_transformer.timestamp_to_datetime(1640995200, is_milliseconds=False)
        assert dt.year == 2022
        
        # Test string timestamp
        dt = mock_transformer.timestamp_to_datetime("1640995200000")
        assert dt.year == 2022
        
        # Test invalid timestamp
        dt = mock_transformer.timestamp_to_datetime("invalid")
        assert isinstance(dt, datetime)
        assert dt.tzinfo == timezone.utc
    
    def test_safe_decimal_conversion(self, mock_transformer):
        """Test safe decimal conversion."""
        # Test valid decimal string
        assert mock_transformer.safe_decimal_conversion("123.45") == Decimal("123.45")
        
        # Test valid integer
        assert mock_transformer.safe_decimal_conversion(123) == Decimal("123")
        
        # Test None value
        assert mock_transformer.safe_decimal_conversion(None) is None
        
        # Test empty string
        assert mock_transformer.safe_decimal_conversion("") is None
        
        # Test invalid value
        assert mock_transformer.safe_decimal_conversion("invalid") is None
    
    def test_normalize_trade_side(self, mock_transformer):
        """Test trade side normalization."""
        # Test buy variations
        assert mock_transformer.normalize_trade_side("buy") == "buy"
        assert mock_transformer.normalize_trade_side("BUY") == "buy"
        assert mock_transformer.normalize_trade_side("bid") == "buy"
        assert mock_transformer.normalize_trade_side("b") == "buy"
        assert mock_transformer.normalize_trade_side("1") == "buy"
        
        # Test sell variations
        assert mock_transformer.normalize_trade_side("sell") == "sell"
        assert mock_transformer.normalize_trade_side("SELL") == "sell"
        assert mock_transformer.normalize_trade_side("ask") == "sell"
        assert mock_transformer.normalize_trade_side("s") == "sell"
        assert mock_transformer.normalize_trade_side("0") == "sell"
        
        # Test unknown value
        assert mock_transformer.normalize_trade_side("unknown") == "buy"
    
    def test_validate_symbol_format(self, mock_transformer):
        """Test symbol format validation."""
        # Test valid symbol
        assert mock_transformer.validate_symbol_format("btc-usdt") == "BTC-USDT"
        assert mock_transformer.validate_symbol_format("BTC-USDT") == "BTC-USDT"
        
        # Test symbol with whitespace
        assert mock_transformer.validate_symbol_format(" btc-usdt ") == "BTC-USDT"
        
        # Test invalid symbols
        with pytest.raises(ValueError):
            mock_transformer.validate_symbol_format("")
        with pytest.raises(ValueError):
            mock_transformer.validate_symbol_format(None)
    
    def test_get_transformer_info(self, mock_transformer):
        """Test transformer info retrieval."""
        info = mock_transformer.get_transformer_info()
        assert info['exchange'] == "mock"
        assert info['component'] == "mock_transformer"
        assert 'capabilities' in info
        assert info['capabilities']['trade_transformation'] is True
        assert info['capabilities']['orderbook_transformation'] is True
        assert info['capabilities']['ticker_transformation'] is True


class TestUnifiedDataTransformer:
    """Test unified data transformer functionality."""
    
    def test_transform_trade_data(self, unified_transformer, sample_trade_data):
        """Test trade data transformation."""
        result = unified_transformer.transform_trade_data(sample_trade_data, "BTC-USDT")
        assert isinstance(result, StandardizedTrade)
        assert result.symbol == "BTC-USDT"
        assert result.trade_id == "123456"
        assert result.price == Decimal("50000.50")
        assert result.size == Decimal("0.1")
        assert result.side == "buy"
        assert result.exchange == "mock"
    
    def test_transform_orderbook_data(self, unified_transformer, sample_orderbook_data):
        """Test orderbook data transformation."""
        result = unified_transformer.transform_orderbook_data(sample_orderbook_data, "BTC-USDT")
        assert result is not None
        assert result['symbol'] == "BTC-USDT"
        assert result['exchange'] == "mock"
        assert len(result['asks']) == 2
        assert len(result['bids']) == 2
    
    def test_transform_ticker_data(self, unified_transformer, sample_ticker_data):
        """Test ticker data transformation."""
        result = unified_transformer.transform_ticker_data(sample_ticker_data, "BTC-USDT")
        assert result is not None
        assert result['symbol'] == "BTC-USDT"
        assert result['exchange'] == "mock"
        assert result['last'] == Decimal("50000.50")
    
    def test_batch_transform_trades(self, unified_transformer):
        """Test batch trade transformation."""
        raw_trades = [
            {
                'id': '123456',
                'price': '50000.50',
                'size': '0.1',
                'side': 'buy',
                'timestamp': 1640995200000
            },
            {
                'id': '123457',
                'price': '50001.00',
                'size': '0.2',
                'side': 'sell',
                'timestamp': 1640995201000
            }
        ]
        
        results = unified_transformer.batch_transform_trades(raw_trades, "BTC-USDT")
        assert len(results) == 2
        assert all(isinstance(r, StandardizedTrade) for r in results)
        assert results[0].trade_id == "123456"
        assert results[1].trade_id == "123457"
    
    def test_get_transformer_info(self, unified_transformer):
        """Test unified transformer info retrieval."""
        info = unified_transformer.get_transformer_info()
        assert info['exchange'] == "mock"
        assert 'unified_component' in info
        assert info['batch_processing'] is True
        assert info['candle_aggregation'] is True


class TestOKXDataTransformer:
    """Test OKX-specific data transformer functionality."""
    
    def test_transform_trade_data(self, okx_transformer, sample_okx_trade_data):
        """Test OKX trade data transformation."""
        result = okx_transformer.transform_trade_data(sample_okx_trade_data, "BTC-USDT")
        assert isinstance(result, StandardizedTrade)
        assert result.symbol == "BTC-USDT"
        assert result.trade_id == "123456"
        assert result.price == Decimal("50000.50")
        assert result.size == Decimal("0.1")
        assert result.side == "buy"
        assert result.exchange == "okx"
    
    def test_transform_orderbook_data(self, okx_transformer, sample_okx_orderbook_data):
        """Test OKX orderbook data transformation."""
        result = okx_transformer.transform_orderbook_data(sample_okx_orderbook_data, "BTC-USDT")
        assert result is not None
        assert result['symbol'] == "BTC-USDT"
        assert result['exchange'] == "okx"
        assert len(result['asks']) == 2
        assert len(result['bids']) == 2
    
    def test_transform_ticker_data(self, okx_transformer, sample_okx_ticker_data):
        """Test OKX ticker data transformation."""
        result = okx_transformer.transform_ticker_data(sample_okx_ticker_data, "BTC-USDT")
        assert result is not None
        assert result['symbol'] == "BTC-USDT"
        assert result['exchange'] == "okx"
        assert result['last'] == Decimal("50000.50")
        assert result['bid'] == Decimal("49999.00")
        assert result['ask'] == Decimal("50001.00")
        assert result['open_24h'] == Decimal("49000.00")
        assert result['high_24h'] == Decimal("51000.00")
        assert result['low_24h'] == Decimal("48000.00")
        assert result['volume_24h'] == Decimal("1000.0")


class TestStandaloneTransformationFunctions:
    """Test standalone transformation utility functions."""
    
    def test_create_standardized_trade(self):
        """Test standardized trade creation."""
        trade = create_standardized_trade(
            symbol="BTC-USDT",
            trade_id="123456",
            price="50000.50",
            size="0.1",
            side="buy",
            timestamp=1640995200000,
            exchange="test",
            is_milliseconds=True
        )
        
        assert isinstance(trade, StandardizedTrade)
        assert trade.symbol == "BTC-USDT"
        assert trade.trade_id == "123456"
        assert trade.price == Decimal("50000.50")
        assert trade.size == Decimal("0.1")
        assert trade.side == "buy"
        assert trade.exchange == "test"
        assert trade.timestamp.year == 2022
        
        # Test with datetime input
        dt = datetime(2022, 1, 1, tzinfo=timezone.utc)
        trade = create_standardized_trade(
            symbol="BTC-USDT",
            trade_id="123456",
            price="50000.50",
            size="0.1",
            side="buy",
            timestamp=dt,
            exchange="test"
        )
        assert trade.timestamp == dt
        
        # Test invalid inputs
        with pytest.raises(ValueError):
            create_standardized_trade(
                symbol="BTC-USDT",
                trade_id="123456",
                price="invalid",
                size="0.1",
                side="buy",
                timestamp=1640995200000,
                exchange="test"
            )
        
        with pytest.raises(ValueError):
            create_standardized_trade(
                symbol="BTC-USDT",
                trade_id="123456",
                price="50000.50",
                size="0.1",
                side="invalid",
                timestamp=1640995200000,
                exchange="test"
            )
    
    def test_batch_create_standardized_trades(self):
        """Test batch trade creation."""
        raw_trades = [
            {'id': '123456', 'px': '50000.50', 'sz': '0.1', 'side': 'buy', 'ts': 1640995200000},
            {'id': '123457', 'px': '50001.00', 'sz': '0.2', 'side': 'sell', 'ts': 1640995201000}
        ]
        
        field_mapping = {
            'trade_id': 'id',
            'price': 'px',
            'size': 'sz',
            'side': 'side',
            'timestamp': 'ts'
        }
        
        trades = batch_create_standardized_trades(
            raw_trades=raw_trades,
            symbol="BTC-USDT",
            exchange="test",
            field_mapping=field_mapping
        )
        
        assert len(trades) == 2
        assert all(isinstance(t, StandardizedTrade) for t in trades)
        assert trades[0].trade_id == "123456"
        assert trades[0].price == Decimal("50000.50")
        assert trades[1].trade_id == "123457"
        assert trades[1].side == "sell"