""" Tests for the common transformation utilities. This module provides comprehensive test coverage for the base transformation utilities used across all exchanges. """ import pytest from datetime import datetime, timezone from decimal import Decimal from typing import Dict, Any from data.common.transformation import ( BaseDataTransformer, UnifiedDataTransformer, create_standardized_trade, batch_create_standardized_trades ) from data.common.data_types import StandardizedTrade from data.exchanges.okx.data_processor import OKXDataTransformer class MockDataTransformer(BaseDataTransformer): """Mock transformer for testing base functionality.""" def __init__(self, component_name: str = "mock_transformer"): super().__init__("mock", component_name) def transform_trade_data(self, raw_data: Dict[str, Any], symbol: str) -> StandardizedTrade: return create_standardized_trade( symbol=symbol, trade_id=raw_data['id'], price=raw_data['price'], size=raw_data['size'], side=raw_data['side'], timestamp=raw_data['timestamp'], exchange="mock", raw_data=raw_data ) def transform_orderbook_data(self, raw_data: Dict[str, Any], symbol: str) -> Dict[str, Any]: return { 'symbol': symbol, 'asks': raw_data.get('asks', []), 'bids': raw_data.get('bids', []), 'timestamp': self.timestamp_to_datetime(raw_data['timestamp']), 'exchange': 'mock', 'raw_data': raw_data } def transform_ticker_data(self, raw_data: Dict[str, Any], symbol: str) -> Dict[str, Any]: return { 'symbol': symbol, 'last': self.safe_decimal_conversion(raw_data.get('last')), 'timestamp': self.timestamp_to_datetime(raw_data['timestamp']), 'exchange': 'mock', 'raw_data': raw_data } @pytest.fixture def mock_transformer(): """Create mock transformer instance.""" return MockDataTransformer() @pytest.fixture def unified_transformer(mock_transformer): """Create unified transformer instance.""" return UnifiedDataTransformer(mock_transformer) @pytest.fixture def okx_transformer(): """Create OKX transformer instance.""" return OKXDataTransformer("test_okx_transformer") @pytest.fixture def sample_trade_data(): """Sample trade data for testing.""" return { 'id': '123456', 'price': '50000.50', 'size': '0.1', 'side': 'buy', 'timestamp': 1640995200000 # 2022-01-01 00:00:00 UTC } @pytest.fixture def sample_okx_trade_data(): """Sample OKX trade data for testing.""" return { 'instId': 'BTC-USDT', 'tradeId': '123456', 'px': '50000.50', 'sz': '0.1', 'side': 'buy', 'ts': '1640995200000' } @pytest.fixture def sample_orderbook_data(): """Sample orderbook data for testing.""" return { 'asks': [['50100.5', '1.5'], ['50200.0', '2.0']], 'bids': [['49900.5', '1.0'], ['49800.0', '2.5']], 'timestamp': 1640995200000 } @pytest.fixture def sample_okx_orderbook_data(): """Sample OKX orderbook data for testing.""" return { 'instId': 'BTC-USDT', 'asks': [['50100.5', '1.5'], ['50200.0', '2.0']], 'bids': [['49900.5', '1.0'], ['49800.0', '2.5']], 'ts': '1640995200000' } @pytest.fixture def sample_ticker_data(): """Sample ticker data for testing.""" return { 'last': '50000.50', 'timestamp': 1640995200000 } @pytest.fixture def sample_okx_ticker_data(): """Sample OKX ticker data for testing.""" return { 'instId': 'BTC-USDT', 'last': '50000.50', 'bidPx': '49999.00', 'askPx': '50001.00', 'open24h': '49000.00', 'high24h': '51000.00', 'low24h': '48000.00', 'vol24h': '1000.0', 'ts': '1640995200000' } class TestBaseDataTransformer: """Test base data transformer functionality.""" def test_timestamp_to_datetime(self, mock_transformer): """Test timestamp conversion to datetime.""" # Test millisecond timestamp dt = mock_transformer.timestamp_to_datetime(1640995200000) assert isinstance(dt, datetime) assert dt.tzinfo == timezone.utc assert dt.year == 2022 assert dt.month == 1 assert dt.day == 1 # Test second timestamp dt = mock_transformer.timestamp_to_datetime(1640995200, is_milliseconds=False) assert dt.year == 2022 # Test string timestamp dt = mock_transformer.timestamp_to_datetime("1640995200000") assert dt.year == 2022 # Test invalid timestamp dt = mock_transformer.timestamp_to_datetime("invalid") assert isinstance(dt, datetime) assert dt.tzinfo == timezone.utc def test_safe_decimal_conversion(self, mock_transformer): """Test safe decimal conversion.""" # Test valid decimal string assert mock_transformer.safe_decimal_conversion("123.45") == Decimal("123.45") # Test valid integer assert mock_transformer.safe_decimal_conversion(123) == Decimal("123") # Test None value assert mock_transformer.safe_decimal_conversion(None) is None # Test empty string assert mock_transformer.safe_decimal_conversion("") is None # Test invalid value assert mock_transformer.safe_decimal_conversion("invalid") is None def test_normalize_trade_side(self, mock_transformer): """Test trade side normalization.""" # Test buy variations assert mock_transformer.normalize_trade_side("buy") == "buy" assert mock_transformer.normalize_trade_side("BUY") == "buy" assert mock_transformer.normalize_trade_side("bid") == "buy" assert mock_transformer.normalize_trade_side("b") == "buy" assert mock_transformer.normalize_trade_side("1") == "buy" # Test sell variations assert mock_transformer.normalize_trade_side("sell") == "sell" assert mock_transformer.normalize_trade_side("SELL") == "sell" assert mock_transformer.normalize_trade_side("ask") == "sell" assert mock_transformer.normalize_trade_side("s") == "sell" assert mock_transformer.normalize_trade_side("0") == "sell" # Test unknown value assert mock_transformer.normalize_trade_side("unknown") == "buy" def test_validate_symbol_format(self, mock_transformer): """Test symbol format validation.""" # Test valid symbol assert mock_transformer.validate_symbol_format("btc-usdt") == "BTC-USDT" assert mock_transformer.validate_symbol_format("BTC-USDT") == "BTC-USDT" # Test symbol with whitespace assert mock_transformer.validate_symbol_format(" btc-usdt ") == "BTC-USDT" # Test invalid symbols with pytest.raises(ValueError): mock_transformer.validate_symbol_format("") with pytest.raises(ValueError): mock_transformer.validate_symbol_format(None) def test_get_transformer_info(self, mock_transformer): """Test transformer info retrieval.""" info = mock_transformer.get_transformer_info() assert info['exchange'] == "mock" assert info['component'] == "mock_transformer" assert 'capabilities' in info assert info['capabilities']['trade_transformation'] is True assert info['capabilities']['orderbook_transformation'] is True assert info['capabilities']['ticker_transformation'] is True class TestUnifiedDataTransformer: """Test unified data transformer functionality.""" def test_transform_trade_data(self, unified_transformer, sample_trade_data): """Test trade data transformation.""" result = unified_transformer.transform_trade_data(sample_trade_data, "BTC-USDT") assert isinstance(result, StandardizedTrade) assert result.symbol == "BTC-USDT" assert result.trade_id == "123456" assert result.price == Decimal("50000.50") assert result.size == Decimal("0.1") assert result.side == "buy" assert result.exchange == "mock" def test_transform_orderbook_data(self, unified_transformer, sample_orderbook_data): """Test orderbook data transformation.""" result = unified_transformer.transform_orderbook_data(sample_orderbook_data, "BTC-USDT") assert result is not None assert result['symbol'] == "BTC-USDT" assert result['exchange'] == "mock" assert len(result['asks']) == 2 assert len(result['bids']) == 2 def test_transform_ticker_data(self, unified_transformer, sample_ticker_data): """Test ticker data transformation.""" result = unified_transformer.transform_ticker_data(sample_ticker_data, "BTC-USDT") assert result is not None assert result['symbol'] == "BTC-USDT" assert result['exchange'] == "mock" assert result['last'] == Decimal("50000.50") def test_batch_transform_trades(self, unified_transformer): """Test batch trade transformation.""" raw_trades = [ { 'id': '123456', 'price': '50000.50', 'size': '0.1', 'side': 'buy', 'timestamp': 1640995200000 }, { 'id': '123457', 'price': '50001.00', 'size': '0.2', 'side': 'sell', 'timestamp': 1640995201000 } ] results = unified_transformer.batch_transform_trades(raw_trades, "BTC-USDT") assert len(results) == 2 assert all(isinstance(r, StandardizedTrade) for r in results) assert results[0].trade_id == "123456" assert results[1].trade_id == "123457" def test_get_transformer_info(self, unified_transformer): """Test unified transformer info retrieval.""" info = unified_transformer.get_transformer_info() assert info['exchange'] == "mock" assert 'unified_component' in info assert info['batch_processing'] is True assert info['candle_aggregation'] is True class TestOKXDataTransformer: """Test OKX-specific data transformer functionality.""" def test_transform_trade_data(self, okx_transformer, sample_okx_trade_data): """Test OKX trade data transformation.""" result = okx_transformer.transform_trade_data(sample_okx_trade_data, "BTC-USDT") assert isinstance(result, StandardizedTrade) assert result.symbol == "BTC-USDT" assert result.trade_id == "123456" assert result.price == Decimal("50000.50") assert result.size == Decimal("0.1") assert result.side == "buy" assert result.exchange == "okx" def test_transform_orderbook_data(self, okx_transformer, sample_okx_orderbook_data): """Test OKX orderbook data transformation.""" result = okx_transformer.transform_orderbook_data(sample_okx_orderbook_data, "BTC-USDT") assert result is not None assert result['symbol'] == "BTC-USDT" assert result['exchange'] == "okx" assert len(result['asks']) == 2 assert len(result['bids']) == 2 def test_transform_ticker_data(self, okx_transformer, sample_okx_ticker_data): """Test OKX ticker data transformation.""" result = okx_transformer.transform_ticker_data(sample_okx_ticker_data, "BTC-USDT") assert result is not None assert result['symbol'] == "BTC-USDT" assert result['exchange'] == "okx" assert result['last'] == Decimal("50000.50") assert result['bid'] == Decimal("49999.00") assert result['ask'] == Decimal("50001.00") assert result['open_24h'] == Decimal("49000.00") assert result['high_24h'] == Decimal("51000.00") assert result['low_24h'] == Decimal("48000.00") assert result['volume_24h'] == Decimal("1000.0") class TestStandaloneTransformationFunctions: """Test standalone transformation utility functions.""" def test_create_standardized_trade(self): """Test standardized trade creation.""" trade = create_standardized_trade( symbol="BTC-USDT", trade_id="123456", price="50000.50", size="0.1", side="buy", timestamp=1640995200000, exchange="test", is_milliseconds=True ) assert isinstance(trade, StandardizedTrade) assert trade.symbol == "BTC-USDT" assert trade.trade_id == "123456" assert trade.price == Decimal("50000.50") assert trade.size == Decimal("0.1") assert trade.side == "buy" assert trade.exchange == "test" assert trade.timestamp.year == 2022 # Test with datetime input dt = datetime(2022, 1, 1, tzinfo=timezone.utc) trade = create_standardized_trade( symbol="BTC-USDT", trade_id="123456", price="50000.50", size="0.1", side="buy", timestamp=dt, exchange="test" ) assert trade.timestamp == dt # Test invalid inputs with pytest.raises(ValueError): create_standardized_trade( symbol="BTC-USDT", trade_id="123456", price="invalid", size="0.1", side="buy", timestamp=1640995200000, exchange="test" ) with pytest.raises(ValueError): create_standardized_trade( symbol="BTC-USDT", trade_id="123456", price="50000.50", size="0.1", side="invalid", timestamp=1640995200000, exchange="test" ) def test_batch_create_standardized_trades(self): """Test batch trade creation.""" raw_trades = [ {'id': '123456', 'px': '50000.50', 'sz': '0.1', 'side': 'buy', 'ts': 1640995200000}, {'id': '123457', 'px': '50001.00', 'sz': '0.2', 'side': 'sell', 'ts': 1640995201000} ] field_mapping = { 'trade_id': 'id', 'price': 'px', 'size': 'sz', 'side': 'side', 'timestamp': 'ts' } trades = batch_create_standardized_trades( raw_trades=raw_trades, symbol="BTC-USDT", exchange="test", field_mapping=field_mapping ) assert len(trades) == 2 assert all(isinstance(t, StandardizedTrade) for t in trades) assert trades[0].trade_id == "123456" assert trades[0].price == Decimal("50000.50") assert trades[1].trade_id == "123457" assert trades[1].side == "sell"