- Introduced a new transformation module that includes safety limits for trade operations, enhancing data integrity and preventing errors. - Refactored existing transformation logic into dedicated classes and functions, improving modularity and maintainability. - Added detailed validation for trade sizes, prices, and symbol formats, ensuring compliance with trading rules. - Implemented logging for significant operations and validation checks, aiding in monitoring and debugging. - Created a changelog to document the new features and changes, providing clarity for future development. - Developed extensive unit tests to cover the new functionality, ensuring reliability and preventing regressions. These changes significantly enhance the architecture of the transformation module, making it more robust and easier to manage.
228 lines
6.4 KiB
Python
228 lines
6.4 KiB
Python
"""
|
|
Base data transformer class.
|
|
|
|
This module provides the base class for all data transformers
|
|
with common functionality and interface definitions.
|
|
"""
|
|
|
|
import logging
|
|
from typing import Dict, Any, Optional, List
|
|
from datetime import datetime
|
|
from decimal import Decimal
|
|
|
|
from ..data_types import StandardizedTrade
|
|
from .trade import create_standardized_trade, batch_create_standardized_trades
|
|
from .time_utils import timestamp_to_datetime
|
|
from .numeric_utils import safe_decimal_conversion
|
|
from .normalization import normalize_trade_side, validate_symbol_format
|
|
|
|
|
|
class BaseDataTransformer:
|
|
"""Base class for all data transformers."""
|
|
|
|
def __init__(
|
|
self,
|
|
exchange: str,
|
|
component_name: str = "base_transformer",
|
|
logger: Optional[logging.Logger] = None
|
|
):
|
|
"""
|
|
Initialize base transformer.
|
|
|
|
Args:
|
|
exchange: Exchange name
|
|
component_name: Component name for logging
|
|
logger: Optional logger instance
|
|
"""
|
|
self.exchange = exchange
|
|
self.component_name = component_name
|
|
self.logger = logger or logging.getLogger(component_name)
|
|
|
|
def timestamp_to_datetime(
|
|
self,
|
|
timestamp: Any,
|
|
is_milliseconds: bool = True
|
|
) -> datetime:
|
|
"""Convert timestamp to datetime."""
|
|
return timestamp_to_datetime(
|
|
timestamp,
|
|
is_milliseconds,
|
|
logger=self.logger,
|
|
component_name=self.component_name
|
|
)
|
|
|
|
def safe_decimal_conversion(
|
|
self,
|
|
value: Any,
|
|
field_name: str = "value"
|
|
) -> Optional[Decimal]:
|
|
"""Convert value to Decimal safely."""
|
|
return safe_decimal_conversion(
|
|
value,
|
|
field_name,
|
|
logger=self.logger,
|
|
component_name=self.component_name
|
|
)
|
|
|
|
def normalize_trade_side(
|
|
self,
|
|
side: str
|
|
) -> str:
|
|
"""Normalize trade side."""
|
|
try:
|
|
return normalize_trade_side(
|
|
side,
|
|
logger=self.logger,
|
|
component_name=self.component_name
|
|
)
|
|
except ValueError as e:
|
|
self.logger.warning(
|
|
f"{self.component_name}: Unknown trade side: {side}, defaulting to 'buy'"
|
|
)
|
|
return 'buy'
|
|
|
|
def validate_symbol_format(
|
|
self,
|
|
symbol: str
|
|
) -> str:
|
|
"""Validate symbol format."""
|
|
return validate_symbol_format(
|
|
symbol,
|
|
logger=self.logger,
|
|
component_name=self.component_name
|
|
)
|
|
|
|
def get_transformer_info(self) -> Dict[str, Any]:
|
|
"""Get transformer information."""
|
|
return {
|
|
"exchange": self.exchange,
|
|
"component": self.component_name,
|
|
"capabilities": {
|
|
"trade_transformation": True,
|
|
"orderbook_transformation": True,
|
|
"ticker_transformation": True,
|
|
"batch_processing": True
|
|
}
|
|
}
|
|
|
|
def transform_trade_data(
|
|
self,
|
|
raw_data: Dict[str, Any],
|
|
symbol: str
|
|
) -> StandardizedTrade:
|
|
"""
|
|
Transform raw trade data to standardized format.
|
|
|
|
Args:
|
|
raw_data: Raw trade data
|
|
symbol: Trading symbol
|
|
|
|
Returns:
|
|
StandardizedTrade object
|
|
|
|
Raises:
|
|
ValueError: If data is invalid
|
|
"""
|
|
raise NotImplementedError("Subclasses must implement transform_trade_data")
|
|
|
|
def transform_orderbook_data(
|
|
self,
|
|
raw_data: Dict[str, Any],
|
|
symbol: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Transform raw orderbook data to standardized format.
|
|
|
|
Args:
|
|
raw_data: Raw orderbook data
|
|
symbol: Trading symbol
|
|
|
|
Returns:
|
|
Standardized orderbook data
|
|
|
|
Raises:
|
|
ValueError: If data is invalid
|
|
"""
|
|
raise NotImplementedError("Subclasses must implement transform_orderbook_data")
|
|
|
|
def transform_ticker_data(
|
|
self,
|
|
raw_data: Dict[str, Any],
|
|
symbol: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Transform raw ticker data to standardized format.
|
|
|
|
Args:
|
|
raw_data: Raw ticker data
|
|
symbol: Trading symbol
|
|
|
|
Returns:
|
|
Standardized ticker data
|
|
|
|
Raises:
|
|
ValueError: If data is invalid
|
|
"""
|
|
raise NotImplementedError("Subclasses must implement transform_ticker_data")
|
|
|
|
def batch_transform_trades(
|
|
self,
|
|
raw_trades: List[Dict[str, Any]],
|
|
symbol: str
|
|
) -> List[StandardizedTrade]:
|
|
"""
|
|
Transform multiple trades in batch.
|
|
|
|
Args:
|
|
raw_trades: List of raw trade data
|
|
symbol: Trading symbol
|
|
|
|
Returns:
|
|
List of StandardizedTrade objects
|
|
|
|
Raises:
|
|
ValueError: If data is invalid
|
|
"""
|
|
return [
|
|
self.transform_trade_data(trade, symbol)
|
|
for trade in raw_trades
|
|
]
|
|
|
|
def transform_trades_batch(
|
|
self,
|
|
raw_trades: List[Dict[str, Any]],
|
|
symbol: str,
|
|
field_mapping: Dict[str, str]
|
|
) -> List[StandardizedTrade]:
|
|
"""
|
|
Transform a batch of raw trades.
|
|
|
|
Args:
|
|
raw_trades: List of raw trade dictionaries
|
|
symbol: Trading symbol
|
|
field_mapping: Field mapping for raw data
|
|
|
|
Returns:
|
|
List of StandardizedTrade objects
|
|
"""
|
|
return batch_create_standardized_trades(
|
|
raw_trades=raw_trades,
|
|
symbol=symbol,
|
|
exchange=self.exchange,
|
|
field_mapping=field_mapping
|
|
)
|
|
|
|
def _log_error(self, message: str, error: Optional[Exception] = None) -> None:
|
|
"""Log error with component context."""
|
|
if error:
|
|
self.logger.error(f"{self.component_name}: {message}: {error}")
|
|
else:
|
|
self.logger.error(f"{self.component_name}: {message}")
|
|
|
|
def _log_warning(self, message: str) -> None:
|
|
"""Log warning with component context."""
|
|
self.logger.warning(f"{self.component_name}: {message}")
|
|
|
|
def _log_info(self, message: str) -> None:
|
|
"""Log info with component context."""
|
|
self.logger.info(f"{self.component_name}: {message}") |