68030730e9
- Introduced a new transformation module that includes safety limits for trade operations, enhancing data integrity and preventing errors. - Refactored existing transformation logic into dedicated classes and functions, improving modularity and maintainability. - Added detailed validation for trade sizes, prices, and symbol formats, ensuring compliance with trading rules. - Implemented logging for significant operations and validation checks, aiding in monitoring and debugging. - Created a changelog to document the new features and changes, providing clarity for future development. - Developed extensive unit tests to cover the new functionality, ensuring reliability and preventing regressions. These changes significantly enhance the architecture of the transformation module, making it more robust and easier to manage.
360 lines
11 KiB
Python
360 lines
11 KiB
Python
"""
|
|
Trade data transformation with safety limits.
|
|
|
|
This module handles the transformation of trade data while enforcing safety limits
|
|
to prevent errors and protect against edge cases.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from decimal import Decimal, InvalidOperation
|
|
from typing import Dict, List, Optional, Any
|
|
|
|
from ..data_types import StandardizedTrade
|
|
from .time_utils import timestamp_to_datetime
|
|
from .numeric_utils import safe_decimal_conversion
|
|
from .normalization import normalize_trade_side, validate_symbol_format, normalize_exchange_name
|
|
from .safety import (
|
|
validate_trade_size,
|
|
validate_trade_price,
|
|
TradeLimits,
|
|
get_trade_limits
|
|
)
|
|
|
|
|
|
# Create a logger for this module
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def create_standardized_trade(
|
|
symbol: str,
|
|
trade_id: str,
|
|
price: Any,
|
|
size: Any,
|
|
side: str,
|
|
timestamp: Any,
|
|
exchange: str,
|
|
raw_data: Optional[Dict[str, Any]] = None,
|
|
is_milliseconds: bool = True
|
|
) -> StandardizedTrade:
|
|
"""
|
|
Utility function to create StandardizedTrade with proper validation.
|
|
|
|
Args:
|
|
symbol: Trading symbol
|
|
trade_id: Trade identifier
|
|
price: Trade price (any numeric type)
|
|
size: Trade size (any numeric type)
|
|
side: Trade side ('buy' or 'sell')
|
|
timestamp: Trade timestamp
|
|
exchange: Exchange name
|
|
raw_data: Original raw data
|
|
is_milliseconds: True if timestamp is in milliseconds
|
|
|
|
Returns:
|
|
StandardizedTrade object
|
|
|
|
Raises:
|
|
ValueError: If data is invalid
|
|
"""
|
|
# Validate symbol
|
|
if not symbol or not isinstance(symbol, str):
|
|
raise ValueError(f"Invalid symbol: {symbol}")
|
|
|
|
# Validate trade_id
|
|
if not trade_id:
|
|
raise ValueError(f"Invalid trade_id: {trade_id}")
|
|
|
|
# Convert timestamp
|
|
try:
|
|
if isinstance(timestamp, (int, float, str)):
|
|
dt = timestamp_to_datetime(timestamp, is_milliseconds)
|
|
elif isinstance(timestamp, datetime):
|
|
dt = timestamp
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
else:
|
|
raise ValueError(f"Invalid timestamp type: {type(timestamp)}")
|
|
except Exception as e:
|
|
raise ValueError(f"Invalid timestamp: {timestamp}") from e
|
|
|
|
# Convert price and size to Decimal
|
|
try:
|
|
if not price or not size:
|
|
raise ValueError("Price and size must not be empty")
|
|
|
|
decimal_price = safe_decimal_conversion(price, "price")
|
|
decimal_size = safe_decimal_conversion(size, "size")
|
|
|
|
if decimal_price is None or decimal_size is None:
|
|
raise ValueError("Invalid price or size format")
|
|
|
|
if decimal_price <= 0:
|
|
raise ValueError(f"Price must be positive: {price}")
|
|
if decimal_size <= 0:
|
|
raise ValueError(f"Size must be positive: {size}")
|
|
|
|
except (InvalidOperation, TypeError, ValueError) as e:
|
|
raise ValueError(f"Invalid price or size: {e}")
|
|
|
|
# Normalize side with strict validation
|
|
try:
|
|
if not side or not isinstance(side, str):
|
|
raise ValueError(f"Invalid trade side: {side}")
|
|
|
|
normalized_side = normalize_trade_side(side, logger=logger)
|
|
except ValueError as e:
|
|
logger.error(f"Trade side validation failed: {e}")
|
|
raise ValueError(f"Invalid trade side: {side}")
|
|
|
|
# Normalize symbol and exchange
|
|
try:
|
|
normalized_symbol = validate_symbol_format(symbol)
|
|
normalized_exchange = normalize_exchange_name(exchange)
|
|
except ValueError as e:
|
|
raise ValueError(str(e))
|
|
|
|
return StandardizedTrade(
|
|
symbol=normalized_symbol,
|
|
trade_id=str(trade_id),
|
|
price=decimal_price,
|
|
size=decimal_size,
|
|
side=normalized_side,
|
|
timestamp=dt,
|
|
exchange=normalized_exchange,
|
|
raw_data=raw_data
|
|
)
|
|
|
|
|
|
def batch_create_standardized_trades(
|
|
raw_trades: List[Dict[str, Any]],
|
|
symbol: str,
|
|
exchange: str,
|
|
field_mapping: Dict[str, str],
|
|
is_milliseconds: bool = True
|
|
) -> List[StandardizedTrade]:
|
|
"""
|
|
Batch create standardized trades from raw data.
|
|
|
|
Args:
|
|
raw_trades: List of raw trade dictionaries
|
|
symbol: Trading symbol
|
|
exchange: Exchange name
|
|
field_mapping: Mapping of StandardizedTrade fields to raw data fields
|
|
is_milliseconds: True if timestamps are in milliseconds
|
|
|
|
Returns:
|
|
List of successfully created StandardizedTrade objects
|
|
|
|
Example field_mapping:
|
|
{
|
|
'trade_id': 'id',
|
|
'price': 'px',
|
|
'size': 'sz',
|
|
'side': 'side',
|
|
'timestamp': 'ts'
|
|
}
|
|
"""
|
|
trades = []
|
|
|
|
for raw_trade in raw_trades:
|
|
try:
|
|
trade = create_standardized_trade(
|
|
symbol=symbol,
|
|
trade_id=raw_trade[field_mapping['trade_id']],
|
|
price=raw_trade[field_mapping['price']],
|
|
size=raw_trade[field_mapping['size']],
|
|
side=raw_trade[field_mapping['side']],
|
|
timestamp=raw_trade[field_mapping['timestamp']],
|
|
exchange=exchange,
|
|
raw_data=raw_trade,
|
|
is_milliseconds=is_milliseconds
|
|
)
|
|
trades.append(trade)
|
|
except Exception as e:
|
|
# Log error but continue processing
|
|
print(f"Failed to transform trade: {e}")
|
|
|
|
return trades
|
|
|
|
|
|
class TradeTransformer:
|
|
"""Transform trade data with safety checks."""
|
|
|
|
VALID_SIDES = {'buy', 'sell'}
|
|
|
|
def __init__(self, market_data_provider: Optional[Any] = None):
|
|
"""
|
|
Initialize transformer.
|
|
|
|
Args:
|
|
market_data_provider: Optional provider of market data for price validation
|
|
"""
|
|
self.market_data_provider = market_data_provider
|
|
|
|
def normalize_trade_side(self, side: str) -> str:
|
|
"""
|
|
Normalize trade side to standard format.
|
|
|
|
Args:
|
|
side: Trade side indicator
|
|
|
|
Returns:
|
|
Normalized trade side ('buy' or 'sell')
|
|
|
|
Raises:
|
|
ValueError: If side is invalid
|
|
"""
|
|
side_lower = str(side).lower().strip()
|
|
|
|
# Handle common variations
|
|
if side_lower in {'buy', 'bid', 'long', '1', 'true'}:
|
|
return 'buy'
|
|
elif side_lower in {'sell', 'ask', 'short', '0', 'false'}:
|
|
return 'sell'
|
|
|
|
raise ValueError(f"Invalid trade side: {side}")
|
|
|
|
def normalize_trade_size(
|
|
self,
|
|
size: Any,
|
|
price: Any,
|
|
symbol: str
|
|
) -> Decimal:
|
|
"""
|
|
Normalize and validate trade size.
|
|
|
|
Args:
|
|
size: Raw trade size
|
|
price: Trade price for notional calculations
|
|
symbol: Trading pair symbol
|
|
|
|
Returns:
|
|
Normalized trade size as Decimal
|
|
|
|
Raises:
|
|
ValueError: If size is invalid or violates limits
|
|
"""
|
|
try:
|
|
size_decimal = Decimal(str(size))
|
|
price_decimal = Decimal(str(price))
|
|
except (TypeError, ValueError) as e:
|
|
raise ValueError(f"Invalid trade size or price format: {e}")
|
|
|
|
if size_decimal <= 0:
|
|
raise ValueError(f"Trade size must be positive: {size}")
|
|
|
|
# Get limits and validate
|
|
limits = get_trade_limits(symbol)
|
|
|
|
# Round to appropriate precision
|
|
size_decimal = round(size_decimal, limits.size_precision)
|
|
|
|
# Validate against limits
|
|
validate_trade_size(
|
|
size_decimal,
|
|
price_decimal,
|
|
symbol,
|
|
logger
|
|
)
|
|
|
|
return size_decimal
|
|
|
|
def normalize_trade_price(
|
|
self,
|
|
price: Any,
|
|
symbol: str
|
|
) -> Decimal:
|
|
"""
|
|
Normalize and validate trade price.
|
|
|
|
Args:
|
|
price: Raw trade price
|
|
symbol: Trading pair symbol
|
|
|
|
Returns:
|
|
Normalized price as Decimal
|
|
|
|
Raises:
|
|
ValueError: If price is invalid or violates limits
|
|
"""
|
|
try:
|
|
price_decimal = Decimal(str(price))
|
|
except (TypeError, ValueError) as e:
|
|
raise ValueError(f"Invalid price format: {e}")
|
|
|
|
if price_decimal <= 0:
|
|
raise ValueError(f"Price must be positive: {price}")
|
|
|
|
# Get limits and round to appropriate precision
|
|
limits = get_trade_limits(symbol)
|
|
price_decimal = round(price_decimal, limits.price_precision)
|
|
|
|
# Get market price if available
|
|
market_price = None
|
|
if self.market_data_provider is not None:
|
|
try:
|
|
market_price = self.market_data_provider.get_price(symbol)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to get market price for {symbol}: {e}")
|
|
|
|
# Validate against limits and market price
|
|
validate_trade_price(
|
|
price_decimal,
|
|
market_price,
|
|
symbol,
|
|
logger
|
|
)
|
|
|
|
return price_decimal
|
|
|
|
def transform_trade(
|
|
self,
|
|
trade_data: Dict[str, Any]
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Transform trade data with safety checks.
|
|
|
|
Args:
|
|
trade_data: Raw trade data
|
|
|
|
Returns:
|
|
Transformed trade data with normalized values
|
|
|
|
Raises:
|
|
ValueError: If any validation fails
|
|
"""
|
|
if not isinstance(trade_data, dict):
|
|
raise ValueError(f"Trade data must be a dictionary: {trade_data}")
|
|
|
|
# Required fields
|
|
required = {'symbol', 'side', 'size', 'price'}
|
|
missing = required - set(trade_data.keys())
|
|
if missing:
|
|
raise ValueError(f"Missing required fields: {missing}")
|
|
|
|
# Validate and normalize symbol
|
|
symbol = str(trade_data['symbol']).upper()
|
|
validate_symbol_format(symbol, logger)
|
|
|
|
# Transform with safety checks
|
|
transformed = {
|
|
'symbol': symbol,
|
|
'side': self.normalize_trade_side(trade_data['side']),
|
|
'size': self.normalize_trade_size(
|
|
trade_data['size'],
|
|
trade_data['price'],
|
|
symbol
|
|
),
|
|
'price': self.normalize_trade_price(
|
|
trade_data['price'],
|
|
symbol
|
|
)
|
|
}
|
|
|
|
# Copy any additional fields
|
|
for key, value in trade_data.items():
|
|
if key not in transformed:
|
|
transformed[key] = value
|
|
|
|
return transformed |