vasily/TCPDashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance logging capabilities across data collection components

Browse Source 
- Added optional logger parameter to various classes including `BaseDataCollector`, `CollectorManager`, `RealTimeCandleProcessor`, and `BatchCandleProcessor` to support conditional logging.
- Implemented error-only logging mode, allowing components to log only error and critical messages when specified.
- Updated logging calls to utilize new helper methods for improved readability and maintainability.
- Enhanced documentation to include details on the new logging system and its usage across components.
- Ensured that child components inherit the logger from their parent components for consistent logging behavior.
This commit is contained in:
Vasily.onl
2025-06-01 14:42:29 +08:00
parent 0697be75da
commit bc13cfcbe0
11 changed files with 1179 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
data/common/aggregation.py
View File

@@ -30,7 +30,6 @@ from .data_types import (
CandleProcessingConfig,
ProcessingStats
)
from utils.logger import get_logger
class TimeframeBucket:
@@ -183,7 +182,8 @@ class RealTimeCandleProcessor:
symbol: str,
exchange: str,
config: Optional[CandleProcessingConfig] = None,
component_name: str = "realtime_candle_processor"):
component_name: str = "realtime_candle_processor",
logger = None):
"""
Initialize real-time candle processor.
@@ -197,7 +197,7 @@ class RealTimeCandleProcessor:
self.exchange = exchange
self.config = config or CandleProcessingConfig()
self.component_name = component_name
self.logger = get_logger(self.component_name)
self.logger = logger
# Current buckets for each timeframe
self.current_buckets: Dict[str, TimeframeBucket] = {}
@@ -208,12 +208,14 @@ class RealTimeCandleProcessor:
# Statistics
self.stats = ProcessingStats(active_timeframes=len(self.config.timeframes))
self.logger.info(f"Initialized real-time candle processor for {symbol} on {exchange} with timeframes: {self.config.timeframes}")
if self.logger:
self.logger.info(f"{self.component_name}: Initialized real-time candle processor for {symbol} on {exchange} with timeframes: {self.config.timeframes}")
def add_candle_callback(self, callback: Callable[[OHLCVCandle], None]) -> None:
"""Add callback function to receive completed candles."""
self.candle_callbacks.append(callback)
self.logger.debug(f"Added candle callback: {callback.__name__ if hasattr(callback, '__name__') else str(callback)}")
if self.logger:
self.logger.debug(f"{self.component_name}: Added candle callback: {callback.__name__ if hasattr(callback, '__name__') else str(callback)}")
def process_trade(self, trade: StandardizedTrade) -> List[OHLCVCandle]:
"""
@@ -250,7 +252,8 @@ class RealTimeCandleProcessor:
return completed_candles
except Exception as e:
self.logger.error(f"Error processing trade for {self.symbol}: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error processing trade for {self.symbol}: {e}")
self.stats.errors_count += 1
return []
@@ -292,12 +295,14 @@ class RealTimeCandleProcessor:
# Add trade to current bucket
if not current_bucket.add_trade(trade):
# This should never happen if logic is correct
self.logger.warning(f"Trade {trade.timestamp} could not be added to bucket {current_bucket.start_time}-{current_bucket.end_time}")
if self.logger:
self.logger.warning(f"{self.component_name}: Trade {trade.timestamp} could not be added to bucket {current_bucket.start_time}-{current_bucket.end_time}")
return completed_candle
except Exception as e:
self.logger.error(f"Error processing trade for timeframe {timeframe}: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error processing trade for timeframe {timeframe}: {e}")
self.stats.errors_count += 1
return None
@@ -353,7 +358,8 @@ class RealTimeCandleProcessor:
for callback in self.candle_callbacks:
callback(candle)
except Exception as e:
self.logger.error(f"Error in candle callback: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error in candle callback: {e}")
self.stats.errors_count += 1
def get_current_candles(self, incomplete: bool = True) -> List[OHLCVCandle]:
@@ -408,7 +414,8 @@ class BatchCandleProcessor:
symbol: str,
exchange: str,
timeframes: List[str],
component_name: str = "batch_candle_processor"):
component_name: str = "batch_candle_processor",
logger = None):
"""
Initialize batch candle processor.
@@ -422,12 +429,13 @@ class BatchCandleProcessor:
self.exchange = exchange
self.timeframes = timeframes
self.component_name = component_name
self.logger = get_logger(self.component_name)
self.logger = logger
# Statistics
self.stats = ProcessingStats(active_timeframes=len(timeframes))
self.logger.info(f"Initialized batch candle processor for {symbol} on {exchange}")
if self.logger:
self.logger.info(f"{self.component_name}: Initialized batch candle processor for {symbol} on {exchange}")
def process_trades_to_candles(self, trades: Iterator[StandardizedTrade]) -> List[OHLCVCandle]:
"""
@@ -469,11 +477,13 @@ class BatchCandleProcessor:
if all_candles:
self.stats.last_candle_time = max(candle.end_time for candle in all_candles)
self.logger.info(f"Batch processed {self.stats.trades_processed} trades to {len(all_candles)} candles")
if self.logger:
self.logger.info(f"{self.component_name}: Batch processed {self.stats.trades_processed} trades to {len(all_candles)} candles")
return all_candles
except Exception as e:
self.logger.error(f"Error in batch processing trades to candles: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error in batch processing trades to candles: {e}")
self.stats.errors_count += 1
return []

55
data/common/transformation.py
View File

@@ -12,7 +12,6 @@ from abc import ABC, abstractmethod
from .data_types import StandardizedTrade, OHLCVCandle, DataValidationResult
from .aggregation import BatchCandleProcessor
from utils.logger import get_logger
class BaseDataTransformer(ABC):
@@ -25,7 +24,8 @@ class BaseDataTransformer(ABC):
def __init__(self,
exchange_name: str,
component_name: str = "base_data_transformer"):
component_name: str = "base_data_transformer",
logger = None):
"""
Initialize base data transformer.
@@ -35,9 +35,10 @@ class BaseDataTransformer(ABC):
"""
self.exchange_name = exchange_name
self.component_name = component_name
self.logger = get_logger(self.component_name)
self.logger = logger
self.logger.info(f"Initialized base data transformer for {exchange_name}")
if self.logger:
self.logger.info(f"{self.component_name}: Initialized base data transformer for {exchange_name}")
# Abstract methods that must be implemented by subclasses
@@ -87,7 +88,8 @@ class BaseDataTransformer(ABC):
return dt
except Exception as e:
self.logger.error(f"Error converting timestamp {timestamp}: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error converting timestamp {timestamp}: {e}")
# Return current time as fallback
return datetime.now(timezone.utc)
@@ -107,7 +109,8 @@ class BaseDataTransformer(ABC):
return None
return Decimal(str(value))
except Exception as e:
self.logger.warning(f"Failed to convert {field_name} '{value}' to Decimal: {e}")
if self.logger:
self.logger.warning(f"{self.component_name}: Failed to convert {field_name} '{value}' to Decimal: {e}")
return None
def normalize_trade_side(self, side: str) -> str:
@@ -125,10 +128,11 @@ class BaseDataTransformer(ABC):
# Handle common variations
if normalized in ['buy', 'bid', 'b', '1']:
return 'buy'
elif normalized in ['sell', 'ask', 's', '0']:
elif normalized in ['sell', 'ask', 's', '0']:
return 'sell'
else:
self.logger.warning(f"Unknown trade side: {side}, defaulting to 'buy'")
if self.logger:
self.logger.warning(f"{self.component_name}: Unknown trade side: {side}, defaulting to 'buy'")
return 'buy'
def validate_symbol_format(self, symbol: str) -> str:
@@ -165,7 +169,8 @@ class BaseDataTransformer(ABC):
Returns:
StandardizedTrade or None if transformation failed
"""
self.logger.warning("transform_database_record not implemented for this exchange")
if self.logger:
self.logger.warning(f"{self.component_name}: transform_database_record not implemented for this exchange")
return None
def get_transformer_info(self) -> Dict[str, Any]:
@@ -201,7 +206,8 @@ class UnifiedDataTransformer:
def __init__(self,
exchange_transformer: BaseDataTransformer,
component_name: str = "unified_data_transformer"):
component_name: str = "unified_data_transformer",
logger = None):
"""
Initialize unified data transformer.
@@ -211,9 +217,10 @@ class UnifiedDataTransformer:
"""
self.exchange_transformer = exchange_transformer
self.component_name = component_name
self.logger = get_logger(self.component_name)
self.logger = logger
self.logger.info(f"Initialized unified data transformer with {exchange_transformer.exchange_name} transformer")
if self.logger:
self.logger.info(f"{self.component_name}: Initialized unified data transformer with {exchange_transformer.exchange_name} transformer")
def transform_trade_data(self, raw_data: Dict[str, Any], symbol: str) -> Optional[StandardizedTrade]:
"""
@@ -229,7 +236,8 @@ class UnifiedDataTransformer:
try:
return self.exchange_transformer.transform_trade_data(raw_data, symbol)
except Exception as e:
self.logger.error(f"Error in trade transformation: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error in trade transformation: {e}")
return None
def transform_orderbook_data(self, raw_data: Dict[str, Any], symbol: str) -> Optional[Dict[str, Any]]:
@@ -246,7 +254,8 @@ class UnifiedDataTransformer:
try:
return self.exchange_transformer.transform_orderbook_data(raw_data, symbol)
except Exception as e:
self.logger.error(f"Error in orderbook transformation: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error in orderbook transformation: {e}")
return None
def transform_ticker_data(self, raw_data: Dict[str, Any], symbol: str) -> Optional[Dict[str, Any]]:
@@ -263,7 +272,8 @@ class UnifiedDataTransformer:
try:
return self.exchange_transformer.transform_ticker_data(raw_data, symbol)
except Exception as e:
self.logger.error(f"Error in ticker transformation: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error in ticker transformation: {e}")
return None
def process_trades_to_candles(self,
@@ -296,11 +306,13 @@ class UnifiedDataTransformer:
candles = processor.process_trades_to_candles(trades)
self.logger.info(f"Processed {processor.get_stats()['trades_processed']} trades to {len(candles)} candles")
if self.logger:
self.logger.info(f"{self.component_name}: Processed {processor.get_stats()['trades_processed']} trades to {len(candles)} candles")
return candles
except Exception as e:
self.logger.error(f"Error processing trades to candles: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error processing trades to candles: {e}")
return []
def batch_transform_trades(self,
@@ -327,10 +339,12 @@ class UnifiedDataTransformer:
else:
errors += 1
except Exception as e:
self.logger.error(f"Error transforming trade: {e}")
if self.logger:
self.logger.error(f"{self.component_name}: Error transforming trade: {e}")
errors += 1
self.logger.info(f"Batch transformed {len(transformed_trades)} trades successfully, {errors} errors")
if self.logger:
self.logger.info(f"{self.component_name}: Batch transformed {len(transformed_trades)} trades successfully, {errors} errors")
return transformed_trades
def get_transformer_info(self) -> Dict[str, Any]:
@@ -457,8 +471,7 @@ def batch_create_standardized_trades(raw_trades: List[Dict[str, Any]],
trades.append(trade)
except Exception as e:
# Log error but continue processing
logger = get_logger("batch_transform")
logger.warning(f"Failed to transform trade: {e}")
print(f"Failed to transform trade: {e}")
return trades

10
data/common/validation.py
View File

@@ -12,7 +12,6 @@ from typing import Dict, List, Optional, Any, Union, Pattern
from abc import ABC, abstractmethod
from .data_types import DataValidationResult, StandardizedTrade, TradeSide
from utils.logger import get_logger
class ValidationResult:
@@ -35,17 +34,19 @@ class BaseDataValidator(ABC):
def __init__(self,
exchange_name: str,
component_name: str = "base_data_validator"):
component_name: str = "base_data_validator",
logger = None):
"""
Initialize base data validator.
Args:
exchange_name: Name of the exchange (e.g., 'okx', 'binance')
component_name: Name for logging
logger: Logger instance. If None, no logging will be performed.
"""
self.exchange_name = exchange_name
self.component_name = component_name
self.logger = get_logger(self.component_name)
self.logger = logger
# Common validation patterns
self._numeric_pattern = re.compile(r'^-?\d*\.?\d+$')
@@ -64,7 +65,8 @@ class BaseDataValidator(ABC):
self._min_timestamp = 1000000000000 # 2001-09-09 (reasonable minimum)
self._max_timestamp = 9999999999999 # 2286-11-20 (reasonable maximum)
self.logger.debug(f"Initialized base data validator for {exchange_name}")
if self.logger:
self.logger.debug(f"{self.component_name}: Initialized {exchange_name} data validator")
# Abstract methods that must be implemented by subclasses