vasily/TCPDashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove OKX configuration file and enhance data collector with timeframes support

Browse Source 
- Deleted the `okx_config.json` file as part of the configuration refactor.
- Updated `BaseDataCollector` to include an optional `timeframes` parameter for more flexible data collection.
- Modified `DataCollectionService` and `OKXCollector` to pass and utilize the new `timeframes` parameter.
- Enhanced `ExchangeCollectorConfig` to validate timeframes, ensuring they are provided and correctly formatted.
- Updated documentation to reflect the new configurable timeframes feature, improving clarity for users.

These changes streamline the configuration process and improve the flexibility of data collection, aligning with project standards for maintainability and usability.
This commit is contained in:
Ajasra
2025-06-07 15:46:24 +08:00
parent 24394d7b92
commit 90cb450640
8 changed files with 93 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
data/base_collector.py
View File

@@ -115,9 +115,11 @@ class BaseDataCollector(ABC):
exchange_name: str,
symbols: List[str],
data_types: Optional[List[DataType]] = None,
timeframes: Optional[List[str]] = None,
component_name: Optional[str] = None,
auto_restart: bool = True,
health_check_interval: float = 30.0,
logger = None,
log_errors_only: bool = False):
"""
@@ -127,6 +129,7 @@ class BaseDataCollector(ABC):
exchange_name: Name of the exchange (e.g., 'okx', 'binance')
symbols: List of trading symbols to collect data for
data_types: Types of data to collect (default: [DataType.CANDLE])
timeframes: List of timeframes to collect (e.g., ['1s', '1m', '5m'])
component_name: Name for logging (default: based on exchange_name)
auto_restart: Enable automatic restart on failures (default: True)
health_check_interval: Seconds between health checks (default: 30.0)
@@ -136,6 +139,7 @@ class BaseDataCollector(ABC):
self.exchange_name = exchange_name.lower()
self.symbols = set(symbols)
self.data_types = data_types or [DataType.CANDLE]
self.timeframes = timeframes or ['1m', '5m'] # Default timeframes if none provided
self.auto_restart = auto_restart
self.health_check_interval = health_check_interval
self.log_errors_only = log_errors_only
@@ -187,6 +191,7 @@ class BaseDataCollector(ABC):
self.component_name = component
if not self.log_errors_only:
self.logger.info(f"{self.component_name}: Initialized {self.exchange_name} data collector for symbols: {', '.join(symbols)}")
self.logger.info(f"{self.component_name}: Using timeframes: {', '.join(self.timeframes)}")
else:
self.component_name = component_name or f"{self.exchange_name}_collector"
@@ -581,6 +586,7 @@ class BaseDataCollector(ABC):
'should_be_running': self._should_be_running,
'symbols': list(self.symbols),
'data_types': [dt.value for dt in self.data_types],
'timeframes': self.timeframes,
'auto_restart': self.auto_restart,
'health': {
'time_since_heartbeat': time_since_heartbeat,
@@ -637,7 +643,9 @@ class BaseDataCollector(ABC):
'last_heartbeat': self._last_heartbeat.isoformat() if self._last_heartbeat else None,
'last_data_received': self._last_data_received.isoformat() if self._last_data_received else None,
'should_be_running': self._should_be_running,
'is_running': self._running
'is_running': self._running,
'timeframes': self.timeframes,
'data_types': [dt.value for dt in self.data_types]
}
def add_symbol(self, symbol: str) -> None:

4
data/collection_service.py
View File

@@ -228,6 +228,7 @@ class DataCollectionService:
exchange=exchange_name,
symbol=symbol,
data_types=data_types,
timeframes=timeframes, # Pass timeframes to config
auto_restart=data_collection_config.get('auto_restart', True),
health_check_interval=data_collection_config.get('health_check_interval', 120.0),
store_raw_data=data_collection_config.get('store_raw_data', True),
@@ -235,7 +236,8 @@ class DataCollectionService:
'component_name': f"{exchange_name}_collector_{symbol.replace('-', '_').lower()}",
'logger': self.logger,
'log_errors_only': True, # Clean logging - only errors and essential events
'force_update_candles': self.config.get('database', {}).get('force_update_candles', False)
'force_update_candles': self.config.get('database', {}).get('force_update_candles', False),
'timeframes': timeframes # Pass timeframes to collector
}
)

19
data/exchanges/factory.py
View File

@@ -7,10 +7,11 @@ from different exchanges based on configuration.
import importlib
from typing import Dict, List, Optional, Any, Type, Tuple
from dataclasses import dataclass
from dataclasses import dataclass, field
from utils.logger import get_logger
from ..base_collector import BaseDataCollector, DataType
from ..common import CandleProcessingConfig
from .registry import EXCHANGE_REGISTRY, get_supported_exchanges, get_exchange_info
from .exceptions import (
ExchangeError,
@@ -29,6 +30,7 @@ class ExchangeCollectorConfig:
exchange: str
symbol: str
data_types: List[DataType]
timeframes: List[str] = field(default_factory=lambda: ['1m', '5m']) # Default timeframes
auto_restart: bool = True
health_check_interval: float = 30.0
store_raw_data: bool = True
@@ -42,6 +44,8 @@ class ExchangeCollectorConfig:
raise InvalidConfigurationError("Symbol cannot be empty")
if not self.data_types:
raise InvalidConfigurationError("At least one data type must be specified")
if not self.timeframes:
raise InvalidConfigurationError("At least one timeframe must be specified")
logger.debug(f"Created collector config for {self.exchange} {self.symbol}")
@@ -92,12 +96,23 @@ class ExchangeFactory:
'data_types': config.data_types,
'auto_restart': config.auto_restart,
'health_check_interval': config.health_check_interval,
'store_raw_data': config.store_raw_data
'store_raw_data': config.store_raw_data,
'timeframes': config.timeframes # Pass timeframes to collector
}
# Add any custom parameters
if config.custom_params:
# If custom_params contains a candle_config key, use it, otherwise create one
if 'candle_config' not in config.custom_params:
config.custom_params['candle_config'] = CandleProcessingConfig(
timeframes=config.timeframes
)
collector_args.update(config.custom_params)
else:
# Create default candle config if no custom params
collector_args['candle_config'] = CandleProcessingConfig(
timeframes=config.timeframes
)
# Create and return the collector instance
logger.info(f"Successfully created collector for {exchange_name} {config.symbol}")

18
data/exchanges/okx/collector.py
View File

@@ -15,7 +15,9 @@ from ...base_collector import (
BaseDataCollector, DataType, CollectorStatus, MarketDataPoint,
OHLCVData, DataValidationError, ConnectionError
)
from ...common import StandardizedTrade, OHLCVCandle
from ...common import (
StandardizedTrade, OHLCVCandle, CandleProcessingConfig
)
from .websocket import (
OKXWebSocketClient, OKXSubscription, OKXChannelType,
ConnectionState, OKXWebSocketError
@@ -53,6 +55,8 @@ class OKXCollector(BaseDataCollector):
health_check_interval: float = 30.0,
store_raw_data: bool = True,
force_update_candles: bool = False,
timeframes: Optional[List[str]] = None,
candle_config: Optional[CandleProcessingConfig] = None,
logger = None,
log_errors_only: bool = False):
"""
@@ -66,6 +70,8 @@ class OKXCollector(BaseDataCollector):
health_check_interval: Seconds between health checks
store_raw_data: Whether to store raw data for debugging
force_update_candles: If True, update existing candles; if False, keep existing candles unchanged
timeframes: List of timeframes to collect (e.g., ['1s', '5s', '1m'])
candle_config: Optional CandleProcessingConfig instance (will create one if not provided)
logger: Logger instance for conditional logging (None for no logging)
log_errors_only: If True and logger provided, only log error-level messages
"""
@@ -82,6 +88,7 @@ class OKXCollector(BaseDataCollector):
exchange_name="okx",
symbols=[symbol],
data_types=data_types,
timeframes=timeframes, # Pass timeframes to base collector
component_name=component_name,
auto_restart=auto_restart,
health_check_interval=health_check_interval,
@@ -98,7 +105,12 @@ class OKXCollector(BaseDataCollector):
self._ws_client: Optional[OKXWebSocketClient] = None
# Data processor using new common framework
self._data_processor = OKXDataProcessor(symbol, component_name=f"{component_name}_processor", logger=logger)
self._data_processor = OKXDataProcessor(
symbol,
config=candle_config or CandleProcessingConfig(timeframes=self.timeframes), # Use provided config or create new one
component_name=f"{component_name}_processor",
logger=logger
)
# Add callbacks for processed data
self._data_processor.add_trade_callback(self._on_trade_processed)
@@ -122,6 +134,7 @@ class OKXCollector(BaseDataCollector):
if logger:
logger.info(f"{component_name}: Initialized OKX collector for {symbol} with data types: {[dt.value for dt in data_types]}")
logger.info(f"{component_name}: Using timeframes: {self.timeframes}")
logger.info(f"{component_name}: Using common data processing framework")
async def connect(self) -> bool:
@@ -511,6 +524,7 @@ class OKXCollector(BaseDataCollector):
"websocket_state": self._ws_client.connection_state.value if self._ws_client else "disconnected",
"store_raw_data": self.store_raw_data,
"force_update_candles": self.force_update_candles,
"timeframes": self.timeframes,
"processing_stats": {
"messages_received": self._message_count,
"trades_processed": self._processed_trades,