2890ba2efa
- Introduced `service_config.py` to manage configuration loading, validation, and schema management, enhancing modularity and security. - Created a `ServiceConfig` class for handling configuration with robust error handling and default values. - Refactored `DataCollectionService` to utilize the new `ServiceConfig`, streamlining configuration management and improving readability. - Added a `CollectorFactory` to encapsulate collector creation logic, promoting separation of concerns. - Updated `CollectorManager` and related components to align with the new architecture, ensuring better maintainability. - Enhanced logging practices across the service for improved monitoring and debugging. These changes significantly improve the architecture and maintainability of the data collection service, aligning with project standards for modularity and performance.
330 lines
12 KiB
Python
330 lines
12 KiB
Python
"""
|
|
Service Configuration Manager for data collection service.
|
|
|
|
This module handles configuration loading, validation, schema management,
|
|
and default configuration creation with security enhancements.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import stat
|
|
from pathlib import Path
|
|
from typing import Dict, Any, Optional, List
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass
|
|
class ServiceConfigSchema:
|
|
"""Schema definition for service configuration."""
|
|
exchange: str = "okx"
|
|
connection: Dict[str, Any] = None
|
|
data_collection: Dict[str, Any] = None
|
|
trading_pairs: List[Dict[str, Any]] = None
|
|
logging: Dict[str, Any] = None
|
|
database: Dict[str, Any] = None
|
|
|
|
|
|
class ServiceConfig:
|
|
"""Manages service configuration with validation and security."""
|
|
|
|
def __init__(self, config_path: str = "config/data_collection.json", logger=None):
|
|
"""
|
|
Initialize the service configuration manager.
|
|
|
|
Args:
|
|
config_path: Path to the configuration file
|
|
logger: Logger instance for logging operations
|
|
"""
|
|
self.config_path = config_path
|
|
self.logger = logger
|
|
self._config: Optional[Dict[str, Any]] = None
|
|
|
|
def load_config(self) -> Dict[str, Any]:
|
|
"""
|
|
Load and validate service configuration from JSON file.
|
|
|
|
Returns:
|
|
Dictionary containing the configuration
|
|
|
|
Raises:
|
|
Exception: If configuration loading or validation fails
|
|
"""
|
|
try:
|
|
config_file = Path(self.config_path)
|
|
|
|
# Create default config if it doesn't exist
|
|
if not config_file.exists():
|
|
self._create_default_config(config_file)
|
|
|
|
# Validate file permissions for security
|
|
self._validate_file_permissions(config_file)
|
|
|
|
# Load configuration
|
|
with open(config_file, 'r') as f:
|
|
config = json.load(f)
|
|
|
|
# Validate configuration schema
|
|
validated_config = self._validate_config_schema(config)
|
|
|
|
self._config = validated_config
|
|
|
|
if self.logger:
|
|
self.logger.info(f"✅ Configuration loaded from {self.config_path}")
|
|
|
|
return validated_config
|
|
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.error(f"❌ Failed to load configuration: {e}", exc_info=True)
|
|
raise
|
|
|
|
def _validate_file_permissions(self, config_file: Path) -> None:
|
|
"""
|
|
Validate configuration file permissions for security.
|
|
|
|
Args:
|
|
config_file: Path to the configuration file
|
|
|
|
Raises:
|
|
PermissionError: If file permissions are too permissive
|
|
"""
|
|
try:
|
|
file_stat = config_file.stat()
|
|
file_mode = file_stat.st_mode
|
|
|
|
# Check if file is readable by others (security risk)
|
|
if file_mode & stat.S_IROTH:
|
|
if self.logger:
|
|
self.logger.warning(f"⚠️ Configuration file {config_file} is readable by others")
|
|
|
|
# Check if file is writable by others (security risk)
|
|
if file_mode & stat.S_IWOTH:
|
|
if self.logger:
|
|
self.logger.warning(f"⚠️ Configuration file {config_file} is writable by others")
|
|
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.warning(f"⚠️ Could not validate file permissions: {e}")
|
|
|
|
def _validate_config_schema(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Validate configuration against schema and apply defaults.
|
|
|
|
Args:
|
|
config: Raw configuration dictionary
|
|
|
|
Returns:
|
|
Validated configuration with defaults applied
|
|
"""
|
|
validated = config.copy()
|
|
|
|
# Validate required fields
|
|
required_fields = ['exchange', 'trading_pairs']
|
|
for field in required_fields:
|
|
if field not in validated:
|
|
raise ValueError(f"Missing required configuration field: {field}")
|
|
|
|
# Apply defaults for optional sections
|
|
if 'connection' not in validated:
|
|
validated['connection'] = self._get_default_connection_config()
|
|
|
|
if 'data_collection' not in validated:
|
|
validated['data_collection'] = self._get_default_data_collection_config()
|
|
|
|
if 'logging' not in validated:
|
|
validated['logging'] = self._get_default_logging_config()
|
|
|
|
if 'database' not in validated:
|
|
validated['database'] = self._get_default_database_config()
|
|
|
|
# Validate trading pairs
|
|
self._validate_trading_pairs(validated['trading_pairs'])
|
|
|
|
return validated
|
|
|
|
def _validate_trading_pairs(self, trading_pairs: List[Dict[str, Any]]) -> None:
|
|
"""
|
|
Validate trading pairs configuration.
|
|
|
|
Args:
|
|
trading_pairs: List of trading pair configurations
|
|
|
|
Raises:
|
|
ValueError: If trading pairs configuration is invalid
|
|
"""
|
|
if not trading_pairs:
|
|
raise ValueError("At least one trading pair must be configured")
|
|
|
|
for i, pair in enumerate(trading_pairs):
|
|
if 'symbol' not in pair:
|
|
raise ValueError(f"Trading pair {i} missing required 'symbol' field")
|
|
|
|
symbol = pair['symbol']
|
|
if not isinstance(symbol, str) or '-' not in symbol:
|
|
raise ValueError(f"Invalid symbol format: {symbol}. Expected format: 'BASE-QUOTE'")
|
|
|
|
# Validate data types
|
|
data_types = pair.get('data_types', ['trade'])
|
|
valid_data_types = ['trade', 'orderbook', 'ticker', 'candle']
|
|
for dt in data_types:
|
|
if dt not in valid_data_types:
|
|
raise ValueError(f"Invalid data type '{dt}' for {symbol}. Valid types: {valid_data_types}")
|
|
|
|
def _create_default_config(self, config_file: Path) -> None:
|
|
"""
|
|
Create a default configuration file.
|
|
|
|
Args:
|
|
config_file: Path where the configuration file should be created
|
|
"""
|
|
default_config = {
|
|
"exchange": "okx",
|
|
"connection": self._get_default_connection_config(),
|
|
"data_collection": self._get_default_data_collection_config(),
|
|
"trading_pairs": self._get_default_trading_pairs(),
|
|
"logging": self._get_default_logging_config(),
|
|
"database": self._get_default_database_config()
|
|
}
|
|
|
|
# Ensure directory exists
|
|
config_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Write configuration file
|
|
with open(config_file, 'w') as f:
|
|
json.dump(default_config, f, indent=2)
|
|
|
|
# Set secure file permissions (owner read/write only)
|
|
try:
|
|
os.chmod(config_file, stat.S_IRUSR | stat.S_IWUSR)
|
|
except Exception as e:
|
|
if self.logger:
|
|
self.logger.warning(f"⚠️ Could not set secure file permissions: {e}")
|
|
|
|
if self.logger:
|
|
self.logger.info(f"📄 Created default configuration: {config_file}")
|
|
|
|
def _get_default_connection_config(self) -> Dict[str, Any]:
|
|
"""Get default connection configuration."""
|
|
return {
|
|
"public_ws_url": "wss://ws.okx.com:8443/ws/v5/public",
|
|
"private_ws_url": "wss://ws.okx.com:8443/ws/v5/private",
|
|
"ping_interval": 25.0,
|
|
"pong_timeout": 10.0,
|
|
"max_reconnect_attempts": 5,
|
|
"reconnect_delay": 5.0
|
|
}
|
|
|
|
def _get_default_data_collection_config(self) -> Dict[str, Any]:
|
|
"""Get default data collection configuration."""
|
|
return {
|
|
"store_raw_data": True,
|
|
"health_check_interval": 120.0,
|
|
"auto_restart": True,
|
|
"buffer_size": 1000
|
|
}
|
|
|
|
def _get_default_trading_pairs(self) -> List[Dict[str, Any]]:
|
|
"""Get default trading pairs configuration."""
|
|
return [
|
|
{
|
|
"symbol": "BTC-USDT",
|
|
"enabled": True,
|
|
"data_types": ["trade", "orderbook"],
|
|
"timeframes": ["1m", "5m", "15m", "1h"],
|
|
"channels": {
|
|
"trades": "trades",
|
|
"orderbook": "books5",
|
|
"ticker": "tickers"
|
|
}
|
|
},
|
|
{
|
|
"symbol": "ETH-USDT",
|
|
"enabled": True,
|
|
"data_types": ["trade", "orderbook"],
|
|
"timeframes": ["1m", "5m", "15m", "1h"],
|
|
"channels": {
|
|
"trades": "trades",
|
|
"orderbook": "books5",
|
|
"ticker": "tickers"
|
|
}
|
|
}
|
|
]
|
|
|
|
def _get_default_logging_config(self) -> Dict[str, Any]:
|
|
"""Get default logging configuration."""
|
|
return {
|
|
"component_name_template": "okx_collector_{symbol}",
|
|
"log_level": "INFO",
|
|
"verbose": False
|
|
}
|
|
|
|
def _get_default_database_config(self) -> Dict[str, Any]:
|
|
"""Get default database configuration."""
|
|
return {
|
|
"store_processed_data": True,
|
|
"store_raw_data": True,
|
|
"force_update_candles": False,
|
|
"batch_size": 100,
|
|
"flush_interval": 5.0
|
|
}
|
|
|
|
def get_config(self) -> Dict[str, Any]:
|
|
"""
|
|
Get the current configuration.
|
|
|
|
Returns:
|
|
Current configuration dictionary
|
|
|
|
Raises:
|
|
RuntimeError: If configuration has not been loaded
|
|
"""
|
|
if self._config is None:
|
|
raise RuntimeError("Configuration has not been loaded. Call load_config() first.")
|
|
return self._config.copy()
|
|
|
|
def get_exchange_config(self) -> Dict[str, Any]:
|
|
"""Get exchange-specific configuration."""
|
|
config = self.get_config()
|
|
return {
|
|
'exchange': config['exchange'],
|
|
'connection': config['connection']
|
|
}
|
|
|
|
def get_enabled_trading_pairs(self) -> List[Dict[str, Any]]:
|
|
"""Get list of enabled trading pairs."""
|
|
config = self.get_config()
|
|
trading_pairs = config.get('trading_pairs', [])
|
|
return [pair for pair in trading_pairs if pair.get('enabled', True)]
|
|
|
|
def get_data_collection_config(self) -> Dict[str, Any]:
|
|
"""Get data collection configuration."""
|
|
config = self.get_config()
|
|
return config.get('data_collection', {})
|
|
|
|
def update_config(self, updates: Dict[str, Any]) -> None:
|
|
"""
|
|
Update configuration with new values.
|
|
|
|
Args:
|
|
updates: Dictionary of configuration updates
|
|
"""
|
|
if self._config is None:
|
|
raise RuntimeError("Configuration has not been loaded. Call load_config() first.")
|
|
|
|
self._config.update(updates)
|
|
|
|
# Optionally save to file
|
|
if self.logger:
|
|
self.logger.info("Configuration updated in memory")
|
|
|
|
def save_config(self) -> None:
|
|
"""Save current configuration to file."""
|
|
if self._config is None:
|
|
raise RuntimeError("Configuration has not been loaded. Call load_config() first.")
|
|
|
|
config_file = Path(self.config_path)
|
|
with open(config_file, 'w') as f:
|
|
json.dump(self._config, f, indent=2)
|
|
|
|
if self.logger:
|
|
self.logger.info(f"Configuration saved to {self.config_path}") |