""" Service Configuration Manager for data collection service. This module handles configuration loading, validation, schema management, and default configuration creation with security enhancements. """ import json import os import stat from pathlib import Path from typing import Dict, Any, Optional, List from dataclasses import dataclass @dataclass class ServiceConfigSchema: """Schema definition for service configuration.""" exchange: str = "okx" connection: Dict[str, Any] = None data_collection: Dict[str, Any] = None trading_pairs: List[Dict[str, Any]] = None logging: Dict[str, Any] = None database: Dict[str, Any] = None class ServiceConfig: """Manages service configuration with validation and security.""" def __init__(self, config_path: str = "config/data_collection.json", logger=None): """ Initialize the service configuration manager. Args: config_path: Path to the configuration file logger: Logger instance for logging operations """ self.config_path = config_path self.logger = logger self._config: Optional[Dict[str, Any]] = None def load_config(self) -> Dict[str, Any]: """ Load and validate service configuration from JSON file. Returns: Dictionary containing the configuration Raises: Exception: If configuration loading or validation fails """ try: config_file = Path(self.config_path) # Create default config if it doesn't exist if not config_file.exists(): self._create_default_config(config_file) # Validate file permissions for security self._validate_file_permissions(config_file) # Load configuration with open(config_file, 'r') as f: config = json.load(f) # Validate configuration schema validated_config = self._validate_config_schema(config) self._config = validated_config if self.logger: self.logger.info(f"✅ Configuration loaded from {self.config_path}") return validated_config except Exception as e: if self.logger: self.logger.error(f"❌ Failed to load configuration: {e}", exc_info=True) raise def _validate_file_permissions(self, config_file: Path) -> None: """ Validate configuration file permissions for security. Args: config_file: Path to the configuration file Raises: PermissionError: If file permissions are too permissive """ try: file_stat = config_file.stat() file_mode = file_stat.st_mode # Check if file is readable by others (security risk) if file_mode & stat.S_IROTH: if self.logger: self.logger.warning(f"⚠️ Configuration file {config_file} is readable by others") # Check if file is writable by others (security risk) if file_mode & stat.S_IWOTH: if self.logger: self.logger.warning(f"⚠️ Configuration file {config_file} is writable by others") except Exception as e: if self.logger: self.logger.warning(f"⚠️ Could not validate file permissions: {e}") def _validate_config_schema(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Validate configuration against schema and apply defaults. Args: config: Raw configuration dictionary Returns: Validated configuration with defaults applied """ validated = config.copy() # Validate required fields required_fields = ['exchange', 'trading_pairs'] for field in required_fields: if field not in validated: raise ValueError(f"Missing required configuration field: {field}") # Apply defaults for optional sections if 'connection' not in validated: validated['connection'] = self._get_default_connection_config() if 'data_collection' not in validated: validated['data_collection'] = self._get_default_data_collection_config() if 'logging' not in validated: validated['logging'] = self._get_default_logging_config() if 'database' not in validated: validated['database'] = self._get_default_database_config() # Validate trading pairs self._validate_trading_pairs(validated['trading_pairs']) return validated def _validate_trading_pairs(self, trading_pairs: List[Dict[str, Any]]) -> None: """ Validate trading pairs configuration. Args: trading_pairs: List of trading pair configurations Raises: ValueError: If trading pairs configuration is invalid """ if not trading_pairs: raise ValueError("At least one trading pair must be configured") for i, pair in enumerate(trading_pairs): if 'symbol' not in pair: raise ValueError(f"Trading pair {i} missing required 'symbol' field") symbol = pair['symbol'] if not isinstance(symbol, str) or '-' not in symbol: raise ValueError(f"Invalid symbol format: {symbol}. Expected format: 'BASE-QUOTE'") # Validate data types data_types = pair.get('data_types', ['trade']) valid_data_types = ['trade', 'orderbook', 'ticker', 'candle'] for dt in data_types: if dt not in valid_data_types: raise ValueError(f"Invalid data type '{dt}' for {symbol}. Valid types: {valid_data_types}") def _create_default_config(self, config_file: Path) -> None: """ Create a default configuration file. Args: config_file: Path where the configuration file should be created """ default_config = { "exchange": "okx", "connection": self._get_default_connection_config(), "data_collection": self._get_default_data_collection_config(), "trading_pairs": self._get_default_trading_pairs(), "logging": self._get_default_logging_config(), "database": self._get_default_database_config() } # Ensure directory exists config_file.parent.mkdir(parents=True, exist_ok=True) # Write configuration file with open(config_file, 'w') as f: json.dump(default_config, f, indent=2) # Set secure file permissions (owner read/write only) try: os.chmod(config_file, stat.S_IRUSR | stat.S_IWUSR) except Exception as e: if self.logger: self.logger.warning(f"⚠️ Could not set secure file permissions: {e}") if self.logger: self.logger.info(f"📄 Created default configuration: {config_file}") def _get_default_connection_config(self) -> Dict[str, Any]: """Get default connection configuration.""" return { "public_ws_url": "wss://ws.okx.com:8443/ws/v5/public", "private_ws_url": "wss://ws.okx.com:8443/ws/v5/private", "ping_interval": 25.0, "pong_timeout": 10.0, "max_reconnect_attempts": 5, "reconnect_delay": 5.0 } def _get_default_data_collection_config(self) -> Dict[str, Any]: """Get default data collection configuration.""" return { "store_raw_data": True, "health_check_interval": 120.0, "auto_restart": True, "buffer_size": 1000 } def _get_default_trading_pairs(self) -> List[Dict[str, Any]]: """Get default trading pairs configuration.""" return [ { "symbol": "BTC-USDT", "enabled": True, "data_types": ["trade", "orderbook"], "timeframes": ["1m", "5m", "15m", "1h"], "channels": { "trades": "trades", "orderbook": "books5", "ticker": "tickers" } }, { "symbol": "ETH-USDT", "enabled": True, "data_types": ["trade", "orderbook"], "timeframes": ["1m", "5m", "15m", "1h"], "channels": { "trades": "trades", "orderbook": "books5", "ticker": "tickers" } } ] def _get_default_logging_config(self) -> Dict[str, Any]: """Get default logging configuration.""" return { "component_name_template": "okx_collector_{symbol}", "log_level": "INFO", "verbose": False } def _get_default_database_config(self) -> Dict[str, Any]: """Get default database configuration.""" return { "store_processed_data": True, "store_raw_data": True, "force_update_candles": False, "batch_size": 100, "flush_interval": 5.0 } def get_config(self) -> Dict[str, Any]: """ Get the current configuration. Returns: Current configuration dictionary Raises: RuntimeError: If configuration has not been loaded """ if self._config is None: raise RuntimeError("Configuration has not been loaded. Call load_config() first.") return self._config.copy() def get_exchange_config(self) -> Dict[str, Any]: """Get exchange-specific configuration.""" config = self.get_config() return { 'exchange': config['exchange'], 'connection': config['connection'] } def get_enabled_trading_pairs(self) -> List[Dict[str, Any]]: """Get list of enabled trading pairs.""" config = self.get_config() trading_pairs = config.get('trading_pairs', []) return [pair for pair in trading_pairs if pair.get('enabled', True)] def get_data_collection_config(self) -> Dict[str, Any]: """Get data collection configuration.""" config = self.get_config() return config.get('data_collection', {}) def update_config(self, updates: Dict[str, Any]) -> None: """ Update configuration with new values. Args: updates: Dictionary of configuration updates """ if self._config is None: raise RuntimeError("Configuration has not been loaded. Call load_config() first.") self._config.update(updates) # Optionally save to file if self.logger: self.logger.info("Configuration updated in memory") def save_config(self) -> None: """Save current configuration to file.""" if self._config is None: raise RuntimeError("Configuration has not been loaded. Call load_config() first.") config_file = Path(self.config_path) with open(config_file, 'w') as f: json.dump(self._config, f, indent=2) if self.logger: self.logger.info(f"Configuration saved to {self.config_path}")