vasily/TCPDashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance error handling and security measures in data collection services

Browse Source 
- Implemented `_sanitize_error` method in `DataCollectionService` and `CollectorManager` to prevent leaking internal error details.
- Improved error handling across various methods by catching specific exceptions and logging sanitized messages with `exc_info=True`.
- Added file permission validation in `ServiceConfig` to ensure secure configuration file handling, including detailed logging for permission issues.
- Refactored logging practices to enhance clarity and maintainability, ensuring consistent error reporting.

These changes significantly bolster the security and robustness of the data collection services, aligning with project standards for error handling and maintainability.
This commit is contained in:
Vasily.onl
2025-06-10 13:12:13 +08:00
parent 6ab40414e7
commit c28e4a9aaf
5 changed files with 312 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
data/collection_service.py
View File

@@ -71,13 +71,57 @@ class DataCollectionService:
self.logger.info("🚀 Data Collection Service initialized")
self.logger.info(f"📁 Configuration: {config_path}")
def _sanitize_error(self, message: str) -> str:
"""
Sanitize error message to prevent leaking internal details.
Args:
message: Original error message
Returns:
Sanitized error message
"""
# Remove sensitive patterns that might leak internal information
sensitive_patterns = [
'password=',
'token=',
'key=',
'secret=',
'auth=',
'api_key=',
'api_secret=',
'access_token=',
'refresh_token='
]
sanitized = message
for pattern in sensitive_patterns:
if pattern.lower() in sanitized.lower():
# Replace the value part after = with [REDACTED]
parts = sanitized.split(pattern)
if len(parts) > 1:
# Find the end of the value (space, comma, or end of string)
value_part = parts[1]
end_chars = [' ', ',', ')', ']', '}', '\n', '\t']
end_idx = len(value_part)
for char in end_chars:
char_idx = value_part.find(char)
if char_idx != -1 and char_idx < end_idx:
end_idx = char_idx
# Replace the value with [REDACTED]
sanitized = parts[0] + pattern + '[REDACTED]' + value_part[end_idx:]
return sanitized
async def initialize_collectors(self) -> bool:
"""Initialize all data collectors based on configuration."""
try:
collectors = await self.collector_factory.create_collectors_from_config(self.config)
if not collectors:
self.logger.error("❌ No collectors were successfully created")
self.logger.error("❌ No collectors were successfully created", exc_info=True)
return False
for collector in collectors:
@@ -88,8 +132,22 @@ class DataCollectionService:
self.logger.info(f"✅ Successfully initialized {len(collectors)} data collectors")
return True
except (KeyError, AttributeError, TypeError) as e:
# Handle configuration and data structure errors
sanitized_message = self._sanitize_error(f"❌ Configuration error initializing collectors: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
except (ConnectionError, OSError, IOError) as e:
# Handle connection and I/O related errors
sanitized_message = self._sanitize_error(f"❌ Connection/IO error initializing collectors: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
except Exception as e:
self.logger.error(f"❌ Failed to initialize collectors: {e}", exc_info=True)
# Catch any other unexpected errors
sanitized_message = self._sanitize_error(f"❌ Unexpected error initializing collectors: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
@@ -117,12 +175,26 @@ class DataCollectionService:
self.logger.info(f"📈 Active collectors: {self.stats['collectors_running']}")
return True
else:
self.logger.error("❌ Failed to start data collectors")
self.logger.error("Failed to start data collectors", exc_info=True)
self.stats['errors_count'] += 1
return False
except (ConnectionError, OSError, IOError) as e:
# Handle database and connection errors
sanitized_message = self._sanitize_error(f"Database/Connection error starting service: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
except (AttributeError, TypeError, ValueError) as e:
# Handle configuration and data validation errors
sanitized_message = self._sanitize_error(f"❌ Configuration error starting service: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
except Exception as e:
self.logger.error(f"❌ Failed to start service: {e}", exc_info=True)
# Catch any other unexpected errors
sanitized_message = self._sanitize_error(f"❌ Unexpected error starting service: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
@@ -144,8 +216,19 @@ class DataCollectionService:
self.logger.info("✅ Data Collection Service stopped gracefully")
self.logger.info(f"📊 Total uptime: {self.stats['total_uptime_seconds']:.1f} seconds")
except (asyncio.CancelledError, KeyboardInterrupt):
# Handle graceful shutdown scenarios
self.logger.warning("Service shutdown was interrupted")
self.stats['errors_count'] += 1
except (ConnectionError, OSError, IOError) as e:
# Handle connection and I/O related errors during shutdown
sanitized_message = self._sanitize_error(f"Connection/IO error during service shutdown: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
except Exception as e:
self.logger.error(f"❌ Error during service shutdown: {e}", exc_info=True)
# Catch any other unexpected errors during shutdown
sanitized_message = self._sanitize_error(f"Unexpected error during service shutdown: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
def get_status(self) -> Dict[str, Any]:
@@ -234,8 +317,20 @@ class DataCollectionService:
return True
except (asyncio.CancelledError, KeyboardInterrupt):
# Handle graceful shutdown scenarios
self.logger.info("Service run was cancelled gracefully")
return True
except (asyncio.TimeoutError, ConnectionError, OSError, IOError) as e:
# Handle timeout, connection and I/O related errors
sanitized_message = self._sanitize_error(f"Connection/Timeout error during service run: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
except Exception as e:
self.logger.error(f"❌ Service error: {e}", exc_info=True)
# Catch any other unexpected errors
sanitized_message = self._sanitize_error(f"Unexpected service error: {e}")
self.logger.error(sanitized_message, exc_info=True)
self.stats['errors_count'] += 1
return False
finally:

8
data/collector_factory.py
View File

@@ -71,7 +71,7 @@ class CollectorFactory:
return collector
else:
if self.logger:
self.logger.error(f"Failed to create collector for {symbol}")
self.logger.error(f"Failed to create collector for {symbol}", exc_info=True)
return None
except Exception as e:
@@ -94,13 +94,13 @@ class CollectorFactory:
# Validate required fields
if 'symbol' not in pair_config:
if self.logger:
self.logger.error("Trading pair missing required 'symbol' field")
self.logger.error("Trading pair missing required 'symbol' field", exc_info=True)
return None
symbol = pair_config['symbol']
if not isinstance(symbol, str) or '-' not in symbol:
if self.logger:
self.logger.error(f"Invalid symbol format: {symbol}. Expected format: 'BASE-QUOTE'")
self.logger.error(f"Invalid symbol format: {symbol}. Expected format: 'BASE-QUOTE'", exc_info=True)
return None
# Apply defaults and validate data types
@@ -239,7 +239,7 @@ class CollectorFactory:
else:
symbol = pair_config.get('symbol', 'unknown')
if self.logger:
self.logger.error(f"Failed to create collector for {symbol}")
self.logger.error(f"Failed to create collector for {symbol}", exc_info=True)
if self.logger:
self.logger.info(f"Successfully created {len(collectors)} collectors")

51
data/collector_manager.py
View File

@@ -56,6 +56,19 @@ class CollectorManager:
if self.logger_manager.is_debug_enabled():
self.logger_manager.log_info(f"Initialized collector manager: {manager_name}")
def _sanitize_error(self, message: str) -> str:
"""
Sanitize error message to prevent leaking internal details.
Args:
message: Original error message
Returns:
Sanitized error message
"""
# Delegate to the logger manager's sanitization method
return self.logger_manager._sanitize_error(message)
def add_collector(self, collector: BaseDataCollector, config: Optional[CollectorConfig] = None) -> None:
"""Add a collector to be managed."""
self.lifecycle_manager.add_collector(collector, config)
@@ -107,9 +120,28 @@ class CollectorManager:
self.logger_manager.log_info(f"Collector manager started - Managing {enabled_count} collectors")
return True
except Exception as e:
except (asyncio.CancelledError, KeyboardInterrupt):
# Handle graceful shutdown scenarios
self.status = ManagerStatus.ERROR
self.logger_manager.log_error(f"Failed to start collector manager: {e}", exc_info=True)
self.logger_manager.log_warning("Collector manager startup was cancelled")
return False
except (ConnectionError, OSError, IOError) as e:
# Handle connection and I/O related errors
self.status = ManagerStatus.ERROR
sanitized_message = self._sanitize_error(f"Connection/IO error starting collector manager: {e}")
self.logger_manager.log_error(sanitized_message, exc_info=True)
return False
except (AttributeError, TypeError, ValueError) as e:
# Handle configuration and data validation errors
self.status = ManagerStatus.ERROR
sanitized_message = self._sanitize_error(f"Configuration error starting collector manager: {e}")
self.logger_manager.log_error(sanitized_message, exc_info=True)
return False
except Exception as e:
# Catch any other unexpected errors
self.status = ManagerStatus.ERROR
sanitized_message = self._sanitize_error(f"Unexpected error starting collector manager: {e}")
self.logger_manager.log_error(sanitized_message, exc_info=True)
return False
async def stop(self) -> None:
@@ -144,9 +176,20 @@ class CollectorManager:
self.status = ManagerStatus.STOPPED
self.logger_manager.log_info("Collector manager stopped")
except Exception as e:
except (asyncio.CancelledError, KeyboardInterrupt):
# Handle graceful shutdown scenarios
self.status = ManagerStatus.ERROR
self.logger_manager.log_error(f"Error stopping collector manager: {e}", exc_info=True)
self.logger_manager.log_warning("Collector manager shutdown was interrupted")
except (ConnectionError, OSError, IOError) as e:
# Handle connection and I/O related errors during shutdown
self.status = ManagerStatus.ERROR
sanitized_message = self._sanitize_error(f"Connection/IO error stopping collector manager: {e}")
self.logger_manager.log_error(sanitized_message, exc_info=True)
except Exception as e:
# Catch any other unexpected errors during shutdown
self.status = ManagerStatus.ERROR
sanitized_message = self._sanitize_error(f"Unexpected error stopping collector manager: {e}")
self.logger_manager.log_error(sanitized_message, exc_info=True)
async def restart_collector(self, collector_name: str) -> bool:
"""Restart a specific collector."""