docs
This commit is contained in:
Vasily.onl
File diff suppressed because it is too large
Load Diff
@@ -27,6 +27,22 @@ The TCP Dashboard implements a sophisticated conditional logging system that all
|
||||
3. **Logger Inheritance**: Parent components pass their logger to child components
|
||||
4. **Hierarchical Structure**: Log files are organized by component hierarchy
|
||||
|
||||
### Component Hierarchy
|
||||
|
||||
```
|
||||
Top-level Application (individual logger)
|
||||
├── ProductionManager (individual logger)
|
||||
│ ├── DataSaver (receives logger from ProductionManager)
|
||||
│ ├── DataValidator (receives logger from ProductionManager)
|
||||
│ ├── DatabaseConnection (receives logger from ProductionManager)
|
||||
│ └── CollectorManager (individual logger)
|
||||
│ ├── OKX collector BTC-USD (individual logger)
|
||||
│ │ ├── DataAggregator (receives logger from OKX collector)
|
||||
│ │ ├── DataTransformer (receives logger from OKX collector)
|
||||
│ │ └── DataProcessor (receives logger from OKX collector)
|
||||
│ └── Another collector...
|
||||
```
|
||||
|
||||
### Usage Patterns
|
||||
|
||||
#### 1. No Logging
|
||||
@@ -134,24 +150,48 @@ class ComponentExample:
|
||||
self.logger = logger
|
||||
self.log_errors_only = log_errors_only
|
||||
|
||||
# Conditional logging helpers
|
||||
self._log_debug = self._create_conditional_logger('debug')
|
||||
self._log_info = self._create_conditional_logger('info')
|
||||
self._log_warning = self._create_conditional_logger('warning')
|
||||
self._log_error = self._create_conditional_logger('error')
|
||||
self._log_critical = self._create_conditional_logger('critical')
|
||||
def _log_debug(self, message: str) -> None:
|
||||
"""Log debug message if logger is available and not in errors-only mode."""
|
||||
if self.logger and not self.log_errors_only:
|
||||
self.logger.debug(message)
|
||||
|
||||
def _create_conditional_logger(self, level):
|
||||
"""Create conditional logging function based on configuration."""
|
||||
if not self.logger:
|
||||
return lambda msg: None # No-op if no logger
|
||||
def _log_info(self, message: str) -> None:
|
||||
"""Log info message if logger is available and not in errors-only mode."""
|
||||
if self.logger and not self.log_errors_only:
|
||||
self.logger.info(message)
|
||||
|
||||
def _log_warning(self, message: str) -> None:
|
||||
"""Log warning message if logger is available and not in errors-only mode."""
|
||||
if self.logger and not self.log_errors_only:
|
||||
self.logger.warning(message)
|
||||
|
||||
def _log_error(self, message: str, exc_info: bool = False) -> None:
|
||||
"""Log error message if logger is available (always logs errors)."""
|
||||
if self.logger:
|
||||
self.logger.error(message, exc_info=exc_info)
|
||||
|
||||
def _log_critical(self, message: str, exc_info: bool = False) -> None:
|
||||
"""Log critical message if logger is available (always logs critical)."""
|
||||
if self.logger:
|
||||
self.logger.critical(message, exc_info=exc_info)
|
||||
```
|
||||
|
||||
#### Child Component Pattern
|
||||
|
||||
Child components receive logger from parent:
|
||||
|
||||
```python
|
||||
class OKXCollector(BaseDataCollector):
|
||||
def __init__(self, symbol: str, logger=None, log_errors_only=False):
|
||||
super().__init__(..., logger=logger, log_errors_only=log_errors_only)
|
||||
|
||||
log_func = getattr(self.logger, level)
|
||||
|
||||
if level in ['debug', 'info', 'warning'] and self.log_errors_only:
|
||||
return lambda msg: None # Suppress non-error messages
|
||||
|
||||
return log_func # Normal logging
|
||||
# Pass logger to child components
|
||||
self._data_processor = OKXDataProcessor(
|
||||
symbol,
|
||||
logger=self.logger # Pass parent's logger
|
||||
)
|
||||
self._data_validator = DataValidator(logger=self.logger)
|
||||
self._data_transformer = DataTransformer(logger=self.logger)
|
||||
```
|
||||
|
||||
#### Supported Components
|
||||
@@ -178,179 +218,6 @@ The following components support conditional logging:
|
||||
- Parameters: `logger=None`
|
||||
- Data processing with conditional logging
|
||||
|
||||
### Best Practices for Conditional Logging
|
||||
|
||||
#### 1. Logger Inheritance
|
||||
```python
|
||||
# Parent component creates logger
|
||||
parent_logger = get_logger('parent_system')
|
||||
parent = ParentComponent(logger=parent_logger)
|
||||
|
||||
# Pass logger to children for consistent hierarchy
|
||||
child1 = ChildComponent(logger=parent_logger)
|
||||
child2 = ChildComponent(logger=parent_logger, log_errors_only=True)
|
||||
child3 = ChildComponent(logger=None) # No logging
|
||||
```
|
||||
|
||||
#### 2. Environment-Based Configuration
|
||||
```python
|
||||
import os
|
||||
from utils.logger import get_logger
|
||||
|
||||
def create_system_logger():
|
||||
"""Create logger based on environment."""
|
||||
env = os.getenv('ENVIRONMENT', 'development')
|
||||
|
||||
if env == 'production':
|
||||
return get_logger('production_system', log_level='INFO', verbose=False)
|
||||
elif env == 'testing':
|
||||
return None # No logging during tests
|
||||
else:
|
||||
return get_logger('dev_system', log_level='DEBUG', verbose=True)
|
||||
|
||||
# Use in components
|
||||
system_logger = create_system_logger()
|
||||
manager = CollectorManager(logger=system_logger)
|
||||
```
|
||||
|
||||
#### 3. Conditional Error-Only Mode
|
||||
```python
|
||||
def create_collector_with_logging_strategy(symbol, strategy='normal'):
|
||||
"""Create collector with different logging strategies."""
|
||||
base_logger = get_logger(f'collector_{symbol.lower().replace("-", "_")}')
|
||||
|
||||
if strategy == 'silent':
|
||||
return OKXCollector(symbol, logger=None)
|
||||
elif strategy == 'errors_only':
|
||||
return OKXCollector(symbol, logger=base_logger, log_errors_only=True)
|
||||
else:
|
||||
return OKXCollector(symbol, logger=base_logger)
|
||||
|
||||
# Usage
|
||||
btc_collector = create_collector_with_logging_strategy('BTC-USDT', 'normal')
|
||||
eth_collector = create_collector_with_logging_strategy('ETH-USDT', 'errors_only')
|
||||
ada_collector = create_collector_with_logging_strategy('ADA-USDT', 'silent')
|
||||
```
|
||||
|
||||
#### 4. Performance Optimization
|
||||
```python
|
||||
class OptimizedComponent:
|
||||
def __init__(self, logger=None, log_errors_only=False):
|
||||
self.logger = logger
|
||||
self.log_errors_only = log_errors_only
|
||||
|
||||
# Pre-compute logging capabilities for performance
|
||||
self.can_log_debug = logger and not log_errors_only
|
||||
self.can_log_info = logger and not log_errors_only
|
||||
self.can_log_warning = logger and not log_errors_only
|
||||
self.can_log_error = logger is not None
|
||||
self.can_log_critical = logger is not None
|
||||
|
||||
def process_data(self, data):
|
||||
if self.can_log_debug:
|
||||
self.logger.debug(f"Processing {len(data)} records")
|
||||
|
||||
# ... processing logic ...
|
||||
|
||||
if self.can_log_info:
|
||||
self.logger.info("Data processing completed")
|
||||
```
|
||||
|
||||
### Migration Guide
|
||||
|
||||
#### From Standard Logging
|
||||
```python
|
||||
# Old approach
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class OldComponent:
|
||||
def __init__(self):
|
||||
self.logger = logger
|
||||
|
||||
# New conditional approach
|
||||
from utils.logger import get_logger
|
||||
|
||||
class NewComponent:
|
||||
def __init__(self, logger=None, log_errors_only=False):
|
||||
self.logger = logger
|
||||
self.log_errors_only = log_errors_only
|
||||
|
||||
# Add conditional logging helpers
|
||||
self._setup_conditional_logging()
|
||||
```
|
||||
|
||||
#### Gradual Adoption
|
||||
1. **Phase 1**: Add optional logger parameters to new components
|
||||
2. **Phase 2**: Update existing components to support conditional logging
|
||||
3. **Phase 3**: Implement hierarchical logging structure
|
||||
4. **Phase 4**: Add error-only logging mode
|
||||
|
||||
### Testing Conditional Logging
|
||||
|
||||
#### Test Script Example
|
||||
```python
|
||||
# test_conditional_logging.py
|
||||
from utils.logger import get_logger
|
||||
from data.collector_manager import CollectorManager
|
||||
from data.exchanges.okx.collector import OKXCollector
|
||||
|
||||
def test_no_logging():
|
||||
"""Test components work without loggers."""
|
||||
manager = CollectorManager(logger=None)
|
||||
collector = OKXCollector("BTC-USDT", logger=None)
|
||||
print("✓ No logging test passed")
|
||||
|
||||
def test_with_logging():
|
||||
"""Test components work with loggers."""
|
||||
logger = get_logger('test_system')
|
||||
manager = CollectorManager(logger=logger)
|
||||
collector = OKXCollector("BTC-USDT", logger=logger)
|
||||
print("✓ With logging test passed")
|
||||
|
||||
def test_error_only():
|
||||
"""Test error-only logging mode."""
|
||||
logger = get_logger('test_errors')
|
||||
collector = OKXCollector("BTC-USDT", logger=logger, log_errors_only=True)
|
||||
print("✓ Error-only logging test passed")
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_no_logging()
|
||||
test_with_logging()
|
||||
test_error_only()
|
||||
print("✅ All conditional logging tests passed!")
|
||||
```
|
||||
|
||||
## Log Format
|
||||
|
||||
All log messages follow this unified format:
|
||||
```
|
||||
[YYYY-MM-DD HH:MM:SS - LEVEL - message]
|
||||
```
|
||||
|
||||
Example:
|
||||
```
|
||||
[2024-01-15 14:30:25 - INFO - Bot started successfully]
|
||||
[2024-01-15 14:30:26 - ERROR - Connection failed: timeout]
|
||||
```
|
||||
|
||||
## File Organization
|
||||
|
||||
Logs are organized in a hierarchical structure:
|
||||
```
|
||||
logs/
|
||||
├── app/
|
||||
│ ├── 2024-01-15.txt
|
||||
│ └── 2024-01-16.txt
|
||||
├── bot_manager/
|
||||
│ ├── 2024-01-15.txt
|
||||
│ └── 2024-01-16.txt
|
||||
├── data_collector/
|
||||
│ └── 2024-01-15.txt
|
||||
└── strategies/
|
||||
└── 2024-01-15.txt
|
||||
```
|
||||
|
||||
## Basic Usage
|
||||
|
||||
### Import and Initialize
|
||||
@@ -414,6 +281,38 @@ class BotManager:
|
||||
self.logger.info(f"Bot {bot_id} stopped")
|
||||
```
|
||||
|
||||
## Log Format
|
||||
|
||||
All log messages follow this unified format:
|
||||
```
|
||||
[YYYY-MM-DD HH:MM:SS - LEVEL - message]
|
||||
```
|
||||
|
||||
Example:
|
||||
```
|
||||
[2024-01-15 14:30:25 - INFO - Bot started successfully]
|
||||
[2024-01-15 14:30:26 - ERROR - Connection failed: timeout]
|
||||
```
|
||||
|
||||
## File Organization
|
||||
|
||||
Logs are organized in a hierarchical structure:
|
||||
```
|
||||
logs/
|
||||
├── tcp_dashboard/
|
||||
│ ├── 2024-01-15.txt
|
||||
│ └── 2024-01-16.txt
|
||||
├── production_manager/
|
||||
│ ├── 2024-01-15.txt
|
||||
│ └── 2024-01-16.txt
|
||||
├── collector_manager/
|
||||
│ └── 2024-01-15.txt
|
||||
├── okx_collector_btc_usdt/
|
||||
│ └── 2024-01-15.txt
|
||||
└── okx_collector_eth_usdt/
|
||||
└── 2024-01-15.txt
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Logger Parameters
|
||||
@@ -487,6 +386,84 @@ logger = get_logger('bot_manager', max_log_files=14)
|
||||
- Deletes older files automatically
|
||||
- Based on file modification time, not filename
|
||||
|
||||
## Best Practices for Conditional Logging
|
||||
|
||||
### 1. Logger Inheritance
|
||||
```python
|
||||
# Parent component creates logger
|
||||
parent_logger = get_logger('parent_system')
|
||||
parent = ParentComponent(logger=parent_logger)
|
||||
|
||||
# Pass logger to children for consistent hierarchy
|
||||
child1 = ChildComponent(logger=parent_logger)
|
||||
child2 = ChildComponent(logger=parent_logger, log_errors_only=True)
|
||||
child3 = ChildComponent(logger=None) # No logging
|
||||
```
|
||||
|
||||
### 2. Environment-Based Configuration
|
||||
```python
|
||||
import os
|
||||
from utils.logger import get_logger
|
||||
|
||||
def create_system_logger():
|
||||
"""Create logger based on environment."""
|
||||
env = os.getenv('ENVIRONMENT', 'development')
|
||||
|
||||
if env == 'production':
|
||||
return get_logger('production_system', log_level='INFO', verbose=False)
|
||||
elif env == 'testing':
|
||||
return None # No logging during tests
|
||||
else:
|
||||
return get_logger('dev_system', log_level='DEBUG', verbose=True)
|
||||
|
||||
# Use in components
|
||||
system_logger = create_system_logger()
|
||||
manager = CollectorManager(logger=system_logger)
|
||||
```
|
||||
|
||||
### 3. Conditional Error-Only Mode
|
||||
```python
|
||||
def create_collector_with_logging_strategy(symbol, strategy='normal'):
|
||||
"""Create collector with different logging strategies."""
|
||||
base_logger = get_logger(f'collector_{symbol.lower().replace("-", "_")}')
|
||||
|
||||
if strategy == 'silent':
|
||||
return OKXCollector(symbol, logger=None)
|
||||
elif strategy == 'errors_only':
|
||||
return OKXCollector(symbol, logger=base_logger, log_errors_only=True)
|
||||
else:
|
||||
return OKXCollector(symbol, logger=base_logger)
|
||||
|
||||
# Usage
|
||||
btc_collector = create_collector_with_logging_strategy('BTC-USDT', 'normal')
|
||||
eth_collector = create_collector_with_logging_strategy('ETH-USDT', 'errors_only')
|
||||
ada_collector = create_collector_with_logging_strategy('ADA-USDT', 'silent')
|
||||
```
|
||||
|
||||
### 4. Performance Optimization
|
||||
```python
|
||||
class OptimizedComponent:
|
||||
def __init__(self, logger=None, log_errors_only=False):
|
||||
self.logger = logger
|
||||
self.log_errors_only = log_errors_only
|
||||
|
||||
# Pre-compute logging capabilities for performance
|
||||
self.can_log_debug = logger and not log_errors_only
|
||||
self.can_log_info = logger and not log_errors_only
|
||||
self.can_log_warning = logger and not log_errors_only
|
||||
self.can_log_error = logger is not None
|
||||
self.can_log_critical = logger is not None
|
||||
|
||||
def process_data(self, data):
|
||||
if self.can_log_debug:
|
||||
self.logger.debug(f"Processing {len(data)} records")
|
||||
|
||||
# ... processing logic ...
|
||||
|
||||
if self.can_log_info:
|
||||
self.logger.info("Data processing completed")
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Manual Log Cleanup
|
||||
@@ -671,16 +648,37 @@ if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f"Data: {expensive_serialization(data)}")
|
||||
```
|
||||
|
||||
## Integration with Existing Code
|
||||
## Migration Guide
|
||||
|
||||
The logging system is designed to be gradually adopted:
|
||||
### Updating Existing Components
|
||||
|
||||
1. **Start with new modules**: Use the unified logger in new code
|
||||
2. **Replace existing logging**: Gradually migrate existing logging to the unified system
|
||||
3. **No breaking changes**: Existing code continues to work
|
||||
1. **Add logger parameter to constructor**:
|
||||
```python
|
||||
def __init__(self, ..., logger=None, log_errors_only=False):
|
||||
```
|
||||
|
||||
### Migration Example
|
||||
2. **Add conditional logging helpers**:
|
||||
```python
|
||||
def _log_debug(self, message: str) -> None:
|
||||
if self.logger and not self.log_errors_only:
|
||||
self.logger.debug(message)
|
||||
```
|
||||
|
||||
3. **Update all logging calls**:
|
||||
```python
|
||||
# Before
|
||||
self.logger.info("Message")
|
||||
|
||||
# After
|
||||
self._log_info("Message")
|
||||
```
|
||||
|
||||
4. **Pass logger to child components**:
|
||||
```python
|
||||
child = ChildComponent(logger=self.logger)
|
||||
```
|
||||
|
||||
### From Standard Logging
|
||||
```python
|
||||
# Old logging (if any existed)
|
||||
import logging
|
||||
@@ -692,13 +690,113 @@ from utils.logger import get_logger
|
||||
logger = get_logger('component_name', verbose=True)
|
||||
```
|
||||
|
||||
### Gradual Adoption
|
||||
1. **Phase 1**: Add optional logger parameters to new components
|
||||
2. **Phase 2**: Update existing components to support conditional logging
|
||||
3. **Phase 3**: Implement hierarchical logging structure
|
||||
4. **Phase 4**: Add error-only logging mode
|
||||
|
||||
## Testing
|
||||
|
||||
### Testing Conditional Logging
|
||||
|
||||
#### Test Script Example
|
||||
```python
|
||||
# test_conditional_logging.py
|
||||
from utils.logger import get_logger
|
||||
from data.collector_manager import CollectorManager
|
||||
from data.exchanges.okx.collector import OKXCollector
|
||||
|
||||
def test_no_logging():
|
||||
"""Test components work without loggers."""
|
||||
manager = CollectorManager(logger=None)
|
||||
collector = OKXCollector("BTC-USDT", logger=None)
|
||||
print("✓ No logging test passed")
|
||||
|
||||
def test_with_logging():
|
||||
"""Test components work with loggers."""
|
||||
logger = get_logger('test_system')
|
||||
manager = CollectorManager(logger=logger)
|
||||
collector = OKXCollector("BTC-USDT", logger=logger)
|
||||
print("✓ With logging test passed")
|
||||
|
||||
def test_error_only():
|
||||
"""Test error-only logging mode."""
|
||||
logger = get_logger('test_errors')
|
||||
collector = OKXCollector("BTC-USDT", logger=logger, log_errors_only=True)
|
||||
print("✓ Error-only logging test passed")
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_no_logging()
|
||||
test_with_logging()
|
||||
test_error_only()
|
||||
print("✅ All conditional logging tests passed!")
|
||||
```
|
||||
|
||||
### Testing Changes
|
||||
|
||||
```python
|
||||
# Test without logger
|
||||
component = MyComponent(logger=None)
|
||||
# Should work without errors, no logging
|
||||
|
||||
# Test with logger
|
||||
logger = get_logger('test_component')
|
||||
component = MyComponent(logger=logger)
|
||||
# Should log normally
|
||||
|
||||
# Test error-only mode
|
||||
component = MyComponent(logger=logger, log_errors_only=True)
|
||||
# Should only log errors
|
||||
```
|
||||
|
||||
### Basic System Test
|
||||
|
||||
Run a simple test to verify the logging system:
|
||||
```bash
|
||||
python -c "from utils.logger import get_logger; logger = get_logger('test', verbose=True); logger.info('Test message'); print('Check logs/test/ directory')"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Permission errors**: Ensure the application has write permissions to the project directory
|
||||
2. **Disk space**: Monitor disk usage and adjust log retention with `max_log_files`
|
||||
3. **Threading issues**: The logger is thread-safe, but check for application-level concurrency issues
|
||||
4. **Too many console messages**: Adjust `verbose` parameter or log levels
|
||||
|
||||
### Debug Mode
|
||||
|
||||
Enable debug logging to troubleshoot issues:
|
||||
```python
|
||||
logger = get_logger('component_name', 'DEBUG', verbose=True)
|
||||
```
|
||||
|
||||
### Console Output Issues
|
||||
|
||||
```python
|
||||
# Force console output regardless of environment
|
||||
logger = get_logger('component_name', verbose=True)
|
||||
|
||||
# Check environment variables
|
||||
import os
|
||||
print(f"VERBOSE_LOGGING: {os.getenv('VERBOSE_LOGGING')}")
|
||||
print(f"LOG_TO_CONSOLE: {os.getenv('LOG_TO_CONSOLE')}")
|
||||
```
|
||||
|
||||
### Fallback Logging
|
||||
|
||||
If file logging fails, the system automatically falls back to console logging with a warning message.
|
||||
|
||||
## Integration with Existing Code
|
||||
|
||||
The logging system is designed to be gradually adopted:
|
||||
|
||||
1. **Start with new modules**: Use the unified logger in new code
|
||||
2. **Replace existing logging**: Gradually migrate existing logging to the unified system
|
||||
3. **No breaking changes**: Existing code continues to work
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Automatic Cleanup Benefits
|
||||
@@ -735,49 +833,4 @@ find logs/ -name "*.txt" -size +10M
|
||||
find logs/ -name "*.txt" | cut -d'/' -f2 | sort | uniq -c
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Permission errors**: Ensure the application has write permissions to the project directory
|
||||
2. **Disk space**: Monitor disk usage and adjust log retention with `max_log_files`
|
||||
3. **Threading issues**: The logger is thread-safe, but check for application-level concurrency issues
|
||||
4. **Too many console messages**: Adjust `verbose` parameter or log levels
|
||||
|
||||
### Debug Mode
|
||||
|
||||
Enable debug logging to troubleshoot issues:
|
||||
```python
|
||||
logger = get_logger('component_name', 'DEBUG', verbose=True)
|
||||
```
|
||||
|
||||
### Console Output Issues
|
||||
|
||||
```python
|
||||
# Force console output regardless of environment
|
||||
logger = get_logger('component_name', verbose=True)
|
||||
|
||||
# Check environment variables
|
||||
import os
|
||||
print(f"VERBOSE_LOGGING: {os.getenv('VERBOSE_LOGGING')}")
|
||||
print(f"LOG_TO_CONSOLE: {os.getenv('LOG_TO_CONSOLE')}")
|
||||
```
|
||||
|
||||
### Fallback Logging
|
||||
|
||||
If file logging fails, the system automatically falls back to console logging with a warning message.
|
||||
|
||||
## New Features Summary
|
||||
|
||||
### Verbose Parameter
|
||||
- Controls console logging output
|
||||
- Respects log levels (DEBUG shows all, ERROR shows only errors)
|
||||
- Uses environment variables as default (`VERBOSE_LOGGING` or `LOG_TO_CONSOLE`)
|
||||
- Can be explicitly set to `True`/`False` to override environment
|
||||
|
||||
### Automatic Cleanup
|
||||
- Enabled by default (`clean_old_logs=True`)
|
||||
- Triggered when new log files are created (date changes)
|
||||
- Keeps most recent `max_log_files` files (default: 30)
|
||||
- Component-specific retention policies
|
||||
- Non-blocking operation with error handling
|
||||
This conditional logging system provides maximum flexibility while maintaining clean, maintainable code that works in all scenarios.
|
||||
Reference in New Issue
Block a user