8c23489ff0
- Added `realtime_execution.py` for real-time strategy execution, enabling live signal generation and integration with the dashboard's chart refresh cycle. - Introduced `data_integration.py` to manage market data orchestration, caching, and technical indicator calculations for strategy signal generation. - Implemented `validation.py` for comprehensive validation and quality assessment of strategy-generated signals, ensuring reliability and consistency. - Developed `batch_processing.py` to facilitate efficient backtesting of multiple strategies across large datasets with memory management and performance optimization. - Updated `__init__.py` files to include new modules and ensure proper exports, enhancing modularity and maintainability. - Enhanced unit tests for the new features, ensuring robust functionality and adherence to project standards. These changes establish a solid foundation for real-time strategy execution and data integration, aligning with project goals for modularity, performance, and maintainability.
478 lines
18 KiB
Python
478 lines
18 KiB
Python
"""
|
|
Tests for Strategy Signal Validation Pipeline
|
|
|
|
This module tests signal validation, filtering, and quality assessment
|
|
functionality for strategy-generated signals.
|
|
"""
|
|
|
|
import pytest
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import patch
|
|
|
|
from strategies.validation import StrategySignalValidator, ValidationConfig
|
|
from strategies.data_types import StrategySignal, SignalType
|
|
|
|
|
|
class TestValidationConfig:
|
|
"""Tests for ValidationConfig dataclass."""
|
|
|
|
def test_default_config(self):
|
|
"""Test default validation configuration."""
|
|
config = ValidationConfig()
|
|
|
|
assert config.min_confidence == 0.0
|
|
assert config.max_confidence == 1.0
|
|
assert config.required_metadata_fields == []
|
|
assert config.allowed_signal_types == list(SignalType)
|
|
assert config.price_tolerance_percent == 5.0
|
|
|
|
def test_custom_config(self):
|
|
"""Test custom validation configuration."""
|
|
config = ValidationConfig(
|
|
min_confidence=0.3,
|
|
max_confidence=0.9,
|
|
required_metadata_fields=['indicator1', 'indicator2'],
|
|
allowed_signal_types=[SignalType.BUY, SignalType.SELL],
|
|
price_tolerance_percent=2.0
|
|
)
|
|
|
|
assert config.min_confidence == 0.3
|
|
assert config.max_confidence == 0.9
|
|
assert config.required_metadata_fields == ['indicator1', 'indicator2']
|
|
assert config.allowed_signal_types == [SignalType.BUY, SignalType.SELL]
|
|
assert config.price_tolerance_percent == 2.0
|
|
|
|
|
|
class TestStrategySignalValidator:
|
|
"""Tests for StrategySignalValidator class."""
|
|
|
|
@pytest.fixture
|
|
def validator(self):
|
|
"""Create validator with default configuration."""
|
|
return StrategySignalValidator()
|
|
|
|
@pytest.fixture
|
|
def strict_validator(self):
|
|
"""Create validator with strict configuration."""
|
|
config = ValidationConfig(
|
|
min_confidence=0.5,
|
|
max_confidence=1.0,
|
|
required_metadata_fields=['rsi', 'macd'],
|
|
allowed_signal_types=[SignalType.BUY, SignalType.SELL]
|
|
)
|
|
return StrategySignalValidator(config)
|
|
|
|
@pytest.fixture
|
|
def valid_signal(self):
|
|
"""Create a valid strategy signal for testing."""
|
|
return StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='BTC-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.BUY,
|
|
price=50000.0,
|
|
confidence=0.8,
|
|
metadata={'rsi': 30, 'macd': 0.05}
|
|
)
|
|
|
|
def test_initialization(self, validator):
|
|
"""Test validator initialization."""
|
|
assert validator.config is not None
|
|
assert validator.logger is not None
|
|
assert validator._validation_stats['total_signals_validated'] == 0
|
|
assert validator._validation_stats['valid_signals'] == 0
|
|
assert validator._validation_stats['invalid_signals'] == 0
|
|
|
|
def test_validate_valid_signal(self, validator, valid_signal):
|
|
"""Test validation of a completely valid signal."""
|
|
is_valid, errors = validator.validate_signal(valid_signal)
|
|
|
|
assert is_valid is True
|
|
assert errors == []
|
|
assert validator._validation_stats['total_signals_validated'] == 1
|
|
assert validator._validation_stats['valid_signals'] == 1
|
|
assert validator._validation_stats['invalid_signals'] == 0
|
|
|
|
def test_validate_invalid_confidence_low(self, validator, valid_signal):
|
|
"""Test validation with confidence too low."""
|
|
valid_signal.confidence = -0.1
|
|
|
|
is_valid, errors = validator.validate_signal(valid_signal)
|
|
|
|
assert is_valid is False
|
|
assert len(errors) == 1
|
|
assert "Invalid confidence" in errors[0]
|
|
assert validator._validation_stats['invalid_signals'] == 1
|
|
|
|
def test_validate_invalid_confidence_high(self, validator, valid_signal):
|
|
"""Test validation with confidence too high."""
|
|
valid_signal.confidence = 1.5
|
|
|
|
is_valid, errors = validator.validate_signal(valid_signal)
|
|
|
|
assert is_valid is False
|
|
assert len(errors) == 1
|
|
assert "Invalid confidence" in errors[0]
|
|
|
|
def test_validate_invalid_signal_type(self, strict_validator, valid_signal):
|
|
"""Test validation with disallowed signal type."""
|
|
valid_signal.signal_type = SignalType.HOLD
|
|
|
|
is_valid, errors = strict_validator.validate_signal(valid_signal)
|
|
|
|
assert is_valid is False
|
|
assert len(errors) == 1
|
|
assert "Signal type" in errors[0] and "not in allowed types" in errors[0]
|
|
|
|
def test_validate_invalid_price(self, validator, valid_signal):
|
|
"""Test validation with invalid price."""
|
|
valid_signal.price = -100.0
|
|
|
|
is_valid, errors = validator.validate_signal(valid_signal)
|
|
|
|
assert is_valid is False
|
|
assert len(errors) == 1
|
|
assert "Invalid price" in errors[0]
|
|
|
|
def test_validate_missing_required_metadata(self, strict_validator, valid_signal):
|
|
"""Test validation with missing required metadata."""
|
|
valid_signal.metadata = {'rsi': 30} # Missing 'macd'
|
|
|
|
is_valid, errors = strict_validator.validate_signal(valid_signal)
|
|
|
|
assert is_valid is False
|
|
assert len(errors) == 1
|
|
assert "Missing required metadata fields" in errors[0]
|
|
assert "macd" in errors[0]
|
|
|
|
def test_validate_multiple_errors(self, strict_validator, valid_signal):
|
|
"""Test validation with multiple errors."""
|
|
valid_signal.confidence = 1.5 # Too high
|
|
valid_signal.price = -100.0 # Invalid
|
|
valid_signal.signal_type = SignalType.HOLD # Not allowed
|
|
valid_signal.metadata = {} # Missing required fields
|
|
|
|
is_valid, errors = strict_validator.validate_signal(valid_signal)
|
|
|
|
assert is_valid is False
|
|
assert len(errors) == 4
|
|
assert any("confidence" in error for error in errors)
|
|
assert any("price" in error for error in errors)
|
|
assert any("Signal type" in error for error in errors)
|
|
assert any("Missing required metadata" in error for error in errors)
|
|
|
|
def test_validation_statistics_tracking(self, validator, valid_signal):
|
|
"""Test that validation statistics are properly tracked."""
|
|
# Validate multiple signals
|
|
validator.validate_signal(valid_signal) # Valid
|
|
|
|
invalid_signal = valid_signal
|
|
invalid_signal.confidence = 1.5 # Invalid
|
|
validator.validate_signal(invalid_signal) # Invalid
|
|
|
|
stats = validator._validation_stats
|
|
assert stats['total_signals_validated'] == 2
|
|
assert stats['valid_signals'] == 1
|
|
assert stats['invalid_signals'] == 1
|
|
assert len(stats['validation_errors']) > 0
|
|
|
|
def test_validate_signals_batch(self, validator, valid_signal):
|
|
"""Test batch validation of multiple signals."""
|
|
# Create a mix of valid and invalid signals
|
|
signals = [
|
|
valid_signal, # Valid
|
|
StrategySignal( # Invalid confidence
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='ETH-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.SELL,
|
|
price=3000.0,
|
|
confidence=1.5, # Invalid
|
|
metadata={}
|
|
),
|
|
StrategySignal( # Valid
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='BNB-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.BUY,
|
|
price=300.0,
|
|
confidence=0.7,
|
|
metadata={}
|
|
)
|
|
]
|
|
|
|
valid_signals, invalid_signals = validator.validate_signals_batch(signals)
|
|
|
|
assert len(valid_signals) == 2
|
|
assert len(invalid_signals) == 1
|
|
assert invalid_signals[0].confidence == 1.5
|
|
|
|
def test_filter_signals_by_confidence(self, validator, valid_signal):
|
|
"""Test filtering signals by confidence threshold."""
|
|
signals = [
|
|
valid_signal, # confidence 0.8
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='ETH-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.SELL,
|
|
price=3000.0,
|
|
confidence=0.3, # Low confidence
|
|
metadata={}
|
|
),
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='BNB-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.BUY,
|
|
price=300.0,
|
|
confidence=0.9, # High confidence
|
|
metadata={}
|
|
)
|
|
]
|
|
|
|
# Filter with threshold 0.5
|
|
filtered_signals = validator.filter_signals_by_confidence(signals, min_confidence=0.5)
|
|
|
|
assert len(filtered_signals) == 2
|
|
assert all(signal.confidence >= 0.5 for signal in filtered_signals)
|
|
assert filtered_signals[0].confidence == 0.8
|
|
assert filtered_signals[1].confidence == 0.9
|
|
|
|
def test_filter_signals_by_type(self, validator, valid_signal):
|
|
"""Test filtering signals by allowed types."""
|
|
signals = [
|
|
valid_signal, # BUY
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='ETH-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.SELL,
|
|
price=3000.0,
|
|
confidence=0.8,
|
|
metadata={}
|
|
),
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='BNB-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.HOLD,
|
|
price=300.0,
|
|
confidence=0.7,
|
|
metadata={}
|
|
)
|
|
]
|
|
|
|
# Filter to only allow BUY and SELL
|
|
filtered_signals = validator.filter_signals_by_type(
|
|
signals,
|
|
allowed_types=[SignalType.BUY, SignalType.SELL]
|
|
)
|
|
|
|
assert len(filtered_signals) == 2
|
|
assert filtered_signals[0].signal_type == SignalType.BUY
|
|
assert filtered_signals[1].signal_type == SignalType.SELL
|
|
|
|
def test_get_validation_statistics(self, validator, valid_signal):
|
|
"""Test comprehensive validation statistics."""
|
|
# Validate some signals to generate statistics
|
|
validator.validate_signal(valid_signal) # Valid
|
|
|
|
invalid_signal = valid_signal
|
|
invalid_signal.confidence = -0.1 # Invalid
|
|
validator.validate_signal(invalid_signal) # Invalid
|
|
|
|
stats = validator.get_validation_statistics()
|
|
|
|
assert stats['total_signals_validated'] == 2
|
|
assert stats['valid_signals'] == 1
|
|
assert stats['invalid_signals'] == 1
|
|
assert stats['validation_success_rate'] == 0.5
|
|
assert stats['validation_failure_rate'] == 0.5
|
|
assert 'validation_errors' in stats
|
|
|
|
def test_transform_signal_confidence(self, validator, valid_signal):
|
|
"""Test signal confidence transformation."""
|
|
original_confidence = valid_signal.confidence # 0.8
|
|
|
|
# Test confidence multiplier
|
|
transformed_signal = validator.transform_signal_confidence(
|
|
valid_signal,
|
|
confidence_multiplier=1.2
|
|
)
|
|
|
|
assert transformed_signal.confidence == original_confidence * 1.2
|
|
assert transformed_signal.symbol == valid_signal.symbol
|
|
assert transformed_signal.signal_type == valid_signal.signal_type
|
|
assert transformed_signal.price == valid_signal.price
|
|
|
|
# Test confidence cap
|
|
capped_signal = validator.transform_signal_confidence(
|
|
valid_signal,
|
|
confidence_multiplier=2.0, # Would exceed 1.0
|
|
max_confidence=1.0
|
|
)
|
|
|
|
assert capped_signal.confidence == 1.0 # Capped at max
|
|
|
|
def test_enrich_signal_metadata(self, validator, valid_signal):
|
|
"""Test signal metadata enrichment."""
|
|
additional_metadata = {
|
|
'validation_timestamp': datetime.now(timezone.utc).isoformat(),
|
|
'validation_status': 'approved',
|
|
'risk_score': 0.2
|
|
}
|
|
|
|
enriched_signal = validator.enrich_signal_metadata(valid_signal, additional_metadata)
|
|
|
|
# Original metadata should be preserved
|
|
assert enriched_signal.metadata['rsi'] == 30
|
|
assert enriched_signal.metadata['macd'] == 0.05
|
|
|
|
# New metadata should be added
|
|
assert enriched_signal.metadata['validation_status'] == 'approved'
|
|
assert enriched_signal.metadata['risk_score'] == 0.2
|
|
assert 'validation_timestamp' in enriched_signal.metadata
|
|
|
|
# Other properties should remain unchanged
|
|
assert enriched_signal.confidence == valid_signal.confidence
|
|
assert enriched_signal.signal_type == valid_signal.signal_type
|
|
|
|
def test_transform_signals_batch(self, validator, valid_signal):
|
|
"""Test batch signal transformation."""
|
|
signals = [
|
|
valid_signal,
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='ETH-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.SELL,
|
|
price=3000.0,
|
|
confidence=0.6,
|
|
metadata={'ema': 2950}
|
|
)
|
|
]
|
|
|
|
additional_metadata = {'batch_id': 'test_batch_001'}
|
|
|
|
transformed_signals = validator.transform_signals_batch(
|
|
signals,
|
|
confidence_multiplier=1.1,
|
|
additional_metadata=additional_metadata
|
|
)
|
|
|
|
assert len(transformed_signals) == 2
|
|
|
|
# Check confidence transformation
|
|
assert transformed_signals[0].confidence == 0.8 * 1.1
|
|
assert transformed_signals[1].confidence == 0.6 * 1.1
|
|
|
|
# Check metadata enrichment
|
|
assert transformed_signals[0].metadata['batch_id'] == 'test_batch_001'
|
|
assert transformed_signals[1].metadata['batch_id'] == 'test_batch_001'
|
|
|
|
# Verify original metadata preserved
|
|
assert transformed_signals[0].metadata['rsi'] == 30
|
|
assert transformed_signals[1].metadata['ema'] == 2950
|
|
|
|
def test_calculate_signal_quality_metrics(self, validator, valid_signal):
|
|
"""Test signal quality metrics calculation."""
|
|
signals = [
|
|
valid_signal, # confidence 0.8, has metadata
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='ETH-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.SELL,
|
|
price=3000.0,
|
|
confidence=0.9, # High confidence
|
|
metadata={'volume_spike': True}
|
|
),
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='BNB-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.HOLD,
|
|
price=300.0,
|
|
confidence=0.4, # Low confidence
|
|
metadata=None # No metadata
|
|
)
|
|
]
|
|
|
|
metrics = validator.calculate_signal_quality_metrics(signals)
|
|
|
|
assert metrics['total_signals'] == 3
|
|
assert metrics['confidence_metrics']['average'] == round((0.8 + 0.9 + 0.4) / 3, 3)
|
|
assert metrics['confidence_metrics']['minimum'] == 0.4
|
|
assert metrics['confidence_metrics']['maximum'] == 0.9
|
|
assert metrics['confidence_metrics']['high_confidence_count'] == 2 # >= 0.7
|
|
assert metrics['quality_score'] == round((2/3) * 100, 1) # 66.7%
|
|
assert metrics['metadata_completeness_percentage'] == round((2/3) * 100, 1)
|
|
|
|
# Check signal type distribution
|
|
assert metrics['signal_type_distribution']['buy'] == 1
|
|
assert metrics['signal_type_distribution']['sell'] == 1
|
|
assert metrics['signal_type_distribution']['hold'] == 1
|
|
|
|
# Check recommendations
|
|
assert isinstance(metrics['recommendations'], list)
|
|
assert len(metrics['recommendations']) > 0
|
|
|
|
def test_calculate_signal_quality_metrics_empty(self, validator):
|
|
"""Test quality metrics with empty signal list."""
|
|
metrics = validator.calculate_signal_quality_metrics([])
|
|
|
|
assert 'error' in metrics
|
|
assert metrics['error'] == 'No signals provided for quality analysis'
|
|
|
|
def test_generate_quality_recommendations(self, validator):
|
|
"""Test quality recommendation generation."""
|
|
# Test low confidence signals
|
|
low_confidence_signals = [
|
|
StrategySignal(
|
|
timestamp=datetime.now(timezone.utc),
|
|
symbol='BTC-USDT',
|
|
timeframe='1h',
|
|
signal_type=SignalType.BUY,
|
|
price=50000.0,
|
|
confidence=0.3, # Low confidence
|
|
metadata=None # No metadata
|
|
)
|
|
]
|
|
|
|
recommendations = validator._generate_quality_recommendations(low_confidence_signals)
|
|
|
|
assert any("confidence" in rec.lower() for rec in recommendations)
|
|
assert any("metadata" in rec.lower() for rec in recommendations)
|
|
|
|
def test_generate_validation_report(self, validator, valid_signal):
|
|
"""Test comprehensive validation report generation."""
|
|
# Generate some validation activity
|
|
validator.validate_signal(valid_signal) # Valid
|
|
|
|
invalid_signal = valid_signal
|
|
invalid_signal.confidence = -0.1 # Invalid
|
|
validator.validate_signal(invalid_signal) # Invalid
|
|
|
|
report = validator.generate_validation_report()
|
|
|
|
assert 'report_timestamp' in report
|
|
assert 'validation_summary' in report
|
|
assert 'error_analysis' in report
|
|
assert 'configuration' in report
|
|
assert 'health_status' in report
|
|
|
|
# Check validation summary
|
|
summary = report['validation_summary']
|
|
assert summary['total_validated'] == 2
|
|
assert '50.0%' in summary['success_rate']
|
|
assert '50.0%' in summary['failure_rate']
|
|
|
|
# Check configuration
|
|
config = report['configuration']
|
|
assert config['min_confidence'] == 0.0
|
|
assert config['max_confidence'] == 1.0
|
|
assert isinstance(config['allowed_signal_types'], list)
|
|
|
|
# Check health status
|
|
assert report['health_status'] in ['good', 'needs_attention'] |