""" Comprehensive Indicator Comparison Test Suite This module provides testing framework to compare original indicators from cycles module with new implementations in IncrementalTrader module to ensure mathematical equivalence. """ import pandas as pd import numpy as np import matplotlib.pyplot as plt import matplotlib.dates as mdates from datetime import datetime import sys import os from pathlib import Path # Add project root to path project_root = Path(__file__).parent.parent sys.path.insert(0, str(project_root)) # Import original indicators from cycles.IncStrategies.indicators import ( MovingAverageState as OriginalMA, ExponentialMovingAverageState as OriginalEMA, ATRState as OriginalATR, SimpleATRState as OriginalSimpleATR, SupertrendState as OriginalSupertrend, RSIState as OriginalRSI, SimpleRSIState as OriginalSimpleRSI, BollingerBandsState as OriginalBB, BollingerBandsOHLCState as OriginalBBOHLC ) # Import new indicators from IncrementalTrader.strategies.indicators import ( MovingAverageState as NewMA, ExponentialMovingAverageState as NewEMA, ATRState as NewATR, SimpleATRState as NewSimpleATR, SupertrendState as NewSupertrend, RSIState as NewRSI, SimpleRSIState as NewSimpleRSI, BollingerBandsState as NewBB, BollingerBandsOHLCState as NewBBOHLC ) class IndicatorComparisonTester: """Test framework for comparing original and new indicator implementations.""" def __init__(self, data_file: str = "data/btcusd_1-min_data.csv", sample_size: int = 10000): """ Initialize the tester with data. Args: data_file: Path to the CSV data file sample_size: Number of data points to use for testing (None for all data) """ self.data_file = data_file self.sample_size = sample_size self.data = None self.results = {} # Create results directory self.results_dir = Path("test/results") self.results_dir.mkdir(exist_ok=True) def load_data(self): """Load and prepare the data for testing.""" print(f"Loading data from {self.data_file}...") # Load data df = pd.read_csv(self.data_file) # Convert timestamp to datetime df['datetime'] = pd.to_datetime(df['Timestamp'], unit='s') # Take sample if specified if self.sample_size and len(df) > self.sample_size: # Take the most recent data df = df.tail(self.sample_size).reset_index(drop=True) self.data = df print(f"Loaded {len(df)} data points from {df['datetime'].iloc[0]} to {df['datetime'].iloc[-1]}") def compare_moving_averages(self, periods=[20, 50]): """Compare Moving Average implementations.""" print("\n=== Testing Moving Averages ===") for period in periods: print(f"Testing MA({period})...") # Initialize indicators original_ma = OriginalMA(period) new_ma = NewMA(period) original_values = [] new_values = [] # Process data for _, row in self.data.iterrows(): price = row['Close'] original_ma.update(price) new_ma.update(price) original_values.append(original_ma.get_current_value() if original_ma.is_warmed_up() else np.nan) new_values.append(new_ma.get_current_value() if new_ma.is_warmed_up() else np.nan) # Store results self.results[f'MA_{period}'] = { 'original': original_values, 'new': new_values, 'dates': self.data['datetime'].tolist() } # Calculate differences diff = np.array(new_values) - np.array(original_values) valid_diff = diff[~np.isnan(diff)] print(f" Max difference: {np.max(np.abs(valid_diff)):.10f}") print(f" Mean difference: {np.mean(np.abs(valid_diff)):.10f}") print(f" Std difference: {np.std(valid_diff):.10f}") def compare_exponential_moving_averages(self, periods=[20, 50]): """Compare Exponential Moving Average implementations.""" print("\n=== Testing Exponential Moving Averages ===") for period in periods: print(f"Testing EMA({period})...") # Initialize indicators original_ema = OriginalEMA(period) new_ema = NewEMA(period) original_values = [] new_values = [] # Process data for _, row in self.data.iterrows(): price = row['Close'] original_ema.update(price) new_ema.update(price) original_values.append(original_ema.value if original_ema.is_ready else np.nan) new_values.append(new_ema.value if new_ema.is_ready else np.nan) # Store results self.results[f'EMA_{period}'] = { 'original': original_values, 'new': new_values, 'dates': self.data['datetime'].tolist() } # Calculate differences diff = np.array(new_values) - np.array(original_values) valid_diff = diff[~np.isnan(diff)] print(f" Max difference: {np.max(np.abs(valid_diff)):.10f}") print(f" Mean difference: {np.mean(np.abs(valid_diff)):.10f}") print(f" Std difference: {np.std(valid_diff):.10f}") def compare_atr(self, periods=[14]): """Compare ATR implementations.""" print("\n=== Testing ATR ===") for period in periods: print(f"Testing ATR({period})...") # Initialize indicators original_atr = OriginalATR(period) new_atr = NewATR(period) original_values = [] new_values = [] # Process data for _, row in self.data.iterrows(): high, low, close = row['High'], row['Low'], row['Close'] original_atr.update(high, low, close) new_atr.update(high, low, close) original_values.append(original_atr.value if original_atr.is_ready else np.nan) new_values.append(new_atr.value if new_atr.is_ready else np.nan) # Store results self.results[f'ATR_{period}'] = { 'original': original_values, 'new': new_values, 'dates': self.data['datetime'].tolist() } # Calculate differences diff = np.array(new_values) - np.array(original_values) valid_diff = diff[~np.isnan(diff)] print(f" Max difference: {np.max(np.abs(valid_diff)):.10f}") print(f" Mean difference: {np.mean(np.abs(valid_diff)):.10f}") print(f" Std difference: {np.std(valid_diff):.10f}") def compare_simple_atr(self, periods=[14]): """Compare Simple ATR implementations.""" print("\n=== Testing Simple ATR ===") for period in periods: print(f"Testing SimpleATR({period})...") # Initialize indicators original_atr = OriginalSimpleATR(period) new_atr = NewSimpleATR(period) original_values = [] new_values = [] # Process data for _, row in self.data.iterrows(): high, low, close = row['High'], row['Low'], row['Close'] original_atr.update(high, low, close) new_atr.update(high, low, close) original_values.append(original_atr.value if original_atr.is_ready else np.nan) new_values.append(new_atr.value if new_atr.is_ready else np.nan) # Store results self.results[f'SimpleATR_{period}'] = { 'original': original_values, 'new': new_values, 'dates': self.data['datetime'].tolist() } # Calculate differences diff = np.array(new_values) - np.array(original_values) valid_diff = diff[~np.isnan(diff)] print(f" Max difference: {np.max(np.abs(valid_diff)):.10f}") print(f" Mean difference: {np.mean(np.abs(valid_diff)):.10f}") print(f" Std difference: {np.std(valid_diff):.10f}") def compare_supertrend(self, periods=[10], multipliers=[3.0]): """Compare Supertrend implementations.""" print("\n=== Testing Supertrend ===") for period in periods: for multiplier in multipliers: print(f"Testing Supertrend({period}, {multiplier})...") # Initialize indicators original_st = OriginalSupertrend(period, multiplier) new_st = NewSupertrend(period, multiplier) original_values = [] new_values = [] original_trends = [] new_trends = [] # Process data for _, row in self.data.iterrows(): high, low, close = row['High'], row['Low'], row['Close'] original_st.update(high, low, close) new_st.update(high, low, close) original_values.append(original_st.value if original_st.is_ready else np.nan) new_values.append(new_st.value if new_st.is_ready else np.nan) original_trends.append(original_st.trend if original_st.is_ready else 0) new_trends.append(new_st.trend if new_st.is_ready else 0) # Store results key = f'Supertrend_{period}_{multiplier}' self.results[key] = { 'original': original_values, 'new': new_values, 'original_trend': original_trends, 'new_trend': new_trends, 'dates': self.data['datetime'].tolist() } # Calculate differences diff = np.array(new_values) - np.array(original_values) valid_diff = diff[~np.isnan(diff)] trend_diff = np.array(new_trends) - np.array(original_trends) trend_matches = np.sum(trend_diff == 0) / len(trend_diff) * 100 print(f" Max difference: {np.max(np.abs(valid_diff)):.10f}") print(f" Mean difference: {np.mean(np.abs(valid_diff)):.10f}") print(f" Trend match: {trend_matches:.2f}%") def compare_rsi(self, periods=[14]): """Compare RSI implementations.""" print("\n=== Testing RSI ===") for period in periods: print(f"Testing RSI({period})...") # Initialize indicators original_rsi = OriginalRSI(period) new_rsi = NewRSI(period) original_values = [] new_values = [] # Process data for _, row in self.data.iterrows(): price = row['Close'] original_rsi.update(price) new_rsi.update(price) original_values.append(original_rsi.value if original_rsi.is_ready else np.nan) new_values.append(new_rsi.value if new_rsi.is_ready else np.nan) # Store results self.results[f'RSI_{period}'] = { 'original': original_values, 'new': new_values, 'dates': self.data['datetime'].tolist() } # Calculate differences diff = np.array(new_values) - np.array(original_values) valid_diff = diff[~np.isnan(diff)] print(f" Max difference: {np.max(np.abs(valid_diff)):.10f}") print(f" Mean difference: {np.mean(np.abs(valid_diff)):.10f}") print(f" Std difference: {np.std(valid_diff):.10f}") def compare_simple_rsi(self, periods=[14]): """Compare Simple RSI implementations.""" print("\n=== Testing Simple RSI ===") for period in periods: print(f"Testing SimpleRSI({period})...") # Initialize indicators original_rsi = OriginalSimpleRSI(period) new_rsi = NewSimpleRSI(period) original_values = [] new_values = [] # Process data for _, row in self.data.iterrows(): price = row['Close'] original_rsi.update(price) new_rsi.update(price) original_values.append(original_rsi.value if original_rsi.is_ready else np.nan) new_values.append(new_rsi.value if new_rsi.is_ready else np.nan) # Store results self.results[f'SimpleRSI_{period}'] = { 'original': original_values, 'new': new_values, 'dates': self.data['datetime'].tolist() } # Calculate differences diff = np.array(new_values) - np.array(original_values) valid_diff = diff[~np.isnan(diff)] print(f" Max difference: {np.max(np.abs(valid_diff)):.10f}") print(f" Mean difference: {np.mean(np.abs(valid_diff)):.10f}") print(f" Std difference: {np.std(valid_diff):.10f}") def compare_bollinger_bands(self, periods=[20], std_devs=[2.0]): """Compare Bollinger Bands implementations.""" print("\n=== Testing Bollinger Bands ===") for period in periods: for std_dev in std_devs: print(f"Testing BollingerBands({period}, {std_dev})...") # Initialize indicators original_bb = OriginalBB(period, std_dev) new_bb = NewBB(period, std_dev) original_upper = [] original_middle = [] original_lower = [] new_upper = [] new_middle = [] new_lower = [] # Process data for _, row in self.data.iterrows(): price = row['Close'] original_bb.update(price) new_bb.update(price) if original_bb.is_ready: original_upper.append(original_bb.upper) original_middle.append(original_bb.middle) original_lower.append(original_bb.lower) else: original_upper.append(np.nan) original_middle.append(np.nan) original_lower.append(np.nan) if new_bb.is_ready: new_upper.append(new_bb.upper) new_middle.append(new_bb.middle) new_lower.append(new_bb.lower) else: new_upper.append(np.nan) new_middle.append(np.nan) new_lower.append(np.nan) # Store results key = f'BB_{period}_{std_dev}' self.results[key] = { 'original_upper': original_upper, 'original_middle': original_middle, 'original_lower': original_lower, 'new_upper': new_upper, 'new_middle': new_middle, 'new_lower': new_lower, 'dates': self.data['datetime'].tolist() } # Calculate differences for band in ['upper', 'middle', 'lower']: orig = np.array(locals()[f'original_{band}']) new = np.array(locals()[f'new_{band}']) diff = new - orig valid_diff = diff[~np.isnan(diff)] print(f" {band.capitalize()} band - Max diff: {np.max(np.abs(valid_diff)):.10f}, " f"Mean diff: {np.mean(np.abs(valid_diff)):.10f}") def plot_comparison(self, indicator_name: str, save_plot: bool = True): """Plot comparison between original and new indicator implementations.""" if indicator_name not in self.results: print(f"No results found for {indicator_name}") return result = self.results[indicator_name] dates = pd.to_datetime(result['dates']) # Create figure fig, axes = plt.subplots(2, 1, figsize=(15, 10)) fig.suptitle(f'{indicator_name} - Original vs New Implementation Comparison', fontsize=16) # Plot 1: Overlay comparison ax1 = axes[0] if 'original' in result and 'new' in result: # Standard indicator comparison ax1.plot(dates, result['original'], label='Original', alpha=0.7, linewidth=1) ax1.plot(dates, result['new'], label='New', alpha=0.7, linewidth=1, linestyle='--') ax1.set_title(f'{indicator_name} Values Comparison') ax1.legend() ax1.grid(True, alpha=0.3) # Plot 2: Difference ax2 = axes[1] diff = np.array(result['new']) - np.array(result['original']) ax2.plot(dates, diff, color='red', alpha=0.7) ax2.set_title(f'{indicator_name} Difference (New - Original)') ax2.axhline(y=0, color='black', linestyle='-', alpha=0.5) ax2.grid(True, alpha=0.3) elif 'original_upper' in result: # Bollinger Bands comparison ax1.plot(dates, result['original_upper'], label='Original Upper', alpha=0.7) ax1.plot(dates, result['original_middle'], label='Original Middle', alpha=0.7) ax1.plot(dates, result['original_lower'], label='Original Lower', alpha=0.7) ax1.plot(dates, result['new_upper'], label='New Upper', alpha=0.7, linestyle='--') ax1.plot(dates, result['new_middle'], label='New Middle', alpha=0.7, linestyle='--') ax1.plot(dates, result['new_lower'], label='New Lower', alpha=0.7, linestyle='--') ax1.set_title(f'{indicator_name} Bollinger Bands Comparison') ax1.legend() ax1.grid(True, alpha=0.3) # Plot 2: Differences for all bands ax2 = axes[1] for band in ['upper', 'middle', 'lower']: orig = np.array(result[f'original_{band}']) new = np.array(result[f'new_{band}']) diff = new - orig ax2.plot(dates, diff, label=f'{band.capitalize()} diff', alpha=0.7) ax2.set_title(f'{indicator_name} Differences (New - Original)') ax2.axhline(y=0, color='black', linestyle='-', alpha=0.5) ax2.legend() ax2.grid(True, alpha=0.3) # Format x-axis for ax in axes: ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) ax.xaxis.set_major_locator(mdates.DayLocator(interval=max(1, len(dates)//10))) plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) plt.tight_layout() if save_plot: plot_path = self.results_dir / f"{indicator_name}_comparison.png" plt.savefig(plot_path, dpi=300, bbox_inches='tight') print(f"Plot saved to {plot_path}") plt.show() def plot_all_comparisons(self): """Plot comparisons for all tested indicators.""" print("\n=== Generating Comparison Plots ===") for indicator_name in self.results.keys(): print(f"Plotting {indicator_name}...") self.plot_comparison(indicator_name, save_plot=True) plt.close('all') # Close plots to save memory def generate_summary_report(self): """Generate a summary report of all comparisons.""" print("\n=== Summary Report ===") report_lines = [] report_lines.append("# Indicator Comparison Summary Report") report_lines.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") report_lines.append(f"Data file: {self.data_file}") report_lines.append(f"Sample size: {len(self.data)} data points") report_lines.append("") for indicator_name, result in self.results.items(): report_lines.append(f"## {indicator_name}") if 'original' in result and 'new' in result: # Standard indicator diff = np.array(result['new']) - np.array(result['original']) valid_diff = diff[~np.isnan(diff)] if len(valid_diff) > 0: report_lines.append(f"- Max absolute difference: {np.max(np.abs(valid_diff)):.10f}") report_lines.append(f"- Mean absolute difference: {np.mean(np.abs(valid_diff)):.10f}") report_lines.append(f"- Standard deviation: {np.std(valid_diff):.10f}") report_lines.append(f"- Valid data points: {len(valid_diff)}") # Check if differences are negligible if np.max(np.abs(valid_diff)) < 1e-10: report_lines.append("- ✅ **PASSED**: Implementations are mathematically equivalent") elif np.max(np.abs(valid_diff)) < 1e-6: report_lines.append("- ⚠️ **WARNING**: Small differences detected (likely floating point precision)") else: report_lines.append("- ❌ **FAILED**: Significant differences detected") else: report_lines.append("- ❌ **ERROR**: No valid data points for comparison") elif 'original_upper' in result: # Bollinger Bands all_passed = True for band in ['upper', 'middle', 'lower']: orig = np.array(result[f'original_{band}']) new = np.array(result[f'new_{band}']) diff = new - orig valid_diff = diff[~np.isnan(diff)] if len(valid_diff) > 0: max_diff = np.max(np.abs(valid_diff)) report_lines.append(f"- {band.capitalize()} band max diff: {max_diff:.10f}") if max_diff >= 1e-6: all_passed = False if all_passed: report_lines.append("- ✅ **PASSED**: All bands are mathematically equivalent") else: report_lines.append("- ❌ **FAILED**: Significant differences in one or more bands") report_lines.append("") # Save report report_path = self.results_dir / "comparison_summary.md" with open(report_path, 'w') as f: f.write('\n'.join(report_lines)) print(f"Summary report saved to {report_path}") # Print summary to console print('\n'.join(report_lines)) def run_all_tests(self): """Run all indicator comparison tests.""" print("Starting comprehensive indicator comparison tests...") # Load data self.load_data() # Run all comparisons self.compare_moving_averages() self.compare_exponential_moving_averages() self.compare_atr() self.compare_simple_atr() self.compare_supertrend() self.compare_rsi() self.compare_simple_rsi() self.compare_bollinger_bands() # Generate plots and reports self.plot_all_comparisons() self.generate_summary_report() print("\n✅ All tests completed! Check the test/results/ directory for detailed outputs.") if __name__ == "__main__": # Run the comprehensive test suite tester = IndicatorComparisonTester(sample_size=5000) # Use 5000 data points for faster testing tester.run_all_tests()