From b49e39dcb490d291875386583c50e8057a3ed29d Mon Sep 17 00:00:00 2001 From: "Vasily.onl" Date: Fri, 6 Jun 2025 15:06:17 +0800 Subject: [PATCH] Implement multi-timeframe support for indicators - Enhanced the `UserIndicator` class to include an optional `timeframe` attribute for custom indicator timeframes. - Updated the `get_indicator_data` method in `MarketDataIntegrator` to fetch and calculate indicators based on the specified timeframe, ensuring proper data alignment and handling. - Modified the `ChartBuilder` to pass the correct DataFrame for plotting indicators with different timeframes. - Added UI elements in the indicator modal for selecting timeframes, improving user experience. - Updated relevant JSON templates to include the new `timeframe` field for all indicators. - Refactored the `prepare_chart_data` function to ensure it returns a DataFrame with a `DatetimeIndex` for consistent calculations. This commit enhances the flexibility and usability of the indicator system, allowing users to analyze data across various timeframes. --- components/charts/builder.py | 17 +- components/charts/data_integration.py | 56 +++++- components/charts/indicator_manager.py | 28 ++- components/charts/utils.py | 7 +- .../templates/bollinger_bands_template.json | 8 +- config/indicators/templates/ema_template.json | 6 + .../indicators/templates/macd_template.json | 11 +- config/indicators/templates/rsi_template.json | 6 + config/indicators/templates/sma_template.json | 6 + .../user_indicators/ema_b869638d.json | 20 ++ .../user_indicators/ema_bfbf3a1d.json | 20 ++ .../user_indicators/macd_307935a7.json | 3 +- dashboard/callbacks/indicators.py | 60 +++--- dashboard/components/data_analysis.py | 21 +- dashboard/components/indicator_modal.py | 21 ++ data/common/indicators.py | 110 +++++----- docs/components/technical-indicators.md | 190 +++++++----------- tasks/tasks-indicator-timeframe-feature.md | 38 ++++ tasks/tasks-refactor-indicator-calculation.md | 36 ++++ 19 files changed, 417 insertions(+), 247 deletions(-) create mode 100644 config/indicators/user_indicators/ema_b869638d.json create mode 100644 config/indicators/user_indicators/ema_bfbf3a1d.json create mode 100644 tasks/tasks-indicator-timeframe-feature.md create mode 100644 tasks/tasks-refactor-indicator-calculation.md diff --git a/components/charts/builder.py b/components/charts/builder.py index a2eca6c..968938a 100644 --- a/components/charts/builder.py +++ b/components/charts/builder.py @@ -489,7 +489,12 @@ class ChartBuilder: if all_indicator_configs: indicator_data_map = self.data_integrator.get_indicator_data( - df, all_indicator_configs, indicator_manager + main_df=df, + main_timeframe=timeframe, + indicator_configs=all_indicator_configs, + indicator_manager=indicator_manager, + symbol=symbol, + exchange="okx" ) for indicator_id, indicator_df in indicator_data_map.items(): @@ -499,7 +504,9 @@ class ChartBuilder: continue if indicator_df is not None and not indicator_df.empty: - final_df = pd.merge(final_df, indicator_df, on='timestamp', how='left') + # Add a suffix to the indicator's columns before joining to prevent overlap + # when multiple indicators of the same type are added. + final_df = final_df.join(indicator_df, how='left', rsuffix=f'_{indicator.id}') # Determine target row for plotting target_row = 1 # Default to overlay on the main chart @@ -511,7 +518,7 @@ class ChartBuilder: if indicator.type == 'bollinger_bands': if all(c in indicator_df.columns for c in ['upper_band', 'lower_band', 'middle_band']): # Prepare data for the filled area - x_vals = indicator_df['timestamp'] + x_vals = indicator_df.index y_upper = indicator_df['upper_band'] y_lower = indicator_df['lower_band'] @@ -522,7 +529,7 @@ class ChartBuilder: # Add the transparent fill trace fig.add_trace(go.Scatter( - x=pd.concat([x_vals, x_vals[::-1]]), + x=pd.concat([x_vals.to_series(), x_vals.to_series()[::-1]]), y=pd.concat([y_upper, y_lower[::-1]]), fill='toself', fillcolor=fill_color, @@ -540,7 +547,7 @@ class ChartBuilder: for col in indicator_df.columns: if col != 'timestamp': fig.add_trace(go.Scatter( - x=indicator_df['timestamp'], + x=indicator_df.index, y=indicator_df[col], mode='lines', name=f"{indicator.name} ({col})", diff --git a/components/charts/data_integration.py b/components/charts/data_integration.py index 8dd0686..495bf3f 100644 --- a/components/charts/data_integration.py +++ b/components/charts/data_integration.py @@ -460,8 +460,11 @@ class MarketDataIntegrator: def get_indicator_data( self, main_df: pd.DataFrame, + main_timeframe: str, indicator_configs: List['IndicatorLayerConfig'], - indicator_manager: 'IndicatorManager' + indicator_manager: 'IndicatorManager', + symbol: str, + exchange: str = "okx" ) -> Dict[str, pd.DataFrame]: indicator_data_map = {} @@ -477,21 +480,62 @@ class MarketDataIntegrator: continue try: - # The new `calculate` method in TechnicalIndicators handles DataFrame input + # Determine the timeframe and data to use + target_timeframe = indicator.timeframe + + if target_timeframe and target_timeframe != main_timeframe: + # Custom timeframe: fetch new data + days_back = (main_df.index.max() - main_df.index.min()).days + 2 # Add buffer + + raw_candles, _ = self.get_market_data_for_indicators( + symbol=symbol, + timeframe=target_timeframe, + days_back=days_back, + exchange=exchange + ) + + if not raw_candles: + self.logger.warning(f"No data for indicator '{indicator.name}' on timeframe {target_timeframe}") + continue + + from components.charts.utils import prepare_chart_data + indicator_df = prepare_chart_data(raw_candles) + else: + # Use main chart's dataframe + indicator_df = main_df + + # Calculate the indicator indicator_result_pkg = self.indicators.calculate( indicator.type, - main_df, + indicator_df, **indicator.parameters ) - if indicator_result_pkg and 'data' in indicator_result_pkg and indicator_result_pkg['data']: - # The result is a list of IndicatorResult objects. Convert to DataFrame. + if indicator_result_pkg and indicator_result_pkg.get('data'): indicator_results = indicator_result_pkg['data'] + + if not indicator_results: + self.logger.warning(f"Indicator '{indicator.name}' produced no results.") + continue + result_df = pd.DataFrame([ {'timestamp': r.timestamp, **r.values} for r in indicator_results ]) - indicator_data_map[indicator.id] = result_df + result_df['timestamp'] = pd.to_datetime(result_df['timestamp']) + result_df.set_index('timestamp', inplace=True) + + # Ensure timezone consistency before reindexing + if result_df.index.tz is None: + result_df = result_df.tz_localize('UTC') + result_df = result_df.tz_convert(main_df.index.tz) + + # Align data to main_df's index to handle different timeframes + if not result_df.index.equals(main_df.index): + aligned_df = result_df.reindex(main_df.index, method='ffill') + indicator_data_map[indicator.id] = aligned_df + else: + indicator_data_map[indicator.id] = result_df else: self.logger.warning(f"No data returned for indicator '{indicator.name}'") diff --git a/components/charts/indicator_manager.py b/components/charts/indicator_manager.py index f3a21f1..a83622b 100644 --- a/components/charts/indicator_manager.py +++ b/components/charts/indicator_manager.py @@ -60,6 +60,7 @@ class UserIndicator: display_type: str # DisplayType parameters: Dict[str, Any] styling: IndicatorStyling + timeframe: Optional[str] = None visible: bool = True created_date: str = "" modified_date: str = "" @@ -82,6 +83,7 @@ class UserIndicator: 'display_type': self.display_type, 'parameters': self.parameters, 'styling': asdict(self.styling), + 'timeframe': self.timeframe, 'visible': self.visible, 'created_date': self.created_date, 'modified_date': self.modified_date @@ -101,6 +103,7 @@ class UserIndicator: display_type=data['display_type'], parameters=data.get('parameters', {}), styling=styling, + timeframe=data.get('timeframe'), visible=data.get('visible', True), created_date=data.get('created_date', ''), modified_date=data.get('modified_date', '') @@ -244,7 +247,7 @@ class IndicatorManager: def create_indicator(self, name: str, indicator_type: str, parameters: Dict[str, Any], description: str = "", color: str = "#007bff", - display_type: str = None) -> Optional[UserIndicator]: + display_type: str = None, timeframe: Optional[str] = None) -> Optional[UserIndicator]: """ Create a new indicator. @@ -255,6 +258,7 @@ class IndicatorManager: description: Optional description color: Color for chart display display_type: overlay or subplot (auto-detected if None) + timeframe: Optional timeframe for the indicator Returns: Created UserIndicator instance or None if error @@ -278,7 +282,8 @@ class IndicatorManager: type=indicator_type, display_type=display_type, parameters=parameters, - styling=styling + styling=styling, + timeframe=timeframe ) # Save to file @@ -309,16 +314,19 @@ class IndicatorManager: return False # Update fields - for field, value in updates.items(): - if hasattr(indicator, field): - if field == 'styling' and isinstance(value, dict): - # Update styling fields - for style_field, style_value in value.items(): - if hasattr(indicator.styling, style_field): - setattr(indicator.styling, style_field, style_value) + for key, value in updates.items(): + if hasattr(indicator, key): + if key == 'styling' and isinstance(value, dict): + # Update nested styling fields + for style_key, style_value in value.items(): + if hasattr(indicator.styling, style_key): + setattr(indicator.styling, style_key, style_value) + elif key == 'parameters' and isinstance(value, dict): + indicator.parameters.update(value) else: - setattr(indicator, field, value) + setattr(indicator, key, value) + # Save updated indicator return self.save_indicator(indicator) except Exception as e: diff --git a/components/charts/utils.py b/components/charts/utils.py index 2dd2ee2..07bf1c6 100644 --- a/components/charts/utils.py +++ b/components/charts/utils.py @@ -139,9 +139,10 @@ def prepare_chart_data(candles: List[Dict[str, Any]]) -> pd.DataFrame: if col in df.columns: df[col] = pd.to_numeric(df[col], errors='coerce') - # Sort by timestamp - df = df.sort_values('timestamp').reset_index(drop=True) - + # Sort by timestamp and set it as the index, keeping the column + df = df.sort_values('timestamp') + df.index = pd.to_datetime(df['timestamp']) + # Handle missing volume data if 'volume' not in df.columns: df['volume'] = 0 diff --git a/config/indicators/templates/bollinger_bands_template.json b/config/indicators/templates/bollinger_bands_template.json index 34ccacb..cc58e84 100644 --- a/config/indicators/templates/bollinger_bands_template.json +++ b/config/indicators/templates/bollinger_bands_template.json @@ -3,6 +3,7 @@ "description": "Bollinger Bands volatility indicator", "type": "bollinger_bands", "display_type": "overlay", + "timeframe": null, "default_parameters": { "period": 20, "std_dev": 2.0 @@ -20,7 +21,12 @@ "min": 0.5, "max": 5.0, "default": 2.0, - "description": "Standard deviation multiplier" + "description": "Standard deviation for Bollinger Bands" + }, + "timeframe": { + "type": "string", + "default": null, + "description": "Indicator timeframe (e.g., '1h', '4h'). Null for chart timeframe." } }, "default_styling": { diff --git a/config/indicators/templates/ema_template.json b/config/indicators/templates/ema_template.json index b26a5d6..c066726 100644 --- a/config/indicators/templates/ema_template.json +++ b/config/indicators/templates/ema_template.json @@ -3,6 +3,7 @@ "description": "Exponential Moving Average indicator", "type": "ema", "display_type": "overlay", + "timeframe": null, "default_parameters": { "period": 12 }, @@ -13,6 +14,11 @@ "max": 200, "default": 12, "description": "Period for EMA calculation" + }, + "timeframe": { + "type": "string", + "default": null, + "description": "Indicator timeframe (e.g., '1h', '4h'). Null for chart timeframe." } }, "default_styling": { diff --git a/config/indicators/templates/macd_template.json b/config/indicators/templates/macd_template.json index 828c6f8..d7073fd 100644 --- a/config/indicators/templates/macd_template.json +++ b/config/indicators/templates/macd_template.json @@ -3,6 +3,7 @@ "description": "Moving Average Convergence Divergence", "type": "macd", "display_type": "subplot", + "timeframe": null, "default_parameters": { "fast_period": 12, "slow_period": 26, @@ -28,11 +29,17 @@ "min": 2, "max": 30, "default": 9, - "description": "Signal line period" + "description": "Signal line period for MACD" + }, + "timeframe": { + "type": "string", + "default": null, + "description": "Indicator timeframe (e.g., '1h', '4h'). Null for chart timeframe." } }, "default_styling": { "color": "#fd7e14", - "line_width": 2 + "line_width": 2, + "macd_line_color": "#007bff" } } \ No newline at end of file diff --git a/config/indicators/templates/rsi_template.json b/config/indicators/templates/rsi_template.json index d1619dc..27085ab 100644 --- a/config/indicators/templates/rsi_template.json +++ b/config/indicators/templates/rsi_template.json @@ -3,6 +3,7 @@ "description": "RSI oscillator indicator", "type": "rsi", "display_type": "subplot", + "timeframe": null, "default_parameters": { "period": 14 }, @@ -13,6 +14,11 @@ "max": 50, "default": 14, "description": "Period for RSI calculation" + }, + "timeframe": { + "type": "string", + "default": null, + "description": "Indicator timeframe (e.g., '1h', '4h'). Null for chart timeframe." } }, "default_styling": { diff --git a/config/indicators/templates/sma_template.json b/config/indicators/templates/sma_template.json index e6a9935..cbb9323 100644 --- a/config/indicators/templates/sma_template.json +++ b/config/indicators/templates/sma_template.json @@ -3,6 +3,7 @@ "description": "Simple Moving Average indicator", "type": "sma", "display_type": "overlay", + "timeframe": null, "default_parameters": { "period": 20 }, @@ -13,6 +14,11 @@ "max": 200, "default": 20, "description": "Period for SMA calculation" + }, + "timeframe": { + "type": "string", + "default": null, + "description": "Indicator timeframe (e.g., '1h', '4h'). Null for chart timeframe." } }, "default_styling": { diff --git a/config/indicators/user_indicators/ema_b869638d.json b/config/indicators/user_indicators/ema_b869638d.json new file mode 100644 index 0000000..083193e --- /dev/null +++ b/config/indicators/user_indicators/ema_b869638d.json @@ -0,0 +1,20 @@ +{ + "id": "ema_b869638d", + "name": "EMA 12 (15 minutes)", + "description": "", + "type": "ema", + "display_type": "overlay", + "parameters": { + "period": 12 + }, + "styling": { + "color": "#007bff", + "line_width": 2, + "opacity": 1.0, + "line_style": "solid" + }, + "timeframe": "15m", + "visible": true, + "created_date": "2025-06-06T06:56:54.181578+00:00", + "modified_date": "2025-06-06T06:56:54.181578+00:00" +} \ No newline at end of file diff --git a/config/indicators/user_indicators/ema_bfbf3a1d.json b/config/indicators/user_indicators/ema_bfbf3a1d.json new file mode 100644 index 0000000..80bfab9 --- /dev/null +++ b/config/indicators/user_indicators/ema_bfbf3a1d.json @@ -0,0 +1,20 @@ +{ + "id": "ema_bfbf3a1d", + "name": "EMA 12 (5 minutes)", + "description": "", + "type": "ema", + "display_type": "overlay", + "parameters": { + "period": 12 + }, + "styling": { + "color": "#007bff", + "line_width": 2, + "opacity": 1.0, + "line_style": "solid" + }, + "timeframe": "5m", + "visible": true, + "created_date": "2025-06-06T07:02:34.613543+00:00", + "modified_date": "2025-06-06T07:02:34.613543+00:00" +} \ No newline at end of file diff --git a/config/indicators/user_indicators/macd_307935a7.json b/config/indicators/user_indicators/macd_307935a7.json index bb4e439..d21ec23 100644 --- a/config/indicators/user_indicators/macd_307935a7.json +++ b/config/indicators/user_indicators/macd_307935a7.json @@ -15,7 +15,8 @@ "opacity": 1.0, "line_style": "solid" }, + "timeframe": "1h", "visible": true, "created_date": "2025-06-04T04:16:35.459602+00:00", - "modified_date": "2025-06-04T04:16:35.459602+00:00" + "modified_date": "2025-06-06T07:03:58.642238+00:00" } \ No newline at end of file diff --git a/dashboard/callbacks/indicators.py b/dashboard/callbacks/indicators.py index 0efbc83..e502cd1 100644 --- a/dashboard/callbacks/indicators.py +++ b/dashboard/callbacks/indicators.py @@ -96,6 +96,7 @@ def register_indicator_callbacks(app): [State('indicator-name-input', 'value'), State('indicator-type-dropdown', 'value'), State('indicator-description-input', 'value'), + State('indicator-timeframe-dropdown', 'value'), State('indicator-color-input', 'value'), State('indicator-line-width-slider', 'value'), # SMA parameters @@ -115,7 +116,7 @@ def register_indicator_callbacks(app): State('edit-indicator-store', 'data')], prevent_initial_call=True ) - def save_new_indicator(n_clicks, name, indicator_type, description, color, line_width, + def save_new_indicator(n_clicks, name, indicator_type, description, timeframe, color, line_width, sma_period, ema_period, rsi_period, macd_fast, macd_slow, macd_signal, bb_period, bb_stddev, edit_data): @@ -161,7 +162,8 @@ def register_indicator_callbacks(app): name=name, description=description or "", parameters=parameters, - styling={'color': color or "#007bff", 'line_width': line_width or 2} + styling={'color': color or "#007bff", 'line_width': line_width or 2}, + timeframe=timeframe or None ) if success: @@ -176,7 +178,8 @@ def register_indicator_callbacks(app): indicator_type=indicator_type, parameters=parameters, description=description or "", - color=color or "#007bff" + color=color or "#007bff", + timeframe=timeframe or None ) if not new_indicator: @@ -384,6 +387,7 @@ def register_indicator_callbacks(app): Output('indicator-name-input', 'value'), Output('indicator-type-dropdown', 'value'), Output('indicator-description-input', 'value'), + Output('indicator-timeframe-dropdown', 'value'), Output('indicator-color-input', 'value'), Output('edit-indicator-store', 'data'), # Add parameter field outputs @@ -403,7 +407,7 @@ def register_indicator_callbacks(app): """Load indicator data for editing.""" ctx = callback_context if not ctx.triggered or not any(edit_clicks): - return no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update + return [no_update] * 15 # Find which button was clicked triggered_id = ctx.triggered[0]['prop_id'] @@ -418,41 +422,42 @@ def register_indicator_callbacks(app): if indicator: # Store indicator ID for update - edit_data = {'indicator_id': indicator_id, 'mode': 'edit', 'open_modal': True} + edit_data = {'indicator_id': indicator_id, 'mode': 'edit'} # Extract parameter values based on indicator type params = indicator.parameters # Default parameter values - sma_period = 20 - ema_period = 12 - rsi_period = 14 - macd_fast = 12 - macd_slow = 26 - macd_signal = 9 - bb_period = 20 - bb_stddev = 2.0 + sma_period = None + ema_period = None + rsi_period = None + macd_fast = None + macd_slow = None + macd_signal = None + bb_period = None + bb_stddev = None # Update with actual saved values if indicator.type == 'sma': - sma_period = params.get('period', 20) + sma_period = params.get('period') elif indicator.type == 'ema': - ema_period = params.get('period', 12) + ema_period = params.get('period') elif indicator.type == 'rsi': - rsi_period = params.get('period', 14) + rsi_period = params.get('period') elif indicator.type == 'macd': - macd_fast = params.get('fast_period', 12) - macd_slow = params.get('slow_period', 26) - macd_signal = params.get('signal_period', 9) + macd_fast = params.get('fast_period') + macd_slow = params.get('slow_period') + macd_signal = params.get('signal_period') elif indicator.type == 'bollinger_bands': - bb_period = params.get('period', 20) - bb_stddev = params.get('std_dev', 2.0) + bb_period = params.get('period') + bb_stddev = params.get('std_dev') return ( - "✏️ Edit Indicator", + f"✏️ Edit Indicator: {indicator.name}", indicator.name, indicator.type, indicator.description, + indicator.timeframe, indicator.styling.color, edit_data, sma_period, @@ -465,17 +470,18 @@ def register_indicator_callbacks(app): bb_stddev ) else: - return no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update + return [no_update] * 15 except Exception as e: logger.error(f"Indicator callback: Error loading indicator for edit: {e}") - return no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update + return [no_update] * 15 # Reset modal form when closed or saved @app.callback( [Output('indicator-name-input', 'value', allow_duplicate=True), Output('indicator-type-dropdown', 'value', allow_duplicate=True), Output('indicator-description-input', 'value', allow_duplicate=True), + Output('indicator-timeframe-dropdown', 'value', allow_duplicate=True), Output('indicator-color-input', 'value', allow_duplicate=True), Output('indicator-line-width-slider', 'value'), Output('modal-title', 'children', allow_duplicate=True), @@ -494,9 +500,7 @@ def register_indicator_callbacks(app): prevent_initial_call=True ) def reset_modal_form(cancel_clicks, save_clicks): - """Reset the modal form when it's closed or saved.""" - if cancel_clicks or save_clicks: - return "", None, "", "#007bff", 2, "📊 Add New Indicator", None, 20, 12, 14, 12, 26, 9, 20, 2.0 - return no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update + """Reset the modal form to its default state.""" + return "", "", "", "", "", 2, "📊 Add New Indicator", None, 20, 12, 14, 12, 26, 9, 20, 2.0 logger.info("Indicator callbacks: registered successfully") \ No newline at end of file diff --git a/dashboard/components/data_analysis.py b/dashboard/components/data_analysis.py index 5f5816b..1ca6e23 100644 --- a/dashboard/components/data_analysis.py +++ b/dashboard/components/data_analysis.py @@ -562,21 +562,28 @@ def get_market_statistics(df: pd.DataFrame, symbol: str, timeframe: str) -> html """ Generate a comprehensive market statistics component from a DataFrame. """ + if df.empty: + return html.Div("No data available for statistics.", className="text-center text-muted") + try: - volume_analyzer = VolumeAnalyzer() + # Get statistics price_analyzer = PriceMovementAnalyzer() + volume_analyzer = VolumeAnalyzer() - volume_stats = volume_analyzer.get_volume_statistics(df) price_stats = price_analyzer.get_price_movement_statistics(df) + volume_stats = volume_analyzer.get_volume_statistics(df) - if 'error' in volume_stats or 'error' in price_stats: - error_msg = volume_stats.get('error') or price_stats.get('error') + # Format key statistics for display + start_date = df.index.min().strftime('%Y-%m-%d %H:%M') + end_date = df.index.max().strftime('%Y-%m-%d %H:%M') + + # Check for errors from analyzers + if 'error' in price_stats or 'error' in volume_stats: + error_msg = price_stats.get('error') or volume_stats.get('error') return html.Div(f"Error generating statistics: {error_msg}", style={'color': 'red'}) # Time range for display - start_date = df['timestamp'].min().strftime('%Y-%m-%d %H:%M') - end_date = df['timestamp'].max().strftime('%Y-%m-%d %H:%M') - days_back = (df['timestamp'].max() - df['timestamp'].min()).days + days_back = (df.index.max() - df.index.min()).days time_status = f"📅 Analysis Range: {start_date} to {end_date} (~{days_back} days)" return html.Div([ diff --git a/dashboard/components/indicator_modal.py b/dashboard/components/indicator_modal.py index 0de8afb..244a93a 100644 --- a/dashboard/components/indicator_modal.py +++ b/dashboard/components/indicator_modal.py @@ -33,6 +33,27 @@ def create_indicator_modal(): placeholder='Select indicator type', ), width=12) ], className="mb-3"), + dbc.Row([ + dbc.Col(dbc.Label("Timeframe (Optional):"), width=12), + dbc.Col(dcc.Dropdown( + id='indicator-timeframe-dropdown', + options=[ + {'label': 'Chart Timeframe', 'value': ''}, + {'label': "1 Second", 'value': '1s'}, + {'label': "5 Seconds", 'value': '5s'}, + {'label': "15 Seconds", 'value': '15s'}, + {'label': "30 Seconds", 'value': '30s'}, + {'label': '1 Minute', 'value': '1m'}, + {'label': '5 Minutes', 'value': '5m'}, + {'label': '15 Minutes', 'value': '15m'}, + {'label': '1 Hour', 'value': '1h'}, + {'label': '4 Hours', 'value': '4h'}, + {'label': '1 Day', 'value': '1d'}, + ], + value='', + placeholder='Defaults to chart timeframe' + ), width=12), + ], className="mb-3"), dbc.Row([ dbc.Col(dbc.Label("Description (Optional):"), width=12), dbc.Col(dcc.Textarea( diff --git a/data/common/indicators.py b/data/common/indicators.py index a09aa57..c482f66 100644 --- a/data/common/indicators.py +++ b/data/common/indicators.py @@ -74,7 +74,7 @@ class TechnicalIndicators: if self.logger: self.logger.info("TechnicalIndicators: Initialized indicator calculator") - def prepare_dataframe(self, candles: List[OHLCVCandle]) -> pd.DataFrame: + def _prepare_dataframe_from_list(self, candles: List[OHLCVCandle]) -> pd.DataFrame: """ Convert OHLCV candles to pandas DataFrame for efficient calculations. @@ -112,20 +112,19 @@ class TechnicalIndicators: return df - def sma(self, candles: List[OHLCVCandle], period: int, + def sma(self, df: pd.DataFrame, period: int, price_column: str = 'close') -> List[IndicatorResult]: """ Calculate Simple Moving Average (SMA). Args: - candles: List of OHLCV candles + df: DataFrame with OHLCV data period: Number of periods for moving average price_column: Price column to use ('open', 'high', 'low', 'close') Returns: List of indicator results with SMA values """ - df = self.prepare_dataframe(candles) if df.empty or len(df) < period: return [] @@ -147,20 +146,19 @@ class TechnicalIndicators: return results - def ema(self, candles: List[OHLCVCandle], period: int, + def ema(self, df: pd.DataFrame, period: int, price_column: str = 'close') -> List[IndicatorResult]: """ Calculate Exponential Moving Average (EMA). Args: - candles: List of OHLCV candles + df: DataFrame with OHLCV data period: Number of periods for moving average price_column: Price column to use ('open', 'high', 'low', 'close') Returns: List of indicator results with EMA values """ - df = self.prepare_dataframe(candles) if df.empty or len(df) < period: return [] @@ -183,20 +181,19 @@ class TechnicalIndicators: return results - def rsi(self, candles: List[OHLCVCandle], period: int = 14, + def rsi(self, df: pd.DataFrame, period: int = 14, price_column: str = 'close') -> List[IndicatorResult]: """ Calculate Relative Strength Index (RSI). Args: - candles: List of OHLCV candles + df: DataFrame with OHLCV data period: Number of periods for RSI calculation (default 14) price_column: Price column to use ('open', 'high', 'low', 'close') Returns: List of indicator results with RSI values """ - df = self.prepare_dataframe(candles) if df.empty or len(df) < period + 1: return [] @@ -234,14 +231,14 @@ class TechnicalIndicators: return results - def macd(self, candles: List[OHLCVCandle], + def macd(self, df: pd.DataFrame, fast_period: int = 12, slow_period: int = 26, signal_period: int = 9, price_column: str = 'close') -> List[IndicatorResult]: """ Calculate Moving Average Convergence Divergence (MACD). Args: - candles: List of OHLCV candles + df: DataFrame with OHLCV data fast_period: Fast EMA period (default 12) slow_period: Slow EMA period (default 26) signal_period: Signal line EMA period (default 9) @@ -250,8 +247,7 @@ class TechnicalIndicators: Returns: List of indicator results with MACD, signal, and histogram values """ - df = self.prepare_dataframe(candles) - if df.empty or len(df) < slow_period + signal_period: + if df.empty or len(df) < slow_period: return [] # Calculate fast and slow EMAs @@ -271,7 +267,7 @@ class TechnicalIndicators: results = [] for i, (timestamp, row) in enumerate(df.iterrows()): # Only return results after minimum period - if i >= slow_period + signal_period - 1: + if i >= slow_period - 1: if not (pd.isna(row['macd']) or pd.isna(row['signal']) or pd.isna(row['histogram'])): result = IndicatorResult( timestamp=timestamp, @@ -293,21 +289,20 @@ class TechnicalIndicators: return results - def bollinger_bands(self, candles: List[OHLCVCandle], period: int = 20, + def bollinger_bands(self, df: pd.DataFrame, period: int = 20, std_dev: float = 2.0, price_column: str = 'close') -> List[IndicatorResult]: """ Calculate Bollinger Bands. Args: - candles: List of OHLCV candles + df: DataFrame with OHLCV data period: Number of periods for moving average (default 20) - std_dev: Number of standard deviations for bands (default 2.0) + std_dev: Number of standard deviations (default 2.0) price_column: Price column to use ('open', 'high', 'low', 'close') Returns: List of indicator results with upper band, middle band (SMA), and lower band """ - df = self.prepare_dataframe(candles) if df.empty or len(df) < period: return [] @@ -417,64 +412,53 @@ class TechnicalIndicators: def calculate(self, indicator_type: str, candles: Union[pd.DataFrame, List[OHLCVCandle]], **kwargs) -> Optional[Dict[str, Any]]: """ - Generic method to calculate any supported indicator by type. + Calculate a single indicator with dynamic dispatch. Args: - indicator_type: The type of indicator to calculate (e.g., 'sma', 'ema'). - candles: The input data, either a DataFrame or a list of OHLCVCandle objects. - **kwargs: Keyword arguments for the specific indicator function. + indicator_type: Name of the indicator (e.g., 'sma', 'ema') + candles: List of OHLCV candles or a pre-prepared DataFrame + **kwargs: Indicator-specific parameters (e.g., period=20) Returns: A dictionary containing the indicator results, or None if the type is unknown. """ - # If input is a DataFrame, convert it to list of OHLCVCandle objects. - # This is a temporary adaptation to the existing methods. - # Future optimization should standardize on DataFrames. - if isinstance(candles, pd.DataFrame): - from .data_types import OHLCVCandle - - # Ensure required columns are present - required_cols = {'open', 'high', 'low', 'close', 'volume'} - if not required_cols.issubset(candles.columns): - if self.logger: - self.logger.error("Indicators: DataFrame missing required columns for OHLCVCandle conversion.") - return None - - symbol = kwargs.get('symbol', 'UNKNOWN') - timeframe = kwargs.get('timeframe', 'UNKNOWN') - - candles_list = [ - OHLCVCandle( - symbol=symbol, - timeframe=timeframe, - start_time=row['timestamp'], - end_time=row['timestamp'], - open=Decimal(str(row['open'])), - high=Decimal(str(row['high'])), - low=Decimal(str(row['low'])), - close=Decimal(str(row['close'])), - volume=Decimal(str(row['volume'])), - trade_count=int(row.get('trade_count', 0)) - ) for _, row in candles.iterrows() - ] - candles = candles_list - + # Get the indicator calculation method indicator_method = getattr(self, indicator_type, None) - if indicator_method and callable(indicator_method): - # We need to construct a proper IndicatorResult object here - # For now, let's adapt to what the methods return - raw_result = indicator_method(candles, **kwargs) + if not indicator_method: + if self.logger: + self.logger.error(f"TechnicalIndicators: Unknown indicator type '{indicator_type}'") + return None + + try: + # Prepare DataFrame if input is a list of candles + if isinstance(candles, list): + df = self._prepare_dataframe_from_list(candles) + elif isinstance(candles, pd.DataFrame): + df = candles + else: + raise TypeError("Input 'candles' must be a list of OHLCVCandle objects or a pandas DataFrame.") + + if df.empty: + return {'data': [], 'metadata': {}} + + # Call the indicator method + raw_result = indicator_method(df, **kwargs) + + # Extract metadata from the first result if available + metadata = raw_result[0].metadata if raw_result else {} # The methods return List[IndicatorResult], let's package that if raw_result: return { - "data": raw_result + "data": raw_result, + "metadata": metadata } return None - if self.logger: - self.logger.warning(f"TechnicalIndicators: Unknown indicator type '{indicator_type}'") - return None + except Exception as e: + if self.logger: + self.logger.error(f"TechnicalIndicators: Error calculating {indicator_type}: {e}") + return None def create_default_indicators_config() -> Dict[str, Dict[str, Any]]: diff --git a/docs/components/technical-indicators.md b/docs/components/technical-indicators.md index 818cc49..2aaadf3 100644 --- a/docs/components/technical-indicators.md +++ b/docs/components/technical-indicators.md @@ -1,175 +1,123 @@ # Technical Indicators Module -The Technical Indicators module provides comprehensive technical analysis capabilities for the TCP Trading Platform. It's designed to handle sparse OHLCV data efficiently and integrates seamlessly with the platform's aggregation strategy. +The Technical Indicators module provides a suite of common technical analysis tools. It is designed to work efficiently with pandas DataFrames, which is the standard data structure for time-series analysis in the TCP Trading Platform. ## Overview -The module implements five core technical indicators commonly used in trading: +The module has been refactored to be **DataFrame-centric**. All calculation methods now expect a pandas DataFrame with a `DatetimeIndex` and the required OHLCV columns (`open`, `high`, `low`, `close`, `volume`). This change simplifies the data pipeline, improves performance through vectorization, and ensures consistency across the platform. -- **Simple Moving Average (SMA)** - Average price over a specified period -- **Exponential Moving Average (EMA)** - Weighted average giving more importance to recent prices -- **Relative Strength Index (RSI)** - Momentum oscillator measuring speed and change of price movements -- **Moving Average Convergence Divergence (MACD)** - Trend-following momentum indicator -- **Bollinger Bands** - Volatility indicator with upper and lower bands around a moving average +The module implements five core technical indicators: + +- **Simple Moving Average (SMA)** +- **Exponential Moving Average (EMA)** +- **Relative Strength Index (RSI)** +- **Moving Average Convergence Divergence (MACD)** +- **Bollinger Bands** ## Key Features -### Sparse Data Handling -- **No Interpolation**: Preserves gaps in timestamp data without artificial interpolation -- **Efficient Processing**: Uses pandas for vectorized calculations -- **Right-Aligned Timestamps**: Follows the platform's aggregation strategy convention -- **Robust Error Handling**: Gracefully handles insufficient data and edge cases - -### Performance Optimized -- **Vectorized Calculations**: Leverages pandas and numpy for fast computation -- **Batch Processing**: Calculate multiple indicators simultaneously -- **Memory Efficient**: Processes data in chunks without excessive memory usage - -### Flexible Configuration -- **JSON Configuration**: Define indicator parameters via configuration files -- **Multiple Price Columns**: Calculate indicators on open, high, low, or close prices -- **Custom Parameters**: Adjust periods, standard deviations, and other parameters -- **Validation**: Built-in configuration validation +- **DataFrame-Centric Design**: Operates directly on pandas DataFrames for performance and simplicity. +- **Vectorized Calculations**: Leverages pandas and numpy for high-speed computation. +- **Flexible `calculate` Method**: A single entry point for calculating any supported indicator by name. +- **Standardized Output**: All methods return a DataFrame containing the calculated indicator values, indexed by timestamp. ## Usage Examples -### Basic Usage +### Preparing the DataFrame + +Before you can calculate indicators, you need a properly formatted pandas DataFrame. The `prepare_chart_data` utility is the recommended way to create one from a list of candle dictionaries. ```python +from components.charts.utils import prepare_chart_data from data.common.indicators import TechnicalIndicators -from data.common.data_types import OHLCVCandle -# Initialize indicators calculator -indicators = TechnicalIndicators() +# Assume 'candles' is a list of OHLCV dictionaries from the database +# candles = fetch_market_data(...) -# Calculate Simple Moving Average -sma_results = indicators.sma(candles, period=20) +# Prepare the DataFrame +df = prepare_chart_data(candles) -# Calculate Exponential Moving Average -ema_results = indicators.ema(candles, period=12) - -# Calculate RSI -rsi_results = indicators.rsi(candles, period=14) - -# Calculate MACD -macd_results = indicators.macd(candles, fast_period=12, slow_period=26, signal_period=9) - -# Calculate Bollinger Bands -bb_results = indicators.bollinger_bands(candles, period=20, std_dev=2.0) +# df is now ready for indicator calculations +# It has a DatetimeIndex and the necessary OHLCV columns. ``` -### Multiple Indicators +### Basic Indicator Calculation + +Once you have a prepared DataFrame, you can calculate indicators directly. ```python -# Define configuration for multiple indicators -config = { - 'sma_20': {'type': 'sma', 'period': 20}, - 'sma_50': {'type': 'sma', 'period': 50}, - 'ema_12': {'type': 'ema', 'period': 12}, - 'rsi_14': {'type': 'rsi', 'period': 14}, - 'macd': {'type': 'macd'}, - 'bb_20': {'type': 'bollinger_bands', 'period': 20} -} +# Initialize the calculator +indicators = TechnicalIndicators() -# Calculate all indicators at once -results = indicators.calculate_multiple_indicators(candles, config) +# Calculate a Simple Moving Average +sma_df = indicators.sma(df, period=20) -# Access individual indicator results -sma_20_values = results['sma_20'] -rsi_values = results['rsi_14'] -macd_values = results['macd'] +# Calculate an Exponential Moving Average +ema_df = indicators.ema(df, period=12) + +# sma_df and ema_df are pandas DataFrames containing the results. +``` + +### Using the `calculate` Method + +The most flexible way to compute an indicator is with the `calculate` method, which accepts the indicator type as a string. + +```python +# Calculate RSI using the generic method +rsi_pkg = indicators.calculate('rsi', df, period=14) +if rsi_pkg: + rsi_df = rsi_pkg['data'] + +# Calculate MACD with custom parameters +macd_pkg = indicators.calculate('macd', df, fast_period=10, slow_period=30, signal_period=8) +if macd_pkg: + macd_df = macd_pkg['data'] ``` ### Using Different Price Columns -```python -# Calculate SMA on high prices instead of close -sma_high = indicators.sma(candles, period=20, price_column='high') - -# Calculate EMA on low prices -ema_low = indicators.ema(candles, period=12, price_column='low') - -# Calculate RSI on open prices -rsi_open = indicators.rsi(candles, period=14, price_column='open') -``` - -### Default Configuration +You can specify which price column (`open`, `high`, `low`, or `close`) to use for the calculation. ```python -from data.common.indicators import create_default_indicators_config +# Calculate SMA on the 'high' price +sma_high_df = indicators.sma(df, period=20, price_column='high') -# Get default configuration -default_config = create_default_indicators_config() - -# Calculate using defaults -results = indicators.calculate_multiple_indicators(candles, default_config) +# Calculate RSI on the 'open' price +rsi_open_pkg = indicators.calculate('rsi', df, period=14, price_column='open') ``` ## Indicator Details +The following details the parameters and the columns returned in the result DataFrame for each indicator. + ### Simple Moving Average (SMA) -Calculates the arithmetic mean of prices over a specified period. - -**Parameters:** -- `period`: Number of periods (default: 20) -- `price_column`: Price column to use (default: 'close') - -**Returns:** -- `sma`: Simple moving average value +- **Parameters**: `period` (int), `price_column` (str, default: 'close') +- **Returned Columns**: `sma` ### Exponential Moving Average (EMA) -Calculates exponentially weighted moving average, giving more weight to recent prices. - -**Parameters:** -- `period`: Number of periods (default: 20) -- `price_column`: Price column to use (default: 'close') - -**Returns:** -- `ema`: Exponential moving average value +- **Parameters**: `period` (int), `price_column` (str, default: 'close') +- **Returned Columns**: `ema` ### Relative Strength Index (RSI) -Momentum oscillator that measures the speed and change of price movements. - -**Parameters:** -- `period`: Number of periods (default: 14) -- `price_column`: Price column to use (default: 'close') - -**Returns:** -- `rsi`: RSI value (0-100 range) +- **Parameters**: `period` (int), `price_column` (str, default: 'close') +- **Returned Columns**: `rsi` ### MACD (Moving Average Convergence Divergence) -Trend-following momentum indicator showing the relationship between two moving averages. - -**Parameters:** -- `fast_period`: Fast EMA period (default: 12) -- `slow_period`: Slow EMA period (default: 26) -- `signal_period`: Signal line EMA period (default: 9) -- `price_column`: Price column to use (default: 'close') - -**Returns:** -- `macd`: MACD line (fast EMA - slow EMA) -- `signal`: Signal line (EMA of MACD) -- `histogram`: MACD histogram (MACD - Signal) +- **Parameters**: `fast_period` (int), `slow_period` (int), `signal_period` (int), `price_column` (str, default: 'close') +- **Returned Columns**: `macd`, `signal`, `histogram` ### Bollinger Bands -Volatility indicator consisting of a moving average and two standard deviation bands. +- **Parameters**: `period` (int), `std_dev` (float), `price_column` (str, default: 'close') +- **Returned Columns**: `upper_band`, `middle_band`, `lower_band` -**Parameters:** -- `period`: Number of periods for moving average (default: 20) -- `std_dev`: Number of standard deviations (default: 2.0) -- `price_column`: Price column to use (default: 'close') +## Integration with the TCP Platform -**Returns:** -- `upper_band`: Upper Bollinger Band -- `middle_band`: Middle band (SMA) -- `lower_band`: Lower Bollinger Band -- `bandwidth`: Band width relative to middle band -- `percent_b`: %B indicator (position within bands) +The refactored `TechnicalIndicators` module is now tightly integrated with the `ChartBuilder`, which handles all data preparation and calculation automatically when indicators are added to a chart. For custom analysis or strategy development, you can use the class directly as shown in the examples above. The key is to always start with a properly prepared DataFrame using `prepare_chart_data`. ## Data Structures diff --git a/tasks/tasks-indicator-timeframe-feature.md b/tasks/tasks-indicator-timeframe-feature.md new file mode 100644 index 0000000..85808e3 --- /dev/null +++ b/tasks/tasks-indicator-timeframe-feature.md @@ -0,0 +1,38 @@ +## Relevant Files + +- `config/indicators/templates/*.json` - Indicator configuration templates to be updated with the new `timeframe` field. +- `components/charts/indicator_manager.py` - To add `timeframe` to the `UserIndicator` dataclass and related methods. +- `dashboard/layouts/market_data.py` - To add UI elements for selecting the indicator timeframe. +- `dashboard/callbacks/indicators.py` - To handle the new `timeframe` input from the UI. +- `components/charts/data_integration.py` - To implement the core logic for fetching data and calculating indicators on different timeframes. +- `components/charts/builder.py` - To ensure the new indicator data is correctly passed to the chart. + +### Notes + +- The core of the changes will be in `components/charts/data_integration.py`. +- Careful data alignment (reindexing and forward-filling) will be crucial for correct visualization. + +## Tasks + +- [x] 1.0 Update Indicator Configuration + - [x] 1.1 Add an optional `timeframe` field to all JSON templates in `config/indicators/templates/`. + - [x] 1.2 Update the `UserIndicator` dataclass in `components/charts/indicator_manager.py` to include `timeframe: Optional[str]`. + - [x] 1.3 Modify `create_indicator` in `IndicatorManager` to accept a `timeframe` parameter. + - [x] 1.4 Update `UserIndicator.from_dict` and `to_dict` to handle the new `timeframe` field. +- [x] 2.0 Implement Multi-Timeframe Data Fetching and Calculation + - [x] 2.1 In `components/charts/data_integration.py`, modify `get_indicator_data`. + - [x] 2.2 If a custom timeframe is present, call `get_market_data_for_indicators` to fetch new data. + - [x] 2.3 If no custom timeframe is set, use the existing `main_df`. + - [x] 2.4 Pass the correct DataFrame to `self.indicators.calculate`. +- [x] 3.0 Align and Merge Indicator Data for Plotting + - [x] 3.1 After calculation, reindex the indicator DataFrame to match the `main_df`'s timestamp index. + - [x] 3.2 Use forward-fill (`ffill`) to handle missing values from reindexing. + - [x] 3.3 Add the aligned data to `indicator_data_map`. +- [x] 4.0 Update UI for Indicator Timeframe Selection + - [x] 4.1 In `dashboard/layouts/market_data.py`, add a `dcc.Dropdown` for timeframe selection in the indicator modal. + - [x] 4.2 In `dashboard/callbacks/indicators.py`, update the save indicator callback to read the timeframe value. + - [x] 4.3 Pass the selected timeframe to `indicator_manager.create_indicator` or `update_indicator`. +- [ ] 5.0 Testing and Validation + - [ ] 5.1 Write unit tests for custom timeframe data fetching and alignment. + - [ ] 5.2 Manually test creating and viewing indicators with various timeframes (higher, lower, and same as chart). + - [ ] 5.3 Verify visual correctness and data integrity on the chart. \ No newline at end of file diff --git a/tasks/tasks-refactor-indicator-calculation.md b/tasks/tasks-refactor-indicator-calculation.md new file mode 100644 index 0000000..e1a0026 --- /dev/null +++ b/tasks/tasks-refactor-indicator-calculation.md @@ -0,0 +1,36 @@ +## Relevant Files + +- `data/common/indicators.py` - This is the primary file to be refactored. The `TechnicalIndicators` class will be modified to be DataFrame-centric. +- `components/charts/utils.py` - The `prepare_chart_data` function in this file needs to be corrected to ensure it properly creates and returns a DataFrame with a `DatetimeIndex`. +- `components/charts/data_integration.py` - This file's `get_indicator_data` method will be simplified to pass the correctly prepared DataFrame to the calculation engine. +- `app_new.py` - The main application file, which will be used to run the dashboard and perform end-to-end testing. + +### Notes + +- The goal of this refactoring is to create a more robust and maintainable data pipeline for indicator calculations, preventing recurring data type and index errors. +- Pay close attention to ensuring that DataFrames have a consistent `DatetimeIndex` with proper timezone information throughout the pipeline. + +## Tasks + +- [x] 1.0 Refactor `TechnicalIndicators` Class in `data/common/indicators.py` to be DataFrame-centric. + - [x] 1.1 Modify `sma`, `ema`, `rsi`, `macd`, and `bollinger_bands` methods to accept a pre-formatted DataFrame as their primary input, not a list of candles. + - [x] 1.2 Remove the redundant `prepare_dataframe` call from within each individual indicator method. + - [x] 1.3 Rename `prepare_dataframe` to `_prepare_dataframe_from_list` to signify its new role as a private helper for converting list-based data. + - [x] 1.4 Update the main `calculate` method to be the single point of data preparation, handling both DataFrame and list inputs. + +- [x] 2.0 Correct DataFrame Preparation in `components/charts/utils.py`. + - [x] 2.1 Review the `prepare_chart_data` function to identify why the `DatetimeIndex` is being dropped. + - [x] 2.2 Modify the function to ensure it returns a DataFrame with the `timestamp` column correctly set as the index, without a `reset_index()` call at the end. + +- [x] 3.0 Simplify Data Flow in `components/charts/data_integration.py`. + - [x] 3.1 In the `get_indicator_data` function, remove the workaround that converts the DataFrame to a list of dictionaries (`to_dict('records')`). + - [x] 3.2 Ensure the function correctly handles both main and custom timeframes, passing the appropriate DataFrame to the calculation engine. + - [x] 3.3 Verify that the final `reindex` operation works correctly with the consistent DataFrame structure. + +- [x] 4.0 End-to-End Testing and Validation. + - [x] 4.1 Run the dashboard and test the indicator plotting functionality with both matching and custom timeframes. + - [x] 4.2 Verify that no new errors appear in the console during chart interaction. +- [x] 5.0 Update Indicators documentation to reflect the new DataFrame-centric approach. + - [x] 5.1 Review the documentation in the `/docs` directory related to indicators. + - [x] 5.2 Update the documentation to explain that the calculation engine now uses DataFrames. + - [x] 5.3 Provide clear examples of how to use the refactored `TechnicalIndicators` class. \ No newline at end of file