Add complete time series aggregation example and refactor OKXCollector for repository pattern
- Introduced `example_complete_series_aggregation.py` to demonstrate time series aggregation, emitting candles even when no trades occur. - Implemented `CompleteSeriesProcessor` extending `RealTimeCandleProcessor` to handle time-based candle emission and empty candle creation. - Refactored `OKXCollector` to utilize the new repository pattern for database operations, enhancing modularity and maintainability. - Updated database operations to centralize data handling through `DatabaseOperations`, improving error handling and logging. - Enhanced documentation to include details on the new aggregation example and repository pattern implementation, ensuring clarity for users.
This commit is contained in:
Vasily.onl
@@ -21,7 +21,7 @@ from .websocket import (
|
||||
ConnectionState, OKXWebSocketError
|
||||
)
|
||||
from .data_processor import OKXDataProcessor
|
||||
from database.connection import get_db_manager, get_raw_data_manager
|
||||
from database.operations import get_database_operations, DatabaseOperationError
|
||||
from database.models import MarketData, RawTrade
|
||||
|
||||
|
||||
@@ -104,9 +104,8 @@ class OKXCollector(BaseDataCollector):
|
||||
self._data_processor.add_trade_callback(self._on_trade_processed)
|
||||
self._data_processor.add_candle_callback(self._on_candle_processed)
|
||||
|
||||
# Database managers
|
||||
self._db_manager = None
|
||||
self._raw_data_manager = None
|
||||
# Database operations using new repository pattern
|
||||
self._db_operations = None
|
||||
|
||||
# Data processing counters
|
||||
self._message_count = 0
|
||||
@@ -136,10 +135,8 @@ class OKXCollector(BaseDataCollector):
|
||||
if self.logger:
|
||||
self.logger.info(f"{self.component_name}: Connecting OKX collector for {self.symbol}")
|
||||
|
||||
# Initialize database managers
|
||||
self._db_manager = get_db_manager()
|
||||
if self.store_raw_data:
|
||||
self._raw_data_manager = get_raw_data_manager()
|
||||
# Initialize database operations using repository pattern
|
||||
self._db_operations = get_database_operations(self.logger)
|
||||
|
||||
# Create WebSocket client
|
||||
ws_component_name = f"okx_ws_{self.symbol.replace('-', '_').lower()}"
|
||||
@@ -370,22 +367,17 @@ class OKXCollector(BaseDataCollector):
|
||||
data_point: Raw market data point (trade, orderbook, ticker)
|
||||
"""
|
||||
try:
|
||||
if not self._db_manager:
|
||||
if not self._db_operations:
|
||||
return
|
||||
|
||||
# Store raw market data points in raw_trades table
|
||||
with self._db_manager.get_session() as session:
|
||||
raw_trade = RawTrade(
|
||||
exchange="okx",
|
||||
symbol=data_point.symbol,
|
||||
timestamp=data_point.timestamp,
|
||||
data_type=data_point.data_type.value,
|
||||
raw_data=data_point.data
|
||||
)
|
||||
session.add(raw_trade)
|
||||
if self.logger:
|
||||
self.logger.debug(f"{self.component_name}: Stored raw data: {data_point.data_type.value} for {data_point.symbol}")
|
||||
# Store raw market data points in raw_trades table using repository
|
||||
success = self._db_operations.raw_trades.insert_market_data_point(data_point)
|
||||
if success and self.logger:
|
||||
self.logger.debug(f"{self.component_name}: Stored raw data: {data_point.data_type.value} for {data_point.symbol}")
|
||||
|
||||
except DatabaseOperationError as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"{self.component_name}: Database error storing raw market data: {e}")
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"{self.component_name}: Error storing raw market data: {e}")
|
||||
@@ -402,70 +394,22 @@ class OKXCollector(BaseDataCollector):
|
||||
candle: Completed OHLCV candle
|
||||
"""
|
||||
try:
|
||||
if not self._db_manager:
|
||||
if not self._db_operations:
|
||||
return
|
||||
|
||||
# Use right-aligned timestamp (end_time) following industry standard
|
||||
candle_timestamp = candle.end_time
|
||||
# Store completed candles using repository pattern
|
||||
success = self._db_operations.market_data.upsert_candle(candle, self.force_update_candles)
|
||||
|
||||
# Store completed candles in market_data table with configurable duplicate handling
|
||||
with self._db_manager.get_session() as session:
|
||||
if self.force_update_candles:
|
||||
# Force update: Overwrite existing candles with new data
|
||||
upsert_query = """
|
||||
INSERT INTO market_data (
|
||||
exchange, symbol, timeframe, timestamp,
|
||||
open, high, low, close, volume, trades_count,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
:exchange, :symbol, :timeframe, :timestamp,
|
||||
:open, :high, :low, :close, :volume, :trades_count,
|
||||
NOW(), NOW()
|
||||
)
|
||||
ON CONFLICT (exchange, symbol, timeframe, timestamp)
|
||||
DO UPDATE SET
|
||||
open = EXCLUDED.open,
|
||||
high = EXCLUDED.high,
|
||||
low = EXCLUDED.low,
|
||||
close = EXCLUDED.close,
|
||||
volume = EXCLUDED.volume,
|
||||
trades_count = EXCLUDED.trades_count,
|
||||
updated_at = NOW()
|
||||
"""
|
||||
action_type = "Updated"
|
||||
else:
|
||||
# Keep existing: Ignore duplicates, preserve first candle
|
||||
upsert_query = """
|
||||
INSERT INTO market_data (
|
||||
exchange, symbol, timeframe, timestamp,
|
||||
open, high, low, close, volume, trades_count,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
:exchange, :symbol, :timeframe, :timestamp,
|
||||
:open, :high, :low, :close, :volume, :trades_count,
|
||||
NOW(), NOW()
|
||||
)
|
||||
ON CONFLICT (exchange, symbol, timeframe, timestamp)
|
||||
DO NOTHING
|
||||
"""
|
||||
action_type = "Stored"
|
||||
|
||||
session.execute(upsert_query, {
|
||||
'exchange': candle.exchange,
|
||||
'symbol': candle.symbol,
|
||||
'timeframe': candle.timeframe,
|
||||
'timestamp': candle_timestamp,
|
||||
'open': float(candle.open),
|
||||
'high': float(candle.high),
|
||||
'low': float(candle.low),
|
||||
'close': float(candle.close),
|
||||
'volume': float(candle.volume),
|
||||
'trades_count': candle.trade_count
|
||||
})
|
||||
|
||||
if self.logger:
|
||||
self.logger.info(f"{self.component_name}: {action_type} candle: {candle.symbol} {candle.timeframe} at {candle_timestamp} (force_update={self.force_update_candles}) - OHLCV: {candle.open}/{candle.high}/{candle.low}/{candle.close}, Vol: {candle.volume}, Trades: {candle.trade_count}")
|
||||
if success and self.logger:
|
||||
action = "Updated" if self.force_update_candles else "Stored"
|
||||
self.logger.info(f"{self.component_name}: {action} candle: {candle.symbol} {candle.timeframe} at {candle.end_time} (force_update={self.force_update_candles}) - OHLCV: {candle.open}/{candle.high}/{candle.low}/{candle.close}, Vol: {candle.volume}, Trades: {candle.trade_count}")
|
||||
|
||||
except DatabaseOperationError as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"{self.component_name}: Database error storing completed candle: {e}")
|
||||
# Log candle details for debugging
|
||||
self.logger.error(f"{self.component_name}: Failed candle details: {candle.symbol} {candle.timeframe} {candle.end_time} - OHLCV: {candle.open}/{candle.high}/{candle.low}/{candle.close}")
|
||||
self._error_count += 1
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"{self.component_name}: Error storing completed candle: {e}")
|
||||
@@ -482,19 +426,24 @@ class OKXCollector(BaseDataCollector):
|
||||
raw_message: Raw WebSocket message
|
||||
"""
|
||||
try:
|
||||
if not self._raw_data_manager or 'data' not in raw_message:
|
||||
if not self._db_operations or 'data' not in raw_message:
|
||||
return
|
||||
|
||||
# Store each data item as a separate raw data record
|
||||
# Store each data item as a separate raw data record using repository
|
||||
for data_item in raw_message['data']:
|
||||
self._raw_data_manager.store_raw_data(
|
||||
success = self._db_operations.raw_trades.insert_raw_websocket_data(
|
||||
exchange="okx",
|
||||
symbol=self.symbol,
|
||||
data_type=f"raw_{channel}", # Prefix with 'raw_' to distinguish from processed data
|
||||
raw_data=data_item,
|
||||
timestamp=datetime.now(timezone.utc)
|
||||
)
|
||||
if not success and self.logger:
|
||||
self.logger.warning(f"{self.component_name}: Failed to store raw WebSocket data for {channel}")
|
||||
|
||||
except DatabaseOperationError as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"{self.component_name}: Database error storing raw WebSocket data: {e}")
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"{self.component_name}: Error storing raw WebSocket data: {e}")
|
||||
|
||||
Reference in New Issue
Block a user