Add complete time series aggregation example and refactor OKXCollector for repository pattern

- Introduced `example_complete_series_aggregation.py` to demonstrate time series aggregation, emitting candles even when no trades occur.
- Implemented `CompleteSeriesProcessor` extending `RealTimeCandleProcessor` to handle time-based candle emission and empty candle creation.
- Refactored `OKXCollector` to utilize the new repository pattern for database operations, enhancing modularity and maintainability.
- Updated database operations to centralize data handling through `DatabaseOperations`, improving error handling and logging.
- Enhanced documentation to include details on the new aggregation example and repository pattern implementation, ensuring clarity for users.
This commit is contained in:
Vasily.onl
2025-06-02 13:27:01 +08:00
parent 5b4547edd5
commit cffc54b648
11 changed files with 1460 additions and 149 deletions

View File

@@ -14,7 +14,7 @@ from datetime import datetime, timezone
from data.exchanges.okx import OKXCollector
from data.base_collector import DataType
from database.connection import DatabaseConnection
from database.operations import get_database_operations
from utils.logger import get_logger
# Global test state
@@ -36,12 +36,15 @@ signal.signal(signal.SIGTERM, signal_handler)
async def check_database_connection():
"""Check if database connection is available."""
try:
db_manager = DatabaseConnection()
# Test connection
with db_manager.get_session() as session:
session.execute("SELECT 1")
print("✅ Database connection successful")
return True
db_operations = get_database_operations()
# Test connection using the new repository pattern
is_healthy = db_operations.health_check()
if is_healthy:
print("✅ Database connection successful")
return True
else:
print("❌ Database health check failed")
return False
except Exception as e:
print(f"❌ Database connection failed: {e}")
print(" Make sure your database is running and configured correctly")
@@ -49,18 +52,22 @@ async def check_database_connection():
async def count_stored_data():
"""Count raw trades and candles in database."""
"""Count raw trades and candles in database using repository pattern."""
try:
db_manager = DatabaseConnection()
with db_manager.get_session() as session:
# Count raw trades
raw_count = session.execute("SELECT COUNT(*) FROM raw_trades WHERE exchange = 'okx'").scalar()
# Count market data candles
candle_count = session.execute("SELECT COUNT(*) FROM market_data WHERE exchange = 'okx'").scalar()
print(f"📊 Database counts: Raw trades: {raw_count}, Candles: {candle_count}")
return raw_count, candle_count
db_operations = get_database_operations()
# Get database statistics using the new operations module
stats = db_operations.get_stats()
if 'error' in stats:
print(f"❌ Error getting database stats: {stats['error']}")
return 0, 0
raw_count = stats.get('raw_trade_count', 0)
candle_count = stats.get('candle_count', 0)
print(f"📊 Database counts: Raw trades: {raw_count}, Candles: {candle_count}")
return raw_count, candle_count
except Exception as e:
print(f"❌ Error counting database records: {e}")
return 0, 0