2890ba2efa
- Introduced `service_config.py` to manage configuration loading, validation, and schema management, enhancing modularity and security. - Created a `ServiceConfig` class for handling configuration with robust error handling and default values. - Refactored `DataCollectionService` to utilize the new `ServiceConfig`, streamlining configuration management and improving readability. - Added a `CollectorFactory` to encapsulate collector creation logic, promoting separation of concerns. - Updated `CollectorManager` and related components to align with the new architecture, ensuring better maintainability. - Enhanced logging practices across the service for improved monitoring and debugging. These changes significantly improve the architecture and maintainability of the data collection service, aligning with project standards for modularity and performance.
119 lines
3.6 KiB
Python
119 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Clean Production OKX Data Collector
|
|
|
|
Simplified production script using the new DataCollectionService architecture.
|
|
Provides clean console output with minimal logging for production environments.
|
|
"""
|
|
|
|
import asyncio
|
|
import signal
|
|
import sys
|
|
import time
|
|
import json
|
|
from typing import Optional
|
|
from pathlib import Path
|
|
|
|
# Add project root to path
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
# Set environment for clean production logging
|
|
import os
|
|
os.environ['DEBUG'] = 'false'
|
|
|
|
# Suppress verbose SQLAlchemy logging
|
|
import logging
|
|
logging.getLogger('sqlalchemy').setLevel(logging.CRITICAL)
|
|
logging.getLogger('sqlalchemy.engine').setLevel(logging.CRITICAL)
|
|
logging.getLogger('sqlalchemy.pool').setLevel(logging.CRITICAL)
|
|
logging.getLogger('sqlalchemy.dialects').setLevel(logging.CRITICAL)
|
|
logging.getLogger('sqlalchemy.orm').setLevel(logging.CRITICAL)
|
|
|
|
from data.collection_service import run_data_collection_service
|
|
|
|
|
|
async def get_config_timeframes(config_path: str) -> str:
|
|
"""Get timeframes from configuration for display."""
|
|
try:
|
|
with open(config_path, 'r') as f:
|
|
config = json.load(f)
|
|
# Get unique timeframes from all enabled trading pairs
|
|
all_timeframes = set()
|
|
for pair in config.get('trading_pairs', []):
|
|
if pair.get('enabled', True):
|
|
all_timeframes.update(pair.get('timeframes', ['1m', '5m']))
|
|
return ', '.join(sorted(all_timeframes))
|
|
except:
|
|
return "configured timeframes"
|
|
|
|
|
|
async def run_clean_production(duration_hours: Optional[float] = None) -> bool:
|
|
"""Run production collector with clean output."""
|
|
|
|
# Configuration path - use the new service config format
|
|
config_path = "config/data_collection.json"
|
|
|
|
try:
|
|
# Get timeframes for display
|
|
timeframes_str = await get_config_timeframes(config_path)
|
|
|
|
# Header
|
|
print("OKX PRODUCTION DATA COLLECTOR")
|
|
print("="*50)
|
|
if duration_hours:
|
|
print(f"Duration: {duration_hours} hours")
|
|
else:
|
|
print(f"Duration: Indefinite (until stopped)")
|
|
print(f"Timeframes: {timeframes_str}")
|
|
print(f"Database: Raw trades + aggregated candles")
|
|
print(f"Logs: logs/ directory")
|
|
print("="*50)
|
|
|
|
# Start data collection using the new service
|
|
print("Starting data collection service...")
|
|
success = await run_data_collection_service(config_path, duration_hours)
|
|
|
|
if success:
|
|
print("Data collection completed successfully")
|
|
return True
|
|
else:
|
|
print("Data collection failed")
|
|
return False
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
return False
|
|
|
|
|
|
def main():
|
|
"""Main entry point."""
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Clean Production OKX Data Collector")
|
|
parser.add_argument(
|
|
"--hours",
|
|
type=float,
|
|
help="Collection duration in hours (default: indefinite until stopped manually)"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Validate arguments
|
|
if args.hours is not None and args.hours <= 0:
|
|
print("Duration must be positive")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
success = asyncio.run(run_clean_production(args.hours))
|
|
sys.exit(0 if success else 1)
|
|
except KeyboardInterrupt:
|
|
print("\nInterrupted by user")
|
|
sys.exit(0)
|
|
except Exception as e:
|
|
print(f"Fatal error: {e}")
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |