f6cb1485b1
- Introduced a comprehensive data collection framework, including `CollectorServiceConfig`, `BaseDataCollector`, and `CollectorManager`, enhancing modularity and maintainability. - Developed `CollectorFactory` for streamlined collector creation, promoting separation of concerns and improved configuration handling. - Enhanced `DataCollectionService` to utilize the new architecture, ensuring robust error handling and logging practices. - Added `TaskManager` for efficient management of asynchronous tasks, improving performance and resource management. - Implemented health monitoring and auto-recovery features in `CollectorManager`, ensuring reliable operation of data collectors. - Updated imports across the codebase to reflect the new structure, ensuring consistent access to components. These changes significantly improve the architecture and maintainability of the data collection service, aligning with project standards for modularity, performance, and error handling.
140 lines
4.0 KiB
Python
140 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Start Data Collection Service
|
|
|
|
Simple script to start the cryptocurrency data collection service
|
|
with clean console output and proper configuration.
|
|
|
|
Usage:
|
|
python scripts/start_data_collection.py [options]
|
|
|
|
Examples:
|
|
# Start with default configuration (indefinite run)
|
|
python scripts/start_data_collection.py
|
|
|
|
# Run for 8 hours with default config
|
|
python scripts/start_data_collection.py --hours 8
|
|
|
|
# Use custom configuration file
|
|
python scripts/start_data_collection.py --config config/my_config.json
|
|
|
|
# Run for 24 hours with custom config
|
|
python scripts/start_data_collection.py --config config/production.json --hours 24
|
|
"""
|
|
|
|
import asyncio
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add project root to path
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
from data.collector.collection_service import run_data_collection_service
|
|
|
|
|
|
def display_banner(config_path: str, duration_hours: float = None):
|
|
"""Display service startup banner."""
|
|
print("🚀 CRYPTOCURRENCY DATA COLLECTION SERVICE")
|
|
print("=" * 55)
|
|
print(f"📁 Configuration: {config_path}")
|
|
|
|
if duration_hours:
|
|
print(f"⏱️ Duration: {duration_hours} hours")
|
|
else:
|
|
print("⏱️ Duration: Indefinite (until stopped)")
|
|
|
|
print("📊 Logging: Essential events only (connections, errors)")
|
|
print("💾 Storage: PostgreSQL + Redis")
|
|
print("🔍 Monitor: python scripts/monitor_clean.py")
|
|
print("⏹️ Stop: Ctrl+C")
|
|
print("=" * 55)
|
|
|
|
|
|
def main():
|
|
"""Main entry point."""
|
|
parser = argparse.ArgumentParser(
|
|
description="Start Cryptocurrency Data Collection Service",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Start with default configuration (indefinite)
|
|
python scripts/start_data_collection.py
|
|
|
|
# Run for 8 hours
|
|
python scripts/start_data_collection.py --hours 8
|
|
|
|
# Use custom configuration
|
|
python scripts/start_data_collection.py --config config/custom.json
|
|
|
|
# Production run for 24 hours
|
|
python scripts/start_data_collection.py --config config/production.json --hours 24
|
|
|
|
Configuration:
|
|
The service will create a default configuration file if none exists.
|
|
Default location: config/data_collection.json
|
|
|
|
The configuration includes:
|
|
- Exchange settings (OKX by default)
|
|
- Trading pairs (BTC-USDT, ETH-USDT by default)
|
|
- Data types and timeframes
|
|
- Health monitoring settings
|
|
"""
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--config',
|
|
default="config/data_collection.json",
|
|
help='Configuration file path (default: config/data_collection.json)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--hours',
|
|
type=float,
|
|
help='Collection duration in hours (default: indefinite until Ctrl+C)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--quiet',
|
|
action='store_true',
|
|
help='Suppress banner and start directly'
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Validate arguments
|
|
if args.hours is not None and args.hours <= 0:
|
|
print("❌ Duration must be positive")
|
|
sys.exit(1)
|
|
|
|
# Display banner unless quiet mode
|
|
if not args.quiet:
|
|
display_banner(args.config, args.hours)
|
|
|
|
try:
|
|
# Start the service
|
|
print("🎯 Starting service..." if not args.quiet else "")
|
|
|
|
success = asyncio.run(run_data_collection_service(
|
|
config_path=args.config,
|
|
duration_hours=args.hours
|
|
))
|
|
|
|
if success:
|
|
print("✅ Service completed successfully" if not args.quiet else "")
|
|
sys.exit(0)
|
|
else:
|
|
print("❌ Service failed" if not args.quiet else "")
|
|
sys.exit(1)
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n👋 Service interrupted by user")
|
|
sys.exit(0)
|
|
except Exception as e:
|
|
print(f"❌ Fatal error: {e}")
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |