Remove complete time series aggregation example and add data collection service implementation
- Deleted `example_complete_series_aggregation.py` as it is no longer needed. - Introduced `data_collection_service.py`, a production-ready service for cryptocurrency market data collection with clean logging and robust error handling. - Added configuration management for multiple trading pairs and exchanges, supporting health monitoring and graceful shutdown. - Created `data_collection.json` for service configuration, including exchange settings and logging preferences. - Updated `CandleProcessingConfig` to reflect changes in timeframes for candle processing. - Enhanced documentation to cover the new data collection service and its configuration, ensuring clarity for users.
This commit is contained in:
140
scripts/start_data_collection.py
Normal file
140
scripts/start_data_collection.py
Normal file
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Start Data Collection Service
|
||||
|
||||
Simple script to start the cryptocurrency data collection service
|
||||
with clean console output and proper configuration.
|
||||
|
||||
Usage:
|
||||
python scripts/start_data_collection.py [options]
|
||||
|
||||
Examples:
|
||||
# Start with default configuration (indefinite run)
|
||||
python scripts/start_data_collection.py
|
||||
|
||||
# Run for 8 hours with default config
|
||||
python scripts/start_data_collection.py --hours 8
|
||||
|
||||
# Use custom configuration file
|
||||
python scripts/start_data_collection.py --config config/my_config.json
|
||||
|
||||
# Run for 24 hours with custom config
|
||||
python scripts/start_data_collection.py --config config/production.json --hours 24
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from data.collection_service import run_data_collection_service
|
||||
|
||||
|
||||
def display_banner(config_path: str, duration_hours: float = None):
|
||||
"""Display service startup banner."""
|
||||
print("🚀 CRYPTOCURRENCY DATA COLLECTION SERVICE")
|
||||
print("=" * 55)
|
||||
print(f"📁 Configuration: {config_path}")
|
||||
|
||||
if duration_hours:
|
||||
print(f"⏱️ Duration: {duration_hours} hours")
|
||||
else:
|
||||
print("⏱️ Duration: Indefinite (until stopped)")
|
||||
|
||||
print("📊 Logging: Essential events only (connections, errors)")
|
||||
print("💾 Storage: PostgreSQL + Redis")
|
||||
print("🔍 Monitor: python scripts/monitor_clean.py")
|
||||
print("⏹️ Stop: Ctrl+C")
|
||||
print("=" * 55)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Start Cryptocurrency Data Collection Service",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Start with default configuration (indefinite)
|
||||
python scripts/start_data_collection.py
|
||||
|
||||
# Run for 8 hours
|
||||
python scripts/start_data_collection.py --hours 8
|
||||
|
||||
# Use custom configuration
|
||||
python scripts/start_data_collection.py --config config/custom.json
|
||||
|
||||
# Production run for 24 hours
|
||||
python scripts/start_data_collection.py --config config/production.json --hours 24
|
||||
|
||||
Configuration:
|
||||
The service will create a default configuration file if none exists.
|
||||
Default location: config/data_collection.json
|
||||
|
||||
The configuration includes:
|
||||
- Exchange settings (OKX by default)
|
||||
- Trading pairs (BTC-USDT, ETH-USDT by default)
|
||||
- Data types and timeframes
|
||||
- Health monitoring settings
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--config',
|
||||
default="config/data_collection.json",
|
||||
help='Configuration file path (default: config/data_collection.json)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--hours',
|
||||
type=float,
|
||||
help='Collection duration in hours (default: indefinite until Ctrl+C)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--quiet',
|
||||
action='store_true',
|
||||
help='Suppress banner and start directly'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate arguments
|
||||
if args.hours is not None and args.hours <= 0:
|
||||
print("❌ Duration must be positive")
|
||||
sys.exit(1)
|
||||
|
||||
# Display banner unless quiet mode
|
||||
if not args.quiet:
|
||||
display_banner(args.config, args.hours)
|
||||
|
||||
try:
|
||||
# Start the service
|
||||
print("🎯 Starting service..." if not args.quiet else "")
|
||||
|
||||
success = asyncio.run(run_data_collection_service(
|
||||
config_path=args.config,
|
||||
duration_hours=args.hours
|
||||
))
|
||||
|
||||
if success:
|
||||
print("✅ Service completed successfully" if not args.quiet else "")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("❌ Service failed" if not args.quiet else "")
|
||||
sys.exit(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n👋 Service interrupted by user")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"❌ Fatal error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user