412 lines
15 KiB
Python
412 lines
15 KiB
Python
|
"""
|
|||
|
|
Demonstration of running multiple data collectors in parallel.
|
|||
|
|
|
|||
|
|
This example shows how to set up and manage multiple collectors simultaneously,
|
|||
|
|
each collecting data from different exchanges or different symbols.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import asyncio
|
|||
|
|
from datetime import datetime, timezone
|
|||
|
|
from typing import Dict, Any
|
|||
|
|
|
|||
|
|
from data import (
|
|||
|
|
BaseDataCollector, DataType, CollectorStatus, MarketDataPoint,
|
|||
|
|
CollectorManager, CollectorConfig
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class DemoExchangeCollector(BaseDataCollector):
|
|||
|
|
"""Demo collector simulating different exchanges."""
|
|||
|
|
|
|||
|
|
def __init__(self,
|
|||
|
|
exchange_name: str,
|
|||
|
|
symbols: list,
|
|||
|
|
message_interval: float = 1.0,
|
|||
|
|
base_price: float = 50000):
|
|||
|
|
"""
|
|||
|
|
Initialize demo collector.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
exchange_name: Name of the exchange (okx, binance, coinbase, etc.)
|
|||
|
|
symbols: Trading symbols to collect
|
|||
|
|
message_interval: Seconds between simulated messages
|
|||
|
|
base_price: Base price for simulation
|
|||
|
|
"""
|
|||
|
|
super().__init__(exchange_name, symbols, [DataType.TICKER])
|
|||
|
|
self.message_interval = message_interval
|
|||
|
|
self.base_price = base_price
|
|||
|
|
self.connected = False
|
|||
|
|
self.subscribed = False
|
|||
|
|
self.message_count = 0
|
|||
|
|
|
|||
|
|
async def connect(self) -> bool:
|
|||
|
|
"""Simulate connection to exchange."""
|
|||
|
|
print(f"🔌 [{self.exchange_name.upper()}] Connecting...")
|
|||
|
|
await asyncio.sleep(0.2) # Simulate connection delay
|
|||
|
|
self.connected = True
|
|||
|
|
print(f"✅ [{self.exchange_name.upper()}] Connected successfully")
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
async def disconnect(self) -> None:
|
|||
|
|
"""Simulate disconnection from exchange."""
|
|||
|
|
print(f"🔌 [{self.exchange_name.upper()}] Disconnecting...")
|
|||
|
|
await asyncio.sleep(0.1)
|
|||
|
|
self.connected = False
|
|||
|
|
self.subscribed = False
|
|||
|
|
print(f"❌ [{self.exchange_name.upper()}] Disconnected")
|
|||
|
|
|
|||
|
|
async def subscribe_to_data(self, symbols: list, data_types: list) -> bool:
|
|||
|
|
"""Simulate subscription to data streams."""
|
|||
|
|
if not self.connected:
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
print(f"📡 [{self.exchange_name.upper()}] Subscribing to {len(symbols)} symbols")
|
|||
|
|
await asyncio.sleep(0.1)
|
|||
|
|
self.subscribed = True
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
async def unsubscribe_from_data(self, symbols: list, data_types: list) -> bool:
|
|||
|
|
"""Simulate unsubscription from data streams."""
|
|||
|
|
print(f"📡 [{self.exchange_name.upper()}] Unsubscribing from data streams")
|
|||
|
|
self.subscribed = False
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
async def _process_message(self, message: Any) -> MarketDataPoint:
|
|||
|
|
"""Process simulated market data message."""
|
|||
|
|
self.message_count += 1
|
|||
|
|
|
|||
|
|
# Create realistic price variation
|
|||
|
|
price_variation = (self.message_count % 100 - 50) * 10
|
|||
|
|
current_price = self.base_price + price_variation
|
|||
|
|
|
|||
|
|
data_point = MarketDataPoint(
|
|||
|
|
exchange=self.exchange_name,
|
|||
|
|
symbol=message['symbol'],
|
|||
|
|
timestamp=datetime.now(timezone.utc),
|
|||
|
|
data_type=DataType.TICKER,
|
|||
|
|
data={
|
|||
|
|
'price': current_price,
|
|||
|
|
'volume': message.get('volume', 1.0 + (self.message_count % 10) * 0.1),
|
|||
|
|
'bid': current_price - 0.5,
|
|||
|
|
'ask': current_price + 0.5,
|
|||
|
|
'timestamp': datetime.now(timezone.utc).isoformat()
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return data_point
|
|||
|
|
|
|||
|
|
async def _handle_messages(self) -> None:
|
|||
|
|
"""Simulate receiving and processing messages."""
|
|||
|
|
if not self.connected or not self.subscribed:
|
|||
|
|
await asyncio.sleep(0.1)
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
# Process each symbol
|
|||
|
|
for symbol in self.symbols:
|
|||
|
|
try:
|
|||
|
|
# Create simulated message
|
|||
|
|
simulated_message = {
|
|||
|
|
'symbol': symbol,
|
|||
|
|
'volume': 1.5 + (self.message_count % 5) * 0.2
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Process the message
|
|||
|
|
data_point = await self._process_message(simulated_message)
|
|||
|
|
if data_point:
|
|||
|
|
self._stats['messages_processed'] += 1
|
|||
|
|
await self._notify_callbacks(data_point)
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
self.logger.error(f"Error processing message for {symbol}: {e}")
|
|||
|
|
raise e
|
|||
|
|
|
|||
|
|
# Wait before next batch of messages
|
|||
|
|
await asyncio.sleep(self.message_interval)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def create_data_callback(exchange_name: str):
|
|||
|
|
"""Create a data callback function for a specific exchange."""
|
|||
|
|
|
|||
|
|
def data_callback(data_point: MarketDataPoint):
|
|||
|
|
print(f"📊 {exchange_name.upper():8} | {data_point.symbol:10} | "
|
|||
|
|
f"${data_point.data.get('price', 0):8.2f} | "
|
|||
|
|
f"Vol: {data_point.data.get('volume', 0):.2f} | "
|
|||
|
|
f"{data_point.timestamp.strftime('%H:%M:%S')}")
|
|||
|
|
|
|||
|
|
return data_callback
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def demo_parallel_collectors():
|
|||
|
|
"""Demonstrate running multiple collectors in parallel."""
|
|||
|
|
print("=" * 80)
|
|||
|
|
print("🚀 PARALLEL COLLECTORS DEMONSTRATION")
|
|||
|
|
print("=" * 80)
|
|||
|
|
print("Running multiple exchange collectors simultaneously...")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# Create manager
|
|||
|
|
manager = CollectorManager(
|
|||
|
|
"parallel_demo_manager",
|
|||
|
|
global_health_check_interval=10.0 # Check every 10 seconds
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# Define exchange configurations
|
|||
|
|
exchange_configs = [
|
|||
|
|
{
|
|||
|
|
'name': 'okx',
|
|||
|
|
'symbols': ['BTC-USDT', 'ETH-USDT'],
|
|||
|
|
'interval': 1.0,
|
|||
|
|
'base_price': 45000
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
'name': 'binance',
|
|||
|
|
'symbols': ['BTC-USDT', 'ETH-USDT', 'SOL-USDT'],
|
|||
|
|
'interval': 1.5,
|
|||
|
|
'base_price': 45100
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
'name': 'coinbase',
|
|||
|
|
'symbols': ['BTC-USD', 'ETH-USD'],
|
|||
|
|
'interval': 2.0,
|
|||
|
|
'base_price': 44900
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
'name': 'kraken',
|
|||
|
|
'symbols': ['XBTUSD', 'ETHUSD'],
|
|||
|
|
'interval': 1.2,
|
|||
|
|
'base_price': 45050
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
# Create and configure collectors
|
|||
|
|
for config in exchange_configs:
|
|||
|
|
# Create collector
|
|||
|
|
collector = DemoExchangeCollector(
|
|||
|
|
exchange_name=config['name'],
|
|||
|
|
symbols=config['symbols'],
|
|||
|
|
message_interval=config['interval'],
|
|||
|
|
base_price=config['base_price']
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# Add data callback
|
|||
|
|
callback = create_data_callback(config['name'])
|
|||
|
|
collector.add_data_callback(DataType.TICKER, callback)
|
|||
|
|
|
|||
|
|
# Add to manager with configuration
|
|||
|
|
collector_config = CollectorConfig(
|
|||
|
|
name=f"{config['name']}_collector",
|
|||
|
|
exchange=config['name'],
|
|||
|
|
symbols=config['symbols'],
|
|||
|
|
data_types=['ticker'],
|
|||
|
|
auto_restart=True,
|
|||
|
|
health_check_interval=15.0,
|
|||
|
|
enabled=True
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
manager.add_collector(collector, collector_config)
|
|||
|
|
print(f"➕ Added {config['name'].upper()} collector with {len(config['symbols'])} symbols")
|
|||
|
|
|
|||
|
|
print(f"\n📝 Total collectors added: {len(manager.list_collectors())}")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# Start all collectors in parallel
|
|||
|
|
print("🏁 Starting all collectors...")
|
|||
|
|
start_time = asyncio.get_event_loop().time()
|
|||
|
|
|
|||
|
|
success = await manager.start()
|
|||
|
|
if not success:
|
|||
|
|
print("❌ Failed to start collector manager")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
startup_time = asyncio.get_event_loop().time() - start_time
|
|||
|
|
print(f"✅ All collectors started in {startup_time:.2f} seconds")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
print("📊 DATA STREAM (All exchanges running in parallel):")
|
|||
|
|
print("-" * 80)
|
|||
|
|
|
|||
|
|
# Monitor for a period
|
|||
|
|
monitoring_duration = 30 # seconds
|
|||
|
|
for i in range(monitoring_duration):
|
|||
|
|
await asyncio.sleep(1)
|
|||
|
|
|
|||
|
|
# Print status every 10 seconds
|
|||
|
|
if i % 10 == 0 and i > 0:
|
|||
|
|
status = manager.get_status()
|
|||
|
|
print()
|
|||
|
|
print(f"⏰ STATUS UPDATE ({i}s):")
|
|||
|
|
print(f" Running collectors: {len(manager.get_running_collectors())}")
|
|||
|
|
print(f" Failed collectors: {len(manager.get_failed_collectors())}")
|
|||
|
|
print(f" Total restarts: {status['statistics']['restarts_performed']}")
|
|||
|
|
print("-" * 80)
|
|||
|
|
|
|||
|
|
# Final status report
|
|||
|
|
print()
|
|||
|
|
print("📈 FINAL STATUS REPORT:")
|
|||
|
|
print("=" * 80)
|
|||
|
|
|
|||
|
|
status = manager.get_status()
|
|||
|
|
print(f"Manager Status: {status['manager_status']}")
|
|||
|
|
print(f"Total Collectors: {status['total_collectors']}")
|
|||
|
|
print(f"Running Collectors: {len(manager.get_running_collectors())}")
|
|||
|
|
print(f"Failed Collectors: {len(manager.get_failed_collectors())}")
|
|||
|
|
print(f"Total Restarts: {status['statistics']['restarts_performed']}")
|
|||
|
|
|
|||
|
|
# Individual collector statistics
|
|||
|
|
print("\n📊 INDIVIDUAL COLLECTOR STATS:")
|
|||
|
|
for collector_name in manager.list_collectors():
|
|||
|
|
collector_status = manager.get_collector_status(collector_name)
|
|||
|
|
if collector_status:
|
|||
|
|
stats = collector_status['status']['statistics']
|
|||
|
|
health = collector_status['health']
|
|||
|
|
|
|||
|
|
print(f"\n{collector_name.upper()}:")
|
|||
|
|
print(f" Status: {collector_status['status']['status']}")
|
|||
|
|
print(f" Messages Processed: {stats['messages_processed']}")
|
|||
|
|
print(f" Uptime: {stats.get('uptime_seconds', 0):.1f}s")
|
|||
|
|
print(f" Errors: {stats['errors']}")
|
|||
|
|
print(f" Healthy: {health['is_healthy']}")
|
|||
|
|
|
|||
|
|
# Stop all collectors
|
|||
|
|
print("\n🛑 Stopping all collectors...")
|
|||
|
|
await manager.stop()
|
|||
|
|
print("✅ All collectors stopped successfully")
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def demo_dynamic_management():
|
|||
|
|
"""Demonstrate dynamic addition/removal of collectors."""
|
|||
|
|
print("\n" + "=" * 80)
|
|||
|
|
print("🔄 DYNAMIC COLLECTOR MANAGEMENT")
|
|||
|
|
print("=" * 80)
|
|||
|
|
|
|||
|
|
manager = CollectorManager("dynamic_manager")
|
|||
|
|
|
|||
|
|
# Start with one collector
|
|||
|
|
collector1 = DemoExchangeCollector("exchange_a", ["BTC-USDT"], 1.0)
|
|||
|
|
collector1.add_data_callback(DataType.TICKER, create_data_callback("exchange_a"))
|
|||
|
|
manager.add_collector(collector1)
|
|||
|
|
|
|||
|
|
await manager.start()
|
|||
|
|
print("✅ Started with 1 collector")
|
|||
|
|
await asyncio.sleep(3)
|
|||
|
|
|
|||
|
|
# Add second collector while system is running
|
|||
|
|
collector2 = DemoExchangeCollector("exchange_b", ["ETH-USDT"], 1.5)
|
|||
|
|
collector2.add_data_callback(DataType.TICKER, create_data_callback("exchange_b"))
|
|||
|
|
manager.add_collector(collector2)
|
|||
|
|
|
|||
|
|
print("➕ Added second collector while running")
|
|||
|
|
await asyncio.sleep(3)
|
|||
|
|
|
|||
|
|
# Add third collector
|
|||
|
|
collector3 = DemoExchangeCollector("exchange_c", ["SOL-USDT"], 2.0)
|
|||
|
|
collector3.add_data_callback(DataType.TICKER, create_data_callback("exchange_c"))
|
|||
|
|
manager.add_collector(collector3)
|
|||
|
|
|
|||
|
|
print("➕ Added third collector")
|
|||
|
|
await asyncio.sleep(5)
|
|||
|
|
|
|||
|
|
# Show current status
|
|||
|
|
print(f"\n📊 Current Status: {len(manager.get_running_collectors())} collectors running")
|
|||
|
|
|
|||
|
|
# Disable one collector
|
|||
|
|
collectors = manager.list_collectors()
|
|||
|
|
if len(collectors) > 1:
|
|||
|
|
manager.disable_collector(collectors[1])
|
|||
|
|
print(f"⏸️ Disabled collector: {collectors[1]}")
|
|||
|
|
await asyncio.sleep(3)
|
|||
|
|
|
|||
|
|
# Re-enable
|
|||
|
|
if len(collectors) > 1:
|
|||
|
|
manager.enable_collector(collectors[1])
|
|||
|
|
print(f"▶️ Re-enabled collector: {collectors[1]}")
|
|||
|
|
await asyncio.sleep(3)
|
|||
|
|
|
|||
|
|
print(f"\n📊 Final Status: {len(manager.get_running_collectors())} collectors running")
|
|||
|
|
|
|||
|
|
await manager.stop()
|
|||
|
|
print("✅ Dynamic management demo complete")
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def demo_performance_monitoring():
|
|||
|
|
"""Demonstrate performance monitoring across multiple collectors."""
|
|||
|
|
print("\n" + "=" * 80)
|
|||
|
|
print("📈 PERFORMANCE MONITORING")
|
|||
|
|
print("=" * 80)
|
|||
|
|
|
|||
|
|
manager = CollectorManager("performance_monitor", global_health_check_interval=5.0)
|
|||
|
|
|
|||
|
|
# Create collectors with different performance characteristics
|
|||
|
|
configs = [
|
|||
|
|
("fast_exchange", ["BTC-USDT"], 0.5), # Fast updates
|
|||
|
|
("medium_exchange", ["ETH-USDT"], 1.0), # Medium updates
|
|||
|
|
("slow_exchange", ["SOL-USDT"], 2.0), # Slow updates
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
for exchange, symbols, interval in configs:
|
|||
|
|
collector = DemoExchangeCollector(exchange, symbols, interval)
|
|||
|
|
collector.add_data_callback(DataType.TICKER, create_data_callback(exchange))
|
|||
|
|
manager.add_collector(collector)
|
|||
|
|
|
|||
|
|
await manager.start()
|
|||
|
|
print("✅ Started performance monitoring demo")
|
|||
|
|
|
|||
|
|
# Monitor performance for 20 seconds
|
|||
|
|
for i in range(4):
|
|||
|
|
await asyncio.sleep(5)
|
|||
|
|
|
|||
|
|
print(f"\n📊 PERFORMANCE SNAPSHOT ({(i+1)*5}s):")
|
|||
|
|
print("-" * 60)
|
|||
|
|
|
|||
|
|
for collector_name in manager.list_collectors():
|
|||
|
|
status = manager.get_collector_status(collector_name)
|
|||
|
|
if status:
|
|||
|
|
stats = status['status']['statistics']
|
|||
|
|
health = status['health']
|
|||
|
|
|
|||
|
|
msg_rate = stats['messages_processed'] / max(stats.get('uptime_seconds', 1), 1)
|
|||
|
|
|
|||
|
|
print(f"{collector_name:15} | "
|
|||
|
|
f"Rate: {msg_rate:5.1f}/s | "
|
|||
|
|
f"Total: {stats['messages_processed']:4d} | "
|
|||
|
|
f"Errors: {stats['errors']:2d} | "
|
|||
|
|
f"Health: {'✅' if health['is_healthy'] else '❌'}")
|
|||
|
|
|
|||
|
|
await manager.stop()
|
|||
|
|
print("\n✅ Performance monitoring demo complete")
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def main():
|
|||
|
|
"""Run all parallel collector demonstrations."""
|
|||
|
|
print("🎯 MULTIPLE COLLECTORS PARALLEL EXECUTION DEMO")
|
|||
|
|
print("This demonstration shows the CollectorManager running multiple collectors simultaneously\n")
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# Main parallel demo
|
|||
|
|
await demo_parallel_collectors()
|
|||
|
|
|
|||
|
|
# Dynamic management demo
|
|||
|
|
await demo_dynamic_management()
|
|||
|
|
|
|||
|
|
# Performance monitoring demo
|
|||
|
|
await demo_performance_monitoring()
|
|||
|
|
|
|||
|
|
print("\n" + "=" * 80)
|
|||
|
|
print("🎉 ALL PARALLEL EXECUTION DEMOS COMPLETED!")
|
|||
|
|
print("=" * 80)
|
|||
|
|
print("\nKey takeaways:")
|
|||
|
|
print("✅ Multiple collectors run truly in parallel")
|
|||
|
|
print("✅ Each collector operates independently")
|
|||
|
|
print("✅ Collectors can be added/removed while system is running")
|
|||
|
|
print("✅ Centralized health monitoring across all collectors")
|
|||
|
|
print("✅ Individual performance tracking per collector")
|
|||
|
|
print("✅ Coordinated lifecycle management")
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"❌ Demo failed with error: {e}")
|
|||
|
|
import traceback
|
|||
|
|
traceback.print_exc()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
asyncio.run(main())
|