Implement enhanced data collection system with health monitoring and management
- Introduced `BaseDataCollector` and `CollectorManager` classes for standardized data collection and centralized management. - Added health monitoring features, including auto-restart capabilities and detailed status reporting for collectors. - Updated `env.template` to include new logging and health check configurations. - Enhanced documentation in `docs/data_collectors.md` to provide comprehensive guidance on the new data collection system. - Added unit tests for `BaseDataCollector` and `CollectorManager` to ensure reliability and functionality.
This commit is contained in:
Vasily.onl
309
examples/collector_demo.py
Normal file
309
examples/collector_demo.py
Normal file
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
Demonstration of the enhanced data collector system with health monitoring and auto-restart.
|
||||
|
||||
This example shows how to:
|
||||
1. Create data collectors with health monitoring
|
||||
2. Use the collector manager for coordinated management
|
||||
3. Monitor collector health and handle failures
|
||||
4. Enable/disable collectors dynamically
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
from data import (
|
||||
BaseDataCollector, DataType, CollectorStatus, MarketDataPoint,
|
||||
CollectorManager, CollectorConfig
|
||||
)
|
||||
|
||||
|
||||
class DemoDataCollector(BaseDataCollector):
|
||||
"""
|
||||
Demo implementation of a data collector for demonstration purposes.
|
||||
|
||||
This collector simulates receiving market data and can be configured
|
||||
to fail periodically to demonstrate auto-restart functionality.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
exchange_name: str,
|
||||
symbols: list,
|
||||
fail_every_n_messages: int = 0,
|
||||
connection_delay: float = 0.1):
|
||||
"""
|
||||
Initialize demo collector.
|
||||
|
||||
Args:
|
||||
exchange_name: Name of the exchange
|
||||
symbols: Trading symbols to collect
|
||||
fail_every_n_messages: Simulate failure every N messages (0 = no failures)
|
||||
connection_delay: Simulated connection delay
|
||||
"""
|
||||
super().__init__(exchange_name, symbols, [DataType.TICKER])
|
||||
self.fail_every_n_messages = fail_every_n_messages
|
||||
self.connection_delay = connection_delay
|
||||
self.message_count = 0
|
||||
self.connected = False
|
||||
self.subscribed = False
|
||||
|
||||
async def connect(self) -> bool:
|
||||
"""Simulate connection to exchange."""
|
||||
print(f"[{self.exchange_name}] Connecting...")
|
||||
await asyncio.sleep(self.connection_delay)
|
||||
self.connected = True
|
||||
print(f"[{self.exchange_name}] Connected successfully")
|
||||
return True
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
"""Simulate disconnection from exchange."""
|
||||
print(f"[{self.exchange_name}] Disconnecting...")
|
||||
await asyncio.sleep(self.connection_delay / 2)
|
||||
self.connected = False
|
||||
self.subscribed = False
|
||||
print(f"[{self.exchange_name}] Disconnected")
|
||||
|
||||
async def subscribe_to_data(self, symbols: list, data_types: list) -> bool:
|
||||
"""Simulate subscription to data streams."""
|
||||
if not self.connected:
|
||||
return False
|
||||
|
||||
print(f"[{self.exchange_name}] Subscribing to {len(symbols)} symbols: {', '.join(symbols)}")
|
||||
await asyncio.sleep(0.05)
|
||||
self.subscribed = True
|
||||
return True
|
||||
|
||||
async def unsubscribe_from_data(self, symbols: list, data_types: list) -> bool:
|
||||
"""Simulate unsubscription from data streams."""
|
||||
print(f"[{self.exchange_name}] Unsubscribing from data streams")
|
||||
self.subscribed = False
|
||||
return True
|
||||
|
||||
async def _process_message(self, message: Any) -> Optional[MarketDataPoint]:
|
||||
"""Process simulated market data message."""
|
||||
self.message_count += 1
|
||||
|
||||
# Simulate periodic failures if configured
|
||||
if (self.fail_every_n_messages > 0 and
|
||||
self.message_count % self.fail_every_n_messages == 0):
|
||||
raise Exception(f"Simulated failure after {self.message_count} messages")
|
||||
|
||||
# Create mock market data
|
||||
data_point = MarketDataPoint(
|
||||
exchange=self.exchange_name,
|
||||
symbol=message['symbol'],
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
data_type=DataType.TICKER,
|
||||
data={
|
||||
'price': message['price'],
|
||||
'volume': message.get('volume', 100),
|
||||
'timestamp': datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
)
|
||||
|
||||
return data_point
|
||||
|
||||
async def _handle_messages(self) -> None:
|
||||
"""Simulate receiving and processing messages."""
|
||||
if not self.connected or not self.subscribed:
|
||||
await asyncio.sleep(0.1)
|
||||
return
|
||||
|
||||
# Simulate receiving data for each symbol
|
||||
for symbol in self.symbols:
|
||||
try:
|
||||
# Create simulated message
|
||||
simulated_message = {
|
||||
'symbol': symbol,
|
||||
'price': 50000 + (self.message_count % 1000), # Fake price that changes
|
||||
'volume': 1.5
|
||||
}
|
||||
|
||||
# Process the message
|
||||
data_point = await self._process_message(simulated_message)
|
||||
if data_point:
|
||||
self._stats['messages_processed'] += 1
|
||||
await self._notify_callbacks(data_point)
|
||||
|
||||
except Exception as e:
|
||||
# This will trigger reconnection logic
|
||||
raise e
|
||||
|
||||
# Simulate processing delay
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
|
||||
async def data_callback(data_point: MarketDataPoint):
|
||||
"""Callback function to handle received data."""
|
||||
print(f"📊 Data received: {data_point.exchange} - {data_point.symbol} - "
|
||||
f"Price: {data_point.data.get('price')} at {data_point.timestamp.strftime('%H:%M:%S')}")
|
||||
|
||||
|
||||
async def monitor_collectors(manager: CollectorManager, duration: int = 30):
|
||||
"""Monitor collector status and print updates."""
|
||||
print(f"\n🔍 Starting monitoring for {duration} seconds...")
|
||||
|
||||
for i in range(duration):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
status = manager.get_status()
|
||||
running = len(manager.get_running_collectors())
|
||||
failed = len(manager.get_failed_collectors())
|
||||
|
||||
if i % 5 == 0: # Print status every 5 seconds
|
||||
print(f"⏰ Status at {i+1}s: {running} running, {failed} failed, "
|
||||
f"{status['statistics']['restarts_performed']} restarts")
|
||||
|
||||
print("🏁 Monitoring complete")
|
||||
|
||||
|
||||
async def demo_basic_usage():
|
||||
"""Demonstrate basic collector usage."""
|
||||
print("=" * 60)
|
||||
print("🚀 Demo 1: Basic Data Collector Usage")
|
||||
print("=" * 60)
|
||||
|
||||
# Create a stable collector
|
||||
collector = DemoDataCollector("demo_exchange", ["BTC-USDT", "ETH-USDT"])
|
||||
|
||||
# Add data callback
|
||||
collector.add_data_callback(DataType.TICKER, data_callback)
|
||||
|
||||
# Start the collector
|
||||
print("Starting collector...")
|
||||
success = await collector.start()
|
||||
if success:
|
||||
print("✅ Collector started successfully")
|
||||
|
||||
# Let it run for a few seconds
|
||||
await asyncio.sleep(5)
|
||||
|
||||
# Show status
|
||||
status = collector.get_status()
|
||||
print(f"📈 Messages processed: {status['statistics']['messages_processed']}")
|
||||
print(f"⏱️ Uptime: {status['statistics']['uptime_seconds']:.1f}s")
|
||||
|
||||
# Stop the collector
|
||||
await collector.stop()
|
||||
print("✅ Collector stopped")
|
||||
else:
|
||||
print("❌ Failed to start collector")
|
||||
|
||||
|
||||
async def demo_manager_usage():
|
||||
"""Demonstrate collector manager usage."""
|
||||
print("\n" + "=" * 60)
|
||||
print("🎛️ Demo 2: Collector Manager Usage")
|
||||
print("=" * 60)
|
||||
|
||||
# Create manager
|
||||
manager = CollectorManager("demo_manager", global_health_check_interval=3.0)
|
||||
|
||||
# Create multiple collectors
|
||||
stable_collector = DemoDataCollector("stable_exchange", ["BTC-USDT"])
|
||||
failing_collector = DemoDataCollector("failing_exchange", ["ETH-USDT"],
|
||||
fail_every_n_messages=5) # Fails every 5 messages
|
||||
|
||||
# Add data callbacks
|
||||
stable_collector.add_data_callback(DataType.TICKER, data_callback)
|
||||
failing_collector.add_data_callback(DataType.TICKER, data_callback)
|
||||
|
||||
# Add collectors to manager
|
||||
manager.add_collector(stable_collector)
|
||||
manager.add_collector(failing_collector)
|
||||
|
||||
print(f"📝 Added {len(manager.list_collectors())} collectors to manager")
|
||||
|
||||
# Start manager
|
||||
success = await manager.start()
|
||||
if success:
|
||||
print("✅ Manager started successfully")
|
||||
|
||||
# Monitor for a while
|
||||
await monitor_collectors(manager, duration=15)
|
||||
|
||||
# Show final status
|
||||
status = manager.get_status()
|
||||
print(f"\n📊 Final Statistics:")
|
||||
print(f" - Total restarts: {status['statistics']['restarts_performed']}")
|
||||
print(f" - Running collectors: {len(manager.get_running_collectors())}")
|
||||
print(f" - Failed collectors: {len(manager.get_failed_collectors())}")
|
||||
|
||||
# Stop manager
|
||||
await manager.stop()
|
||||
print("✅ Manager stopped")
|
||||
else:
|
||||
print("❌ Failed to start manager")
|
||||
|
||||
|
||||
async def demo_dynamic_management():
|
||||
"""Demonstrate dynamic collector management."""
|
||||
print("\n" + "=" * 60)
|
||||
print("🔄 Demo 3: Dynamic Collector Management")
|
||||
print("=" * 60)
|
||||
|
||||
# Create manager
|
||||
manager = CollectorManager("dynamic_manager", global_health_check_interval=2.0)
|
||||
|
||||
# Start with one collector
|
||||
collector1 = DemoDataCollector("exchange_1", ["BTC-USDT"])
|
||||
collector1.add_data_callback(DataType.TICKER, data_callback)
|
||||
|
||||
manager.add_collector(collector1)
|
||||
await manager.start()
|
||||
|
||||
print("✅ Started with 1 collector")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Add second collector
|
||||
collector2 = DemoDataCollector("exchange_2", ["ETH-USDT"])
|
||||
collector2.add_data_callback(DataType.TICKER, data_callback)
|
||||
manager.add_collector(collector2)
|
||||
|
||||
print("➕ Added second collector")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Disable first collector
|
||||
collector_names = manager.list_collectors()
|
||||
manager.disable_collector(collector_names[0])
|
||||
|
||||
print("⏸️ Disabled first collector")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Re-enable first collector
|
||||
manager.enable_collector(collector_names[0])
|
||||
|
||||
print("▶️ Re-enabled first collector")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Show final status
|
||||
status = manager.get_status()
|
||||
print(f"📊 Final state: {len(manager.get_running_collectors())} running collectors")
|
||||
|
||||
await manager.stop()
|
||||
print("✅ Dynamic demo complete")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all demonstrations."""
|
||||
print("🎯 Data Collector System Demonstration")
|
||||
print("This demo shows health monitoring and auto-restart capabilities\n")
|
||||
|
||||
try:
|
||||
# Run demonstrations
|
||||
await demo_basic_usage()
|
||||
await demo_manager_usage()
|
||||
await demo_dynamic_management()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("🎉 All demonstrations completed successfully!")
|
||||
print("=" * 60)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Demo failed with error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
412
examples/parallel_collectors_demo.py
Normal file
412
examples/parallel_collectors_demo.py
Normal file
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
Demonstration of running multiple data collectors in parallel.
|
||||
|
||||
This example shows how to set up and manage multiple collectors simultaneously,
|
||||
each collecting data from different exchanges or different symbols.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any
|
||||
|
||||
from data import (
|
||||
BaseDataCollector, DataType, CollectorStatus, MarketDataPoint,
|
||||
CollectorManager, CollectorConfig
|
||||
)
|
||||
|
||||
|
||||
class DemoExchangeCollector(BaseDataCollector):
|
||||
"""Demo collector simulating different exchanges."""
|
||||
|
||||
def __init__(self,
|
||||
exchange_name: str,
|
||||
symbols: list,
|
||||
message_interval: float = 1.0,
|
||||
base_price: float = 50000):
|
||||
"""
|
||||
Initialize demo collector.
|
||||
|
||||
Args:
|
||||
exchange_name: Name of the exchange (okx, binance, coinbase, etc.)
|
||||
symbols: Trading symbols to collect
|
||||
message_interval: Seconds between simulated messages
|
||||
base_price: Base price for simulation
|
||||
"""
|
||||
super().__init__(exchange_name, symbols, [DataType.TICKER])
|
||||
self.message_interval = message_interval
|
||||
self.base_price = base_price
|
||||
self.connected = False
|
||||
self.subscribed = False
|
||||
self.message_count = 0
|
||||
|
||||
async def connect(self) -> bool:
|
||||
"""Simulate connection to exchange."""
|
||||
print(f"🔌 [{self.exchange_name.upper()}] Connecting...")
|
||||
await asyncio.sleep(0.2) # Simulate connection delay
|
||||
self.connected = True
|
||||
print(f"✅ [{self.exchange_name.upper()}] Connected successfully")
|
||||
return True
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
"""Simulate disconnection from exchange."""
|
||||
print(f"🔌 [{self.exchange_name.upper()}] Disconnecting...")
|
||||
await asyncio.sleep(0.1)
|
||||
self.connected = False
|
||||
self.subscribed = False
|
||||
print(f"❌ [{self.exchange_name.upper()}] Disconnected")
|
||||
|
||||
async def subscribe_to_data(self, symbols: list, data_types: list) -> bool:
|
||||
"""Simulate subscription to data streams."""
|
||||
if not self.connected:
|
||||
return False
|
||||
|
||||
print(f"📡 [{self.exchange_name.upper()}] Subscribing to {len(symbols)} symbols")
|
||||
await asyncio.sleep(0.1)
|
||||
self.subscribed = True
|
||||
return True
|
||||
|
||||
async def unsubscribe_from_data(self, symbols: list, data_types: list) -> bool:
|
||||
"""Simulate unsubscription from data streams."""
|
||||
print(f"📡 [{self.exchange_name.upper()}] Unsubscribing from data streams")
|
||||
self.subscribed = False
|
||||
return True
|
||||
|
||||
async def _process_message(self, message: Any) -> MarketDataPoint:
|
||||
"""Process simulated market data message."""
|
||||
self.message_count += 1
|
||||
|
||||
# Create realistic price variation
|
||||
price_variation = (self.message_count % 100 - 50) * 10
|
||||
current_price = self.base_price + price_variation
|
||||
|
||||
data_point = MarketDataPoint(
|
||||
exchange=self.exchange_name,
|
||||
symbol=message['symbol'],
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
data_type=DataType.TICKER,
|
||||
data={
|
||||
'price': current_price,
|
||||
'volume': message.get('volume', 1.0 + (self.message_count % 10) * 0.1),
|
||||
'bid': current_price - 0.5,
|
||||
'ask': current_price + 0.5,
|
||||
'timestamp': datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
)
|
||||
|
||||
return data_point
|
||||
|
||||
async def _handle_messages(self) -> None:
|
||||
"""Simulate receiving and processing messages."""
|
||||
if not self.connected or not self.subscribed:
|
||||
await asyncio.sleep(0.1)
|
||||
return
|
||||
|
||||
# Process each symbol
|
||||
for symbol in self.symbols:
|
||||
try:
|
||||
# Create simulated message
|
||||
simulated_message = {
|
||||
'symbol': symbol,
|
||||
'volume': 1.5 + (self.message_count % 5) * 0.2
|
||||
}
|
||||
|
||||
# Process the message
|
||||
data_point = await self._process_message(simulated_message)
|
||||
if data_point:
|
||||
self._stats['messages_processed'] += 1
|
||||
await self._notify_callbacks(data_point)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error processing message for {symbol}: {e}")
|
||||
raise e
|
||||
|
||||
# Wait before next batch of messages
|
||||
await asyncio.sleep(self.message_interval)
|
||||
|
||||
|
||||
def create_data_callback(exchange_name: str):
|
||||
"""Create a data callback function for a specific exchange."""
|
||||
|
||||
def data_callback(data_point: MarketDataPoint):
|
||||
print(f"📊 {exchange_name.upper():8} | {data_point.symbol:10} | "
|
||||
f"${data_point.data.get('price', 0):8.2f} | "
|
||||
f"Vol: {data_point.data.get('volume', 0):.2f} | "
|
||||
f"{data_point.timestamp.strftime('%H:%M:%S')}")
|
||||
|
||||
return data_callback
|
||||
|
||||
|
||||
async def demo_parallel_collectors():
|
||||
"""Demonstrate running multiple collectors in parallel."""
|
||||
print("=" * 80)
|
||||
print("🚀 PARALLEL COLLECTORS DEMONSTRATION")
|
||||
print("=" * 80)
|
||||
print("Running multiple exchange collectors simultaneously...")
|
||||
print()
|
||||
|
||||
# Create manager
|
||||
manager = CollectorManager(
|
||||
"parallel_demo_manager",
|
||||
global_health_check_interval=10.0 # Check every 10 seconds
|
||||
)
|
||||
|
||||
# Define exchange configurations
|
||||
exchange_configs = [
|
||||
{
|
||||
'name': 'okx',
|
||||
'symbols': ['BTC-USDT', 'ETH-USDT'],
|
||||
'interval': 1.0,
|
||||
'base_price': 45000
|
||||
},
|
||||
{
|
||||
'name': 'binance',
|
||||
'symbols': ['BTC-USDT', 'ETH-USDT', 'SOL-USDT'],
|
||||
'interval': 1.5,
|
||||
'base_price': 45100
|
||||
},
|
||||
{
|
||||
'name': 'coinbase',
|
||||
'symbols': ['BTC-USD', 'ETH-USD'],
|
||||
'interval': 2.0,
|
||||
'base_price': 44900
|
||||
},
|
||||
{
|
||||
'name': 'kraken',
|
||||
'symbols': ['XBTUSD', 'ETHUSD'],
|
||||
'interval': 1.2,
|
||||
'base_price': 45050
|
||||
}
|
||||
]
|
||||
|
||||
# Create and configure collectors
|
||||
for config in exchange_configs:
|
||||
# Create collector
|
||||
collector = DemoExchangeCollector(
|
||||
exchange_name=config['name'],
|
||||
symbols=config['symbols'],
|
||||
message_interval=config['interval'],
|
||||
base_price=config['base_price']
|
||||
)
|
||||
|
||||
# Add data callback
|
||||
callback = create_data_callback(config['name'])
|
||||
collector.add_data_callback(DataType.TICKER, callback)
|
||||
|
||||
# Add to manager with configuration
|
||||
collector_config = CollectorConfig(
|
||||
name=f"{config['name']}_collector",
|
||||
exchange=config['name'],
|
||||
symbols=config['symbols'],
|
||||
data_types=['ticker'],
|
||||
auto_restart=True,
|
||||
health_check_interval=15.0,
|
||||
enabled=True
|
||||
)
|
||||
|
||||
manager.add_collector(collector, collector_config)
|
||||
print(f"➕ Added {config['name'].upper()} collector with {len(config['symbols'])} symbols")
|
||||
|
||||
print(f"\n📝 Total collectors added: {len(manager.list_collectors())}")
|
||||
print()
|
||||
|
||||
# Start all collectors in parallel
|
||||
print("🏁 Starting all collectors...")
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
|
||||
success = await manager.start()
|
||||
if not success:
|
||||
print("❌ Failed to start collector manager")
|
||||
return
|
||||
|
||||
startup_time = asyncio.get_event_loop().time() - start_time
|
||||
print(f"✅ All collectors started in {startup_time:.2f} seconds")
|
||||
print()
|
||||
|
||||
print("📊 DATA STREAM (All exchanges running in parallel):")
|
||||
print("-" * 80)
|
||||
|
||||
# Monitor for a period
|
||||
monitoring_duration = 30 # seconds
|
||||
for i in range(monitoring_duration):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Print status every 10 seconds
|
||||
if i % 10 == 0 and i > 0:
|
||||
status = manager.get_status()
|
||||
print()
|
||||
print(f"⏰ STATUS UPDATE ({i}s):")
|
||||
print(f" Running collectors: {len(manager.get_running_collectors())}")
|
||||
print(f" Failed collectors: {len(manager.get_failed_collectors())}")
|
||||
print(f" Total restarts: {status['statistics']['restarts_performed']}")
|
||||
print("-" * 80)
|
||||
|
||||
# Final status report
|
||||
print()
|
||||
print("📈 FINAL STATUS REPORT:")
|
||||
print("=" * 80)
|
||||
|
||||
status = manager.get_status()
|
||||
print(f"Manager Status: {status['manager_status']}")
|
||||
print(f"Total Collectors: {status['total_collectors']}")
|
||||
print(f"Running Collectors: {len(manager.get_running_collectors())}")
|
||||
print(f"Failed Collectors: {len(manager.get_failed_collectors())}")
|
||||
print(f"Total Restarts: {status['statistics']['restarts_performed']}")
|
||||
|
||||
# Individual collector statistics
|
||||
print("\n📊 INDIVIDUAL COLLECTOR STATS:")
|
||||
for collector_name in manager.list_collectors():
|
||||
collector_status = manager.get_collector_status(collector_name)
|
||||
if collector_status:
|
||||
stats = collector_status['status']['statistics']
|
||||
health = collector_status['health']
|
||||
|
||||
print(f"\n{collector_name.upper()}:")
|
||||
print(f" Status: {collector_status['status']['status']}")
|
||||
print(f" Messages Processed: {stats['messages_processed']}")
|
||||
print(f" Uptime: {stats.get('uptime_seconds', 0):.1f}s")
|
||||
print(f" Errors: {stats['errors']}")
|
||||
print(f" Healthy: {health['is_healthy']}")
|
||||
|
||||
# Stop all collectors
|
||||
print("\n🛑 Stopping all collectors...")
|
||||
await manager.stop()
|
||||
print("✅ All collectors stopped successfully")
|
||||
|
||||
|
||||
async def demo_dynamic_management():
|
||||
"""Demonstrate dynamic addition/removal of collectors."""
|
||||
print("\n" + "=" * 80)
|
||||
print("🔄 DYNAMIC COLLECTOR MANAGEMENT")
|
||||
print("=" * 80)
|
||||
|
||||
manager = CollectorManager("dynamic_manager")
|
||||
|
||||
# Start with one collector
|
||||
collector1 = DemoExchangeCollector("exchange_a", ["BTC-USDT"], 1.0)
|
||||
collector1.add_data_callback(DataType.TICKER, create_data_callback("exchange_a"))
|
||||
manager.add_collector(collector1)
|
||||
|
||||
await manager.start()
|
||||
print("✅ Started with 1 collector")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Add second collector while system is running
|
||||
collector2 = DemoExchangeCollector("exchange_b", ["ETH-USDT"], 1.5)
|
||||
collector2.add_data_callback(DataType.TICKER, create_data_callback("exchange_b"))
|
||||
manager.add_collector(collector2)
|
||||
|
||||
print("➕ Added second collector while running")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Add third collector
|
||||
collector3 = DemoExchangeCollector("exchange_c", ["SOL-USDT"], 2.0)
|
||||
collector3.add_data_callback(DataType.TICKER, create_data_callback("exchange_c"))
|
||||
manager.add_collector(collector3)
|
||||
|
||||
print("➕ Added third collector")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
# Show current status
|
||||
print(f"\n📊 Current Status: {len(manager.get_running_collectors())} collectors running")
|
||||
|
||||
# Disable one collector
|
||||
collectors = manager.list_collectors()
|
||||
if len(collectors) > 1:
|
||||
manager.disable_collector(collectors[1])
|
||||
print(f"⏸️ Disabled collector: {collectors[1]}")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Re-enable
|
||||
if len(collectors) > 1:
|
||||
manager.enable_collector(collectors[1])
|
||||
print(f"▶️ Re-enabled collector: {collectors[1]}")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
print(f"\n📊 Final Status: {len(manager.get_running_collectors())} collectors running")
|
||||
|
||||
await manager.stop()
|
||||
print("✅ Dynamic management demo complete")
|
||||
|
||||
|
||||
async def demo_performance_monitoring():
|
||||
"""Demonstrate performance monitoring across multiple collectors."""
|
||||
print("\n" + "=" * 80)
|
||||
print("📈 PERFORMANCE MONITORING")
|
||||
print("=" * 80)
|
||||
|
||||
manager = CollectorManager("performance_monitor", global_health_check_interval=5.0)
|
||||
|
||||
# Create collectors with different performance characteristics
|
||||
configs = [
|
||||
("fast_exchange", ["BTC-USDT"], 0.5), # Fast updates
|
||||
("medium_exchange", ["ETH-USDT"], 1.0), # Medium updates
|
||||
("slow_exchange", ["SOL-USDT"], 2.0), # Slow updates
|
||||
]
|
||||
|
||||
for exchange, symbols, interval in configs:
|
||||
collector = DemoExchangeCollector(exchange, symbols, interval)
|
||||
collector.add_data_callback(DataType.TICKER, create_data_callback(exchange))
|
||||
manager.add_collector(collector)
|
||||
|
||||
await manager.start()
|
||||
print("✅ Started performance monitoring demo")
|
||||
|
||||
# Monitor performance for 20 seconds
|
||||
for i in range(4):
|
||||
await asyncio.sleep(5)
|
||||
|
||||
print(f"\n📊 PERFORMANCE SNAPSHOT ({(i+1)*5}s):")
|
||||
print("-" * 60)
|
||||
|
||||
for collector_name in manager.list_collectors():
|
||||
status = manager.get_collector_status(collector_name)
|
||||
if status:
|
||||
stats = status['status']['statistics']
|
||||
health = status['health']
|
||||
|
||||
msg_rate = stats['messages_processed'] / max(stats.get('uptime_seconds', 1), 1)
|
||||
|
||||
print(f"{collector_name:15} | "
|
||||
f"Rate: {msg_rate:5.1f}/s | "
|
||||
f"Total: {stats['messages_processed']:4d} | "
|
||||
f"Errors: {stats['errors']:2d} | "
|
||||
f"Health: {'✅' if health['is_healthy'] else '❌'}")
|
||||
|
||||
await manager.stop()
|
||||
print("\n✅ Performance monitoring demo complete")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all parallel collector demonstrations."""
|
||||
print("🎯 MULTIPLE COLLECTORS PARALLEL EXECUTION DEMO")
|
||||
print("This demonstration shows the CollectorManager running multiple collectors simultaneously\n")
|
||||
|
||||
try:
|
||||
# Main parallel demo
|
||||
await demo_parallel_collectors()
|
||||
|
||||
# Dynamic management demo
|
||||
await demo_dynamic_management()
|
||||
|
||||
# Performance monitoring demo
|
||||
await demo_performance_monitoring()
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("🎉 ALL PARALLEL EXECUTION DEMOS COMPLETED!")
|
||||
print("=" * 80)
|
||||
print("\nKey takeaways:")
|
||||
print("✅ Multiple collectors run truly in parallel")
|
||||
print("✅ Each collector operates independently")
|
||||
print("✅ Collectors can be added/removed while system is running")
|
||||
print("✅ Centralized health monitoring across all collectors")
|
||||
print("✅ Individual performance tracking per collector")
|
||||
print("✅ Coordinated lifecycle management")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Demo failed with error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user