Remove complete time series aggregation example and add data collection service implementation
- Deleted `example_complete_series_aggregation.py` as it is no longer needed. - Introduced `data_collection_service.py`, a production-ready service for cryptocurrency market data collection with clean logging and robust error handling. - Added configuration management for multiple trading pairs and exchanges, supporting health monitoring and graceful shutdown. - Created `data_collection.json` for service configuration, including exchange settings and logging preferences. - Updated `CandleProcessingConfig` to reflect changes in timeframes for candle processing. - Enhanced documentation to cover the new data collection service and its configuration, ensuring clarity for users.
This commit is contained in:
69
config/data_collection.json
Normal file
69
config/data_collection.json
Normal file
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"exchange": "okx",
|
||||
"connection": {
|
||||
"public_ws_url": "wss://ws.okx.com:8443/ws/v5/public",
|
||||
"private_ws_url": "wss://ws.okx.com:8443/ws/v5/private",
|
||||
"ping_interval": 25.0,
|
||||
"pong_timeout": 10.0,
|
||||
"max_reconnect_attempts": 5,
|
||||
"reconnect_delay": 5.0
|
||||
},
|
||||
"data_collection": {
|
||||
"store_raw_data": true,
|
||||
"health_check_interval": 120.0,
|
||||
"auto_restart": true,
|
||||
"buffer_size": 1000
|
||||
},
|
||||
"trading_pairs": [
|
||||
{
|
||||
"symbol": "BTC-USDT",
|
||||
"enabled": true,
|
||||
"data_types": [
|
||||
"trade",
|
||||
"orderbook"
|
||||
],
|
||||
"timeframes": [
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"1h"
|
||||
],
|
||||
"channels": {
|
||||
"trades": "trades",
|
||||
"orderbook": "books5",
|
||||
"ticker": "tickers"
|
||||
}
|
||||
},
|
||||
{
|
||||
"symbol": "ETH-USDT",
|
||||
"enabled": true,
|
||||
"data_types": [
|
||||
"trade",
|
||||
"orderbook"
|
||||
],
|
||||
"timeframes": [
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"1h"
|
||||
],
|
||||
"channels": {
|
||||
"trades": "trades",
|
||||
"orderbook": "books5",
|
||||
"ticker": "tickers"
|
||||
}
|
||||
}
|
||||
],
|
||||
"logging": {
|
||||
"component_name_template": "okx_collector_{symbol}",
|
||||
"log_level": "INFO",
|
||||
"verbose": false
|
||||
},
|
||||
"database": {
|
||||
"store_processed_data": true,
|
||||
"store_raw_data": true,
|
||||
"force_update_candles": false,
|
||||
"batch_size": 100,
|
||||
"flush_interval": 5.0
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user