memory optimization

main.py

@@ -8,6 +8,10 @@ import time
 import signal
 import queue
 
+# Memory tuning flags (set to True only if you need in-memory histories or latest snapshot)
+ENABLE_HISTORY = False
+STORE_LATEST_BOOK = False
+
 latest_book = {'bids': [], 'asks': [], 'timestamp': None}
 book_history = deque(maxlen=200)  # Keep last 100 book updates (~20-60 seconds)
 trade_history = deque(maxlen=500)  # Keep last 1000 trades (~2-10 minutes)
@@ -134,13 +138,14 @@ def handle_message(data, instrument, db, trade_history, book_history, latest_boo
                 'side': trade.get('side'),
                 'timestamp': trade.get('ts')
             })
-            ts = float(trade.get('ts', time.time() * 1000))
-            trade_history.append({
-                'price': trade.get('px'),
-                'size': trade.get('sz'),
-                'side': trade.get('side'),
-                'timestamp': ts
-            })
+            if ENABLE_HISTORY:
+                ts = float(trade.get('ts', time.time() * 1000))
+                trade_history.append({
+                    'price': trade.get('px'),
+                    'size': trade.get('sz'),
+                    'side': trade.get('side'),
+                    'timestamp': ts
+                })
     elif 'arg' in msg and msg['arg'].get('channel', '').startswith('books'):
         for book in msg.get('data', []):
             db.insert_book({
@@ -149,15 +154,17 @@ def handle_message(data, instrument, db, trade_history, book_history, latest_boo
                 'asks': book.get('asks'),
                 'timestamp': book.get('ts')
             })
-            latest_book['bids'] = book.get('bids', [])
-            latest_book['asks'] = book.get('asks', [])
-            latest_book['timestamp'] = book.get('ts')
-            ts = float(book.get('ts', time.time() * 1000))
-            book_history.append({
-                'bids': book.get('bids', []),
-                'asks': book.get('asks', []),
-                'timestamp': ts
-            })
+            if STORE_LATEST_BOOK:
+                latest_book['bids'] = book.get('bids', [])
+                latest_book['asks'] = book.get('asks', [])
+                latest_book['timestamp'] = book.get('ts')
+            if ENABLE_HISTORY:
+                ts = float(book.get('ts', time.time() * 1000))
+                book_history.append({
+                    'bids': book.get('bids', []),
+                    'asks': book.get('asks', []),
+                    'timestamp': ts
+                })
     else:
         logging.info(f"Unknown message for {instrument}: {msg}")
     return True, now
@@ -216,16 +223,6 @@ def main():
         "BTC-USDT"
     ]
 
-    # Configure logging to both file and stdout
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s %(levelname)s %(message)s',
-        handlers=[
-            logging.FileHandler('market_data_collector.log', mode='a'),
-            logging.StreamHandler()
-        ]
-    )
-    
     try:
         logging.info("Starting Market Data Collector")
         logging.info(f"Initializing instruments: {instruments}")

okx_client.py

@@ -4,7 +4,6 @@ import hmac
 import hashlib
 import base64
 import json
-import pandas as pd
 import threading
 import requests
 import websocket