Refactor desktop application to enhance OHLC and OBI visualizations. Introduced dynamic candle width based on timestamps, improved data handling for metrics, and added crosshair functionality for better data inspection. Updated UI layout for improved user experience and integrated real-time data updates from the processor.
This commit is contained in:
parent
ebf232317c
commit
65dab17424
1002
desktop_app.py
1002
desktop_app.py
File diff suppressed because it is too large
Load Diff
132
level_parser.py
132
level_parser.py
@ -1,85 +1,87 @@
|
||||
"""Level parsing utilities for orderbook data."""
|
||||
"""Ultra-fast level parsing for strings like:
|
||||
"[['110173.4', '0.0000454', '0', '4'], ['110177.1', '0', '0', '0'], ...]"
|
||||
"""
|
||||
|
||||
import json
|
||||
import ast
|
||||
import logging
|
||||
from typing import List, Any, Tuple
|
||||
from typing import List, Tuple, Any
|
||||
|
||||
|
||||
def normalize_levels(levels: Any) -> List[List[float]]:
|
||||
"""
|
||||
Convert string-encoded levels into [[price, size], ...] floats.
|
||||
|
||||
Filters out zero/negative sizes. Supports JSON and Python literal formats.
|
||||
Return [[price, size], ...] with size > 0 only (floats).
|
||||
Assumes 'levels' is a single-quoted list-of-lists string as above.
|
||||
"""
|
||||
if not levels or levels == '[]':
|
||||
return []
|
||||
|
||||
parsed = _parse_string_to_list(levels)
|
||||
if not parsed:
|
||||
return []
|
||||
|
||||
pairs: List[List[float]] = []
|
||||
for item in parsed:
|
||||
price, size = _extract_price_size(item)
|
||||
if price is None or size is None:
|
||||
continue
|
||||
try:
|
||||
p, s = float(price), float(size)
|
||||
if s > 0:
|
||||
pairs.append([p, s])
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not pairs:
|
||||
logging.debug("normalize_levels: no valid pairs parsed from input")
|
||||
return pairs
|
||||
pairs = _fast_pairs(levels)
|
||||
# filter strictly positive sizes
|
||||
return [[p, s] for (p, s) in pairs if s > 0.0]
|
||||
|
||||
|
||||
def parse_levels_including_zeros(levels: Any) -> List[Tuple[float, float]]:
|
||||
"""
|
||||
Parse levels into (price, size) tuples including zero sizes for deletions.
|
||||
|
||||
Similar to normalize_levels but preserves zero sizes (for orderbook deletions).
|
||||
Return [(price, size), ...] (floats), preserving zeros for deletions.
|
||||
Assumes 'levels' is a single-quoted list-of-lists string as above.
|
||||
"""
|
||||
if not levels or levels == '[]':
|
||||
return _fast_pairs(levels)
|
||||
|
||||
|
||||
# ----------------- internal: fast path -----------------
|
||||
|
||||
def _fast_pairs(levels: Any) -> List[Tuple[float, float]]:
|
||||
"""
|
||||
Extremely fast parser for inputs like:
|
||||
"[['110173.4','0.0000454','0','4'],['110177.1','0','0','0'], ...]"
|
||||
Keeps only the first two fields from each row and converts to float.
|
||||
"""
|
||||
if not levels:
|
||||
return []
|
||||
|
||||
parsed = _parse_string_to_list(levels)
|
||||
if not parsed:
|
||||
# If already a list (rare in your pipeline), fall back to simple handling
|
||||
if isinstance(levels, (list, tuple)):
|
||||
out: List[Tuple[float, float]] = []
|
||||
for item in levels:
|
||||
if isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
try:
|
||||
p = float(item[0]); s = float(item[1])
|
||||
out.append((p, s))
|
||||
except Exception:
|
||||
continue
|
||||
return out
|
||||
|
||||
# Expect a string: strip outer brackets and single quotes fast
|
||||
s = str(levels).strip()
|
||||
if len(s) < 5: # too short to contain "[[...]]"
|
||||
return []
|
||||
|
||||
results: List[Tuple[float, float]] = []
|
||||
for item in parsed:
|
||||
price, size = _extract_price_size(item)
|
||||
if price is None or size is None:
|
||||
# Remove the outermost [ and ] quickly (tolerant)
|
||||
if s[0] == '[':
|
||||
s = s[1:]
|
||||
if s and s[-1] == ']':
|
||||
s = s[:-1]
|
||||
|
||||
# Remove *all* single quotes (input uses single quotes, not JSON)
|
||||
s = s.replace("'", "")
|
||||
|
||||
# Now s looks like: [[110173.4, 0.0000454, 0, 4], [110177.1, 0, 0, 0], ...]
|
||||
# Split into rows on "],", then strip brackets/spaces per row
|
||||
rows = s.split("],")
|
||||
out: List[Tuple[float, float]] = []
|
||||
|
||||
for row in rows:
|
||||
row = row.strip()
|
||||
# strip any leading/trailing brackets/spaces
|
||||
if row.startswith('['):
|
||||
row = row[1:]
|
||||
if row.endswith(']'):
|
||||
row = row[:-1]
|
||||
|
||||
# fast split by commas and take first two fields
|
||||
cols = row.split(',')
|
||||
if len(cols) < 2:
|
||||
continue
|
||||
try:
|
||||
p, s = float(price), float(size)
|
||||
if s >= 0:
|
||||
results.append((p, s))
|
||||
p = float(cols[0].strip())
|
||||
s_ = float(cols[1].strip())
|
||||
out.append((p, s_))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _parse_string_to_list(levels: Any) -> List[Any]:
|
||||
"""Parse string levels to list, trying JSON first then literal_eval."""
|
||||
try:
|
||||
parsed = json.loads(levels)
|
||||
except Exception:
|
||||
try:
|
||||
parsed = ast.literal_eval(levels)
|
||||
except Exception:
|
||||
return []
|
||||
return parsed if isinstance(parsed, list) else []
|
||||
|
||||
|
||||
def _extract_price_size(item: Any) -> Tuple[Any, Any]:
|
||||
"""Extract price and size from dict or list/tuple format."""
|
||||
if isinstance(item, dict):
|
||||
return item.get("price", item.get("p")), item.get("size", item.get("s"))
|
||||
elif isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
return item[0], item[1]
|
||||
return None, None
|
||||
return out
|
||||
|
||||
44
main.py
44
main.py
@ -10,15 +10,12 @@ from ohlc_processor import OHLCProcessor
|
||||
from desktop_app import MainWindow
|
||||
import sys
|
||||
from PySide6.QtWidgets import QApplication
|
||||
from PySide6.QtCore import Signal, QTimer
|
||||
|
||||
|
||||
def main(instrument: str = typer.Argument(..., help="Instrument to backtest, e.g. BTC-USDT"),
|
||||
start_date: str = typer.Argument(..., help="Start date, e.g. 2025-07-01"),
|
||||
end_date: str = typer.Argument(..., help="End date, e.g. 2025-08-01"),
|
||||
window_seconds: int = typer.Option(60, help="OHLC window size in seconds")):
|
||||
"""
|
||||
Process orderbook data and visualize OHLC charts in real-time.
|
||||
"""
|
||||
end_date: str = typer.Argument(..., help="End date, e.g. 2025-08-01")):
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
start_date = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
@ -39,10 +36,18 @@ def main(instrument: str = typer.Argument(..., help="Instrument to backtest, e.g
|
||||
|
||||
logging.info(f"Found {len(db_paths)} database files: {[p.name for p in db_paths]}")
|
||||
|
||||
processor = OHLCProcessor(window_seconds=window_seconds)
|
||||
processor = OHLCProcessor(aggregate_window_seconds=60 * 60)
|
||||
|
||||
app = QApplication(sys.argv)
|
||||
desktop_app = MainWindow()
|
||||
desktop_app.show()
|
||||
|
||||
timer = QTimer()
|
||||
timer.timeout.connect(lambda: desktop_app.update_data(processor))
|
||||
timer.start(1000)
|
||||
|
||||
|
||||
def process_data():
|
||||
"""Process database data in a separate thread."""
|
||||
try:
|
||||
for db_path in db_paths:
|
||||
db_name_parts = db_path.name.split(".")[0].split("-")
|
||||
@ -64,30 +69,19 @@ def main(instrument: str = typer.Argument(..., help="Instrument to backtest, e.g
|
||||
for orderbook_update, trades in db_interpreter.stream():
|
||||
batch_count += 1
|
||||
|
||||
processor.process_trades(trades)
|
||||
processor.update_orderbook(orderbook_update)
|
||||
processor.process_trades(trades)
|
||||
# desktop_app.update_data(processor)
|
||||
|
||||
|
||||
processor.finalize()
|
||||
logging.info("Data processing completed")
|
||||
except Exception as e:
|
||||
logging.error(f"Error in data processing: {e}")
|
||||
|
||||
try:
|
||||
app = QApplication(sys.argv)
|
||||
desktop_app = MainWindow()
|
||||
|
||||
desktop_app.setup_data_processor(processor)
|
||||
desktop_app.show()
|
||||
|
||||
logging.info("Desktop visualizer started")
|
||||
|
||||
data_thread = threading.Thread(target=process_data, daemon=True)
|
||||
data_thread.start()
|
||||
|
||||
app.exec()
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to start desktop visualizer: {e}")
|
||||
data_thread = threading.Thread(target=process_data, daemon=True)
|
||||
data_thread.start()
|
||||
|
||||
app.exec()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import logging
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional, List
|
||||
|
||||
|
||||
class MetricsCalculator:
|
||||
@ -7,6 +7,20 @@ class MetricsCalculator:
|
||||
self.cvd_cumulative = 0.0
|
||||
self.obi_value = 0.0
|
||||
|
||||
# --- per-bucket state ---
|
||||
self._b_ts_start: Optional[int] = None
|
||||
self._b_ts_end: Optional[int] = None
|
||||
self._obi_o: Optional[float] = None
|
||||
self._obi_h: Optional[float] = None
|
||||
self._obi_l: Optional[float] = None
|
||||
self._obi_c: Optional[float] = None
|
||||
|
||||
# final series rows: [ts_start, ts_end, obi_o, obi_h, obi_l, obi_c, cvd]
|
||||
self._series: List[List[float]] = []
|
||||
|
||||
# ------------------------------
|
||||
# CVD
|
||||
# ------------------------------
|
||||
def update_cvd_from_trade(self, side: str, size: float) -> None:
|
||||
if side == "buy":
|
||||
volume_delta = float(size)
|
||||
@ -14,8 +28,56 @@ class MetricsCalculator:
|
||||
volume_delta = -float(size)
|
||||
else:
|
||||
logging.warning(f"Unknown trade side '{side}', treating as neutral")
|
||||
|
||||
volume_delta = 0.0
|
||||
self.cvd_cumulative += volume_delta
|
||||
|
||||
# ------------------------------
|
||||
# OBI
|
||||
# ------------------------------
|
||||
def update_obi_from_book(self, total_bids: float, total_asks: float) -> None:
|
||||
self.obi_value = float(total_bids - total_asks)
|
||||
# update H/L/C if a bucket is open
|
||||
if self._b_ts_start is not None:
|
||||
v = self.obi_value
|
||||
if self._obi_o is None:
|
||||
self._obi_o = self._obi_h = self._obi_l = self._obi_c = v
|
||||
else:
|
||||
self._obi_h = max(self._obi_h, v)
|
||||
self._obi_l = min(self._obi_l, v)
|
||||
self._obi_c = v
|
||||
|
||||
# ------------------------------
|
||||
# Bucket lifecycle
|
||||
# ------------------------------
|
||||
def begin_bucket(self, ts_start_ms: int, ts_end_ms: int) -> None:
|
||||
self._b_ts_start = int(ts_start_ms)
|
||||
self._b_ts_end = int(ts_end_ms)
|
||||
v = float(self.obi_value)
|
||||
self._obi_o = self._obi_h = self._obi_l = self._obi_c = v
|
||||
|
||||
def finalize_bucket(self) -> None:
|
||||
if self._b_ts_start is None or self._b_ts_end is None:
|
||||
return
|
||||
o = float(self._obi_o if self._obi_o is not None else self.obi_value)
|
||||
h = float(self._obi_h if self._obi_h is not None else self.obi_value)
|
||||
l = float(self._obi_l if self._obi_l is not None else self.obi_value)
|
||||
c = float(self._obi_c if self._obi_c is not None else self.obi_value)
|
||||
self._series.append([
|
||||
self._b_ts_start, self._b_ts_end, o, h, l, c, float(self.cvd_cumulative)
|
||||
])
|
||||
# reset
|
||||
self._b_ts_start = self._b_ts_end = None
|
||||
self._obi_o = self._obi_h = self._obi_l = self._obi_c = None
|
||||
|
||||
def add_flat_bucket(self, ts_start_ms: int, ts_end_ms: int) -> None:
|
||||
v = float(self.obi_value)
|
||||
self._series.append([
|
||||
int(ts_start_ms), int(ts_end_ms),
|
||||
v, v, v, v, float(self.cvd_cumulative)
|
||||
])
|
||||
|
||||
# ------------------------------
|
||||
# Output
|
||||
# ------------------------------
|
||||
def get_series(self):
|
||||
return self._series
|
||||
|
||||
@ -1,70 +1,191 @@
|
||||
import logging
|
||||
from typing import List, Any, Dict, Tuple
|
||||
from typing import List, Any, Dict, Tuple, Optional
|
||||
|
||||
from viz_io import add_ohlc_bar, upsert_ohlc_bar, _atomic_write_json, DEPTH_FILE
|
||||
from db_interpreter import OrderbookUpdate
|
||||
from level_parser import normalize_levels, parse_levels_including_zeros
|
||||
from level_parser import parse_levels_including_zeros
|
||||
from orderbook_manager import OrderbookManager
|
||||
from metrics_calculator import MetricsCalculator
|
||||
|
||||
|
||||
class OHLCProcessor:
|
||||
"""
|
||||
Processes trade data and orderbook updates into OHLC bars and depth snapshots.
|
||||
|
||||
This class aggregates individual trades into time-windowed OHLC (Open, High, Low, Close)
|
||||
bars and maintains an in-memory orderbook state for depth visualization. It also
|
||||
calculates Order Book Imbalance (OBI) and Cumulative Volume Delta (CVD) metrics.
|
||||
|
||||
The processor uses throttled updates to balance visualization responsiveness with
|
||||
I/O efficiency, emitting intermediate updates during active windows.
|
||||
|
||||
Attributes:
|
||||
window_seconds: Time window duration for OHLC aggregation
|
||||
depth_levels_per_side: Number of top price levels to maintain per side
|
||||
trades_processed: Total number of trades processed
|
||||
bars_created: Total number of OHLC bars created
|
||||
cvd_cumulative: Running cumulative volume delta (via metrics calculator)
|
||||
Time-bucketed OHLC aggregator with gap-bar filling and metric hooks.
|
||||
|
||||
- Bars are aligned to fixed buckets of length `aggregate_window_seconds`.
|
||||
- If there is a gap (no trades for one or more buckets), synthetic zero-volume
|
||||
candles are emitted with O=H=L=C=last_close AND a flat metrics bucket is added.
|
||||
- By default, the next bar's OPEN is the previous bar's CLOSE (configurable via
|
||||
`carry_forward_open`).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.current_bar = None
|
||||
def __init__(self, aggregate_window_seconds: int, carry_forward_open: bool = True) -> None:
|
||||
self.aggregate_window_seconds = int(aggregate_window_seconds)
|
||||
self._bucket_ms = self.aggregate_window_seconds * 1000
|
||||
|
||||
self.carry_forward_open = carry_forward_open
|
||||
|
||||
self.current_bar: Optional[Dict[str, Any]] = None
|
||||
self._current_bucket_index: Optional[int] = None
|
||||
self._last_close: Optional[float] = None
|
||||
|
||||
self.trades_processed = 0
|
||||
self.bars: List[Dict[str, Any]] = []
|
||||
|
||||
self._orderbook = OrderbookManager()
|
||||
self._metrics = MetricsCalculator()
|
||||
|
||||
@property
|
||||
def cvd_cumulative(self) -> float:
|
||||
"""Access cumulative CVD from metrics calculator."""
|
||||
return self._metrics.cvd_cumulative
|
||||
# -----------------------
|
||||
# Internal helpers
|
||||
# -----------------------
|
||||
def _new_bar(self, bucket_start_ms: int, open_price: float) -> Dict[str, Any]:
|
||||
return {
|
||||
"timestamp_start": bucket_start_ms,
|
||||
"timestamp_end": bucket_start_ms + self._bucket_ms,
|
||||
"open": float(open_price),
|
||||
"high": float(open_price),
|
||||
"low": float(open_price),
|
||||
"close": float(open_price),
|
||||
"volume": 0.0,
|
||||
}
|
||||
|
||||
def _emit_gap_bars(self, from_index: int, to_index: int) -> None:
|
||||
"""
|
||||
Emit empty buckets strictly between from_index and to_index.
|
||||
Each synthetic bar has zero volume and O=H=L=C=last_close.
|
||||
Also emit a flat metrics bucket for each gap.
|
||||
"""
|
||||
if self._last_close is None:
|
||||
return
|
||||
|
||||
for bi in range(from_index + 1, to_index):
|
||||
start_ms = bi * self._bucket_ms
|
||||
gap_bar = self._new_bar(start_ms, self._last_close)
|
||||
self.bars.append(gap_bar)
|
||||
|
||||
# metrics: add a flat bucket to keep OBI/CVD time-continuous
|
||||
try:
|
||||
self._metrics.add_flat_bucket(start_ms, start_ms + self._bucket_ms)
|
||||
except Exception as e:
|
||||
logging.debug(f"metrics add_flat_bucket error (ignored): {e}")
|
||||
|
||||
# -----------------------
|
||||
# Public API
|
||||
# -----------------------
|
||||
def process_trades(self, trades: List[Tuple[Any, ...]]) -> None:
|
||||
"""
|
||||
trades: iterables like (trade_id, trade_id_str, price, size, side, timestamp_ms, ...)
|
||||
timestamp_ms expected in milliseconds.
|
||||
"""
|
||||
if not trades:
|
||||
return
|
||||
|
||||
# Ensure time-ascending order; if upstream guarantees it, you can skip.
|
||||
trades = sorted(trades, key=lambda t: int(t[5]))
|
||||
|
||||
for trade in trades:
|
||||
trade_id, trade_id_str, price, size, side, timestamp_ms = trade[:6]
|
||||
price = float(price)
|
||||
size = float(size)
|
||||
timestamp_ms = int(timestamp_ms)
|
||||
self.trades_processed += 1
|
||||
|
||||
self._metrics.update_cvd_from_trade(side, size)
|
||||
# Metrics driven by trades: update CVD
|
||||
try:
|
||||
self._metrics.update_cvd_from_trade(side, size)
|
||||
except Exception as e:
|
||||
logging.debug(f"CVD update error (ignored): {e}")
|
||||
|
||||
if not self.current_bar:
|
||||
self.current_bar = {
|
||||
'open': float(price),
|
||||
'high': float(price),
|
||||
'low': float(price),
|
||||
'close': float(price)
|
||||
}
|
||||
self.current_bar['high'] = max(self.current_bar['high'], float(price))
|
||||
self.current_bar['low'] = min(self.current_bar['low'], float(price))
|
||||
self.current_bar['close'] = float(price)
|
||||
self.current_bar['volume'] += float(size)
|
||||
# Determine this trade's bucket
|
||||
bucket_index = timestamp_ms // self._bucket_ms
|
||||
bucket_start = bucket_index * self._bucket_ms
|
||||
|
||||
# New bucket?
|
||||
if self._current_bucket_index is None or bucket_index != self._current_bucket_index:
|
||||
# finalize prior bar
|
||||
if self.current_bar is not None:
|
||||
self.bars.append(self.current_bar)
|
||||
self._last_close = self.current_bar["close"]
|
||||
# finalize metrics for the prior bucket window
|
||||
try:
|
||||
self._metrics.finalize_bucket()
|
||||
except Exception as e:
|
||||
logging.debug(f"metrics finalize_bucket error (ignored): {e}")
|
||||
|
||||
# handle gaps
|
||||
if self._current_bucket_index is not None and bucket_index > self._current_bucket_index + 1:
|
||||
self._emit_gap_bars(self._current_bucket_index, bucket_index)
|
||||
|
||||
# pick open price policy
|
||||
if self.carry_forward_open and self._last_close is not None:
|
||||
open_for_new = self._last_close
|
||||
else:
|
||||
open_for_new = price
|
||||
|
||||
self.current_bar = self._new_bar(bucket_start, open_for_new)
|
||||
self._current_bucket_index = bucket_index
|
||||
|
||||
# begin a new metrics bucket aligned to this bar window
|
||||
try:
|
||||
self._metrics.begin_bucket(bucket_start, bucket_start + self._bucket_ms)
|
||||
except Exception as e:
|
||||
logging.debug(f"metrics begin_bucket error (ignored): {e}")
|
||||
|
||||
# Update current bucket with this trade
|
||||
b = self.current_bar
|
||||
if b is None:
|
||||
# Should not happen, but guard anyway
|
||||
b = self._new_bar(bucket_start, price)
|
||||
self.current_bar = b
|
||||
self._current_bucket_index = bucket_index
|
||||
try:
|
||||
self._metrics.begin_bucket(bucket_start, bucket_start + self._bucket_ms)
|
||||
except Exception as e:
|
||||
logging.debug(f"metrics begin_bucket (guard) error (ignored): {e}")
|
||||
|
||||
b["high"] = max(b["high"], price)
|
||||
b["low"] = min(b["low"], price)
|
||||
b["close"] = price
|
||||
b["volume"] += size
|
||||
# keep timestamp_end snapped to bucket boundary
|
||||
b["timestamp_end"] = bucket_start + self._bucket_ms
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Emit the in-progress bar (if any). Call at the end of a run/backtest."""
|
||||
if self.current_bar is not None:
|
||||
self.bars.append(self.current_bar)
|
||||
self._last_close = self.current_bar["close"]
|
||||
self.current_bar = None
|
||||
# finalize any open metrics bucket
|
||||
try:
|
||||
self._metrics.finalize_bucket()
|
||||
except Exception as e:
|
||||
logging.debug(f"metrics finalize_bucket on flush error (ignored): {e}")
|
||||
|
||||
def update_orderbook(self, ob_update: OrderbookUpdate) -> None:
|
||||
"""
|
||||
Apply orderbook deltas and refresh OBI metrics.
|
||||
Call this frequently (on each OB update) so intra-bucket OBI highs/lows track the book.
|
||||
"""
|
||||
bids_updates = parse_levels_including_zeros(ob_update.bids)
|
||||
asks_updates = parse_levels_including_zeros(ob_update.asks)
|
||||
|
||||
self._orderbook.apply_updates(bids_updates, asks_updates)
|
||||
|
||||
total_bids, total_asks = self._orderbook.get_total_volume()
|
||||
self._metrics.update_obi_from_book(total_bids, total_asks)
|
||||
|
||||
try:
|
||||
self._metrics.update_obi_from_book(total_bids, total_asks)
|
||||
except Exception as e:
|
||||
logging.debug(f"OBI update error (ignored): {e}")
|
||||
|
||||
# -----------------------
|
||||
# UI-facing helpers
|
||||
# -----------------------
|
||||
def get_metrics_series(self):
|
||||
"""
|
||||
Returns a list of rows:
|
||||
[timestamp_start_ms, timestamp_end_ms, obi_open, obi_high, obi_low, obi_close, cvd_value]
|
||||
"""
|
||||
try:
|
||||
return self._metrics.get_series()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user