import threading import time import queue from google.oauth2.service_account import Credentials import gspread import math import numpy as np from collections import defaultdict class GSheetBatchPusher(threading.Thread): def __init__(self, queue, timestamp, spreadsheet_name, interval=60, logging=None): super().__init__(daemon=True) self.queue = queue self.timestamp = timestamp self.spreadsheet_name = spreadsheet_name self.interval = interval self._stop_event = threading.Event() self.logging = logging def run(self): while not self._stop_event.is_set(): self.push_all() time.sleep(self.interval) # Final push on stop self.push_all() def stop(self): self._stop_event.set() def push_all(self): batch_results = [] batch_trades = [] while True: try: results, trades = self.queue.get_nowait() batch_results.extend(results) batch_trades.extend(trades) except queue.Empty: break if batch_results or batch_trades: self.write_results_per_combination_gsheet(batch_results, batch_trades, self.timestamp, self.spreadsheet_name) def write_results_per_combination_gsheet(self, results_rows, trade_rows, timestamp, spreadsheet_name="GlimBit Backtest Results"): scopes = [ "https://www.googleapis.com/auth/spreadsheets", "https://www.googleapis.com/auth/drive" ] creds = Credentials.from_service_account_file('credentials/service_account.json', scopes=scopes) gc = gspread.authorize(creds) sh = gc.open(spreadsheet_name) try: worksheet = sh.worksheet("Results") except gspread.exceptions.WorksheetNotFound: worksheet = sh.add_worksheet(title="Results", rows="1000", cols="20") # Clear the worksheet before writing new results worksheet.clear() # Updated fieldnames to match your data rows fieldnames = [ "timeframe", "stop_loss_pct", "n_trades", "n_stop_loss", "win_rate", "max_drawdown", "avg_trade", "profit_ratio", "initial_usd", "final_usd" ] def to_native(val): if isinstance(val, (np.generic, np.ndarray)): val = val.item() if hasattr(val, 'isoformat'): return val.isoformat() # Handle inf, -inf, nan if isinstance(val, float): if math.isinf(val): return "∞" if val > 0 else "-∞" if math.isnan(val): return "" return val # Write header if sheet is empty if len(worksheet.get_all_values()) == 0: worksheet.append_row(fieldnames) for row in results_rows: values = [to_native(row.get(field, "")) for field in fieldnames] worksheet.append_row(values) trades_fieldnames = [ "entry_time", "exit_time", "entry_price", "exit_price", "profit_pct", "type" ] trades_by_combo = defaultdict(list) for trade in trade_rows: tf = trade.get("timeframe") sl = trade.get("stop_loss_pct") trades_by_combo[(tf, sl)].append(trade) for (tf, sl), trades in trades_by_combo.items(): sl_percent = int(round(sl * 100)) sheet_name = f"Trades_{tf}_ST{sl_percent}%" try: trades_ws = sh.worksheet(sheet_name) except gspread.exceptions.WorksheetNotFound: trades_ws = sh.add_worksheet(title=sheet_name, rows="1000", cols="20") # Clear the trades worksheet before writing new trades trades_ws.clear() if len(trades_ws.get_all_values()) == 0: trades_ws.append_row(trades_fieldnames) for trade in trades: trade_row = [to_native(trade.get(field, "")) for field in trades_fieldnames] try: trades_ws.append_row(trade_row) except gspread.exceptions.APIError as e: if '429' in str(e): if self.logging is not None: self.logging.warning(f"Google Sheets API quota exceeded (429). Please wait one minute. Will retry on next batch push. Sheet: {sheet_name}") # Re-queue the failed batch for retry self.queue.put((results_rows, trade_rows)) return # Stop pushing for this batch, will retry next interval else: raise