Cycles/main.py

import pandas as pd
import numpy as np
import logging
import concurrent.futures
import os
import datetime
import queue

from cycles.trend_detector_simple import TrendDetectorSimple
from cycles.taxes import Taxes
from cycles.utils.storage import Storage
from cycles.utils.gsheets import GSheetBatchPusher
from cycles.utils.system import SystemUtils

# Set up logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[
        logging.FileHandler("backtest.log"),
        logging.StreamHandler()
    ]
)

# Global queue for batching Google Sheets updates
results_queue = queue.Queue()

def process_timeframe_data(min1_df, df, stop_loss_pcts, rule_name, initial_usd, debug=False):
    """Process the entire timeframe with all stop loss values (no monthly split)"""
    df = df.copy().reset_index(drop=True)
    trend_detector = TrendDetectorSimple(df, verbose=False)

    results_rows = []
    trade_rows = []
    for stop_loss_pct in stop_loss_pcts:
        results = trend_detector.backtest_meta_supertrend(
            min1_df,
            initial_usd=initial_usd,
            stop_loss_pct=stop_loss_pct,
            debug=debug
        )
        n_trades = results["n_trades"]
        trades = results.get('trades', [])
        wins = [1 for t in trades if t['exit'] is not None and t['exit'] > t['entry']]
        n_winning_trades = len(wins)
        total_profit = sum(trade['profit_pct'] for trade in trades)
        total_loss = sum(-trade['profit_pct'] for trade in trades if trade['profit_pct'] < 0)
        win_rate = n_winning_trades / n_trades if n_trades > 0 else 0
        avg_trade = total_profit / n_trades if n_trades > 0 else 0
        profit_ratio = total_profit / total_loss if total_loss > 0 else float('inf')
        cumulative_profit = 0
        max_drawdown = 0
        peak = 0
        for trade in trades:
            cumulative_profit += trade['profit_pct']
            if cumulative_profit > peak:
                peak = cumulative_profit
            drawdown = peak - cumulative_profit
            if drawdown > max_drawdown:
                max_drawdown = drawdown
        final_usd = initial_usd
        for trade in trades:
            final_usd *= (1 + trade['profit_pct'])
        row = {
            "timeframe": rule_name,
            "stop_loss_pct": stop_loss_pct,
            "n_trades": n_trades,
            "n_stop_loss": sum(1 for trade in trades if 'type' in trade and trade['type'] == 'STOP'),
            "win_rate": win_rate,
            "max_drawdown": max_drawdown,
            "avg_trade": avg_trade,
            "total_profit": total_profit,
            "total_loss": total_loss,
            "profit_ratio": profit_ratio,            
            "initial_usd": initial_usd,
            "final_usd": final_usd,
        }
        results_rows.append(row)
        for trade in trades:
            trade_rows.append({
                "timeframe": rule_name,
                "stop_loss_pct": stop_loss_pct,
                "entry_time": trade.get("entry_time"),
                "exit_time": trade.get("exit_time"),
                "entry_price": trade.get("entry"),
                "exit_price": trade.get("exit"),
                "profit_pct": trade.get("profit_pct"),
                "type": trade.get("type", ""),
            })
        logging.info(f"Timeframe: {rule_name}, Stop Loss: {stop_loss_pct}, Trades: {n_trades}")
        if debug:
            for trade in trades:
                if trade['type'] == 'STOP':
                    print(trade)
            for trade in trades:
                if trade['profit_pct'] < -0.09:  # or whatever is close to -0.10
                    print("Large loss trade:", trade)
    return results_rows, trade_rows

def process_timeframe(timeframe_info, debug=False):
    """Process a single (timeframe, stop_loss_pct) combination (no monthly split)"""
    rule, data_1min, stop_loss_pct, initial_usd = timeframe_info
    if rule == "1T":
        df = data_1min.copy()
    else:
        df = data_1min.resample(rule).agg({
            'open': 'first',
            'high': 'max',
            'low': 'min',
            'close': 'last',
            'volume': 'sum'
        }).dropna()
    df = df.reset_index()
    # Only process one stop loss
    results_rows, all_trade_rows = process_timeframe_data(data_1min, df, [stop_loss_pct], rule, initial_usd, debug=debug)
    return results_rows, all_trade_rows

def aggregate_results(all_rows):
    """Aggregate results per stop_loss_pct and per rule (timeframe)"""
    from collections import defaultdict

    grouped = defaultdict(list)
    for row in all_rows:
        key = (row['timeframe'], row['stop_loss_pct'])
        grouped[key].append(row)

    summary_rows = []
    for (rule, stop_loss_pct), rows in grouped.items():
        n_months = len(rows)
        total_trades = sum(r['n_trades'] for r in rows)
        total_stop_loss = sum(r['n_stop_loss'] for r in rows)
        avg_win_rate = np.mean([r['win_rate'] for r in rows])
        avg_max_drawdown = np.mean([r['max_drawdown'] for r in rows])
        avg_avg_trade = np.mean([r['avg_trade'] for r in rows])
        avg_profit_ratio = np.mean([r['profit_ratio'] for r in rows])

        # Calculate final USD
        final_usd = np.mean([r.get('final_usd', initial_usd) for r in rows])

        summary_rows.append({
            "timeframe": rule,
            "stop_loss_pct": stop_loss_pct,
            "n_trades": total_trades,
            "n_stop_loss": total_stop_loss,
            "win_rate": avg_win_rate,
            "max_drawdown": avg_max_drawdown,
            "avg_trade": avg_avg_trade,
            "profit_ratio": avg_profit_ratio,
            "initial_usd": initial_usd,
            "final_usd": final_usd,
        })
    return summary_rows

def get_nearest_price(df, target_date):
        if len(df) == 0:
            return None, None
        target_ts = pd.to_datetime(target_date)
        nearest_idx = df.index.get_indexer([target_ts], method='nearest')[0]
        nearest_time = df.index[nearest_idx]
        price = df.iloc[nearest_idx]['close']
        return nearest_time, price

if __name__ == "__main__":
    # Configuration
    # start_date = '2022-01-01'
    # stop_date = '2023-01-01'
    start_date = '2024-05-15'
    stop_date = '2025-05-15'
    initial_usd = 10000
    debug = False
    
    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M")

    storage = Storage(logging=logging)
    system_utils = SystemUtils(logging=logging)

    timeframes = ["1D"]
    stop_loss_pcts = [0.01, 0.02, 0.03]

    # Load data once
    data_1min = storage.load_data('btcusd_1-min_data.csv', start_date, stop_date)

    nearest_start_time, start_price = get_nearest_price(data_1min, start_date)
    nearest_stop_time, stop_price = get_nearest_price(data_1min, stop_date)

    logging.info(f"Price at start_date ({start_date}) [nearest timestamp: {nearest_start_time}]: {start_price}")
    logging.info(f"Price at stop_date ({stop_date}) [nearest timestamp: {nearest_stop_time}]: {stop_price}")
    
    tasks = [
        (name, data_1min, stop_loss_pct, initial_usd)
        for name in timeframes
        for stop_loss_pct in stop_loss_pcts
    ]
    
    workers = system_utils.get_optimal_workers()

    # Start the background batch pusher
    # spreadsheet_name = "GlimBit Backtest Results"
    # batch_pusher = GSheetBatchPusher(results_queue, timestamp, spreadsheet_name, interval=65)
    # batch_pusher.start()
    
    # Process tasks with optimized concurrency
    with concurrent.futures.ProcessPoolExecutor(max_workers=workers) as executor:
        futures = {executor.submit(process_timeframe, task, debug): task for task in tasks}
        all_results_rows = []
        all_trade_rows = []
        for future in concurrent.futures.as_completed(futures):
            results, trades = future.result()
            if results or trades:
                all_results_rows.extend(results)
                all_trade_rows.extend(trades)
                # results_queue.put((results, trades))  # Enqueue for batch update

    # After all tasks, flush any remaining updates
    # batch_pusher.stop()
    # batch_pusher.join()

    # Ensure all batches are pushed, even after 429 errors
    # while not results_queue.empty():
    #     logging.info("Waiting for Google Sheets quota to reset. Retrying batch push in 60 seconds...")
    #     time.sleep(65)
    #     batch_pusher.push_all()

    # Write all results to a single CSV file
    combined_filename = os.path.join(f"{timestamp}_backtest_combined.csv")
    combined_fieldnames = [
        "timeframe", "stop_loss_pct", "n_trades", "n_stop_loss", "win_rate",
        "max_drawdown", "avg_trade", "profit_ratio", "final_usd"
    ]
    storage.write_results_combined(combined_filename, combined_fieldnames, all_results_rows)

    # --- Add taxes to combined results CSV ---
    # taxes = Taxes()  # Default 20% tax rate
    # taxed_filename = combined_filename.replace('.csv', '_taxed.csv')
    # taxes.add_taxes_to_results_csv(combined_filename, taxed_filename, profit_col='total_profit')
    # logging.info(f"Taxed results written to {taxed_filename}")

    # --- Write trades to separate CSVs per timeframe and stop loss ---
    # Collect all trades from each task (need to run tasks to collect trades)
    # Since only all_results_rows is collected above, we need to also collect all trades.
    # To do this, modify the above loop to collect all trades as well.
    # But for now, let's assume you have a list all_trade_rows (list of dicts)
    # If not, you need to collect it in the ProcessPoolExecutor loop above.

    # --- BEGIN: Collect all trades from each task ---
    # To do this, modify the ProcessPoolExecutor loop above:
    # all_results_rows = []
    # all_trade_rows = []
    # ...
    # for future in concurrent.futures.as_completed(futures):
    #     results, trades = future.result()
    #     if results or trades:
    #         all_results_rows.extend(results)
    #         all_trade_rows.extend(trades)
    # --- END: Collect all trades from each task ---

    # Now, group all_trade_rows by (timeframe, stop_loss_pct)


    trades_fieldnames = [
        "entry_time", "exit_time", "entry_price", "exit_price", "profit_pct", "type"
    ]
    storage.write_trades(all_trade_rows, trades_fieldnames)