Cycles/cycles/utils/system.py

import os
import psutil

class SystemUtils:

    def __init__(self, logging=None):
        self.logging = logging

    def get_optimal_workers(self):
        """Determine optimal number of worker processes based on system resources"""
        cpu_count = os.cpu_count() or 4
        memory_gb = psutil.virtual_memory().total / (1024**3)
        
        # OPTIMIZATION: More aggressive worker allocation for better performance
        workers_by_memory = max(1, int(memory_gb / 2))  # 2GB per worker
        workers_by_cpu = max(1, int(cpu_count * 0.8))  # Use 80% of CPU cores
        optimal_workers = min(workers_by_cpu, workers_by_memory, 8)  # Cap at 8 workers
        
        if self.logging is not None:
            self.logging.info(f"Using {optimal_workers} workers for processing (CPU-based: {workers_by_cpu}, Memory-based: {workers_by_memory})")
        return optimal_workers