import os import psutil class SystemUtils: def __init__(self, logging=None): self.logging = logging def get_optimal_workers(self): """Determine optimal number of worker processes based on system resources""" cpu_count = os.cpu_count() or 4 memory_gb = psutil.virtual_memory().total / (1024**3) # OPTIMIZATION: More aggressive worker allocation for better performance workers_by_memory = max(1, int(memory_gb / 2)) # 2GB per worker workers_by_cpu = max(1, int(cpu_count * 0.8)) # Use 80% of CPU cores optimal_workers = min(workers_by_cpu, workers_by_memory, 8) # Cap at 8 workers if self.logging is not None: self.logging.info(f"Using {optimal_workers} workers for processing (CPU-based: {workers_by_cpu}, Memory-based: {workers_by_memory})") return optimal_workers