21 lines
888 B
Python
21 lines
888 B
Python
import os
|
|
import psutil
|
|
|
|
class SystemUtils:
|
|
|
|
def __init__(self, logging=None):
|
|
self.logging = logging
|
|
|
|
def get_optimal_workers(self):
|
|
"""Determine optimal number of worker processes based on system resources"""
|
|
cpu_count = os.cpu_count() or 4
|
|
memory_gb = psutil.virtual_memory().total / (1024**3)
|
|
|
|
# OPTIMIZATION: More aggressive worker allocation for better performance
|
|
workers_by_memory = max(1, int(memory_gb / 2)) # 2GB per worker
|
|
workers_by_cpu = max(1, int(cpu_count * 0.8)) # Use 80% of CPU cores
|
|
optimal_workers = min(workers_by_cpu, workers_by_memory, 8) # Cap at 8 workers
|
|
|
|
if self.logging is not None:
|
|
self.logging.info(f"Using {optimal_workers} workers for processing (CPU-based: {workers_by_cpu}, Memory-based: {workers_by_memory})")
|
|
return optimal_workers |