Safer calculation of default n_threads

abetlen · abetlen · commit da539cc2ee8c · 2023-04-06T21:22:19.000-04:00
diff --git a/llama_cpp/server/__main__.py b/llama_cpp/server/__main__.py
@@ -28,7 +28,7 @@ class Settings(BaseSettings):
     model: str
     n_ctx: int = 2048
     n_batch: int = 8
-    n_threads: int = int(os.cpu_count() / 2) or 1
+    n_threads: int = ((os.cpu_count() or 2) // 2) or 1
     f16_kv: bool = True
     use_mlock: bool = False     # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out...
     embedding: bool = True
