Merge branch 'abetlen:main' into main

ryan1117001 · web-flow · commit 8628db83b232 · 2024-08-15T20:11:32.000-04:00
diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py
@@ -179,11 +179,11 @@ def token_eot(self) -> int:
         assert self.model is not None
         return llama_cpp.llama_token_eot(self.model)
 
-    def add_bos_token(self) -> int:
+    def add_bos_token(self) -> bool:
         assert self.model is not None
         return llama_cpp.llama_add_bos_token(self.model)
 
-    def add_eos_token(self) -> int:
+    def add_eos_token(self) -> bool:
         assert self.model is not None
         return llama_cpp.llama_add_eos_token(self.model)
 
@@ -691,8 +691,8 @@ def _detokenize_bpe(model: _LlamaModel, tokens: List[int]) -> str:
 def _should_add_bos(model: _LlamaModel) -> bool:
     assert model.model is not None
     add_bos = llama_cpp.llama_add_bos_token(model.model)
-    if add_bos != -1:
-        return add_bos != 0
+    if add_bos:
+        return add_bos
     else:
         return llama_cpp.llama_vocab_type(model.model) == llama_cpp.LLAMA_VOCAB_TYPE_SPM
 
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -198,6 +198,7 @@ def __init__(
             A Llama instance.
         """
         self.verbose = verbose
+        self._stack = contextlib.ExitStack()
 
         set_verbose(verbose)
 
@@ -365,8 +366,6 @@ def __init__(
         if not os.path.exists(model_path):
             raise ValueError(f"Model path does not exist: {model_path}")
 
-        self._stack = contextlib.ExitStack()
-
         self._model = self._stack.enter_context(
             contextlib.closing(
                 _LlamaModel(
@@ -420,6 +419,15 @@ def __init__(
                 raise RuntimeError(
                     f"Failed to initialize LoRA adapter from lora path: {self.lora_path}"
                 )
+
+            def free_lora_adapter():
+                if self._lora_adapter is None:
+                    return
+                llama_cpp.llama_lora_adapter_free(self._lora_adapter)
+                self._lora_adapter = None
+
+            self._stack.callback(free_lora_adapter)
+
             assert self._ctx.ctx is not None
             if llama_cpp.llama_lora_adapter_set(
                 self._ctx.ctx, self._lora_adapter, self.lora_scale
@@ -1058,13 +1066,13 @@ def _create_completion(
 
         if (
             (isinstance(prompt, list) and suffix is None)
-            or self._model.add_bos_token() == 0
+            or not self._model.add_bos_token()
             or bos_tokens[:1] == [-1]
         ):
             bos_tokens = []
 
         if (isinstance(prompt, list) and suffix is None) or (
-            self._model.add_eos_token() != 1 and sep_token_id == -1
+            not self._model.add_eos_token() and sep_token_id == -1
         ):
             eos_tokens = []
 
@@ -2090,14 +2098,9 @@ def pooling_type(self) -> str:
 
     def close(self) -> None:
         """Explicitly free the model from memory."""
-        if hasattr(self,'_stack'):
-            if self._stack is not None:
-                self._stack.close()
+        self._stack.close()
 
     def __del__(self) -> None:
-        if hasattr(self,'_lora_adapter'):
-            if self._lora_adapter is not None:
-                llama_cpp.llama_lora_adapter_free(self._lora_adapter)
         self.close()
 
     @staticmethod
diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py
@@ -314,6 +314,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa
 #     LLAMA_VOCAB_PRE_TYPE_TEKKEN         = 20,
 #     LLAMA_VOCAB_PRE_TYPE_SMOLLM         = 21,
 #     LLAMA_VOCAB_PRE_TYPE_CODESHELL      = 22,
+#     LLAMA_VOCAB_PRE_TYPE_BLOOM          = 23,
+#     LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH   = 24,
 # };
 LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0
 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1
@@ -338,20 +340,20 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa
 LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20
 LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21
 LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22
+LLAMA_VOCAB_PRE_TYPE_BLOOM = 23
+LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24
 
 
 # // note: these values should be synchronized with ggml_rope
 # // TODO: maybe move this enum to ggml.h (ggml_rope_type)
 # enum llama_rope_type {
 #     LLAMA_ROPE_TYPE_NONE = -1,
 #     LLAMA_ROPE_TYPE_NORM =  0,
-#     LLAMA_ROPE_TYPE_NEOX =  2,
-#     LLAMA_ROPE_TYPE_GLM  =  4,
+#     LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
 # };
 LLAMA_ROPE_TYPE_NONE = -1
 LLAMA_ROPE_TYPE_NORM = 0
-LLAMA_ROPE_TYPE_NEOX = 2
-LLAMA_ROPE_TYPE_GLM = 4
+LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2
 
 
 # enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
@@ -2741,19 +2743,15 @@ def llama_token_nl(model: llama_model_p, /) -> int:
     ...
 
 
-# // Returns -1 if unknown, 1 for true or 0 for false.
-# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model);
-@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_int32)
-def llama_add_bos_token(model: llama_model_p, /) -> int:
-    """Returns -1 if unknown, 1 for true or 0 for false."""
+# LLAMA_API bool llama_add_bos_token(const struct llama_model * model);
+@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool)
+def llama_add_bos_token(model: llama_model_p, /) -> bool:
     ...
 
 
-# // Returns -1 if unknown, 1 for true or 0 for false.
-# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model);
-@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_int32)
-def llama_add_eos_token(model: llama_model_p, /) -> int:
-    """Returns -1 if unknown, 1 for true or 0 for false."""
+# LLAMA_API bool llama_add_eos_token(const struct llama_model * model);
+@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool)
+def llama_add_eos_token(model: llama_model_p, /) -> bool:
     ...
 
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -41,6 +41,10 @@ test = [
     "pytest>=7.4.0",
     "httpx>=0.24.1",
     "scipy>=1.10",
+    "fastapi>=0.100.0",
+    "sse-starlette>=1.6.1",
+    "starlette-context>=0.3.6,<0.4",
+    "pydantic-settings>=2.0.1",
 ]
 dev = [
     "black>=23.3.0",
diff --git a/vendor/llama.cpp b/vendor/llama.cpp
@@ -1 +1 @@
-Subproject commit fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2
+Subproject commit 4b9afbbe9037f8a2d659097c0c7d9fce32c6494c
