session hack

BlackLotus · web-flow · commit ce4c44f69528 · 2023-04-02T13:51:43.000+02:00
hacked together in git webui as an example
untested - do not use
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -275,6 +275,7 @@ def _create_completion(
         repeat_penalty: float = 1.1,
         top_k: int = 40,
         stream: bool = False,
+        interactive: bool = False,
     ) -> Union[
         Generator[Completion, None, None],
         Generator[CompletionChunk, None, None],
@@ -299,13 +300,23 @@ def _create_completion(
             stop_sequences = []
 
         finish_reason = None
-        for token in self.generate(
+        if interactive:
+            self._session = self.generate(
+            prompt_tokens,
+            top_k=top_k,
+            top_p=top_p,
+            temp=temperature,
+            repeat_penalty=repeat_penalty,)
+            session = self._session
+        else:
+            session = self.generate(
             prompt_tokens,
             top_k=top_k,
             top_p=top_p,
             temp=temperature,
             repeat_penalty=repeat_penalty,
-        ):
+        )
+        for token in session:
             if token == llama_cpp.llama_token_eos():
                 text = self.detokenize(completion_tokens)
                 finish_reason = "stop"
