- (llama.cpp) Update llama.cpp add support for LLaMa 2 70B

Llama2 70b must set the `n_gqa` parameter (grouped-query attention factor) to 8 when loading:

The low-level API is a direct [`ctypes`](https://docs.python.org/3/library/ctypes.html) binding to the C API provided by `llama.cpp`.

Below is a short example demonstrating how to use the low-level API to tokenize a prompt:

RUN apt update && apt install -y libopenblas-dev ninja-build build-essential

RUN python -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings

CMD python3 -m llama_cpp.server

from . import llama_cpp

from .llama_types import *

import numpy as np

import numpy.typing as npt

class BaseLlamaCache(ABC):

"""Base cache class for a llama.cpp model."""

rope_freq_base: float = 10000.0,

rope_freq_scale: float = 1.0,

n_gqa: Optional[int] = None, # (TEMPORARY) must be 8 for llama2 70b

verbose: bool = True,

):

"""Load a llama.cpp model from `model_path`.

self.params = llama_cpp.llama_context_default_params()

self.params.n_ctx = n_ctx

self.params.seed = seed

self.params.f16_kv = f16_kv

self.params.logits_all = logits_all

self.params.low_vram = low_vram

self.tensor_split = tensor_split

if self.tensor_split is not None:

# Type conversion and expand the list to the length of LLAMA_MAX_DEVICES

if rms_norm_eps is not None:

self.params.rms_norm_eps = rms_norm_eps

self.last_n_tokens_size = last_n_tokens_size

self.n_batch = min(n_ctx, n_batch)

if not os.path.exists(model_path):

raise ValueError(f"Model path does not exist: {model_path}")

assert self.model is not None

assert self.ctx is not None

sorted=sorted,

)

self._candidates = candidates

self._candidates_data_id = np.arange(self._n_vocab, dtype=np.intc) # type: ignore

self._candidates_data_p = np.zeros(self._n_vocab, dtype=np.single)

assert self.ctx is not None

output = b""

for token in tokens:

)

return output

def set_cache(self, cache: Optional[BaseLlamaCache]):

mirostat_eta: llama_cpp.c_float,

penalize_nl: bool = True,

logits_processor: Optional[LogitsProcessorList] = None,

):

assert self.ctx is not None

assert self.n_tokens > 0

)

if not penalize_nl:

candidates.data[self._token_nl].logit = llama_cpp.c_float(nl_logit)

if temp.value == 0.0:

ctx=self.ctx,

candidates=llama_cpp.ctypes.byref(candidates), # type: ignore

)

candidates=llama_cpp.ctypes.byref(candidates), # type: ignore

temp=temp,

)

ctx=self.ctx,

candidates=llama_cpp.ctypes.byref(candidates), # type: ignore

tau=mirostat_tau,

candidates=llama_cpp.ctypes.byref(candidates), # type: ignore

temp=temp,

)

ctx=self.ctx,

candidates=llama_cpp.ctypes.byref(candidates), # type: ignore

tau=mirostat_tau,

candidates=llama_cpp.ctypes.byref(candidates), # type: ignore

temp=temp,

)

ctx=self.ctx,

candidates=llama_cpp.ctypes.byref(candidates), # type: ignore

)

def sample(

self,

mirostat_tau: float = 5.0,

penalize_nl: bool = True,

logits_processor: Optional[LogitsProcessorList] = None,

):

"""Sample a token from the model.

mirostat_eta=llama_cpp.c_float(mirostat_eta),

penalize_nl=penalize_nl,

logits_processor=logits_processor,

)

def generate(

mirostat_eta: float = 0.1,

logits_processor: Optional[LogitsProcessorList] = None,

stopping_criteria: Optional[StoppingCriteriaList] = None,

) -> Generator[int, Optional[Sequence[int]], None]:

"""Create a generator of tokens from a prompt.

assert self.ctx is not None

if reset and len(self._input_ids) > 0:

longest_prefix = 0

for a, b in zip(self._input_ids, tokens[:-1]):

if reset:

self.reset()

while True:

self.eval(tokens)

token = self.sample(

mirostat_tau=mirostat_tau,

mirostat_eta=mirostat_eta,

logits_processor=logits_processor,

)

if stopping_criteria is not None and stopping_criteria(

self._input_ids, self._scores[-1, :]

model: Optional[str] = None,

stopping_criteria: Optional[StoppingCriteriaList] = None,

logits_processor: Optional[LogitsProcessorList] = None,

) -> Union[Iterator[Completion], Iterator[CompletionChunk]]:

assert self.ctx is not None

repeat_penalty=repeat_penalty,

stopping_criteria=stopping_criteria,

logits_processor=logits_processor,

):

if token == self._token_eos:

text = self.detokenize(completion_tokens)

for token in remaining_tokens:

token_end_position += len(self.detokenize([token]))

# Check if stop sequence is in the token

break

logprobs_or_none: Optional[CompletionLogprobs] = None

if logprobs is not None:

model: Optional[str] = None,

stopping_criteria: Optional[StoppingCriteriaList] = None,

logits_processor: Optional[LogitsProcessorList] = None,

) -> Union[Completion, Iterator[CompletionChunk]]:

"""Generate text from a prompt.

model=model,

stopping_criteria=stopping_criteria,

logits_processor=logits_processor,

)

if stream:

chunks: Iterator[CompletionChunk] = completion_or_chunks

model: Optional[str] = None,

stopping_criteria: Optional[StoppingCriteriaList] = None,

logits_processor: Optional[LogitsProcessorList] = None,

) -> Union[Completion, Iterator[CompletionChunk]]:

"""Generate text from a prompt.

model=model,

stopping_criteria=stopping_criteria,

logits_processor=logits_processor,

)

def _convert_text_completion_to_chat(

mirostat_eta: float = 0.1,

model: Optional[str] = None,

logits_processor: Optional[LogitsProcessorList] = None,

) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:

"""Generate a chat completion from a list of messages.

mirostat_eta=mirostat_eta,

model=model,

logits_processor=logits_processor,

)

if stream:

chunks: Iterator[CompletionChunk] = completion_or_chunks # type: ignore

return self._convert_text_completion_to_chat(completion)

def __del__(self):

llama_cpp.llama_free_model(self.model)

self.model = None

llama_cpp.llama_free(self.ctx)

self.ctx = None

assert self.ctx is not None

return LlamaTokenizer(self)

"""Return the end-of-sequence token."""

"""Return the beginning-of-sequence token."""

"""Return the newline token."""

@staticmethod

def logits_to_logprobs(logits: List[float]) -> List[float]: