Added cache size to settins object.

abetlen · abetlen · commit 14da46f16e46 · 2023-05-07T19:33:17.000-04:00
diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py
@@ -45,6 +45,10 @@ class Settings(BaseSettings):
         default=False,
         description="Use a cache to reduce processing times for evaluated prompts.",
     )
+    cache_size: int = Field(
+        default=2 << 30,
+        description="The size of the cache in bytes. Only used if cache is True.",
+    )
     vocab_only: bool = Field(
         default=False, description="Whether to only return the vocabulary."
     )
@@ -89,7 +93,7 @@ def create_app(settings: Optional[Settings] = None):
         verbose=settings.verbose,
     )
     if settings.cache:
-        cache = llama_cpp.LlamaCache()
+        cache = llama_cpp.LlamaCache(capacity_bytes=settings.cache_size)
         llama.set_cache(cache)
     return app
 

Original file line number	Diff line number	Diff line change
`@@ -45,6 +45,10 @@ class Settings(BaseSettings):`
`45`	`45`	`default=False,`
`46`	`46`	`description="Use a cache to reduce processing times for evaluated prompts.",`
`47`	`47`	`)`
	`48`	`+ cache_size: int = Field(`
	`49`	`+ default=2 << 30,`
	`50`	`+ description="The size of the cache in bytes. Only used if cache is True.",`
	`51`	`+ )`
`48`	`52`	`vocab_only: bool = Field(`
`49`	`53`	`default=False, description="Whether to only return the vocabulary."`
`50`	`54`	`)`
`@@ -89,7 +93,7 @@ def create_app(settings: Optional[Settings] = None):`
`89`	`93`	`verbose=settings.verbose,`
`90`	`94`	`)`
`91`	`95`	`if settings.cache:`
`92`		`- cache = llama_cpp.LlamaCache()`
	`96`	`+ cache = llama_cpp.LlamaCache(capacity_bytes=settings.cache_size)`
`93`	`97`	`llama.set_cache(cache)`
`94`	`98`	`return app`
`95`	`99`