Skip to content

Commit

Permalink
Add async support to semantic cache extension (#214)
Browse files Browse the repository at this point in the history
This PR introduces new async compliant methods to the semantic cache
class using lazy index construction. Because the `AsyncSearchIndex`
requires an async redis python client, we needed to construct that class
lazily upon first usage within the semantic cache class. This PR fixes
some unclosed connection errors and is also in support of
BerriAI/litellm#5412 at LiteLLM.
  • Loading branch information
tylerhutcherson authored Sep 6, 2024
1 parent 5f1e046 commit 973d431
Show file tree
Hide file tree
Showing 10 changed files with 588 additions and 109 deletions.
30 changes: 28 additions & 2 deletions redisvl/extensions/llmcache/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@ def set_ttl(self, ttl: Optional[int] = None):
self._ttl = None

def clear(self) -> None:
"""Clear the LLMCache of all keys in the index."""
"""Clear the cache of all keys in the index."""
raise NotImplementedError

async def aclear(self) -> None:
"""Async clear the cache of all keys in the index."""
raise NotImplementedError

def check(
Expand All @@ -41,6 +45,17 @@ def check(
num_results: int = 1,
return_fields: Optional[List[str]] = None,
) -> List[dict]:
"""Check the cache based on a prompt or vector."""
raise NotImplementedError

async def acheck(
self,
prompt: Optional[str] = None,
vector: Optional[List[float]] = None,
num_results: int = 1,
return_fields: Optional[List[str]] = None,
) -> List[dict]:
"""Async check the cache based on a prompt or vector."""
raise NotImplementedError

def store(
Expand All @@ -50,7 +65,18 @@ def store(
vector: Optional[List[float]] = None,
metadata: Optional[dict] = {},
) -> str:
"""Stores the specified key-value pair in the cache along with
"""Store the specified key-value pair in the cache along with
metadata."""
raise NotImplementedError

async def astore(
self,
prompt: str,
response: str,
vector: Optional[List[float]] = None,
metadata: Optional[dict] = {},
) -> str:
"""Async store the specified key-value pair in the cache along with
metadata."""
raise NotImplementedError

Expand Down
Loading

0 comments on commit 973d431

Please sign in to comment.