From 58af0379cb7b8a22fb93f9733084bb622ea955d9 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:40:59 -0800 Subject: [PATCH 1/6] update to use route rather than decision --- pyproject.toml | 2 +- semantic_router/__init__.py | 4 +- semantic_router/layer.py | 92 +++++++++++++++++------------------ semantic_router/schema.py | 12 ++--- walkthrough.ipynb | 96 ++++++++++++++++++++++++++----------- 5 files changed, 123 insertions(+), 83 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76ac7539..9abd9c7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "semantic-router" -version = "0.0.5" +version = "0.0.7" description = "Super fast semantic router for AI decision making" authors = [ "James Briggs ", diff --git a/semantic_router/__init__.py b/semantic_router/__init__.py index 734906f8..b10cc687 100644 --- a/semantic_router/__init__.py +++ b/semantic_router/__init__.py @@ -1,3 +1,3 @@ -from .layer import DecisionLayer, HybridDecisionLayer +from .layer import RouteLayer, HybridRouteLayer -__all__ = ["DecisionLayer", "HybridDecisionLayer"] +__all__ = ["RouteLayer", "HybridRouteLayer"] diff --git a/semantic_router/layer.py b/semantic_router/layer.py index adff961c..c29f190f 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -9,15 +9,15 @@ BM25Encoder, ) from semantic_router.linear import similarity_matrix, top_scores -from semantic_router.schema import Decision +from semantic_router.schema import Route -class DecisionLayer: +class RouteLayer: index = None categories = None score_threshold = 0.82 - def __init__(self, encoder: BaseEncoder, decisions: list[Decision] = []): + def __init__(self, encoder: BaseEncoder, routes: list[Route] = []): self.encoder = encoder # decide on default threshold based on encoder if isinstance(encoder, OpenAIEncoder): @@ -26,11 +26,11 @@ def __init__(self, encoder: BaseEncoder, decisions: list[Decision] = []): self.score_threshold = 0.3 else: self.score_threshold = 0.82 - # if decisions list has been passed, we initialize index now - if decisions: + # if routes list has been passed, we initialize index now + if routes: # initialize index now - for decision in tqdm(decisions): - self._add_decision(decision=decision) + for route in tqdm(routes): + self._add_route(route=route) def __call__(self, text: str) -> str | None: results = self._query(text) @@ -41,18 +41,18 @@ def __call__(self, text: str) -> str | None: else: return None - def add(self, decision: Decision): - self._add_decision(decision=decision) + def add(self, route: Route): + self._add_route(route=route) - def _add_decision(self, decision: Decision): + def _add_route(self, route: Route): # create embeddings - embeds = self.encoder(decision.utterances) + embeds = self.encoder(route.utterances) - # create decision array + # create route array if self.categories is None: - self.categories = np.array([decision.name] * len(embeds)) + self.categories = np.array([route.name] * len(embeds)) else: - str_arr = np.array([decision.name] * len(embeds)) + str_arr = np.array([route.name] * len(embeds)) self.categories = np.concatenate([self.categories, str_arr]) # create utterance array (the index) if self.index is None: @@ -73,10 +73,10 @@ def _query(self, text: str, top_k: int = 5): # calculate similarity matrix sim = similarity_matrix(xq, self.index) scores, idx = top_scores(sim, top_k) - # get the utterance categories (decision names) - decisions = self.categories[idx] if self.categories is not None else [] + # get the utterance categories (route names) + routes = self.categories[idx] if self.categories is not None else [] return [ - {"decision": d, "score": s.item()} for d, s in zip(decisions, scores) + {"route": d, "score": s.item()} for d, s in zip(routes, scores) ] else: return [] @@ -85,15 +85,15 @@ def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float scores_by_class = {} for result in query_results: score = result["score"] - decision = result["decision"] - if decision in scores_by_class: - scores_by_class[decision].append(score) + route = result["route"] + if route in scores_by_class: + scores_by_class[route].append(score) else: - scores_by_class[decision] = [score] + scores_by_class[route] = [score] # Calculate total score for each class total_scores = { - decision: sum(scores) for decision, scores in scores_by_class.items() + route: sum(scores) for route, scores in scores_by_class.items() } top_class = max(total_scores, key=lambda x: total_scores[x], default=None) @@ -107,14 +107,14 @@ def _pass_threshold(self, scores: list[float], threshold: float) -> bool: return False -class HybridDecisionLayer: +class HybridRouteLayer: index = None sparse_index = None categories = None score_threshold = 0.82 def __init__( - self, encoder: BaseEncoder, decisions: list[Decision] = [], alpha: float = 0.3 + self, encoder: BaseEncoder, routes: list[Route] = [], alpha: float = 0.3 ): self.encoder = encoder self.sparse_encoder = BM25Encoder() @@ -126,11 +126,11 @@ def __init__( self.score_threshold = 0.3 else: self.score_threshold = 0.82 - # if decisions list has been passed, we initialize index now - if decisions: + # if routes list has been passed, we initialize index now + if routes: # initialize index now - for decision in tqdm(decisions): - self._add_decision(decision=decision) + for route in tqdm(routes): + self._add_route(route=route) def __call__(self, text: str) -> str | None: results = self._query(text) @@ -141,25 +141,25 @@ def __call__(self, text: str) -> str | None: else: return None - def add(self, decision: Decision): - self._add_decision(decision=decision) + def add(self, route: Route): + self._add_route(route=route) - def _add_decision(self, decision: Decision): + def _add_route(self, route: Route): # create embeddings - dense_embeds = np.array(self.encoder(decision.utterances)) # * self.alpha + dense_embeds = np.array(self.encoder(route.utterances)) # * self.alpha sparse_embeds = np.array( - self.sparse_encoder(decision.utterances) + self.sparse_encoder(route.utterances) ) # * (1 - self.alpha) - # create decision array + # create route array if self.categories is None: - self.categories = np.array([decision.name] * len(decision.utterances)) - self.utterances = np.array(decision.utterances) + self.categories = np.array([route.name] * len(route.utterances)) + self.utterances = np.array(route.utterances) else: - str_arr = np.array([decision.name] * len(decision.utterances)) + str_arr = np.array([route.name] * len(route.utterances)) self.categories = np.concatenate([self.categories, str_arr]) self.utterances = np.concatenate( - [self.utterances, np.array(decision.utterances)] + [self.utterances, np.array(route.utterances)] ) # create utterance array (the dense index) if self.index is None: @@ -199,10 +199,10 @@ def _query(self, text: str, top_k: int = 5): top_k = min(top_k, total_sim.shape[0]) idx = np.argpartition(total_sim, -top_k)[-top_k:] scores = total_sim[idx] - # get the utterance categories (decision names) - decisions = self.categories[idx] if self.categories is not None else [] + # get the utterance categories (route names) + routes = self.categories[idx] if self.categories is not None else [] return [ - {"decision": d, "score": s.item()} for d, s in zip(decisions, scores) + {"route": d, "score": s.item()} for d, s in zip(routes, scores) ] else: return [] @@ -217,15 +217,15 @@ def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float scores_by_class = {} for result in query_results: score = result["score"] - decision = result["decision"] - if decision in scores_by_class: - scores_by_class[decision].append(score) + route = result["route"] + if route in scores_by_class: + scores_by_class[route].append(score) else: - scores_by_class[decision] = [score] + scores_by_class[route] = [score] # Calculate total score for each class total_scores = { - decision: sum(scores) for decision, scores in scores_by_class.items() + route: sum(scores) for route, scores in scores_by_class.items() } top_class = max(total_scores, key=lambda x: total_scores[x], default=None) diff --git a/semantic_router/schema.py b/semantic_router/schema.py index 37a43dd4..3763db03 100644 --- a/semantic_router/schema.py +++ b/semantic_router/schema.py @@ -10,7 +10,7 @@ ) -class Decision(BaseModel): +class Route(BaseModel): name: str utterances: list[str] description: str | None = None @@ -45,12 +45,12 @@ def __call__(self, texts: list[str]) -> list[float]: @dataclass class SemanticSpace: id: str - decisions: list[Decision] + routes: list[Route] encoder: str = "" - def __init__(self, decisions: list[Decision] = []): + def __init__(self, routes: list[Route] = []): self.id = "" - self.decisions = decisions + self.routes = routes - def add(self, decision: Decision): - self.decisions.append(decision) + def add(self, route: Route): + self.routes.append(route) diff --git a/walkthrough.ipynb b/walkthrough.ipynb index 81bb3ec2..6731ee0a 100644 --- a/walkthrough.ipynb +++ b/walkthrough.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The Semantic Router library can be used as a super fast decision making layer on top of LLMs. That means rather than waiting on a slow agent to decide what to do, we can use the magic of semantic vector space to make decisions. Cutting decision making time down from seconds to milliseconds." + "The Semantic Router library can be used as a super fast route making layer on top of LLMs. That means rather than waiting on a slow agent to decide what to do, we can use the magic of semantic vector space to make routes. Cutting route making time down from seconds to milliseconds." ] }, { @@ -41,18 +41,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We start by defining a dictionary mapping decisions to example phrases that should trigger those decisions." + "We start by defining a dictionary mapping routes to example phrases that should trigger those routes." ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantic_router.schema import Decision\n", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jamesbriggs/opt/anaconda3/envs/decision-layer/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n" + ] + } + ], + "source": [ + "from semantic_router.schema import Route\n", "\n", - "politics = Decision(\n", + "politics = Route(\n", " name=\"politics\",\n", " utterances=[\n", " \"isn't politics the best thing ever\",\n", @@ -74,11 +84,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "chitchat = Decision(\n", + "chitchat = Route(\n", " name=\"chitchat\",\n", " utterances=[\n", " \"how's the weather today?\",\n", @@ -89,7 +99,7 @@ " ]\n", ")\n", "\n", - "decisions = [politics, chitchat]" + "routes = [politics, chitchat]" ] }, { @@ -101,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -109,7 +119,7 @@ "from getpass import getpass\n", "import os\n", "\n", - "os.environ[\"COHERE_API_KEY\"] = os.environ[\"COHERE_API_KEY\"] or \\\n", + "os.environ[\"COHERE_API_KEY\"] = os.getenv(\"COHERE_API_KEY\") or \\\n", " getpass(\"Enter Cohere API Key: \")\n", "\n", "encoder = CohereEncoder()" @@ -119,18 +129,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now we define the `DecisionLayer`. When called, the decision layer will consume text (a query) and output the category (`Decision`) it belongs to — to initialize a `DecisionLayer` we need our `encoder` model and a list of `decisions`." + "Now we define the `RouteLayer`. When called, the route layer will consume text (a query) and output the category (`Route`) it belongs to — to initialize a `RouteLayer` we need our `encoder` model and a list of `routes`." ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantic_router import DecisionLayer\n", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 2/2 [00:01<00:00, 1.04it/s]\n" + ] + } + ], + "source": [ + "from semantic_router.layer import RouteLayer\n", "\n", - "dl = DecisionLayer(encoder=encoder, decisions=decisions)" + "dl = RouteLayer(encoder=encoder, routes=routes)" ] }, { @@ -142,18 +160,40 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'politics'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dl(\"don't you love politics?\")" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'chitchat'" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dl(\"how's the weather today?\")" ] @@ -162,12 +202,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Both are classified accurately, what if we send a query that is unrelated to our existing `Decision` objects?" + "Both are classified accurately, what if we send a query that is unrelated to our existing `Route` objects?" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ From 4570bc5f66109dfa9336ca2b5fb0ee60aafe6941 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:44:10 -0800 Subject: [PATCH 2/6] use routes --- tests/unit/test_layer.py | 220 +++++++++++++++++++------------------- tests/unit/test_schema.py | 18 ++-- 2 files changed, 119 insertions(+), 119 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index a746c4ec..0571071a 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -2,10 +2,10 @@ from semantic_router.encoders import BaseEncoder, CohereEncoder, OpenAIEncoder from semantic_router.layer import ( - DecisionLayer, - HybridDecisionLayer, + RouteLayer, + HybridRouteLayer, ) # Replace with the actual module name -from semantic_router.schema import Decision +from semantic_router.schema import Route def mock_encoder_call(utterances): @@ -38,159 +38,159 @@ def openai_encoder(mocker): @pytest.fixture -def decisions(): +def routes(): return [ - Decision(name="Decision 1", utterances=["Hello", "Hi"]), - Decision(name="Decision 2", utterances=["Goodbye", "Bye", "Au revoir"]), + Route(name="Route 1", utterances=["Hello", "Hi"]), + Route(name="Route 2", utterances=["Goodbye", "Bye", "Au revoir"]), ] -class TestDecisionLayer: - def test_initialization(self, openai_encoder, decisions): - decision_layer = DecisionLayer(encoder=openai_encoder, decisions=decisions) - assert decision_layer.score_threshold == 0.82 - assert len(decision_layer.index) == 5 - assert len(set(decision_layer.categories)) == 2 +class TestRouteLayer: + def test_initialization(self, openai_encoder, routes): + route_layer = RouteLayer(encoder=openai_encoder, routes=routes) + assert route_layer.score_threshold == 0.82 + assert len(route_layer.index) == 5 + assert len(set(route_layer.categories)) == 2 def test_initialization_different_encoders(self, cohere_encoder, openai_encoder): - decision_layer_cohere = DecisionLayer(encoder=cohere_encoder) - assert decision_layer_cohere.score_threshold == 0.3 - - decision_layer_openai = DecisionLayer(encoder=openai_encoder) - assert decision_layer_openai.score_threshold == 0.82 - - def test_add_decision(self, openai_encoder): - decision_layer = DecisionLayer(encoder=openai_encoder) - decision = Decision(name="Decision 3", utterances=["Yes", "No"]) - decision_layer.add(decision) - assert len(decision_layer.index) == 2 - assert len(set(decision_layer.categories)) == 1 - - def test_add_multiple_decisions(self, openai_encoder, decisions): - decision_layer = DecisionLayer(encoder=openai_encoder) - for decision in decisions: - decision_layer.add(decision) - assert len(decision_layer.index) == 5 - assert len(set(decision_layer.categories)) == 2 - - def test_query_and_classification(self, openai_encoder, decisions): - decision_layer = DecisionLayer(encoder=openai_encoder, decisions=decisions) - query_result = decision_layer("Hello") - assert query_result in ["Decision 1", "Decision 2"] + route_layer_cohere = RouteLayer(encoder=cohere_encoder) + assert route_layer_cohere.score_threshold == 0.3 + + route_layer_openai = RouteLayer(encoder=openai_encoder) + assert route_layer_openai.score_threshold == 0.82 + + def test_add_route(self, openai_encoder): + route_layer = RouteLayer(encoder=openai_encoder) + route = Route(name="Route 3", utterances=["Yes", "No"]) + route_layer.add(route) + assert len(route_layer.index) == 2 + assert len(set(route_layer.categories)) == 1 + + def test_add_multiple_routes(self, openai_encoder, routes): + route_layer = RouteLayer(encoder=openai_encoder) + for route in routes: + route_layer.add(route) + assert len(route_layer.index) == 5 + assert len(set(route_layer.categories)) == 2 + + def test_query_and_classification(self, openai_encoder, routes): + route_layer = RouteLayer(encoder=openai_encoder, routes=routes) + query_result = route_layer("Hello") + assert query_result in ["Route 1", "Route 2"] def test_query_with_no_index(self, openai_encoder): - decision_layer = DecisionLayer(encoder=openai_encoder) - assert decision_layer("Anything") is None + route_layer = RouteLayer(encoder=openai_encoder) + assert route_layer("Anything") is None - def test_semantic_classify(self, openai_encoder, decisions): - decision_layer = DecisionLayer(encoder=openai_encoder, decisions=decisions) - classification, score = decision_layer._semantic_classify( + def test_semantic_classify(self, openai_encoder, routes): + route_layer = RouteLayer(encoder=openai_encoder, routes=routes) + classification, score = route_layer._semantic_classify( [ - {"decision": "Decision 1", "score": 0.9}, - {"decision": "Decision 2", "score": 0.1}, + {"route": "Route 1", "score": 0.9}, + {"route": "Route 2", "score": 0.1}, ] ) - assert classification == "Decision 1" + assert classification == "Route 1" assert score == [0.9] - def test_semantic_classify_multiple_decisions(self, openai_encoder, decisions): - decision_layer = DecisionLayer(encoder=openai_encoder, decisions=decisions) - classification, score = decision_layer._semantic_classify( + def test_semantic_classify_multiple_routes(self, openai_encoder, routes): + route_layer = RouteLayer(encoder=openai_encoder, routes=routes) + classification, score = route_layer._semantic_classify( [ - {"decision": "Decision 1", "score": 0.9}, - {"decision": "Decision 2", "score": 0.1}, - {"decision": "Decision 1", "score": 0.8}, + {"route": "Route 1", "score": 0.9}, + {"route": "Route 2", "score": 0.1}, + {"route": "Route 1", "score": 0.8}, ] ) - assert classification == "Decision 1" + assert classification == "Route 1" assert score == [0.9, 0.8] def test_pass_threshold(self, openai_encoder): - decision_layer = DecisionLayer(encoder=openai_encoder) - assert not decision_layer._pass_threshold([], 0.5) - assert decision_layer._pass_threshold([0.6, 0.7], 0.5) + route_layer = RouteLayer(encoder=openai_encoder) + assert not route_layer._pass_threshold([], 0.5) + assert route_layer._pass_threshold([0.6, 0.7], 0.5) def test_failover_score_threshold(self, base_encoder): - decision_layer = DecisionLayer(encoder=base_encoder) - assert decision_layer.score_threshold == 0.82 + route_layer = RouteLayer(encoder=base_encoder) + assert route_layer.score_threshold == 0.82 -class TestHybridDecisionLayer: - def test_initialization(self, openai_encoder, decisions): - decision_layer = HybridDecisionLayer( - encoder=openai_encoder, decisions=decisions +class TestHybridRouteLayer: + def test_initialization(self, openai_encoder, routes): + route_layer = HybridRouteLayer( + encoder=openai_encoder, routes=routes ) - assert decision_layer.score_threshold == 0.82 - assert len(decision_layer.index) == 5 - assert len(set(decision_layer.categories)) == 2 + assert route_layer.score_threshold == 0.82 + assert len(route_layer.index) == 5 + assert len(set(route_layer.categories)) == 2 def test_initialization_different_encoders(self, cohere_encoder, openai_encoder): - decision_layer_cohere = HybridDecisionLayer(encoder=cohere_encoder) - assert decision_layer_cohere.score_threshold == 0.3 - - decision_layer_openai = HybridDecisionLayer(encoder=openai_encoder) - assert decision_layer_openai.score_threshold == 0.82 - - def test_add_decision(self, openai_encoder): - decision_layer = HybridDecisionLayer(encoder=openai_encoder) - decision = Decision(name="Decision 3", utterances=["Yes", "No"]) - decision_layer.add(decision) - assert len(decision_layer.index) == 2 - assert len(set(decision_layer.categories)) == 1 - - def test_add_multiple_decisions(self, openai_encoder, decisions): - decision_layer = HybridDecisionLayer(encoder=openai_encoder) - for decision in decisions: - decision_layer.add(decision) - assert len(decision_layer.index) == 5 - assert len(set(decision_layer.categories)) == 2 - - def test_query_and_classification(self, openai_encoder, decisions): - decision_layer = HybridDecisionLayer( - encoder=openai_encoder, decisions=decisions + route_layer_cohere = HybridRouteLayer(encoder=cohere_encoder) + assert route_layer_cohere.score_threshold == 0.3 + + route_layer_openai = HybridRouteLayer(encoder=openai_encoder) + assert route_layer_openai.score_threshold == 0.82 + + def test_add_route(self, openai_encoder): + route_layer = HybridRouteLayer(encoder=openai_encoder) + route = Route(name="Route 3", utterances=["Yes", "No"]) + route_layer.add(route) + assert len(route_layer.index) == 2 + assert len(set(route_layer.categories)) == 1 + + def test_add_multiple_routes(self, openai_encoder, routes): + route_layer = HybridRouteLayer(encoder=openai_encoder) + for route in routes: + route_layer.add(route) + assert len(route_layer.index) == 5 + assert len(set(route_layer.categories)) == 2 + + def test_query_and_classification(self, openai_encoder, routes): + route_layer = HybridRouteLayer( + encoder=openai_encoder, routes=routes ) - query_result = decision_layer("Hello") - assert query_result in ["Decision 1", "Decision 2"] + query_result = route_layer("Hello") + assert query_result in ["Route 1", "Route 2"] def test_query_with_no_index(self, openai_encoder): - decision_layer = HybridDecisionLayer(encoder=openai_encoder) - assert decision_layer("Anything") is None + route_layer = HybridRouteLayer(encoder=openai_encoder) + assert route_layer("Anything") is None - def test_semantic_classify(self, openai_encoder, decisions): - decision_layer = HybridDecisionLayer( - encoder=openai_encoder, decisions=decisions + def test_semantic_classify(self, openai_encoder, routes): + route_layer = HybridRouteLayer( + encoder=openai_encoder, routes=routes ) - classification, score = decision_layer._semantic_classify( + classification, score = route_layer._semantic_classify( [ - {"decision": "Decision 1", "score": 0.9}, - {"decision": "Decision 2", "score": 0.1}, + {"route": "Route 1", "score": 0.9}, + {"route": "Route 2", "score": 0.1}, ] ) - assert classification == "Decision 1" + assert classification == "Route 1" assert score == [0.9] - def test_semantic_classify_multiple_decisions(self, openai_encoder, decisions): - decision_layer = HybridDecisionLayer( - encoder=openai_encoder, decisions=decisions + def test_semantic_classify_multiple_routes(self, openai_encoder, routes): + route_layer = HybridRouteLayer( + encoder=openai_encoder, routes=routes ) - classification, score = decision_layer._semantic_classify( + classification, score = route_layer._semantic_classify( [ - {"decision": "Decision 1", "score": 0.9}, - {"decision": "Decision 2", "score": 0.1}, - {"decision": "Decision 1", "score": 0.8}, + {"route": "Route 1", "score": 0.9}, + {"route": "Route 2", "score": 0.1}, + {"route": "Route 1", "score": 0.8}, ] ) - assert classification == "Decision 1" + assert classification == "Route 1" assert score == [0.9, 0.8] def test_pass_threshold(self, openai_encoder): - decision_layer = HybridDecisionLayer(encoder=openai_encoder) - assert not decision_layer._pass_threshold([], 0.5) - assert decision_layer._pass_threshold([0.6, 0.7], 0.5) + route_layer = HybridRouteLayer(encoder=openai_encoder) + assert not route_layer._pass_threshold([], 0.5) + assert route_layer._pass_threshold([0.6, 0.7], 0.5) def test_failover_score_threshold(self, base_encoder): - decision_layer = HybridDecisionLayer(encoder=base_encoder) - assert decision_layer.score_threshold == 0.82 + route_layer = HybridRouteLayer(encoder=base_encoder) + assert route_layer.score_threshold == 0.82 # Add more tests for edge cases and error handling as needed. diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index 2563bf0b..9128835a 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -2,7 +2,7 @@ from semantic_router.schema import ( CohereEncoder, - Decision, + Route, Encoder, EncoderType, OpenAIEncoder, @@ -46,16 +46,16 @@ class TestSemanticSpaceDataclass: def test_semanticspace_initialization(self): semantic_space = SemanticSpace() assert semantic_space.id == "" - assert semantic_space.decisions == [] + assert semantic_space.routes == [] - def test_semanticspace_add_decision(self): - decision = Decision( + def test_semanticspace_add_route(self): + route = Route( name="test", utterances=["hello", "hi"], description="greeting" ) semantic_space = SemanticSpace() - semantic_space.add(decision) + semantic_space.add(route) - assert len(semantic_space.decisions) == 1 - assert semantic_space.decisions[0].name == "test" - assert semantic_space.decisions[0].utterances == ["hello", "hi"] - assert semantic_space.decisions[0].description == "greeting" + assert len(semantic_space.routes) == 1 + assert semantic_space.routes[0].name == "test" + assert semantic_space.routes[0].utterances == ["hello", "hi"] + assert semantic_space.routes[0].description == "greeting" From 5aa6c77e8772f8611f3a70d82c9101f8fb01e9d6 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:47:09 -0800 Subject: [PATCH 3/6] lint --- semantic_router/layer.py | 16 ++++------------ tests/unit/test_layer.py | 16 ++++------------ tests/unit/test_schema.py | 4 +--- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index c29f190f..23e3ec69 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -75,9 +75,7 @@ def _query(self, text: str, top_k: int = 5): scores, idx = top_scores(sim, top_k) # get the utterance categories (route names) routes = self.categories[idx] if self.categories is not None else [] - return [ - {"route": d, "score": s.item()} for d, s in zip(routes, scores) - ] + return [{"route": d, "score": s.item()} for d, s in zip(routes, scores)] else: return [] @@ -92,9 +90,7 @@ def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float scores_by_class[route] = [score] # Calculate total score for each class - total_scores = { - route: sum(scores) for route, scores in scores_by_class.items() - } + total_scores = {route: sum(scores) for route, scores in scores_by_class.items()} top_class = max(total_scores, key=lambda x: total_scores[x], default=None) # Return the top class and its associated scores @@ -201,9 +197,7 @@ def _query(self, text: str, top_k: int = 5): scores = total_sim[idx] # get the utterance categories (route names) routes = self.categories[idx] if self.categories is not None else [] - return [ - {"route": d, "score": s.item()} for d, s in zip(routes, scores) - ] + return [{"route": d, "score": s.item()} for d, s in zip(routes, scores)] else: return [] @@ -224,9 +218,7 @@ def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float scores_by_class[route] = [score] # Calculate total score for each class - total_scores = { - route: sum(scores) for route, scores in scores_by_class.items() - } + total_scores = {route: sum(scores) for route, scores in scores_by_class.items()} top_class = max(total_scores, key=lambda x: total_scores[x], default=None) # Return the top class and its associated scores diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 0571071a..34ee8b99 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -117,9 +117,7 @@ def test_failover_score_threshold(self, base_encoder): class TestHybridRouteLayer: def test_initialization(self, openai_encoder, routes): - route_layer = HybridRouteLayer( - encoder=openai_encoder, routes=routes - ) + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) assert route_layer.score_threshold == 0.82 assert len(route_layer.index) == 5 assert len(set(route_layer.categories)) == 2 @@ -146,9 +144,7 @@ def test_add_multiple_routes(self, openai_encoder, routes): assert len(set(route_layer.categories)) == 2 def test_query_and_classification(self, openai_encoder, routes): - route_layer = HybridRouteLayer( - encoder=openai_encoder, routes=routes - ) + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) query_result = route_layer("Hello") assert query_result in ["Route 1", "Route 2"] @@ -157,9 +153,7 @@ def test_query_with_no_index(self, openai_encoder): assert route_layer("Anything") is None def test_semantic_classify(self, openai_encoder, routes): - route_layer = HybridRouteLayer( - encoder=openai_encoder, routes=routes - ) + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) classification, score = route_layer._semantic_classify( [ {"route": "Route 1", "score": 0.9}, @@ -170,9 +164,7 @@ def test_semantic_classify(self, openai_encoder, routes): assert score == [0.9] def test_semantic_classify_multiple_routes(self, openai_encoder, routes): - route_layer = HybridRouteLayer( - encoder=openai_encoder, routes=routes - ) + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) classification, score = route_layer._semantic_classify( [ {"route": "Route 1", "score": 0.9}, diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index 9128835a..7c954805 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -49,9 +49,7 @@ def test_semanticspace_initialization(self): assert semantic_space.routes == [] def test_semanticspace_add_route(self): - route = Route( - name="test", utterances=["hello", "hi"], description="greeting" - ) + route = Route(name="test", utterances=["hello", "hi"], description="greeting") semantic_space = SemanticSpace() semantic_space.add(route) From 9c4cdf8531581b4c8f6348f5570922b36790e326 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:01:07 +0200 Subject: [PATCH 4/6] fix tests --- coverage.xml | 211 ++++++++++++++++--------------- docs/examples/hybrid-layer.ipynb | 56 ++++---- semantic_router/__init__.py | 3 +- semantic_router/hybrid_layer.py | 139 ++++++++++++++++++++ semantic_router/layer.py | 131 ------------------- tests/unit/test_hybrid_layer.py | 118 +++++++++++++++++ tests/unit/test_layer.py | 80 +----------- tests/unit/test_schema.py | 2 +- walkthrough.ipynb | 2 +- 9 files changed, 401 insertions(+), 341 deletions(-) create mode 100644 semantic_router/hybrid_layer.py create mode 100644 tests/unit/test_hybrid_layer.py diff --git a/coverage.xml b/coverage.xml index 3c9c2e7c..612cac46 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -12,10 +12,11 @@ - + + - + @@ -23,149 +24,157 @@ - + - - + + - - + + + + + - + - - - + + - - + + + - - - + + + + + + + + - + + - + + + - + - - - + + + + + + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/examples/hybrid-layer.ipynb b/docs/examples/hybrid-layer.ipynb index 98fccf69..8b1da5ae 100644 --- a/docs/examples/hybrid-layer.ipynb +++ b/docs/examples/hybrid-layer.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The Hybrid Layer in the Semantic Router library can improve decision making performance particularly for niche use-cases that contain specific terminology, such as finance or medical. It helps us provide more importance to decision making based on the keywords contained in our utterances and user queries." + "The Hybrid Layer in the Semantic Router library can improve making performance particularly for niche use-cases that contain specific terminology, such as finance or medical. It helps us provide more importance to making based on the keywords contained in our utterances and user queries." ] }, { @@ -34,36 +34,37 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install -qU semantic-router==0.0.5" + "!pip install -qU semantic-router==0.0.6" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We start by defining a dictionary mapping decisions to example phrases that should trigger those decisions." + "We start by defining a dictionary mapping s to example phrases that should trigger those s." ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "ename": "ImportError", + "evalue": "cannot import name 'Route' from 'semantic_router.schema' (/Users/jakit/customers/aurelio/semantic-router/.venv/lib/python3.11/site-packages/semantic_router/schema.py)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/Users/jakit/customers/aurelio/semantic-router/docs/examples/hybrid-layer.ipynb Cell 7\u001b[0m line \u001b[0;36m1\n\u001b[0;32m----> 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39msemantic_router\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mschema\u001b[39;00m \u001b[39mimport\u001b[39;00m Route\n\u001b[1;32m 3\u001b[0m politics \u001b[39m=\u001b[39m Route(\n\u001b[1;32m 4\u001b[0m name\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mpolitics\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m 5\u001b[0m utterances\u001b[39m=\u001b[39m[\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 12\u001b[0m ],\n\u001b[1;32m 13\u001b[0m )\n", + "\u001b[0;31mImportError\u001b[0m: cannot import name 'Route' from 'semantic_router.schema' (/Users/jakit/customers/aurelio/semantic-router/.venv/lib/python3.11/site-packages/semantic_router/schema.py)" + ] + } + ], + "source": [ + "from semantic_router.schema import Route\n", "\n", - "os.environ[\"COHERE_API_KEY\"] = \"BQBiUqqjDRsYl1QKKux4JsqKdDkjyInS5T3Z3eJP\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantic_router.schema import Decision\n", - "\n", - "politics = Decision(\n", + "politics = Route(\n", " name=\"politics\",\n", " utterances=[\n", " \"isn't politics the best thing ever\",\n", @@ -89,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "chitchat = Decision(\n", + "chitchat = Route(\n", " name=\"chitchat\",\n", " utterances=[\n", " \"how's the weather today?\",\n", @@ -100,7 +101,7 @@ " ],\n", ")\n", "\n", - "chitchat = Decision(\n", + "chitchat = Route(\n", " name=\"chitchat\",\n", " utterances=[\n", " \"how's the weather today?\",\n", @@ -111,7 +112,7 @@ " ],\n", ")\n", "\n", - "decisions = [politics, chitchat]" + "routes = [politics, chitchat]" ] }, { @@ -127,6 +128,7 @@ "metadata": {}, "outputs": [], "source": [ + "import os\n", "from semantic_router.encoders import CohereEncoder\n", "from getpass import getpass\n", "\n", @@ -141,7 +143,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now we define the `DecisionLayer`. When called, the decision layer will consume text (a query) and output the category (`Decision`) it belongs to — to initialize a `DecisionLayer` we need our `encoder` model and a list of `decisions`." + "Now we define the `RouteLayer`. When called, the route layer will consume text (a query) and output the category (`Route`) it belongs to — to initialize a `RouteLayer` we need our `encoder` model and a list of `routes`." ] }, { @@ -150,9 +152,9 @@ "metadata": {}, "outputs": [], "source": [ - "from semantic_router.layer import HybridDecisionLayer\n", + "from semantic_router.hybrid_layer import HybridRouteLayer\n", "\n", - "dl = HybridDecisionLayer(encoder=encoder, decisions=decisions)" + "dl = HybridRouteLayer(encoder=encoder, routes=routes)" ] }, { @@ -197,7 +199,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/semantic_router/__init__.py b/semantic_router/__init__.py index b10cc687..0c445bea 100644 --- a/semantic_router/__init__.py +++ b/semantic_router/__init__.py @@ -1,3 +1,4 @@ -from .layer import RouteLayer, HybridRouteLayer +from .hybrid_layer import HybridRouteLayer +from .layer import RouteLayer __all__ = ["RouteLayer", "HybridRouteLayer"] diff --git a/semantic_router/hybrid_layer.py b/semantic_router/hybrid_layer.py new file mode 100644 index 00000000..3e9508d6 --- /dev/null +++ b/semantic_router/hybrid_layer.py @@ -0,0 +1,139 @@ +import numpy as np +from numpy.linalg import norm +from tqdm.auto import tqdm + +from semantic_router.encoders import ( + BaseEncoder, + BM25Encoder, + CohereEncoder, + OpenAIEncoder, +) +from semantic_router.schema import Route + + +class HybridRouteLayer: + index = None + sparse_index = None + categories = None + score_threshold = 0.82 + + def __init__( + self, encoder: BaseEncoder, routes: list[Route] = [], alpha: float = 0.3 + ): + self.encoder = encoder + self.sparse_encoder = BM25Encoder() + self.alpha = alpha + # decide on default threshold based on encoder + if isinstance(encoder, OpenAIEncoder): + self.score_threshold = 0.82 + elif isinstance(encoder, CohereEncoder): + self.score_threshold = 0.3 + else: + self.score_threshold = 0.82 + # if routes list has been passed, we initialize index now + if routes: + # initialize index now + for route in tqdm(routes): + self._add_route(route=route) + + def __call__(self, text: str) -> str | None: + results = self._query(text) + top_class, top_class_scores = self._semantic_classify(results) + passed = self._pass_threshold(top_class_scores, self.score_threshold) + if passed: + return top_class + else: + return None + + def add(self, route: Route): + self._add_route(route=route) + + def _add_route(self, route: Route): + # create embeddings + dense_embeds = np.array(self.encoder(route.utterances)) # * self.alpha + sparse_embeds = np.array( + self.sparse_encoder(route.utterances) + ) # * (1 - self.alpha) + + # create route array + if self.categories is None: + self.categories = np.array([route.name] * len(route.utterances)) + self.utterances = np.array(route.utterances) + else: + str_arr = np.array([route.name] * len(route.utterances)) + self.categories = np.concatenate([self.categories, str_arr]) + self.utterances = np.concatenate( + [self.utterances, np.array(route.utterances)] + ) + # create utterance array (the dense index) + if self.index is None: + self.index = dense_embeds + else: + self.index = np.concatenate([self.index, dense_embeds]) + # create sparse utterance array + if self.sparse_index is None: + self.sparse_index = sparse_embeds + else: + self.sparse_index = np.concatenate([self.sparse_index, sparse_embeds]) + + def _query(self, text: str, top_k: int = 5): + """Given some text, encodes and searches the index vector space to + retrieve the top_k most similar records. + """ + # create dense query vector + xq_d = np.array(self.encoder([text])) + xq_d = np.squeeze(xq_d) # Reduce to 1d array. + # create sparse query vector + xq_s = np.array(self.sparse_encoder([text])) + xq_s = np.squeeze(xq_s) + # convex scaling + xq_d, xq_s = self._convex_scaling(xq_d, xq_s) + + if self.index is not None: + # calculate dense vec similarity + index_norm = norm(self.index, axis=1) + xq_d_norm = norm(xq_d.T) + sim_d = np.dot(self.index, xq_d.T) / (index_norm * xq_d_norm) + # calculate sparse vec similarity + sparse_norm = norm(self.sparse_index, axis=1) + xq_s_norm = norm(xq_s.T) + sim_s = np.dot(self.sparse_index, xq_s.T) / (sparse_norm * xq_s_norm) + total_sim = sim_d + sim_s + # get indices of top_k records + top_k = min(top_k, total_sim.shape[0]) + idx = np.argpartition(total_sim, -top_k)[-top_k:] + scores = total_sim[idx] + # get the utterance categories (route names) + routes = self.categories[idx] if self.categories is not None else [] + return [{"route": d, "score": s.item()} for d, s in zip(routes, scores)] + else: + return [] + + def _convex_scaling(self, dense: list[float], sparse: list[float]): + # scale sparse and dense vecs + dense = np.array(dense) * self.alpha + sparse = np.array(sparse) * (1 - self.alpha) + return dense, sparse + + def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float]]: + scores_by_class = {} + for result in query_results: + score = result["score"] + route = result["route"] + if route in scores_by_class: + scores_by_class[route].append(score) + else: + scores_by_class[route] = [score] + + # Calculate total score for each class + total_scores = {route: sum(scores) for route, scores in scores_by_class.items()} + top_class = max(total_scores, key=lambda x: total_scores[x], default=None) + + # Return the top class and its associated scores + return str(top_class), scores_by_class.get(top_class, []) + + def _pass_threshold(self, scores: list[float], threshold: float) -> bool: + if scores: + return max(scores) > threshold + else: + return False diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 591e8f08..efa4862d 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -1,10 +1,7 @@ import numpy as np -from numpy.linalg import norm -from tqdm.auto import tqdm from semantic_router.encoders import ( BaseEncoder, - BM25Encoder, CohereEncoder, OpenAIEncoder, ) @@ -121,131 +118,3 @@ def _pass_threshold(self, scores: list[float], threshold: float) -> bool: return max(scores) > threshold else: return False - - -class HybridRouteLayer: - index = None - sparse_index = None - categories = None - score_threshold = 0.82 - - def __init__( - self, encoder: BaseEncoder, routes: list[Route] = [], alpha: float = 0.3 - ): - self.encoder = encoder - self.sparse_encoder = BM25Encoder() - self.alpha = alpha - # decide on default threshold based on encoder - if isinstance(encoder, OpenAIEncoder): - self.score_threshold = 0.82 - elif isinstance(encoder, CohereEncoder): - self.score_threshold = 0.3 - else: - self.score_threshold = 0.82 - # if routes list has been passed, we initialize index now - if routes: - # initialize index now - for route in tqdm(routes): - self._add_route(route=route) - - def __call__(self, text: str) -> str | None: - results = self._query(text) - top_class, top_class_scores = self._semantic_classify(results) - passed = self._pass_threshold(top_class_scores, self.score_threshold) - if passed: - return top_class - else: - return None - - def add(self, route: Route): - self._add_route(route=route) - - def _add_route(self, route: Route): - # create embeddings - dense_embeds = np.array(self.encoder(route.utterances)) # * self.alpha - sparse_embeds = np.array( - self.sparse_encoder(route.utterances) - ) # * (1 - self.alpha) - - # create route array - if self.categories is None: - self.categories = np.array([route.name] * len(route.utterances)) - self.utterances = np.array(route.utterances) - else: - str_arr = np.array([route.name] * len(route.utterances)) - self.categories = np.concatenate([self.categories, str_arr]) - self.utterances = np.concatenate( - [self.utterances, np.array(route.utterances)] - ) - # create utterance array (the dense index) - if self.index is None: - self.index = dense_embeds - else: - self.index = np.concatenate([self.index, dense_embeds]) - # create sparse utterance array - if self.sparse_index is None: - self.sparse_index = sparse_embeds - else: - self.sparse_index = np.concatenate([self.sparse_index, sparse_embeds]) - - def _query(self, text: str, top_k: int = 5): - """Given some text, encodes and searches the index vector space to - retrieve the top_k most similar records. - """ - # create dense query vector - xq_d = np.array(self.encoder([text])) - xq_d = np.squeeze(xq_d) # Reduce to 1d array. - # create sparse query vector - xq_s = np.array(self.sparse_encoder([text])) - xq_s = np.squeeze(xq_s) - # convex scaling - xq_d, xq_s = self._convex_scaling(xq_d, xq_s) - - if self.index is not None: - # calculate dense vec similarity - index_norm = norm(self.index, axis=1) - xq_d_norm = norm(xq_d.T) - sim_d = np.dot(self.index, xq_d.T) / (index_norm * xq_d_norm) - # calculate sparse vec similarity - sparse_norm = norm(self.sparse_index, axis=1) - xq_s_norm = norm(xq_s.T) - sim_s = np.dot(self.sparse_index, xq_s.T) / (sparse_norm * xq_s_norm) - total_sim = sim_d + sim_s - # get indices of top_k records - top_k = min(top_k, total_sim.shape[0]) - idx = np.argpartition(total_sim, -top_k)[-top_k:] - scores = total_sim[idx] - # get the utterance categories (route names) - routes = self.categories[idx] if self.categories is not None else [] - return [{"route": d, "score": s.item()} for d, s in zip(routes, scores)] - else: - return [] - - def _convex_scaling(self, dense: list[float], sparse: list[float]): - # scale sparse and dense vecs - dense = np.array(dense) * self.alpha - sparse = np.array(sparse) * (1 - self.alpha) - return dense, sparse - - def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float]]: - scores_by_class = {} - for result in query_results: - score = result["score"] - route = result["route"] - if route in scores_by_class: - scores_by_class[route].append(score) - else: - scores_by_class[route] = [score] - - # Calculate total score for each class - total_scores = {route: sum(scores) for route, scores in scores_by_class.items()} - top_class = max(total_scores, key=lambda x: total_scores[x], default=None) - - # Return the top class and its associated scores - return str(top_class), scores_by_class.get(top_class, []) - - def _pass_threshold(self, scores: list[float], threshold: float) -> bool: - if scores: - return max(scores) > threshold - else: - return False diff --git a/tests/unit/test_hybrid_layer.py b/tests/unit/test_hybrid_layer.py new file mode 100644 index 00000000..94720cd8 --- /dev/null +++ b/tests/unit/test_hybrid_layer.py @@ -0,0 +1,118 @@ +import pytest + +from semantic_router.encoders import BaseEncoder, CohereEncoder, OpenAIEncoder +from semantic_router.hybrid_layer import HybridRouteLayer +from semantic_router.schema import Route + + +def mock_encoder_call(utterances): + # Define a mapping of utterances to return values + mock_responses = { + "Hello": [0.1, 0.2, 0.3], + "Hi": [0.4, 0.5, 0.6], + "Goodbye": [0.7, 0.8, 0.9], + "Bye": [1.0, 1.1, 1.2], + "Au revoir": [1.3, 1.4, 1.5], + } + return [mock_responses.get(u, [0, 0, 0]) for u in utterances] + + +@pytest.fixture +def base_encoder(): + return BaseEncoder(name="test-encoder") + + +@pytest.fixture +def cohere_encoder(mocker): + mocker.patch.object(CohereEncoder, "__call__", side_effect=mock_encoder_call) + return CohereEncoder(name="test-cohere-encoder", cohere_api_key="test_api_key") + + +@pytest.fixture +def openai_encoder(mocker): + mocker.patch.object(OpenAIEncoder, "__call__", side_effect=mock_encoder_call) + return OpenAIEncoder(name="test-openai-encoder", openai_api_key="test_api_key") + + +@pytest.fixture +def routes(): + return [ + Route(name="Route 1", utterances=["Hello", "Hi"]), + Route(name="Route 2", utterances=["Goodbye", "Bye", "Au revoir"]), + ] + + +class TestHybridRouteLayer: + def test_initialization(self, openai_encoder, routes): + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) + assert route_layer.index is not None and route_layer.categories is not None + assert route_layer.score_threshold == 0.82 + assert len(route_layer.index) == 5 + assert len(set(route_layer.categories)) == 2 + + def test_initialization_different_encoders(self, cohere_encoder, openai_encoder): + route_layer_cohere = HybridRouteLayer(encoder=cohere_encoder) + assert route_layer_cohere.score_threshold == 0.3 + + route_layer_openai = HybridRouteLayer(encoder=openai_encoder) + assert route_layer_openai.score_threshold == 0.82 + + def test_add_route(self, openai_encoder): + route_layer = HybridRouteLayer(encoder=openai_encoder) + route = Route(name="Route 3", utterances=["Yes", "No"]) + route_layer.add(route) + assert route_layer.index is not None and route_layer.categories is not None + assert len(route_layer.index) == 2 + assert len(set(route_layer.categories)) == 1 + + def test_add_multiple_routes(self, openai_encoder, routes): + route_layer = HybridRouteLayer(encoder=openai_encoder) + for route in routes: + route_layer.add(route) + assert route_layer.index is not None and route_layer.categories is not None + assert len(route_layer.index) == 5 + assert len(set(route_layer.categories)) == 2 + + def test_query_and_classification(self, openai_encoder, routes): + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) + query_result = route_layer("Hello") + assert query_result in ["Route 1", "Route 2"] + + def test_query_with_no_index(self, openai_encoder): + route_layer = HybridRouteLayer(encoder=openai_encoder) + assert route_layer("Anything") is None + + def test_semantic_classify(self, openai_encoder, routes): + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) + classification, score = route_layer._semantic_classify( + [ + {"route": "Route 1", "score": 0.9}, + {"route": "Route 2", "score": 0.1}, + ] + ) + assert classification == "Route 1" + assert score == [0.9] + + def test_semantic_classify_multiple_routes(self, openai_encoder, routes): + route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) + classification, score = route_layer._semantic_classify( + [ + {"route": "Route 1", "score": 0.9}, + {"route": "Route 2", "score": 0.1}, + {"route": "Route 1", "score": 0.8}, + ] + ) + assert classification == "Route 1" + assert score == [0.9, 0.8] + + def test_pass_threshold(self, openai_encoder): + route_layer = HybridRouteLayer(encoder=openai_encoder) + assert not route_layer._pass_threshold([], 0.5) + assert route_layer._pass_threshold([0.6, 0.7], 0.5) + + def test_failover_score_threshold(self, base_encoder): + route_layer = HybridRouteLayer(encoder=base_encoder) + assert route_layer.score_threshold == 0.82 + + +# Add more tests for edge cases and error handling as needed. diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index d5f698be..66e0d53b 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -1,12 +1,7 @@ import pytest from semantic_router.encoders import BaseEncoder, CohereEncoder, OpenAIEncoder -from semantic_router.layer import ( - HybridRouteLayer, - RouteLayer, -) - -# Replace with the actual module name +from semantic_router.layer import RouteLayer from semantic_router.schema import Route @@ -130,77 +125,4 @@ def test_failover_score_threshold(self, base_encoder): assert route_layer.score_threshold == 0.82 -class TestHybridRouteLayer: - def test_initialization(self, openai_encoder, routes): - route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) - assert route_layer.index is not None and route_layer.categories is not None - assert route_layer.score_threshold == 0.82 - assert len(route_layer.index) == 5 - assert len(set(route_layer.categories)) == 2 - - def test_initialization_different_encoders(self, cohere_encoder, openai_encoder): - route_layer_cohere = HybridRouteLayer(encoder=cohere_encoder) - assert route_layer_cohere.score_threshold == 0.3 - - route_layer_openai = HybridRouteLayer(encoder=openai_encoder) - assert route_layer_openai.score_threshold == 0.82 - - def test_add_route(self, openai_encoder): - route_layer = HybridRouteLayer(encoder=openai_encoder) - route = Route(name="Route 3", utterances=["Yes", "No"]) - route_layer.add(route) - assert route_layer.index is not None and route_layer.categories is not None - assert len(route_layer.index) == 2 - assert len(set(route_layer.categories)) == 1 - - def test_add_multiple_routes(self, openai_encoder, routes): - route_layer = HybridRouteLayer(encoder=openai_encoder) - for route in routes: - route_layer.add(route) - assert route_layer.index is not None and route_layer.categories is not None - assert len(route_layer.index) == 5 - assert len(set(route_layer.categories)) == 2 - - def test_query_and_classification(self, openai_encoder, routes): - route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) - query_result = route_layer("Hello") - assert query_result in ["Route 1", "Route 2"] - - def test_query_with_no_index(self, openai_encoder): - route_layer = HybridRouteLayer(encoder=openai_encoder) - assert route_layer("Anything") is None - - def test_semantic_classify(self, openai_encoder, routes): - route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) - classification, score = route_layer._semantic_classify( - [ - {"route": "Route 1", "score": 0.9}, - {"route": "Route 2", "score": 0.1}, - ] - ) - assert classification == "Route 1" - assert score == [0.9] - - def test_semantic_classify_multiple_routes(self, openai_encoder, routes): - route_layer = HybridRouteLayer(encoder=openai_encoder, routes=routes) - classification, score = route_layer._semantic_classify( - [ - {"route": "Route 1", "score": 0.9}, - {"route": "Route 2", "score": 0.1}, - {"route": "Route 1", "score": 0.8}, - ] - ) - assert classification == "Route 1" - assert score == [0.9, 0.8] - - def test_pass_threshold(self, openai_encoder): - route_layer = HybridRouteLayer(encoder=openai_encoder) - assert not route_layer._pass_threshold([], 0.5) - assert route_layer._pass_threshold([0.6, 0.7], 0.5) - - def test_failover_score_threshold(self, base_encoder): - route_layer = HybridRouteLayer(encoder=base_encoder) - assert route_layer.score_threshold == 0.82 - - # Add more tests for edge cases and error handling as needed. diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index 7c954805..f471755c 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -2,10 +2,10 @@ from semantic_router.schema import ( CohereEncoder, - Route, Encoder, EncoderType, OpenAIEncoder, + Route, SemanticSpace, ) diff --git a/walkthrough.ipynb b/walkthrough.ipynb index a4265e5a..d31a88dc 100644 --- a/walkthrough.ipynb +++ b/walkthrough.ipynb @@ -128,7 +128,7 @@ "metadata": {}, "outputs": [], "source": [ - "from semantic_router.layer import RouteLayer\n", + "from semantic_router.router import RouteLayer\n", "\n", "dl = RouteLayer(encoder=encoder, routes=routes)" ] From b743510ebbb6225b4778919df759bb4e2602cee1 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:07:57 +0200 Subject: [PATCH 5/6] linting --- coverage.xml | 80 +++++++++++++++++-------------- semantic_router/hybrid_layer.py | 6 ++- semantic_router/utils/__init__.py | 0 3 files changed, 47 insertions(+), 39 deletions(-) create mode 100644 semantic_router/utils/__init__.py diff --git a/coverage.xml b/coverage.xml index 612cac46..755c321e 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -22,88 +22,90 @@ - - - + + + - - + + - + - - - + + + - + - - + + - - + + - + - + - + - - + + - - - + + + - + - - - + + + - + - + - + - + + - + - + - - - + + + - + + @@ -361,6 +363,10 @@ + + + + diff --git a/semantic_router/hybrid_layer.py b/semantic_router/hybrid_layer.py index 3e9508d6..a0452a31 100644 --- a/semantic_router/hybrid_layer.py +++ b/semantic_router/hybrid_layer.py @@ -1,6 +1,7 @@ import numpy as np from numpy.linalg import norm from tqdm.auto import tqdm +from semantic_router.utils.logger import logger from semantic_router.encoders import ( BaseEncoder, @@ -89,7 +90,7 @@ def _query(self, text: str, top_k: int = 5): # convex scaling xq_d, xq_s = self._convex_scaling(xq_d, xq_s) - if self.index is not None: + if self.index is not None and self.sparse_index is not None: # calculate dense vec similarity index_norm = norm(self.index, axis=1) xq_d_norm = norm(xq_d.T) @@ -107,9 +108,10 @@ def _query(self, text: str, top_k: int = 5): routes = self.categories[idx] if self.categories is not None else [] return [{"route": d, "score": s.item()} for d, s in zip(routes, scores)] else: + logger.warning("No index found. Please add routes to the layer.") return [] - def _convex_scaling(self, dense: list[float], sparse: list[float]): + def _convex_scaling(self, dense: np.ndarray, sparse: np.ndarray): # scale sparse and dense vecs dense = np.array(dense) * self.alpha sparse = np.array(sparse) * (1 - self.alpha) diff --git a/semantic_router/utils/__init__.py b/semantic_router/utils/__init__.py new file mode 100644 index 00000000..e69de29b From 671cd0738bce63cfb5312cdfe9b66f6711a8051e Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:37:31 +0200 Subject: [PATCH 6/6] added test + lint + codecov --- Makefile | 1 + README.md | 1 + coverage.xml | 99 +++++++++++++++++++------------- poetry.lock | 49 +++++++++++++++- pyproject.toml | 4 ++ semantic_router/encoders/base.py | 2 +- semantic_router/encoders/bm25.py | 25 +++++--- semantic_router/hybrid_layer.py | 10 +++- semantic_router/layer.py | 10 +++- semantic_router/schema.py | 2 +- tests/unit/encoders/test_bm25.py | 19 ++++++ 11 files changed, 164 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index 3a3c42cd..8de202fa 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ lint_diff: PYTHON_FILES=$(shell git diff --name-only --diff-filter=d main | grep lint lint_diff: poetry run black $(PYTHON_FILES) --check poetry run ruff . + poetry run mypy $(PYTHON_FILES) test: poetry run pytest -vv -n 20 --cov=semantic_router --cov-report=term-missing --cov-report=xml --cov-fail-under=100 diff --git a/README.md b/README.md index 9dac4222..b4b3c0e3 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ GitHub Issues GitHub Pull Requests + Github License

diff --git a/coverage.xml b/coverage.xml index 755c321e..8e6ca91d 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -22,8 +22,8 @@ - - + + @@ -102,10 +102,13 @@ - + + + + @@ -115,68 +118,73 @@ - + - + - + - - - - + + + + - - - - + + + + - + - + - + - - - - + + + + - + - - + + - - + + - + - + + - + - - - + + + - + + + + + @@ -271,31 +279,40 @@ - - + + - + + + + - - + + - - + + + + + + + + diff --git a/poetry.lock b/poetry.lock index 3bedc8de..b459e6ba 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1065,6 +1065,53 @@ files = [ {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] +[[package]] +name = "mypy" +version = "1.7.1" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:12cce78e329838d70a204293e7b29af9faa3ab14899aec397798a4b41be7f340"}, + {file = "mypy-1.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1484b8fa2c10adf4474f016e09d7a159602f3239075c7bf9f1627f5acf40ad49"}, + {file = "mypy-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31902408f4bf54108bbfb2e35369877c01c95adc6192958684473658c322c8a5"}, + {file = "mypy-1.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f2c2521a8e4d6d769e3234350ba7b65ff5d527137cdcde13ff4d99114b0c8e7d"}, + {file = "mypy-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:fcd2572dd4519e8a6642b733cd3a8cfc1ef94bafd0c1ceed9c94fe736cb65b6a"}, + {file = "mypy-1.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4b901927f16224d0d143b925ce9a4e6b3a758010673eeded9b748f250cf4e8f7"}, + {file = "mypy-1.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2f7f6985d05a4e3ce8255396df363046c28bea790e40617654e91ed580ca7c51"}, + {file = "mypy-1.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:944bdc21ebd620eafefc090cdf83158393ec2b1391578359776c00de00e8907a"}, + {file = "mypy-1.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9c7ac372232c928fff0645d85f273a726970c014749b924ce5710d7d89763a28"}, + {file = "mypy-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:f6efc9bd72258f89a3816e3a98c09d36f079c223aa345c659622f056b760ab42"}, + {file = "mypy-1.7.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6dbdec441c60699288adf051f51a5d512b0d818526d1dcfff5a41f8cd8b4aaf1"}, + {file = "mypy-1.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4fc3d14ee80cd22367caaaf6e014494415bf440980a3045bf5045b525680ac33"}, + {file = "mypy-1.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c6e4464ed5f01dc44dc9821caf67b60a4e5c3b04278286a85c067010653a0eb"}, + {file = "mypy-1.7.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:d9b338c19fa2412f76e17525c1b4f2c687a55b156320acb588df79f2e6fa9fea"}, + {file = "mypy-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:204e0d6de5fd2317394a4eff62065614c4892d5a4d1a7ee55b765d7a3d9e3f82"}, + {file = "mypy-1.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:84860e06ba363d9c0eeabd45ac0fde4b903ad7aa4f93cd8b648385a888e23200"}, + {file = "mypy-1.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8c5091ebd294f7628eb25ea554852a52058ac81472c921150e3a61cdd68f75a7"}, + {file = "mypy-1.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40716d1f821b89838589e5b3106ebbc23636ffdef5abc31f7cd0266db936067e"}, + {file = "mypy-1.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5cf3f0c5ac72139797953bd50bc6c95ac13075e62dbfcc923571180bebb662e9"}, + {file = "mypy-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:78e25b2fd6cbb55ddfb8058417df193f0129cad5f4ee75d1502248e588d9e0d7"}, + {file = "mypy-1.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:75c4d2a6effd015786c87774e04331b6da863fc3fc4e8adfc3b40aa55ab516fe"}, + {file = "mypy-1.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2643d145af5292ee956aa0a83c2ce1038a3bdb26e033dadeb2f7066fb0c9abce"}, + {file = "mypy-1.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75aa828610b67462ffe3057d4d8a4112105ed211596b750b53cbfe182f44777a"}, + {file = "mypy-1.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ee5d62d28b854eb61889cde4e1dbc10fbaa5560cb39780c3995f6737f7e82120"}, + {file = "mypy-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:72cf32ce7dd3562373f78bd751f73c96cfb441de147cc2448a92c1a308bd0ca6"}, + {file = "mypy-1.7.1-py3-none-any.whl", hash = "sha256:f7c5d642db47376a0cc130f0de6d055056e010debdaf0707cd2b0fc7e7ef30ea"}, + {file = "mypy-1.7.1.tar.gz", hash = "sha256:fcb6d9afb1b6208b4c712af0dafdc650f518836065df0d4fb1d800f5d6773db2"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -2055,4 +2102,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "b17b9fd9486d6c744c41a31ab54f7871daba1e2d4166fda228033c5858f6f9d8" +content-hash = "58bf19052f05863cb4623e85a73de5758d581ff539cfb69f0920e57f6cb035d0" diff --git a/pyproject.toml b/pyproject.toml index 61a95510..5a8e18e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ pytest = "^7.4.3" pytest-mock = "^3.12.0" pytest-cov = "^4.1.0" pytest-xdist = "^3.5.0" +mypy = "^1.7.1" [build-system] requires = ["poetry-core"] @@ -36,3 +37,6 @@ build-backend = "poetry.core.masonry.api" [tool.ruff.per-file-ignores] "*.ipynb" = ["E402"] + +[tool.mypy] +ignore_missing_imports = true diff --git a/semantic_router/encoders/base.py b/semantic_router/encoders/base.py index b6de1f89..632ebc79 100644 --- a/semantic_router/encoders/base.py +++ b/semantic_router/encoders/base.py @@ -7,5 +7,5 @@ class BaseEncoder(BaseModel): class Config: arbitrary_types_allowed = True - def __call__(self, docs: list[str]) -> list[float]: + def __call__(self, docs: list[str]) -> list[list[float]]: raise NotImplementedError("Subclasses must implement this method") diff --git a/semantic_router/encoders/bm25.py b/semantic_router/encoders/bm25.py index 0d498197..c9da628e 100644 --- a/semantic_router/encoders/bm25.py +++ b/semantic_router/encoders/bm25.py @@ -1,29 +1,36 @@ +from typing import Any + from pinecone_text.sparse import BM25Encoder as encoder from semantic_router.encoders import BaseEncoder class BM25Encoder(BaseEncoder): - model: encoder | None = None + model: Any | None = None idx_mapping: dict[int, int] | None = None def __init__(self, name: str = "bm25"): super().__init__(name=name) - # initialize BM25 encoder with default params (trained on MSMarco) self.model = encoder.default() - self.idx_mapping = { - idx: i - for i, idx in enumerate(self.model.get_params()["doc_freq"]["indices"]) - } + + params = self.model.get_params() + doc_freq = params["doc_freq"] + if isinstance(doc_freq, dict): + indices = doc_freq["indices"] + self.idx_mapping = {int(idx): i for i, idx in enumerate(indices)} + else: + raise TypeError("Expected a dictionary for 'doc_freq'") def __call__(self, docs: list[str]) -> list[list[float]]: + if self.model is None or self.idx_mapping is None: + raise ValueError("Model or index mapping is not initialized.") if len(docs) == 1: sparse_dicts = self.model.encode_queries(docs) elif len(docs) > 1: sparse_dicts = self.model.encode_documents(docs) else: raise ValueError("No documents to encode.") - # convert sparse dict to sparse vector + embeds = [[0.0] * len(self.idx_mapping)] * len(docs) for i, output in enumerate(sparse_dicts): indices = output["indices"] @@ -32,9 +39,9 @@ def __call__(self, docs: list[str]) -> list[list[float]]: if idx in self.idx_mapping: position = self.idx_mapping[idx] embeds[i][position] = val - else: - print(idx, "not in encoder.idx_mapping") return embeds def fit(self, docs: list[str]): + if self.model is None: + raise ValueError("Model is not initialized.") self.model.fit(docs) diff --git a/semantic_router/hybrid_layer.py b/semantic_router/hybrid_layer.py index a0452a31..dec6336e 100644 --- a/semantic_router/hybrid_layer.py +++ b/semantic_router/hybrid_layer.py @@ -1,7 +1,6 @@ import numpy as np from numpy.linalg import norm from tqdm.auto import tqdm -from semantic_router.utils.logger import logger from semantic_router.encoders import ( BaseEncoder, @@ -10,6 +9,7 @@ OpenAIEncoder, ) from semantic_router.schema import Route +from semantic_router.utils.logger import logger class HybridRouteLayer: @@ -118,7 +118,7 @@ def _convex_scaling(self, dense: np.ndarray, sparse: np.ndarray): return dense, sparse def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float]]: - scores_by_class = {} + scores_by_class: dict[str, list[float]] = {} for result in query_results: score = result["score"] route = result["route"] @@ -132,7 +132,11 @@ def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float top_class = max(total_scores, key=lambda x: total_scores[x], default=None) # Return the top class and its associated scores - return str(top_class), scores_by_class.get(top_class, []) + if top_class is not None: + return str(top_class), scores_by_class.get(top_class, []) + else: + logger.warning("No classification found for semantic classifier.") + return "", [] def _pass_threshold(self, scores: list[float], threshold: float) -> bool: if scores: diff --git a/semantic_router/layer.py b/semantic_router/layer.py index efa4862d..cb408c5c 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -7,6 +7,7 @@ ) from semantic_router.linear import similarity_matrix, top_scores from semantic_router.schema import Route +from semantic_router.utils.logger import logger class RouteLayer: @@ -94,10 +95,11 @@ def _query(self, text: str, top_k: int = 5): routes = self.categories[idx] if self.categories is not None else [] return [{"route": d, "score": s.item()} for d, s in zip(routes, scores)] else: + logger.warning("No index found for route layer.") return [] def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float]]: - scores_by_class = {} + scores_by_class: dict[str, list[float]] = {} for result in query_results: score = result["score"] route = result["route"] @@ -111,7 +113,11 @@ def _semantic_classify(self, query_results: list[dict]) -> tuple[str, list[float top_class = max(total_scores, key=lambda x: total_scores[x], default=None) # Return the top class and its associated scores - return str(top_class), scores_by_class.get(top_class, []) + if top_class is not None: + return str(top_class), scores_by_class.get(top_class, []) + else: + logger.warning("No classification found for semantic classifier.") + return "", [] def _pass_threshold(self, scores: list[float], threshold: float) -> bool: if scores: diff --git a/semantic_router/schema.py b/semantic_router/schema.py index 3763db03..007cddcb 100644 --- a/semantic_router/schema.py +++ b/semantic_router/schema.py @@ -38,7 +38,7 @@ def __init__(self, type: str, name: str): elif self.type == EncoderType.COHERE: self.model = CohereEncoder(name) - def __call__(self, texts: list[str]) -> list[float]: + def __call__(self, texts: list[str]) -> list[list[float]]: return self.model(texts) diff --git a/tests/unit/encoders/test_bm25.py b/tests/unit/encoders/test_bm25.py index c1987151..e654d7bb 100644 --- a/tests/unit/encoders/test_bm25.py +++ b/tests/unit/encoders/test_bm25.py @@ -33,3 +33,22 @@ def test_call_method_no_word(self, bm25_encoder): assert all( isinstance(sublist, list) for sublist in result ), "Each item in result should be a list" + + def test_init_with_non_dict_doc_freq(self, mocker): + mock_encoder = mocker.MagicMock() + mock_encoder.get_params.return_value = {"doc_freq": "not a dict"} + mocker.patch( + "pinecone_text.sparse.BM25Encoder.default", return_value=mock_encoder + ) + with pytest.raises(TypeError): + BM25Encoder() + + def test_call_method_with_uninitialized_model_or_mapping(self, bm25_encoder): + bm25_encoder.model = None + with pytest.raises(ValueError): + bm25_encoder(["test"]) + + def test_fit_with_uninitialized_model(self, bm25_encoder): + bm25_encoder.model = None + with pytest.raises(ValueError): + bm25_encoder.fit(["test"])