From 52eae9ea78b4fa7512ea102b0d8be30772ef0442 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 18 Sep 2023 11:50:26 +0200 Subject: [PATCH 1/3] Update Langchain example with the first beta version of the new Langchain integration --- .../notebooks/Comet_with_Langchain.ipynb | 232 +++++++----------- 1 file changed, 82 insertions(+), 150 deletions(-) diff --git a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb index bb637913..f831751b 100644 --- a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb +++ b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb @@ -24,7 +24,7 @@ " \"Open\n", "\n", "\n", - "**Example Project:** [Comet with LangChain](https://www.comet.com/examples/comet-example-langchain/view/b5ZThK6OFdhKWVSP3fDfRtrNF/panels?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook)" + "**Example Project:** [Comet with LangChain](https://www.comet.com/lothiraldan/langchain-integration-beta/prompts?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook)" ] }, { @@ -47,7 +47,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U comet_ml \"langchain>=0.0.162\" openai google-search-results spacy textstat pandas\n", + "%pip install -U comet_llm \"langchain>=0.0.162\" openai google-search-results spacy textstat pandas\n", "\n", "import sys\n", "\n", @@ -65,7 +65,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can grab your [Comet API Key here](https://www.comet.com/signup?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook) or click the link after intializing Comet" + "You can grab your [Comet API Key here](https://www.comet.com/signup?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook):" ] }, { @@ -74,9 +74,10 @@ "metadata": {}, "outputs": [], "source": [ - "import comet_ml\n", + "import comet_llm\n", "\n", - "comet_ml.init(project_name=\"comet-example-langchain\")" + "# os.environ['COMET_API_KEY'] = YOUR-COMET-API-KEY\n", + "# os.environ['COMET_PROJECT_NAME'] = YOUR-PROJECT-NAME" ] }, { @@ -101,9 +102,67 @@ "source": [ "import os\n", "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", + "# os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", "# os.environ[\"OPENAI_ORGANIZATION\"] = \"...\"\n", - "os.environ[\"SERPAPI_API_KEY\"] = \"...\"" + "# os.environ[\"SERPAPI_API_KEY\"] = \"...\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define the beta-version of the Langchain callback" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.callbacks.tracers.base import BaseTracer\n", + "\n", + "\n", + "def get_run_type(run):\n", + " if isinstance(run.run_type, str):\n", + " return run.run_type\n", + " elif hasattr(run.run_type, \"value\"):\n", + " return run.run_type.value\n", + " else:\n", + " return str(run.run_type)\n", + "\n", + "\n", + "class CometTracer(BaseTracer):\n", + " def __init__(self, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self.span_map = {}\n", + "\n", + " def _persist_run(self, run) -> None:\n", + " comet_llm.end_chain(outputs=run.outputs)\n", + "\n", + " def _start_trace(self, run) -> None:\n", + " if not run.parent_run_id:\n", + " # This is the first run, which maps to a chain\n", + " comet_llm.start_chain(inputs=run.inputs)\n", + " else:\n", + " span = comet_llm.Span(\n", + " inputs=run.inputs,\n", + " category=get_run_type(run),\n", + " metadata=run.extra,\n", + " name=run.name,\n", + " ).__enter__()\n", + " self.span_map[run.id] = span\n", + " super()._start_trace(run)\n", + "\n", + " def _end_trace(self, run) -> None:\n", + " if not run.parent_run_id:\n", + " pass\n", + " # Langchain will call _persist_run for us\n", + " else:\n", + " span = self.span_map[run.id]\n", + " span.set_outputs(outputs=run.outputs)\n", + " span.__exit__(None, None, None)\n", + " super()._end_trace(run)" ] }, { @@ -121,21 +180,15 @@ "source": [ "from datetime import datetime\n", "\n", - "from langchain.callbacks import CometCallbackHandler\n", "from langchain.llms import OpenAI\n", "\n", - "comet_callback = CometCallbackHandler(\n", - " project_name=\"comet-example-langchain\",\n", - " complexity_metrics=True,\n", - " stream_logs=True,\n", - " tags=[\"llm\"],\n", - " visualizations=[\"dep\"],\n", - ")\n", - "llm = OpenAI(temperature=0.9, callbacks=[comet_callback], verbose=True)\n", + "comet_tracer = CometTracer()\n", + "callbacks = [comet_tracer]\n", + "llm = OpenAI(temperature=0.9, callbacks=callbacks, verbose=True)\n", "\n", - "llm_result = llm.generate([\"Tell me a joke\", \"Tell me a poem\", \"Tell me a fact\"] * 3)\n", - "print(\"LLM result\", llm_result)\n", - "comet_callback.flush_tracker(llm, finish=True)" + "for prompt in [\"Tell me a joke\", \"Tell me a poem\", \"Tell me a fact\"]:\n", + " llm_result = llm.generate([prompt], callbacks=[comet_tracer])\n", + " print(\"LLM result\", llm_result)" ] }, { @@ -151,18 +204,12 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.callbacks import CometCallbackHandler\n", "from langchain.chains import LLMChain\n", "from langchain.llms import OpenAI\n", "from langchain.prompts import PromptTemplate\n", "\n", - "comet_callback = CometCallbackHandler(\n", - " complexity_metrics=True,\n", - " project_name=\"comet-example-langchain\",\n", - " stream_logs=True,\n", - " tags=[\"synopsis-chain\"],\n", - ")\n", - "callbacks = [comet_callback]\n", + "tracer = CometTracer()\n", + "callbacks = [tracer]\n", "\n", "llm = OpenAI(temperature=0.9, callbacks=callbacks, verbose=True)\n", "\n", @@ -175,8 +222,7 @@ ")\n", "\n", "test_prompts = [{\"title\": \"Documentary about Bigfoot in Paris\"}]\n", - "print(synopsis_chain.apply(test_prompts))\n", - "comet_callback.flush_tracker(synopsis_chain, finish=True)" + "print(synopsis_chain.apply(test_prompts))" ] }, { @@ -193,16 +239,11 @@ "outputs": [], "source": [ "from langchain.agents import initialize_agent, load_tools\n", - "from langchain.callbacks import CometCallbackHandler\n", "from langchain.llms import OpenAI\n", "\n", - "comet_callback = CometCallbackHandler(\n", - " project_name=\"comet-example-langchain\",\n", - " complexity_metrics=True,\n", - " stream_logs=True,\n", - " tags=[\"agent\"],\n", - ")\n", - "callbacks = [comet_callback]\n", + "\n", + "tracer = CometTracer()\n", + "callbacks = [tracer]\n", "\n", "llm = OpenAI(temperature=0.9, callbacks=callbacks, verbose=True)\n", "\n", @@ -215,118 +256,9 @@ " verbose=True,\n", ")\n", "agent.run(\n", - " \"Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?\"\n", - ")\n", - "comet_callback.flush_tracker(agent, finish=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Scenario 4: Using Custom Evaluation Metrics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `CometCallbackManager` also allows you to define and use Custom Evaluation Metrics to assess generated outputs from your model. Let's take a look at how this works. \n", - "\n", - "\n", - "In the snippet below, we will use the [ROUGE](https://huggingface.co/spaces/evaluate-metric/rouge) metric to evaluate the quality of a generated summary of an input prompt. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install rouge-score" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from rouge_score import rouge_scorer\n", - "\n", - "from langchain.callbacks import CometCallbackHandler\n", - "from langchain.chains import LLMChain\n", - "from langchain.llms import OpenAI\n", - "from langchain.prompts import PromptTemplate\n", - "\n", - "\n", - "class Rouge:\n", - " def __init__(self, reference):\n", - " self.reference = reference\n", - " self.scorer = rouge_scorer.RougeScorer([\"rougeLsum\"], use_stemmer=True)\n", - "\n", - " def compute_metric(self, generation, prompt_idx, gen_idx):\n", - " prediction = generation.text\n", - " results = self.scorer.score(target=self.reference, prediction=prediction)\n", - "\n", - " return {\n", - " \"rougeLsum_score\": results[\"rougeLsum\"].fmeasure,\n", - " \"reference\": self.reference,\n", - " }\n", - "\n", - "\n", - "reference = \"\"\"\n", - "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building.\n", - "It was the first structure to reach a height of 300 metres.\n", - "\n", - "It is now taller than the Chrysler Building in New York City by 5.2 metres (17 ft)\n", - "Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France .\n", - "\"\"\"\n", - "rouge_score = Rouge(reference=reference)\n", - "\n", - "template = \"\"\"Given the following article, it is your job to write a summary.\n", - "Article:\n", - "{article}\n", - "Summary: This is the summary for the above article:\"\"\"\n", - "prompt_template = PromptTemplate(input_variables=[\"article\"], template=template)\n", - "\n", - "comet_callback = CometCallbackHandler(\n", - " project_name=\"comet-example-langchain\",\n", - " complexity_metrics=False,\n", - " stream_logs=True,\n", - " tags=[\"custom_metrics\"],\n", - " custom_metrics=rouge_score.compute_metric,\n", - ")\n", - "callbacks = [comet_callback]\n", - "\n", - "llm = OpenAI(temperature=0.9, callbacks=callbacks, verbose=True)\n", - "\n", - "synopsis_chain = LLMChain(\n", - " llm=llm, prompt=prompt_template, callbacks=callbacks, verbose=True\n", - ")\n", - "\n", - "test_prompts = [\n", - " {\n", - " \"article\": \"\"\"\n", - " The tower is 324 metres (1,063 ft) tall, about the same height as\n", - " an 81-storey building, and the tallest structure in Paris. Its base is square,\n", - " measuring 125 metres (410 ft) on each side.\n", - " During its construction, the Eiffel Tower surpassed the\n", - " Washington Monument to become the tallest man-made structure in the world,\n", - " a title it held for 41 years until the Chrysler Building\n", - " in New York City was finished in 1930.\n", - "\n", - " It was the first structure to reach a height of 300 metres.\n", - " Due to the addition of a broadcasting aerial at the top of the tower in 1957,\n", - " it is now taller than the Chrysler Building by 5.2 metres (17 ft).\n", - "\n", - " Excluding transmitters, the Eiffel Tower is the second tallest\n", - " free-standing structure in France after the Millau Viaduct.\n", - " \"\"\"\n", - " }\n", - "]\n", - "print(synopsis_chain.apply(test_prompts))\n", - "comet_callback.flush_tracker(synopsis_chain, finish=True)" + " \"Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?\",\n", + " callbacks=callbacks,\n", + ")" ] } ], @@ -346,7 +278,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.9.1" } }, "nbformat": 4, From e0274c3a4e4b7f03a33e3ae7125a8edeb2855580 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Fri, 6 Oct 2023 14:47:23 +0200 Subject: [PATCH 2/3] Use comet_llm.init --- .../llm/langchain/notebooks/Comet_with_Langchain.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb index f831751b..1ad1de86 100644 --- a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb +++ b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb @@ -70,12 +70,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import comet_llm\n", "\n", + "comet_llm.init(project=\"comet-example-langchain-llm\")\n", "# os.environ['COMET_API_KEY'] = YOUR-COMET-API-KEY\n", "# os.environ['COMET_PROJECT_NAME'] = YOUR-PROJECT-NAME" ] From c7fa86b2bd8366afe17d911243d0a2a2bd6c8aa4 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Fri, 26 Jan 2024 16:08:56 +0100 Subject: [PATCH 3/3] Update langchain example for new tracer mechanism --- .../notebooks/Comet_with_Langchain.ipynb | 117 ++++-------------- 1 file changed, 26 insertions(+), 91 deletions(-) diff --git a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb index 1ad1de86..dd302c28 100644 --- a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb +++ b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb @@ -18,13 +18,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In this guide we will demonstrate how to track your Langchain Experiments, Evaluation Metrics, and LLM Sessions with [Comet](https://www.comet.com/site/?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook). \n", + "In this guide we will demonstrate how to track your Langchain prompts, Chains, and Agents with [Comet](https://www.comet.com/site/?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook).\n", "\n", - "\n", - " \"Open\n", - "\n", "\n", - "**Example Project:** [Comet with LangChain](https://www.comet.com/lothiraldan/langchain-integration-beta/prompts?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook)" + "**Example Project:** [Comet with LangChain](https://www.comet.com/examples/comet-example-langchain-llm/prompts?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook)" ] }, { @@ -47,11 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U comet_llm \"langchain>=0.0.162\" openai google-search-results spacy textstat pandas\n", - "\n", - "import sys\n", - "\n", - "!{sys.executable} -m spacy download en_core_web_sm" + "%pip install -U comet_llm \"langchain>=0.1.3\" \"langchain-openai\" openai numexpr" ] }, { @@ -61,22 +54,15 @@ "### Initialize Comet and Set your Credentials" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can grab your [Comet API Key here](https://www.comet.com/signup?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook):" - ] - }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import comet_llm\n", "\n", - "comet_llm.init(project=\"comet-example-langchain-llm\")\n", + "comet_llm.init(project=\"comet-example-langchain-llm-notebook\")\n", "# os.environ['COMET_API_KEY'] = YOUR-COMET-API-KEY\n", "# os.environ['COMET_PROJECT_NAME'] = YOUR-PROJECT-NAME" ] @@ -104,15 +90,19 @@ "import os\n", "\n", "# os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", - "# os.environ[\"OPENAI_ORGANIZATION\"] = \"...\"\n", - "# os.environ[\"SERPAPI_API_KEY\"] = \"...\"" + "# os.environ[\"OPENAI_ORGANIZATION\"] = \"...\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Define the beta-version of the Langchain callback" + "# Tracing with Comet\n", + "\n", + "There are two ways to trace your LangChains executions with Comet:\n", + "\n", + "1. Setting the `LANGCHAIN_COMET_TRACING` environment variable to \"true\". This is the recommended way.\n", + "2. Import the `CometTracer` manually and pass it explicitely." ] }, { @@ -121,49 +111,8 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.callbacks.tracers.base import BaseTracer\n", - "\n", - "\n", - "def get_run_type(run):\n", - " if isinstance(run.run_type, str):\n", - " return run.run_type\n", - " elif hasattr(run.run_type, \"value\"):\n", - " return run.run_type.value\n", - " else:\n", - " return str(run.run_type)\n", - "\n", - "\n", - "class CometTracer(BaseTracer):\n", - " def __init__(self, **kwargs):\n", - " super().__init__(**kwargs)\n", - " self.span_map = {}\n", - "\n", - " def _persist_run(self, run) -> None:\n", - " comet_llm.end_chain(outputs=run.outputs)\n", - "\n", - " def _start_trace(self, run) -> None:\n", - " if not run.parent_run_id:\n", - " # This is the first run, which maps to a chain\n", - " comet_llm.start_chain(inputs=run.inputs)\n", - " else:\n", - " span = comet_llm.Span(\n", - " inputs=run.inputs,\n", - " category=get_run_type(run),\n", - " metadata=run.extra,\n", - " name=run.name,\n", - " ).__enter__()\n", - " self.span_map[run.id] = span\n", - " super()._start_trace(run)\n", - "\n", - " def _end_trace(self, run) -> None:\n", - " if not run.parent_run_id:\n", - " pass\n", - " # Langchain will call _persist_run for us\n", - " else:\n", - " span = self.span_map[run.id]\n", - " span.set_outputs(outputs=run.outputs)\n", - " span.__exit__(None, None, None)\n", - " super()._end_trace(run)" + "os.environ[\"LANGCHAIN_COMET_TRACING\"] = \"true\"\n", + "from langchain.callbacks.tracers.comet import CometTracer" ] }, { @@ -181,14 +130,12 @@ "source": [ "from datetime import datetime\n", "\n", - "from langchain.llms import OpenAI\n", + "from langchain_openai import OpenAI\n", "\n", - "comet_tracer = CometTracer()\n", - "callbacks = [comet_tracer]\n", - "llm = OpenAI(temperature=0.9, callbacks=callbacks, verbose=True)\n", + "llm = OpenAI(temperature=0.9, verbose=True)\n", "\n", "for prompt in [\"Tell me a joke\", \"Tell me a poem\", \"Tell me a fact\"]:\n", - " llm_result = llm.generate([prompt], callbacks=[comet_tracer])\n", + " llm_result = llm.generate([prompt])\n", " print(\"LLM result\", llm_result)" ] }, @@ -206,21 +153,16 @@ "outputs": [], "source": [ "from langchain.chains import LLMChain\n", - "from langchain.llms import OpenAI\n", + "from langchain_openai import OpenAI\n", "from langchain.prompts import PromptTemplate\n", "\n", - "tracer = CometTracer()\n", - "callbacks = [tracer]\n", - "\n", - "llm = OpenAI(temperature=0.9, callbacks=callbacks, verbose=True)\n", + "llm = OpenAI(temperature=0.9, verbose=True)\n", "\n", "template = \"\"\"You are a playwright. Given the title of play, it is your job to write a synopsis for that title.\n", "Title: {title}\n", "Playwright: This is a synopsis for the above play:\"\"\"\n", "prompt_template = PromptTemplate(input_variables=[\"title\"], template=template)\n", - "synopsis_chain = LLMChain(\n", - " llm=llm, prompt=prompt_template, callbacks=callbacks, verbose=True\n", - ")\n", + "synopsis_chain = LLMChain(llm=llm, prompt=prompt_template)\n", "\n", "test_prompts = [{\"title\": \"Documentary about Bigfoot in Paris\"}]\n", "print(synopsis_chain.apply(test_prompts))" @@ -240,25 +182,18 @@ "outputs": [], "source": [ "from langchain.agents import initialize_agent, load_tools\n", - "from langchain.llms import OpenAI\n", - "\n", - "\n", - "tracer = CometTracer()\n", - "callbacks = [tracer]\n", + "from langchain_openai import OpenAI\n", "\n", - "llm = OpenAI(temperature=0.9, callbacks=callbacks, verbose=True)\n", + "llm = OpenAI(temperature=0.9)\n", "\n", - "tools = load_tools([\"serpapi\", \"llm-math\"], llm=llm, callbacks=callbacks, verbose=True)\n", + "tools = load_tools([\"llm-math\"], llm=llm)\n", "agent = initialize_agent(\n", " tools,\n", " llm,\n", " agent=\"zero-shot-react-description\",\n", - " callbacks=callbacks,\n", - " verbose=True,\n", ")\n", "agent.run(\n", - " \"Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?\",\n", - " callbacks=callbacks,\n", + " \"What is 2 raised to .123243 power?\",\n", ")" ] } @@ -279,9 +214,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.10.12" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }