From f074ded5cd9c6d17d774af067fedfecc920b05a9 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 12 Jun 2023 09:45:02 +0200 Subject: [PATCH 1/4] Add helloworld example for CometLLM --- .github/workflows/test-examples.yml | 1 + .../llm/comet-llm/CometLLM_hello_world.ipynb | 157 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 integrations/llm/comet-llm/CometLLM_hello_world.ipynb diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index cf59ed46..cecbdb1e 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -17,6 +17,7 @@ jobs: - guides/get-started/Comet_Quickstart.ipynb - guides/manage_data/A_Guide_to_Remote_Artifacts.ipynb - guides/manage_data/Introduction_to_Artifacts.ipynb + - integrations/llm/comet-llm/CometLLM_hello_world.ipynb - integrations/model-evaluation/gradio/notebooks/Gradio_and_Comet.ipynb - integrations/model-evaluation/gradio/notebooks/Logging_Model_Inferences_with_Comet_and_Gradio.ipynb - integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb diff --git a/integrations/llm/comet-llm/CometLLM_hello_world.ipynb b/integrations/llm/comet-llm/CometLLM_hello_world.ipynb new file mode 100644 index 00000000..8058e634 --- /dev/null +++ b/integrations/llm/comet-llm/CometLLM_hello_world.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "PWVljpddz_vN" + }, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A0-thQauBRRL" + }, + "source": [ + "[Comet](https://www.comet.com/site/products/ml-experiment-tracking/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch) is an MLOps Platform that is designed to help Data Scientists and Teams build better models faster! Comet provides tooling to track, Explain, Manage, and Monitor your models in a single place! It works with Jupyter Notebooks and Scripts and most importantly it's 100% free to get started!\n", + "\n", + "[PyTorch](https://pytorch.org/) is a popular open source machine learning framework based on the Torch library, used for applications such as computer vision and natural language processing.\n", + "\n", + "PyTorch enables fast, flexible experimentation and efficient production through a user-friendly front-end, distributed training, and ecosystem of tools and libraries.\n", + "\n", + "Instrument PyTorch with Comet to start managing experiments, create dataset versions and track hyperparameters for faster and easier reproducibility and collaboration.\n", + "\n", + "[Find more information about our integration with Pytorch](https://www.comet.ml/docs/v2/integrations/ml-frameworks/pytorch/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch)\n", + "\n", + "Curious about how Comet can help you build better models, faster? Find out more about [Comet](https://www.comet.com/site/products/ml-experiment-tracking/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch) and our [other integrations](https://www.comet.com/docs/v2/integrations/overview/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch)\n", + "\n", + "Get a preview for what's to come. Check out a completed experiment created from this notebook [here](https://www.comet.com/examples/comet-example-pytorch-notebook/view/KH44QDCwjScAzejsflz4injPf/panels?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "V2UZtdWitSLf" + }, + "source": [ + "# Install Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vIQsPNvatQIU" + }, + "outputs": [], + "source": [ + "%pip install comet_llm" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lpCFdN33tday" + }, + "source": [ + "# Initialize Comet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kGyz_i-dtfk4" + }, + "outputs": [], + "source": [ + "import comet_ml\n", + "\n", + "comet_ml.init(project_name=\"comet-example-llm-hello-world\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Log a single prompt and response\n", + "\n", + "This minimal example logs only the minimum amount of data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import comet_llm\n", + "\n", + "comet_llm.log_prompt(\n", + " prompt=\"What is your name?\",\n", + " output=\" My name is Alex.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Log everything\n", + "\n", + "You can also logs a lot more data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import comet_llm\n", + "\n", + "comet_llm.log_prompt(\n", + " prompt=\"Answer the question and if the question can't be answered, say \\\"I don't know\\\"\\n\\n---\\n\\nQuestion: What is your name?\\nAnswer:\",\n", + " prompt_template=\"Answer the question and if the question can't be answered, say \\\"I don't know\\\"\\n\\n---\\n\\nQuestion: {{question}}?\\nAnswer:\",\n", + " prompt_template_variables={\"question\": \"What is your name?\"},\n", + " metadata={\n", + " \"usage.prompt_tokens\": 7,\n", + " \"usage.completion_tokens\": 5,\n", + " \"usage.total_tokens\": 12,\n", + " },\n", + " output=\" My name is Alex.\",\n", + " duration=16.598,\n", + ")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "Comet and Pytorch.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} From 70279012a64932dca3cd9db5b69a8f06626f7f99 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 12 Jun 2023 10:21:25 +0200 Subject: [PATCH 2/4] Move example to the right directory --- .github/workflows/test-examples.yml | 2 +- .../llm/comet-llm/{ => notebooks}/CometLLM_hello_world.ipynb | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename integrations/llm/comet-llm/{ => notebooks}/CometLLM_hello_world.ipynb (100%) diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index cecbdb1e..bc1a4890 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -17,7 +17,7 @@ jobs: - guides/get-started/Comet_Quickstart.ipynb - guides/manage_data/A_Guide_to_Remote_Artifacts.ipynb - guides/manage_data/Introduction_to_Artifacts.ipynb - - integrations/llm/comet-llm/CometLLM_hello_world.ipynb + - integrations/llm/comet-llm/notebools/CometLLM_hello_world.ipynb - integrations/model-evaluation/gradio/notebooks/Gradio_and_Comet.ipynb - integrations/model-evaluation/gradio/notebooks/Logging_Model_Inferences_with_Comet_and_Gradio.ipynb - integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb diff --git a/integrations/llm/comet-llm/CometLLM_hello_world.ipynb b/integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb similarity index 100% rename from integrations/llm/comet-llm/CometLLM_hello_world.ipynb rename to integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb From ae7be64c2869f951250a0dcdb05f4a51e6a25f78 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 12 Jun 2023 11:29:34 +0200 Subject: [PATCH 3/4] Update introduction text for the CometLLM hello world example --- .../comet-llm/notebooks/CometLLM_hello_world.ipynb | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb b/integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb index 8058e634..cad9da07 100644 --- a/integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb +++ b/integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb @@ -15,19 +15,9 @@ "id": "A0-thQauBRRL" }, "source": [ - "[Comet](https://www.comet.com/site/products/ml-experiment-tracking/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch) is an MLOps Platform that is designed to help Data Scientists and Teams build better models faster! Comet provides tooling to track, Explain, Manage, and Monitor your models in a single place! It works with Jupyter Notebooks and Scripts and most importantly it's 100% free to get started!\n", + "[Comet’s LLMOps tools](https://www.comet.com/site/products/llmops/) are designed to allow users to leverage the latest advancement in Prompt Management and query models in Comet to iterate quicker, identify performance bottlenecks, and visualize the internal state of the Prompt Chains.\n", "\n", - "[PyTorch](https://pytorch.org/) is a popular open source machine learning framework based on the Torch library, used for applications such as computer vision and natural language processing.\n", - "\n", - "PyTorch enables fast, flexible experimentation and efficient production through a user-friendly front-end, distributed training, and ecosystem of tools and libraries.\n", - "\n", - "Instrument PyTorch with Comet to start managing experiments, create dataset versions and track hyperparameters for faster and easier reproducibility and collaboration.\n", - "\n", - "[Find more information about our integration with Pytorch](https://www.comet.ml/docs/v2/integrations/ml-frameworks/pytorch/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch)\n", - "\n", - "Curious about how Comet can help you build better models, faster? Find out more about [Comet](https://www.comet.com/site/products/ml-experiment-tracking/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch) and our [other integrations](https://www.comet.com/docs/v2/integrations/overview/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch)\n", - "\n", - "Get a preview for what's to come. Check out a completed experiment created from this notebook [here](https://www.comet.com/examples/comet-example-pytorch-notebook/view/KH44QDCwjScAzejsflz4injPf/panels?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch).\n" + "Get a preview for what's to come. Check out a completed project created from this notebook [here](https://www.comet.com/examples/comet-example-llm-hello-world/prompts).\n" ] }, { From 08b69faaf8ae13a438054f8634ff43e2c1ca725b Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 26 Jun 2023 16:18:25 +0200 Subject: [PATCH 4/4] Fix typo in the test matrix --- .github/workflows/test-examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index bc1a4890..110d3602 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -17,7 +17,7 @@ jobs: - guides/get-started/Comet_Quickstart.ipynb - guides/manage_data/A_Guide_to_Remote_Artifacts.ipynb - guides/manage_data/Introduction_to_Artifacts.ipynb - - integrations/llm/comet-llm/notebools/CometLLM_hello_world.ipynb + - integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb - integrations/model-evaluation/gradio/notebooks/Gradio_and_Comet.ipynb - integrations/model-evaluation/gradio/notebooks/Logging_Model_Inferences_with_Comet_and_Gradio.ipynb - integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb