diff --git a/docs/use_case_evals.ipynb b/docs/use_case_evals.ipynb new file mode 100644 index 00000000..4e765eb8 --- /dev/null +++ b/docs/use_case_evals.ipynb @@ -0,0 +1,269 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Meeting Search & Analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Single-meeting summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "SCENARIO = \"\"\"Customer specifies a meeting date\n", + "and seeks a summary of the meeting, optionally\n", + "specifying topics of interest.\"\"\"\n", + "\n", + "MEETING_DATE = \"\"\"DATE GOES HERE\"\"\"\n", + "\n", + "test_prompts = [\n", + " f\"\"\"\n", + " Please summarize the key discussion points, public comments, and votes taken\n", + " during the Board of Commissioners meeting on {MEETING_DATE}, including any\n", + " ordinance amendments or resolutions passed.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Generate a detailed summary of the Planning Board meeting on {MEETING_DATE},\n", + " including all zoning requests, variance applications, and site plan reviews,\n", + " along with the board's decisions on each item.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Provide a summary of the joint Board of Commissioners and Planning Board\n", + " workshop from {MEETING_DATE}, highlighting the discussion on the comprehensive\n", + " plan updates and future development guidelines.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Create a concise summary of the Board of Commissioners' budget hearing \n", + " on {MEETING_DATE}, including public comments, tax rate discussions, and\n", + " final budget allocations for municipal departments.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Summarize the Planning Board's special session on {MEETING_DATE} regarding\n", + " the proposed mixed-use development project, including staff recommendations,\n", + " public comments, and the board's conditions for approval.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Please provide a breakdown of the Board of Commissioners meeting on {MEETING_DATE},\n", + " focusing on infrastructure improvement proposals, contract awards, and updates\n", + " on ongoing municipal projects.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Summarize the Planning Board's subdivision review meeting from {MEETING_DATE},\n", + " including environmental impact discussions, traffic study results, and\n", + " conditions placed on preliminary approvals.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Create a detailed overview of the Board of Commissioners' public hearing\n", + " on {MEETING_DATE} regarding the proposed zoning ordinance changes, including\n", + " citizen feedback and final voting results.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Provide a summary of the {MEETING_DATE} Planning Board meeting focused on\n", + " the annual review of development standards, including proposed amendments\n", + " to parking requirements and landscaping regulations.\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Generate a comprehensive summary of the Board of Commissioners' quarterly \n", + " strategic planning session on {MEETING_DATE}, including updates on major\n", + " town initiatives, capital improvement projects, and interlocal agreements.\n", + " \"\"\"\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Specific Topic Search & Analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Topic Updates\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "SCENARIO = \"\"\"Customer specifies a topic and requests an update \n", + "regarding recent developments or historical timelines\"\"\"\n", + "\n", + "SEARCH_TOPIC = \"\"\"\"\"\"\n", + "\n", + "TIME_FRAME = \"\"\"\"\"\"\n", + "\n", + "test_prompts = [\n", + " \n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Topic History" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Customer specifies a topic and requests an overview \n", + "# of historical timelines \n", + "\n", + "SCENARIO = \"\"\"Customer specifies a topic and requests an update \n", + "regarding recent developments or historical timelines\"\"\"\n", + "\n", + "SEARCH_TOPIC = \"\"\"\"\"\"\n", + "\n", + "test_prompts = [\n", + " \"\"\"\n", + " Generate a comprehensive report summarizing \n", + " the discussions and decisions made at town meetings\n", + " related to the {SEARCH_TOPIC}.\n", + " \"\"\",\n", + " \n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Specific Event Search & Analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Event Finder\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "SCENARIO = \"\"\"Customer can describe an event or decision which \n", + "occurred in a certain meeting and wants to know the date of that\n", + "meeting\"\"\"\n", + "\n", + "EVENT_DESCRIPTION = \"\"\"\"\"\"\n", + "\n", + "test_prompts = [\n", + " \n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Trends Analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Citizen Concerns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "SCENARIO = \"\"\"Customer specifies a timeframe and \n", + "seeks a summary of issues or concerns commonly brought up \n", + "by citizens at meetings in that time frame\"\"\"\n", + "\n", + "TIME_FRAME = \"\"\"\"\"\"\n", + "\n", + "test_prompts = [\n", + " f\"\"\"\n", + " Analyze the transcripts of the past several months \n", + " of town meetings to identify the top issues or \n", + " concerns raised by residents, and any emerging\n", + " trends or patterns.\n", + " \"\"\",\n", + " f\"\"\"\n", + " Compile and analyze the transcripts of all \n", + " town meetings over the past year to identify \n", + " trends in community priorities and sentiments.\n", + " \"\"\",\n", + "\n", + "\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "##### Top Topics & Topic Trends" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "SCENARIO = \"\"\"\"Customer specifies a time frame and seeks \n", + "a summary of top discussion topics and emerging trends.\"\"\"\n", + "\n", + "TIME_FRAME = \"\"\"\"\"\"\n", + "\n", + "test_prompts = [\n", + " \n", + "]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}