Merge pull request #39 from dsba6010-llm-applications/neal-logan/add-…

…use-case-evals

Added a notebook to /docs with test prompts organized by use case

docs/use_case_evals.ipynb

@@ -0,0 +1,269 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Meeting Search & Analysis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Single-meeting summary"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SCENARIO = \"\"\"Customer specifies a meeting date\n",
+    "and seeks a summary of the meeting, optionally\n",
+    "specifying topics of interest.\"\"\"\n",
+    "\n",
+    "MEETING_DATE = \"\"\"DATE GOES HERE\"\"\"\n",
+    "\n",
+    "test_prompts = [\n",
+    "    f\"\"\"\n",
+    "    Please summarize the key discussion points, public comments, and votes taken\n",
+    "    during the Board of Commissioners meeting on {MEETING_DATE}, including any\n",
+    "    ordinance amendments or resolutions passed.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Generate a detailed summary of the Planning Board meeting on {MEETING_DATE},\n",
+    "    including all zoning requests, variance applications, and site plan reviews,\n",
+    "    along with the board's decisions on each item.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Provide a summary of the joint Board of Commissioners and Planning Board\n",
+    "    workshop from {MEETING_DATE}, highlighting the discussion on the comprehensive\n",
+    "    plan updates and future development guidelines.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Create a concise summary of the Board of Commissioners' budget hearing \n",
+    "    on {MEETING_DATE}, including public comments, tax rate discussions, and\n",
+    "    final budget allocations for municipal departments.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Summarize the Planning Board's special session on {MEETING_DATE} regarding\n",
+    "    the proposed mixed-use development project, including staff recommendations,\n",
+    "    public comments, and the board's conditions for approval.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Please provide a breakdown of the Board of Commissioners meeting on {MEETING_DATE},\n",
+    "    focusing on infrastructure improvement proposals, contract awards, and updates\n",
+    "    on ongoing municipal projects.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Summarize the Planning Board's subdivision review meeting from {MEETING_DATE},\n",
+    "    including environmental impact discussions, traffic study results, and\n",
+    "    conditions placed on preliminary approvals.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Create a detailed overview of the Board of Commissioners' public hearing\n",
+    "    on {MEETING_DATE} regarding the proposed zoning ordinance changes, including\n",
+    "    citizen feedback and final voting results.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Provide a summary of the {MEETING_DATE} Planning Board meeting focused on\n",
+    "    the annual review of development standards, including proposed amendments\n",
+    "    to parking requirements and landscaping regulations.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "    f\"\"\"\n",
+    "    Generate a comprehensive summary of the Board of Commissioners' quarterly \n",
+    "    strategic planning session on {MEETING_DATE}, including updates on major\n",
+    "    town initiatives, capital improvement projects, and interlocal agreements.\n",
+    "    \"\"\"\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Specific Topic Search & Analysis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Topic Updates\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SCENARIO = \"\"\"Customer specifies a topic and requests an update \n",
+    "regarding recent developments or historical timelines\"\"\"\n",
+    "\n",
+    "SEARCH_TOPIC = \"\"\"\"\"\"\n",
+    "\n",
+    "TIME_FRAME = \"\"\"\"\"\"\n",
+    "\n",
+    "test_prompts = [\n",
+    "    \n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Topic History"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Customer specifies a topic and requests an overview \n",
+    "# of historical timelines  \n",
+    "\n",
+    "SCENARIO = \"\"\"Customer specifies a topic and requests an update \n",
+    "regarding recent developments or historical timelines\"\"\"\n",
+    "\n",
+    "SEARCH_TOPIC = \"\"\"\"\"\"\n",
+    "\n",
+    "test_prompts = [\n",
+    "    \"\"\"\n",
+    "    Generate a comprehensive report summarizing \n",
+    "    the discussions and decisions made at town meetings\n",
+    "    related to the {SEARCH_TOPIC}.\n",
+    "    \"\"\",\n",
+    "    \n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Specific Event Search & Analysis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Event Finder\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SCENARIO = \"\"\"Customer can describe an event or decision which \n",
+    "occurred in a certain meeting and wants to know the date of that\n",
+    "meeting\"\"\"\n",
+    "\n",
+    "EVENT_DESCRIPTION = \"\"\"\"\"\"\n",
+    "\n",
+    "test_prompts = [\n",
+    "    \n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Trends Analysis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Citizen Concerns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "SCENARIO = \"\"\"Customer specifies a timeframe and \n",
+    "seeks a summary of issues or concerns commonly brought up \n",
+    "by citizens at meetings in that time frame\"\"\"\n",
+    "\n",
+    "TIME_FRAME = \"\"\"\"\"\"\n",
+    "\n",
+    "test_prompts = [\n",
+    "    f\"\"\"\n",
+    "    Analyze the transcripts of the past several months \n",
+    "    of town meetings to identify the top issues or \n",
+    "    concerns raised by residents, and any emerging\n",
+    "    trends or patterns.\n",
+    "    \"\"\",\n",
+    "    f\"\"\"\n",
+    "    Compile and analyze the transcripts of all \n",
+    "    town meetings over the past year to identify \n",
+    "    trends in community priorities and sentiments.\n",
+    "    \"\"\",\n",
+    "\n",
+    "\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "##### Top Topics & Topic Trends"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SCENARIO = \"\"\"\"Customer specifies a time frame and seeks \n",
+    "a summary of top discussion topics and emerging trends.\"\"\"\n",
+    "\n",
+    "TIME_FRAME = \"\"\"\"\"\"\n",
+    "\n",
+    "test_prompts = [\n",
+    "    \n",
+    "]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}