From fa550baf23363372f15cc6e85e806cf649515ae4 Mon Sep 17 00:00:00 2001 From: Aboli Rajendra Kasar Date: Wed, 20 Nov 2024 13:54:22 -0500 Subject: [PATCH] Updated use cases --- docs/use_case_evals.ipynb | 255 ++++++++++++++++++++++++++++++++------ 1 file changed, 215 insertions(+), 40 deletions(-) diff --git a/docs/use_case_evals.ipynb b/docs/use_case_evals.ipynb index 4e765eb8..1e75612e 100644 --- a/docs/use_case_evals.ipynb +++ b/docs/use_case_evals.ipynb @@ -112,12 +112,66 @@ "SCENARIO = \"\"\"Customer specifies a topic and requests an update \n", "regarding recent developments or historical timelines\"\"\"\n", "\n", - "SEARCH_TOPIC = \"\"\"\"\"\"\n", + "SEARCH_TOPIC = \"\"\"TOPIC GOES HERE\"\"\"\n", "\n", - "TIME_FRAME = \"\"\"\"\"\"\n", + "TIME_FRAME = \"\"\"SELECT TIME FRAME\"\"\"\n", "\n", "test_prompts = [\n", + "\n", + " f\"\"\"\n", + " Show me all discussions and decisions about the {SEARCH_TOPIC} in {TIME_FRAME} across town meetings, \n", + " including funding allocations, timeline updates, and community feedback. Please \n", + " include any related votes or resolutions.\n", + " \"\"\",\n", " \n", + " f\"\"\"\n", + " Summarize all discussions regarding the {SEARCH_TOPIC} in {TIME_FRAME} including planning board \n", + " recommendations, public comments, and any decisions made across different meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Find all mentions and discussions of the proposed {SEARCH_TOPIC} in {TIME_FRAME} including \n", + " cost analysis, implementation timeline, and community feedback from any relevant \n", + " town meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Collect all discussions and decisions related to the {SEARCH_TOPIC} in {TIME_FRAME} across \n", + " meetings, including initial proposals, amendments, final allocations, and any \n", + " public comments or concerns raised.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Provide a summary of all discussions about the {SEARCH_TOPIC} implementation, in {TIME_FRAME}\n", + " including technology requirements, training needs, and budget allocations from \n", + " any relevant meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Find all mentions and decisions regarding the {SEARCH_TOPIC} in {TIME_FRAME} , including facility\n", + " plans, budget discussions, and timeline updates across different meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Summarize all discussions about the {SEARCH_TOPIC} in {TIME_FRAME} , including proposed solutions, \n", + " community feedback, and implementation decisions from any relevant meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Show all discussions and decisions related to the {SEARCH_TOPIC} in {TIME_FRAME} , \n", + " including costs, logistics, and public education plans across different meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Compile all discussions about {SEARCH_TOPIC} in {TIME_FRAME} including coverage \n", + " areas, schedule changes, and budget implications from any relevant town meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Provide a summary of all discussions and decisions regarding the {SEARCH_TOPIC} in {TIME_FRAME} , \n", + " including proposed locations, funding mechanisms, and community feedback across \n", + " different meetings.\n", + " \"\"\"\n", "]" ] }, @@ -140,15 +194,69 @@ "SCENARIO = \"\"\"Customer specifies a topic and requests an update \n", "regarding recent developments or historical timelines\"\"\"\n", "\n", - "SEARCH_TOPIC = \"\"\"\"\"\"\n", + "SEARCH_TOPIC = \"\"\"TOPIC GOES HERE\"\"\"\n", "\n", "test_prompts = [\n", - " \"\"\"\n", - " Generate a comprehensive report summarizing \n", - " the discussions and decisions made at town meetings\n", - " related to the {SEARCH_TOPIC}.\n", + " f\"\"\"\n", + " Please provide a chronological timeline of all discussions and decisions about \n", + " {SEARCH_TOPIC}, starting from its initial proposal. Include key milestones, \n", + " funding decisions, and implementation updates, with particular focus on developments \n", + " in the last three months.\n", " \"\"\",\n", - " \n", + "\n", + " f\"\"\"\n", + " Show me how the town's {SEARCH_TOPIC} policy has evolved over time, including \n", + " initial proposals, amendments, and community feedback. Focus on recent changes \n", + " and upcoming decision points from the last six months of meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " What's the current status of the {SEARCH_TOPIC} project? Please provide \n", + " a timeline of key decisions and changes, highlighting recent developments \n", + " and next steps discussed in recent meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " How has the {SEARCH_TOPIC} budget developed over the past year? Include \n", + " initial proposals, modifications, and final allocations, with emphasis on \n", + " recent adjustments and future planning discussions.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Create a progress report on the town's {SEARCH_TOPIC} program, from its \n", + " initial proposal through current implementation. Focus on recent participation \n", + " rates, challenges discussed, and proposed modifications from recent meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Provide an update on the {SEARCH_TOPIC}, including its full history and timeline. \n", + " Highlight recent construction progress, budget updates, and any modifications \n", + " to the original plan discussed in recent meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Show the development and impact of the {SEARCH_TOPIC} since its inception. \n", + " Include initial goals, implementation challenges, and recent feedback or \n", + " modifications discussed in town meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Trace the evolution of the town's {SEARCH_TOPIC} from initial proposals through \n", + " current status. Emphasize recent funding decisions, service expansions, and \n", + " upcoming plans discussed in meetings.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " How has the town's {SEARCH_TOPIC} initiative progressed since its start? \n", + " Create a timeline showing key decisions, implementation steps, and recent \n", + " developments, including any modifications to goals or timelines.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Provide a comprehensive update on the {SEARCH_TOPIC}, from initial proposals \n", + " through current implementation. Focus on recent technical updates, training \n", + " programs, and any performance metrics discussed in recent meetings.\n", + " \"\"\"\n", "]" ] }, @@ -176,10 +284,48 @@ "occurred in a certain meeting and wants to know the date of that\n", "meeting\"\"\"\n", "\n", - "EVENT_DESCRIPTION = \"\"\"\"\"\"\n", + "EVENT_DESCRIPTION = \"\"\"EVENT KEYWORDS GOES HERE\"\"\"\n", "\n", "test_prompts = [\n", " \n", + " f\"\"\"\n", + " When was the town council meeting where they {EVENT_DESCRIPTION}? I remember there was significant \n", + " debate about reducing the gymnasium size to cut costs.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " I'm trying to find the meeting where {EVENT_DESCRIPTION}\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Which meeting had the heated debate about {EVENT_DESCRIPTION}?\n", + " \"\"\",\n", + " \n", + " f\"\"\"\n", + " Can you tell me when the council voted to approve the {EVENT_DESCRIPTION}It was a meeting \n", + " where they added {EVENT_DESCRIPTION}\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " I need to know the date of the meeting where the council approved the {EVENT_DESCRIPTION}\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " When was the meeting where the council had {EVENT_DESCRIPTION}\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " I'm looking for the meeting where they appointed the new town manager. There was {EVENT_DESCRIPTION}\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " Which meeting addressed the {EVENT_DESCRIPTION}\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " I need the date of the meeting where they {EVENT_DESCRIPTION}\n", + " \"\"\"\n", + " \n", "]" ] }, @@ -205,50 +351,79 @@ "source": [ "\n", "SCENARIO = \"\"\"Customer specifies a timeframe and \n", - "seeks a summary of issues or concerns commonly brought up \n", + "seeks a summary of issues or concerns commonly brought up/top concerns\n", "by citizens at meetings in that time frame\"\"\"\n", "\n", - "TIME_FRAME = \"\"\"\"\"\"\n", + "TIME_FRAME = \"\"\"SELECT TIME FRAME\"\"\" #e.g.,LAST 3 MONTHS | 20XX-20XX | AFTER 20XX\n", "\n", "test_prompts = [\n", " f\"\"\"\n", - " Analyze the transcripts of the past several months \n", - " of town meetings to identify the top issues or \n", - " concerns raised by residents, and any emerging\n", - " trends or patterns.\n", + " Analyze the transcripts of the past {TIME_FRAME} of town meetings to identify the \n", + " top issues or concerns raised by residents, and any emerging trends or patterns.\n", " \"\"\",\n", + "\n", " f\"\"\"\n", - " Compile and analyze the transcripts of all \n", - " town meetings over the past year to identify \n", + " Compile and analyze the transcripts of all town meetings over {TIME_FRAME} to identify \n", " trends in community priorities and sentiments.\n", " \"\"\",\n", "\n", + " f\"\"\"\n", + " Analyze all public comment sections from {TIME_FRAME} and identify the most \n", + " frequently raised concerns by citizens. Include the number of times each issue \n", + " was mentioned and any patterns in how these concerns were addressed.\n", + " \"\"\",\n", "\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "##### Top Topics & Topic Trends" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "SCENARIO = \"\"\"\"Customer specifies a time frame and seeks \n", - "a summary of top discussion topics and emerging trends.\"\"\"\n", + " f\"\"\"\n", + " What were the top 5 issues raised by residents during public comments \n", + " in the {TIME_FRAME}? Please show how these concerns evolved over time \n", + " and whether they resulted in any council actions or responses.\n", + " \"\"\",\n", "\n", - "TIME_FRAME = \"\"\"\"\"\"\n", + " f\"\"\"\n", + " Looking at meetings from {TIME_FRAME} what were the most common concerns \n", + " raised by residents from different neighborhoods? Please group the issues \n", + " by geographic area and show any patterns in the types of concerns raised.\n", + " \"\"\",\n", "\n", - "test_prompts = [\n", - " \n", + " f\"\"\"\n", + " For {TIME_FRAME}, analyze the meetings to show which issues generated the \n", + " most citizen participation. Include the number of speakers per topic and \n", + " the average time spent discussing each concern.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " During the past {TIME_FRAME}, what were the recurring citizen concerns \n", + " that received formal responses from the council? Show the progression \n", + " from initial concern to official response or action.\n", + " \"\"\",\n", + " f\"\"\"\n", + " Compare the primary citizen concerns raised in {TIME_FRAME} versus {TIME_FRAME}.\n", + " Highlight any seasonal patterns in the types of issues brought \n", + " up and how they were addressed.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " For the period of {TIME_FRAME}, summarize all citizen comments \n", + " related to development projects. Group them by project type (residential, \n", + " commercial, mixed-use) and show the main concerns for each category.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " During the {TIME_FRAME}, what were the most common funding-related concerns \n", + " raised by citizens? Include how these concerns influenced budget discussions and decisions.\n", + " \"\"\"\n", + "\n", + " f\"\"\"\n", + " Analyze all citizen comments about town services from the {TIME_FRAME}.\n", + " Identify recurring complaints or suggestions, and track any improvements \n", + " or changes made in response to these comments.\n", + " \"\"\",\n", + "\n", + " f\"\"\"\n", + " What environmental issues were most frequently raised by citizens during \n", + " {TIME_FRAME}? Show how these concerns evolved throughout the year and any \n", + " resulting policy discussions or changes.\n", + " \"\"\"\n", "]" ] }