diff --git a/llm-server/custom_types/bot_message.py b/llm-server/custom_types/bot_message.py index b77b6c0e2..ac21bf400 100644 --- a/llm-server/custom_types/bot_message.py +++ b/llm-server/custom_types/bot_message.py @@ -1,17 +1,17 @@ -from typing import List, Optional +from typing import List from langchain.pydantic_v1 import BaseModel, Field from langchain.output_parsers import PydanticOutputParser + class BotMessage(BaseModel): bot_message: str = Field(description="Message from the bot") ids: List[str] = Field(description="List of IDs") - missing_information: Optional[str] = Field(description="Incase of ambiguity ask user follow up question") - - + + # Set up a parser + inject instructions into the prompt template. bot_message_parser = PydanticOutputParser(pydantic_object=BotMessage) # bot_message_parser.parse(input_string) def parse_bot_message(input: str) -> BotMessage: - return bot_message_parser.parse(input) \ No newline at end of file + return bot_message_parser.parse(input) diff --git a/llm-server/models/migrations/versions/d845330c4432_add_system_summary_prompt_to_chatbot.py b/llm-server/models/migrations/versions/d845330c4432_add_system_summary_prompt_to_chatbot.py new file mode 100644 index 000000000..cdde79e81 --- /dev/null +++ b/llm-server/models/migrations/versions/d845330c4432_add_system_summary_prompt_to_chatbot.py @@ -0,0 +1,41 @@ +"""Add system_summary_prompt to Chatbot + +Revision ID: d845330c4432 +Revises: 86c78095b920 +Create Date: 2023-12-12 14:35:53.182454 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "d845330c4432" +down_revision: Union[str, None] = "86c78095b920" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade(): + if ( + not op.get_bind() + .execute(sa.text("SHOW COLUMNS FROM chatbots LIKE 'summary_prompt'")) + .fetchone() + ): + op.add_column("chatbots", sa.Column("summary_prompt", sa.Text(), nullable=True)) + + op.execute( + """ + UPDATE chatbots + SET summary_prompt = "Given a JSON response, summarize the key information in a concise manner. Include relevant details, references, and links if present. Format the summary in Markdown for clarity and readability." + """ + ) + + op.alter_column( + "chatbots", "summary_prompt", existing_type=sa.TEXT(), nullable=False + ) + + +def downgrade(): + op.drop_column("chatbots", "system_summary_prompt") diff --git a/llm-server/routes/chat/chat_controller.py b/llm-server/routes/chat/chat_controller.py index 32c05c42e..a70297dbe 100644 --- a/llm-server/routes/chat/chat_controller.py +++ b/llm-server/routes/chat/chat_controller.py @@ -158,10 +158,13 @@ async def send_chat(): headers=headers_from_json, server_base_url=server_base_url, app=app_name, + summary_prompt=str(bot.summary_prompt), ) if response_data["response"]: - upsert_analytics_record(chatbot_id=str(bot.id), successful_operations=1, total_operations=1) + upsert_analytics_record( + chatbot_id=str(bot.id), successful_operations=1, total_operations=1 + ) create_chat_history(str(bot.id), session_id, True, message) create_chat_history( str(bot.id), @@ -170,7 +173,12 @@ async def send_chat(): response_data["response"] or response_data["error"] or "", ) elif response_data["error"]: - upsert_analytics_record(chatbot_id=str(bot.id), successful_operations=0, total_operations=1, logs=response_data["error"]) + upsert_analytics_record( + chatbot_id=str(bot.id), + successful_operations=0, + total_operations=1, + logs=response_data["error"], + ) return jsonify( {"type": "text", "response": {"text": response_data["response"]}} diff --git a/llm-server/routes/root_service.py b/llm-server/routes/root_service.py index 42dcf870c..3aabfd1f8 100644 --- a/llm-server/routes/root_service.py +++ b/llm-server/routes/root_service.py @@ -47,6 +47,7 @@ chat = get_chat_model() + def validate_steps(steps: List[str], swagger_doc: ResolvingParser): try: paths = swagger_doc.specification.get("paths", {}) @@ -67,7 +68,11 @@ def validate_steps(steps: List[str], swagger_doc: ResolvingParser): if all(x in operationIds for x in steps): return True else: - logger.warn("Model has hallucinated, made up operation id", steps=steps, operationIds=operationIds) + logger.warn( + "Model has hallucinated, made up operation id", + steps=steps, + operationIds=operationIds, + ) return False except Exception as e: @@ -75,7 +80,6 @@ def validate_steps(steps: List[str], swagger_doc: ResolvingParser): return False - async def handle_request( text: str, swagger_url: str, @@ -85,6 +89,7 @@ async def handle_request( headers: Dict[str, str], server_base_url: str, app: Optional[str], + summary_prompt: str, ) -> ResponseDict: log_user_request(text) check_required_fields(base_prompt, text, swagger_url) @@ -112,22 +117,16 @@ async def handle_request( prev_conversations=prev_conversations, flows=flows, bot_id=bot_id, - base_prompt=base_prompt + base_prompt=base_prompt, ) - - if step.missing_information is not None and len(step.missing_information) >= 10: - return { - "error": None, - "response": step.bot_message + "\n" + step.missing_information - } if len(step.ids) > 0: swagger_doc = get_swagger_doc(swagger_url) fl = validate_steps(step.ids, swagger_doc) - + if fl is False: return {"error": None, "response": step.bot_message} - + response = await handle_api_calls( ids=step.ids, swagger_doc=swagger_doc, @@ -135,9 +134,9 @@ async def handle_request( bot_id=bot_id, headers=headers, server_base_url=server_base_url, - session_id=session_id, text=text, swagger_url=swagger_url, + summary_prompt=summary_prompt, ) logger.info( @@ -196,6 +195,7 @@ def get_swagger_doc(swagger_url: str) -> ResolvingParser: else: return ResolvingParser(spec_string=swagger_doc) + async def handle_api_calls( ids: List[str], swagger_doc: ResolvingParser, @@ -204,8 +204,8 @@ async def handle_api_calls( server_base_url: str, swagger_url: Optional[str], app: Optional[str], - session_id: str, bot_id: str, + summary_prompt: str, ) -> ResponseDict: _workflow = create_workflow_from_operation_ids(ids, swagger_doc, text) output = await run_workflow( @@ -214,6 +214,7 @@ async def handle_api_calls( WorkflowData(text, headers, server_base_url, swagger_url, app), app, bot_id=bot_id, + summary_prompt=summary_prompt, ) _workflow["swagger_url"] = swagger_url diff --git a/llm-server/routes/workflow/extractors/convert_json_to_text.py b/llm-server/routes/workflow/extractors/convert_json_to_text.py index 9ab86cee8..823bd72bf 100644 --- a/llm-server/routes/workflow/extractors/convert_json_to_text.py +++ b/llm-server/routes/workflow/extractors/convert_json_to_text.py @@ -17,20 +17,10 @@ def convert_json_to_text( api_response: Dict[str, Any], api_request_data: Dict[str, Any], bot_id: str, + summary_prompt: str, ) -> str: chat = get_chat_model() - - api_summarizer_template = None - system_message = SystemMessage( - content="You are an ai assistant that can summarize api responses" - ) - prompt_templates = load_prompts(bot_id) - api_summarizer_template = ( - prompt_templates.api_summarizer if prompt_templates else None - ) - - if api_summarizer_template is not None: - system_message = SystemMessage(content=api_summarizer_template) + system_message = SystemMessage(content=summary_prompt) messages = [ system_message, @@ -41,12 +31,14 @@ def convert_json_to_text( HumanMessage( content="Here is the response from the apis: {}".format(api_response) ), - # HumanMessage( - # content="Here is the api_request_data: {}".format(api_request_data) - # ), ] result = chat(messages) - logger.info("Convert json to text", content=result.content, incident="convert_json_to_text", api_request_data=api_request_data) + logger.info( + "Convert json to text", + content=result.content, + incident="convert_json_to_text", + api_request_data=api_request_data, + ) return cast(str, result.content) diff --git a/llm-server/routes/workflow/utils/process_conversation_step.py b/llm-server/routes/workflow/utils/process_conversation_step.py index b025d86f9..d89781530 100644 --- a/llm-server/routes/workflow/utils/process_conversation_step.py +++ b/llm-server/routes/workflow/utils/process_conversation_step.py @@ -28,15 +28,25 @@ def process_conversation_step( prev_conversations: List[BaseMessage], flows: List[WorkflowFlowType], bot_id: str, - base_prompt: str + base_prompt: str, ): - logger.info("planner data", context=context, api_summaries=api_summaries, prev_conversations=prev_conversations, flows=flows) + logger.info( + "planner data", + context=context, + api_summaries=api_summaries, + prev_conversations=prev_conversations, + flows=flows, + ) if not session_id: raise ValueError("Session id must be defined for chat conversations") messages: List[BaseMessage] = [] messages.append(SystemMessage(content=base_prompt)) - - messages.append(SystemMessage(content="You will have access to a list of api's and some useful information, called context.")) + + messages.append( + SystemMessage( + content="You will have access to a list of api's and some useful information, called context." + ) + ) if len(prev_conversations) > 0: messages.extend(prev_conversations) @@ -71,15 +81,14 @@ def process_conversation_step( messages.append( HumanMessage( - content="""Based on the information provided to you and the conversation history of this conversation, I want you to answer the questions that follow. Your should respond with a json that looks like the following, you must always use the operationIds provided in api summaries. Do not make up an operation id - - { - "ids": ["list", "of", "operationIds", "for apis to be called"], - "bot_message": "your response based on the instructions provided at the beginning", - "missing_information": "Optional Field; Incase of ambiguity ask clarifying questions. You should not worry about the api filters or query, that should be decided by a different agent." - } - - Don't add operation ids if you can reply by merely looking in the conversation history. - """ + content="""Based on the information provided to you and the conversation history of this conversation, I want you to answer the questions that follow. You should respond with a json that looks like the following, you must always use the operationIds provided in api summaries. Do not make up an operation id - +{ + "ids": ["list", "of", "operationIds", "for", "apis", "to", "be", "called"], + "bot_message": "your response based on the instructions provided at the beginning, this could also be clarification if the information provided by the user is not complete / accurate", +} + +Don't add operation ids if you can reply by merely looking in the conversation history. +""" ) ) messages.append( @@ -87,8 +96,7 @@ def process_conversation_step( ) messages.append(HumanMessage(content=user_requirement)) - - + logger.info("messages array", messages=messages) content = cast(str, chat(messages=messages).content) @@ -105,7 +113,7 @@ def process_conversation_step( except OutputParserException as e: logger.warn("Failed to parse json", data=content, err=str(e)) - return BotMessage(bot_message=content, ids=[], missing_information=None) + return BotMessage(bot_message=content, ids=[]) except Exception as e: logger.warn("unexpected error occured", err=str(e)) - return BotMessage(ids=[], bot_message=str(e), missing_information=None) + return BotMessage(ids=[], bot_message=str(e)) diff --git a/llm-server/routes/workflow/utils/run_openapi_ops.py b/llm-server/routes/workflow/utils/run_openapi_ops.py index 55cacf5aa..3391eaf92 100644 --- a/llm-server/routes/workflow/utils/run_openapi_ops.py +++ b/llm-server/routes/workflow/utils/run_openapi_ops.py @@ -2,10 +2,7 @@ from opencopilot_types.workflow_type import WorkflowDataType from routes.workflow.generate_openapi_payload import generate_openapi_payload from utils.make_api_call import make_api_request -from typing import Any, Optional -from routes.workflow.extractors.transform_api_response import ( - transform_api_response_from_schema, -) +from typing import Optional from routes.workflow.extractors.convert_json_to_text import convert_json_to_text from utils.process_app_state import process_state from prance import ResolvingParser @@ -25,6 +22,7 @@ async def run_openapi_operations( server_base_url: str, app: Optional[str], bot_id: str, + summary_prompt: str, ) -> str: api_request_data = {} prev_api_response = "" @@ -47,7 +45,12 @@ async def run_openapi_operations( api_request_data[operation_id] = api_payload.__dict__ api_response = None try: - logger.info("Making API call", incident="make_api_call", body=json.dumps(api_payload.body_schema), params=api_payload.query_params) + logger.info( + "Making API call", + incident="make_api_call", + body=json.dumps(api_payload.body_schema), + params=api_payload.query_params, + ) api_response = make_api_request( headers=headers, **api_payload.__dict__ @@ -59,10 +62,14 @@ async def run_openapi_operations( raise ValueError("API response is not JSON") except Exception as e: - logger.error("Error occurred while making API call", incident="make_api_call_failed", error=str(e)) + logger.error( + "Error occurred while making API call", + incident="make_api_call_failed", + error=str(e), + ) raise e - logger.info("Got the following api response", text = api_response.text) + logger.info("Got the following api response", text=api_response.text) # if a custom transformer function is defined for this operationId use that, otherwise forward it to the llm # so we don't necessarily have to defined mappers for all api endpoints partial_json = load_json_config(app, operation_id) @@ -71,15 +78,15 @@ async def run_openapi_operations( "Config map is not defined for this operationId", incident="config_map_undefined", operation_id=operation_id, - app=app + app=app, ) record_info[operation_id] = api_response.text - + # Removed this because this slows down the bot response instead of speeding it # record_info[operation_id] = transform_api_response_from_schema( # api_payload.endpoint or "", api_response.text # ) - + pass else: logger.info( @@ -99,13 +106,19 @@ async def run_openapi_operations( except Exception as e: logger.error( "Error occurred during workflow check in store", - incident="check_workflow_in_store", - text= text, - headers= headers, - server_base_url= server_base_url, - app= app, + incident="check_workflow_in_store", + text=text, + headers=headers, + server_base_url=server_base_url, + app=app, error=str(e), ) - + return str(e) - return convert_json_to_text(text, record_info, api_request_data, bot_id=bot_id) + return convert_json_to_text( + text, + record_info, + api_request_data, + bot_id=bot_id, + summary_prompt=summary_prompt, + ) diff --git a/llm-server/routes/workflow/utils/run_workflow.py b/llm-server/routes/workflow/utils/run_workflow.py index 03faa7836..d09529344 100644 --- a/llm-server/routes/workflow/utils/run_workflow.py +++ b/llm-server/routes/workflow/utils/run_workflow.py @@ -18,6 +18,7 @@ async def run_workflow( data: WorkflowData, app: Optional[str], bot_id: str, + summary_prompt: str, ) -> ResponseDict: headers = data.headers or Headers() server_base_url = data.server_base_url @@ -34,6 +35,7 @@ async def run_workflow( server_base_url, app, bot_id=bot_id, + summary_prompt=summary_prompt, ) except Exception as e: payload_data = { @@ -53,8 +55,6 @@ async def run_workflow( output: ResponseDict = {"response": result if not error else "", "error": error} - logging.info( - "Workflow output %s", json.dumps(output, separators=(",", ":")) - ) + logging.info("Workflow output %s", json.dumps(output, separators=(",", ":"))) return output diff --git a/llm-server/shared/models/opencopilot_db/chatbot.py b/llm-server/shared/models/opencopilot_db/chatbot.py index 32e81bcfc..08155b74f 100644 --- a/llm-server/shared/models/opencopilot_db/chatbot.py +++ b/llm-server/shared/models/opencopilot_db/chatbot.py @@ -6,21 +6,32 @@ class Chatbot(Base): - __tablename__ = 'chatbots' + __tablename__ = "chatbots" id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) name = Column(String(255)) - email = Column(String(255), nullable=True, default='guest') + email = Column(String(255), nullable=True, default="guest") token = Column(String(255)) website = Column(String(255), nullable=True) - status = Column(String(255), default='draft') + status = Column(String(255), default="draft") prompt_message = Column(Text) swagger_url = Column(Text) enhanced_privacy = Column(Boolean, default=False) smart_sync = Column(Boolean, default=False) created_at = Column(DateTime, default=datetime.datetime.utcnow) - updated_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) + updated_at = Column( + DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow + ) deleted_at = Column(DateTime, nullable=True) + summary_prompt = Column( + Text, + default=( + "Given a JSON response, summarize the key information in a concise manner. " + "Include relevant details, references, and links if present. " + "Format the summary in Markdown for clarity and readability." + ), + ) -Base.metadata.create_all(engine) \ No newline at end of file + +Base.metadata.create_all(engine) diff --git a/llm-server/utils/get_chat_model.py b/llm-server/utils/get_chat_model.py index 41ed2e38e..6ae83a8eb 100644 --- a/llm-server/utils/get_chat_model.py +++ b/llm-server/utils/get_chat_model.py @@ -9,40 +9,25 @@ localip = os.getenv("LOCAL_IP", "localhost") -model_name = os.getenv(llm_consts.model_env_var, CHAT_MODELS.gpt_4_32k) +model_name = os.getenv(llm_consts.model_env_var, CHAT_MODELS.gpt_3_5_turbo_16k) + @lru_cache(maxsize=1) def get_chat_model() -> BaseChatModel: if model_name == CHAT_MODELS.gpt_3_5_turbo: - model = ChatOpenAI( - model=CHAT_MODELS.gpt_3_5_turbo, - temperature=0 - ) + model = ChatOpenAI(model=CHAT_MODELS.gpt_3_5_turbo, temperature=0) elif model_name == CHAT_MODELS.gpt_4_32k: - model = ChatOpenAI( - temperature=0, - model=CHAT_MODELS.gpt_4_32k - ) + model = ChatOpenAI(temperature=0, model=CHAT_MODELS.gpt_4_32k) elif model_name == CHAT_MODELS.gpt_4_1106_preview: - model = ChatOpenAI( - temperature=0, - model=CHAT_MODELS.gpt_4_1106_preview - ) + model = ChatOpenAI(temperature=0, model=CHAT_MODELS.gpt_4_1106_preview) elif model_name == CHAT_MODELS.gpt_3_5_turbo_16k: - model = ChatOpenAI( - model=CHAT_MODELS.gpt_3_5_turbo_16k, - temperature=0 - ) + model = ChatOpenAI(model=CHAT_MODELS.gpt_3_5_turbo_16k, temperature=0) elif model_name == "claude": model = ChatAnthropic( - anthropic_api_key=os.getenv("CLAUDE_API_KEY"), + anthropic_api_key=os.getenv("CLAUDE_API_KEY"), ) elif model_name == "openchat": - model = ChatOllama( - base_url=f"{localip}:11434", - model="openchat", - temperature=0 - ) + model = ChatOllama(base_url=f"{localip}:11434", model="openchat", temperature=0) else: raise ValueError(f"Unsupported model: {model_name}") - return model \ No newline at end of file + return model