diff --git a/README.md b/README.md index 847658cf1..6efe8d594 100644 --- a/README.md +++ b/README.md @@ -69,11 +69,9 @@ You will need the following items to run the sample: - > **IMPORTANT:** For `AzureOpenAI`, if you deployed models `gpt-35-turbo` and `text-embedding-ada-002` with custom names (instead of the default names), also use the parameters: ```powershell - -CompletionModel {DEPLOYMENT_NAME} -EmbeddingModel {DEPLOYMENT_NAME} -PlannerModel {DEPLOYMENT_NAME} + -CompletionModel {DEPLOYMENT_NAME} -EmbeddingModel {DEPLOYMENT_NAME} ``` - > -PlannerModel name will be the same as -CompletionModel - Open the `.\Configure.ps1` script to see all of the available parameters. 1. Run Chat Copilot locally. This step starts both the backend API and frontend application. @@ -153,12 +151,9 @@ You will need the following items to run the sample: --endpoint {AZURE_OPENAI_ENDPOINT} \ --apikey {API_KEY} \ --completionmodel {DEPLOYMENT_NAME} \ - --plannermodel {DEPLOYMENT_NAME} \ --embeddingmodel {DEPLOYMENT_NAME} ``` - `--plannermodel` will be the same name as `--completionmodel` - 1. Run Chat Copilot locally. This step starts both the backend API and frontend application. ```bash diff --git a/integration-tests/ChatCopilotIntegrationTests.csproj b/integration-tests/ChatCopilotIntegrationTests.csproj index debb176fc..78564602e 100644 --- a/integration-tests/ChatCopilotIntegrationTests.csproj +++ b/integration-tests/ChatCopilotIntegrationTests.csproj @@ -14,13 +14,13 @@ - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/memorypipeline/CopilotChatMemoryPipeline.csproj b/memorypipeline/CopilotChatMemoryPipeline.csproj index 0906ac571..fe48d0d9b 100644 --- a/memorypipeline/CopilotChatMemoryPipeline.csproj +++ b/memorypipeline/CopilotChatMemoryPipeline.csproj @@ -14,9 +14,9 @@ - - - + + + diff --git a/memorypipeline/README.md b/memorypipeline/README.md index e7f35030a..270047455 100644 --- a/memorypipeline/README.md +++ b/memorypipeline/README.md @@ -43,7 +43,7 @@ The memorypipeline is only needed when `SemanticMemory:DataIngestion:Orchestrati - RabbitMQ - SimpleQueues: stores messages on your local file system. - [Vector database](https://learn.microsoft.com/en-us/semantic-kernel/memories/vector-db): storage solution for high-dimensional vectors, aka [embeddings](https://github.com/microsoft/semantic-kernel/blob/main/docs/EMBEDDINGS.md). Available options: - - [AzureCognitiveSearch](https://learn.microsoft.com/en-us/azure/search/search-what-is-azure-search) + - [AzureAISearch](https://learn.microsoft.com/en-us/azure/search/search-what-is-azure-search) - [Qdrant](https://github.com/qdrant/qdrant) - SimpleVectorDb - TextFile: stores vectors on your local file system. @@ -72,8 +72,8 @@ The memorypipeline is only needed when `SemanticMemory:DataIngestion:Orchestrati 2. Find the **Url** under **Overview** and the **key** under **Keys** on the portal. 3. Run the following to set up the authentication to the resources: ```bash - dotnet user-secrets set SemanticMemory:Services:AzureCognitiveSearch:Endpoint [your secret] - dotnet user-secrets set SemanticMemory:Services:AzureCognitiveSearch:APIKey [your secret] + dotnet user-secrets set SemanticMemory:Services:AzureAISearch:Endpoint [your secret] + dotnet user-secrets set SemanticMemory:Services:AzureAISearch:APIKey [your secret] ``` ##### RabbitMQ diff --git a/memorypipeline/appsettings.json b/memorypipeline/appsettings.json index d6e9c2dc0..932afd299 100644 --- a/memorypipeline/appsettings.json +++ b/memorypipeline/appsettings.json @@ -18,7 +18,7 @@ // Distributed: asynchronous queue based orchestrator // - DistributedOrchestration is the detailed configuration for OrchestrationType=Distributed // - EmbeddingGeneratorTypes is the list of embedding generator types - // - VectorDbTypes is the list of vector database types + // - MemoryDbTypes is the list of vector database types "DataIngestion": { "OrchestrationType": "Distributed", // @@ -33,17 +33,17 @@ "AzureOpenAIEmbedding" ], // Vectors can be written to multiple storages, e.g. for data migration, A/B testing, etc. - "VectorDbTypes": [ + "MemoryDbTypes": [ "SimpleVectorDb" ] }, // // Memory retrieval configuration - A single EmbeddingGenerator and VectorDb. - // - VectorDbType: Vector database configuration: "SimpleVectorDb" or "AzureCognitiveSearch" or "Qdrant" + // - MemoryDbType: Vector database configuration: "SimpleVectorDb" or "AzureAISearch" or "Qdrant" // - EmbeddingGeneratorType: Embedding generator configuration: "AzureOpenAIEmbedding" or "OpenAI" // "Retrieval": { - "VectorDbType": "SimpleVectorDb", + "MemoryDbType": "SimpleVectorDb", "EmbeddingGeneratorType": "AzureOpenAIEmbedding" }, // @@ -122,9 +122,9 @@ // - APIKey is the key generated to access the service. // - Endpoint is the service endpoint url. // - "AzureCognitiveSearch": { + "AzureAISearch": { "Auth": "ApiKey", - //"APIKey": "", // dotnet user-secrets set "SemanticMemory:Services:AzureCognitiveSearch:APIKey" "MY_ACS_KEY" + //"APIKey": "", // dotnet user-secrets set "SemanticMemory:Services:AzureAISearch:APIKey" "MY_ACS_KEY" "Endpoint": "" }, // diff --git a/scripts/deploy/deploy-azure.ps1 b/scripts/deploy/deploy-azure.ps1 index c82787f6a..7dd91276d 100644 --- a/scripts/deploy/deploy-azure.ps1 +++ b/scripts/deploy/deploy-azure.ps1 @@ -58,10 +58,10 @@ param( # Azure AD cloud instance for authenticating users $AzureAdInstance = "https://login.microsoftonline.com", - [ValidateSet("AzureCognitiveSearch", "Qdrant")] + [ValidateSet("AzureAISearch", "Qdrant")] [string] # What method to use to persist embeddings - $MemoryStore = "AzureCognitiveSearch", + $MemoryStore = "AzureAISearch", [switch] # Don't deploy Cosmos DB for chat storage - Use volatile memory instead diff --git a/scripts/deploy/deploy-azure.sh b/scripts/deploy/deploy-azure.sh index 81f89d522..e6df71d05 100755 --- a/scripts/deploy/deploy-azure.sh +++ b/scripts/deploy/deploy-azure.sh @@ -22,7 +22,7 @@ usage() { echo " -i, --instance AZURE_AD_INSTANCE Azure AD cloud instance for authenticating users" echo " (default: \"https://login.microsoftonline.com\")" echo " -ms, --memory-store Method to use to persist embeddings. Valid values are" - echo " \"AzureCognitiveSearch\" (default) and \"Qdrant\"" + echo " \"AzureAISearch\" (default) and \"Qdrant\"" echo " -nc, --no-cosmos-db Don't deploy Cosmos DB for chat storage - Use volatile memory instead" echo " -ns, --no-speech-services Don't deploy Speech Services to enable speech as chat input" echo " -ws, --deploy-web-searcher-plugin Deploy the web searcher plugin" @@ -184,7 +184,7 @@ az account set -s "$SUBSCRIPTION" : "${REGION:="southcentralus"}" : "${WEB_APP_SVC_SKU:="B1"}" : "${AZURE_AD_INSTANCE:="https://login.microsoftonline.com"}" -: "${MEMORY_STORE:="AzureCognitiveSearch"}" +: "${MEMORY_STORE:="AzureAISearch"}" : "${NO_COSMOS_DB:=false}" : "${NO_SPEECH_SERVICES:=false}" : "${DEPLOY_WEB_SEARCHER_PLUGIN:=false}" diff --git a/scripts/deploy/main.bicep b/scripts/deploy/main.bicep index 5f4813b07..756affb15 100644 --- a/scripts/deploy/main.bicep +++ b/scripts/deploy/main.bicep @@ -37,9 +37,6 @@ param completionModel string = 'gpt-35-turbo' @description('Model to use for text embeddings') param embeddingModel string = 'text-embedding-ada-002' -@description('Completion model the task planner should use') -param plannerModel string = 'gpt-35-turbo' - @description('Azure OpenAI endpoint to use (Azure OpenAI only)') param aiEndpoint string = '' @@ -67,10 +64,10 @@ param deployCosmosDB bool = true @description('What method to use to persist embeddings') @allowed([ - 'AzureCognitiveSearch' + 'AzureAISearch' 'Qdrant' ]) -param memoryStore string = 'AzureCognitiveSearch' +param memoryStore string = 'AzureAISearch' @description('Whether to deploy Azure Speech Services to enable input by voice') param deploySpeechServices bool = true @@ -206,10 +203,6 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = { name: 'Authentication:AzureAd:Scopes' value: 'access_as_user' } - { - name: 'Planner:Model' - value: plannerModel - } { name: 'ChatStore:Type' value: deployCosmosDB ? 'cosmos' : 'volatile' @@ -311,11 +304,11 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = { value: aiService } { - name: 'KernelMemory:DataIngestion:VectorDbTypes:0' + name: 'KernelMemory:DataIngestion:MemoryDbTypes:0' value: memoryStore } { - name: 'KernelMemory:Retrieval:VectorDbType' + name: 'KernelMemory:Retrieval:MemoryDbType' value: memoryStore } { @@ -343,16 +336,16 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = { value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[1].value}' } { - name: 'KernelMemory:Services:AzureCognitiveSearch:Auth' + name: 'KernelMemory:Services:AzureAISearch:Auth' value: 'ApiKey' } { - name: 'KernelMemory:Services:AzureCognitiveSearch:Endpoint' - value: memoryStore == 'AzureCognitiveSearch' ? 'https://${azureCognitiveSearch.name}.search.windows.net' : '' + name: 'KernelMemory:Services:AzureAISearch:Endpoint' + value: memoryStore == 'AzureAISearch' ? 'https://${azureAISearch.name}.search.windows.net' : '' } { - name: 'KernelMemory:Services:AzureCognitiveSearch:APIKey' - value: memoryStore == 'AzureCognitiveSearch' ? azureCognitiveSearch.listAdminKeys().primaryKey : '' + name: 'KernelMemory:Services:AzureAISearch:APIKey' + value: memoryStore == 'AzureAISearch' ? azureAISearch.listAdminKeys().primaryKey : '' } { name: 'KernelMemory:Services:Qdrant:Endpoint' @@ -480,7 +473,7 @@ resource appServiceMemoryPipelineConfig 'Microsoft.Web/sites/config@2022-09-01' value: aiService } { - name: 'KernelMemory:ImageOcrType' + name: 'KernelMemory:DataIngestion:ImageOcrType' value: 'AzureFormRecognizer' } { @@ -496,11 +489,11 @@ resource appServiceMemoryPipelineConfig 'Microsoft.Web/sites/config@2022-09-01' value: aiService } { - name: 'KernelMemory:DataIngestion:VectorDbTypes:0' + name: 'KernelMemory:DataIngestion:MemoryDbTypes:0' value: memoryStore } { - name: 'KernelMemory:Retrieval:VectorDbType' + name: 'KernelMemory:Retrieval:MemoryDbType' value: memoryStore } { @@ -528,16 +521,16 @@ resource appServiceMemoryPipelineConfig 'Microsoft.Web/sites/config@2022-09-01' value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[1].value}' } { - name: 'KernelMemory:Services:AzureCognitiveSearch:Auth' + name: 'KernelMemory:Services:AzureAISearch:Auth' value: 'ApiKey' } { - name: 'KernelMemory:Services:AzureCognitiveSearch:Endpoint' - value: memoryStore == 'AzureCognitiveSearch' ? 'https://${azureCognitiveSearch.name}.search.windows.net' : '' + name: 'KernelMemory:Services:AzureAISearch:Endpoint' + value: memoryStore == 'AzureAISearch' ? 'https://${azureAISearch.name}.search.windows.net' : '' } { - name: 'KernelMemory:Services:AzureCognitiveSearch:APIKey' - value: memoryStore == 'AzureCognitiveSearch' ? azureCognitiveSearch.listAdminKeys().primaryKey : '' + name: 'KernelMemory:Services:AzureAISearch:APIKey' + value: memoryStore == 'AzureAISearch' ? azureAISearch.listAdminKeys().primaryKey : '' } { name: 'KernelMemory:Services:Qdrant:Endpoint' @@ -812,7 +805,7 @@ resource appServiceQdrant 'Microsoft.Web/sites@2022-09-01' = if (memoryStore == } } -resource azureCognitiveSearch 'Microsoft.Search/searchServices@2022-09-01' = if (memoryStore == 'AzureCognitiveSearch') { +resource azureAISearch 'Microsoft.Search/searchServices@2022-09-01' = if (memoryStore == 'AzureAISearch') { name: 'acs-${uniqueName}' location: location sku: { diff --git a/scripts/deploy/main.json b/scripts/deploy/main.json index b38c967d2..27b231f48 100644 --- a/scripts/deploy/main.json +++ b/scripts/deploy/main.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.23.1.45101", - "templateHash": "10280441124249350356" + "version": "0.25.53.49325", + "templateHash": "1335279082624698663" } }, "parameters": { @@ -79,13 +79,6 @@ "description": "Model to use for text embeddings" } }, - "plannerModel": { - "type": "string", - "defaultValue": "gpt-35-turbo", - "metadata": { - "description": "Completion model the task planner should use" - } - }, "aiEndpoint": { "type": "string", "defaultValue": "", @@ -140,9 +133,9 @@ }, "memoryStore": { "type": "string", - "defaultValue": "AzureCognitiveSearch", + "defaultValue": "AzureAISearch", "allowedValues": [ - "AzureCognitiveSearch", + "AzureAISearch", "Qdrant" ], "metadata": { @@ -306,7 +299,7 @@ "use32BitWorkerProcess": false, "vnetRouteAllEnabled": true, "webSocketsEnabled": true, - "appSettings": "[concat(createArray(createObject('name', 'Authentication:Type', 'value', 'AzureAd'), createObject('name', 'Authentication:AzureAd:Instance', 'value', parameters('azureAdInstance')), createObject('name', 'Authentication:AzureAd:TenantId', 'value', parameters('azureAdTenantId')), createObject('name', 'Authentication:AzureAd:ClientId', 'value', parameters('webApiClientId')), createObject('name', 'Authentication:AzureAd:Scopes', 'value', 'access_as_user'), createObject('name', 'Planner:Model', 'value', parameters('plannerModel')), createObject('name', 'ChatStore:Type', 'value', if(parameters('deployCosmosDB'), 'cosmos', 'volatile')), createObject('name', 'ChatStore:Cosmos:Database', 'value', 'CopilotChat'), createObject('name', 'ChatStore:Cosmos:ChatSessionsContainer', 'value', 'chatsessions'), createObject('name', 'ChatStore:Cosmos:ChatMessagesContainer', 'value', 'chatmessages'), createObject('name', 'ChatStore:Cosmos:ChatMemorySourcesContainer', 'value', 'chatmemorysources'), createObject('name', 'ChatStore:Cosmos:ChatParticipantsContainer', 'value', 'chatparticipants'), createObject('name', 'ChatStore:Cosmos:ConnectionString', 'value', if(parameters('deployCosmosDB'), listConnectionStrings(resourceId('Microsoft.DocumentDB/databaseAccounts', toLower(format('cosmos-{0}', variables('uniqueName')))), '2023-04-15').connectionStrings[0].connectionString, '')), createObject('name', 'AzureSpeech:Region', 'value', parameters('location')), createObject('name', 'AzureSpeech:Key', 'value', if(parameters('deploySpeechServices'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('cog-speech-{0}', variables('uniqueName'))), '2022-12-01').key1, '')), createObject('name', 'AllowedOrigins', 'value', '[*]'), createObject('name', 'Kestrel:Endpoints:Https:Url', 'value', 'https://localhost:443'), createObject('name', 'Frontend:AadClientId', 'value', parameters('frontendClientId')), createObject('name', 'Logging:LogLevel:Default', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:CopilotChat.WebApi', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.SemanticKernel', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.AspNetCore.Hosting', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.Hosting.Lifetimel', 'value', 'Warning'), createObject('name', 'Logging:ApplicationInsights:LogLevel:Default', 'value', 'Warning'), createObject('name', 'APPLICATIONINSIGHTS_CONNECTION_STRING', 'value', reference(resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName'))), '2020-02-02').ConnectionString), createObject('name', 'ApplicationInsightsAgent_EXTENSION_VERSION', 'value', '~2'), createObject('name', 'KernelMemory:ContentStorageType', 'value', 'AzureBlobs'), createObject('name', 'KernelMemory:TextGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:OrchestrationType', 'value', 'Distributed'), createObject('name', 'KernelMemory:DataIngestion:DistributedOrchestration:QueueType', 'value', 'AzureQueue'), createObject('name', 'KernelMemory:DataIngestion:EmbeddingGeneratorTypes:0', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:VectorDbTypes:0', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:VectorDbType', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:EmbeddingGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:Services:AzureBlobs:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureBlobs:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureBlobs:Container', 'value', 'chatmemory'), createObject('name', 'KernelMemory:Services:AzureQueue:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureQueue:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureCognitiveSearch:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureCognitiveSearch:Endpoint', 'value', if(equals(parameters('memoryStore'), 'AzureCognitiveSearch'), format('https://{0}.search.windows.net', format('acs-{0}', variables('uniqueName'))), '')), createObject('name', 'KernelMemory:Services:AzureCognitiveSearch:APIKey', 'value', if(equals(parameters('memoryStore'), 'AzureCognitiveSearch'), listAdminKeys(resourceId('Microsoft.Search/searchServices', format('acs-{0}', variables('uniqueName'))), '2022-09-01').primaryKey, '')), createObject('name', 'KernelMemory:Services:Qdrant:Endpoint', 'value', if(equals(parameters('memoryStore'), 'Qdrant'), format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('app-{0}-qdrant', variables('uniqueName'))), '2022-09-01').defaultHostName), '')), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Deployment', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Deployment', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:TextModel', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:OpenAI:EmbeddingModel', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:APIKey', 'value', parameters('aiApiKey')), createObject('name', 'Plugins:0:Name', 'value', 'Klarna Shopping'), createObject('name', 'Plugins:0:ManifestDomain', 'value', 'https://www.klarna.com')), if(parameters('deployWebSearcherPlugin'), createArray(createObject('name', 'Plugins:1:Name', 'value', 'WebSearcher'), createObject('name', 'Plugins:1:ManifestDomain', 'value', format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName'))), '2022-09-01').defaultHostName)), createObject('name', 'Plugins:1:Key', 'value', listkeys(format('{0}/host/default/', resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName')))), '2022-09-01').functionKeys.default)), createArray()))]" + "appSettings": "[concat(createArray(createObject('name', 'Authentication:Type', 'value', 'AzureAd'), createObject('name', 'Authentication:AzureAd:Instance', 'value', parameters('azureAdInstance')), createObject('name', 'Authentication:AzureAd:TenantId', 'value', parameters('azureAdTenantId')), createObject('name', 'Authentication:AzureAd:ClientId', 'value', parameters('webApiClientId')), createObject('name', 'Authentication:AzureAd:Scopes', 'value', 'access_as_user'), createObject('name', 'ChatStore:Type', 'value', if(parameters('deployCosmosDB'), 'cosmos', 'volatile')), createObject('name', 'ChatStore:Cosmos:Database', 'value', 'CopilotChat'), createObject('name', 'ChatStore:Cosmos:ChatSessionsContainer', 'value', 'chatsessions'), createObject('name', 'ChatStore:Cosmos:ChatMessagesContainer', 'value', 'chatmessages'), createObject('name', 'ChatStore:Cosmos:ChatMemorySourcesContainer', 'value', 'chatmemorysources'), createObject('name', 'ChatStore:Cosmos:ChatParticipantsContainer', 'value', 'chatparticipants'), createObject('name', 'ChatStore:Cosmos:ConnectionString', 'value', if(parameters('deployCosmosDB'), listConnectionStrings(resourceId('Microsoft.DocumentDB/databaseAccounts', toLower(format('cosmos-{0}', variables('uniqueName')))), '2023-04-15').connectionStrings[0].connectionString, '')), createObject('name', 'AzureSpeech:Region', 'value', parameters('location')), createObject('name', 'AzureSpeech:Key', 'value', if(parameters('deploySpeechServices'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('cog-speech-{0}', variables('uniqueName'))), '2022-12-01').key1, '')), createObject('name', 'AllowedOrigins', 'value', '[*]'), createObject('name', 'Kestrel:Endpoints:Https:Url', 'value', 'https://localhost:443'), createObject('name', 'Frontend:AadClientId', 'value', parameters('frontendClientId')), createObject('name', 'Logging:LogLevel:Default', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:CopilotChat.WebApi', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.SemanticKernel', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.AspNetCore.Hosting', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.Hosting.Lifetimel', 'value', 'Warning'), createObject('name', 'Logging:ApplicationInsights:LogLevel:Default', 'value', 'Warning'), createObject('name', 'APPLICATIONINSIGHTS_CONNECTION_STRING', 'value', reference(resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName'))), '2020-02-02').ConnectionString), createObject('name', 'ApplicationInsightsAgent_EXTENSION_VERSION', 'value', '~2'), createObject('name', 'KernelMemory:ContentStorageType', 'value', 'AzureBlobs'), createObject('name', 'KernelMemory:TextGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:OrchestrationType', 'value', 'Distributed'), createObject('name', 'KernelMemory:DataIngestion:DistributedOrchestration:QueueType', 'value', 'AzureQueue'), createObject('name', 'KernelMemory:DataIngestion:EmbeddingGeneratorTypes:0', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:MemoryDbTypes:0', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:MemoryDbType', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:EmbeddingGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:Services:AzureBlobs:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureBlobs:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureBlobs:Container', 'value', 'chatmemory'), createObject('name', 'KernelMemory:Services:AzureQueue:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureQueue:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureAISearch:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureAISearch:Endpoint', 'value', if(equals(parameters('memoryStore'), 'AzureAISearch'), format('https://{0}.search.windows.net', format('acs-{0}', variables('uniqueName'))), '')), createObject('name', 'KernelMemory:Services:AzureAISearch:APIKey', 'value', if(equals(parameters('memoryStore'), 'AzureAISearch'), listAdminKeys(resourceId('Microsoft.Search/searchServices', format('acs-{0}', variables('uniqueName'))), '2022-09-01').primaryKey, '')), createObject('name', 'KernelMemory:Services:Qdrant:Endpoint', 'value', if(equals(parameters('memoryStore'), 'Qdrant'), format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('app-{0}-qdrant', variables('uniqueName'))), '2022-09-01').defaultHostName), '')), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Deployment', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Deployment', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:TextModel', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:OpenAI:EmbeddingModel', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:APIKey', 'value', parameters('aiApiKey')), createObject('name', 'Plugins:0:Name', 'value', 'Klarna Shopping'), createObject('name', 'Plugins:0:ManifestDomain', 'value', 'https://www.klarna.com')), if(parameters('deployWebSearcherPlugin'), createArray(createObject('name', 'Plugins:1:Name', 'value', 'WebSearcher'), createObject('name', 'Plugins:1:ManifestDomain', 'value', format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName'))), '2022-09-01').defaultHostName)), createObject('name', 'Plugins:1:Key', 'value', listkeys(format('{0}/host/default/', resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName')))), '2022-09-01').functionKeys.default)), createArray()))]" }, "dependsOn": [ "[resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName')))]", @@ -377,7 +370,7 @@ "value": "[parameters('aiService')]" }, { - "name": "KernelMemory:ImageOcrType", + "name": "KernelMemory:DataIngestion:ImageOcrType", "value": "AzureFormRecognizer" }, { @@ -393,11 +386,11 @@ "value": "[parameters('aiService')]" }, { - "name": "KernelMemory:DataIngestion:VectorDbTypes:0", + "name": "KernelMemory:DataIngestion:MemoryDbTypes:0", "value": "[parameters('memoryStore')]" }, { - "name": "KernelMemory:Retrieval:VectorDbType", + "name": "KernelMemory:Retrieval:MemoryDbType", "value": "[parameters('memoryStore')]" }, { @@ -425,16 +418,16 @@ "value": "[format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)]" }, { - "name": "KernelMemory:Services:AzureCognitiveSearch:Auth", + "name": "KernelMemory:Services:AzureAISearch:Auth", "value": "ApiKey" }, { - "name": "KernelMemory:Services:AzureCognitiveSearch:Endpoint", - "value": "[if(equals(parameters('memoryStore'), 'AzureCognitiveSearch'), format('https://{0}.search.windows.net', format('acs-{0}', variables('uniqueName'))), '')]" + "name": "KernelMemory:Services:AzureAISearch:Endpoint", + "value": "[if(equals(parameters('memoryStore'), 'AzureAISearch'), format('https://{0}.search.windows.net', format('acs-{0}', variables('uniqueName'))), '')]" }, { - "name": "KernelMemory:Services:AzureCognitiveSearch:APIKey", - "value": "[if(equals(parameters('memoryStore'), 'AzureCognitiveSearch'), listAdminKeys(resourceId('Microsoft.Search/searchServices', format('acs-{0}', variables('uniqueName'))), '2022-09-01').primaryKey, '')]" + "name": "KernelMemory:Services:AzureAISearch:APIKey", + "value": "[if(equals(parameters('memoryStore'), 'AzureAISearch'), listAdminKeys(resourceId('Microsoft.Search/searchServices', format('acs-{0}', variables('uniqueName'))), '2022-09-01').primaryKey, '')]" }, { "name": "KernelMemory:Services:Qdrant:Endpoint", @@ -754,7 +747,7 @@ ] }, { - "condition": "[equals(parameters('memoryStore'), 'AzureCognitiveSearch')]", + "condition": "[equals(parameters('memoryStore'), 'AzureAISearch')]", "type": "Microsoft.Search/searchServices", "apiVersion": "2022-09-01", "name": "[format('acs-{0}', variables('uniqueName'))]", diff --git a/shared/ConfigurationBuilderExtensions.cs b/shared/ConfigurationBuilderExtensions.cs new file mode 100644 index 000000000..e3c5a0b50 --- /dev/null +++ b/shared/ConfigurationBuilderExtensions.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.KernelMemory.Configuration; + +namespace CopilotChat.Shared; + +internal static class ConfigurationBuilderExtensions +{ + // ASP.NET env var + private const string AspnetEnvVar = "ASPNETCORE_ENVIRONMENT"; + + public static void AddKMConfigurationSources( + this IConfigurationBuilder builder, + bool useAppSettingsFiles = true, + bool useEnvVars = true, + bool useSecretManager = true, + string? settingsDirectory = null) + { + // Load env var name, either Development or Production + var env = Environment.GetEnvironmentVariable(AspnetEnvVar) ?? string.Empty; + + // Detect the folder containing configuration files + settingsDirectory ??= Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + ?? Directory.GetCurrentDirectory(); + builder.SetBasePath(settingsDirectory); + + // Add configuration files as sources + if (useAppSettingsFiles) + { + // Add appsettings.json, typically used for default settings, without credentials + var main = Path.Join(settingsDirectory, "appsettings.json"); + if (!File.Exists(main)) + { + throw new ConfigurationException($"appsettings.json not found. Directory: {settingsDirectory}"); + } + + builder.AddJsonFile(main, optional: false); + + // Add appsettings.development.json, used for local overrides and credentials + if (env.Equals("development", StringComparison.OrdinalIgnoreCase)) + { + var f1 = Path.Join(settingsDirectory, "appsettings.development.json"); + var f2 = Path.Join(settingsDirectory, "appsettings.Development.json"); + if (File.Exists(f1)) + { + builder.AddJsonFile(f1, optional: false); + } + else if (File.Exists(f2)) + { + builder.AddJsonFile(f2, optional: false); + } + } + + // Add appsettings.production.json, used for production settings and credentials + if (env.Equals("production", StringComparison.OrdinalIgnoreCase)) + { + var f1 = Path.Join(settingsDirectory, "appsettings.production.json"); + var f2 = Path.Join(settingsDirectory, "appsettings.Production.json"); + if (File.Exists(f1)) + { + builder.AddJsonFile(f1, optional: false); + } + else if (File.Exists(f2)) + { + builder.AddJsonFile(f2, optional: false); + } + } + } + + // Add Secret Manager as source + if (useSecretManager) + { + // GetEntryAssembly method can return null if the library is loaded + // from an unmanaged application, in which case UserSecrets are not supported. + var entryAssembly = Assembly.GetEntryAssembly(); + + // Support for user secrets. Secret Manager doesn't encrypt the stored secrets and + // shouldn't be treated as a trusted store. It's for development purposes only. + // see: https://learn.microsoft.com/aspnet/core/security/app-secrets?#secret-manager + if (entryAssembly != null && env.Equals("development", StringComparison.OrdinalIgnoreCase)) + { + builder.AddUserSecrets(entryAssembly, optional: true); + } + } + + // Add environment variables as source. + // Environment variables can override all the settings provided by the previous sources. + if (useEnvVars) + { + // Support for environment variables overriding the config files + builder.AddEnvironmentVariables(); + } + } +} diff --git a/shared/CopilotChatShared.csproj b/shared/CopilotChatShared.csproj index a70b39e9b..6d3a7bc6e 100644 --- a/shared/CopilotChatShared.csproj +++ b/shared/CopilotChatShared.csproj @@ -9,7 +9,10 @@ - + + + + diff --git a/shared/KernelMemoryBuilderExtensions.cs b/shared/KernelMemoryBuilderExtensions.cs new file mode 100644 index 000000000..1b82ff732 --- /dev/null +++ b/shared/KernelMemoryBuilderExtensions.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Configuration; +using Microsoft.KernelMemory; + +namespace CopilotChat.Shared; + +/// +/// Kernel Memory builder extensions for apps using settings in appsettings.json +/// and using IConfiguration. +/// +public static class KernelMemoryBuilderExtensions +{ + /// + /// Configure the builder using settings stored in the specified directory. + /// If directory is empty, use the current assembly folder + /// + /// KernelMemory builder instance + /// Directory containing appsettings.json (incl. dev/prod) + public static IKernelMemoryBuilder FromAppSettings(this IKernelMemoryBuilder builder, string? settingsDirectory = null) + { + return new ServiceConfiguration(settingsDirectory).PrepareBuilder(builder); + } + + /// + /// Configure the builder using settings from the given KernelMemoryConfig and IConfiguration instances. + /// + /// KernelMemory builder instance + /// KM configuration + /// Dependencies configuration, e.g. queue, embedding, storage, etc. + public static IKernelMemoryBuilder FromMemoryConfiguration( + this IKernelMemoryBuilder builder, + KernelMemoryConfig memoryConfiguration, + IConfiguration servicesConfiguration) + { + return new ServiceConfiguration(servicesConfiguration, memoryConfiguration).PrepareBuilder(builder); + } +} diff --git a/shared/MemoryClientBuilderExtensions.cs b/shared/MemoryClientBuilderExtensions.cs index 27bdba12e..e9d3f215a 100644 --- a/shared/MemoryClientBuilderExtensions.cs +++ b/shared/MemoryClientBuilderExtensions.cs @@ -10,11 +10,11 @@ namespace CopilotChat.Shared; /// public static class MemoryClientBuilderExtensions { - public static KernelMemoryBuilder WithCustomOcr(this KernelMemoryBuilder builder, IConfiguration configuration) + public static IKernelMemoryBuilder WithCustomOcr(this IKernelMemoryBuilder builder, IConfiguration configuration) { var ocrEngine = configuration.CreateCustomOcr(); - if (ocrEngine != null) + if (ocrEngine is not null) { builder.WithCustomImageOcr(ocrEngine); } diff --git a/shared/Ocr/ConfigurationExtensions.cs b/shared/Ocr/ConfigurationExtensions.cs index 095ae53b6..3d477e2c0 100644 --- a/shared/Ocr/ConfigurationExtensions.cs +++ b/shared/Ocr/ConfigurationExtensions.cs @@ -4,7 +4,7 @@ using CopilotChat.Shared.Ocr.Tesseract; using Microsoft.Extensions.Configuration; using Microsoft.KernelMemory.Configuration; -using Microsoft.KernelMemory.DataFormats.Image; +using Microsoft.KernelMemory.DataFormats; namespace CopilotChat.Shared.Ocr; diff --git a/shared/Ocr/Tesseract/TesseractOcrEngine.cs b/shared/Ocr/Tesseract/TesseractOcrEngine.cs index 5d9722292..75881c90c 100644 --- a/shared/Ocr/Tesseract/TesseractOcrEngine.cs +++ b/shared/Ocr/Tesseract/TesseractOcrEngine.cs @@ -3,7 +3,7 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Microsoft.KernelMemory.DataFormats.Image; +using Microsoft.KernelMemory.DataFormats; using Tesseract; namespace CopilotChat.Shared.Ocr.Tesseract; diff --git a/shared/ServiceConfiguration.cs b/shared/ServiceConfiguration.cs new file mode 100644 index 000000000..8f61dde25 --- /dev/null +++ b/shared/ServiceConfiguration.cs @@ -0,0 +1,515 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.AI; +using Microsoft.KernelMemory.Configuration; +using Microsoft.KernelMemory.ContentStorage.DevTools; +using Microsoft.KernelMemory.MemoryStorage; +using Microsoft.KernelMemory.MemoryStorage.DevTools; +using Microsoft.KernelMemory.Pipeline.Queue.DevTools; +using Microsoft.KernelMemory.Postgres; + +namespace CopilotChat.Shared; + +internal sealed class ServiceConfiguration +{ + // Content of appsettings.json, used to access dynamic data under "Services" + private IConfiguration _rawAppSettings; + + // Normalized configuration + private KernelMemoryConfig _memoryConfiguration; + + // appsettings.json root node name + private const string ConfigRoot = "KernelMemory"; + + // ASP.NET env var + private const string AspnetEnvVar = "ASPNETCORE_ENVIRONMENT"; + + // OpenAI env var + private const string OpenAIEnvVar = "OPENAI_API_KEY"; + + public ServiceConfiguration(string? settingsDirectory = null) + : this(ReadAppSettings(settingsDirectory)) + { + } + + public ServiceConfiguration(IConfiguration rawAppSettings) + : this(rawAppSettings, + rawAppSettings.GetSection(ConfigRoot).Get() + ?? throw new ConfigurationException($"Unable to load Kernel Memory settings from the given configuration. " + + $"There should be a '{ConfigRoot}' root node, " + + $"with data mapping to '{nameof(KernelMemoryConfig)}'")) + { + } + + public ServiceConfiguration( + IConfiguration rawAppSettings, + KernelMemoryConfig memoryConfiguration) + { + this._rawAppSettings = rawAppSettings ?? throw new ConfigurationException("The given app settings configuration is NULL"); + this._memoryConfiguration = memoryConfiguration ?? throw new ConfigurationException("The given memory configuration is NULL"); + + if (!this.MinimumConfigurationIsAvailable(false)) { this.SetupForOpenAI(); } + + this.MinimumConfigurationIsAvailable(true); + } + + public IKernelMemoryBuilder PrepareBuilder(IKernelMemoryBuilder builder) + { + return this.BuildUsingConfiguration(builder); + } + + private IKernelMemoryBuilder BuildUsingConfiguration(IKernelMemoryBuilder builder) + { + if (this._memoryConfiguration == null) + { + throw new ConfigurationException("The given memory configuration is NULL"); + } + + if (this._rawAppSettings == null) + { + throw new ConfigurationException("The given app settings configuration is NULL"); + } + + // Required by ctors expecting KernelMemoryConfig via DI + builder.AddSingleton(this._memoryConfiguration); + + this.ConfigureMimeTypeDetectionDependency(builder); + + this.ConfigureTextPartitioning(builder); + + this.ConfigureQueueDependency(builder); + + this.ConfigureStorageDependency(builder); + + // The ingestion embedding generators is a list of generators that the "gen_embeddings" handler uses, + // to generate embeddings for each partition. While it's possible to use multiple generators (e.g. to compare embedding quality) + // only one generator is used when searching by similarity, and the generator used for search is not in this list. + // - config.DataIngestion.EmbeddingGeneratorTypes => list of generators, embeddings to generate and store in memory DB + // - config.Retrieval.EmbeddingGeneratorType => one embedding generator, used to search, and usually injected into Memory DB constructor + + this.ConfigureIngestionEmbeddingGenerators(builder); + + this.ConfigureSearchClient(builder); + + this.ConfigureRetrievalEmbeddingGenerator(builder); + + // The ingestion Memory DBs is a list of DBs where handlers write records to. While it's possible + // to write to multiple DBs, e.g. for replication purpose, there is only one Memory DB used to + // read/search, and it doesn't come from this list. See "config.Retrieval.MemoryDbType". + // Note: use the aux service collection to avoid mixing ingestion and retrieval dependencies. + + this.ConfigureIngestionMemoryDb(builder); + + this.ConfigureRetrievalMemoryDb(builder); + + this.ConfigureTextGenerator(builder); + + this.ConfigureImageOCR(builder); + + return builder; + } + + private static IConfiguration ReadAppSettings(string? settingsDirectory) + { + var builder = new ConfigurationBuilder(); + builder.AddKMConfigurationSources(settingsDirectory: settingsDirectory); + return builder.Build(); + } + + private void ConfigureQueueDependency(IKernelMemoryBuilder builder) + { + if (string.Equals(this._memoryConfiguration.DataIngestion.OrchestrationType, "Distributed", StringComparison.OrdinalIgnoreCase)) + { + switch (this._memoryConfiguration.DataIngestion.DistributedOrchestration.QueueType) + { + case string y1 when y1.Equals("AzureQueue", StringComparison.OrdinalIgnoreCase): + case string y2 when y2.Equals("AzureQueues", StringComparison.OrdinalIgnoreCase): + // Check 2 keys for backward compatibility + builder.Services.AddAzureQueuesOrchestration(this.GetServiceConfig("AzureQueues") + ?? this.GetServiceConfig("AzureQueue")); + break; + + case string y when y.Equals("RabbitMQ", StringComparison.OrdinalIgnoreCase): + // Check 2 keys for backward compatibility + builder.Services.AddRabbitMQOrchestration(this.GetServiceConfig("RabbitMQ") + ?? this.GetServiceConfig("RabbitMq")); + break; + + case string y when y.Equals("SimpleQueues", StringComparison.OrdinalIgnoreCase): + builder.Services.AddSimpleQueues(this.GetServiceConfig("SimpleQueues")); + break; + + default: + // NOOP - allow custom implementations, via WithCustomIngestionQueueClientFactory() + break; + } + } + } + + private void ConfigureStorageDependency(IKernelMemoryBuilder builder) + { + switch (this._memoryConfiguration.ContentStorageType) + { + case string x1 when x1.Equals("AzureBlob", StringComparison.OrdinalIgnoreCase): + case string x2 when x2.Equals("AzureBlobs", StringComparison.OrdinalIgnoreCase): + // Check 2 keys for backward compatibility + builder.Services.AddAzureBlobsAsContentStorage(this.GetServiceConfig("AzureBlobs") + ?? this.GetServiceConfig("AzureBlob")); + break; + + case string x when x.Equals("SimpleFileStorage", StringComparison.OrdinalIgnoreCase): + builder.Services.AddSimpleFileStorageAsContentStorage(this.GetServiceConfig("SimpleFileStorage")); + break; + + default: + // NOOP - allow custom implementations, via WithCustomStorage() + break; + } + } + + private void ConfigureTextPartitioning(IKernelMemoryBuilder builder) + { + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + if (this._memoryConfiguration.DataIngestion.TextPartitioning != null) + { + this._memoryConfiguration.DataIngestion.TextPartitioning.Validate(); + builder.WithCustomTextPartitioningOptions(this._memoryConfiguration.DataIngestion.TextPartitioning); + } + } + + private void ConfigureMimeTypeDetectionDependency(IKernelMemoryBuilder builder) + { + builder.WithDefaultMimeTypeDetection(); + } + + private void ConfigureIngestionEmbeddingGenerators(IKernelMemoryBuilder builder) + { + // Note: using multiple embeddings is not fully supported yet and could cause write errors or incorrect search results + if (this._memoryConfiguration.DataIngestion.EmbeddingGeneratorTypes.Count > 1) + { + throw new NotSupportedException("Using multiple embedding generators is currently unsupported. " + + "You may contact the team if this feature is required, or workaround this exception " + + "using KernelMemoryBuilder methods explicitly."); + } + + foreach (var type in this._memoryConfiguration.DataIngestion.EmbeddingGeneratorTypes) + { + switch (type) + { + case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase): + case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddAzureOpenAIEmbeddingGeneration(this.GetServiceConfig("AzureOpenAIEmbedding"))); + builder.AddIngestionEmbeddingGenerator(instance); + break; + } + + case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddOpenAITextEmbeddingGeneration(this.GetServiceConfig("OpenAI"))); + builder.AddIngestionEmbeddingGenerator(instance); + break; + } + + default: + // NOOP - allow custom implementations, via WithCustomEmbeddingGeneration() + break; + } + } + } + + private void ConfigureIngestionMemoryDb(IKernelMemoryBuilder builder) + { + foreach (var type in this._memoryConfiguration.DataIngestion.MemoryDbTypes) + { + switch (type) + { + default: + throw new ConfigurationException( + $"Unknown Memory DB option '{type}'. " + + "To use a custom Memory DB, set the configuration value to an empty string, " + + "and inject the custom implementation using `IKernelMemoryBuilder.WithCustomMemoryDb(...)`"); + + case "": + // NOOP - allow custom implementations, via WithCustomMemoryDb() + break; + + case string x when x.Equals("AzureAISearch", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddAzureAISearchAsMemoryDb(this.GetServiceConfig("AzureAISearch")) + ); + builder.AddIngestionMemoryDb(instance); + break; + } + + case string x when x.Equals("Qdrant", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddQdrantAsMemoryDb(this.GetServiceConfig("Qdrant")) + ); + builder.AddIngestionMemoryDb(instance); + break; + } + + case string x when x.Equals("Postgres", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddPostgresAsMemoryDb(this.GetServiceConfig("Postgres")) + ); + builder.AddIngestionMemoryDb(instance); + break; + } + + case string x when x.Equals("Redis", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddRedisAsMemoryDb(this.GetServiceConfig("Redis")) + ); + builder.AddIngestionMemoryDb(instance); + break; + } + + case string x when x.Equals("SimpleVectorDb", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddSimpleVectorDbAsMemoryDb(this.GetServiceConfig("SimpleVectorDb")) + ); + builder.AddIngestionMemoryDb(instance); + break; + } + + case string x when x.Equals("SimpleTextDb", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddSimpleTextDbAsMemoryDb(this.GetServiceConfig("SimpleTextDb")) + ); + builder.AddIngestionMemoryDb(instance); + break; + } + } + } + } + + private void ConfigureSearchClient(IKernelMemoryBuilder builder) + { + // Search settings + builder.WithSearchClientConfig(this._memoryConfiguration.Retrieval.SearchClient); + } + + private void ConfigureRetrievalEmbeddingGenerator(IKernelMemoryBuilder builder) + { + // Retrieval embeddings - ITextEmbeddingGeneration interface + switch (this._memoryConfiguration.Retrieval.EmbeddingGeneratorType) + { + case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase): + case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase): + builder.Services.AddAzureOpenAIEmbeddingGeneration(this.GetServiceConfig("AzureOpenAIEmbedding")); + break; + + case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): + builder.Services.AddOpenAITextEmbeddingGeneration(this.GetServiceConfig("OpenAI")); + break; + + default: + // NOOP - allow custom implementations, via WithCustomEmbeddingGeneration() + break; + } + } + + private void ConfigureRetrievalMemoryDb(IKernelMemoryBuilder builder) + { + // Retrieval Memory DB - IMemoryDb interface + switch (this._memoryConfiguration.Retrieval.MemoryDbType) + { + case string x when x.Equals("AzureAISearch", StringComparison.OrdinalIgnoreCase): + builder.Services.AddAzureAISearchAsMemoryDb(this.GetServiceConfig("AzureAISearch")); + break; + + case string x when x.Equals("Qdrant", StringComparison.OrdinalIgnoreCase): + builder.Services.AddQdrantAsMemoryDb(this.GetServiceConfig("Qdrant")); + break; + + case string x when x.Equals("Postgres", StringComparison.OrdinalIgnoreCase): + builder.Services.AddPostgresAsMemoryDb(this.GetServiceConfig("Postgres")); + break; + + case string x when x.Equals("Redis", StringComparison.OrdinalIgnoreCase): + builder.Services.AddRedisAsMemoryDb(this.GetServiceConfig("Redis")); + break; + + case string x when x.Equals("SimpleVectorDb", StringComparison.OrdinalIgnoreCase): + builder.Services.AddSimpleVectorDbAsMemoryDb(this.GetServiceConfig("SimpleVectorDb")); + break; + + case string x when x.Equals("SimpleTextDb", StringComparison.OrdinalIgnoreCase): + builder.Services.AddSimpleTextDbAsMemoryDb(this.GetServiceConfig("SimpleTextDb")); + break; + + default: + // NOOP - allow custom implementations, via WithCustomMemoryDb() + break; + } + } + + private void ConfigureTextGenerator(IKernelMemoryBuilder builder) + { + // Text generation + switch (this._memoryConfiguration.TextGeneratorType) + { + case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase): + case string y when y.Equals("AzureOpenAIText", StringComparison.OrdinalIgnoreCase): + builder.Services.AddAzureOpenAITextGeneration(this.GetServiceConfig("AzureOpenAIText")); + break; + + case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): + builder.Services.AddOpenAITextGeneration(this.GetServiceConfig("OpenAI")); + break; + + default: + // NOOP - allow custom implementations, via WithCustomTextGeneration() + break; + } + } + + private void ConfigureImageOCR(IKernelMemoryBuilder builder) + { + // Image OCR + switch (this._memoryConfiguration.DataIngestion.ImageOcrType) + { + case string y when string.IsNullOrWhiteSpace(y): + case string x when x.Equals("None", StringComparison.OrdinalIgnoreCase): + break; + + case string x when x.Equals("AzureAIDocIntel", StringComparison.OrdinalIgnoreCase): + builder.Services.AddAzureAIDocIntel(this.GetServiceConfig("AzureAIDocIntel")); + break; + + default: + // NOOP - allow custom implementations, via WithCustomImageOCR() + break; + } + } + + /// + /// Check the configuration for minimum requirements + /// + /// Whether to throw or return false when the config is incomplete + /// Whether the configuration is valid + private bool MinimumConfigurationIsAvailable(bool throwOnError) + { + // Check if text generation settings + if (string.IsNullOrEmpty(this._memoryConfiguration.TextGeneratorType)) + { + if (!throwOnError) { return false; } + + throw new ConfigurationException("Text generation (TextGeneratorType) is not configured in Kernel Memory."); + } + + // Check embedding generation ingestion settings + if (this._memoryConfiguration.DataIngestion.EmbeddingGenerationEnabled) + { + if (this._memoryConfiguration.DataIngestion.EmbeddingGeneratorTypes.Count == 0) + { + if (!throwOnError) { return false; } + + throw new ConfigurationException("Data ingestion embedding generation (DataIngestion.EmbeddingGeneratorTypes) is not configured in Kernel Memory."); + } + } + + // Check embedding generation retrieval settings + if (string.IsNullOrEmpty(this._memoryConfiguration.Retrieval.EmbeddingGeneratorType)) + { + if (!throwOnError) { return false; } + + throw new ConfigurationException("Retrieval embedding generation (Retrieval.EmbeddingGeneratorType) is not configured in Kernel Memory."); + } + + return true; + } + + /// + /// Rewrite configuration using OpenAI, if possible. + /// + private void SetupForOpenAI() + { + string openAIKey = Environment.GetEnvironmentVariable(OpenAIEnvVar)?.Trim() ?? string.Empty; + if (string.IsNullOrEmpty(openAIKey)) + { + return; + } + + var inMemoryConfig = new Dictionary + { + { $"{ConfigRoot}:Services:OpenAI:APIKey", openAIKey }, + { $"{ConfigRoot}:TextGeneratorType", "OpenAI" }, + { $"{ConfigRoot}:DataIngestion:EmbeddingGeneratorTypes:0", "OpenAI" }, + { $"{ConfigRoot}:Retrieval:EmbeddingGeneratorType", "OpenAI" } + }; + + var newAppSettings = new ConfigurationBuilder(); + newAppSettings.AddConfiguration(this._rawAppSettings); + newAppSettings.AddInMemoryCollection(inMemoryConfig); + + this._rawAppSettings = newAppSettings.Build(); + this._memoryConfiguration = this._rawAppSettings.GetSection(ConfigRoot).Get()!; + } + + /// + /// Get an instance of T, using dependencies available in the builder, + /// except for existing service descriptors for T. Replace/Use the + /// given action to define T's implementation. + /// Return an instance of T built using the definition provided by + /// the action. + /// + /// KM builder + /// Action used to configure the service collection + /// Target type/interface + private T GetServiceInstance(IKernelMemoryBuilder builder, Action addCustomService) + { + // Clone the list of service descriptors, skipping T descriptor + IServiceCollection services = new ServiceCollection(); + foreach (ServiceDescriptor d in builder.Services) + { + if (d.ServiceType == typeof(T)) { continue; } + + services.Add(d); + } + + // Add the custom T descriptor + addCustomService.Invoke(services); + + // Build and return an instance of T, as defined by `addCustomService` + return services.BuildServiceProvider().GetService() + ?? throw new ConfigurationException($"Unable to build {nameof(T)}"); + } + + /// + /// Read a dependency configuration from IConfiguration + /// Data is usually retrieved from KernelMemory:Services:{serviceName}, e.g. when using appsettings.json + /// { + /// "KernelMemory": { + /// "Services": { + /// "{serviceName}": { + /// ... + /// ... + /// } + /// } + /// } + /// } + /// + /// Name of the dependency + /// Type of configuration to return + /// Configuration instance, settings for the dependency specified + private T GetServiceConfig(string serviceName) + { + return this._memoryConfiguration.GetServiceConfig(this._rawAppSettings, serviceName); + } +} diff --git a/tools/importdocument/ImportDocument.csproj b/tools/importdocument/ImportDocument.csproj index 18a012fb3..243fc414d 100644 --- a/tools/importdocument/ImportDocument.csproj +++ b/tools/importdocument/ImportDocument.csproj @@ -16,9 +16,9 @@ - + - + diff --git a/webapi/Controllers/ChatController.cs b/webapi/Controllers/ChatController.cs index 22de83e09..0d7116d2c 100644 --- a/webapi/Controllers/ChatController.cs +++ b/webapi/Controllers/ChatController.cs @@ -7,6 +7,7 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Reflection; +using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; @@ -28,14 +29,10 @@ using Microsoft.Extensions.Options; using Microsoft.Graph; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; -using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; -using Microsoft.SemanticKernel.Functions.OpenAPI.OpenAI; -using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Plugins.MsGraph; using Microsoft.SemanticKernel.Plugins.MsGraph.Connectors; using Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Client; +using Microsoft.SemanticKernel.Plugins.OpenApi; namespace CopilotChat.WebApi.Controllers; @@ -50,12 +47,10 @@ public class ChatController : ControllerBase, IDisposable private readonly List _disposables; private readonly ITelemetryService _telemetryService; private readonly ServiceOptions _serviceOptions; - private readonly PlannerOptions _plannerOptions; private readonly IDictionary _plugins; private const string ChatPluginName = nameof(ChatPlugin); private const string ChatFunctionName = "Chat"; - private const string ProcessPlanFunctionName = "ProcessPlan"; private const string GeneratingResponseClientCall = "ReceiveBotResponseStatus"; public ChatController( @@ -63,7 +58,6 @@ public ChatController( IHttpClientFactory httpClientFactory, ITelemetryService telemetryService, IOptions serviceOptions, - IOptions plannerOptions, IDictionary plugins) { this._logger = logger; @@ -71,7 +65,6 @@ public ChatController( this._telemetryService = telemetryService; this._disposables = new List(); this._serviceOptions = serviceOptions.Value; - this._plannerOptions = plannerOptions.Value; this._plugins = plugins; } @@ -80,8 +73,6 @@ public ChatController( /// /// Semantic kernel obtained through dependency injection. /// Message Hub that performs the real time relay service. - /// Planner to use to create function sequences. - /// Converter to use for converting Asks. /// Repository of chat sessions. /// Repository of chat participants. /// Auth info for the current request. @@ -96,9 +87,8 @@ public ChatController( [ProducesResponseType(StatusCodes.Status404NotFound)] [ProducesResponseType(StatusCodes.Status504GatewayTimeout)] public async Task ChatAsync( - [FromServices] IKernel kernel, + [FromServices] Kernel kernel, [FromServices] IHubContext messageRelayHubContext, - [FromServices] CopilotChatPlanner planner, [FromServices] ChatSessionRepository chatSessionRepository, [FromServices] ChatParticipantRepository chatParticipantRepository, [FromServices] IAuthInfo authInfo, @@ -107,105 +97,35 @@ public async Task ChatAsync( { this._logger.LogDebug("Chat message received."); - return await this.HandleRequest(ChatFunctionName, kernel, messageRelayHubContext, planner, chatSessionRepository, chatParticipantRepository, authInfo, ask, chatId.ToString()); - } - - /// - /// Invokes the chat function to process and/or execute plan. - /// - /// Semantic kernel obtained through dependency injection. - /// Message Hub that performs the real time relay service. - /// Planner to use to create function sequences. - /// Converter to use for converting Asks. - /// Repository of chat sessions. - /// Repository of chat participants. - /// Auth info for the current request. - /// Prompt along with its parameters. - /// Chat ID. - /// Results containing the response from the model. - [Route("chats/{chatId:guid}/plan")] - [HttpPost] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [ProducesResponseType(StatusCodes.Status403Forbidden)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - [ProducesResponseType(StatusCodes.Status504GatewayTimeout)] - public async Task ProcessPlanAsync( - [FromServices] IKernel kernel, - [FromServices] IHubContext messageRelayHubContext, - [FromServices] CopilotChatPlanner planner, - [FromServices] ChatSessionRepository chatSessionRepository, - [FromServices] ChatParticipantRepository chatParticipantRepository, - [FromServices] IAuthInfo authInfo, - [FromBody] ExecutePlanParameters ask, - [FromRoute] Guid chatId) - { - this._logger.LogDebug("plan request received."); + string chatIdString = chatId.ToString(); - return await this.HandleRequest(ProcessPlanFunctionName, kernel, messageRelayHubContext, planner, chatSessionRepository, chatParticipantRepository, authInfo, ask, chatId.ToString()); - } - - /// - /// Invokes given function of ChatPlugin. - /// - /// Name of the ChatPlugin function to invoke. - /// Semantic kernel obtained through dependency injection. - /// Message Hub that performs the real time relay service. - /// Planner to use to create function sequences. - /// Converter to use for converting Asks. - /// Repository of chat sessions. - /// Repository of chat participants. - /// Auth info for the current request. - /// Prompt along with its parameters. - /// - /// Results containing the response from the model. - private async Task HandleRequest( - string functionName, - IKernel kernel, - IHubContext messageRelayHubContext, - CopilotChatPlanner planner, - ChatSessionRepository chatSessionRepository, - ChatParticipantRepository chatParticipantRepository, - IAuthInfo authInfo, - Ask ask, - string chatId) - { // Put ask's variables in the context we will use. - var contextVariables = GetContextVariables(ask, authInfo, chatId); + var contextVariables = GetContextVariables(ask, authInfo, chatIdString); // Verify that the chat exists and that the user has access to it. ChatSession? chat = null; - if (!(await chatSessionRepository.TryFindByIdAsync(chatId, callback: c => chat = c))) + if (!(await chatSessionRepository.TryFindByIdAsync(chatIdString, callback: c => chat = c))) { return this.NotFound("Failed to find chat session for the chatId specified in variables."); } - if (!(await chatParticipantRepository.IsUserInChatAsync(authInfo.UserId, chatId))) + if (!(await chatParticipantRepository.IsUserInChatAsync(authInfo.UserId, chatIdString))) { return this.Forbid("User does not have access to the chatId specified in variables."); } // Register plugins that have been enabled var openApiPluginAuthHeaders = this.GetPluginAuthHeaders(this.HttpContext.Request.Headers); - await this.RegisterPlannerFunctionsAsync(planner, openApiPluginAuthHeaders, contextVariables); + await this.RegisterFunctionsAsync(kernel, openApiPluginAuthHeaders, contextVariables); // Register hosted plugins that have been enabled - await this.RegisterPlannerHostedFunctionsUsedAsync(planner, chat!.EnabledPlugins); + await this.RegisterHostedFunctionsAsync(kernel, chat!.EnabledPlugins); // Get the function to invoke - ISKFunction? function = null; - try - { - function = kernel.Functions.GetFunction(ChatPluginName, functionName); - } - catch (SKException ex) - { - this._logger.LogError("Failed to find {PluginName}/{FunctionName} on server: {Exception}", ChatPluginName, functionName, ex); - return this.NotFound($"Failed to find {ChatPluginName}/{functionName} on server"); - } + KernelFunction? chatFunction = kernel.Plugins.GetFunction(ChatPluginName, ChatFunctionName); // Run the function. - KernelResult? result = null; + FunctionResult? result = null; try { using CancellationTokenSource? cts = this._serviceOptions.TimeoutLimitInS is not null @@ -213,30 +133,31 @@ private async Task HandleRequest( ? new CancellationTokenSource(TimeSpan.FromSeconds((double)this._serviceOptions.TimeoutLimitInS)) : null; - result = await kernel.RunAsync(function!, contextVariables, cts?.Token ?? default); - this._telemetryService.TrackPluginFunction(ChatPluginName, functionName, true); + result = await kernel.InvokeAsync(chatFunction!, contextVariables, cts?.Token ?? default); + this._telemetryService.TrackPluginFunction(ChatPluginName, ChatFunctionName, true); } catch (Exception ex) { if (ex is OperationCanceledException || ex.InnerException is OperationCanceledException) { // Log the timeout and return a 504 response - this._logger.LogError("The {FunctionName} operation timed out.", functionName); - return this.StatusCode(StatusCodes.Status504GatewayTimeout, $"The chat {functionName} timed out."); + this._logger.LogError("The {FunctionName} operation timed out.", ChatFunctionName); + return this.StatusCode(StatusCodes.Status504GatewayTimeout, $"The chat {ChatFunctionName} timed out."); } - this._telemetryService.TrackPluginFunction(ChatPluginName, functionName, false); - throw ex; + this._telemetryService.TrackPluginFunction(ChatPluginName, ChatFunctionName, false); + + throw; } AskResult chatAskResult = new() { - Value = result.GetValue() ?? string.Empty, - Variables = contextVariables.Select(v => new KeyValuePair(v.Key, v.Value)) + Value = result.ToString() ?? string.Empty, + Variables = contextVariables.Select(v => new KeyValuePair(v.Key, v.Value)) }; // Broadcast AskResult to all users - await messageRelayHubContext.Clients.Group(chatId).SendAsync(GeneratingResponseClientCall, chatId, null); + await messageRelayHubContext.Clients.Group(chatIdString).SendAsync(GeneratingResponseClientCall, chatIdString, null); return this.Ok(chatAskResult); } @@ -267,18 +188,18 @@ private Dictionary GetPluginAuthHeaders(IHeaderDictionary header } /// - /// Register functions with the planner's kernel. + /// Register functions with the kernel. /// - private async Task RegisterPlannerFunctionsAsync(CopilotChatPlanner planner, Dictionary authHeaders, ContextVariables variables) + private async Task RegisterFunctionsAsync(Kernel kernel, Dictionary authHeaders, KernelArguments variables) { - // Register authenticated functions with the planner's kernel only if the request includes an auth header for the plugin. + // Register authenticated functions with the kernel only if the request includes an auth header for the plugin. // GitHub if (authHeaders.TryGetValue("GITHUB", out string? GithubAuthHeader)) { this._logger.LogInformation("Enabling GitHub plugin."); BearerAuthenticationProvider authenticationProvider = new(() => Task.FromResult(GithubAuthHeader)); - await planner.Kernel.ImportOpenApiPluginFunctionsAsync( + await kernel.ImportPluginFromOpenApiAsync( pluginName: "GitHubPlugin", filePath: Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "Plugins", "OpenApi/GitHubPlugin/openapi.json"), new OpenApiFunctionExecutionParameters @@ -292,15 +213,15 @@ await planner.Kernel.ImportOpenApiPluginFunctionsAsync( { this._logger.LogInformation("Registering Jira plugin"); var authenticationProvider = new BasicAuthenticationProvider(() => { return Task.FromResult(JiraAuthHeader); }); - var hasServerUrlOverride = variables.TryGetValue("jira-server-url", out string? serverUrlOverride); + var hasServerUrlOverride = variables.TryGetValue("jira-server-url", out object? serverUrlOverride); - await planner.Kernel.ImportOpenApiPluginFunctionsAsync( + await kernel.ImportPluginFromOpenApiAsync( pluginName: "JiraPlugin", filePath: Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "Plugins", "OpenApi/JiraPlugin/openapi.json"), new OpenApiFunctionExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync, - ServerUrlOverride = hasServerUrlOverride ? new Uri(serverUrlOverride!) : null, + ServerUrlOverride = hasServerUrlOverride ? new Uri(serverUrlOverride!.ToString()!) : null, }); } @@ -309,16 +230,16 @@ await planner.Kernel.ImportOpenApiPluginFunctionsAsync( { this._logger.LogInformation("Enabling Microsoft Graph plugin(s)."); BearerAuthenticationProvider authenticationProvider = new(() => Task.FromResult(GraphAuthHeader)); - GraphServiceClient graphServiceClient = this.CreateGraphServiceClient(authenticationProvider.AuthenticateRequestAsync); + GraphServiceClient graphServiceClient = this.CreateGraphServiceClient(authenticationProvider.GraphClientAuthenticateRequestAsync); - planner.Kernel.ImportFunctions(new TaskListPlugin(new MicrosoftToDoConnector(graphServiceClient)), "todo"); - planner.Kernel.ImportFunctions(new CalendarPlugin(new OutlookCalendarConnector(graphServiceClient)), "calendar"); - planner.Kernel.ImportFunctions(new EmailPlugin(new OutlookMailConnector(graphServiceClient)), "email"); + kernel.ImportPluginFromObject(new TaskListPlugin(new MicrosoftToDoConnector(graphServiceClient)), "todo"); + kernel.ImportPluginFromObject(new CalendarPlugin(new OutlookCalendarConnector(graphServiceClient)), "calendar"); + kernel.ImportPluginFromObject(new EmailPlugin(new OutlookMailConnector(graphServiceClient)), "email"); } - if (variables.TryGetValue("customPlugins", out string? customPluginsString)) + if (variables.TryGetValue("customPlugins", out object? customPluginsString)) { - CustomPlugin[]? customPlugins = JsonSerializer.Deserialize(customPluginsString); + CustomPlugin[]? customPlugins = JsonSerializer.Deserialize(customPluginsString!.ToString()!); if (customPlugins != null) { @@ -326,24 +247,24 @@ await planner.Kernel.ImportOpenApiPluginFunctionsAsync( { if (authHeaders.TryGetValue(plugin.AuthHeaderTag.ToUpperInvariant(), out string? PluginAuthValue)) { - // Register the ChatGPT plugin with the planner's kernel. + // Register the ChatGPT plugin with the kernel. this._logger.LogInformation("Enabling {0} plugin.", plugin.NameForHuman); // TODO: [Issue #44] Support other forms of auth. Currently, we only support user PAT or no auth. var requiresAuth = !plugin.AuthType.Equals("none", StringComparison.OrdinalIgnoreCase); - OpenAIAuthenticateRequestAsyncCallback authCallback = (request, _, _) => + Task authCallback(HttpRequestMessage request, string _, OpenAIAuthenticationConfig __, CancellationToken ___ = default) { request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", PluginAuthValue); return Task.CompletedTask; - }; + } - await planner.Kernel.ImportOpenAIPluginFunctionsAsync( + await kernel.ImportPluginFromOpenAIAsync( $"{plugin.NameForModel}Plugin", PluginUtils.GetPluginManifestUri(plugin.ManifestDomain), new OpenAIFunctionExecutionParameters { - HttpClient = this._httpClientFactory.CreateClient("Plugin"), + HttpClient = this._httpClientFactory.CreateClient(), IgnoreNonCompliantErrors = true, AuthCallback = requiresAuth ? authCallback : null }); @@ -377,7 +298,7 @@ private GraphServiceClient CreateGraphServiceClient(AuthenticateRequestAsyncDele return graphServiceClient; } - private async Task RegisterPlannerHostedFunctionsUsedAsync(CopilotChatPlanner planner, HashSet enabledPlugins) + private async Task RegisterHostedFunctionsAsync(Kernel kernel, HashSet enabledPlugins) { foreach (string enabledPlugin in enabledPlugins) { @@ -385,20 +306,20 @@ private async Task RegisterPlannerHostedFunctionsUsedAsync(CopilotChatPlanner pl { this._logger.LogDebug("Enabling hosted plugin {0}.", plugin.Name); - OpenAIAuthenticateRequestAsyncCallback authCallback = (request, _, _) => + Task authCallback(HttpRequestMessage request, string _, OpenAIAuthenticationConfig __, CancellationToken ___ = default) { request.Headers.Add("X-Functions-Key", plugin.Key); return Task.CompletedTask; - }; + } - // Register the ChatGPT plugin with the planner's kernel. - await planner.Kernel.ImportOpenAIPluginFunctionsAsync( + // Register the ChatGPT plugin with the kernel. + await kernel.ImportPluginFromOpenAIAsync( PluginUtils.SanitizePluginName(plugin.Name), PluginUtils.GetPluginManifestUri(plugin.ManifestDomain), new OpenAIFunctionExecutionParameters { - HttpClient = this._httpClientFactory.CreateClient("Plugin"), + HttpClient = this._httpClientFactory.CreateClient(), IgnoreNonCompliantErrors = true, AuthCallback = authCallback }); @@ -412,21 +333,23 @@ await planner.Kernel.ImportOpenAIPluginFunctionsAsync( return; } - private static ContextVariables GetContextVariables(Ask ask, IAuthInfo authInfo, string chatId) + private static KernelArguments GetContextVariables(Ask ask, IAuthInfo authInfo, string chatId) { const string UserIdKey = "userId"; const string UserNameKey = "userName"; const string ChatIdKey = "chatId"; + const string MessageKey = "message"; - var contextVariables = new ContextVariables(ask.Input); + var contextVariables = new KernelArguments(); foreach (var variable in ask.Variables) { - contextVariables.Set(variable.Key, variable.Value); + contextVariables[variable.Key] = variable.Value; } - contextVariables.Set(UserIdKey, authInfo.UserId); - contextVariables.Set(UserNameKey, authInfo.Name); - contextVariables.Set(ChatIdKey, chatId); + contextVariables[UserIdKey] = authInfo.UserId; + contextVariables[UserNameKey] = authInfo.Name; + contextVariables[ChatIdKey] = chatId; + contextVariables[MessageKey] = ask.Input; return contextVariables; } @@ -453,3 +376,79 @@ public void Dispose() GC.SuppressFinalize(this); } } + +/// +/// Retrieves authentication content (e.g. username/password, API key) via the provided delegate and +/// applies it to HTTP requests using the "basic" authentication scheme. +/// +public class BasicAuthenticationProvider +{ + private readonly Func> _credentialsDelegate; + + /// + /// Creates an instance of the class. + /// + /// Delegate for retrieving credentials. + public BasicAuthenticationProvider(Func> credentialsDelegate) + { + this._credentialsDelegate = credentialsDelegate; + } + + /// + /// Applies the authentication content to the provided HTTP request message. + /// + /// The HTTP request message. + /// The cancellation token. + public async Task AuthenticateRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) + { + // Base64 encode + string encodedContent = Convert.ToBase64String(Encoding.UTF8.GetBytes(await this._credentialsDelegate().ConfigureAwait(false))); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", encodedContent); + } +} + +/// +/// Retrieves a token via the provided delegate and applies it to HTTP requests using the +/// "bearer" authentication scheme. +/// +public class BearerAuthenticationProvider +{ + private readonly Func> _bearerTokenDelegate; + + /// + /// Creates an instance of the class. + /// + /// Delegate to retrieve the bearer token. + public BearerAuthenticationProvider(Func> bearerTokenDelegate) + { + this._bearerTokenDelegate = bearerTokenDelegate; + } + + /// + /// Applies the token to the provided HTTP request message. + /// + /// The HTTP request message. + public async Task AuthenticateRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) + { + var token = await this._bearerTokenDelegate().ConfigureAwait(false); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + } + + /// + /// Applies the token to the provided HTTP request message. + /// + /// The HTTP request message. + public async Task GraphClientAuthenticateRequestAsync(HttpRequestMessage request) + { + await this.AuthenticateRequestAsync(request); + } + + /// + /// Applies the token to the provided HTTP request message. + /// + /// The HTTP request message. + public async Task OpenAIAuthenticateRequestAsync(HttpRequestMessage request, string pluginName, OpenAIAuthenticationConfig openAIAuthConfig, CancellationToken cancellationToken = default) + { + await this.AuthenticateRequestAsync(request, cancellationToken); + } +} diff --git a/webapi/Controllers/PluginController.cs b/webapi/Controllers/PluginController.cs index 9c5d3a802..6a795fc31 100644 --- a/webapi/Controllers/PluginController.cs +++ b/webapi/Controllers/PluginController.cs @@ -15,7 +15,6 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; namespace CopilotChat.WebApi.Controllers; @@ -55,9 +54,9 @@ public async Task GetPluginManifest([FromQuery] Uri manifestDomai { using var request = new HttpRequestMessage(HttpMethod.Get, PluginUtils.GetPluginManifestUri(manifestDomain)); // Need to set the user agent to avoid 403s from some sites. - request.Headers.Add("User-Agent", Telemetry.HttpUserAgent); + request.Headers.Add("User-Agent", "Semantic-Kernel"); - using HttpClient client = this._httpClientFactory.CreateClient("Plugin"); + using HttpClient client = this._httpClientFactory.CreateClient(); var response = await client.SendAsync(request); if (!response.IsSuccessStatusCode) { diff --git a/webapi/CopilotChatWebApi.csproj b/webapi/CopilotChatWebApi.csproj index ead1836e4..1bda3fda9 100644 --- a/webapi/CopilotChatWebApi.csproj +++ b/webapi/CopilotChatWebApi.csproj @@ -1,4 +1,4 @@ - + CopilotChat.WebApi net6.0 @@ -7,6 +7,7 @@ enable disable 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 + SKEXP0003,SKEXP0011,SKEXP0021,SKEXP0026,SKEXP0042,SKEXP0050,SKEXP0052,SKEXP0053,SKEXP0060 @@ -17,23 +18,22 @@ - + - + - - - - - - - - - - - - - + + + + + + + + + + + + @@ -57,17 +57,17 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/webapi/Extensions/ISemanticMemoryClientExtensions.cs b/webapi/Extensions/ISemanticMemoryClientExtensions.cs index 646505a21..59d3d9cbb 100644 --- a/webapi/Extensions/ISemanticMemoryClientExtensions.cs +++ b/webapi/Extensions/ISemanticMemoryClientExtensions.cs @@ -21,7 +21,7 @@ namespace CopilotChat.WebApi.Extensions; /// internal static class ISemanticMemoryClientExtensions { - private static readonly List pipelineSteps = new() { "extract", "partition", "gen_embeddings", "save_embeddings" }; + private static readonly List pipelineSteps = new() { "extract", "partition", "gen_embeddings", "save_records" }; /// /// Inject . @@ -32,7 +32,7 @@ public static void AddSemanticMemoryServices(this WebApplicationBuilder appBuild var memoryConfig = serviceProvider.GetRequiredService>().Value; - var ocrType = memoryConfig.ImageOcrType; + var ocrType = memoryConfig.DataIngestion.ImageOcrType; var hasOcr = !string.IsNullOrWhiteSpace(ocrType) && !ocrType.Equals(MemoryConfiguration.NoneType, StringComparison.OrdinalIgnoreCase); var pipelineType = memoryConfig.DataIngestion.OrchestrationType; @@ -54,7 +54,7 @@ public static void AddSemanticMemoryServices(this WebApplicationBuilder appBuild } } - IKernelMemory memory = memoryBuilder.FromConfiguration( + IKernelMemory memory = memoryBuilder.FromMemoryConfiguration( memoryConfig, appBuilder.Configuration ).Build(); diff --git a/webapi/Extensions/SemanticKernelExtensions.cs b/webapi/Extensions/SemanticKernelExtensions.cs index 884a19c66..0bafb441b 100644 --- a/webapi/Extensions/SemanticKernelExtensions.cs +++ b/webapi/Extensions/SemanticKernelExtensions.cs @@ -20,7 +20,6 @@ using Microsoft.Extensions.Options; using Microsoft.KernelMemory; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Plugins.Core; namespace CopilotChat.WebApi.Extensions; @@ -33,19 +32,13 @@ internal static class SemanticKernelExtensions /// /// Delegate to register functions with a Semantic Kernel /// - public delegate Task RegisterFunctionsWithKernel(IServiceProvider sp, IKernel kernel); + public delegate Task RegisterFunctionsWithKernel(IServiceProvider sp, Kernel kernel); /// /// Delegate for any complimentary setup of the kernel, i.e., registering custom plugins, etc. /// See webapi/README.md#Add-Custom-Setup-to-Chat-Copilot's-Kernel for more details. /// - public delegate Task KernelSetupHook(IServiceProvider sp, IKernel kernel); - - /// - /// Delegate to register plugins with the planner's kernel (i.e., omits plugins not required to generate bot response). - /// See webapi/README.md#Add-Custom-Plugin-Registration-to-the-Planner's-Kernel for more details. - /// - public delegate Task RegisterFunctionsWithPlannerHook(IServiceProvider sp, IKernel kernel); + public delegate Task KernelSetupHook(IServiceProvider sp, Kernel kernel); /// /// Add Semantic Kernel services @@ -55,7 +48,7 @@ public static WebApplicationBuilder AddSemanticKernelServices(this WebApplicatio builder.InitializeKernelProvider(); // Semantic Kernel - builder.Services.AddScoped( + builder.Services.AddScoped( sp => { var provider = sp.GetRequiredService(); @@ -82,34 +75,7 @@ public static WebApplicationBuilder AddSemanticKernelServices(this WebApplicatio } /// - /// Add Planner services - /// - public static WebApplicationBuilder AddPlannerServices(this WebApplicationBuilder builder) - { - builder.InitializeKernelProvider(); - - builder.Services.AddScoped(sp => - { - sp.WithBotConfig(builder.Configuration); - var plannerOptions = sp.GetRequiredService>(); - - var provider = sp.GetRequiredService(); - var plannerKernel = provider.GetPlannerKernel(); - - // Invoke custom plugin registration for planner's kernel. - sp.GetService()?.Invoke(sp, plannerKernel); - - return new CopilotChatPlanner(plannerKernel, plannerOptions?.Value, sp.GetRequiredService>()); - }); - - // Register any custom plugins with the planner's kernel. - builder.Services.AddPlannerSetupHook(); - - return builder; - } - - /// - /// Add Planner services + /// Add embedding model /// public static WebApplicationBuilder AddBotConfig(this WebApplicationBuilder builder) { @@ -129,29 +95,13 @@ public static IServiceCollection AddKernelSetupHook(this IServiceCollection serv return services; } - /// - /// Register custom hook for registering plugins with the planner's kernel. - /// These plugins will be persistent and available to the planner on every request. - /// Transient plugins requiring auth or configured by the webapp should be registered in RegisterPlannerFunctionsAsync of ChatController. - /// - /// The delegate to register plugins with the planner's kernel. If null, defaults to local runtime plugin registration using RegisterPluginsAsync. - public static IServiceCollection AddPlannerSetupHook(this IServiceCollection services, RegisterFunctionsWithPlannerHook? registerPluginsHook = null) - { - // Default to local runtime plugin registration. - registerPluginsHook ??= RegisterPluginsAsync; - - // Add the hook to the service collection - services.AddScoped(sp => registerPluginsHook); - return services; - } - /// /// Register the chat plugin with the kernel. /// - public static IKernel RegisterChatPlugin(this IKernel kernel, IServiceProvider sp) + public static Kernel RegisterChatPlugin(this Kernel kernel, IServiceProvider sp) { // Chat plugin - kernel.ImportFunctions( + kernel.ImportPluginFromObject( new ChatPlugin( kernel, memoryClient: sp.GetRequiredService(), @@ -161,7 +111,6 @@ public static IKernel RegisterChatPlugin(this IKernel kernel, IServiceProvider s promptOptions: sp.GetRequiredService>(), documentImportOptions: sp.GetRequiredService>(), contentSafety: sp.GetService(), - planner: sp.GetRequiredService(), logger: sp.GetRequiredService>()), nameof(ChatPlugin)); @@ -176,13 +125,13 @@ private static void InitializeKernelProvider(this WebApplicationBuilder builder) /// /// Register functions with the main kernel responsible for handling Chat Copilot requests. /// - private static Task RegisterChatCopilotFunctionsAsync(IServiceProvider sp, IKernel kernel) + private static Task RegisterChatCopilotFunctionsAsync(IServiceProvider sp, Kernel kernel) { // Chat Copilot functions kernel.RegisterChatPlugin(sp); // Time plugin - kernel.ImportFunctions(new TimePlugin(), nameof(TimePlugin)); + kernel.ImportPluginFromObject(new TimePlugin(), nameof(TimePlugin)); return Task.CompletedTask; } @@ -190,7 +139,7 @@ private static Task RegisterChatCopilotFunctionsAsync(IServiceProvider sp, IKern /// /// Register plugins with a given kernel. /// - private static Task RegisterPluginsAsync(IServiceProvider sp, IKernel kernel) + private static Task RegisterPluginsAsync(IServiceProvider sp, Kernel kernel) { var logger = kernel.LoggerFactory.CreateLogger(nameof(Kernel)); @@ -202,9 +151,9 @@ private static Task RegisterPluginsAsync(IServiceProvider sp, IKernel kernel) { try { - kernel.ImportSemanticFunctionsFromDirectory(options.SemanticPluginsDirectory, Path.GetFileName(subDir)!); + kernel.ImportPluginFromPromptDirectory(options.SemanticPluginsDirectory, Path.GetFileName(subDir)!); } - catch (SKException ex) + catch (KernelException ex) { logger.LogError("Could not load plugin from {Directory}: {Message}", subDir, ex.Message); } @@ -231,9 +180,9 @@ private static Task RegisterPluginsAsync(IServiceProvider sp, IKernel kernel) try { var plugin = Activator.CreateInstance(classType); - kernel.ImportFunctions(plugin!, classType.Name!); + kernel.ImportPluginFromObject(plugin!, classType.Name!); } - catch (SKException ex) + catch (KernelException ex) { logger.LogError("Could not load plugin from file {File}: {Details}", file, ex.Message); } diff --git a/webapi/Extensions/ServiceExtensions.cs b/webapi/Extensions/ServiceExtensions.cs index 76a812a27..ac873a110 100644 --- a/webapi/Extensions/ServiceExtensions.cs +++ b/webapi/Extensions/ServiceExtensions.cs @@ -21,7 +21,7 @@ using Microsoft.Extensions.Options; using Microsoft.Identity.Web; using Microsoft.KernelMemory; -using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.KernelMemory.Diagnostics; namespace CopilotChat.WebApi.Extensions; @@ -53,8 +53,6 @@ public static IServiceCollection AddOptions(this IServiceCollection services, Co // Chat prompt options AddOptions(PromptsOptions.PropertyName); - AddOptions(PlannerOptions.PropertyName); - AddOptions(ContentSafetyOptions.PropertyName); AddOptions(MemoryConfiguration.KernelMemorySection); diff --git a/webapi/Models/Request/ExecutePlanParameters.cs b/webapi/Models/Request/ExecutePlanParameters.cs deleted file mode 100644 index ff0bcb6d7..000000000 --- a/webapi/Models/Request/ExecutePlanParameters.cs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using CopilotChat.WebApi.Models.Response; - -namespace CopilotChat.WebApi.Models.Request; - -public class ExecutePlanParameters : Ask -{ - public ProposedPlan? Plan { get; set; } -} diff --git a/webapi/Models/Response/AskResult.cs b/webapi/Models/Response/AskResult.cs index c6f4f6175..e7a1a9d06 100644 --- a/webapi/Models/Response/AskResult.cs +++ b/webapi/Models/Response/AskResult.cs @@ -9,5 +9,5 @@ public class AskResult { public string Value { get; set; } = string.Empty; - public IEnumerable>? Variables { get; set; } = Enumerable.Empty>(); + public IEnumerable>? Variables { get; set; } = Enumerable.Empty>(); } diff --git a/webapi/Models/Response/BotResponsePrompt.cs b/webapi/Models/Response/BotResponsePrompt.cs index ccd0d8bb4..4f7cb46c6 100644 --- a/webapi/Models/Response/BotResponsePrompt.cs +++ b/webapi/Models/Response/BotResponsePrompt.cs @@ -1,12 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. using System.Text.Json.Serialization; -using ChatCompletionContextMessages = Microsoft.SemanticKernel.AI.ChatCompletion.ChatHistory; +using Microsoft.SemanticKernel.ChatCompletion; namespace CopilotChat.WebApi.Models.Response; /// -/// The fianl prompt sent to generate bot response. +/// The final prompt sent to generate bot response. /// public class BotResponsePrompt { @@ -40,27 +40,20 @@ public class BotResponsePrompt [JsonPropertyName("chatHistory")] public string ChatHistory { get; set; } = string.Empty; - /// - /// Relevant additional knowledge extracted using a planner. - /// - [JsonPropertyName("externalInformation")] - public SemanticDependency ExternalInformation { get; set; } - /// /// The collection of context messages associated with this chat completions request. /// See https://learn.microsoft.com/en-us/dotnet/api/azure.ai.openai.chatcompletionsoptions.messages?view=azure-dotnet-preview#azure-ai-openai-chatcompletionsoptions-messages. /// [JsonPropertyName("metaPromptTemplate")] - public ChatCompletionContextMessages MetaPromptTemplate { get; set; } = new(); + public ChatHistory MetaPromptTemplate { get; set; } = new(); public BotResponsePrompt( string systemInstructions, string audience, string userIntent, string chatMemories, - SemanticDependency externalInformation, string chatHistory, - ChatCompletionContextMessages metaPromptTemplate + ChatHistory metaPromptTemplate ) { this.SystemPersona = systemInstructions; @@ -68,7 +61,6 @@ ChatCompletionContextMessages metaPromptTemplate this.UserIntent = userIntent; this.PastMemories = chatMemories; this.ChatHistory = chatHistory; - this.ExternalInformation = externalInformation; this.MetaPromptTemplate = metaPromptTemplate; } } diff --git a/webapi/Models/Response/PlanExecutionMetadata.cs b/webapi/Models/Response/PlanExecutionMetadata.cs deleted file mode 100644 index 2c8256330..000000000 --- a/webapi/Models/Response/PlanExecutionMetadata.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace CopilotChat.WebApi.Models.Response; - -/// -/// Metadata about plan execution. -/// -public class PlanExecutionMetadata -{ - /// - /// Steps taken execution stat. - /// - [JsonPropertyName("stepsTaken")] - public string StepsTaken { get; set; } = string.Empty; - - /// - /// Time taken to fulfil the goal. - /// Format: hh:mm:ss - /// - [JsonPropertyName("timeTaken")] - public string TimeTaken { get; set; } = string.Empty; - - /// - /// Functions used execution stat. - /// - [JsonPropertyName("functionsUsed")] - public string FunctionsUsed { get; set; } = string.Empty; - - /// - /// Planner type. - /// - [JsonPropertyName("plannerType")] - public PlanType PlannerType { get; set; } = PlanType.Stepwise; - - /// - /// Raw result of the planner. - /// - [JsonIgnore] - public string RawResult { get; set; } = string.Empty; - - public PlanExecutionMetadata(string stepsTaken, string timeTaken, string functionsUsed, string rawResult) - { - this.StepsTaken = stepsTaken; - this.TimeTaken = timeTaken; - this.FunctionsUsed = functionsUsed; - this.RawResult = rawResult; - } -} diff --git a/webapi/Models/Response/ProposedPlan.cs b/webapi/Models/Response/ProposedPlan.cs deleted file mode 100644 index 37fd5753c..000000000 --- a/webapi/Models/Response/ProposedPlan.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.Planning; - -namespace CopilotChat.WebApi.Models.Response; - -// Type of Plan -public enum PlanType -{ - Action, // single-step - Sequential, // multi-step - Stepwise, // MRKL style planning -} - -// State of Plan -public enum PlanState -{ - NoOp, // Plan has not received any user input - Approved, - Rejected, - Derived, // Plan has been derived from a previous plan; used when user wants to re-run a plan. -} - -/// -/// Information about a single proposed plan. -/// -public class ProposedPlan -{ - /// - /// Plan object to be approved, rejected, or executed. - /// - [JsonPropertyName("proposedPlan")] - public Plan Plan { get; set; } - - /// - /// Indicates whether plan is Action (single-step) or Sequential (multi-step). - /// - [JsonPropertyName("type")] - public PlanType Type { get; set; } - - /// - /// State of plan - /// - [JsonPropertyName("state")] - public PlanState State { get; set; } - - /// - /// User intent that serves as goal of plan. - /// - [JsonPropertyName("userIntent")] - public string UserIntent { get; set; } - - /// - /// Original user input that prompted this plan. - /// - [JsonPropertyName("originalUserInput")] - public string OriginalUserInput { get; set; } - - /// - /// Id tracking bot message of plan in chat history when it was first generated. - /// - [JsonPropertyName("generatedPlanMessageId")] - public string? GeneratedPlanMessageId { get; set; } = null; - - /// - /// Create a new proposed plan. - /// - /// Proposed plan object - public ProposedPlan(Plan plan, PlanType type, PlanState state, string userIntent, string originalUserInput, string? generatedPlanMessageId = null) - { - this.Plan = plan; - this.Type = type; - this.State = state; - this.UserIntent = userIntent; - this.OriginalUserInput = originalUserInput; - this.GeneratedPlanMessageId = generatedPlanMessageId; - } -} diff --git a/webapi/Models/Response/SemanticDependency.cs b/webapi/Models/Response/SemanticDependency.cs deleted file mode 100644 index a3ddfaa08..000000000 --- a/webapi/Models/Response/SemanticDependency.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace CopilotChat.WebApi.Models.Response; - -public interface ISemanticDependency -{ - /// - /// Result of the dependency. This is the output that's injected into the prompt. - /// - [JsonPropertyName("result")] - string Result { get; } - - /// - /// Type of dependency, if any. - /// - [JsonPropertyName("type")] - string? Type { get; } -} - -/// -/// Information about semantic dependencies of the prompt. -/// -public class SemanticDependency : ISemanticDependency -{ - /// - [JsonPropertyName("result")] - public string Result { get; set; } = string.Empty; - - /// - [JsonPropertyName("type")] - public string Type { get; set; } = string.Empty; - - /// - /// Context of the dependency. This can be either the prompt template or planner details. - /// - [JsonPropertyName("context")] - public T? Context { get; set; } = default; - - public SemanticDependency(string result, T? context = default, string? type = null) - { - this.Result = result; - this.Context = context; - this.Type = type ?? typeof(T).Name; - } -} diff --git a/webapi/Models/Storage/CopilotChatMessage.cs b/webapi/Models/Storage/CopilotChatMessage.cs index 5316a9f37..c0aa13a98 100644 --- a/webapi/Models/Storage/CopilotChatMessage.cs +++ b/webapi/Models/Storage/CopilotChatMessage.cs @@ -5,7 +5,6 @@ using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; -using System.Text.RegularExpressions; using CopilotChat.WebApi.Models.Response; using CopilotChat.WebApi.Storage; @@ -161,7 +160,7 @@ public CopilotChatMessage( /// Total token usage of response completion public static CopilotChatMessage CreateBotResponseMessage(string chatId, string content, string prompt, IEnumerable? citations, IDictionary? tokenUsage = null) { - return new CopilotChatMessage("Bot", "Bot", chatId, content, prompt, citations, AuthorRoles.Bot, IsPlan(content) ? ChatMessageType.Plan : ChatMessageType.Message, tokenUsage); + return new CopilotChatMessage("Bot", "Bot", chatId, content, prompt, citations, AuthorRoles.Bot, ChatMessageType.Message, tokenUsage); } /// @@ -185,42 +184,19 @@ public string ToFormattedString() var messagePrefix = $"[{this.Timestamp.ToString("G", CultureInfo.CurrentCulture)}]"; switch (this.Type) { - case ChatMessageType.Plan: - { - var planMessageContent = "proposed a plan."; - if (this.Content.Contains("proposedPlan\":", StringComparison.InvariantCultureIgnoreCase)) - { - // Try to extract user intent from the plan proposal. - string pattern = ".*User Intent:User intent: (.*)(?=\"})"; - Match match = Regex.Match(this.Content, pattern); - if (match.Success) - { - string userIntent = match.Groups[1].Value.Trim(); - planMessageContent = $"proposed a plan to fulfill user intent: {userIntent}"; - } - } - - return $"{messagePrefix} {this.UserName} {planMessageContent}"; - } - case ChatMessageType.Document: - { var documentMessage = DocumentMessageContent.FromString(this.Content); var documentMessageContent = (documentMessage != null) ? documentMessage.ToFormattedString() : "documents"; return $"{messagePrefix} {this.UserName} uploaded: {documentMessageContent}"; - } + case ChatMessageType.Plan: // Fall through case ChatMessageType.Message: - { return $"{messagePrefix} {this.UserName} said: {this.Content}"; - } default: - { // This should never happen. throw new InvalidOperationException($"Unknown message type: {this.Type}"); - } } } @@ -242,16 +218,4 @@ public override string ToString() { return JsonSerializer.Deserialize(json, SerializerSettings); } - - /// - /// Check if the response is a Plan. - /// This is a copy of the `isPlan` function on the frontend. - /// - /// The response from the bot. - /// True if the response represents Plan, false otherwise. - private static bool IsPlan(string response) - { - var planPrefix = "proposedPlan\":"; - return response.Contains(planPrefix, StringComparison.InvariantCulture); - } } diff --git a/webapi/Options/MemoryStoreType.cs b/webapi/Options/MemoryStoreType.cs index 6d92b3f85..d68152336 100644 --- a/webapi/Options/MemoryStoreType.cs +++ b/webapi/Options/MemoryStoreType.cs @@ -28,9 +28,9 @@ public enum MemoryStoreType Qdrant, /// - /// Azure Cognitive Search persistent memory store. + /// Azure AI Search persistent memory store. /// - AzureCognitiveSearch, + AzureAISearch, } public static class MemoryStoreTypeExtensions @@ -44,10 +44,10 @@ public static class MemoryStoreTypeExtensions /// The memory store type. public static MemoryStoreType GetMemoryStoreType(this KernelMemoryConfig memoryOptions, IConfiguration configuration) { - var type = memoryOptions.Retrieval.VectorDbType; - if (type.Equals("AzureCognitiveSearch", StringComparison.OrdinalIgnoreCase)) + var type = memoryOptions.Retrieval.MemoryDbType; + if (type.Equals("AzureAISearch", StringComparison.OrdinalIgnoreCase)) { - return MemoryStoreType.AzureCognitiveSearch; + return MemoryStoreType.AzureAISearch; } else if (type.Equals("Qdrant", StringComparison.OrdinalIgnoreCase)) { diff --git a/webapi/Options/PlannerOptions.cs b/webapi/Options/PlannerOptions.cs deleted file mode 100644 index 672771049..000000000 --- a/webapi/Options/PlannerOptions.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -using System.ComponentModel.DataAnnotations; -using CopilotChat.WebApi.Models.Response; -using Microsoft.SemanticKernel.Planners; - -namespace CopilotChat.WebApi.Options; - -/// -/// Configuration options for the planner. -/// -public class PlannerOptions -{ - /// - /// Options to handle planner errors. - /// - public class ErrorOptions - { - /// - /// Whether to allow retries on planner errors. - /// - public bool AllowRetries { get; set; } = true; - - // - // Whether to allow missing functions in the sequential plan on creation. If set to true, the - // plan will be created with missing functions as no-op steps. If set to false (default), - // the plan creation will fail if any functions are missing. - // - public bool AllowMissingFunctions { get; set; } = true; - - /// - /// Max retries allowed. - /// - [Range(1, 5)] - public int MaxRetriesAllowed { get; set; } = 3; - } - - public const string PropertyName = "Planner"; - - /// - /// The model name used by the planner. - /// - [Required] - public string Model { get; set; } = string.Empty; - - /// - /// The type of planner to used to create plan. - /// - [Required] - public PlanType Type { get; set; } = PlanType.Action; - - /// - /// The minimum relevancy score for a function to be considered during plan creation when using SequentialPlanner - /// - [Range(0, 1.0)] - public double? RelevancyThreshold { get; set; } = 0; - - /// - /// The maximum number of seconds to wait for a response from a plugin. - /// If this is not set, timeout limit will be 100s, which is the default timeout setting for HttpClient. - /// - [Range(0, int.MaxValue)] - public double PluginTimeoutLimitInS { get; set; } = 100; - - /// - /// Options on how to handle planner errors. - /// - public ErrorOptions ErrorHandling { get; set; } = new ErrorOptions(); - - /// - /// Optional flag to indicate whether to use the planner result as the bot response. - /// - [RequiredOnPropertyValue(nameof(Type), PlanType.Stepwise)] - public bool UseStepwiseResultAsBotResponse { get; set; } = false; - - /// - /// The configuration for the stepwise planner. - /// - [RequiredOnPropertyValue(nameof(Type), PlanType.Stepwise)] - public StepwisePlannerConfig StepwisePlannerConfig { get; set; } = new StepwisePlannerConfig(); -} diff --git a/webapi/Options/PromptsOptions.cs b/webapi/Options/PromptsOptions.cs index d6be8a336..5dd1edc6a 100644 --- a/webapi/Options/PromptsOptions.cs +++ b/webapi/Options/PromptsOptions.cs @@ -31,12 +31,6 @@ public class PromptsOptions /// internal double MemoriesResponseContextWeight { get; } = 0.6; - /// - /// Weight of information returned from planner (i.e., responses from OpenAPI functions). - /// Contextual prompt excludes all the system commands and user intent. - /// - internal double ExternalInformationContextWeight { get; } = 0.3; - /// /// Upper bound of the relevancy score of a kernel memory to be included in the final prompt. /// The actual relevancy score is determined by the memory balance. @@ -61,21 +55,6 @@ public class PromptsOptions [Required, NotEmptyOrWhitespace] public string SystemDescription { get; set; } = string.Empty; [Required, NotEmptyOrWhitespace] public string SystemResponse { get; set; } = string.Empty; - /// - /// Context bot message for meta prompt when using external information acquired from a plan. - /// - [Required, NotEmptyOrWhitespace] public string ProposedPlanBotMessage { get; set; } = string.Empty; - - /// - /// Supplement to help guide model in using data. - /// - [Required, NotEmptyOrWhitespace] public string PlanResultsDescription { get; set; } = string.Empty; - - /// - /// Supplement to help guide model in using a response from StepwisePlanner. - /// - [Required, NotEmptyOrWhitespace] public string StepwisePlannerSupplement { get; set; } = string.Empty; - internal string[] SystemAudiencePromptComponents => new string[] { this.SystemAudience, diff --git a/webapi/Plugins/Chat/ChatPlugin.cs b/webapi/Plugins/Chat/ChatPlugin.cs index 3e8579650..7e33951c1 100644 --- a/webapi/Plugins/Chat/ChatPlugin.cs +++ b/webapi/Plugins/Chat/ChatPlugin.cs @@ -6,7 +6,6 @@ using System.Globalization; using System.Linq; using System.Text.Json; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using CopilotChat.WebApi.Auth; @@ -23,14 +22,8 @@ using Microsoft.IdentityModel.Tokens; using Microsoft.KernelMemory; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.ChatCompletion; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Basic; -using ChatCompletionContextMessages = Microsoft.SemanticKernel.AI.ChatCompletion.ChatHistory; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.OpenAI; using CopilotChatMessage = CopilotChat.WebApi.Models.Storage.CopilotChatMessage; namespace CopilotChat.WebApi.Plugins.Chat; @@ -45,7 +38,7 @@ public class ChatPlugin /// A kernel instance to create a completion function since each invocation /// of the function will generate a new prompt dynamically. /// - private readonly IKernel _kernel; + private readonly Kernel _kernel; /// /// Client for the kernel memory service. @@ -82,11 +75,6 @@ public class ChatPlugin /// private readonly SemanticMemoryRetriever _semanticMemoryRetriever; - /// - /// A plugin instance to acquire external information. - /// - private readonly ExternalInformationPlugin _externalInformationPlugin; - /// /// Azure content safety moderator. /// @@ -96,14 +84,13 @@ public class ChatPlugin /// Create a new instance of . /// public ChatPlugin( - IKernel kernel, + Kernel kernel, IKernelMemory memoryClient, ChatMessageRepository chatMessageRepository, ChatSessionRepository chatSessionRepository, IHubContext messageRelayHubContext, IOptions promptOptions, IOptions documentImportOptions, - CopilotChatPlanner planner, ILogger logger, AzureContentSafety? contentSafety = null) { @@ -122,19 +109,15 @@ public ChatPlugin( memoryClient, logger); - this._externalInformationPlugin = new ExternalInformationPlugin( - promptOptions, - planner, - logger); this._contentSafety = contentSafety; } /// /// Extract user intent from the conversation history. /// - /// The SKContext. + /// The KernelArguments. /// The cancellation token. - private async Task ExtractUserIntentAsync(SKContext context, CancellationToken cancellationToken = default) + private async Task ExtractUserIntentAsync(KernelArguments kernelArguments, CancellationToken cancellationToken = default) { var tokenLimit = this._promptOptions.CompletionTokenLimit; var historyTokenBudget = @@ -149,23 +132,24 @@ private async Task ExtractUserIntentAsync(SKContext context, Cancellatio ); // Clone the context to avoid modifying the original context variables. - var intentExtractionContext = context.Clone(); - intentExtractionContext.Variables.Set("tokenLimit", historyTokenBudget.ToString(new NumberFormatInfo())); - intentExtractionContext.Variables.Set("knowledgeCutoff", this._promptOptions.KnowledgeCutoffDate); + KernelArguments intentExtractionContext = new(kernelArguments); + intentExtractionContext["tokenLimit"] = historyTokenBudget.ToString(new NumberFormatInfo()); + intentExtractionContext["knowledgeCutoff"] = this._promptOptions.KnowledgeCutoffDate; - var completionFunction = this._kernel.CreateSemanticFunction( + var completionFunction = this._kernel.CreateFunctionFromPrompt( this._promptOptions.SystemIntentExtraction, - pluginName: nameof(ChatPlugin), + this.CreateIntentCompletionSettings(), + functionName: nameof(ChatPlugin), description: "Complete the prompt."); var result = await completionFunction.InvokeAsync( + this._kernel, intentExtractionContext, - this.CreateIntentCompletionSettings(), cancellationToken ); // Get token usage from ChatCompletion result and add to context - TokenUtils.GetFunctionTokenUsage(result, context, this._logger, "SystemIntentExtraction"); + TokenUtils.GetFunctionTokenUsage(result, intentExtractionContext, this._logger, "SystemIntentExtraction"); return $"User intent: {result}"; } @@ -176,7 +160,7 @@ private async Task ExtractUserIntentAsync(SKContext context, Cancellatio /// /// The SKContext. /// The cancellation token. - private async Task ExtractAudienceAsync(SKContext context, CancellationToken cancellationToken = default) + private async Task ExtractAudienceAsync(KernelArguments context, CancellationToken cancellationToken = default) { var tokenLimit = this._promptOptions.CompletionTokenLimit; var historyTokenBudget = @@ -190,17 +174,18 @@ private async Task ExtractAudienceAsync(SKContext context, CancellationT ); // Clone the context to avoid modifying the original context variables. - var audienceExtractionContext = context.Clone(); - audienceExtractionContext.Variables.Set("tokenLimit", historyTokenBudget.ToString(new NumberFormatInfo())); + KernelArguments audienceExtractionContext = new(context); + audienceExtractionContext["tokenLimit"] = historyTokenBudget.ToString(new NumberFormatInfo()); - var completionFunction = this._kernel.CreateSemanticFunction( + var completionFunction = this._kernel.CreateFunctionFromPrompt( this._promptOptions.SystemAudienceExtraction, - pluginName: nameof(ChatPlugin), + this.CreateIntentCompletionSettings(), + functionName: nameof(ChatPlugin), description: "Complete the prompt."); var result = await completionFunction.InvokeAsync( + this._kernel, audienceExtractionContext, - this.CreateIntentCompletionSettings(), cancellationToken ); @@ -216,7 +201,7 @@ private async Task ExtractAudienceAsync(SKContext context, CancellationT /// but the ChatHistory type is not supported when calling from a rendered prompt, so this wrapper bypasses the chatHistory parameter. /// /// The cancellation token. - [SKFunction, Description("Extract chat history")] + [KernelFunction, Description("Extract chat history")] public Task ExtractChatHistory( [Description("Chat ID to extract history from")] string chatId, [Description("Maximum number of tokens")] int tokenLimit, @@ -226,23 +211,23 @@ public Task ExtractChatHistory( } /// - /// Extract chat history within token limit as a formatted string and optionally update the ChatCompletionContextMessages object with the allotted messages + /// Extract chat history within token limit as a formatted string and optionally update the ChatHistory object with the allotted messages /// /// Chat ID to extract history from. /// Maximum number of tokens. - /// Optional ChatCompletionContextMessages object tracking allotted messages. + /// Optional ChatHistory object tracking allotted messages. /// The cancellation token. /// Chat history as a string. private async Task GetAllowedChatHistoryAsync( string chatId, int tokenLimit, - ChatCompletionContextMessages? chatHistory = null, + ChatHistory? chatHistory = null, CancellationToken cancellationToken = default) { var messages = await this._chatMessageRepository.FindByChatIdAsync(chatId); var sortedMessages = messages.OrderByDescending(m => m.Timestamp); - ChatCompletionContextMessages allottedChatHistory = new(); + ChatHistory allottedChatHistory = new(); var remainingToken = tokenLimit; string historyText = string.Empty; @@ -255,23 +240,6 @@ private async Task GetAllowedChatHistoryAsync( continue; } - // Plan object is not meaningful content in generating bot response, so shorten to intent only to save on tokens - if (chatMessage.Type == CopilotChatMessage.ChatMessageType.Plan) - { - formattedMessage = "Bot proposed plan"; - - // Try to extract the user intent for more context - string pattern = @"User intent: (.*)(?=\.""})"; - Match match = Regex.Match(chatMessage.Content, pattern); - if (match.Success) - { - string userIntent = match.Groups[1].Value.Trim(); - formattedMessage = $"Bot proposed plan to help fulfill goal: {userIntent}."; - } - - formattedMessage = $"[{chatMessage.Timestamp.ToString("G", CultureInfo.CurrentCulture)}] {formattedMessage}"; - } - var promptRole = chatMessage.AuthorRole == CopilotChatMessage.AuthorRoles.Bot ? AuthorRole.System : AuthorRole.User; var tokenCount = chatHistory is not null ? TokenUtils.GetContextMessageTokenCount(promptRole, formattedMessage) : TokenUtils.TokenCount(formattedMessage); @@ -281,7 +249,7 @@ private async Task GetAllowedChatHistoryAsync( if (chatMessage.AuthorRole == CopilotChatMessage.AuthorRoles.Bot) { // Message doesn't have to be formatted for bot. This helps with asserting a natural language response from the LLM (no date or author preamble). - var botMessage = chatMessage.Type == CopilotChatMessage.ChatMessageType.Plan ? formattedMessage : chatMessage.Content; + var botMessage = chatMessage.Content; allottedChatHistory.AddAssistantMessage(botMessage.Trim()); } else @@ -301,8 +269,7 @@ private async Task GetAllowedChatHistoryAsync( } } - allottedChatHistory.Reverse(); - chatHistory?.AddRange(allottedChatHistory); + chatHistory?.AddRange(allottedChatHistory.Reverse()); return $"Chat history:\n{historyText.Trim()}"; } @@ -313,14 +280,14 @@ private async Task GetAllowedChatHistoryAsync( /// prompt that will be rendered by the template engine. /// /// The cancellation token. - [SKFunction, Description("Get chat response")] - public async Task ChatAsync( + [KernelFunction, Description("Get chat response")] + public async Task ChatAsync( [Description("The new message")] string message, [Description("Unique and persistent identifier for the user")] string userId, [Description("Name of the user")] string userName, [Description("Unique and persistent identifier for the chat")] string chatId, [Description("Type of the message")] string messageType, - SKContext context, + KernelArguments context, CancellationToken cancellationToken = default) { // Set the system description in the prompt options @@ -331,15 +298,15 @@ public async Task ChatAsync( var newUserMessage = await this.SaveNewMessageAsync(message, userId, userName, chatId, messageType, cancellationToken); // Clone the context to avoid modifying the original context variables. - var chatContext = context.Clone(); - chatContext.Variables.Set("knowledgeCutoff", this._promptOptions.KnowledgeCutoffDate); + KernelArguments chatContext = new(context); + chatContext["knowledgeCutoff"] = this._promptOptions.KnowledgeCutoffDate; CopilotChatMessage chatMessage = await this.GetChatResponseAsync(chatId, userId, chatContext, newUserMessage, cancellationToken); - context.Variables.Update(chatMessage.Content); + context["input"] = chatMessage.Content; if (chatMessage.TokenUsage != null) { - context.Variables.Set("tokenUsage", JsonSerializer.Serialize(chatMessage.TokenUsage)); + context["tokenUsage"] = JsonSerializer.Serialize(chatMessage.TokenUsage); } else { @@ -349,183 +316,22 @@ public async Task ChatAsync( return context; } - /// - /// This is the entry point for handling a plan, whether the user approves, cancels, or re-runs it. - /// - /// The cancellation token. - [SKFunction, Description("Process a plan")] - public async Task ProcessPlanAsync( - [Description("The new message")] string message, - [Description("Unique and persistent identifier for the user")] string userId, - [Description("Name of the user")] string userName, - [Description("Unique and persistent identifier for the chat")] string chatId, - [Description("Proposed plan object"), DefaultValue(null), SKName("proposedPlan")] string? planJson, - SKContext context, - CancellationToken cancellationToken = default) - { - // Ensure that plan exists in ask's context variables. - // If a plan was returned at this point, that means it is a - // 1. Proposed plan that was approved or cancelled, or - // 2. Saved plan being re-run. - ProposedPlan? deserializedPlan; - if (string.IsNullOrWhiteSpace(planJson) || (deserializedPlan = JsonSerializer.Deserialize(planJson)) == null) - { - throw new ArgumentException("Plan does not exist in request context. Ensure that a valid plan is saved in the ask's variables"); - } - - // Ensure plan is actionable by supported planners. - if (!(deserializedPlan.Type == PlanType.Action || deserializedPlan.Type == PlanType.Sequential)) - { - throw new ArgumentException($"Plan of type {deserializedPlan.Type} cannot be executed. Only Action or Sequential plans can be re-run at this time."); - } - - // Set the system description in the prompt options. - await this.SetSystemDescriptionAsync(chatId, cancellationToken); - - // Clone the context to avoid modifying the original context variables. - var chatContext = context.Clone(); - chatContext.Variables.Set("knowledgeCutoff", this._promptOptions.KnowledgeCutoffDate); - - // Save this new message to memory such that subsequent chat responses can use it - await this.UpdateBotResponseStatusOnClientAsync(chatId, "Saving user message to chat history", cancellationToken); - var newUserMessage = await this.SaveNewMessageAsync(message, userId, userName, chatId, CopilotChatMessage.ChatMessageType.Message.ToString(), cancellationToken); - - // If GeneratedPlanMessageId exists on plan object, update that message with new plan state. - // This signals that this plan was freshly proposed by the model and already saved as a bot response message in chat history. - if (!string.IsNullOrEmpty(deserializedPlan.GeneratedPlanMessageId)) - { - await this.UpdateChatMessageContentAsync(planJson, deserializedPlan.GeneratedPlanMessageId, chatId, cancellationToken); - } - - // If plan was derived from a previous plan, create a new bot message with the plan content and save to chat history. - if (deserializedPlan.State == PlanState.Derived) - { - await this.SaveNewResponseAsync( - planJson, - deserializedPlan.Plan.Description, - chatId, - userId, - cancellationToken - ); - } - - CopilotChatMessage chatMessage; - if (deserializedPlan.State == PlanState.Rejected) - { - // Use a hardcoded response if user cancelled plan - await this.UpdateBotResponseStatusOnClientAsync(chatId, "Cancelling plan", cancellationToken); - chatMessage = await this.SaveNewResponseAsync( - "I am sorry the plan did not meet your goals.", - string.Empty, - chatId, - userId, - cancellationToken, - TokenUtils.EmptyTokenUsages() - ); - } - else if (deserializedPlan.State == PlanState.Approved || deserializedPlan.State == PlanState.Derived) - { - // Render system instruction components and create the meta-prompt template - var systemInstructions = await AsyncUtils.SafeInvokeAsync( - () => this.RenderSystemInstructions(chatId, chatContext, cancellationToken), nameof(RenderSystemInstructions)); - var chatCompletion = this._kernel.GetService(); - var promptTemplate = chatCompletion.CreateNewChat(systemInstructions); - string chatHistoryString = ""; - - // Add original user input that prompted plan template - promptTemplate.AddUserMessage(deserializedPlan.OriginalUserInput); - chatHistoryString += "\n" + PromptUtils.FormatChatHistoryMessage(CopilotChatMessage.AuthorRoles.User, deserializedPlan.OriginalUserInput); - - // Add bot message proposal as prompt context message - chatContext.Variables.Set("planFunctions", this._externalInformationPlugin.FormattedFunctionsString(deserializedPlan.Plan)); - var promptTemplateFactory = new BasicPromptTemplateFactory(); - var proposedPlanTemplate = promptTemplateFactory.Create(this._promptOptions.ProposedPlanBotMessage, new PromptTemplateConfig()); - var proposedPlanBotMessage = await proposedPlanTemplate.RenderAsync(chatContext, cancellationToken); - promptTemplate.AddAssistantMessage(proposedPlanBotMessage); - chatHistoryString += "\n" + PromptUtils.FormatChatHistoryMessage(CopilotChatMessage.AuthorRoles.Bot, proposedPlanBotMessage); - - // Add user approval message as prompt context message - promptTemplate.AddUserMessage("Yes, proceed"); - chatHistoryString += "\n" + PromptUtils.FormatChatHistoryMessage(CopilotChatMessage.AuthorRoles.User, "Yes, proceed"); - - // Add user intent behind plan - // TODO: [Issue #51] Consider regenerating user intent if plan was modified - promptTemplate.AddSystemMessage(deserializedPlan.UserIntent); - - // Render system supplement to help guide model in using data. - var promptSupplementTemplate = promptTemplateFactory.Create(this._promptOptions.PlanResultsDescription, new PromptTemplateConfig()); - var promptSupplement = await promptSupplementTemplate.RenderAsync(chatContext, cancellationToken); - - // Calculate remaining token budget and execute plan - await this.UpdateBotResponseStatusOnClientAsync(chatId, "Executing plan", cancellationToken); - var remainingTokenBudget = this.GetChatContextTokenLimit(promptTemplate) - TokenUtils.GetContextMessageTokenCount(AuthorRole.System, promptSupplement); - - try - { - var planResult = await this.AcquireExternalInformationAsync(chatContext, deserializedPlan.UserIntent, remainingTokenBudget, cancellationToken, deserializedPlan.Plan); - promptTemplate.AddSystemMessage(planResult); - - // Calculate token usage of prompt template - chatContext.Variables.Set(TokenUtils.GetFunctionKey(this._logger, "SystemMetaPrompt")!, TokenUtils.GetContextMessagesTokenCount(promptTemplate).ToString(CultureInfo.CurrentCulture)); - - // TODO: [Issue #150, sk#2106] Accommodate different planner contexts once core team finishes work to return prompt and token usage. - var plannerDetails = new SemanticDependency(planResult, null, deserializedPlan.Type.ToString()); - - // Get bot response and stream to client - var promptView = new BotResponsePrompt(systemInstructions, "", deserializedPlan.UserIntent, "", plannerDetails, chatHistoryString, promptTemplate); - chatMessage = await this.HandleBotResponseAsync(chatId, userId, chatContext, promptView, cancellationToken); - - if (chatMessage.TokenUsage != null) - { - context.Variables.Set("tokenUsage", JsonSerializer.Serialize(chatMessage.TokenUsage)); - } - else - { - this._logger.LogWarning("ChatPlugin.ProcessPlan token usage unknown. Ensure token management has been implemented correctly."); - } - } - catch (Exception ex) - { - // Use a hardcoded response if plan failed. - // TODO: [Issue #150, sk#2106] Check planner token usage, if any, on failure - chatMessage = await this.SaveNewResponseAsync( - $"Oops, I encountered an issue processing your plan. Please review the plan logic and access permissions required for the plan, then try running it again from the Plans tab.\n\nError details: {ex.Message}", - string.Empty, - chatId, - userId, - cancellationToken, - TokenUtils.EmptyTokenUsages() - ); - - throw new SKException("Failed to process plan.", ex); - } - - context.Variables.Update(chatMessage.Content); - } - else - { - throw new ArgumentException($"Plan inactionable in current state: {deserializedPlan.State}"); - } - - return context; - } - /// /// Generate the necessary chat context to create a prompt then invoke the model to get a response. /// /// The chat ID /// The user ID - /// The SKContext. + /// The KernelArguments. /// ChatMessage object representing new user message. /// The cancellation token. /// The created chat message containing the model-generated response. - private async Task GetChatResponseAsync(string chatId, string userId, SKContext chatContext, CopilotChatMessage userMessage, CancellationToken cancellationToken) + private async Task GetChatResponseAsync(string chatId, string userId, KernelArguments chatContext, CopilotChatMessage userMessage, CancellationToken cancellationToken) { // Render system instruction components and create the meta-prompt template var systemInstructions = await AsyncUtils.SafeInvokeAsync( () => this.RenderSystemInstructions(chatId, chatContext, cancellationToken), nameof(RenderSystemInstructions)); - var chatCompletion = this._kernel.GetService(); - var promptTemplate = chatCompletion.CreateNewChat(systemInstructions); + var chatCompletion = this._kernel.GetRequiredService(); + ChatHistory chatHistory = new(systemInstructions); // Bypass audience extraction if Auth is disabled var audience = string.Empty; @@ -535,53 +341,18 @@ private async Task GetChatResponseAsync(string chatId, strin await this.UpdateBotResponseStatusOnClientAsync(chatId, "Extracting audience", cancellationToken); audience = await AsyncUtils.SafeInvokeAsync( () => this.GetAudienceAsync(chatContext, cancellationToken), nameof(GetAudienceAsync)); - promptTemplate.AddSystemMessage(audience); + chatHistory.AddSystemMessage(audience); } // Extract user intent from the conversation history. await this.UpdateBotResponseStatusOnClientAsync(chatId, "Extracting user intent", cancellationToken); var userIntent = await AsyncUtils.SafeInvokeAsync( () => this.GetUserIntentAsync(chatContext, cancellationToken), nameof(GetUserIntentAsync)); - promptTemplate.AddSystemMessage(userIntent); + chatHistory.AddSystemMessage(userIntent); // Calculate the remaining token budget. await this.UpdateBotResponseStatusOnClientAsync(chatId, "Calculating remaining token budget", cancellationToken); - var remainingTokenBudget = this.GetChatContextTokenLimit(promptTemplate, userMessage.ToFormattedString()); - - // Acquire external information from planner - await this.UpdateBotResponseStatusOnClientAsync(chatId, "Acquiring external information from planner", cancellationToken); - var externalInformationTokenLimit = (int)(remainingTokenBudget * this._promptOptions.ExternalInformationContextWeight); - var planResult = await AsyncUtils.SafeInvokeAsync( - () => this.AcquireExternalInformationAsync(chatContext, userIntent, externalInformationTokenLimit, cancellationToken: cancellationToken), nameof(AcquireExternalInformationAsync)); - - // Extract additional details about stepwise planner execution in chat context - var plannerDetails = new SemanticDependency( - this._externalInformationPlugin.StepwiseThoughtProcess?.RawResult ?? planResult, - this._externalInformationPlugin.StepwiseThoughtProcess - ); - - // If plan is suggested, send back to user for approval before running - var proposedPlan = this._externalInformationPlugin.ProposedPlan; - if (proposedPlan != null) - { - // Save a new response to the chat history with the proposed plan content - return await this.SaveNewResponseAsync( - JsonSerializer.Serialize(proposedPlan), - proposedPlan.Plan.Description, - chatId, - userId, - cancellationToken, - // TODO: [Issue #2106] Accommodate plan token usage differently - this.GetTokenUsages(chatContext) - ); - } - - // If plan result is to be used as bot response, save the Stepwise result as a new response to the chat history and return. - if (this._externalInformationPlugin.UseStepwiseResultAsBotResponse(planResult)) - { - var promptDetails = new BotResponsePrompt("", "", userIntent, "", plannerDetails, "", new ChatHistory()); - return await this.HandleBotResponseAsync(chatId, userId, chatContext, promptDetails, cancellationToken, null, this._externalInformationPlugin.StepwiseThoughtProcess!.RawResult); - } + var remainingTokenBudget = this.GetChatContextTokenLimit(chatHistory, userMessage.ToFormattedString()); // Query relevant semantic and document memories await this.UpdateBotResponseStatusOnClientAsync(chatId, "Extracting semantic and document memories", cancellationToken); @@ -590,28 +361,22 @@ private async Task GetChatResponseAsync(string chatId, strin if (!string.IsNullOrWhiteSpace(memoryText)) { - promptTemplate.AddSystemMessage(memoryText); + chatHistory.AddSystemMessage(memoryText); } // Fill in the chat history with remaining token budget. - string chatHistory = string.Empty; - var chatHistoryTokenBudget = remainingTokenBudget - TokenUtils.GetContextMessageTokenCount(AuthorRole.System, memoryText) - TokenUtils.GetContextMessageTokenCount(AuthorRole.System, planResult); + string allowedChatHistory = string.Empty; + var allowedChatHistoryTokenBudget = remainingTokenBudget - TokenUtils.GetContextMessageTokenCount(AuthorRole.System, memoryText); // Append previous messages await this.UpdateBotResponseStatusOnClientAsync(chatId, "Extracting chat history", cancellationToken); - chatHistory = await this.GetAllowedChatHistoryAsync(chatId, chatHistoryTokenBudget, promptTemplate, cancellationToken); - - // Append the plan result last, if exists, to imply precedence. - if (!string.IsNullOrWhiteSpace(planResult)) - { - promptTemplate.AddSystemMessage(planResult); - } + allowedChatHistory = await this.GetAllowedChatHistoryAsync(chatId, allowedChatHistoryTokenBudget, chatHistory, cancellationToken); // Calculate token usage of prompt template - chatContext.Variables.Set(TokenUtils.GetFunctionKey(this._logger, "SystemMetaPrompt")!, TokenUtils.GetContextMessagesTokenCount(promptTemplate).ToString(CultureInfo.CurrentCulture)); + chatContext[TokenUtils.GetFunctionKey(this._logger, "SystemMetaPrompt")!] = TokenUtils.GetContextMessagesTokenCount(chatHistory).ToString(CultureInfo.CurrentCulture); // Stream the response to the client - var promptView = new BotResponsePrompt(systemInstructions, audience, userIntent, memoryText, plannerDetails, chatHistory, promptTemplate); + var promptView = new BotResponsePrompt(systemInstructions, audience, userIntent, memoryText, allowedChatHistory, chatHistory); return await this.HandleBotResponseAsync(chatId, userId, chatContext, promptView, cancellationToken, citationMap.Values.AsEnumerable()); } @@ -619,17 +384,16 @@ private async Task GetChatResponseAsync(string chatId, strin /// Helper function to render system instruction components. /// /// The chat ID - /// The SKContext. + /// The KernelArguments. /// The cancellation token. - private async Task RenderSystemInstructions(string chatId, SKContext context, CancellationToken cancellationToken) + private async Task RenderSystemInstructions(string chatId, KernelArguments context, CancellationToken cancellationToken) { // Render system instruction components await this.UpdateBotResponseStatusOnClientAsync(chatId, "Initializing prompt", cancellationToken); - var promptTemplateFactory = new BasicPromptTemplateFactory(); - var template = promptTemplateFactory.Create(this._promptOptions.SystemPersona, new PromptTemplateConfig()); - - return await template.RenderAsync(context, cancellationToken); + var promptTemplateFactory = new KernelPromptTemplateFactory(); + var promptTemplate = promptTemplateFactory.Create(new PromptTemplateConfig(this._promptOptions.SystemPersona)); + return await promptTemplate.RenderAsync(this._kernel, context, cancellationToken); } /// @@ -643,7 +407,7 @@ private async Task RenderSystemInstructions(string chatId, SKContext con private async Task HandleBotResponseAsync( string chatId, string userId, - SKContext chatContext, + KernelArguments chatContext, BotResponsePrompt promptView, CancellationToken cancellationToken, IEnumerable? citations = null, @@ -701,16 +465,16 @@ await AsyncUtils.SafeInvokeAsync( /// extract the audience from a conversation history. /// /// The cancellation token. - private async Task GetAudienceAsync(SKContext context, CancellationToken cancellationToken) + private async Task GetAudienceAsync(KernelArguments context, CancellationToken cancellationToken) { - SKContext audienceContext = context.Clone(); + KernelArguments audienceContext = new(context); var audience = await this.ExtractAudienceAsync(audienceContext, cancellationToken); // Copy token usage into original chat context var functionKey = TokenUtils.GetFunctionKey(this._logger, "SystemAudienceExtraction")!; - if (audienceContext.Variables.TryGetValue(functionKey, out string? tokenUsage)) + if (audienceContext.TryGetValue(functionKey, out object? tokenUsage)) { - context.Variables.Set(functionKey, tokenUsage); + context[functionKey] = tokenUsage; } return audience; @@ -721,38 +485,21 @@ private async Task GetAudienceAsync(SKContext context, CancellationToken /// extract the user intent from the conversation history. /// /// The cancellation token. - private async Task GetUserIntentAsync(SKContext context, CancellationToken cancellationToken) + private async Task GetUserIntentAsync(KernelArguments context, CancellationToken cancellationToken) { - SKContext intentContext = context.Clone(); + KernelArguments intentContext = new(context); string userIntent = await this.ExtractUserIntentAsync(intentContext, cancellationToken); // Copy token usage into original chat context var functionKey = TokenUtils.GetFunctionKey(this._logger, "SystemIntentExtraction")!; - if (intentContext.Variables.TryGetValue(functionKey!, out string? tokenUsage)) + if (intentContext.TryGetValue(functionKey!, out object? tokenUsage)) { - context.Variables.Set(functionKey!, tokenUsage); + context[functionKey!] = tokenUsage; } return userIntent; } - /// - /// Helper function that creates the correct context variables to acquire external information. - /// - /// The plan. - /// The SKContext. - /// The user intent. - /// Maximum number of tokens. - /// The cancellation token. - private async Task AcquireExternalInformationAsync(SKContext context, string userIntent, int tokenLimit, CancellationToken cancellationToken, Plan? plan = null) - { - SKContext planContext = context.Clone(); - planContext.Variables.Set("tokenLimit", tokenLimit.ToString(new NumberFormatInfo())); - return plan is not null - ? await this._externalInformationPlugin.ExecutePlanAsync(planContext, plan, cancellationToken) - : await this._externalInformationPlugin.InvokePlannerAsync(userIntent, planContext, cancellationToken); - } - /// /// Save a new message to the chat history. /// @@ -849,26 +596,27 @@ private async Task UpdateChatMessageContentAsync(string updatedResponse, string } /// - /// Create `OpenAIRequestSettings` for chat response. Parameters are read from the PromptSettings class. + /// Create `OpenAIPromptExecutionSettings` for chat response. Parameters are read from the PromptSettings class. /// - private OpenAIRequestSettings CreateChatRequestSettings() + private OpenAIPromptExecutionSettings CreateChatRequestSettings() { - return new OpenAIRequestSettings + return new OpenAIPromptExecutionSettings { MaxTokens = this._promptOptions.ResponseTokenLimit, Temperature = this._promptOptions.ResponseTemperature, TopP = this._promptOptions.ResponseTopP, FrequencyPenalty = this._promptOptions.ResponseFrequencyPenalty, - PresencePenalty = this._promptOptions.ResponsePresencePenalty + PresencePenalty = this._promptOptions.ResponsePresencePenalty, + ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions }; } /// - /// Create `OpenAIRequestSettings` for intent response. Parameters are read from the PromptSettings class. + /// Create `OpenAIPromptExecutionSettings` for intent response. Parameters are read from the PromptSettings class. /// - private OpenAIRequestSettings CreateIntentCompletionSettings() + private OpenAIPromptExecutionSettings CreateIntentCompletionSettings() { - return new OpenAIRequestSettings + return new OpenAIPromptExecutionSettings { MaxTokens = this._promptOptions.ResponseTokenLimit, Temperature = this._promptOptions.IntentTemperature, @@ -886,7 +634,7 @@ private OpenAIRequestSettings CreateIntentCompletionSettings() /// All current messages to use for chat completion /// The user message. /// The remaining token limit. - private int GetChatContextTokenLimit(ChatCompletionContextMessages promptTemplate, string userInput = "") + private int GetChatContextTokenLimit(ChatHistory promptTemplate, string userInput = "") { return this._promptOptions.CompletionTokenLimit - TokenUtils.GetContextMessagesTokenCount(promptTemplate) @@ -897,19 +645,22 @@ private int GetChatContextTokenLimit(ChatCompletionContextMessages promptTemplat /// /// Gets token usage totals for each semantic function if not undefined. /// - /// Context maintained during response generation. + /// Context maintained during response generation. /// String representing bot response. If null, response is still being generated or was hardcoded. /// Dictionary containing function to token usage mapping for each total that's defined. - private Dictionary GetTokenUsages(SKContext chatContext, string? content = null) + private Dictionary GetTokenUsages(KernelArguments kernelArguments, string? content = null) { var tokenUsageDict = new Dictionary(StringComparer.OrdinalIgnoreCase); // Total token usage of each semantic function foreach (string function in TokenUtils.semanticFunctions.Values) { - if (chatContext.Variables.TryGetValue($"{function}TokenUsage", out string? tokenUsage)) + if (kernelArguments.TryGetValue($"{function}TokenUsage", out object? tokenUsage)) { - tokenUsageDict.Add(function, int.Parse(tokenUsage, CultureInfo.InvariantCulture)); + if (tokenUsage is string tokenUsageString) + { + tokenUsageDict.Add(function, int.Parse(tokenUsageString, CultureInfo.InvariantCulture)); + } } } @@ -938,11 +689,12 @@ private async Task StreamResponseToClientAsync( IEnumerable? citations = null) { // Create the stream - var chatCompletion = this._kernel.GetService(); + var chatCompletion = this._kernel.GetRequiredService(); var stream = - chatCompletion.GenerateMessageStreamAsync( + chatCompletion.GetStreamingChatMessageContentsAsync( prompt.MetaPromptTemplate, this.CreateChatRequestSettings(), + this._kernel, cancellationToken); // Create message on client @@ -956,7 +708,7 @@ private async Task StreamResponseToClientAsync( ); // Stream the message to the client - await foreach (string contentPiece in stream) + await foreach (var contentPiece in stream) { chatMessage.Content += contentPiece; await this.UpdateMessageOnClient(chatMessage, cancellationToken); diff --git a/webapi/Plugins/Chat/CopilotChatPlanner.cs b/webapi/Plugins/Chat/CopilotChatPlanner.cs deleted file mode 100644 index f88783c3d..000000000 --- a/webapi/Plugins/Chat/CopilotChatPlanner.cs +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using CopilotChat.WebApi.Models.Response; -using CopilotChat.WebApi.Options; -using Microsoft.Extensions.Logging; -using Microsoft.IdentityModel.Tokens; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planners; -using Microsoft.SemanticKernel.Planning; - -namespace CopilotChat.WebApi.Plugins.Chat; - -/// -/// A lightweight wrapper around a planner to allow for curating which functions are available to it. -/// -public class CopilotChatPlanner -{ - /// - /// High level logger. - /// - private readonly ILogger _logger; - - /// - /// The planner's kernel. - /// - public IKernel Kernel { get; } - - /// - /// Options for the planner. - /// - private readonly PlannerOptions? _plannerOptions; - - /// - /// Gets the pptions for the planner. - /// - public PlannerOptions? PlannerOptions => this._plannerOptions; - - /// - /// Flag to indicate that a variable is unknown and needs to be filled in by the user. - /// This is used to flag any inputs that had dependencies from removed steps. - /// - private const string UnknownVariableFlag = "$???"; - - /// - /// Regex to match variable names from plan parameters. - /// Valid variable names can contain letters, numbers, underscores, and dashes but can't start with a number. - /// Matches: $variableName, $variable_name, $variable-name, $some_variable_Name, $variableName123, $variableName_123, $variableName-123 - /// Does not match: $123variableName, $100 $200 - /// - private const string VariableRegex = @"\$([A-Za-z]+[_-]*[\w]+)"; - - /// - /// Supplemental text to add to the plan goal if PlannerOptions.Type is set to Stepwise. - /// Helps the planner know when to bail out to request additional user input. - /// - private const string StepwisePlannerSupplement = "If you need more information to fulfill this request, return with a request for additional user input."; - - /// - /// Initializes a new instance of the class. - /// - /// The planner's kernel. - public CopilotChatPlanner(IKernel plannerKernel, PlannerOptions? plannerOptions, ILogger logger) - { - this.Kernel = plannerKernel; - this._plannerOptions = plannerOptions; - this._logger = logger; - } - - /// - /// Create a plan for a goal. - /// - /// The goal to create a plan for. - /// Logger from context. - /// The cancellation token. - /// The plan. - public async Task CreatePlanAsync(string goal, ILogger logger, CancellationToken cancellationToken = default) - { - var plannerFunctionsView = this.Kernel.Functions.GetFunctionViews(); - if (plannerFunctionsView.IsNullOrEmpty()) - { - // No functions are available - return an empty plan. - return new Plan(goal); - } - - Plan plan; - - try - { - switch (this._plannerOptions?.Type) - { - case PlanType.Sequential: - plan = await new SequentialPlanner( - this.Kernel, - new SequentialPlannerConfig - { - SemanticMemoryConfig = - { - RelevancyThreshold = this._plannerOptions?.RelevancyThreshold, - }, - // Allow plan to be created with missing functions - AllowMissingFunctions = this._plannerOptions?.ErrorHandling.AllowMissingFunctions ?? false - } - ).CreatePlanAsync(goal, cancellationToken); - break; - default: - plan = await new ActionPlanner(this.Kernel).CreatePlanAsync(goal, cancellationToken); - break; - } - } - catch (SKException) - { - // No relevant functions are available - return an empty plan. - return new Plan(goal); - } - - return this._plannerOptions!.ErrorHandling.AllowMissingFunctions ? this.SanitizePlan(plan, plannerFunctionsView, logger) : plan; - } - - /// - /// Run the stepwise planner. - /// - /// The goal containing user intent and ask context. - /// The context to run the plan in. - /// The cancellation token. - public async Task RunStepwisePlannerAsync(string goal, SKContext context, CancellationToken cancellationToken = default) - { - var config = new StepwisePlannerConfig() - { - MaxTokens = this._plannerOptions?.StepwisePlannerConfig.MaxTokens ?? 2048, - MaxIterations = this._plannerOptions?.StepwisePlannerConfig.MaxIterations ?? 15, - MinIterationTimeMs = this._plannerOptions?.StepwisePlannerConfig.MinIterationTimeMs ?? 1500 - }; - - var sw = Stopwatch.StartNew(); - - try - { - var plan = new StepwisePlanner( - this.Kernel, - config - ).CreatePlan(string.Join("\n", goal, StepwisePlannerSupplement)); - var result = await plan.InvokeAsync(context, cancellationToken: cancellationToken); - - sw.Stop(); - context.Variables.Set("timeTaken", sw.Elapsed.ToString()); - return result; - } - catch (Exception e) - { - this._logger.LogError(e, "Error running stepwise planner"); - throw; - } - } - - /// - /// Scrubs plan of functions not available in planner's kernel - /// and flags any effected input dependencies with '$???' to prompt for user input. - /// Proposed plan object to sanitize. - /// The functions available in the planner's kernel. - /// Logger from context. - /// - private Plan SanitizePlan(Plan plan, IEnumerable availableFunctions, ILogger logger) - { // TODO: [Issue #2256] Re-evaluate this logic once we have a better understanding of how to handle missing functions - List sanitizedSteps = new(); - List availableOutputs = new(); - List unavailableOutputs = new(); - - foreach (var step in plan.Steps) - { - // Check if function exists in planner's kernel - if (this.Kernel.Functions.TryGetFunction(step.PluginName, step.Name, out var function)) - { - availableOutputs.AddRange(step.Outputs); - - // Regex to match variable names - Regex variableRegEx = new(VariableRegex, RegexOptions.Singleline); - - // Check for any inputs that may have dependencies from removed steps - foreach (var input in step.Parameters) - { - // Check if input contains a variable - Match inputVariableMatch = variableRegEx.Match(input.Value); - if (inputVariableMatch.Success) - { - foreach (Capture match in inputVariableMatch.Groups[1].Captures) - { - var inputVariableValue = match.Value; - if (!availableOutputs.Any(output => string.Equals(output, inputVariableValue, StringComparison.OrdinalIgnoreCase))) - { - var overrideValue = - // Use previous step's output if no direct dependency on unavailable functions' outputs - // Else use designated constant for unknowns to prompt for user input - string.Equals("INPUT", input.Key, StringComparison.OrdinalIgnoreCase) - && inputVariableMatch.Groups[1].Captures.Count == 1 - && !unavailableOutputs.Any(output => string.Equals(output, inputVariableValue, StringComparison.OrdinalIgnoreCase)) - ? "$PLAN.RESULT" // TODO: [Issue #2256] Extract constants from Plan class, requires change on kernel team - : UnknownVariableFlag; - step.Parameters.Set(input.Key, Regex.Replace(input.Value, variableRegEx.ToString(), overrideValue)); - } - } - } - } - sanitizedSteps.Add(step); - } - else - { - logger.LogWarning("Function {0} not found in planner's kernel. Removing step from plan.", step.Description); - unavailableOutputs.AddRange(step.Outputs); - } - } - - Plan sanitizedPlan = new(plan.Description, sanitizedSteps.ToArray()); - - // Merge any parameters back into new plan object - foreach (var parameter in plan.Parameters) - { - sanitizedPlan.Parameters[parameter.Key] = parameter.Value; - } - - return sanitizedPlan; - } -} diff --git a/webapi/Plugins/Chat/ExternalInformationPlugin.cs b/webapi/Plugins/Chat/ExternalInformationPlugin.cs deleted file mode 100644 index b76e0936b..000000000 --- a/webapi/Plugins/Chat/ExternalInformationPlugin.cs +++ /dev/null @@ -1,487 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using CopilotChat.WebApi.Models.Response; -using CopilotChat.WebApi.Options; -using CopilotChat.WebApi.Plugins.OpenApi.GitHubPlugin.Model; -using CopilotChat.WebApi.Plugins.OpenApi.JiraPlugin.Model; -using CopilotChat.WebApi.Plugins.Utils; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.IdentityModel.Tokens; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planners; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Basic; - -namespace CopilotChat.WebApi.Plugins.Chat; - -/// -/// This plugin provides the functions to acquire external information. -/// -public class ExternalInformationPlugin -{ - /// - /// High level logger. - /// - private readonly ILogger _logger; - - /// - /// Prompt settings. - /// - private readonly PromptsOptions _promptOptions; - - /// - /// Chat Copilot's planner to gather additional information for the chat context. - /// - private readonly CopilotChatPlanner _planner; - - /// - /// Options for the planner. - /// - public PlannerOptions? PlannerOptions { get; } - - /// - /// Proposed plan to return for approval. - /// - public ProposedPlan? ProposedPlan { get; private set; } - - /// - /// Stepwise thought process to return for view. - /// - public PlanExecutionMetadata? StepwiseThoughtProcess { get; private set; } - - /// - /// Header to indicate plan results. - /// - private const string ResultHeader = "RESULT: "; - - /// - /// Create a new instance of ExternalInformationPlugin. - /// - public ExternalInformationPlugin( - IOptions promptOptions, - CopilotChatPlanner planner, - ILogger logger) - { - this._promptOptions = promptOptions.Value; - this._planner = planner; - this.PlannerOptions = planner.PlannerOptions; - this._logger = logger; - } - - public string FormattedFunctionsString(Plan plan) { return string.Join("; ", this.GetPlanSteps(plan)); } - - /// - /// Invoke planner to generate a new plan or extract relevant additional knowledge. - /// - public async Task InvokePlannerAsync( - string userIntent, - SKContext context, - CancellationToken cancellationToken = default) - { - // TODO: [Issue #2106] Calculate planner and plan token usage - var functions = this._planner.Kernel.Functions.GetFunctionViews(); - if (functions.IsNullOrEmpty()) - { - return string.Empty; - } - - var contextString = this.GetChatContextString(context); - var goal = $"Given the following context, accomplish the user intent.\nContext:\n{contextString}\n{userIntent}"; - - // Run stepwise planner if PlannerOptions.Type == Stepwise - if (this._planner.PlannerOptions?.Type == PlanType.Stepwise) - { - return await this.RunStepwisePlannerAsync(goal, context, cancellationToken); - } - - // Create a plan and set it in context for approval. - Plan? plan = null; - var plannerOptions = this._planner.PlannerOptions ?? new PlannerOptions(); // Use default planner options if planner options are null. - int retriesAvail = plannerOptions.ErrorHandling.AllowRetries - ? plannerOptions.ErrorHandling.MaxRetriesAllowed : 0; - - do - { // TODO: [Issue #2256] Remove InvalidPlan retry logic once Core team stabilizes planner - try - { - plan = await this._planner.CreatePlanAsync(goal, this._logger, cancellationToken); - } - catch (Exception e) - { - if (--retriesAvail >= 0) - { - this._logger.LogWarning("Retrying CreatePlan on error: {0}", e.Message); - continue; - } - throw; - } - } while (plan == null); - - if (plan.Steps.Count > 0) - { - // Merge any variables from ask context into plan parameters as these will be used on plan execution. - // These context variables come from user input, so they are prioritized. - if (plannerOptions.Type == PlanType.Action) - { - // Parameters stored in plan's top level state - this.MergeContextIntoPlan(context.Variables, plan.Parameters); - } - else - { - foreach (var step in plan.Steps) - { - this.MergeContextIntoPlan(context.Variables, step.Parameters); - } - } - - this.ProposedPlan = new ProposedPlan(plan, plannerOptions.Type, PlanState.NoOp, userIntent, context.Variables.Input); - } - - return string.Empty; - } - - public async Task ExecutePlanAsync( - SKContext context, - Plan plan, - CancellationToken cancellationToken = default) - { - // Reload the plan with the planner's kernel so it has full context to be executed - var newPlanContext = this._planner.Kernel.CreateNewContext(context.Variables, this._planner.Kernel.Functions, this._planner.Kernel.LoggerFactory); - string planJson = JsonSerializer.Serialize(plan); - plan = Plan.FromJson(planJson, this._planner.Kernel.Functions); - - // Invoke plan - var functionResult = await plan.InvokeAsync(newPlanContext, null, cancellationToken); - var functionsUsed = $"FUNCTIONS USED: {this.FormattedFunctionsString(plan)}"; - - // TODO: #2581 Account for planner system instructions - int tokenLimit = int.Parse(context.Variables["tokenLimit"], new NumberFormatInfo()) - - TokenUtils.TokenCount(functionsUsed) - - TokenUtils.TokenCount(ResultHeader); - - // The result of the plan may be from an OpenAPI plugin. Attempt to extract JSON from the response. - bool extractJsonFromOpenApi = - this.TryExtractJsonFromOpenApiPlanResult(newPlanContext, newPlanContext.Result, out string planResult); - if (extractJsonFromOpenApi) - { - planResult = this.OptimizeOpenApiPluginJson(planResult, tokenLimit, plan); - } - else - { - // If not, use result of plan execution directly. - planResult = newPlanContext.Variables.Input; - } - - return $"{functionsUsed}\n{ResultHeader}{planResult.Trim()}"; - } - - /// - /// Determines whether to use the stepwise planner result as the bot response, thereby bypassing meta prompt generation and completion. - /// - /// The result obtained from the stepwise planner. - /// - /// True if the stepwise planner result should be used as the bot response, - /// false otherwise. - /// - /// - /// This method checks the following conditions: - /// 1. The plan result is not null, empty, or whitespace. - /// 2. The planner options are specified, and the plan type is set to Stepwise. - /// 3. The UseStepwiseResultAsBotResponse option is enabled. - /// 4. The StepwiseThoughtProcess is not null. - /// - public bool UseStepwiseResultAsBotResponse(string planResult) - { - return !string.IsNullOrWhiteSpace(planResult) - && this.PlannerOptions?.Type == PlanType.Stepwise - && this.PlannerOptions.UseStepwiseResultAsBotResponse - && this.StepwiseThoughtProcess != null; - } - - /// - /// Executes the stepwise planner with a given goal and context, and returns the result along with descriptive text. - /// Also sets any metadata associated with stepwise planner execution. - /// - /// The goal to be achieved by the stepwise planner. - /// The SKContext containing the necessary information for the planner. - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// A formatted string containing the result of the stepwise planner and a supplementary message to guide the model in using the result. - /// - private async Task RunStepwisePlannerAsync(string goal, SKContext context, CancellationToken cancellationToken) - { - var plannerContext = context.Clone(); - var functionResult = await this._planner.RunStepwisePlannerAsync(goal, context, cancellationToken); - - // Populate the execution metadata. - var plannerResult = functionResult.GetValue()?.Trim() ?? string.Empty; - this.StepwiseThoughtProcess = new PlanExecutionMetadata( - plannerContext.Variables["stepsTaken"], - plannerContext.Variables["timeTaken"], - plannerContext.Variables["pluginCount"], - plannerResult); - - // Return empty string if result was not found so it's omitted from the meta prompt. - if (plannerResult.Contains("Result not found, review 'stepsTaken' to see what happened.", StringComparison.OrdinalIgnoreCase)) - { - return string.Empty; - } - - // Parse the steps taken to determine which functions were used. - if (plannerContext.Variables.TryGetValue("stepsTaken", out var stepsTaken)) - { - var steps = JsonSerializer.Deserialize>(stepsTaken); - var functionsUsed = new HashSet(); - steps?.ForEach(step => - { - if (!step.Action.IsNullOrEmpty()) { functionsUsed.Add(step.Action); } - }); - - var planFunctions = string.Join(", ", functionsUsed); - plannerContext.Variables.Set("planFunctions", functionsUsed.Count > 0 ? planFunctions : "N/A"); - } - - // Render the supplement to guide the model in using the result. - var promptTemplateFactory = new BasicPromptTemplateFactory(); - var promptTemplate = promptTemplateFactory.Create(this._promptOptions.StepwisePlannerSupplement, new PromptTemplateConfig()); - var resultSupplement = await promptTemplate.RenderAsync(plannerContext, cancellationToken); - - return $"{resultSupplement}\n\nResult:\n\"{plannerResult}\""; - } - - /// - /// Merge any variables from context into plan parameters. - /// - private void MergeContextIntoPlan(ContextVariables variables, ContextVariables planParams) - { - foreach (var param in planParams) - { - if (param.Key.Equals("INPUT", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (variables.TryGetValue(param.Key, out string? value)) - { - planParams.Set(param.Key, value); - } - } - } - - /// - /// Try to extract json from the planner response as if it were from an OpenAPI plugin. - /// - private bool TryExtractJsonFromOpenApiPlanResult(SKContext context, string openApiPluginResponse, out string json) - { - try - { - JsonNode? jsonNode = JsonNode.Parse(openApiPluginResponse); - string contentType = jsonNode?["contentType"]?.ToString() ?? string.Empty; - if (contentType.StartsWith("application/json", StringComparison.InvariantCultureIgnoreCase)) - { - var content = jsonNode?["content"]?.ToString() ?? string.Empty; - if (!string.IsNullOrWhiteSpace(content)) - { - json = content; - return true; - } - } - } - catch (JsonException) - { - this._logger.LogDebug("Unable to extract JSON from planner response, it is likely not from an OpenAPI plugin."); - } - catch (InvalidOperationException) - { - this._logger.LogDebug("Unable to extract JSON from planner response, it may already be proper JSON."); - } - - json = string.Empty; - - return false; - } - - /// - /// Try to optimize json from the planner response - /// based on token limit - /// - private string OptimizeOpenApiPluginJson(string jsonContent, int tokenLimit, Plan plan) - { - // Remove all new line characters + leading and trailing white space - jsonContent = Regex.Replace(jsonContent.Trim(), @"[\n\r]", string.Empty); - var document = JsonDocument.Parse(jsonContent); - string lastPluginInvoked = plan.Steps[^1].PluginName; - string lastFunctionInvoked = plan.Steps[^1].Name; - bool trimResponse = false; - - // The json will be deserialized based on the response type of the particular operation that was last invoked by the planner - // The response type can be a custom trimmed down json structure, which is useful in staying within the token limit - Type responseType = this.GetOpenApiFunctionResponseType(ref document, ref lastPluginInvoked, ref lastFunctionInvoked, ref trimResponse); - - if (trimResponse) - { - // Deserializing limits the json content to only the fields defined in the respective OpenApi's Model classes - var functionResponse = JsonSerializer.Deserialize(jsonContent, responseType); - jsonContent = functionResponse != null ? JsonSerializer.Serialize(functionResponse) : string.Empty; - document = JsonDocument.Parse(jsonContent); - } - - int jsonContentTokenCount = TokenUtils.TokenCount(jsonContent); - - // Return the JSON content if it does not exceed the token limit - if (jsonContentTokenCount < tokenLimit) - { - return jsonContent; - } - - List itemList = new(); - - // Some APIs will return a JSON response with one property key representing an embedded answer. - // Extract this value for further processing - string resultsDescriptor = string.Empty; - - if (document.RootElement.ValueKind == JsonValueKind.Object) - { - int propertyCount = 0; - foreach (JsonProperty property in document.RootElement.EnumerateObject()) - { - propertyCount++; - } - - if (propertyCount == 1) - { - // Save property name for result interpolation - JsonProperty firstProperty = document.RootElement.EnumerateObject().First(); - tokenLimit -= TokenUtils.TokenCount(firstProperty.Name); - resultsDescriptor = string.Format(CultureInfo.InvariantCulture, "{0}: ", firstProperty.Name); - - // Extract object to be truncated - JsonElement value = firstProperty.Value; - document = JsonDocument.Parse(value.GetRawText()); - } - } - - // Detail Object - // To stay within token limits, attempt to truncate the list of properties - if (document.RootElement.ValueKind == JsonValueKind.Object) - { - foreach (JsonProperty property in document.RootElement.EnumerateObject()) - { - int propertyTokenCount = TokenUtils.TokenCount(property.ToString()); - - if (tokenLimit - propertyTokenCount > 0) - { - itemList.Add(property); - tokenLimit -= propertyTokenCount; - } - else - { - break; - } - } - } - - // Summary (List) Object - // To stay within token limits, attempt to truncate the list of results - if (document.RootElement.ValueKind == JsonValueKind.Array) - { - foreach (JsonElement item in document.RootElement.EnumerateArray()) - { - int itemTokenCount = TokenUtils.TokenCount(item.ToString()); - - if (tokenLimit - itemTokenCount > 0) - { - itemList.Add(item); - tokenLimit -= itemTokenCount; - } - else - { - break; - } - } - } - - return itemList.Count > 0 - ? string.Format(CultureInfo.InvariantCulture, "{0}{1}", resultsDescriptor, JsonSerializer.Serialize(itemList)) - : string.Format(CultureInfo.InvariantCulture, "JSON response from {0} is too large to be consumed at this time.", this._planner.PlannerOptions?.Type == PlanType.Sequential ? "plan" : lastPluginInvoked); - } - - private Type GetOpenApiFunctionResponseType(ref JsonDocument document, ref string lastPluginInvoked, ref string lastFunctionInvoked, ref bool trimResponse) - { - // TODO: [Issue #93] Find a way to determine response type if multiple steps are invoked - Type responseType = typeof(object); // Use a reasonable default response type - - // Different operations under the plugin will return responses as json structures; - // Prune each operation response according to the most important/contextual fields only to avoid going over the token limit - // Check what the last function invoked was and deserialize the JSON content accordingly - if (string.Equals(lastPluginInvoked, "GitHubPlugin", StringComparison.Ordinal)) - { - trimResponse = true; - responseType = this.GetGithubPluginResponseType(ref document); - } - else if (string.Equals(lastPluginInvoked, "JiraPlugin", StringComparison.Ordinal)) - { - trimResponse = true; - responseType = this.GetJiraPluginResponseType(ref document, ref lastFunctionInvoked); - } - - return responseType; - } - - private Type GetGithubPluginResponseType(ref JsonDocument document) - { - return document.RootElement.ValueKind == JsonValueKind.Array ? typeof(PullRequest[]) : typeof(PullRequest); - } - - private Type GetJiraPluginResponseType(ref JsonDocument document, ref string lastFunctionInvoked) - { - if (lastFunctionInvoked == "GetIssue") - { - return document.RootElement.ValueKind == JsonValueKind.Array ? typeof(IssueResponse[]) : typeof(IssueResponse); - } - - return typeof(IssueResponse); - } - - /// - /// Retrieves the steps in a plan that was executed successfully. - /// - /// The plan object. - /// A list of strings representing the successfully executed steps in the plan. - private List GetPlanSteps(Plan plan) - { - List steps = new(); - foreach (var step in plan.Steps) - { - steps.Add($"{step.PluginName}.{step.Name}"); - } - - return steps; - } - - /// - /// Returns a string representation of the chat context, excluding some variables that are only relevant to the ChatPlugin execution context and should be ignored by the planner. - /// This helps clarify the context that is passed to the planner as well as save on tokens. - /// - /// The chat context object that contains the variables and their values. - /// A string with one line per variable, in the format "key: value", except for variables that contain "TokenUsage", "tokenLimit", or "chatId" in their names, which are skipped. - private string GetChatContextString(SKContext context) - { - return string.Join("\n", context.Variables.Where(v => !( - v.Key.Contains("TokenUsage", StringComparison.CurrentCultureIgnoreCase) - || v.Key.Contains("tokenLimit", StringComparison.CurrentCultureIgnoreCase) - || v.Key.Contains("chatId", StringComparison.CurrentCultureIgnoreCase))) - .Select(v => $"{v.Key}: {v.Value}")); - } -} diff --git a/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs b/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs index aa99844fe..9b1c6d4eb 100644 --- a/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs +++ b/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs @@ -11,8 +11,6 @@ using Microsoft.Extensions.Logging; using Microsoft.KernelMemory; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI; -using Microsoft.SemanticKernel.Orchestration; namespace CopilotChat.WebApi.Plugins.Chat; @@ -26,15 +24,15 @@ internal static class SemanticChatMemoryExtractor /// /// The Chat ID. /// The semantic kernel. - /// The Semantic Kernel context. + /// The Semantic Kernel context. /// The prompts options. /// The logger. /// The cancellation token. public static async Task ExtractSemanticChatMemoryAsync( string chatId, IKernelMemory memoryClient, - IKernel kernel, - SKContext context, + Kernel kernel, + KernelArguments kernelArguments, PromptsOptions options, ILogger logger, CancellationToken cancellationToken) @@ -80,21 +78,21 @@ async Task ExtractCognitiveMemoryAsync(string memoryType, st options.ResponseTokenLimit - TokenUtils.TokenCount(memoryPrompt); - var memoryExtractionContext = context.Clone(); - memoryExtractionContext.Variables.Set("tokenLimit", remainingToken.ToString(new NumberFormatInfo())); - memoryExtractionContext.Variables.Set("memoryName", memoryName); - memoryExtractionContext.Variables.Set("format", options.MemoryFormat); - memoryExtractionContext.Variables.Set("knowledgeCutoff", options.KnowledgeCutoffDate); + var memoryExtractionArguments = new KernelArguments(kernelArguments); + memoryExtractionArguments["tokenLimit"] = remainingToken.ToString(new NumberFormatInfo()); + memoryExtractionArguments["memoryName"] = memoryName; + memoryExtractionArguments["format"] = options.MemoryFormat; + memoryExtractionArguments["knowledgeCutoff"] = options.KnowledgeCutoffDate; - var completionFunction = kernel.CreateSemanticFunction(memoryPrompt); + var completionFunction = kernel.CreateFunctionFromPrompt(memoryPrompt); var result = await completionFunction.InvokeAsync( - memoryExtractionContext, - options.ToCompletionSettings(), + kernel, + memoryExtractionArguments, cancellationToken); // Get token usage from ChatCompletion result and add to context // Since there are multiple memory types, total token usage is calculated by cumulating the token usage of each memory type. - TokenUtils.GetFunctionTokenUsage(result, context, logger, $"SystemCognitive_{memoryType}"); + TokenUtils.GetFunctionTokenUsage(result, kernelArguments, logger, $"SystemCognitive_{memoryType}"); SemanticChatMemory memory = SemanticChatMemory.FromJson(result.ToString()); return memory; @@ -131,21 +129,4 @@ await memoryClient.SearchMemoryAsync( } } } - - /// - /// Create a completion settings object for chat response. Parameters are read from the PromptSettings class. - /// - private static OpenAIRequestSettings ToCompletionSettings(this PromptsOptions options) - { - var completionSettings = new OpenAIRequestSettings - { - MaxTokens = options.ResponseTokenLimit, - Temperature = options.ResponseTemperature, - TopP = options.ResponseTopP, - FrequencyPenalty = options.ResponseFrequencyPenalty, - PresencePenalty = options.ResponsePresencePenalty - }; - - return completionSettings; - } } diff --git a/webapi/Plugins/Chat/SemanticMemoryRetriever.cs b/webapi/Plugins/Chat/SemanticMemoryRetriever.cs index fca472e7c..f237c97e7 100644 --- a/webapi/Plugins/Chat/SemanticMemoryRetriever.cs +++ b/webapi/Plugins/Chat/SemanticMemoryRetriever.cs @@ -174,7 +174,7 @@ await this._memoryClient.SearchMemoryAsync( var tokenCount = TokenUtils.TokenCount(result.Memory.Text); if (remainingToken - tokenCount > 0) { - if (result.Citation.Tags.TryGetValue(MemoryTags.TagMemory, out var tag) && tag.Count > 0) + if (result.Memory.Tags.TryGetValue(MemoryTags.TagMemory, out var tag) && tag.Count > 0) { var memoryName = tag.Single()!; var citationSource = CitationSource.FromSemanticMemoryCitation( diff --git a/webapi/Plugins/Utils/AsyncUtils.cs b/webapi/Plugins/Utils/AsyncUtils.cs index 7b71e24f0..1bb48f855 100644 --- a/webapi/Plugins/Utils/AsyncUtils.cs +++ b/webapi/Plugins/Utils/AsyncUtils.cs @@ -2,7 +2,7 @@ using System; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel; namespace CopilotChat.WebApi.Plugins.Utils; @@ -27,7 +27,7 @@ public static async Task SafeInvokeAsync(Func> callback, string fu } catch (Exception ex) { - throw new SKException($"{functionName} failed.", ex); + throw new KernelException($"{functionName} failed.", ex); } } @@ -47,7 +47,7 @@ public static async Task SafeInvokeAsync(Func callback, string functionNam } catch (Exception ex) { - throw new SKException($"{functionName} failed.", ex); + throw new KernelException($"{functionName} failed.", ex); } } } diff --git a/webapi/Plugins/Utils/TokenUtils.cs b/webapi/Plugins/Utils/TokenUtils.cs index dca65c09e..03d45e20a 100644 --- a/webapi/Plugins/Utils/TokenUtils.cs +++ b/webapi/Plugins/Utils/TokenUtils.cs @@ -4,13 +4,10 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text.Json; using Microsoft.Extensions.Logging; -using Microsoft.IdentityModel.Tokens; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.AI.ChatCompletion; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; -using Microsoft.SemanticKernel.Orchestration; -using ChatCompletionContextMessages = Microsoft.SemanticKernel.AI.ChatCompletion.ChatHistory; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; namespace CopilotChat.WebApi.Plugins.Utils; @@ -27,7 +24,6 @@ public static class TokenUtils /// public static readonly Dictionary semanticFunctions = new() { - // TODO: [Issue #2106] Calculate token usage for planner dependencies. { "SystemAudienceExtraction", "audienceExtraction" }, { "SystemIntentExtraction", "userIntentExtraction" }, { "SystemMetaPrompt", "metaPromptTemplate" }, @@ -66,11 +62,11 @@ internal static Dictionary EmptyTokenUsages() /// Gets the total token usage from a Chat or Text Completion result context and adds it as a variable to response context. /// /// Result context from chat model - /// Context maintained during response generation. + /// Context maintained during response generation. /// The logger instance to use for logging errors. /// Name of the function that invoked the chat completion. /// true if token usage is found in result context; otherwise, false. - internal static void GetFunctionTokenUsage(FunctionResult result, SKContext chatContext, ILogger logger, string? functionName = null) + internal static void GetFunctionTokenUsage(FunctionResult result, KernelArguments kernelArguments, ILogger logger, string? functionName = null) { try { @@ -80,15 +76,30 @@ internal static void GetFunctionTokenUsage(FunctionResult result, SKContext chat return; } - var modelResults = result.GetModelResults(); - if (modelResults.IsNullOrEmpty()) + if (result.Metadata is null) + { + logger.LogError("No metadata provided to capture usage details."); + return; + } + + if (!result.Metadata.TryGetValue("Usage", out object? usageObject) || usageObject is null) { logger.LogError("Unable to determine token usage for {0}", functionKey); return; } - var tokenUsage = modelResults!.First().GetResult().Usage.TotalTokens; - chatContext.Variables.Set(functionKey!, tokenUsage.ToString(CultureInfo.InvariantCulture)); + var tokenUsage = 0; + try + { + var jsonObject = JsonSerializer.Deserialize(JsonSerializer.Serialize(usageObject)); + tokenUsage = jsonObject.GetProperty("TotalTokens").GetInt32(); + } + catch (KeyNotFoundException) + { + logger.LogError("Usage details not found in model result."); + } + + kernelArguments[functionKey!] = tokenUsage.ToString(CultureInfo.InvariantCulture); } catch (Exception e) { @@ -114,17 +125,17 @@ internal static int TokenCount(string text) /// /// Author role of the message. /// Content of the message. - internal static int GetContextMessageTokenCount(AuthorRole authorRole, string content) + internal static int GetContextMessageTokenCount(AuthorRole authorRole, string? content) { var tokenCount = authorRole == AuthorRole.System ? TokenCount("\n") : 0; return tokenCount + TokenCount($"role:{authorRole.Label}") + TokenCount($"content:{content}"); } /// - /// Rough token costing of ChatCompletionContextMessages object. + /// Rough token costing of ChatHistory object. /// - /// ChatCompletionContextMessages object to calculate the number of tokens of. - internal static int GetContextMessagesTokenCount(ChatCompletionContextMessages chatHistory) + /// ChatHistory object to calculate the number of tokens of. + internal static int GetContextMessagesTokenCount(ChatHistory chatHistory) { var tokenCount = 0; foreach (var message in chatHistory) diff --git a/webapi/Program.cs b/webapi/Program.cs index 051e0ddcb..bdbf10638 100644 --- a/webapi/Program.cs +++ b/webapi/Program.cs @@ -2,7 +2,6 @@ using System; using System.Diagnostics; -using System.Globalization; using System.Linq; using System.Text.Json; using System.Threading.Tasks; @@ -52,7 +51,6 @@ public static async Task Main(string[] args) builder .AddBotConfig() .AddSemanticKernelServices() - .AddPlannerServices() .AddSemanticMemoryServices(); // Add SignalR as the real time relay service @@ -70,11 +68,6 @@ public static async Task Main(string[] args) // Add named HTTP clients for IHttpClientFactory builder.Services.AddHttpClient(); - builder.Services.AddHttpClient("Plugin", httpClient => - { - int timeout = int.Parse(builder.Configuration["Planner:PluginTimeoutLimitInS"] ?? "100", CultureInfo.InvariantCulture); - httpClient.Timeout = TimeSpan.FromSeconds(timeout); - }); // Add in the rest of the services. builder.Services diff --git a/webapi/README.md b/webapi/README.md index 6da8c9eed..6f597fc99 100644 --- a/webapi/README.md +++ b/webapi/README.md @@ -58,8 +58,8 @@ Running the memory creation pipeline steps in different processes. This means th 1. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:ContentStorageType` to `AzureBlobs`. 2. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:DataIngestion:DistributedOrchestration:QueueType` to `AzureQueue`. -3. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:DataIngestion:VectorDbTypes:0` to `AzureCognitiveSearch`. -4. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:Retrieval:VectorDbType` to `AzureCognitiveSearch`. +3. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:DataIngestion:MemoryDbTypes:0` to `AzureAISearch`. +4. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:Retrieval:MemoryDbType` to `AzureAISearch`. 5. Run the following to set up the authentication to the resources: ```bash @@ -67,8 +67,8 @@ Running the memory creation pipeline steps in different processes. This means th dotnet user-secrets set SemanticMemory:Services:AzureBlobs:ConnectionString [your secret] dotnet user-secrets set SemanticMemory:Services:AzureQueue:Auth ConnectionString # Only needed when running distributed processing dotnet user-secrets set SemanticMemory:Services:AzureQueue:ConnectionString [your secret] # Only needed when running distributed processing - dotnet user-secrets set SemanticMemory:Services:AzureCognitiveSearch:Endpoint [your secret] - dotnet user-secrets set SemanticMemory:Services:AzureCognitiveSearch:APIKey [your secret] + dotnet user-secrets set SemanticMemory:Services:AzureAISearch:Endpoint [your secret] + dotnet user-secrets set SemanticMemory:Services:AzureAISearch:APIKey [your secret] ``` 6. For more information and other options, please refer to the [memorypipeline](../memorypipeline/README.md). @@ -81,27 +81,6 @@ You can also add OpenAI plugins that will be managed by the webapi (as opposed t Please refer to [here](../plugins/README.md) for more details. -## Enabling Sequential Planner - -If you want to use SequentialPlanner (multi-step) instead ActionPlanner (single-step), we recommend using `gpt-4` or `gpt-3.5-turbo` as the planner model. **SequentialPlanner works best with `gpt-4`.** Using `gpt-3.5-turbo` will require using a relevancy filter. - -To enable sequential planner, - -1. In [./webapi/appsettings.json](appsettings.json), set `"Type": "Sequential"` under the `Planner` section. -1. Then, set your preferred Planner model (`gpt-4` or `gpt-3.5-turbo`) under the `AIService` configuration section. -1. If using `gpt-4`, no other changes are required. -1. If using `gpt-3.5-turbo`: change [CopilotChatPlanner.cs](Plugins/Chat/CopilotChatPlanner.cs) to initialize SequentialPlanner with a RelevancyThreshold\*. - - Add `using` statement to top of file: - ``` - using Microsoft.SemanticKernel.Planning.Sequential; - ``` - - The `CreatePlanAsync` method should return the following line if `this._plannerOptions?.Type == "Sequential"` is true: - ``` - return new SequentialPlanner(this.Kernel, new SequentialPlannerConfig { RelevancyThreshold = 0.75 }).CreatePlanAsync(goal); - ``` - \* The `RelevancyThreshold` is a number from 0 to 1 that represents how similar a goal is to a function's name/description/inputs. You want to tune that value when using SequentialPlanner to help keep things scoped while not missing on on things that are relevant or including too many things that really aren't. `0.75` is an arbitrary threshold and we recommend developers play around with this number to see what best fits their scenarios. -1. Restart the `webapi` - Chat Copilot should be now running locally with SequentialPlanner. - ## (Optional) Enabling Cosmos Chat Store. [Azure Cosmos DB](https://learn.microsoft.com/en-us/azure/cosmos-db/introduction) can be used as a persistent chat store for Chat Copilot. Chat stores are used for storing chat sessions, participants, and messages. @@ -246,7 +225,7 @@ Then use a Time chart on the Visual tab. > Though plugins can contain both semantic and native functions, Chat Copilot currently only supports plugins of isolated types due to import limitations, so you must separate your plugins into respective folders for each. -If you wish to load custom plugins into the kernel or planner: +If you wish to load custom plugins into the kernel: 1. Create two new folders under `./Plugins` directory named `./SemanticPlugins` and `./NativePlugins`. There, you can add your custom plugins (synonymous with plugins). 2. Then, comment out the respective options in `appsettings.json`: @@ -261,7 +240,7 @@ If you wish to load custom plugins into the kernel or planner: }, ``` -3. By default, custom plugins are only loaded into planner's kernel for discovery at runtime. If you want to load the plugins into the core chat Kernel, you'll have to add the plugin registration into the `AddSemanticKernelServices` method of `SemanticKernelExtensions.cs`. Uncomment the line with `services.AddKernelSetupHook` and pass in the `RegisterPluginsAsync` hook: +3. If you want to load the plugins into the core chat Kernel, you'll have to add the plugin registration into the `AddSemanticKernelServices` method of `SemanticKernelExtensions.cs`. Uncomment the line with `services.AddKernelSetupHook` and pass in the `RegisterPluginsAsync` hook: ```c# internal static IServiceCollection AddSemanticKernelServices(this IServiceCollection services) @@ -280,11 +259,11 @@ If you wish to load custom plugins into the kernel or planner: If you want to deploy your custom plugins with the webapi, additional configuration is required. You have the following options: -1. **[Recommended]** Create custom setup hooks to import your plugins into the kernel and planner. +1. **[Recommended]** Create custom setup hooks to import your plugins into the kernel. > The default `RegisterPluginsAsync` function uses reflection to import native functions from your custom plugin files. C# reflection is a powerful but slow mechanism that dynamically inspects and invokes types and methods at runtime. It works well for loading a few plugin files, but it can degrade performance and increase memory usage if you have many plugins or complex types. Therefore, we recommend creating your own import function to load your custom plugins manually. This way, you can avoid reflection overhead and have more control over how and when your plugins are loaded. - Create a function to load your custom plugins at build and pass that function as a hook to `AddKernelSetupHook` or `AddPlannerSetupHook` in `SemanticKernelExtensions.cs`. See the [next two sections](#add-custom-setup-to-chat-copilots-kernel) for details on how to do this. This bypasses the need to load the plugins at runtime, and consequently, there's no need to ship the source files for your custom plugins. Remember to comment out the `NativePluginsDirectory` or `SemanticPluginsDirectory` options in `appsettings.json` to prevent any potential pathing errors. + Create a function to load your custom plugins at build and pass that function as a hook to `AddKernelSetupHook` in `SemanticKernelExtensions.cs`. See the [next two sections](#add-custom-setup-to-chat-copilots-kernel) for details on how to do this. This bypasses the need to load the plugins at runtime, and consequently, there's no need to ship the source files for your custom plugins. Remember to comment out the `NativePluginsDirectory` or `SemanticPluginsDirectory` options in `appsettings.json` to prevent any potential pathing errors. Alternatively, @@ -301,26 +280,18 @@ Alternatively, ``` - You will also need to include a line for each skprompt.txt file you have for you plugins to ensure they are included in the build output. - - ```xml - - Always - - ``` - 3. Change the respective directory option to use an absolute path or a different base path, but make sure that the files are accessible from that location. ### Add Custom Setup to Chat Copilot's Kernel -Chat Copilot's Semantic Kernel can be customized with additional plugins or settings by using a custom hook that performs any complimentary setup of the kernel. A custom hook is a delegate that takes an `IServiceProvider` and an `IKernel` as parameters and performs any desired actions on the kernel, such as registering additional plugins, setting kernel options, adding dependency injections, importing data, etc. To use a custom hook, you can pass it as an argument to the `AddKernelSetupHook` call in the `AddSemanticKernelServices` method of `SemanticKernelExtensions.cs`. +Chat Copilot's Semantic Kernel can be customized with additional plugins or settings by using a custom hook that performs any complimentary setup of the kernel. A custom hook is a delegate that takes an `IServiceProvider` and an `Kernel` as parameters and performs any desired actions on the kernel, such as registering additional plugins, setting kernel options, adding dependency injections, importing data, etc. To use a custom hook, you can pass it as an argument to the `AddKernelSetupHook` call in the `AddSemanticKernelServices` method of `SemanticKernelExtensions.cs`. For example, the following code snippet shows how to create a custom hook that registers a plugin called MyPlugin and passes it to `AddKernelSetupHook`: ```c# // Define a custom hook that registers MyPlugin with the kernel -private static Task MyCustomSetupHook(IServiceProvider sp, IKernel kernel) +private static Task MyCustomSetupHook(IServiceProvider sp, Kernel kernel) { // Import your plugin into the kernel with the name "MyPlugin" kernel.ImportFunctions(new MyPlugin(), nameof(MyPlugin)); @@ -346,30 +317,4 @@ internal static IServiceCollection AddSemanticKernelServices(this IServiceCollec return services; } -``` - -### Add Custom Plugin Registration to the Planner's Kernel - -The planner uses a separate kernel instance that can be configured with plugins that are specific to the planning process. Note that these plugins will be persistent across all chat requests. - -To customize the planner's kernel, you can use a custom hook that registers plugins at build time. A custom hook is a delegate that takes an `IServiceProvider` and an `IKernel` as parameters and performs any desired actions on the kernel. By default, the planner will register plugins using `SemanticKernelExtensions.RegisterPluginsAsync` to load files from the `Service.SemanticPluginsDirectory` and `Service.NativePluginsDirectory` option values in `appsettings.json`. - -To use a custom hook, you can pass it as an argument to the `AddPlannerSetupHook` call in the `AddPlannerServices` method of `SemanticKernelExtensions.cs`, which will invoke the hook after the planner's kernel is created. See section above for an example of a custom hook function. - -> Note: This will override the call to `RegisterPluginsAsync`. - -Then in the `AddPlannerServices` method of `SemanticKernelExtensions.cs`, pass your hook into the `services.AddPlannerSetupHook` call: - -```c# - -internal static IServiceCollection AddPlannerServices(this IServiceCollection services) -{ - ... - - // Register any custom plugins with the planner's kernel. - builder.Services.AddPlannerSetupHook(MyCustomSetupHook); - - return services; -} - -``` +``` \ No newline at end of file diff --git a/webapi/Services/AzureContentSafety.cs b/webapi/Services/AzureContentSafety.cs index 507ed3e51..e16094180 100644 --- a/webapi/Services/AzureContentSafety.cs +++ b/webapi/Services/AzureContentSafety.cs @@ -11,7 +11,7 @@ using System.Threading.Tasks; using CopilotChat.WebApi.Models.Response; using Microsoft.AspNetCore.Http; -using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel; namespace CopilotChat.WebApi.Services; @@ -109,13 +109,13 @@ public async Task ImageAnalysisAsync(IFormFile formFile, var body = await response.Content.ReadAsStringAsync(cancellationToken); if (!response.IsSuccessStatusCode || body is null) { - throw new SKException($"[Content Safety] Failed to analyze image. {response.StatusCode}"); + throw new KernelException($"[Content Safety] Failed to analyze image. {response.StatusCode}"); } var result = JsonSerializer.Deserialize(body!); if (result is null) { - throw new SKException($"[Content Safety] Failed to analyze image. Details: {body}"); + throw new KernelException($"[Content Safety] Failed to analyze image. Details: {body}"); } return result; } diff --git a/webapi/Services/MaintenanceMiddleware.cs b/webapi/Services/MaintenanceMiddleware.cs index 6e8983c71..c0e5b5e02 100644 --- a/webapi/Services/MaintenanceMiddleware.cs +++ b/webapi/Services/MaintenanceMiddleware.cs @@ -41,7 +41,7 @@ public MaintenanceMiddleware( this._logger = logger; } - public async Task Invoke(HttpContext ctx, IKernel kernel) + public async Task Invoke(HttpContext ctx, Kernel kernel) { // Skip inspection if _isInMaintenance explicitly false. if (this._isInMaintenance == null || this._isInMaintenance.Value) diff --git a/webapi/Services/SemanticKernelProvider.cs b/webapi/Services/SemanticKernelProvider.cs index 746227d2d..9879d54d7 100644 --- a/webapi/Services/SemanticKernelProvider.cs +++ b/webapi/Services/SemanticKernelProvider.cs @@ -3,19 +3,16 @@ using System; using System.Net.Http; using System.Threading; -using CopilotChat.WebApi.Options; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; -using Microsoft.KernelMemory.MemoryStorage.Qdrant; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI; -using Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; -using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; +using Microsoft.SemanticKernel.Connectors.AzureAISearch; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Connectors.Qdrant; using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Plugins.Memory; namespace CopilotChat.WebApi.Services; @@ -26,40 +23,33 @@ public sealed class SemanticKernelProvider { private static IMemoryStore? _volatileMemoryStore; - private readonly KernelBuilder _builderChat; - private readonly KernelBuilder _builderPlanner; + private readonly IKernelBuilder _builderChat; private readonly MemoryBuilder _builderMemory; public SemanticKernelProvider(IServiceProvider serviceProvider, IConfiguration configuration, IHttpClientFactory httpClientFactory) { this._builderChat = InitializeCompletionKernel(serviceProvider, configuration, httpClientFactory); - this._builderPlanner = InitializePlannerKernel(serviceProvider, configuration, httpClientFactory); this._builderMemory = InitializeMigrationMemory(serviceProvider, configuration, httpClientFactory); } /// /// Produce semantic-kernel with only completion services for chat. /// - public IKernel GetCompletionKernel() => this._builderChat.Build(); - - /// - /// Produce semantic-kernel with only completion services for planner. - /// - public IKernel GetPlannerKernel() => this._builderPlanner.Build(); + public Kernel GetCompletionKernel() => this._builderChat.Build(); /// /// Produce semantic-kernel with kernel memory. /// - public ISemanticTextMemory GetMigrationMemory() => this._builderMemory.Build(); + public ISemanticTextMemory MigrationMemory => this._builderMemory.Build(); - private static KernelBuilder InitializeCompletionKernel( + private static IKernelBuilder InitializeCompletionKernel( IServiceProvider serviceProvider, IConfiguration configuration, IHttpClientFactory httpClientFactory) { - var builder = new KernelBuilder(); + var builder = Kernel.CreateBuilder(); - builder.WithLoggerFactory(serviceProvider.GetRequiredService()); + builder.Services.AddLogging(); var memoryOptions = serviceProvider.GetRequiredService>().Value; @@ -69,7 +59,7 @@ private static KernelBuilder InitializeCompletionKernel( case string y when y.Equals("AzureOpenAIText", StringComparison.OrdinalIgnoreCase): var azureAIOptions = memoryOptions.GetServiceConfig(configuration, "AzureOpenAIText"); #pragma warning disable CA2000 // No need to dispose of HttpClient instances from IHttpClientFactory - builder.WithAzureOpenAIChatCompletionService( + builder.AddAzureOpenAIChatCompletion( azureAIOptions.Deployment, azureAIOptions.Endpoint, azureAIOptions.APIKey, @@ -80,7 +70,7 @@ private static KernelBuilder InitializeCompletionKernel( case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): var openAIOptions = memoryOptions.GetServiceConfig(configuration, "OpenAI"); #pragma warning disable CA2000 // No need to dispose of HttpClient instances from IHttpClientFactory - builder.WithOpenAIChatCompletionService( + builder.AddOpenAIChatCompletion( openAIOptions.TextModel, openAIOptions.APIKey, httpClient: httpClientFactory.CreateClient()); @@ -94,49 +84,6 @@ private static KernelBuilder InitializeCompletionKernel( return builder; } - private static KernelBuilder InitializePlannerKernel( - IServiceProvider serviceProvider, - IConfiguration configuration, - IHttpClientFactory httpClientFactory) - { - var builder = new KernelBuilder(); - - builder.WithLoggerFactory(serviceProvider.GetRequiredService()); - - var memoryOptions = serviceProvider.GetRequiredService>().Value; - var plannerOptions = serviceProvider.GetRequiredService>().Value; - - switch (memoryOptions.TextGeneratorType) - { - case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase): - case string y when y.Equals("AzureOpenAIText", StringComparison.OrdinalIgnoreCase): - var azureAIOptions = memoryOptions.GetServiceConfig(configuration, "AzureOpenAIText"); -#pragma warning disable CA2000 // No need to dispose of HttpClient instances from IHttpClientFactory - builder.WithAzureOpenAIChatCompletionService( - plannerOptions.Model, - azureAIOptions.Endpoint, - azureAIOptions.APIKey, - httpClient: httpClientFactory.CreateClient()); -#pragma warning restore CA2000 - break; - - case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): - var openAIOptions = memoryOptions.GetServiceConfig(configuration, "OpenAI"); -#pragma warning disable CA2000 // No need to dispose of HttpClient instances from IHttpClientFactory - builder.WithOpenAIChatCompletionService( - plannerOptions.Model, - openAIOptions.APIKey, - httpClient: httpClientFactory.CreateClient()); -#pragma warning restore CA2000 - break; - - default: - throw new ArgumentException($"Invalid {nameof(memoryOptions.TextGeneratorType)} value in 'KernelMemory' settings."); - } - - return builder; - } - private static MemoryBuilder InitializeMigrationMemory( IServiceProvider serviceProvider, IConfiguration configuration, @@ -155,7 +102,7 @@ private static MemoryBuilder InitializeMigrationMemory( case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase): var azureAIOptions = memoryOptions.GetServiceConfig(configuration, "AzureOpenAIEmbedding"); #pragma warning disable CA2000 // No need to dispose of HttpClient instances from IHttpClientFactory - builder.WithAzureOpenAITextEmbeddingGenerationService( + builder.WithAzureOpenAITextEmbeddingGeneration( azureAIOptions.Deployment, azureAIOptions.Endpoint, azureAIOptions.APIKey, @@ -166,7 +113,7 @@ private static MemoryBuilder InitializeMigrationMemory( case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): var openAIOptions = memoryOptions.GetServiceConfig(configuration, "OpenAI"); #pragma warning disable CA2000 // No need to dispose of HttpClient instances from IHttpClientFactory - builder.WithOpenAITextEmbeddingGenerationService( + builder.WithOpenAITextEmbeddingGeneration( openAIOptions.EmbeddingModel, openAIOptions.APIKey, httpClient: httpClientFactory.CreateClient()); @@ -180,7 +127,7 @@ private static MemoryBuilder InitializeMigrationMemory( IMemoryStore CreateMemoryStore() { - switch (memoryOptions.Retrieval.VectorDbType) + switch (memoryOptions.Retrieval.MemoryDbType) { case string x when x.Equals("SimpleVectorDb", StringComparison.OrdinalIgnoreCase): // Maintain single instance of volatile memory. @@ -205,12 +152,12 @@ IMemoryStore CreateMemoryStore() qdrantConfig.Endpoint, loggerFactory: serviceProvider.GetRequiredService()); - case string x when x.Equals("AzureCognitiveSearch", StringComparison.OrdinalIgnoreCase): - var acsConfig = memoryOptions.GetServiceConfig(configuration, "AzureCognitiveSearch"); - return new AzureCognitiveSearchMemoryStore(acsConfig.Endpoint, acsConfig.APIKey); + case string x when x.Equals("AzureAISearch", StringComparison.OrdinalIgnoreCase): + var acsConfig = memoryOptions.GetServiceConfig(configuration, "AzureAISearch"); + return new AzureAISearchMemoryStore(acsConfig.Endpoint, acsConfig.APIKey); default: - throw new InvalidOperationException($"Invalid 'VectorDbType' type '{memoryOptions.Retrieval.VectorDbType}'."); + throw new InvalidOperationException($"Invalid 'MemoryDbType' type '{memoryOptions.Retrieval.MemoryDbType}'."); } } } diff --git a/webapi/Storage/CosmosDbContext.cs b/webapi/Storage/CosmosDbContext.cs index 2162b7c4e..29e43d743 100644 --- a/webapi/Storage/CosmosDbContext.cs +++ b/webapi/Storage/CosmosDbContext.cs @@ -56,7 +56,7 @@ public async Task CreateAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } await this._container.CreateItemAsync(entity, new PartitionKey(entity.Partition)); @@ -67,7 +67,7 @@ public async Task DeleteAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } await this._container.DeleteItemAsync(entity.Id, new PartitionKey(entity.Partition)); @@ -97,7 +97,7 @@ public async Task UpsertAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } await this._container.UpsertItemAsync(entity, new PartitionKey(entity.Partition)); diff --git a/webapi/Storage/FileSystemContext.cs b/webapi/Storage/FileSystemContext.cs index 44803f1b8..a84096c7d 100644 --- a/webapi/Storage/FileSystemContext.cs +++ b/webapi/Storage/FileSystemContext.cs @@ -37,7 +37,7 @@ public Task CreateAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } if (this._entities.TryAdd(entity.Id, entity)) @@ -53,7 +53,7 @@ public Task DeleteAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } if (this._entities.TryRemove(entity.Id, out _)) @@ -85,7 +85,7 @@ public Task UpsertAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } if (this._entities.AddOrUpdate(entity.Id, entity, (key, oldValue) => entity) != null) diff --git a/webapi/Storage/Repository.cs b/webapi/Storage/Repository.cs index cce06ae21..7269ceaff 100644 --- a/webapi/Storage/Repository.cs +++ b/webapi/Storage/Repository.cs @@ -29,7 +29,7 @@ public Task CreateAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity ID cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity ID cannot be null or empty."); } return this.StorageContext.CreateAsync(entity); diff --git a/webapi/Storage/VolatileContext.cs b/webapi/Storage/VolatileContext.cs index 122b639d0..6eed96ea9 100644 --- a/webapi/Storage/VolatileContext.cs +++ b/webapi/Storage/VolatileContext.cs @@ -39,7 +39,7 @@ public Task CreateAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } this._entities.TryAdd(entity.Id, entity); @@ -52,7 +52,7 @@ public Task DeleteAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } this._entities.TryRemove(entity.Id, out _); @@ -81,7 +81,7 @@ public Task UpsertAsync(T entity) { if (string.IsNullOrWhiteSpace(entity.Id)) { - throw new ArgumentOutOfRangeException(nameof(entity.Id), "Entity Id cannot be null or empty."); + throw new ArgumentOutOfRangeException(nameof(entity), "Entity Id cannot be null or empty."); } this._entities.AddOrUpdate(entity.Id, entity, (key, oldValue) => entity); diff --git a/webapi/Utilities/PluginUtils.cs b/webapi/Utilities/PluginUtils.cs index b29c44847..58ac661c3 100644 --- a/webapi/Utilities/PluginUtils.cs +++ b/webapi/Utilities/PluginUtils.cs @@ -2,12 +2,11 @@ using System; using CopilotChat.WebApi.Models.Request; -using Microsoft.SemanticKernel.Orchestration; namespace CopilotChat.WebApi.Utilities; /// -/// Converts variables to , inserting some system variables along the way. +/// Converts variables to , inserting some system variables along the way. /// internal static class PluginUtils { diff --git a/webapi/appsettings.json b/webapi/appsettings.json index 4b0f7194e..f11e34665 100644 --- a/webapi/appsettings.json +++ b/webapi/appsettings.json @@ -41,45 +41,6 @@ "Scopes": "access_as_user" // Scopes that the client app requires to access the API } }, - // - // Planner can determine which plugin functions, if any, need to be used to fulfill a user's request. - // https://learn.microsoft.com/en-us/semantic-kernel/concepts-sk/planner - // - Set Planner:Type to "Action" to use the single-step ActionPlanner - // - Set Planner:Type to "Sequential" to enable the multi-step SequentialPlanner - // Note: SequentialPlanner works best with `gpt-4`. See the "Enabling Sequential Planner" section in webapi/README.md for configuration instructions. - // - Set Planner:Type to "Stepwise" to enable MRKL style planning - // - Set Planner:UseStepwiseResultAsBotResponse to "true" to use the result of the planner as the bot response. If false, planner result will be imbedded in meta prompt. - // - This is helpful because the Stepwise Planner returns a sensical chat response as its result. - // - Set Planner:RelevancyThreshold to a decimal between 0 and 1.0. - // - Set Planner:Model to a chat completion model (e.g., gpt-35-turbo, gpt-4). - // - "Planner": { - "Type": "Sequential", - "UseStepwiseResultAsBotResponse": true, - // The minimum relevancy score for a function to be considered. - // Set RelevancyThreshold to a value between 0 and 1 if using the SequentialPlanner or Stepwise planner with gpt-3.5-turbo. - // Ignored when Planner:Type is "Action" - "RelevancyThreshold": "0.25", - "Model": "gpt-35-turbo", // For OpenAI, change to 'gpt-3.5-turbo' (with a period). - // Configuration for the error handling and retry logic. - // - Set AllowRetries to "true" to enable retries. - // - Set AllowMissingFunctions to "true" to allow missing functions in the sequential plan on creation. - // The plan will be created with missing functions as no-op steps. - // - Set MaxRetriesAllowed to the maximum number of retries allowed. If set to 0, no retries will be attempted. - "ErrorHandling": { - "AllowRetries": "true", - "AllowMissingFunctions": "true", - "MaxRetriesAllowed": "3" // Max retries allowed. If set to 0, no retries will be attempted. - }, - // The maximum number of seconds to wait for a response from a plugin. If this is not set, timeout limit will be 100s, which is default timeout setting for HttpClient - // Note: The Service:TimeoutLimitinS option above will take precedence on the broader request lifespan if set. - "PluginTimeoutLimitInS": 100, - "StepwisePlannerConfig": { - "MaxTokens": "2048", - "MaxIterations": "15", - "MinIterationTimeMs": "1500" - } - }, // A list of plugins that will be loaded by the application. // - Name is the NameForHuman of the plugin. // - ManifestDomain is the root domain of the plugin: https://platform.openai.com/docs/plugins/production/domain-verification-and-security @@ -168,9 +129,6 @@ "SystemDescription": "This is a chat between an intelligent AI bot named Copilot and one or more participants. SK stands for Semantic Kernel, the AI platform used to build the bot. The AI was trained on data through 2021 and is not aware of events that have occurred since then. It also has no ability to access data on the Internet, so it should not claim that it can or say that it will go and look things up. Try to be concise with your answers, though it is not required. Knowledge cutoff: {{$knowledgeCutoff}} / Current date: {{TimePlugin.Now}}.", "SystemResponse": "Either return [silence] or provide a response to the last message. ONLY PROVIDE A RESPONSE IF the last message WAS ADDRESSED TO THE 'BOT' OR 'COPILOT'. If it appears the last message was not for you, send [silence] as the bot response.", "InitialBotMessage": "Hello, thank you for democratizing AI's productivity benefits with open source! How can I help you today?", - "ProposedPlanBotMessage": "As an AI language model, my knowledge is based solely on the data that was used to train me, but I can use the following functions to get fresh information: {{$planFunctions}}. Do you agree to proceed?", - "StepwisePlannerSupplement": "This result was obtained using the Stepwise Planner, which used a series of thoughts and actions to fulfill the user intent. The planner attempted to use the following functions to gather necessary information: {{$planFunctions}}.", - "PlanResultsDescription": "This is the result of invoking the functions listed after \"FUNCTIONS USED:\" to retrieve additional information outside of the data you were trained on. This information was retrieved on {{TimePlugin.Now}}. You can use this data to help answer the user's query.", "KnowledgeCutoffDate": "Saturday, January 1, 2022", "SystemAudience": "Below is a chat history between an intelligent AI bot named Copilot with one or more participants.", "SystemAudienceContinuation": "Using the provided chat history, generate a list of names of the participants of this chat. Do not include 'bot' or 'copilot'.The output should be a single rewritten sentence containing only a comma separated list of names. DO NOT offer additional commentary. DO NOT FABRICATE INFORMATION.\nParticipants:", @@ -198,7 +156,6 @@ // Kernel Memory configuration - https://github.com/microsoft/kernel-memory // - ContentStorageType is the storage configuration for memory transfer: "AzureBlobs" or "SimpleFileStorage" // - TextGeneratorType is the AI completion service configuration: "AzureOpenAIText", "AzureOpenAI" or "OpenAI" - // - ImageOcrType is the image OCR configuration: "None" or "AzureFormRecognizer" or "Tesseract" // - DataIngestion is the configuration section for data ingestion pipelines. // - Retrieval is the configuration section for memory retrieval. // - Services is the configuration sections for various memory settings. @@ -206,14 +163,13 @@ "KernelMemory": { "ContentStorageType": "SimpleFileStorage", "TextGeneratorType": "AzureOpenAIText", - "ImageOcrType": "None", // Data ingestion pipelines configuration. // - OrchestrationType is the pipeline orchestration configuration : "InProcess" or "Distributed" // InProcess: in process .NET orchestrator, synchronous/no queues // Distributed: asynchronous queue based orchestrator // - DistributedOrchestration is the detailed configuration for OrchestrationType=Distributed // - EmbeddingGeneratorTypes is the list of embedding generator types - // - VectorDbTypes is the list of vector database types + // - MemoryDbTypes is the list of vector database types "DataIngestion": { "OrchestrationType": "InProcess", // @@ -228,17 +184,19 @@ "AzureOpenAIEmbedding" ], // Vectors can be written to multiple storages, e.g. for data migration, A/B testing, etc. - "VectorDbTypes": [ + "MemoryDbTypes": [ "SimpleVectorDb" - ] + ], + // ImageOcrType is the image OCR configuration: "None", "AzureFormRecognizer" or "Tesseract" + "ImageOcrType": "None" }, // // Memory retrieval configuration - A single EmbeddingGenerator and VectorDb. - // - VectorDbType: Vector database configuration: "SimpleVectorDb" or "AzureCognitiveSearch" or "Qdrant" + // - MemoryDbType: Vector database configuration: "SimpleVectorDb" or "AzureAISearch" or "Qdrant" // - EmbeddingGeneratorType: Embedding generator configuration: "AzureOpenAIEmbedding", "AzureOpenAI" or "OpenAI" // "Retrieval": { - "VectorDbType": "SimpleVectorDb", + "MemoryDbType": "SimpleVectorDb", "EmbeddingGeneratorType": "AzureOpenAIEmbedding" }, // @@ -317,9 +275,9 @@ // - APIKey is the key generated to access the service. // - Endpoint is the service endpoint url. // - "AzureCognitiveSearch": { + "AzureAISearch": { "Auth": "ApiKey", - //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureCognitiveSearch:APIKey" "MY_ACS_KEY" + //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureAISearch:APIKey" "MY_ACS_KEY" "Endpoint": "" }, // diff --git a/webapp/src/libs/hooks/useChat.ts b/webapp/src/libs/hooks/useChat.ts index 703486499..05d017607 100644 --- a/webapp/src/libs/hooks/useChat.ts +++ b/webapp/src/libs/hooks/useChat.ts @@ -42,7 +42,7 @@ export interface GetResponseOptions { messageType: ChatMessageType; value: string; chatId: string; - contextVariables?: IAskVariables[]; + kernelArguments?: IAskVariables[]; processPlan?: boolean; } @@ -108,7 +108,7 @@ export const useChat = () => { } }; - const getResponse = async ({ messageType, value, chatId, contextVariables, processPlan }: GetResponseOptions) => { + const getResponse = async ({ messageType, value, chatId, kernelArguments, processPlan }: GetResponseOptions) => { const chatInput: IChatMessage = { chatId: chatId, timestamp: new Date().getTime(), @@ -135,8 +135,8 @@ export const useChat = () => { ], }; - if (contextVariables) { - ask.variables.push(...contextVariables); + if (kernelArguments) { + ask.variables.push(...kernelArguments); } try { @@ -422,7 +422,7 @@ export const useChat = () => { }; const processPlan = async (chatId: string, planState: PlanState, serializedPlan: string, planGoal?: string) => { - const contextVariables: ContextVariable[] = [ + const kernelArguments: ContextVariable[] = [ { key: 'proposedPlan', value: serializedPlan, @@ -442,7 +442,7 @@ export const useChat = () => { // Send plan back for processing or execution await getResponse({ value: message, - contextVariables, + kernelArguments, messageType: ChatMessageType.Message, chatId: chatId, processPlan: true, diff --git a/webapp/src/redux/features/app/AppState.ts b/webapp/src/redux/features/app/AppState.ts index 027bcad15..5472459dd 100644 --- a/webapp/src/redux/features/app/AppState.ts +++ b/webapp/src/redux/features/app/AppState.ts @@ -66,7 +66,7 @@ export enum FeatureKeys { SimplifiedExperience, PluginsPlannersAndPersonas, AzureContentSafety, - AzureCognitiveSearch, + AzureAISearch, BotAsDocs, MultiUserChat, RLHF, // Reinforcement Learning from Human Feedback @@ -91,9 +91,9 @@ export const Features = { label: 'Azure Content Safety', inactive: true, }, - [FeatureKeys.AzureCognitiveSearch]: { + [FeatureKeys.AzureAISearch]: { enabled: false, - label: 'Azure Cognitive Search', + label: 'Azure AI Search', inactive: true, }, [FeatureKeys.BotAsDocs]: { @@ -127,7 +127,7 @@ export const Settings = [ }, { title: 'Azure AI', - features: [FeatureKeys.AzureContentSafety, FeatureKeys.AzureCognitiveSearch], + features: [FeatureKeys.AzureContentSafety, FeatureKeys.AzureAISearch], stackVertically: true, }, {