Upgrade to Semantic Kernel 1.4 (microsoft#817)

### Motivation and Context Chat Copilot hasn't been upgraded to SK 1 and this hampers development. ### Description Upgrade to SK 1.4 ### Contribution Checklist - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [Contribution Guidelines](https://github.com/microsoft/chat-copilot/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/chat-copilot/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
vlink-group · Feb 27, 2024 · c14f737 · c14f737
1 parent fdef591
commit c14f737
Show file tree

Hide file tree

Showing 53 changed files with 1,082 additions and 1,965 deletions.
diff --git a/README.md b/README.md
@@ -69,11 +69,9 @@ You will need the following items to run the sample:
    - > **IMPORTANT:** For `AzureOpenAI`, if you deployed models `gpt-35-turbo` and `text-embedding-ada-002` with custom names (instead of the default names), also use the parameters:
 
      ```powershell
-     -CompletionModel {DEPLOYMENT_NAME} -EmbeddingModel {DEPLOYMENT_NAME} -PlannerModel {DEPLOYMENT_NAME}
+     -CompletionModel {DEPLOYMENT_NAME} -EmbeddingModel {DEPLOYMENT_NAME}
      ```
 
-     > -PlannerModel name will be the same as -CompletionModel
-
      Open the `.\Configure.ps1` script to see all of the available parameters.
 
 1. Run Chat Copilot locally. This step starts both the backend API and frontend application.
@@ -153,12 +151,9 @@ You will need the following items to run the sample:
                      --endpoint        {AZURE_OPENAI_ENDPOINT} \
                      --apikey          {API_KEY} \
                      --completionmodel {DEPLOYMENT_NAME} \
-                     --plannermodel    {DEPLOYMENT_NAME} \
                      --embeddingmodel  {DEPLOYMENT_NAME}
       ```
 
-      `--plannermodel` will be the same name as `--completionmodel`
-
 1. Run Chat Copilot locally. This step starts both the backend API and frontend application.
 
    ```bash

diff --git a/integration-tests/ChatCopilotIntegrationTests.csproj b/integration-tests/ChatCopilotIntegrationTests.csproj
@@ -14,13 +14,13 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageReference Include="xunit" Version="2.6.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.4">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
+    <PackageReference Include="xunit" Version="2.7.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.7">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="6.0.0">
+    <PackageReference Include="coverlet.collector" Version="6.0.1">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>

diff --git a/memorypipeline/CopilotChatMemoryPipeline.csproj b/memorypipeline/CopilotChatMemoryPipeline.csproj
@@ -14,9 +14,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
-    <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.10.231117.1-preview" />
-    <PackageReference Include="Microsoft.SemanticKernel" Version="1.0.0-beta8" />
+    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
+    <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.29.240219.2" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="1.4.0" />
   </ItemGroup>
 
 </Project>
diff --git a/memorypipeline/README.md b/memorypipeline/README.md
@@ -43,7 +43,7 @@ The memorypipeline is only needed when `SemanticMemory:DataIngestion:Orchestrati
   - RabbitMQ
   - SimpleQueues: stores messages on your local file system.
 - [Vector database](https://learn.microsoft.com/en-us/semantic-kernel/memories/vector-db): storage solution for high-dimensional vectors, aka [embeddings](https://github.com/microsoft/semantic-kernel/blob/main/docs/EMBEDDINGS.md). Available options:
-  - [AzureCognitiveSearch](https://learn.microsoft.com/en-us/azure/search/search-what-is-azure-search)
+  - [AzureAISearch](https://learn.microsoft.com/en-us/azure/search/search-what-is-azure-search)
   - [Qdrant](https://github.com/qdrant/qdrant)
   - SimpleVectorDb
     - TextFile: stores vectors on your local file system.
@@ -72,8 +72,8 @@ The memorypipeline is only needed when `SemanticMemory:DataIngestion:Orchestrati
 2. Find the **Url** under **Overview** and the **key** under **Keys** on the portal.
 3. Run the following to set up the authentication to the resources:
    ```bash
-   dotnet user-secrets set SemanticMemory:Services:AzureCognitiveSearch:Endpoint [your secret]
-   dotnet user-secrets set SemanticMemory:Services:AzureCognitiveSearch:APIKey [your secret]
+   dotnet user-secrets set SemanticMemory:Services:AzureAISearch:Endpoint [your secret]
+   dotnet user-secrets set SemanticMemory:Services:AzureAISearch:APIKey [your secret]
    ```
 
 ##### RabbitMQ

diff --git a/memorypipeline/appsettings.json b/memorypipeline/appsettings.json
@@ -18,7 +18,7 @@
     //      Distributed: asynchronous queue based orchestrator
     // - DistributedOrchestration is the detailed configuration for OrchestrationType=Distributed
     // - EmbeddingGeneratorTypes is the list of embedding generator types
-    // - VectorDbTypes is the list of vector database types
+    // - MemoryDbTypes is the list of vector database types
     "DataIngestion": {
       "OrchestrationType": "Distributed",
       //
@@ -33,17 +33,17 @@
         "AzureOpenAIEmbedding"
       ],
       // Vectors can be written to multiple storages, e.g. for data migration, A/B testing, etc.
-      "VectorDbTypes": [
+      "MemoryDbTypes": [
         "SimpleVectorDb"
       ]
     },
     //
     // Memory retrieval configuration - A single EmbeddingGenerator and VectorDb.
-    // - VectorDbType: Vector database configuration: "SimpleVectorDb" or "AzureCognitiveSearch" or "Qdrant"
+    // - MemoryDbType: Vector database configuration: "SimpleVectorDb" or "AzureAISearch" or "Qdrant"
     // - EmbeddingGeneratorType: Embedding generator configuration: "AzureOpenAIEmbedding" or "OpenAI"
     //
     "Retrieval": {
-      "VectorDbType": "SimpleVectorDb",
+      "MemoryDbType": "SimpleVectorDb",
       "EmbeddingGeneratorType": "AzureOpenAIEmbedding"
     },
     //
@@ -122,9 +122,9 @@
       // - APIKey is the key generated to access the service.
       // - Endpoint is the service endpoint url.
       //
-      "AzureCognitiveSearch": {
+      "AzureAISearch": {
         "Auth": "ApiKey",
-        //"APIKey": "", // dotnet user-secrets set "SemanticMemory:Services:AzureCognitiveSearch:APIKey" "MY_ACS_KEY"
+        //"APIKey": "", // dotnet user-secrets set "SemanticMemory:Services:AzureAISearch:APIKey" "MY_ACS_KEY"
         "Endpoint": ""
       },
       //

diff --git a/scripts/deploy/deploy-azure.ps1 b/scripts/deploy/deploy-azure.ps1
@@ -58,10 +58,10 @@ param(
     # Azure AD cloud instance for authenticating users
     $AzureAdInstance = "https://login.microsoftonline.com",
 
-    [ValidateSet("AzureCognitiveSearch", "Qdrant")]
+    [ValidateSet("AzureAISearch", "Qdrant")]
     [string]
     # What method to use to persist embeddings
-    $MemoryStore = "AzureCognitiveSearch",
+    $MemoryStore = "AzureAISearch",
 
     [switch]
     # Don't deploy Cosmos DB for chat storage - Use volatile memory instead

diff --git a/scripts/deploy/deploy-azure.sh b/scripts/deploy/deploy-azure.sh
@@ -22,7 +22,7 @@ usage() {
     echo "  -i, --instance AZURE_AD_INSTANCE           Azure AD cloud instance for authenticating users"
     echo "                                             (default: \"https://login.microsoftonline.com\")"
     echo "  -ms, --memory-store                        Method to use to persist embeddings. Valid values are"
-    echo "                                             \"AzureCognitiveSearch\" (default) and \"Qdrant\""
+    echo "                                             \"AzureAISearch\" (default) and \"Qdrant\""
     echo "  -nc, --no-cosmos-db                        Don't deploy Cosmos DB for chat storage - Use volatile memory instead"
     echo "  -ns, --no-speech-services                  Don't deploy Speech Services to enable speech as chat input"
     echo "  -ws, --deploy-web-searcher-plugin          Deploy the web searcher plugin"
@@ -184,7 +184,7 @@ az account set -s "$SUBSCRIPTION"
 : "${REGION:="southcentralus"}"
 : "${WEB_APP_SVC_SKU:="B1"}"
 : "${AZURE_AD_INSTANCE:="https://login.microsoftonline.com"}"
-: "${MEMORY_STORE:="AzureCognitiveSearch"}"
+: "${MEMORY_STORE:="AzureAISearch"}"
 : "${NO_COSMOS_DB:=false}"
 : "${NO_SPEECH_SERVICES:=false}"
 : "${DEPLOY_WEB_SEARCHER_PLUGIN:=false}"

diff --git a/scripts/deploy/main.bicep b/scripts/deploy/main.bicep
@@ -37,9 +37,6 @@ param completionModel string = 'gpt-35-turbo'
 @description('Model to use for text embeddings')
 param embeddingModel string = 'text-embedding-ada-002'
 
-@description('Completion model the task planner should use')
-param plannerModel string = 'gpt-35-turbo'
-
 @description('Azure OpenAI endpoint to use (Azure OpenAI only)')
 param aiEndpoint string = ''
 
@@ -67,10 +64,10 @@ param deployCosmosDB bool = true
 
 @description('What method to use to persist embeddings')
 @allowed([
-  'AzureCognitiveSearch'
+  'AzureAISearch'
   'Qdrant'
 ])
-param memoryStore string = 'AzureCognitiveSearch'
+param memoryStore string = 'AzureAISearch'
 
 @description('Whether to deploy Azure Speech Services to enable input by voice')
 param deploySpeechServices bool = true
@@ -206,10 +203,6 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = {
           name: 'Authentication:AzureAd:Scopes'
           value: 'access_as_user'
         }
-        {
-          name: 'Planner:Model'
-          value: plannerModel
-        }
         {
           name: 'ChatStore:Type'
           value: deployCosmosDB ? 'cosmos' : 'volatile'
@@ -311,11 +304,11 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = {
           value: aiService
         }
         {
-          name: 'KernelMemory:DataIngestion:VectorDbTypes:0'
+          name: 'KernelMemory:DataIngestion:MemoryDbTypes:0'
           value: memoryStore
         }
         {
-          name: 'KernelMemory:Retrieval:VectorDbType'
+          name: 'KernelMemory:Retrieval:MemoryDbType'
           value: memoryStore
         }
         {
@@ -343,16 +336,16 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = {
           value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[1].value}'
         }
         {
-          name: 'KernelMemory:Services:AzureCognitiveSearch:Auth'
+          name: 'KernelMemory:Services:AzureAISearch:Auth'
           value: 'ApiKey'
         }
         {
-          name: 'KernelMemory:Services:AzureCognitiveSearch:Endpoint'
-          value: memoryStore == 'AzureCognitiveSearch' ? 'https://${azureCognitiveSearch.name}.search.windows.net' : ''
+          name: 'KernelMemory:Services:AzureAISearch:Endpoint'
+          value: memoryStore == 'AzureAISearch' ? 'https://${azureAISearch.name}.search.windows.net' : ''
         }
         {
-          name: 'KernelMemory:Services:AzureCognitiveSearch:APIKey'
-          value: memoryStore == 'AzureCognitiveSearch' ? azureCognitiveSearch.listAdminKeys().primaryKey : ''
+          name: 'KernelMemory:Services:AzureAISearch:APIKey'
+          value: memoryStore == 'AzureAISearch' ? azureAISearch.listAdminKeys().primaryKey : ''
         }
         {
           name: 'KernelMemory:Services:Qdrant:Endpoint'
@@ -480,7 +473,7 @@ resource appServiceMemoryPipelineConfig 'Microsoft.Web/sites/config@2022-09-01'
         value: aiService
       }
       {
-        name: 'KernelMemory:ImageOcrType'
+        name: 'KernelMemory:DataIngestion:ImageOcrType'
         value: 'AzureFormRecognizer'
       }
       {
@@ -496,11 +489,11 @@ resource appServiceMemoryPipelineConfig 'Microsoft.Web/sites/config@2022-09-01'
         value: aiService
       }
       {
-        name: 'KernelMemory:DataIngestion:VectorDbTypes:0'
+        name: 'KernelMemory:DataIngestion:MemoryDbTypes:0'
         value: memoryStore
       }
       {
-        name: 'KernelMemory:Retrieval:VectorDbType'
+        name: 'KernelMemory:Retrieval:MemoryDbType'
         value: memoryStore
       }
       {
@@ -528,16 +521,16 @@ resource appServiceMemoryPipelineConfig 'Microsoft.Web/sites/config@2022-09-01'
         value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[1].value}'
       }
       {
-        name: 'KernelMemory:Services:AzureCognitiveSearch:Auth'
+        name: 'KernelMemory:Services:AzureAISearch:Auth'
         value: 'ApiKey'
       }
       {
-        name: 'KernelMemory:Services:AzureCognitiveSearch:Endpoint'
-        value: memoryStore == 'AzureCognitiveSearch' ? 'https://${azureCognitiveSearch.name}.search.windows.net' : ''
+        name: 'KernelMemory:Services:AzureAISearch:Endpoint'
+        value: memoryStore == 'AzureAISearch' ? 'https://${azureAISearch.name}.search.windows.net' : ''
       }
       {
-        name: 'KernelMemory:Services:AzureCognitiveSearch:APIKey'
-        value: memoryStore == 'AzureCognitiveSearch' ? azureCognitiveSearch.listAdminKeys().primaryKey : ''
+        name: 'KernelMemory:Services:AzureAISearch:APIKey'
+        value: memoryStore == 'AzureAISearch' ? azureAISearch.listAdminKeys().primaryKey : ''
       }
       {
         name: 'KernelMemory:Services:Qdrant:Endpoint'
@@ -812,7 +805,7 @@ resource appServiceQdrant 'Microsoft.Web/sites@2022-09-01' = if (memoryStore ==
   }
 }
 
-resource azureCognitiveSearch 'Microsoft.Search/searchServices@2022-09-01' = if (memoryStore == 'AzureCognitiveSearch') {
+resource azureAISearch 'Microsoft.Search/searchServices@2022-09-01' = if (memoryStore == 'AzureAISearch') {
   name: 'acs-${uniqueName}'
   location: location
   sku: {