Merge branch 'main' into jayesh-tanna/upload_artifact_v4_v1

Azure · Oct 11, 2024 · 9087ddb · 9087ddb
2 parents a3e7f50 + c7b8161
commit 9087ddb
Show file tree

Hide file tree

Showing 38 changed files with 164 additions and 160 deletions.
diff --git a/.github/test/scripts/check_notebook_output.py b/.github/test/scripts/check_notebook_output.py
@@ -81,6 +81,7 @@
         "google.protobuf.service module is deprecated. RPC implementations should provide code generator plugins "
         "which generate code specific to the RPC implementation. service.py will be removed in Jan 2025"
     ),
+    "from google.protobuf import service as _service",
 ]
 
 with open(full_name, "r") as notebook_file:

diff --git a/cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml b/cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml
@@ -32,7 +32,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
   count_word:
     type: spark
@@ -44,6 +44,6 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
 
diff --git a/cli/jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml b/cli/jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml
@@ -36,7 +36,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
     conf:
       spark.driver.cores: 2
@@ -71,7 +71,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
     conf:
       spark.driver.cores: 2

diff --git a/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml
@@ -22,4 +22,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.2"
+      runtime_version: "3.3"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml
@@ -25,4 +25,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.2"
+      runtime_version: "3.3"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml
@@ -25,4 +25,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.2"
+      runtime_version: "3.3"
diff --git a/cli/jobs/spark/serverless-spark-standalone-default-identity.yml b/cli/jobs/spark/serverless-spark-standalone-default-identity.yml
@@ -31,5 +31,5 @@ args: >-
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.2"
+  runtime_version: "3.3"
 
diff --git a/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml b/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.2"
+  runtime_version: "3.3"
 
diff --git a/cli/jobs/spark/serverless-spark-standalone-user-identity.yml b/cli/jobs/spark/serverless-spark-standalone-user-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.2"
+  runtime_version: "3.3"
 
diff --git a/cli/jobs/spark/setup-attached-resources.sh b/cli/jobs/spark/setup-attached-resources.sh
@@ -27,7 +27,7 @@ az storage account create --name $GEN2_STORAGE_NAME --resource-group $RESOURCE_G
 az storage fs create -n $GEN2_FILE_SYSTEM --account-name $GEN2_STORAGE_NAME
 az synapse workspace create --name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --storage-account $GEN2_STORAGE_NAME --file-system $GEN2_FILE_SYSTEM --sql-admin-login-user $SQL_ADMIN_LOGIN_USER --sql-admin-login-password $RANDOM_STRING --location $LOCATION
 az role assignment create --role "Storage Blob Data Owner" --assignee $AML_USER_MANAGED_ID_OID --scope /subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$GEN2_STORAGE_NAME/blobServices/default/containers/$GEN2_FILE_SYSTEM
-az synapse spark pool create --name $SPARK_POOL_NAME --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --spark-version 3.2 --node-count 3 --node-size Medium --min-node-count 3 --max-node-count 10 --enable-auto-scale true
+az synapse spark pool create --name $SPARK_POOL_NAME --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --spark-version 3.3 --node-count 3 --node-size Medium --min-node-count 3 --max-node-count 10 --enable-auto-scale true
 az synapse workspace firewall-rule create --name allowAll --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --start-ip-address 0.0.0.0 --end-ip-address 255.255.255.255
 
 if [[ "$2" == *"managed-identity"* ]]

diff --git a/...eaidashboard-housing-classification/cli-responsibleaidashboard-housing-classification.yml b/...eaidashboard-housing-classification/cli-responsibleaidashboard-housing-classification.yml
@@ -39,7 +39,7 @@ jobs:
 
   create_rai_job:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_constructor/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_constructor/versions/0.17.0
     limits:
       timeout: 3600
     inputs:
@@ -54,7 +54,7 @@ jobs:
 
   explain_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_explanation/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_explanation/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -63,7 +63,7 @@ jobs:
 
   causal_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_causal/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_causal/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -75,7 +75,7 @@ jobs:
 
   counterfactual_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_counterfactual/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_counterfactual/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -88,14 +88,14 @@ jobs:
     limits:
       timeout: 7200
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_erroranalysis/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_erroranalysis/versions/0.17.0
     inputs:
       rai_insights_dashboard: ${{parent.jobs.create_rai_job.outputs.rai_insights_dashboard}}
       max_depth: 3
 
   gather_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_gather/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_gather/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -107,7 +107,7 @@ jobs:
 
   scorecard_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_score_card/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_score_card/versions/0.17.0
     inputs:
       dashboard: ${{parent.jobs.gather_01.outputs.dashboard}}
       pdf_generation_config:

diff --git a/...bleaidashboard-programmer-regression/cli-responsibleaidashboard-programmer-regression.yml b/...bleaidashboard-programmer-regression/cli-responsibleaidashboard-programmer-regression.yml
@@ -33,7 +33,7 @@ jobs:
 
   create_rai_job:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_constructor/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_constructor/versions/0.17.0
     limits:
       timeout: 3600
     inputs:
@@ -47,7 +47,7 @@ jobs:
 
   explain_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_explanation/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_explanation/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -56,7 +56,7 @@ jobs:
 
   causal_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_causal/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_causal/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -68,7 +68,7 @@ jobs:
 
   counterfactual_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_counterfactual/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_counterfactual/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -81,15 +81,15 @@ jobs:
     limits:
       timeout: 7200
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_erroranalysis/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_erroranalysis/versions/0.17.0
     inputs:
       rai_insights_dashboard: ${{parent.jobs.create_rai_job.outputs.rai_insights_dashboard}}
       max_depth: 3
       filter_features: '["style", "Employer"]'
 
   gather_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_gather/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_insight_gather/versions/0.17.0
     limits:
       timeout: 7200
     inputs:
@@ -101,7 +101,7 @@ jobs:
 
   scorecard_01:
     type: command
-    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_score_card/versions/0.15.0
+    component: azureml://registries/azureml/components/microsoft_azureml_rai_tabular_score_card/versions/0.17.0
     inputs:
       dashboard: ${{parent.jobs.gather_01.outputs.dashboard}}
       pdf_generation_config:

diff --git a/sdk/dotnet/AzureML-Samples-CSharp/Assets/Code/CodeOperations.cs b/sdk/dotnet/AzureML-Samples-CSharp/Assets/Code/CodeOperations.cs
@@ -19,7 +19,7 @@ internal class CodeOperations
     /// <param name="version"></param>
     /// <returns></returns>
     // <GetOrCreateCodeVersionAsync>
-    public static async Task<CodeVersionResource> GetOrCreateCodeVersionAsync(
+    public static async Task<MachineLearningCodeVersionResource> GetOrCreateCodeVersionAsync(
         ArmClient armClient,
         ResourceGroupResource resourceGroup,
         string workspaceName,
@@ -32,13 +32,13 @@ public static async Task<CodeVersionResource> GetOrCreateCodeVersionAsync(
 
         string resourceId = $"{ws.Id}/codes/{codeName}";
         var id = new ResourceIdentifier(resourceId);
-        CodeContainerResource codeContainerDataResource = armClient.GetCodeContainerResource(id);
+        MachineLearningCodeContainerResource codeContainerDataResource = armClient.GetMachineLearningCodeContainerResource(id);
 
-        CodeVersionProperties properties = new CodeVersionProperties { CodeUri = new Uri(codeUri) };
-        CodeVersionData data = new CodeVersionData(properties);
+        MachineLearningCodeVersionProperties properties = new MachineLearningCodeVersionProperties { CodeUri = new Uri(codeUri) };
+        MachineLearningCodeVersionData data = new MachineLearningCodeVersionData(properties);
 
-        ArmOperation<CodeVersionResource> CodeVersionResourceOperation = await codeContainerDataResource.GetCodeVersions().CreateOrUpdateAsync(WaitUntil.Completed, version, data);
-        CodeVersionResource codeVersionResource = CodeVersionResourceOperation.Value;
+        ArmOperation<MachineLearningCodeVersionResource> CodeVersionResourceOperation = await codeContainerDataResource.GetMachineLearningCodeVersions().CreateOrUpdateAsync(WaitUntil.Completed, version, data);
+        MachineLearningCodeVersionResource codeVersionResource = CodeVersionResourceOperation.Value;
         Console.WriteLine($"codeVersionResource {codeVersionResource.Data.Id} created.");
 
         return codeVersionResource;

diff --git a/sdk/dotnet/AzureML-Samples-CSharp/Assets/Component/ComponentOperations.cs b/sdk/dotnet/AzureML-Samples-CSharp/Assets/Component/ComponentOperations.cs
@@ -20,7 +20,7 @@ internal class ComponentOperations
     /// <param name="version"></param>
     /// <returns></returns>
     // <GetOrCreateComponentVersionAsync>
-    public static async Task<ComponentVersionResource> GetOrCreateComponentVersionAsync(
+    public static async Task<MachineLearningComponentVersionResource> GetOrCreateComponentVersionAsync(
         ArmClient armClient,
         ResourceGroupResource resourceGroup,
         string workspaceName,
@@ -34,7 +34,7 @@ public static async Task<ComponentVersionResource> GetOrCreateComponentVersionAs
 
         string resourceId = $"{ws.Id}/components/{componentName}";
         var id = new ResourceIdentifier(resourceId);
-        ComponentContainerResource componentContainerResource = armClient.GetComponentContainerResource(id);
+        MachineLearningComponentContainerResource componentContainerResource = armClient.GetMachineLearningComponentContainerResource(id);
 
         JObject jsonObject = JObject.Parse(@"{
   '$schema': 'https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json',
@@ -80,11 +80,11 @@ public static async Task<ComponentVersionResource> GetOrCreateComponentVersionAs
   },
 }");
 
-        ComponentVersionProperties properties = new ComponentVersionProperties { ComponentSpec = new BinaryData(jsonObject.ToString()) };
-        ComponentVersionData data = new ComponentVersionData(properties);
+        MachineLearningComponentVersionProperties properties = new MachineLearningComponentVersionProperties { ComponentSpec = new BinaryData(jsonObject.ToString()) };
+        MachineLearningComponentVersionData data = new MachineLearningComponentVersionData(properties);
 
-        ArmOperation<ComponentVersionResource> componentVersionResourceOperation = await componentContainerResource.GetComponentVersions().CreateOrUpdateAsync(WaitUntil.Completed, version, data);
-        ComponentVersionResource componentVersionResource = componentVersionResourceOperation.Value;
+        ArmOperation<MachineLearningComponentVersionResource> componentVersionResourceOperation = await componentContainerResource.GetMachineLearningComponentVersions().CreateOrUpdateAsync(WaitUntil.Completed, version, data);
+        MachineLearningComponentVersionResource componentVersionResource = componentVersionResourceOperation.Value;
         Console.WriteLine($"ComponentVersionResource {componentVersionResource.Id} created.");
         return componentVersionResource;
     }
@@ -102,7 +102,7 @@ public static async Task<ComponentVersionResource> GetOrCreateComponentVersionAs
     /// <param name="version"></param>
     /// <returns></returns>
     // <GetOrCreateComponentVersion_Pipeline_Async>
-    public static async Task<ComponentVersionResource> GetOrCreateComponentVersion_Pipeline_Async(
+    public static async Task<MachineLearningComponentVersionResource> GetOrCreateComponentVersion_Pipeline_Async(
         ArmClient armClient,
         ResourceGroupResource resourceGroup,
         string workspaceName,
@@ -116,7 +116,7 @@ public static async Task<ComponentVersionResource> GetOrCreateComponentVersion_P
 
         string resourceId = $"{ws.Id}/components/{componentName}";
         var id = new ResourceIdentifier(resourceId);
-        ComponentContainerResource componentContainerResource = armClient.GetComponentContainerResource(id);
+        MachineLearningComponentContainerResource componentContainerResource = armClient.GetMachineLearningComponentContainerResource(id);
 
         JObject jsonObject = JObject.Parse(@"{
   '$schema': 'https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json',
@@ -150,11 +150,11 @@ public static async Task<ComponentVersionResource> GetOrCreateComponentVersion_P
   },
 }");
 
-        ComponentVersionProperties properties = new ComponentVersionProperties { ComponentSpec = new BinaryData(jsonObject.ToString()) };
-        ComponentVersionData data = new ComponentVersionData(properties);
+        MachineLearningComponentVersionProperties properties = new MachineLearningComponentVersionProperties { ComponentSpec = new BinaryData(jsonObject.ToString()) };
+        MachineLearningComponentVersionData data = new MachineLearningComponentVersionData(properties);
 
-        ArmOperation<ComponentVersionResource> componentVersionResourceOperation = await componentContainerResource.GetComponentVersions().CreateOrUpdateAsync(WaitUntil.Completed, version, data);
-        ComponentVersionResource componentVersionResource = componentVersionResourceOperation.Value;
+        ArmOperation<MachineLearningComponentVersionResource> componentVersionResourceOperation = await componentContainerResource.GetMachineLearningComponentVersions().CreateOrUpdateAsync(WaitUntil.Completed, version, data);
+        MachineLearningComponentVersionResource componentVersionResource = componentVersionResourceOperation.Value;
         Console.WriteLine($"ComponentVersionResource {componentVersionResource.Id} created.");
         return componentVersionResource;
     }