Merge remote-tracking branch 'origin/2.x' into node-discovery-backpor…

…t-15131-to-2.x
SwethaGuptha · Sep 12, 2024 · 924ecd8 · 924ecd8
2 parents b7660db + 1074c71
commit 924ecd8
Show file tree

Hide file tree

Showing 127 changed files with 5,883 additions and 615 deletions.
diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
@@ -32,7 +32,10 @@ jobs:
         if: runner.os == 'macos'
         continue-on-error: true
         run: |
-          brew install docker colima coreutils
+          # Force QEMU 9.0.2 usage
+          curl https://raw.githubusercontent.com/Homebrew/homebrew-core/f1a9cf104a9a51779c7a532b658c490f69974839/Formula/q/qemu.rb > qemu.rb
+          brew install qemu.rb
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install docker colima coreutils
           gtimeout 15m colima start
         shell: bash
       - name: Run Gradle (assemble)

diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml
@@ -59,7 +59,7 @@ jobs:
           sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE }}
@@ -86,7 +86,7 @@ jobs:
           sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE_X }}
@@ -114,7 +114,7 @@ jobs:
           sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ steps.github_app_token.outputs.token }}
           base: main

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,9 +5,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
+- [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651))
 
 ### Dependencies
 - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))
+- Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684))
+- Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863))
+- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862))
 
 ### Changed
 
@@ -17,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737))
 
 ### Security
 

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -20,7 +20,7 @@ woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.13.1
 guava             = 32.1.1-jre
-protobuf          = 3.22.3
+protobuf          = 3.25.4
 jakarta_annotation = 1.3.5
 google_http_client = 1.44.1
 tdigest           = 3.2

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
+distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
@@ -62,7 +62,7 @@ dependencies {
   api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
   api 'com.microsoft.azure:msal4j:1.17.0'
-  api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
+  api 'com.nimbusds:oauth2-oidc-sdk:11.19.1'
   api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api 'com.nimbusds:content-type:2.3'
   api 'com.nimbusds:lang-tag:1.7'

diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1
@@ -0,0 +1 @@
+58db85a807a56ae76baffa519772271ad5808195
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
@@ -148,9 +148,6 @@ thirdPartyAudit {
     'com.google.appengine.api.urlfetch.HTTPResponse',
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
-    'com.google.protobuf.MapFieldBuilder',
-    'com.google.protobuf.MapFieldBuilder$Converter',
-    'com.google.protobuf.MapFieldReflectionAccessor',
     'com.google.protobuf.util.JsonFormat',
     'com.google.protobuf.util.JsonFormat$Parser',
     'com.google.protobuf.util.JsonFormat$Printer',

diff --git a/release-notes/opensearch.release-notes-2.17.0.md b/release-notes/opensearch.release-notes-2.17.0.md
@@ -51,6 +51,8 @@
 - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471))
 - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131))
 - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
+- Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478))
+- [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml
@@ -25,38 +25,40 @@ setup:
         snapshot: test_snapshot
         wait_for_completion: true
 
-  - do:
-      snapshot.status:
-        repository: test_repo_status_1
-        snapshot: test_snapshot
-
-  - is_true: snapshots
-  - match: { snapshots.0.snapshot: test_snapshot }
-  - match: { snapshots.0.state: SUCCESS }
-  - gt: { snapshots.0.stats.incremental.file_count: 0 }
-  - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
-  - gt: { snapshots.0.stats.total.file_count: 0 }
-  - gt: { snapshots.0.stats.total.size_in_bytes: 0 }
-  - is_true: snapshots.0.stats.start_time_in_millis
-## fast in memory snapshots can take less than one millisecond to complete.
-  - gte: { snapshots.0.stats.time_in_millis: 0 }
-
----
-"Get missing snapshot status throws an exception":
-
-  - do:
-      catch: /snapshot_missing_exception.+ is missing/
-      snapshot.status:
-        repository: test_repo_status_1
-        snapshot: test_nonexistent_snapshot
-
----
-"Get missing snapshot status succeeds when ignoreUnavailable is true":
-
-  - do:
-      snapshot.status:
-        repository: test_repo_status_1
-        snapshot: test_nonexistent_snapshot
-        ignore_unavailable: true
+# TODO: fix and unmute tests
 
-  - is_true: snapshots
+#  - do:
+#      snapshot.status:
+#        repository: test_repo_status_1
+#        snapshot: test_snapshot
+#
+#  - is_true: snapshots
+#  - match: { snapshots.0.snapshot: test_snapshot }
+#  - match: { snapshots.0.state: SUCCESS }
+#  - gt: { snapshots.0.stats.incremental.file_count: 0 }
+#  - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
+#  - gt: { snapshots.0.stats.total.file_count: 0 }
+#  - gt: { snapshots.0.stats.total.size_in_bytes: 0 }
+#  - is_true: snapshots.0.stats.start_time_in_millis
+### fast in memory snapshots can take less than one millisecond to complete.
+#  - gte: { snapshots.0.stats.time_in_millis: 0 }
+#
+#---
+#"Get missing snapshot status throws an exception":
+#
+#  - do:
+#      catch: /snapshot_missing_exception.+ is missing/
+#      snapshot.status:
+#        repository: test_repo_status_1
+#        snapshot: test_nonexistent_snapshot
+#
+#---
+#"Get missing snapshot status succeeds when ignoreUnavailable is true":
+#
+#  - do:
+#      snapshot.status:
+#        repository: test_repo_status_1
+#        snapshot: test_nonexistent_snapshot
+#        ignore_unavailable: true
+#
+#  - is_true: snapshots
diff --git a/server/licenses/protobuf-java-3.22.3.jar.sha1 b/server/licenses/protobuf-java-3.22.3.jar.sha1
diff --git a/server/licenses/protobuf-java-3.25.4.jar.sha1 b/server/licenses/protobuf-java-3.25.4.jar.sha1
@@ -0,0 +1 @@
+43fcb86e4a411516c7fc681450f1516de0b862a2
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java
@@ -37,6 +37,8 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.cluster.health.ClusterIndexHealth;
+import org.opensearch.cluster.health.ClusterShardHealth;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.service.ClusterService;
@@ -49,6 +51,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -439,4 +442,164 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
             completionFuture.actionGet(TimeValue.timeValueSeconds(30));
         }
     }
+
+    public void testHealthWithClusterLevelAppliedAtTransportLayer() {
+        createIndex(
+            "test1",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        ensureGreen();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
+        assertTrue(healthResponse.getIndices().isEmpty());
+        assertEquals(1, healthResponse.getActiveShards());
+        assertEquals(1, healthResponse.getActivePrimaryShards());
+        assertEquals(0, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+    }
+
+    public void testHealthWithIndicesLevelAppliedAtTransportLayer() {
+        createIndex(
+            "test1",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        ensureGreen();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setLevel("indices")
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
+
+        assertEquals(1, healthResponse.getActiveShards());
+        assertEquals(1, healthResponse.getActivePrimaryShards());
+        assertEquals(0, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+
+        Map<String, ClusterIndexHealth> indices = healthResponse.getIndices();
+        assertFalse(indices.isEmpty());
+        assertEquals(1, indices.size());
+        for (Map.Entry<String, ClusterIndexHealth> indicesHealth : indices.entrySet()) {
+            String indexName = indicesHealth.getKey();
+            assertEquals("test1", indexName);
+            ClusterIndexHealth indicesHealthValue = indicesHealth.getValue();
+            assertEquals(1, indicesHealthValue.getActiveShards());
+            assertEquals(1, indicesHealthValue.getActivePrimaryShards());
+            assertEquals(0, indicesHealthValue.getInitializingShards());
+            assertEquals(0, indicesHealthValue.getUnassignedShards());
+            assertEquals(0, indicesHealthValue.getDelayedUnassignedShards());
+            assertEquals(0, indicesHealthValue.getRelocatingShards());
+            assertEquals(ClusterHealthStatus.GREEN, indicesHealthValue.getStatus());
+            assertTrue(indicesHealthValue.getShards().isEmpty());
+        }
+    }
+
+    public void testHealthWithShardLevelAppliedAtTransportLayer() {
+        int dataNodes = internalCluster().getDataNodeNames().size();
+        int greenClusterReplicaCount = dataNodes - 1;
+        int yellowClusterReplicaCount = dataNodes;
+
+        createIndex(
+            "test1",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount)
+                .build()
+        );
+        ensureGreen(TimeValue.timeValueSeconds(120), "test1");
+        createIndex(
+            "test2",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount)
+                .build()
+        );
+        ensureGreen(TimeValue.timeValueSeconds(120));
+        client().admin()
+            .indices()
+            .prepareUpdateSettings()
+            .setIndices("test2")
+            .setSettings(Settings.builder().put("index.number_of_replicas", yellowClusterReplicaCount).build())
+            .execute()
+            .actionGet();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setLevel("shards")
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.YELLOW, healthResponse.getStatus());
+
+        assertEquals(2 * dataNodes, healthResponse.getActiveShards());
+        assertEquals(2, healthResponse.getActivePrimaryShards());
+        assertEquals(1, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+
+        Map<String, ClusterIndexHealth> indices = healthResponse.getIndices();
+        assertFalse(indices.isEmpty());
+        assertEquals(2, indices.size());
+        for (Map.Entry<String, ClusterIndexHealth> indicesHealth : indices.entrySet()) {
+            String indexName = indicesHealth.getKey();
+            boolean indexHasMoreReplicas = indexName.equals("test2");
+            ClusterIndexHealth indicesHealthValue = indicesHealth.getValue();
+            assertEquals(dataNodes, indicesHealthValue.getActiveShards());
+            assertEquals(1, indicesHealthValue.getActivePrimaryShards());
+            assertEquals(0, indicesHealthValue.getInitializingShards());
+            assertEquals(indexHasMoreReplicas ? 1 : 0, indicesHealthValue.getUnassignedShards());
+            assertEquals(0, indicesHealthValue.getDelayedUnassignedShards());
+            assertEquals(0, indicesHealthValue.getRelocatingShards());
+            assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, indicesHealthValue.getStatus());
+            Map<Integer, ClusterShardHealth> shards = indicesHealthValue.getShards();
+            assertFalse(shards.isEmpty());
+            assertEquals(1, shards.size());
+            for (Map.Entry<Integer, ClusterShardHealth> shardHealth : shards.entrySet()) {
+                ClusterShardHealth clusterShardHealth = shardHealth.getValue();
+                assertEquals(dataNodes, clusterShardHealth.getActiveShards());
+                assertEquals(indexHasMoreReplicas ? 1 : 0, clusterShardHealth.getUnassignedShards());
+                assertEquals(0, clusterShardHealth.getDelayedUnassignedShards());
+                assertEquals(0, clusterShardHealth.getRelocatingShards());
+                assertEquals(0, clusterShardHealth.getInitializingShards());
+                assertTrue(clusterShardHealth.isPrimaryActive());
+                assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, clusterShardHealth.getStatus());
+            }
+        }
+    }
+
+    public void testHealthWithAwarenessAttributesLevelAppliedAtTransportLayer() {
+        createIndex(
+            "test1",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        ensureGreen();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setLevel("awareness_attributes")
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
+        assertTrue(healthResponse.getIndices().isEmpty());
+        assertNotNull(healthResponse.getClusterAwarenessHealth());
+        assertEquals(1, healthResponse.getActiveShards());
+        assertEquals(1, healthResponse.getActivePrimaryShards());
+        assertEquals(0, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+    }
 }