Merge branch 'main' into its_with_publication

.ci/bwcVersions

@@ -38,3 +38,4 @@ BWC_VERSION:
   - "2.16.0"
   - "2.16.1"
   - "2.17.0"
+  - "2.18.0"

.github/workflows/assemble.yml

@@ -30,11 +30,11 @@ jobs:
       - name: Setup docker (missing on MacOS)
         id: setup_docker
         if: runner.os == 'macos'
-        uses: douglascamata/setup-docker-macos-action@main
         continue-on-error: true
-        with:
-          upgrade-qemu: true
-          colima:  v0.6.8
+        run: |
+          brew install docker colima coreutils
+          gtimeout 15m colima start
+        shell: bash
       - name: Run Gradle (assemble)
         if: runner.os == 'macos' && steps.setup_docker.outcome != 'success'
         run: |

CHANGELOG.md

@@ -47,17 +47,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291)))
 - Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409))
 - Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557))
+- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
 - Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568))
 - [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218))
 - Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010))
 - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508))
 - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131))
+- Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705))
 - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471))
+- Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478))
 - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
+- [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145))
+- Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292))
+- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))
-- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.16.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205))
+- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.17.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205), [#15580](https://github.com/opensearch-project/OpenSearch/pull/15580))
 - OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998))
 - Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420))
 - Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997))
@@ -81,6 +87,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419))
 - Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423))
 - Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469))
+- Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))
+- Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684))
 
 ### Changed
 - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
@@ -108,6 +116,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394))
 - Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069))
 - Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607))
+- Fix remote snapshot file_cache exceeding capacity ([#15077](https://github.com/opensearch-project/OpenSearch/pull/15077))
 
 ### Security
 

buildSrc/version.properties

@@ -20,7 +20,7 @@ woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.13.1
 guava             = 32.1.1-jre
-protobuf          = 3.22.3
+protobuf          = 3.25.4
 jakarta_annotation = 1.3.5
 google_http_client = 1.44.1
 tdigest           = 3.3

libs/core/src/main/java/org/opensearch/Version.java

@@ -109,6 +109,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1);
+    public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version CURRENT = V_3_0_0;
 

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java

@@ -23,6 +23,7 @@
 import org.opensearch.index.query.AbstractQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.query.WithFieldName;
 import org.opensearch.plugin.correlation.core.index.mapper.VectorFieldMapper;
 
 import java.io.IOException;
@@ -36,7 +37,7 @@
  *
  * @opensearch.internal
  */
-public class CorrelationQueryBuilder extends AbstractQueryBuilder<CorrelationQueryBuilder> {
+public class CorrelationQueryBuilder extends AbstractQueryBuilder<CorrelationQueryBuilder> implements WithFieldName {
 
     private static final Logger log = LogManager.getLogger(CorrelationQueryBuilder.class);
     protected static final ParseField VECTOR_FIELD = new ParseField("vector");
@@ -205,6 +206,7 @@ public void setFieldName(String fieldName) {
      * get field name
      * @return field name
      */
+    @Override
     public String fieldName() {
         return fieldName;
     }

plugins/repository-azure/build.gradle

@@ -57,7 +57,7 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
-  api 'com.azure:azure-identity:1.13.0'
+  api 'com.azure:azure-identity:1.13.2'
   // Start of transitive dependencies for azure-identity
   api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"

plugins/repository-azure/licenses/azure-identity-1.13.2.jar.sha1

@@ -0,0 +1 @@
+50a1daef3eb5c6ab2e1351a3e3f5a7649a8fe464

plugins/repository-gcs/build.gradle

@@ -149,9 +149,6 @@ thirdPartyAudit {
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
     'com.google.auth.oauth2.GdchCredentials',
-    'com.google.protobuf.MapFieldBuilder',
-    'com.google.protobuf.MapFieldBuilder$Converter',
-    'com.google.protobuf.MapFieldReflectionAccessor',
     'com.google.protobuf.util.JsonFormat',
     'com.google.protobuf.util.JsonFormat$Parser',
     'com.google.protobuf.util.JsonFormat$Printer',

plugins/repository-hdfs/build.gradle

@@ -76,7 +76,7 @@ dependencies {
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
   api 'org.apache.commons:commons-configuration2:2.11.0'
   api "commons-io:commons-io:${versions.commonsio}"
-  api 'org.apache.commons:commons-lang3:3.16.0'
+  api 'org.apache.commons:commons-lang3:3.17.0'
   implementation 'com.google.re2j:re2j:1.7'
   api 'javax.servlet:servlet-api:2.5'
   api "org.slf4j:slf4j-api:${versions.slf4j}"

plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1

@@ -0,0 +1 @@
+b17d2136f0460dcc0d2016ceefca8723bdf4ee70

server/licenses/protobuf-java-3.25.4.jar.sha1

@@ -0,0 +1 @@
+43fcb86e4a411516c7fc681450f1516de0b862a2

server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java

@@ -37,6 +37,8 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.cluster.health.ClusterIndexHealth;
+import org.opensearch.cluster.health.ClusterShardHealth;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.service.ClusterService;
@@ -49,6 +51,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -439,4 +442,164 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
             completionFuture.actionGet(TimeValue.timeValueSeconds(30));
         }
     }
+
+    public void testHealthWithClusterLevelAppliedAtTransportLayer() {
+        createIndex(
+            "test1",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        ensureGreen();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
+        assertTrue(healthResponse.getIndices().isEmpty());
+        assertEquals(1, healthResponse.getActiveShards());
+        assertEquals(1, healthResponse.getActivePrimaryShards());
+        assertEquals(0, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+    }
+
+    public void testHealthWithIndicesLevelAppliedAtTransportLayer() {
+        createIndex(
+            "test1",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        ensureGreen();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setLevel("indices")
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
+
+        assertEquals(1, healthResponse.getActiveShards());
+        assertEquals(1, healthResponse.getActivePrimaryShards());
+        assertEquals(0, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+
+        Map<String, ClusterIndexHealth> indices = healthResponse.getIndices();
+        assertFalse(indices.isEmpty());
+        assertEquals(1, indices.size());
+        for (Map.Entry<String, ClusterIndexHealth> indicesHealth : indices.entrySet()) {
+            String indexName = indicesHealth.getKey();
+            assertEquals("test1", indexName);
+            ClusterIndexHealth indicesHealthValue = indicesHealth.getValue();
+            assertEquals(1, indicesHealthValue.getActiveShards());
+            assertEquals(1, indicesHealthValue.getActivePrimaryShards());
+            assertEquals(0, indicesHealthValue.getInitializingShards());
+            assertEquals(0, indicesHealthValue.getUnassignedShards());
+            assertEquals(0, indicesHealthValue.getDelayedUnassignedShards());
+            assertEquals(0, indicesHealthValue.getRelocatingShards());
+            assertEquals(ClusterHealthStatus.GREEN, indicesHealthValue.getStatus());
+            assertTrue(indicesHealthValue.getShards().isEmpty());
+        }
+    }
+
+    public void testHealthWithShardLevelAppliedAtTransportLayer() {
+        int dataNodes = internalCluster().getDataNodeNames().size();
+        int greenClusterReplicaCount = dataNodes - 1;
+        int yellowClusterReplicaCount = dataNodes;
+
+        createIndex(
+            "test1",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount)
+                .build()
+        );
+        ensureGreen(TimeValue.timeValueSeconds(120), "test1");
+        createIndex(
+            "test2",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount)
+                .build()
+        );
+        ensureGreen(TimeValue.timeValueSeconds(120));
+        client().admin()
+            .indices()
+            .prepareUpdateSettings()
+            .setIndices("test2")
+            .setSettings(Settings.builder().put("index.number_of_replicas", yellowClusterReplicaCount).build())
+            .execute()
+            .actionGet();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setLevel("shards")
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.YELLOW, healthResponse.getStatus());
+
+        assertEquals(2 * dataNodes, healthResponse.getActiveShards());
+        assertEquals(2, healthResponse.getActivePrimaryShards());
+        assertEquals(1, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+
+        Map<String, ClusterIndexHealth> indices = healthResponse.getIndices();
+        assertFalse(indices.isEmpty());
+        assertEquals(2, indices.size());
+        for (Map.Entry<String, ClusterIndexHealth> indicesHealth : indices.entrySet()) {
+            String indexName = indicesHealth.getKey();
+            boolean indexHasMoreReplicas = indexName.equals("test2");
+            ClusterIndexHealth indicesHealthValue = indicesHealth.getValue();
+            assertEquals(dataNodes, indicesHealthValue.getActiveShards());
+            assertEquals(1, indicesHealthValue.getActivePrimaryShards());
+            assertEquals(0, indicesHealthValue.getInitializingShards());
+            assertEquals(indexHasMoreReplicas ? 1 : 0, indicesHealthValue.getUnassignedShards());
+            assertEquals(0, indicesHealthValue.getDelayedUnassignedShards());
+            assertEquals(0, indicesHealthValue.getRelocatingShards());
+            assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, indicesHealthValue.getStatus());
+            Map<Integer, ClusterShardHealth> shards = indicesHealthValue.getShards();
+            assertFalse(shards.isEmpty());
+            assertEquals(1, shards.size());
+            for (Map.Entry<Integer, ClusterShardHealth> shardHealth : shards.entrySet()) {
+                ClusterShardHealth clusterShardHealth = shardHealth.getValue();
+                assertEquals(dataNodes, clusterShardHealth.getActiveShards());
+                assertEquals(indexHasMoreReplicas ? 1 : 0, clusterShardHealth.getUnassignedShards());
+                assertEquals(0, clusterShardHealth.getDelayedUnassignedShards());
+                assertEquals(0, clusterShardHealth.getRelocatingShards());
+                assertEquals(0, clusterShardHealth.getInitializingShards());
+                assertTrue(clusterShardHealth.isPrimaryActive());
+                assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, clusterShardHealth.getStatus());
+            }
+        }
+    }
+
+    public void testHealthWithAwarenessAttributesLevelAppliedAtTransportLayer() {
+        createIndex(
+            "test1",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        ensureGreen();
+        ClusterHealthResponse healthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setLevel("awareness_attributes")
+            .setApplyLevelAtTransportLayer(true)
+            .execute()
+            .actionGet();
+        assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
+        assertTrue(healthResponse.getIndices().isEmpty());
+        assertNotNull(healthResponse.getClusterAwarenessHealth());
+        assertEquals(1, healthResponse.getActiveShards());
+        assertEquals(1, healthResponse.getActivePrimaryShards());
+        assertEquals(0, healthResponse.getUnassignedShards());
+        assertEquals(0, healthResponse.getInitializingShards());
+        assertEquals(0, healthResponse.getRelocatingShards());
+        assertEquals(0, healthResponse.getDelayedUnassignedShards());
+    }
 }

server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java

@@ -35,12 +35,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
-import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL;
 import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT;
 import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING;
 import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS;
 import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY;
 import static org.opensearch.gateway.remote.RemoteUploadStats.REMOTE_UPLOAD;
 import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE;
 import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING;
@@ -189,7 +189,7 @@ public void testRemoteCleanupDeleteStaleIndexRoutingFiles() throws Exception {
             RemoteStoreEnums.PathType.HASHED_PREFIX.toString()
         )
             .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO)
-            .put(REMOTE_PUBLICATION_EXPERIMENTAL, true);
+            .put(REMOTE_PUBLICATION_SETTING_KEY, true);
 
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;