From a3402d1201ce4b04111a4f3a7a76210c9ded9fd2 Mon Sep 17 00:00:00 2001
From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com>
Date: Mon, 17 Jun 2024 14:25:05 +0530
Subject: [PATCH 001/113] Use remote publication flag to decide which custom
 objects to upload (#14338)

* Simplify updated customs (ClusterState.Custom & Metadata.Custom) persistence logic to remote store

Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
---
 .../RemoteClusterStateAttributesManager.java  |  44 ++--
 .../remote/RemoteClusterStateService.java     |  97 ++++----
 .../remote/RemoteGlobalMetadataManager.java   |  58 +++--
 .../model/RemoteClusterStateBlobStore.java    |   4 +-
 ...oteClusterStateAttributesManagerTests.java | 186 +++++++++++++++-
 .../RemoteClusterStateServiceTests.java       |  91 +++++++-
 .../RemoteGlobalMetadataManagerTests.java     | 209 ++++++++++++++++++
 7 files changed, 591 insertions(+), 98 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java
index b052b6e1a613d..8f986423587d7 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java
@@ -10,6 +10,8 @@
 
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.DiffableUtils;
+import org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer;
 import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity;
 import org.opensearch.common.remote.RemoteWritableEntityStore;
@@ -25,10 +27,9 @@
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
 
 /**
  * A Manager which provides APIs to upload and download attributes of ClusterState to the {@link RemoteClusterStateBlobStore}
@@ -126,18 +127,35 @@ public CheckedRunnable<IOException> getAsyncMetadataReadAction(
         return () -> getStore(blobEntity).readAsync(blobEntity, actionListener);
     }
 
-    public Map<String, ClusterState.Custom> getUpdatedCustoms(ClusterState clusterState, ClusterState previousClusterState) {
-        Map<String, ClusterState.Custom> updatedCustoms = new HashMap<>();
-        Set<String> currentCustoms = new HashSet<>(clusterState.customs().keySet());
-        for (Map.Entry<String, ClusterState.Custom> entry : previousClusterState.customs().entrySet()) {
-            if (currentCustoms.contains(entry.getKey()) && !entry.getValue().equals(clusterState.customs().get(entry.getKey()))) {
-                updatedCustoms.put(entry.getKey(), clusterState.customs().get(entry.getKey()));
-            }
-            currentCustoms.remove(entry.getKey());
+    public DiffableUtils.MapDiff<String, ClusterState.Custom, Map<String, ClusterState.Custom>> getUpdatedCustoms(
+        ClusterState clusterState,
+        ClusterState previousClusterState,
+        boolean isRemotePublicationEnabled,
+        boolean isFirstUpload
+    ) {
+        if (!isRemotePublicationEnabled) {
+            // When isRemotePublicationEnabled is false, we do not want store any custom objects
+            return DiffableUtils.diff(
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                DiffableUtils.getStringKeySerializer(),
+                NonDiffableValueSerializer.getAbstractInstance()
+            );
         }
-        for (String custom : currentCustoms) {
-            updatedCustoms.put(custom, clusterState.customs().get(custom));
+        if (isFirstUpload) {
+            // For first upload of ephemeral metadata, we want to upload all customs
+            return DiffableUtils.diff(
+                Collections.emptyMap(),
+                clusterState.customs(),
+                DiffableUtils.getStringKeySerializer(),
+                NonDiffableValueSerializer.getAbstractInstance()
+            );
         }
-        return updatedCustoms;
+        return DiffableUtils.diff(
+            previousClusterState.customs(),
+            clusterState.customs(),
+            DiffableUtils.getStringKeySerializer(),
+            NonDiffableValueSerializer.getAbstractInstance()
+        );
     }
 }
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index ada29fdb57c57..4a1c9c8615e39 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -39,6 +39,7 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -88,6 +89,7 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL;
 import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD;
 import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2;
 import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS;
@@ -159,6 +161,7 @@ public class RemoteClusterStateService implements Closeable {
     private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged "
         + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata "
         + "updated : [{}], custom metadata updated : [{}], indices routing updated : [{}]";
+    private final boolean isPublicationEnabled;
 
     // ToXContent Params with gateway mode.
     // We are using gateway context mode to persist all custom metadata.
@@ -201,6 +204,9 @@ public RemoteClusterStateService(
             threadPool
         );
         this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService, remoteRoutingTableService);
+        this.isPublicationEnabled = FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL)
+            && RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings)
+            && RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings);
     }
 
     /**
@@ -221,15 +227,15 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat
             clusterState,
             new ArrayList<>(clusterState.metadata().indices().values()),
             emptyMap(),
-            clusterState.metadata().customs(),
+            RemoteGlobalMetadataManager.filterCustoms(clusterState.metadata().customs(), isPublicationEnabled),
             true,
             true,
             true,
-            true,
-            true,
-            true,
-            clusterState.customs(),
-            true,
+            isPublicationEnabled,
+            isPublicationEnabled,
+            isPublicationEnabled,
+            isPublicationEnabled ? clusterState.customs() : Collections.emptyMap(),
+            isPublicationEnabled,
             remoteRoutingTableService.getIndicesRouting(clusterState.getRoutingTable())
         );
         final RemoteClusterStateManifestInfo manifestDetails = remoteManifestManager.uploadManifest(
@@ -285,28 +291,17 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
         }
         assert previousClusterState.metadata().coordinationMetadata().term() == clusterState.metadata().coordinationMetadata().term();
 
-        final Map<String, UploadedMetadataAttribute> customsToBeDeletedFromRemote = new HashMap<>(previousManifest.getCustomMetadataMap());
-        final Map<String, Metadata.Custom> customsToUpload = remoteGlobalMetadataManager.getUpdatedCustoms(
-            clusterState,
-            previousClusterState
-        );
-        final Map<String, UploadedMetadataAttribute> clusterStateCustomsToBeDeleted = new HashMap<>(
+        boolean firstUploadForSplitGlobalMetadata = !previousManifest.hasMetadataAttributesFiles();
+
+        final DiffableUtils.MapDiff<String, Metadata.Custom, Map<String, Metadata.Custom>> customsDiff = remoteGlobalMetadataManager
+            .getCustomsDiff(clusterState, previousClusterState, firstUploadForSplitGlobalMetadata, isPublicationEnabled);
+        final DiffableUtils.MapDiff<String, ClusterState.Custom, Map<String, ClusterState.Custom>> clusterStateCustomsDiff =
+            remoteClusterStateAttributesManager.getUpdatedCustoms(clusterState, previousClusterState, isPublicationEnabled, false);
+        final Map<String, UploadedMetadataAttribute> allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap());
+        final Map<String, UploadedMetadataAttribute> allUploadedClusterStateCustomsMap = new HashMap<>(
             previousManifest.getClusterStateCustomMap()
         );
-        final Map<String, ClusterState.Custom> clusterStateCustomsToUpload = remoteClusterStateAttributesManager.getUpdatedCustoms(
-            clusterState,
-            previousClusterState
-        );
-        final Map<String, UploadedMetadataAttribute> allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap());
-        for (final String custom : clusterState.metadata().customs().keySet()) {
-            // remove all the customs which are present currently
-            customsToBeDeletedFromRemote.remove(custom);
-        }
         final Map<String, IndexMetadata> indicesToBeDeletedFromRemote = new HashMap<>(previousClusterState.metadata().indices());
-        for (final String custom : clusterState.customs().keySet()) {
-            // remove all the custom which are present currently
-            clusterStateCustomsToBeDeleted.remove(custom);
-        }
         int numIndicesUpdated = 0;
         int numIndicesUnchanged = 0;
         final Map<String, ClusterMetadataManifest.UploadedIndexMetadata> allUploadedIndexMetadata = previousManifest.getIndices()
@@ -337,42 +332,44 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
             indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName());
         }
 
-        DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> routingTableDiff = remoteRoutingTableService
+        final DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> routingTableDiff = remoteRoutingTableService
             .getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable());
-        List<IndexRoutingTable> indicesRoutingToUpload = new ArrayList<>();
+        final List<IndexRoutingTable> indicesRoutingToUpload = new ArrayList<>();
         routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v));
 
         UploadedMetadataResults uploadedMetadataResults;
         // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files,
         // If file is empty and codec is 1 then write global metadata.
-        boolean firstUploadForSplitGlobalMetadata = !previousManifest.hasMetadataAttributesFiles();
         boolean updateCoordinationMetadata = firstUploadForSplitGlobalMetadata
             || Metadata.isCoordinationMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
         ;
         boolean updateSettingsMetadata = firstUploadForSplitGlobalMetadata
             || Metadata.isSettingsMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
-        boolean updateTransientSettingsMetadata = firstUploadForSplitGlobalMetadata
-            || Metadata.isTransientSettingsMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
+        boolean updateTransientSettingsMetadata = Metadata.isTransientSettingsMetadataEqual(
+            previousClusterState.metadata(),
+            clusterState.metadata()
+        ) == false;
         boolean updateTemplatesMetadata = firstUploadForSplitGlobalMetadata
             || Metadata.isTemplatesMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
-        // ToDo: check if these needs to be updated or not
-        final boolean updateDiscoveryNodes = clusterState.getNodes().delta(previousClusterState.getNodes()).hasChanges();
-        final boolean updateClusterBlocks = !clusterState.blocks().equals(previousClusterState.blocks());
-        final boolean updateHashesOfConsistentSettings = firstUploadForSplitGlobalMetadata
+
+        final boolean updateDiscoveryNodes = isPublicationEnabled
+            && clusterState.getNodes().delta(previousClusterState.getNodes()).hasChanges();
+        final boolean updateClusterBlocks = isPublicationEnabled && !clusterState.blocks().equals(previousClusterState.blocks());
+        final boolean updateHashesOfConsistentSettings = isPublicationEnabled
             || Metadata.isHashesOfConsistentSettingsEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
 
         uploadedMetadataResults = writeMetadataInParallel(
             clusterState,
             toUpload,
             prevIndexMetadataByName,
-            firstUploadForSplitGlobalMetadata ? clusterState.metadata().customs() : customsToUpload,
+            customsDiff.getUpserts(),
             updateCoordinationMetadata,
             updateSettingsMetadata,
             updateTemplatesMetadata,
             updateDiscoveryNodes,
             updateClusterBlocks,
             updateTransientSettingsMetadata,
-            clusterStateCustomsToUpload,
+            clusterStateCustomsDiff.getUpserts(),
             updateHashesOfConsistentSettings,
             indicesRoutingToUpload
         );
@@ -382,10 +379,11 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
             uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata)
         );
         allUploadedCustomMap.putAll(uploadedMetadataResults.uploadedCustomMetadataMap);
+        allUploadedClusterStateCustomsMap.putAll(uploadedMetadataResults.uploadedClusterStateCustomMetadataMap);
         // remove the data for removed custom/indices
-        customsToBeDeletedFromRemote.keySet().forEach(allUploadedCustomMap::remove);
+        customsDiff.getDeletes().forEach(allUploadedCustomMap::remove);
         indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove);
-        clusterStateCustomsToBeDeleted.keySet().forEach(allUploadedCustomMap::remove);
+        clusterStateCustomsDiff.getDeletes().forEach(allUploadedClusterStateCustomsMap::remove);
 
         if (!updateCoordinationMetadata) {
             uploadedMetadataResults.uploadedCoordinationMetadata = previousManifest.getCoordinationMetadata();
@@ -399,31 +397,24 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
         if (!updateTemplatesMetadata) {
             uploadedMetadataResults.uploadedTemplatesMetadata = previousManifest.getTemplatesMetadata();
         }
-        if (!updateDiscoveryNodes && !firstUploadForSplitGlobalMetadata) {
+        if (!updateDiscoveryNodes) {
             uploadedMetadataResults.uploadedDiscoveryNodes = previousManifest.getDiscoveryNodesMetadata();
         }
-        if (!updateClusterBlocks && !firstUploadForSplitGlobalMetadata) {
+        if (!updateClusterBlocks) {
             uploadedMetadataResults.uploadedClusterBlocks = previousManifest.getClusterBlocksMetadata();
         }
-        if (!updateHashesOfConsistentSettings && !firstUploadForSplitGlobalMetadata) {
+        if (!updateHashesOfConsistentSettings) {
             uploadedMetadataResults.uploadedHashesOfConsistentSettings = previousManifest.getHashesOfConsistentSettings();
         }
-        if (!firstUploadForSplitGlobalMetadata && customsToUpload.isEmpty()) {
-            uploadedMetadataResults.uploadedCustomMetadataMap = previousManifest.getCustomMetadataMap();
-        }
-        if (!firstUploadForSplitGlobalMetadata && clusterStateCustomsToUpload.isEmpty()) {
-            uploadedMetadataResults.uploadedClusterStateCustomMetadataMap = previousManifest.getClusterStateCustomMap();
-        }
         uploadedMetadataResults.uploadedCustomMetadataMap = allUploadedCustomMap;
+        uploadedMetadataResults.uploadedClusterStateCustomMetadataMap = allUploadedClusterStateCustomsMap;
         uploadedMetadataResults.uploadedIndexMetadata = new ArrayList<>(allUploadedIndexMetadata.values());
 
-        List<ClusterMetadataManifest.UploadedIndexMetadata> allUploadedIndicesRouting = new ArrayList<>();
-        allUploadedIndicesRouting = remoteRoutingTableService.getAllUploadedIndicesRouting(
+        uploadedMetadataResults.uploadedIndicesRoutingMetadata = remoteRoutingTableService.getAllUploadedIndicesRouting(
             previousManifest,
             uploadedMetadataResults.uploadedIndicesRoutingMetadata,
             routingTableDiff.getDeletes()
         );
-        uploadedMetadataResults.uploadedIndicesRoutingMetadata = allUploadedIndicesRouting;
 
         final RemoteClusterStateManifestInfo manifestDetails = remoteManifestManager.uploadManifest(
             clusterState,
@@ -448,7 +439,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
             updateCoordinationMetadata,
             updateSettingsMetadata,
             updateTemplatesMetadata,
-            customsToUpload.size(),
+            customsDiff.getUpserts().size(),
             indicesRoutingToUpload.size()
         );
         if (durationMillis >= slowWriteLoggingThreshold.getMillis()) {
@@ -464,7 +455,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
                 updateCoordinationMetadata,
                 updateSettingsMetadata,
                 updateTemplatesMetadata,
-                customsToUpload.size()
+                customsDiff.getUpserts().size()
             );
         } else {
             logger.info("{}; {}", clusterStateUploadTimeMessage, metadataUpdateMessage);
@@ -479,7 +470,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
                 updateCoordinationMetadata,
                 updateSettingsMetadata,
                 updateTemplatesMetadata,
-                customsToUpload.size()
+                customsDiff.getUpserts().size()
             );
         }
         return manifestDetails;
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java
index 3053095368972..2c5aad99adc0c 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java
@@ -10,9 +10,12 @@
 
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.DiffableUtils;
+import org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer;
 import org.opensearch.cluster.coordination.CoordinationMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.Metadata.Custom;
+import org.opensearch.cluster.metadata.Metadata.XContentContext;
 import org.opensearch.cluster.metadata.TemplatesMetadata;
 import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity;
@@ -39,11 +42,12 @@
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
 
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT;
 
@@ -276,29 +280,37 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe
         }
     }
 
-    Map<String, Metadata.Custom> getUpdatedCustoms(ClusterState currentState, ClusterState previousState) {
-        if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) {
-            return new HashMap<>();
-        }
-        Map<String, Metadata.Custom> updatedCustom = new HashMap<>();
-        Set<String> currentCustoms = new HashSet<>(currentState.metadata().customs().keySet());
-        for (Map.Entry<String, Metadata.Custom> cursor : previousState.metadata().customs().entrySet()) {
-            if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) {
-                if (currentCustoms.contains(cursor.getKey())
-                    && !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) {
-                    // If the custom metadata is updated, we need to upload the new version.
-                    updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey()));
-                }
-                currentCustoms.remove(cursor.getKey());
-            }
+    DiffableUtils.MapDiff<String, Metadata.Custom, Map<String, Metadata.Custom>> getCustomsDiff(
+        ClusterState currentState,
+        ClusterState previousState,
+        boolean firstUploadForSplitGlobalMetadata,
+        boolean isRemotePublicationEnabled
+    ) {
+        if (firstUploadForSplitGlobalMetadata) {
+            // For first split global metadata upload, we want to upload all customs
+            return DiffableUtils.diff(
+                Collections.emptyMap(),
+                filterCustoms(currentState.metadata().customs(), isRemotePublicationEnabled),
+                DiffableUtils.getStringKeySerializer(),
+                NonDiffableValueSerializer.getAbstractInstance()
+            );
         }
-        for (String custom : currentCustoms) {
-            Metadata.Custom cursor = currentState.metadata().custom(custom);
-            if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) {
-                updatedCustom.put(custom, cursor);
-            }
+        return DiffableUtils.diff(
+            filterCustoms(previousState.metadata().customs(), isRemotePublicationEnabled),
+            filterCustoms(currentState.metadata().customs(), isRemotePublicationEnabled),
+            DiffableUtils.getStringKeySerializer(),
+            NonDiffableValueSerializer.getAbstractInstance()
+        );
+    }
+
+    public static Map<String, Metadata.Custom> filterCustoms(Map<String, Metadata.Custom> customs, boolean isRemotePublicationEnabled) {
+        if (isRemotePublicationEnabled) {
+            return customs;
         }
-        return updatedCustom;
+        return customs.entrySet()
+            .stream()
+            .filter(e -> e.getValue().context().contains(XContentContext.GATEWAY))
+            .collect(Collectors.toMap(Entry::getKey, Entry::getValue));
     }
 
     boolean isGlobalMetadataEqual(ClusterMetadataManifest first, ClusterMetadataManifest second, String clusterName) {
diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java
index 83326f65f0d43..1dd23443f1252 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java
@@ -72,7 +72,9 @@ public void writeAsync(final U entity, final ActionListener<Void> listener) {
     public T read(final U entity) throws IOException {
         // TODO Add timing logs and tracing
         assert entity.getFullBlobName() != null;
-        return entity.deserialize(transferService.downloadBlob(getBlobPathForDownload(entity), entity.getBlobFileName()));
+        try (InputStream inputStream = transferService.downloadBlob(getBlobPathForDownload(entity), entity.getBlobFileName())) {
+            return entity.deserialize(inputStream);
+        }
     }
 
     @Override
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
index 0aff1c4b0e5e2..41e1546ead164 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
@@ -8,7 +8,13 @@
 
 package org.opensearch.gateway.remote;
 
+import org.opensearch.Version;
 import org.opensearch.action.LatchedActionListener;
+import org.opensearch.cluster.AbstractNamedDiffable;
+import org.opensearch.cluster.ClusterName;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.ClusterState.Custom;
+import org.opensearch.cluster.DiffableUtils;
 import org.opensearch.cluster.block.ClusterBlocks;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.CheckedRunnable;
@@ -16,8 +22,11 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.compress.Compressor;
 import org.opensearch.core.compress.NoneCompressor;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.gateway.remote.model.RemoteClusterBlocks;
 import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes;
 import org.opensearch.gateway.remote.model.RemoteReadResult;
@@ -31,6 +40,9 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -41,6 +53,7 @@
 import static org.opensearch.gateway.remote.model.RemoteClusterBlocksTests.randomClusterBlocks;
 import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES_FORMAT;
 import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodesTests.getDiscoveryNodes;
+import static org.hamcrest.Matchers.is;
 import static org.mockito.ArgumentMatchers.anyIterable;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
@@ -51,7 +64,7 @@ public class RemoteClusterStateAttributesManagerTests extends OpenSearchTestCase
     private BlobStoreTransferService blobStoreTransferService;
     private BlobStoreRepository blobStoreRepository;
     private Compressor compressor;
-    private ThreadPool threadpool = new TestThreadPool(RemoteClusterStateAttributesManagerTests.class.getName());
+    private ThreadPool threadPool = new TestThreadPool(RemoteClusterStateAttributesManagerTests.class.getName());
 
     @Before
     public void setup() throws Exception {
@@ -65,15 +78,15 @@ public void setup() throws Exception {
             "test-cluster",
             blobStoreRepository,
             blobStoreTransferService,
-            namedWriteableRegistry,
-            threadpool
+            writableRegistry(),
+            threadPool
         );
     }
 
     @After
     public void tearDown() throws Exception {
         super.tearDown();
-        threadpool.shutdown();
+        threadPool.shutdown();
     }
 
     public void testGetAsyncMetadataReadAction_DiscoveryNodes() throws IOException {
@@ -138,4 +151,169 @@ public void testGetAsyncMetadataReadAction_ClusterBlocks() throws IOException {
             throw new RuntimeException(e);
         }
     }
+
+    public void testGetUpdatedCustoms() {
+        Map<String, ClusterState.Custom> previousCustoms = Map.of(
+            TestCustom1.TYPE,
+            new TestCustom1("data1"),
+            TestCustom2.TYPE,
+            new TestCustom2("data2"),
+            TestCustom3.TYPE,
+            new TestCustom3("data3")
+        );
+        ClusterState previousState = ClusterState.builder(new ClusterName("test-cluster")).customs(previousCustoms).build();
+
+        Map<String, Custom> currentCustoms = Map.of(
+            TestCustom2.TYPE,
+            new TestCustom2("data2"),
+            TestCustom3.TYPE,
+            new TestCustom3("data3-changed"),
+            TestCustom4.TYPE,
+            new TestCustom4("data4")
+        );
+
+        ClusterState currentState = ClusterState.builder(new ClusterName("test-cluster")).customs(currentCustoms).build();
+
+        DiffableUtils.MapDiff<String, ClusterState.Custom, Map<String, ClusterState.Custom>> customsDiff =
+            remoteClusterStateAttributesManager.getUpdatedCustoms(currentState, previousState, false, randomBoolean());
+        assertThat(customsDiff.getUpserts(), is(Collections.emptyMap()));
+        assertThat(customsDiff.getDeletes(), is(Collections.emptyList()));
+
+        customsDiff = remoteClusterStateAttributesManager.getUpdatedCustoms(currentState, previousState, true, true);
+        assertThat(customsDiff.getUpserts(), is(currentCustoms));
+        assertThat(customsDiff.getDeletes(), is(Collections.emptyList()));
+
+        Map<String, ClusterState.Custom> expectedCustoms = Map.of(
+            TestCustom3.TYPE,
+            new TestCustom3("data3-changed"),
+            TestCustom4.TYPE,
+            new TestCustom4("data4")
+        );
+
+        customsDiff = remoteClusterStateAttributesManager.getUpdatedCustoms(currentState, previousState, true, false);
+        assertThat(customsDiff.getUpserts(), is(expectedCustoms));
+        assertThat(customsDiff.getDeletes(), is(List.of(TestCustom1.TYPE)));
+    }
+
+    private static abstract class AbstractTestCustom extends AbstractNamedDiffable<Custom> implements ClusterState.Custom {
+
+        private final String value;
+
+        AbstractTestCustom(String value) {
+            this.value = value;
+        }
+
+        AbstractTestCustom(StreamInput in) throws IOException {
+            this.value = in.readString();
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(value);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder;
+        }
+
+        @Override
+        public boolean isPrivate() {
+            return true;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            AbstractTestCustom that = (AbstractTestCustom) o;
+
+            if (!value.equals(that.value)) return false;
+
+            return true;
+        }
+
+        @Override
+        public int hashCode() {
+            return value.hashCode();
+        }
+    }
+
+    private static class TestCustom1 extends AbstractTestCustom {
+
+        private static final String TYPE = "custom_1";
+
+        TestCustom1(String value) {
+            super(value);
+        }
+
+        TestCustom1(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+
+    private static class TestCustom2 extends AbstractTestCustom {
+
+        private static final String TYPE = "custom_2";
+
+        TestCustom2(String value) {
+            super(value);
+        }
+
+        TestCustom2(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+
+    private static class TestCustom3 extends AbstractTestCustom {
+
+        private static final String TYPE = "custom_3";
+
+        TestCustom3(String value) {
+            super(value);
+        }
+
+        TestCustom3(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+
+    private static class TestCustom4 extends AbstractTestCustom {
+
+        private static final String TYPE = "custom_4";
+
+        TestCustom4(String value) {
+            super(value);
+        }
+
+        TestCustom4(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
 }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index feae97bae48e9..c8fd982fec1e1 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -12,6 +12,8 @@
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.RepositoryCleanupInProgress;
+import org.opensearch.cluster.RepositoryCleanupInProgress.Entry;
 import org.opensearch.cluster.coordination.CoordinationMetadata;
 import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.IndexMetadata;
@@ -74,6 +76,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -130,6 +133,7 @@ public class RemoteClusterStateServiceTests extends OpenSearchTestCase {
     private BlobStoreRepository blobStoreRepository;
     private BlobStore blobStore;
     private Settings settings;
+    private boolean publicationEnabled;
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
 
     @Before
@@ -154,6 +158,7 @@ public void setup() {
             .put(stateRepoTypeAttributeKey, FsRepository.TYPE)
             .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath")
             .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
+            .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository")
             .build();
 
         clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
@@ -190,6 +195,9 @@ public void setup() {
     public void teardown() throws Exception {
         super.tearDown();
         remoteClusterStateService.close();
+        publicationEnabled = false;
+        Settings nodeSettings = Settings.builder().build();
+        FeatureFlags.initializeFeatureFlags(nodeSettings);
         threadPool.shutdown();
     }
 
@@ -263,9 +271,67 @@ public void testWriteFullMetadataSuccess() throws IOException {
         assertThat(manifest.getSettingsMetadata(), notNullValue());
         assertThat(manifest.getTemplatesMetadata(), notNullValue());
         assertFalse(manifest.getCustomMetadataMap().isEmpty());
+        assertThat(manifest.getClusterBlocksMetadata(), nullValue());
+        assertThat(manifest.getDiscoveryNodesMetadata(), nullValue());
+        assertThat(manifest.getTransientSettingsMetadata(), nullValue());
+        assertThat(manifest.getHashesOfConsistentSettings(), nullValue());
+        assertThat(manifest.getClusterStateCustomMap().size(), is(0));
+    }
+
+    public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException {
+        // TODO Make the publication flag parameterized
+        publicationEnabled = true;
+        Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, publicationEnabled).build();
+        FeatureFlags.initializeFeatureFlags(nodeSettings);
+        remoteClusterStateService = new RemoteClusterStateService(
+            "test-node-id",
+            repositoriesServiceSupplier,
+            settings,
+            clusterService,
+            () -> 0L,
+            threadPool,
+            List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)),
+            writableRegistry()
+        );
+        final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager())
+            .customs(Map.of(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(List.of(new Entry("test-repo", 10L)))))
+            .build();
+        mockBlobStoreObjects();
+        remoteClusterStateService.start();
+        final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid")
+            .getClusterMetadataManifest();
+        final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename");
+        List<UploadedIndexMetadata> indices = List.of(uploadedIndexMetadata);
+
+        final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
+            .indices(indices)
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .stateUUID("state-uuid")
+            .clusterUUID("cluster-uuid")
+            .previousClusterUUID("prev-cluster-uuid")
+            .build();
+
+        assertThat(manifest.getIndices().size(), is(1));
+        assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName()));
+        assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID()));
+        assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue());
+        assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm()));
+        assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion()));
+        assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
+        assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
+        assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID()));
+        assertThat(manifest.getGlobalMetadataFileName(), nullValue());
+        assertThat(manifest.getCoordinationMetadata(), notNullValue());
+        assertThat(manifest.getSettingsMetadata(), notNullValue());
+        assertThat(manifest.getTemplatesMetadata(), notNullValue());
+        assertFalse(manifest.getCustomMetadataMap().isEmpty());
+        assertThat(manifest.getClusterStateCustomMap().size(), is(1));
+        assertThat(manifest.getClusterStateCustomMap().containsKey(RepositoryCleanupInProgress.TYPE), is(true));
     }
 
     public void testWriteFullMetadataInParallelSuccess() throws IOException {
+        // TODO Add test with publication flag enabled
         final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
         AsyncMultiStreamBlobContainer container = (AsyncMultiStreamBlobContainer) mockBlobStoreObjects(AsyncMultiStreamBlobContainer.class);
 
@@ -310,8 +376,8 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
         assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID()));
 
-        assertEquals(11, actionListenerArgumentCaptor.getAllValues().size());
-        assertEquals(11, writeContextArgumentCaptor.getAllValues().size());
+        assertEquals(7, actionListenerArgumentCaptor.getAllValues().size());
+        assertEquals(7, writeContextArgumentCaptor.getAllValues().size());
 
         byte[] writtenBytes = capturedWriteContext.get("metadata")
             .getStreamProvider(Integer.MAX_VALUE)
@@ -584,7 +650,7 @@ private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMe
         assertNotNull(manifest.getCoordinationMetadata());
         assertNotNull(manifest.getSettingsMetadata());
         assertNotNull(manifest.getTemplatesMetadata());
-        assertNotEquals(0, manifest.getCustomMetadataMap().size());
+        assertNotNull(manifest.getCustomMetadataMap());
 
         assertEquals(expectedManifest.getClusterTerm(), manifest.getClusterTerm());
         assertEquals(expectedManifest.getStateVersion(), manifest.getStateVersion());
@@ -769,6 +835,7 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException {
                     .putCustom("custom1", new CustomMetadata1("mock_custom_metadata1"))
                     .putCustom("custom2", new CustomMetadata1("mock_custom_metadata2"))
                     .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3"))
+                    .version(initialClusterState.metadata().version() + 1)
             )
             .build();
 
@@ -784,6 +851,7 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException {
                     .putCustom("custom2", new CustomMetadata1("mock_updated_custom_metadata"))
                     .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3"))
                     .putCustom("custom4", new CustomMetadata1("mock_custom_metadata4"))
+                    .version(clusterState1.metadata().version() + 1)
             )
             .build();
         ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1)
@@ -1313,7 +1381,11 @@ public void testRemoteStateStats() throws IOException {
     }
 
     public void testRemoteRoutingTableNotInitializedWhenDisabled() {
-        assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof NoopRemoteRoutingTableService);
+        if (publicationEnabled) {
+            assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof InternalRemoteRoutingTableService);
+        } else {
+            assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof NoopRemoteRoutingTableService);
+        }
     }
 
     public void testRemoteRoutingTableInitializedWhenEnabled() {
@@ -1737,6 +1809,17 @@ private BlobContainer mockBlobStoreObjects(Class<? extends BlobContainer> blobCo
         final BlobPath blobPath = mock(BlobPath.class);
         when((blobStoreRepository.basePath())).thenReturn(blobPath);
         when(blobPath.add(anyString())).thenReturn(blobPath);
+        when(blobPath.iterator()).thenReturn(new Iterator<String>() {
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public String next() {
+                return null;
+            }
+        });
         when(blobPath.buildAsString()).thenReturn("/blob/path/");
         final BlobContainer blobContainer = mock(blobContainerClazz);
         when(blobContainer.path()).thenReturn(blobPath);
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
index f24f8ddeb1959..bd01bc1ab0cdb 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
@@ -8,7 +8,14 @@
 
 package org.opensearch.gateway.remote;
 
+import org.opensearch.Version;
 import org.opensearch.cluster.ClusterModule;
+import org.opensearch.cluster.ClusterName;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.DiffableUtils;
+import org.opensearch.cluster.metadata.IndexGraveyard;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.metadata.Metadata.XContentContext;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
@@ -19,15 +26,20 @@
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.TestCustomMetadata;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Stream;
 
 import static java.util.stream.Collectors.toList;
+import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -83,4 +95,201 @@ public void testGlobalMetadataUploadWaitTimeSetting() {
         clusterSettings.applySettings(newSettings);
         assertEquals(globalMetadataUploadTimeout, remoteGlobalMetadataManager.getGlobalMetadataUploadTimeout().seconds());
     }
+
+    public void testGetUpdatedCustoms() {
+        Map<String, Metadata.Custom> previousCustoms = Map.of(
+            CustomMetadata1.TYPE,
+            new CustomMetadata1("data1"),
+            CustomMetadata2.TYPE,
+            new CustomMetadata2("data2"),
+            CustomMetadata3.TYPE,
+            new CustomMetadata3("data3")
+        );
+        ClusterState previousState = ClusterState.builder(new ClusterName("test-cluster"))
+            .metadata(Metadata.builder().customs(previousCustoms))
+            .build();
+
+        Map<String, Metadata.Custom> currentCustoms = Map.of(
+            CustomMetadata2.TYPE,
+            new CustomMetadata2("data2"),
+            CustomMetadata3.TYPE,
+            new CustomMetadata3("data3-changed"),
+            CustomMetadata4.TYPE,
+            new CustomMetadata4("data4"),
+            CustomMetadata5.TYPE,
+            new CustomMetadata5("data5")
+        );
+        ClusterState currentState = ClusterState.builder(new ClusterName("test-cluster"))
+            .metadata(Metadata.builder().customs(currentCustoms))
+            .build();
+
+        DiffableUtils.MapDiff<String, Metadata.Custom, Map<String, Metadata.Custom>> customsDiff = remoteGlobalMetadataManager
+            .getCustomsDiff(currentState, previousState, true, false);
+        Map<String, Metadata.Custom> expectedUpserts = Map.of(
+            CustomMetadata2.TYPE,
+            new CustomMetadata2("data2"),
+            CustomMetadata3.TYPE,
+            new CustomMetadata3("data3-changed"),
+            CustomMetadata4.TYPE,
+            new CustomMetadata4("data4"),
+            IndexGraveyard.TYPE,
+            IndexGraveyard.builder().build()
+        );
+        assertThat(customsDiff.getUpserts(), is(expectedUpserts));
+        assertThat(customsDiff.getDeletes(), is(List.of()));
+
+        customsDiff = remoteGlobalMetadataManager.getCustomsDiff(currentState, previousState, false, false);
+        expectedUpserts = Map.of(
+            CustomMetadata3.TYPE,
+            new CustomMetadata3("data3-changed"),
+            CustomMetadata4.TYPE,
+            new CustomMetadata4("data4")
+        );
+        assertThat(customsDiff.getUpserts(), is(expectedUpserts));
+        assertThat(customsDiff.getDeletes(), is(List.of(CustomMetadata1.TYPE)));
+
+        customsDiff = remoteGlobalMetadataManager.getCustomsDiff(currentState, previousState, true, true);
+        expectedUpserts = Map.of(
+            CustomMetadata2.TYPE,
+            new CustomMetadata2("data2"),
+            CustomMetadata3.TYPE,
+            new CustomMetadata3("data3-changed"),
+            CustomMetadata4.TYPE,
+            new CustomMetadata4("data4"),
+            CustomMetadata5.TYPE,
+            new CustomMetadata5("data5"),
+            IndexGraveyard.TYPE,
+            IndexGraveyard.builder().build()
+        );
+        assertThat(customsDiff.getUpserts(), is(expectedUpserts));
+        assertThat(customsDiff.getDeletes(), is(List.of()));
+
+        customsDiff = remoteGlobalMetadataManager.getCustomsDiff(currentState, previousState, false, true);
+        expectedUpserts = Map.of(
+            CustomMetadata3.TYPE,
+            new CustomMetadata3("data3-changed"),
+            CustomMetadata4.TYPE,
+            new CustomMetadata4("data4"),
+            CustomMetadata5.TYPE,
+            new CustomMetadata5("data5")
+        );
+        assertThat(customsDiff.getUpserts(), is(expectedUpserts));
+        assertThat(customsDiff.getDeletes(), is(List.of(CustomMetadata1.TYPE)));
+
+    }
+
+    private static class CustomMetadata1 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_1";
+
+        CustomMetadata1(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    private static class CustomMetadata2 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_2";
+
+        CustomMetadata2(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    private static class CustomMetadata3 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_3";
+
+        CustomMetadata3(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    private static class CustomMetadata4 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_4";
+
+        CustomMetadata4(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    private static class CustomMetadata5 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_5";
+
+        CustomMetadata5(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(XContentContext.API);
+        }
+    }
 }

From 47425a11fc7c3ab3e72679c0e22033ea86cbb7bf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 10:18:58 -0400
Subject: [PATCH 002/113] Bump commons-net:commons-net from 3.10.0 to 3.11.1 in
 /test/fixtures/hdfs-fixture (#14396)

* Bump commons-net:commons-net in /test/fixtures/hdfs-fixture

Bumps commons-net:commons-net from 3.10.0 to 3.11.1.

---
updated-dependencies:
- dependency-name: commons-net:commons-net
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                            | 1 +
 test/fixtures/hdfs-fixture/build.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e1d41ec6b152..fa387e19c3d77 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
 - Bump `netty` from 4.1.110.Final to 4.1.111.Final ([#14356](https://github.com/opensearch-project/OpenSearch/pull/14356))
 - Bump `org.wiremock:wiremock-standalone` from 3.3.1 to 3.6.0 ([#14361](https://github.com/opensearch-project/OpenSearch/pull/14361))
+- Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
 
 ### Changed
 
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index ddb876b46fd1c..a532bf0c6287b 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -70,7 +70,7 @@ dependencies {
   api "org.eclipse.jetty.websocket:javax-websocket-server-impl:${versions.jetty}"
   api 'org.apache.zookeeper:zookeeper:3.9.2'
   api "org.apache.commons:commons-text:1.12.0"
-  api "commons-net:commons-net:3.10.0"
+  api "commons-net:commons-net:3.11.1"
   api "ch.qos.logback:logback-core:1.5.6"
   api "ch.qos.logback:logback-classic:1.2.13"
   api 'org.apache.kerby:kerb-admin:2.0.3'

From 5ec2abeb8200bf60f1f706e66f4364f03e756573 Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Mon, 17 Jun 2024 22:28:03 +0800
Subject: [PATCH 003/113] Fix flaky tests in
 org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceTests (#14264)

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 .../remote/RemoteRoutingTableServiceTests.java  | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
index cc31c1a6e8fd1..839ebe1ff8301 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
@@ -227,11 +227,11 @@ public void testGetIndicesRoutingMapDiffIndexAdded() {
                 .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
                 .build()
-        ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build();
+        ).numberOfShards(between(1, 1000)).numberOfReplicas(randomInt(10)).build();
         RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build();
 
         String indexName2 = randomAlphaOfLength(randomIntBetween(1, 50));
-        int noOfShards = randomInt(1000);
+        int noOfShards = between(1, 1000);
         int noOfReplicas = randomInt(10);
         final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(indexName2).settings(
             Settings.builder()
@@ -252,8 +252,7 @@ public void testGetIndicesRoutingMapDiffIndexAdded() {
 
     public void testGetIndicesRoutingMapDiffShardChanged() {
         String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        final Index index = new Index(indexName, "uuid");
-        int noOfShards = randomInt(1000);
+        int noOfShards = between(1, 1000);
         int noOfReplicas = randomInt(10);
         final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings(
             Settings.builder()
@@ -299,8 +298,7 @@ public void testGetIndicesRoutingMapDiffShardChanged() {
 
     public void testGetIndicesRoutingMapDiffShardDetailChanged() {
         String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        final Index index = new Index(indexName, "uuid");
-        int noOfShards = randomInt(1000);
+        int noOfShards = between(1, 1000);
         int noOfReplicas = randomInt(10);
         final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings(
             Settings.builder()
@@ -328,7 +326,7 @@ public void testGetIndicesRoutingMapDiffIndexDeleted() {
                 .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
                 .build()
-        ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build();
+        ).numberOfShards(between(1, 1000)).numberOfReplicas(randomInt(10)).build();
         RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build();
 
         String indexName2 = randomAlphaOfLength(randomIntBetween(1, 50));
@@ -337,7 +335,7 @@ public void testGetIndicesRoutingMapDiffIndexDeleted() {
                 .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetadata.SETTING_INDEX_UUID, "uuid2")
                 .build()
-        ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build();
+        ).numberOfShards(between(1, 1000)).numberOfReplicas(randomInt(10)).build();
         RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build();
 
         DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> diff = remoteRoutingTableService
@@ -732,7 +730,6 @@ private ClusterState createIndices(int numberOfIndices) {
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
         for (int i = 0; i < numberOfIndices; i++) {
             String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-            final Index index = new Index(indexName, "uuid");
             final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings(
                 Settings.builder()
                     .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
@@ -751,7 +748,7 @@ private ClusterState createClusterState(String indexName) {
                 .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
                 .build()
-        ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build();
+        ).numberOfShards(between(1, 1000)).numberOfReplicas(randomInt(10)).build();
         RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build();
         return ClusterState.builder(ClusterName.DEFAULT)
             .routingTable(routingTable)

From f99a6f8f682a79d052b68eb6610b631dd9ce0e41 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Mon, 17 Jun 2024 11:21:36 -0400
Subject: [PATCH 004/113] Bump reactor from 3.5.17 to 3.5.18 and reactor-netty
 from 1.1.19 to 1.1.20 (#14395)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                                  | 2 ++
 buildSrc/version.properties                                   | 4 ++--
 .../licenses/reactor-netty-core-1.1.19.jar.sha1               | 1 -
 .../licenses/reactor-netty-core-1.1.20.jar.sha1               | 1 +
 .../licenses/reactor-netty-http-1.1.19.jar.sha1               | 1 -
 .../licenses/reactor-netty-http-1.1.20.jar.sha1               | 1 +
 .../licenses/reactor-netty-core-1.1.19.jar.sha1               | 1 -
 .../licenses/reactor-netty-core-1.1.20.jar.sha1               | 1 +
 .../licenses/reactor-netty-http-1.1.19.jar.sha1               | 1 -
 .../licenses/reactor-netty-http-1.1.20.jar.sha1               | 1 +
 server/licenses/reactor-core-3.5.17.jar.sha1                  | 1 -
 server/licenses/reactor-core-3.5.18.jar.sha1                  | 1 +
 12 files changed, 9 insertions(+), 7 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.19.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.19.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1
 delete mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.19.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1
 delete mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.19.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1
 delete mode 100644 server/licenses/reactor-core-3.5.17.jar.sha1
 create mode 100644 server/licenses/reactor-core-3.5.18.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fa387e19c3d77..b6beaa40e6d5a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
 - Bump `netty` from 4.1.110.Final to 4.1.111.Final ([#14356](https://github.com/opensearch-project/OpenSearch/pull/14356))
 - Bump `org.wiremock:wiremock-standalone` from 3.3.1 to 3.6.0 ([#14361](https://github.com/opensearch-project/OpenSearch/pull/14361))
+- Bump `reactor` from 3.5.17 to 3.5.18 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
+- Bump `reactor-netty` from 1.1.19 to 1.1.20 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
 - Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
 
 ### Changed
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index d20c3bd46f304..eb96261b056e3 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -33,8 +33,8 @@ netty             = 4.1.111.Final
 joda              = 2.12.7
 
 # project reactor
-reactor_netty     = 1.1.19
-reactor           = 3.5.17
+reactor_netty     = 1.1.20
+reactor           = 3.5.18
 
 # client dependencies
 httpclient5       = 5.2.1
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.19.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.19.jar.sha1
deleted file mode 100644
index cbcbfcd87d682..0000000000000
--- a/plugins/repository-azure/licenses/reactor-netty-core-1.1.19.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-639e2c63ade6f2a49d7e501ca2264b74d240b448
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1
new file mode 100644
index 0000000000000..2f4d023c88c80
--- /dev/null
+++ b/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1
@@ -0,0 +1 @@
+1a5ef52a470a82d9313e2e1ad8ba064bdbd38948
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.19.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.19.jar.sha1
deleted file mode 100644
index 1eeedfc0926f5..0000000000000
--- a/plugins/repository-azure/licenses/reactor-netty-http-1.1.19.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4bbb1aeb64ecb2b3949c38983032a7f0b0ebd07
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1
new file mode 100644
index 0000000000000..6c031e00e39c1
--- /dev/null
+++ b/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1
@@ -0,0 +1 @@
+8d4ee98405a5856cf0c9d7c1a70f3f14631e3c46
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.19.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.19.jar.sha1
deleted file mode 100644
index cbcbfcd87d682..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.19.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-639e2c63ade6f2a49d7e501ca2264b74d240b448
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1
new file mode 100644
index 0000000000000..2f4d023c88c80
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1
@@ -0,0 +1 @@
+1a5ef52a470a82d9313e2e1ad8ba064bdbd38948
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.19.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.19.jar.sha1
deleted file mode 100644
index 1eeedfc0926f5..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.19.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4bbb1aeb64ecb2b3949c38983032a7f0b0ebd07
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1
new file mode 100644
index 0000000000000..6c031e00e39c1
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1
@@ -0,0 +1 @@
+8d4ee98405a5856cf0c9d7c1a70f3f14631e3c46
\ No newline at end of file
diff --git a/server/licenses/reactor-core-3.5.17.jar.sha1 b/server/licenses/reactor-core-3.5.17.jar.sha1
deleted file mode 100644
index 6663356bab047..0000000000000
--- a/server/licenses/reactor-core-3.5.17.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2cf9b080e3a2d8a5a39948260db5fd1dae54c3ac
\ No newline at end of file
diff --git a/server/licenses/reactor-core-3.5.18.jar.sha1 b/server/licenses/reactor-core-3.5.18.jar.sha1
new file mode 100644
index 0000000000000..c503f768beafa
--- /dev/null
+++ b/server/licenses/reactor-core-3.5.18.jar.sha1
@@ -0,0 +1 @@
+3a8157f7d66d71a407eb77ba12bce72a38c5b4da
\ No newline at end of file

From 21d3aaa8d2235b0e6fc2e9ce69f98b333c5aa293 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 12:06:02 -0400
Subject: [PATCH 005/113] Bump org.apache.commons:commons-configuration2 from
 2.10.1 to 2.11.0 in /plugins/repository-hdfs (#14399)

* Bump org.apache.commons:commons-configuration2

Bumps org.apache.commons:commons-configuration2 from 2.10.1 to 2.11.0.

---
updated-dependencies:
- dependency-name: org.apache.commons:commons-configuration2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                                    | 1 +
 plugins/repository-hdfs/build.gradle                            | 2 +-
 .../licenses/commons-configuration2-2.10.1.jar.sha1             | 1 -
 .../licenses/commons-configuration2-2.11.0.jar.sha1             | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/repository-hdfs/licenses/commons-configuration2-2.10.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/commons-configuration2-2.11.0.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6beaa40e6d5a..b9413d379f88b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `reactor` from 3.5.17 to 3.5.18 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
 - Bump `reactor-netty` from 1.1.19 to 1.1.20 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
 - Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
+- Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
 
 ### Changed
 
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index eb50bd2d0615a..63eb783649884 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -74,7 +74,7 @@ dependencies {
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
-  api 'org.apache.commons:commons-configuration2:2.10.1'
+  api 'org.apache.commons:commons-configuration2:2.11.0'
   api "commons-io:commons-io:${versions.commonsio}"
   api 'org.apache.commons:commons-lang3:3.14.0'
   implementation 'com.google.re2j:re2j:1.7'
diff --git a/plugins/repository-hdfs/licenses/commons-configuration2-2.10.1.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration2-2.10.1.jar.sha1
deleted file mode 100644
index d4c0f8417d357..0000000000000
--- a/plugins/repository-hdfs/licenses/commons-configuration2-2.10.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2b681b3bcddeaa5bf5c2a2939cd77e2f9ad6efda
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-configuration2-2.11.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration2-2.11.0.jar.sha1
new file mode 100644
index 0000000000000..eea24804c5228
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-configuration2-2.11.0.jar.sha1
@@ -0,0 +1 @@
+af5a2c6abe587074c0be1107fcb27fa2fad91304
\ No newline at end of file

From a32035bff597359316760a6861ac02cafb4f2397 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 15:37:00 -0400
Subject: [PATCH 006/113] Bump com.gradle.develocity from 3.17.4 to 3.17.5
 (#14397)

* Bump com.gradle.develocity from 3.17.4 to 3.17.5

Bumps com.gradle.develocity from 3.17.4 to 3.17.5.

---
updated-dependencies:
- dependency-name: com.gradle.develocity
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md    | 1 +
 settings.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b9413d379f88b..347c28792b35b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `reactor-netty` from 1.1.19 to 1.1.20 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
 - Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
+- Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
 
 ### Changed
 
diff --git a/settings.gradle b/settings.gradle
index 888ecc62aac7c..a96d00a4ab863 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -10,7 +10,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "3.17.4"
+  id "com.gradle.develocity" version "3.17.5"
 }
 
 ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE')

From 1d14569622f36fb163ee45e0e52620553a050a7d Mon Sep 17 00:00:00 2001
From: Liyun Xiu <xiliyun@amazon.com>
Date: Mon, 17 Jun 2024 13:11:20 -0700
Subject: [PATCH 007/113] Remove one unnecesary test and simply some code in a
 test (#14360)

Signed-off-by: Liyun Xiu <xiliyun@amazon.com>
---
 .../org/opensearch/ingest/IngestClientIT.java | 50 ++-----------------
 .../opensearch/ingest/IngestServiceTests.java | 27 ++++------
 2 files changed, 15 insertions(+), 62 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java
index dbde31ef1eb65..657d0f178e096 100644
--- a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java
@@ -189,7 +189,7 @@ private void runBulkTestWithRandomDocs(boolean shouldSetBatchSize) throws Except
         int numRequests = scaledRandomIntBetween(32, 128);
         BulkRequest bulkRequest = new BulkRequest();
         if (shouldSetBatchSize) {
-            bulkRequest.batchSize(numRequests);
+            bulkRequest.batchSize(scaledRandomIntBetween(2, numRequests));
         }
         for (int i = 0; i < numRequests; i++) {
             IndexRequest indexRequest = new IndexRequest("index").id(Integer.toString(i)).setPipeline("_id");
@@ -214,6 +214,9 @@ private void runBulkTestWithRandomDocs(boolean shouldSetBatchSize) throws Except
                 );
                 assertThat(indexResponse, notNullValue());
                 assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
+                // verify field of successful doc
+                Map<String, Object> successDoc = client().prepareGet("index", indexResponse.getId()).get().getSourceAsMap();
+                assertThat(successDoc.get("processed"), equalTo(true));
                 assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
             }
         }
@@ -223,51 +226,6 @@ private void runBulkTestWithRandomDocs(boolean shouldSetBatchSize) throws Except
         assertTrue(deletePipelineResponse.isAcknowledged());
     }
 
-    public void testBulkWithIngestFailuresBatch() throws Exception {
-        createIndex("index");
-
-        BytesReference source = BytesReference.bytes(
-            jsonBuilder().startObject()
-                .field("description", "my_pipeline")
-                .startArray("processors")
-                .startObject()
-                .startObject("test")
-                .endObject()
-                .endObject()
-                .endArray()
-                .endObject()
-        );
-        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, MediaTypeRegistry.JSON);
-        client().admin().cluster().putPipeline(putPipelineRequest).get();
-
-        BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.batchSize(2);
-        bulkRequest.add(
-            new IndexRequest("index").id("_fail").setPipeline("_id").source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", true)
-        );
-        bulkRequest.add(
-            new IndexRequest("index").id("_success").setPipeline("_id").source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", false)
-        );
-
-        BulkResponse response = client().bulk(bulkRequest).actionGet();
-        MatcherAssert.assertThat(response.getItems().length, equalTo(bulkRequest.requests().size()));
-
-        Map<String, BulkItemResponse> results = Arrays.stream(response.getItems())
-            .collect(Collectors.toMap(BulkItemResponse::getId, r -> r));
-
-        MatcherAssert.assertThat(results.keySet(), containsInAnyOrder("_fail", "_success"));
-        assertNotNull(results.get("_fail").getFailure());
-        assertNull(results.get("_success").getFailure());
-
-        // verify field of successful doc
-        Map<String, Object> successDoc = client().prepareGet("index", "_success").get().getSourceAsMap();
-        assertThat(successDoc.get("processed"), equalTo(true));
-
-        // cleanup
-        AcknowledgedResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
-        assertTrue(deletePipelineResponse.isAcknowledged());
-    }
-
     public void testBulkWithIngestFailuresAndDropBatch() throws Exception {
         createIndex("index");
 
diff --git a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
index a32cd2c3cad3f..684297c11c140 100644
--- a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
@@ -81,6 +81,7 @@
 import org.junit.Before;
 
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
@@ -88,6 +89,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -97,7 +99,6 @@
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
-import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatcher;
 import org.mockito.invocation.InvocationOnMock;
 
@@ -1923,27 +1924,21 @@ public void testExecuteBulkRequestInBatchWithExceptionAndDropInCallback() {
             return null;
         }).when(mockCompoundProcessor).batchExecute(any(), any());
 
-        @SuppressWarnings("unchecked")
-        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
-        @SuppressWarnings("unchecked")
-        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
-        final IntConsumer dropHandler = mock(IntConsumer.class);
+        final Map<Integer, Exception> failureHandler = new HashMap<>();
+        final Map<Thread, Exception> completionHandler = new HashMap<>();
+        final List<Integer> dropHandler = new ArrayList<>();
         ingestService.executeBulkRequest(
             3,
             bulkRequest.requests(),
-            failureHandler,
-            completionHandler,
-            dropHandler,
+            failureHandler::put,
+            completionHandler::put,
+            dropHandler::add,
             Names.WRITE,
             bulkRequest
         );
-        ArgumentCaptor<Integer> failureSlotCaptor = ArgumentCaptor.forClass(Integer.class);
-        verify(failureHandler, times(1)).accept(failureSlotCaptor.capture(), any());
-        assertEquals(1, failureSlotCaptor.getValue().intValue());
-        ArgumentCaptor<Integer> dropSlotCaptor = ArgumentCaptor.forClass(Integer.class);
-        verify(dropHandler, times(1)).accept(dropSlotCaptor.capture());
-        assertEquals(2, dropSlotCaptor.getValue().intValue());
-        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        assertEquals(Set.of(1), failureHandler.keySet());
+        assertEquals(List.of(2), dropHandler);
+        assertEquals(Set.of(Thread.currentThread()), completionHandler.keySet());
         verify(mockCompoundProcessor, times(1)).batchExecute(any(), any());
         verify(mockCompoundProcessor, never()).execute(any(), any());
     }

From 112704bf2976db8b0dd71980173896029a490cd9 Mon Sep 17 00:00:00 2001
From: Andrew Ross <andrross@amazon.com>
Date: Mon, 17 Jun 2024 13:33:50 -0700
Subject: [PATCH 008/113] Add missing data types to IngestDocument deep copy
 (#14380)

PR #11725 added a new deep copy in the ScriptProcessor flow. If a script
uses a Short or Byte data type then this new deep copy introduced a
regression. This commit fixes that regression.

However, it appears there has been an existing bug where using a
Character type in the same way will fail (this failed before PR 11725).
The failure is different, and appears to be related to something deeping
in the XContent serialization layer. For now, I have fixed the
regression but not yet dug into the failure with the Character data
type. I have added a test that expects this failure.

Resolves #14379

Signed-off-by: Andrew Ross <andrross@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../test/ingest/190_script_processor.yml      | 75 +++++++++++++++++++
 .../org/opensearch/ingest/IngestDocument.java |  3 +
 .../ingest/IngestDocumentTests.java           | 47 ++++++++----
 4 files changed, 111 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 347c28792b35b..a43c0acf3219a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 
 ### Security
 
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml
index a66f02d6b6a6d..984c67d39757d 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml
@@ -278,3 +278,78 @@ teardown:
         body: {source_field: "fooBar", foo: {foo: "bar"}}
   - match: { error.root_cause.0.type: "illegal_argument_exception" }
   - match: { error.root_cause.0.reason: "Iterable object is self-referencing itself (ingest script)" }
+
+---
+"Test painless data types":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "source" : "ctx.byte = (byte)127;ctx.short = (short)32767;ctx.int = (int)2147483647;ctx.long = (long)9223372036854775807L;ctx.float = (float)0.1;ctx.double = (double)0.1;ctx.boolean = (boolean)true"
+                }
+              },
+              {
+                "script" : {
+                  "source" : "ctx.other_field = 'other_field'"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "FooBar"}
+
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source.byte: 127 }
+  - match: { _source.int: 2147483647 }
+  - match: { _source.long: 9223372036854775807 }
+  - gt: { _source.float: 0.0 }
+  - lt: { _source.float: 0.2 }
+  - gt: { _source.double: 0.0 }
+  - lt: { _source.double: 0.2 }
+  - match: { _source.boolean: true }
+
+---
+"Test char type fails":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "source" : "ctx.char = (char)'a'"
+                }
+              },
+              {
+                "script" : {
+                  "source" : "ctx.other_field = 'other_field'"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      catch: bad_request
+      index:
+        index: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "FooBar"}
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
diff --git a/server/src/main/java/org/opensearch/ingest/IngestDocument.java b/server/src/main/java/org/opensearch/ingest/IngestDocument.java
index d975b0014de1f..9ec59e4c275a8 100644
--- a/server/src/main/java/org/opensearch/ingest/IngestDocument.java
+++ b/server/src/main/java/org/opensearch/ingest/IngestDocument.java
@@ -776,6 +776,9 @@ public static Object deepCopy(Object value) {
             byte[] bytes = (byte[]) value;
             return Arrays.copyOf(bytes, bytes.length);
         } else if (value == null
+            || value instanceof Byte
+            || value instanceof Character
+            || value instanceof Short
             || value instanceof String
             || value instanceof Integer
             || value instanceof Long
diff --git a/server/src/test/java/org/opensearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/opensearch/ingest/IngestDocumentTests.java
index be035bc6ef7ea..8b78689acd6d2 100644
--- a/server/src/test/java/org/opensearch/ingest/IngestDocumentTests.java
+++ b/server/src/test/java/org/opensearch/ingest/IngestDocumentTests.java
@@ -62,6 +62,7 @@ public class IngestDocumentTests extends OpenSearchTestCase {
 
     private static final ZonedDateTime BOGUS_TIMESTAMP = ZonedDateTime.of(2016, 10, 23, 0, 0, 0, 0, ZoneOffset.UTC);
     private IngestDocument ingestDocument;
+    private int initialSourceAndMetadataSize;
 
     @Before
     public void setTestIngestDocument() {
@@ -70,7 +71,6 @@ public void setTestIngestDocument() {
         ingestMap.put("timestamp", BOGUS_TIMESTAMP);
         document.put("_ingest", ingestMap);
         document.put("foo", "bar");
-        document.put("int", 123);
         Map<String, Object> innerObject = new HashMap<>();
         innerObject.put("buzz", "hello world");
         innerObject.put("foo_null", null);
@@ -92,7 +92,17 @@ public void setTestIngestDocument() {
         list2.add("bar");
         list2.add("baz");
         document.put("list2", list2);
+        document.put("byte", (byte) 1);
+        document.put("short", (short) 2);
+        document.put("int", Integer.MAX_VALUE);
+        document.put("long", Long.MAX_VALUE);
+        document.put("float", 0.1f);
+        document.put("double", 0.1d);
+        document.put("char", 'a');
+        document.put("string", "A test string \uD83C\uDF89");
+        document.put("datetime", ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"));
         ingestDocument = new IngestDocument("index", "id", null, null, null, document);
+        initialSourceAndMetadataSize = 16; // i.e. ingestDocument.getSourceAndMetadata().size()
     }
 
     public void testSelfReferencingSource() {
@@ -101,11 +111,18 @@ public void testSelfReferencingSource() {
         expectThrows(IllegalArgumentException.class, () -> IngestDocument.deepCopyMap(value));
     }
 
+    public void testCopy() {
+        final IngestDocument copy = new IngestDocument(ingestDocument);
+        assertThat(copy, equalTo(ingestDocument));
+        assertThat(copy.getSourceAndMetadata(), not(sameInstance(ingestDocument.getSourceAndMetadata())));
+        assertThat(copy.getIngestMetadata(), not(sameInstance(ingestDocument.getIngestMetadata())));
+    }
+
     public void testSimpleGetFieldValue() {
         assertThat(ingestDocument.getFieldValue("foo", String.class), equalTo("bar"));
-        assertThat(ingestDocument.getFieldValue("int", Integer.class), equalTo(123));
+        assertThat(ingestDocument.getFieldValue("int", Integer.class), equalTo(Integer.MAX_VALUE));
         assertThat(ingestDocument.getFieldValue("_source.foo", String.class), equalTo("bar"));
-        assertThat(ingestDocument.getFieldValue("_source.int", Integer.class), equalTo(123));
+        assertThat(ingestDocument.getFieldValue("_source.int", Integer.class), equalTo(Integer.MAX_VALUE));
         assertThat(ingestDocument.getFieldValue("_index", String.class), equalTo("index"));
         assertThat(ingestDocument.getFieldValue("_id", String.class), equalTo("id"));
         assertThat(
@@ -578,7 +595,7 @@ public void testAppendFieldValueConvertIntegerToList() {
         @SuppressWarnings("unchecked")
         List<Object> list = (List<Object>) object;
         assertThat(list.size(), equalTo(2));
-        assertThat(list.get(0), equalTo(123));
+        assertThat(list.get(0), equalTo(Integer.MAX_VALUE));
         assertThat(list.get(1), equalTo(456));
     }
 
@@ -589,7 +606,7 @@ public void testAppendFieldValuesConvertIntegerToList() {
         @SuppressWarnings("unchecked")
         List<Object> list = (List<Object>) object;
         assertThat(list.size(), equalTo(3));
-        assertThat(list.get(0), equalTo(123));
+        assertThat(list.get(0), equalTo(Integer.MAX_VALUE));
         assertThat(list.get(1), equalTo(456));
         assertThat(list.get(2), equalTo(789));
     }
@@ -812,23 +829,23 @@ public void testSetFieldValueEmptyName() {
 
     public void testRemoveField() {
         ingestDocument.removeField("foo");
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(7));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize - 1));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("foo"), equalTo(false));
         ingestDocument.removeField("_index");
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(6));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize - 2));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("_index"), equalTo(false));
         ingestDocument.removeField("_source.fizz");
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(5));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize - 3));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(false));
         assertThat(ingestDocument.getIngestMetadata().size(), equalTo(1));
         ingestDocument.removeField("_ingest.timestamp");
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(5));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize - 3));
         assertThat(ingestDocument.getIngestMetadata().size(), equalTo(0));
     }
 
     public void testRemoveInnerField() {
         ingestDocument.removeField("fizz.buzz");
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize));
         assertThat(ingestDocument.getSourceAndMetadata().get("fizz"), instanceOf(Map.class));
         @SuppressWarnings("unchecked")
         Map<String, Object> map = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("fizz");
@@ -837,17 +854,17 @@ public void testRemoveInnerField() {
 
         ingestDocument.removeField("fizz.foo_null");
         assertThat(map.size(), equalTo(2));
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(true));
 
         ingestDocument.removeField("fizz.1");
         assertThat(map.size(), equalTo(1));
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(true));
 
         ingestDocument.removeField("fizz.list");
         assertThat(map.size(), equalTo(0));
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(true));
     }
 
@@ -883,7 +900,7 @@ public void testRemoveSourceObject() {
 
     public void testRemoveIngestObject() {
         ingestDocument.removeField("_ingest");
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(7));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize - 1));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("_ingest"), equalTo(false));
     }
 
@@ -905,7 +922,7 @@ public void testRemoveEmptyPathAfterStrippingOutPrefix() {
 
     public void testListRemoveField() {
         ingestDocument.removeField("list.0.field");
-        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8));
+        assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(initialSourceAndMetadataSize));
         assertThat(ingestDocument.getSourceAndMetadata().containsKey("list"), equalTo(true));
         Object object = ingestDocument.getSourceAndMetadata().get("list");
         assertThat(object, instanceOf(List.class));

From e22b651ed06fb93165a4c029302adb1c504ef722 Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Tue, 18 Jun 2024 16:17:31 +0530
Subject: [PATCH 009/113] [Remote Store] Rate limiter for low priority  uploads
 (#14374)

---------
Signed-off-by: Gaurav Bafna <gbbafna@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../indices/create/RemoteCloneIndexIT.java    | 86 +++++++++++++++++++
 .../MockFsMetadataSupportedRepository.java    |  4 +-
 .../index/store/RemoteDirectory.java          | 18 +++-
 .../RemoteSegmentStoreDirectoryFactory.java   |  1 +
 .../repositories/FilterRepository.java        |  5 ++
 .../opensearch/repositories/Repository.java   |  9 +-
 .../blobstore/BlobStoreRepository.java        | 28 ++++++
 8 files changed, 147 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a43c0acf3219a..6654b478c74f8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 ### Added
 - Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724))
+- [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
index a1122f279c7e4..acbd68fff6dd0 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
@@ -42,24 +42,32 @@
 import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.admin.indices.shrink.ResizeType;
 import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.opensearch.client.Requests;
+import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.query.TermsQueryBuilder;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
+import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.VersionUtils;
+import org.junit.Before;
 
 import java.util.concurrent.ExecutionException;
 
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteCloneIndexIT extends RemoteStoreBaseIntegTestCase {
@@ -69,6 +77,11 @@ protected boolean forbidPrivateIndexSettings() {
         return false;
     }
 
+    @Before
+    public void setup() {
+        asyncUploadMockFsRepo = true;
+    }
+
     public void testCreateCloneIndex() {
         Version version = VersionUtils.randomIndexCompatibleVersion(random());
         int numPrimaryShards = randomIntBetween(1, 5);
@@ -140,6 +153,79 @@ public void testCreateCloneIndex() {
 
     }
 
+    public void testCreateCloneIndexLowPriorityRateLimit() {
+        Version version = VersionUtils.randomIndexCompatibleVersion(random());
+        int numPrimaryShards = 1;
+        prepareCreate("source").setSettings(
+            Settings.builder().put(indexSettings()).put("number_of_shards", numPrimaryShards).put("index.version.created", version)
+        ).get();
+        final int docs = randomIntBetween(0, 128);
+        for (int i = 0; i < docs; i++) {
+            client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+        }
+        ByteSizeValue shardSize = client().admin().indices().prepareStats("source").execute().actionGet().getShards()[0].getStats()
+            .getStore()
+            .size();
+        logger.info("Shard size is {}", shardSize);
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.blocks.write", true)).get();
+        ensureGreen();
+
+        // disable rebalancing to be able to capture the right stats. balancing can move the target primary
+        // making it hard to pin point the source shards.
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none"))
+            .get();
+        try {
+            // apply rate limiter
+            setLowPriorityUploadRate(REPOSITORY_NAME, "1kb");
+            assertAcked(
+                client().admin()
+                    .indices()
+                    .prepareResizeIndex("source", "target")
+                    .setResizeType(ResizeType.CLONE)
+                    .setSettings(Settings.builder().put("index.number_of_replicas", 0).putNull("index.blocks.write").build())
+                    .get()
+            );
+            ensureGreen();
+            long uploadPauseTime = 0L;
+            for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) {
+                uploadPauseTime += repositoriesService.repository(REPOSITORY_NAME).getLowPriorityRemoteUploadThrottleTimeInNanos();
+            }
+            assertThat(uploadPauseTime, greaterThan(TimeValue.timeValueSeconds(randomIntBetween(5, 10)).nanos()));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        } finally {
+            // clean up
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(
+                    Settings.builder()
+                        .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                        .put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), (String) null)
+                )
+                .get();
+        }
+    }
+
+    protected void setLowPriorityUploadRate(String repoName, String value) throws ExecutionException, InterruptedException {
+        GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { repoName });
+        GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get();
+        RepositoryMetadata rmd = res.repositories().get(0);
+        Settings.Builder settings = Settings.builder()
+            .put("location", rmd.settings().get("location"))
+            .put("max_remote_low_priority_upload_bytes_per_sec", value);
+        assertAcked(client().admin().cluster().preparePutRepository(repoName).setType(rmd.type()).setSettings(settings).get());
+    }
+
     public void testCreateCloneIndexFailure() throws ExecutionException, InterruptedException {
         asyncUploadMockFsRepo = false;
         Version version = VersionUtils.randomIndexCompatibleVersion(random());
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/translogmetadata/mocks/MockFsMetadataSupportedRepository.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/translogmetadata/mocks/MockFsMetadataSupportedRepository.java
index 333fba413ce4e..1abacbe5091dd 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/translogmetadata/mocks/MockFsMetadataSupportedRepository.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/translogmetadata/mocks/MockFsMetadataSupportedRepository.java
@@ -16,9 +16,9 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.indices.recovery.RecoverySettings;
-import org.opensearch.repositories.fs.FsRepository;
+import org.opensearch.repositories.fs.ReloadableFsRepository;
 
-public class MockFsMetadataSupportedRepository extends FsRepository {
+public class MockFsMetadataSupportedRepository extends ReloadableFsRepository {
 
     public static Setting<Boolean> TRIGGER_DATA_INTEGRITY_FAILURE = Setting.boolSetting(
         "mock_fs_repository.trigger_data_integrity_failure",
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
index ab76150f8f83d..99f78130ad3ef 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
@@ -64,6 +64,8 @@ public class RemoteDirectory extends Directory {
 
     private final UnaryOperator<OffsetRangeInputStream> uploadRateLimiter;
 
+    private final UnaryOperator<OffsetRangeInputStream> lowPriorityUploadRateLimiter;
+
     private final UnaryOperator<InputStream> downloadRateLimiter;
 
     /**
@@ -76,15 +78,17 @@ public BlobContainer getBlobContainer() {
     }
 
     public RemoteDirectory(BlobContainer blobContainer) {
-        this(blobContainer, UnaryOperator.identity(), UnaryOperator.identity());
+        this(blobContainer, UnaryOperator.identity(), UnaryOperator.identity(), UnaryOperator.identity());
     }
 
     public RemoteDirectory(
         BlobContainer blobContainer,
         UnaryOperator<OffsetRangeInputStream> uploadRateLimiter,
+        UnaryOperator<OffsetRangeInputStream> lowPriorityUploadRateLimiter,
         UnaryOperator<InputStream> downloadRateLimiter
     ) {
         this.blobContainer = blobContainer;
+        this.lowPriorityUploadRateLimiter = lowPriorityUploadRateLimiter;
         this.uploadRateLimiter = uploadRateLimiter;
         this.downloadRateLimiter = downloadRateLimiter;
     }
@@ -357,13 +361,23 @@ private void uploadBlob(
             remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) getBlobContainer()).remoteIntegrityCheckSupported();
         }
         lowPriorityUpload = lowPriorityUpload || contentLength > ByteSizeUnit.GB.toBytes(15);
+        RemoteTransferContainer.OffsetRangeInputStreamSupplier offsetRangeInputStreamSupplier;
+        if (lowPriorityUpload) {
+            offsetRangeInputStreamSupplier = (size, position) -> lowPriorityUploadRateLimiter.apply(
+                new OffsetRangeIndexInputStream(from.openInput(src, ioContext), size, position)
+            );
+        } else {
+            offsetRangeInputStreamSupplier = (size, position) -> uploadRateLimiter.apply(
+                new OffsetRangeIndexInputStream(from.openInput(src, ioContext), size, position)
+            );
+        }
         RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer(
             src,
             remoteFileName,
             contentLength,
             true,
             lowPriorityUpload ? WritePriority.LOW : WritePriority.NORMAL,
-            (size, position) -> uploadRateLimiter.apply(new OffsetRangeIndexInputStream(from.openInput(src, ioContext), size, position)),
+            offsetRangeInputStreamSupplier,
             expectedChecksum,
             remoteIntegrityEnabled
         );
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java
index b965d7ce73ae6..3f6f4eeeef87b 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java
@@ -77,6 +77,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
             RemoteDirectory dataDirectory = new RemoteDirectory(
                 blobStoreRepository.blobStore().blobContainer(dataPath),
                 blobStoreRepository::maybeRateLimitRemoteUploadTransfers,
+                blobStoreRepository::maybeRateLimitLowPriorityRemoteUploadTransfers,
                 blobStoreRepository::maybeRateLimitRemoteDownloadTransfers
             );
 
diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java
index 697ac37c4a175..d700a92ed4bad 100644
--- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java
@@ -142,6 +142,11 @@ public long getRemoteUploadThrottleTimeInNanos() {
         return in.getRemoteUploadThrottleTimeInNanos();
     }
 
+    @Override
+    public long getLowPriorityRemoteUploadThrottleTimeInNanos() {
+        return in.getRemoteUploadThrottleTimeInNanos();
+    }
+
     @Override
     public long getRemoteDownloadThrottleTimeInNanos() {
         return in.getRemoteDownloadThrottleTimeInNanos();
diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java
index b3f1e9ce2eed9..ed30aad7b4dd2 100644
--- a/server/src/main/java/org/opensearch/repositories/Repository.java
+++ b/server/src/main/java/org/opensearch/repositories/Repository.java
@@ -207,10 +207,17 @@ default void deleteSnapshotsAndReleaseLockFiles(
     long getRestoreThrottleTimeInNanos();
 
     /**
-     * Returns restore throttle time in nanoseconds
+     * Returns upload throttle time in nanoseconds
      */
     long getRemoteUploadThrottleTimeInNanos();
 
+    /**
+     * Returns low priority upload throttle time in nanoseconds
+     */
+    default long getLowPriorityRemoteUploadThrottleTimeInNanos() {
+        return 0;
+    }
+
     /**
      * Returns restore throttle time in nanoseconds
      */
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index 1a5701d9204ef..c41e97d278dd5 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -316,6 +316,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private volatile RateLimiter remoteUploadRateLimiter;
 
+    private volatile RateLimiter remoteUploadLowPriorityRateLimiter;
+
     private volatile RateLimiter remoteDownloadRateLimiter;
 
     private final CounterMetric snapshotRateLimitingTimeInNanos = new CounterMetric();
@@ -326,6 +328,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private final CounterMetric remoteUploadRateLimitingTimeInNanos = new CounterMetric();
 
+    private final CounterMetric remoteUploadLowPriorityRateLimitingTimeInNanos = new CounterMetric();
+
     public static final ChecksumBlobStoreFormat<Metadata> GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
         "metadata",
         METADATA_NAME_FORMAT,
@@ -445,6 +449,11 @@ private void readRepositoryMetadata(RepositoryMetadata repositoryMetadata) {
         snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", ByteSizeValue.ZERO);
         remoteUploadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_upload_bytes_per_sec", ByteSizeValue.ZERO);
+        remoteUploadLowPriorityRateLimiter = getRateLimiter(
+            metadata.settings(),
+            "max_remote_low_priority_upload_bytes_per_sec",
+            ByteSizeValue.ZERO
+        );
         remoteDownloadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_download_bytes_per_sec", ByteSizeValue.ZERO);
         readOnly = READONLY_SETTING.get(metadata.settings());
         cacheRepositoryData = CACHE_REPOSITORY_DATA.get(metadata.settings());
@@ -1882,6 +1891,11 @@ public long getRemoteUploadThrottleTimeInNanos() {
         return remoteUploadRateLimitingTimeInNanos.count();
     }
 
+    @Override
+    public long getLowPriorityRemoteUploadThrottleTimeInNanos() {
+        return remoteUploadLowPriorityRateLimitingTimeInNanos.count();
+    }
+
     @Override
     public long getRemoteDownloadThrottleTimeInNanos() {
         return remoteDownloadRateLimitingTimeInNanos.count();
@@ -3177,6 +3191,20 @@ public OffsetRangeInputStream maybeRateLimitRemoteUploadTransfers(OffsetRangeInp
         );
     }
 
+    public OffsetRangeInputStream maybeRateLimitLowPriorityRemoteUploadTransfers(OffsetRangeInputStream offsetRangeInputStream) {
+        return maybeRateLimitRemoteTransfers(
+            maybeRateLimitRemoteTransfers(
+                offsetRangeInputStream,
+                () -> remoteUploadRateLimiter,
+                remoteUploadRateLimitingTimeInNanos,
+                BlobStoreTransferContext.REMOTE_UPLOAD
+            ),
+            () -> remoteUploadLowPriorityRateLimiter,
+            remoteUploadLowPriorityRateLimitingTimeInNanos,
+            BlobStoreTransferContext.REMOTE_UPLOAD
+        );
+    }
+
     public InputStream maybeRateLimitRemoteDownloadTransfers(InputStream inputStream) {
         return maybeRateLimit(
             maybeRateLimit(

From 3a0c0c0b38c0b42bc519c3673d5cd4a1e3379550 Mon Sep 17 00:00:00 2001
From: Sachin Kale <sachinpkale@gmail.com>
Date: Tue, 18 Jun 2024 16:28:44 +0530
Subject: [PATCH 010/113] Fix flaky test
 IndexShardTests.testCommitLevelRestoreShardFromRemoteStore (#14418)

Signed-off-by: Sachin Kale <kalsac@amazon.com>
---
 .../java/org/opensearch/index/shard/IndexShardTests.java  | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index e5bfa8caee79a..3188de13bb00b 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -2956,6 +2956,14 @@ public void testRestoreShardFromRemoteStore(boolean performFlush) throws IOExcep
             )
         );
 
+        // Make sure to drain refreshes from the shard. Otherwise, if the refresh is in-progress, it overlaps with
+        // deletion of segment files in the subsequent code block.
+        for (ReferenceManager.RefreshListener refreshListener : target.getEngine().config().getInternalRefreshListener()) {
+            if (refreshListener instanceof ReleasableRetryableRefreshListener) {
+                ((ReleasableRetryableRefreshListener) refreshListener).drainRefreshes();
+            }
+        }
+
         // Delete files in store directory to restore from remote directory
         Directory storeDirectory = target.store().directory();
 

From 2a55a2a0c61e264d4d68759af9f82ea5e5e0e7d5 Mon Sep 17 00:00:00 2001
From: Prudhvi Godithi <pgodithi@amazon.com>
Date: Tue, 18 Jun 2024 09:14:06 -0700
Subject: [PATCH 011/113] Update DEVELOPER_GUIDE.md to add
 gradle-check-flaky-test-detector automation information (#14417)

Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>
---
 DEVELOPER_GUIDE.md | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
index bc11e7335af49..03cd189aa911e 100644
--- a/DEVELOPER_GUIDE.md
+++ b/DEVELOPER_GUIDE.md
@@ -651,16 +651,18 @@ Note that these snapshots do not follow the Maven [naming convention](https://ma
 
 ### Flaky Tests
 
-OpenSearch has a very large test suite with long running, often failing (flaky), integration tests. Such individual tests are labelled as [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22). Your help is wanted fixing these!
+If you encounter a test failure locally or in CI that is seemingly unrelated to the change in your pull request, it may be a known flaky test or a new test failure. OpenSearch has a very large test suite with long running, often failing (flaky), integration tests. Such individual tests are labelled as [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22). Your help is wanted fixing these!
 
-If you encounter a build/test failure in CI that is unrelated to the change in your pull request, it may be a known flaky test, or a new test failure.
+The automation [gradle-check-flaky-test-detector](https://build.ci.opensearch.org/job/gradle-check-flaky-test-detector/), which runs in OpenSearch public Jenkins, identifies failing flaky issues that are part of post-merge actions. Once a flaky test is identified, the automation creates an issue with detailed report that includes links to all relevant commits, the Gradle check build log, the test report, and pull requests that are impacted with the flaky test failures. This automation leverages data from the [OpenSearch Metrics Project](https://github.com/opensearch-project/opensearch-metrics) to establish a baseline for creating the issue and updating the flaky test report. For all flaky test issues created by automation, visit this [link](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%3Etest-failure+author%3Aopensearch-ci-bot).
+
+If you still see a failing test that is not part of the post merge actions, please do:
+
+* Follow failed CI links, and locate the failing test(s) or use the [Gradle Check Metrics Dashboard](#gradle-check-metrics-dashboard).
+* Copy-paste the failure into a comment of your PR.
+* Search through issues using the name of the failed test for whether this is a known flaky test.
+* If no existing issue is found, open one.
+* Retry CI via the GitHub UX or by pushing an update to your PR.
 
-1. Follow failed CI links, and locate the failing test(s).
-2. Copy-paste the failure into a comment of your PR.
-3. Search through [issues](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) using the name of the failed test for whether this is a known flaky test.
-4. If an existing issue is found, paste a link to the known issue in a comment to your PR.
-5. If no existing issue is found, open one.
-6. Retry CI via the GitHub UX or by pushing an update to your PR.
 
 ### Gradle Check Metrics Dashboard
 

From daf4ac7e826edfb7b60ef93371c30935b29916d4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 18 Jun 2024 12:15:05 -0400
Subject: [PATCH 012/113] Bump com.nimbusds:nimbus-jose-jwt from 9.37.3 to 9.40
 in /plugins/repository-azure (#14398)

* Bump com.nimbusds:nimbus-jose-jwt in /plugins/repository-azure

Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.37.3 to 9.40.
- [Changelog](https://bitbucket.org/connect2id/nimbus-jose-jwt/src/master/CHANGELOG.txt)
- [Commits](https://bitbucket.org/connect2id/nimbus-jose-jwt/branches/compare/9.40..9.37.3)

---
updated-dependencies:
- dependency-name: com.nimbusds:nimbus-jose-jwt
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                               | 1 +
 plugins/repository-azure/build.gradle                      | 7 +------
 .../licenses/nimbus-jose-jwt-9.37.3.jar.sha1               | 1 -
 .../licenses/nimbus-jose-jwt-9.40.jar.sha1                 | 1 +
 4 files changed, 3 insertions(+), 7 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6654b478c74f8..529a5ce57ddf3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `reactor` from 3.5.17 to 3.5.18 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
 - Bump `reactor-netty` from 1.1.19 to 1.1.20 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
 - Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.37.3 to 9.40 ([#14398](https://github.com/opensearch-project/OpenSearch/pull/14398))
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
 - Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
 
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 61e9f71712eaf..0fd30af71dd0a 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -63,7 +63,7 @@ dependencies {
   api "net.java.dev.jna:jna-platform:${versions.jna}"
   api 'com.microsoft.azure:msal4j:1.14.3'
   api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
-  api 'com.nimbusds:nimbus-jose-jwt:9.37.3'
+  api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api 'com.nimbusds:content-type:2.3'
   api 'com.nimbusds:lang-tag:1.7'
   // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,
@@ -219,11 +219,6 @@ thirdPartyAudit {
     'org.bouncycastle.cert.X509CertificateHolder',
     'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
     'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
-    'org.bouncycastle.crypto.InvalidCipherTextException',
-    'org.bouncycastle.crypto.engines.AESEngine',
-    'org.bouncycastle.crypto.modes.GCMBlockCipher',
-    'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
-    'org.bouncycastle.jce.provider.BouncyCastleProvider',
     'org.bouncycastle.openssl.PEMKeyPair',
     'org.bouncycastle.openssl.PEMParser',
     'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
deleted file mode 100644
index 7278cd8994f71..0000000000000
--- a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-700f71ffefd60c16bd8ce711a956967ea9071cec
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1
new file mode 100644
index 0000000000000..83228caf233cc
--- /dev/null
+++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1
@@ -0,0 +1 @@
+42b1dfa0360e4062951b070bac52dd8d96fd7b38
\ No newline at end of file

From 903784b0afe756ee9f3e5eed7120f2289b207682 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Tue, 18 Jun 2024 12:15:34 -0400
Subject: [PATCH 013/113] Bump Apache Lucene to 9.12.0-snapshot-c896995
 (#14414)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 buildSrc/version.properties                                     | 2 +-
 libs/core/licenses/lucene-core-9.11.0.jar.sha1                  | 1 -
 libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1 | 1 +
 libs/core/src/main/java/org/opensearch/Version.java             | 2 +-
 .../lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1 | 1 -
 .../lucene-expressions-9.12.0-snapshot-c896995.jar.sha1         | 1 +
 .../analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1   | 1 -
 .../lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1        | 1 +
 .../licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1           | 1 -
 .../lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1   | 1 +
 .../analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1 | 1 -
 .../lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1       | 1 +
 .../licenses/lucene-analysis-phonetic-9.11.0.jar.sha1           | 1 -
 .../lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1   | 1 +
 .../licenses/lucene-analysis-smartcn-9.11.0.jar.sha1            | 1 -
 .../lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1    | 1 +
 .../licenses/lucene-analysis-stempel-9.11.0.jar.sha1            | 1 -
 .../lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1    | 1 +
 .../licenses/lucene-analysis-morfologik-9.11.0.jar.sha1         | 1 -
 .../lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1 | 1 +
 server/licenses/lucene-analysis-common-9.11.0.jar.sha1          | 1 -
 .../lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1     | 1 +
 server/licenses/lucene-backward-codecs-9.11.0.jar.sha1          | 1 -
 .../lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1     | 1 +
 server/licenses/lucene-core-9.11.0.jar.sha1                     | 1 -
 server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1    | 1 +
 server/licenses/lucene-grouping-9.11.0.jar.sha1                 | 1 -
 .../licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1   | 1 +
 server/licenses/lucene-highlighter-9.11.0.jar.sha1              | 1 -
 .../lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1         | 1 +
 server/licenses/lucene-join-9.11.0.jar.sha1                     | 1 -
 server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1    | 1 +
 server/licenses/lucene-memory-9.11.0.jar.sha1                   | 1 -
 server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1  | 1 +
 server/licenses/lucene-misc-9.11.0.jar.sha1                     | 1 -
 server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1    | 1 +
 server/licenses/lucene-queries-9.11.0.jar.sha1                  | 1 -
 server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1 | 1 +
 server/licenses/lucene-queryparser-9.11.0.jar.sha1              | 1 -
 .../lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1         | 1 +
 server/licenses/lucene-sandbox-9.11.0.jar.sha1                  | 1 -
 server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1 | 1 +
 server/licenses/lucene-spatial-extras-9.11.0.jar.sha1           | 1 -
 .../lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1      | 1 +
 server/licenses/lucene-spatial3d-9.11.0.jar.sha1                | 1 -
 .../licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1  | 1 +
 server/licenses/lucene-suggest-9.11.0.jar.sha1                  | 1 -
 server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1 | 1 +
 48 files changed, 25 insertions(+), 25 deletions(-)
 delete mode 100644 libs/core/licenses/lucene-core-9.11.0.jar.sha1
 create mode 100644 libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1
 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-analysis-common-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-backward-codecs-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-core-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-grouping-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-highlighter-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-join-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-memory-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-misc-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-queries-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-queryparser-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-sandbox-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-extras-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial3d-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1
 delete mode 100644 server/licenses/lucene-suggest-9.11.0.jar.sha1
 create mode 100644 server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index eb96261b056e3..af421b97e12e4 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.11.0
+lucene            = 9.12.0-snapshot-c896995
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9
diff --git a/libs/core/licenses/lucene-core-9.11.0.jar.sha1 b/libs/core/licenses/lucene-core-9.11.0.jar.sha1
deleted file mode 100644
index b0d38c4165581..0000000000000
--- a/libs/core/licenses/lucene-core-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e487755a6814b2a1bc770c26569dcba86873dcf
\ No newline at end of file
diff --git a/libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1 b/libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..299283562fddc
--- /dev/null
+++ b/libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+826b328c37ea7f27c05d685db03bf8d2b00457ff
\ No newline at end of file
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index d99dae2a5e64b..0cb2d4f867c12 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -106,7 +106,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_14_1 = new Version(2140199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_0);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1
deleted file mode 100644
index 29aade3ad4298..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e21d20edee0712472e7c6f605c9d97aeecf16c0
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..6d8d3be59f945
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+9f0321cf2d34fca3f1f9334fdfee2b79d9d27444
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1
deleted file mode 100644
index 6f0501d3312ae..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5c7f2d8eab0fca3fdc3d3e57a7f48a335dc7ac33
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..696803bf63b46
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+e6314f36fb29e208d58c0470f14269c9c36996ba
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1
deleted file mode 100644
index 25031381c9cb3..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-efcf65dda1b4e9d7e83926fd5895a47e491cbf29
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..7a12077d7fc62
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+77fbf1e37af79715f28f66d8cc5b50af2982fc54
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1
deleted file mode 100644
index e27d45b217dad..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-59599d7b8bed2e6bd27d0dad7935c078b98c39cc
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..efed62c7e5e5b
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+a7a4e9c6004c72782e1002e1dcfaf4fbab7887d8
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0.jar.sha1
deleted file mode 100644
index ad5473865537d..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e55f83bb373ac139e313f64e80afe1eb0a75b8c0
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..f2020abcb8ef7
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+42ac148a3769d6eb880d7f184d1917bad48ca303
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0.jar.sha1
deleted file mode 100644
index 68abd162e7266..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1be59d91c45a4de069611fb7f8aa3e8fd26020ec
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..b64e4061311e5
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+adf2a25339ac8722647f8196288c1f5056bbf0de
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0.jar.sha1
deleted file mode 100644
index c5f1521ec3769..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d5b5922acf3743b5a0c542959dd93fca8be333a7
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..f56e7fc5df766
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+a689e3af2015b21b7b4f41a1206b50c44519b6f7
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0.jar.sha1
deleted file mode 100644
index b676ca507467a..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-50fd7b471cbdd6648c4972169f3fc67fae9db7f6
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..30732e3c4a688
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+c875f7706ee81b1fb0b3443767a8c9c52f30abc5
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.11.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.11.0.jar.sha1
deleted file mode 100644
index 7139f6a43a15a..0000000000000
--- a/server/licenses/lucene-analysis-common-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-75a0a333cf1e043102743066c929e65fe51cbcda
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..4b545e061c52f
--- /dev/null
+++ b/server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+73696492c6e59972974cd91e03ad9464e6b5bfcd
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.11.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.11.0.jar.sha1
deleted file mode 100644
index 735e80b60b001..0000000000000
--- a/server/licenses/lucene-backward-codecs-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-db385446bc3fd70e7c6a744276c0a157bd60ee0a
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..ae4ffb2b1800b
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+3cbb29ecc873e8c880a6f32e739655551708dbcf
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.11.0.jar.sha1 b/server/licenses/lucene-core-9.11.0.jar.sha1
deleted file mode 100644
index b0d38c4165581..0000000000000
--- a/server/licenses/lucene-core-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e487755a6814b2a1bc770c26569dcba86873dcf
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..299283562fddc
--- /dev/null
+++ b/server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+826b328c37ea7f27c05d685db03bf8d2b00457ff
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.11.0.jar.sha1 b/server/licenses/lucene-grouping-9.11.0.jar.sha1
deleted file mode 100644
index 562de95605b60..0000000000000
--- a/server/licenses/lucene-grouping-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-882bdaf209b0acb332aa34836616424bcbecf462
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..b0268c98167d3
--- /dev/null
+++ b/server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+a3a7003dc83197523e830f058a3748dbea96cab7
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.11.0.jar.sha1 b/server/licenses/lucene-highlighter-9.11.0.jar.sha1
deleted file mode 100644
index e0ef36d321c9d..0000000000000
--- a/server/licenses/lucene-highlighter-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-44accdc03c5482e602718f7bf91e5940ba4e4870
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..d87927364b5a8
--- /dev/null
+++ b/server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+00eb386915c3cffa9efcef2dc4c406f8a6776afe
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.11.0.jar.sha1 b/server/licenses/lucene-join-9.11.0.jar.sha1
deleted file mode 100644
index 34c618ccfbcc7..0000000000000
--- a/server/licenses/lucene-join-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-32a30ee03ed4f3e43bf63250270b2d4d53050045
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..25a95546ab544
--- /dev/null
+++ b/server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+bb1fc572da7d473bf39672fd8ac323b15a1ffff0
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.11.0.jar.sha1 b/server/licenses/lucene-memory-9.11.0.jar.sha1
deleted file mode 100644
index d730cfb4b7660..0000000000000
--- a/server/licenses/lucene-memory-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b3e80aa6aa3299118e76a23edc23b58f3ba5a515
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..a0b3fd812561c
--- /dev/null
+++ b/server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+05ebfcef0435f4870859a19c93020e24398bb939
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.11.0.jar.sha1 b/server/licenses/lucene-misc-9.11.0.jar.sha1
deleted file mode 100644
index 9be27f004435b..0000000000000
--- a/server/licenses/lucene-misc-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-54fe308908194e1b0697a1157a45c5998c9e1083
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..1e2cc97c37257
--- /dev/null
+++ b/server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+d5747ed1be242b59aa36b0c32b0d3bd26b1d8fb8
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.11.0.jar.sha1 b/server/licenses/lucene-queries-9.11.0.jar.sha1
deleted file mode 100644
index b445610c25858..0000000000000
--- a/server/licenses/lucene-queries-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-987d1286949ddf514b8405fd453ed47bebdfb12d
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..31d4fe2886fc1
--- /dev/null
+++ b/server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+fb6678d7fe035e55c545450682b67be49457ef1b
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.11.0.jar.sha1 b/server/licenses/lucene-queryparser-9.11.0.jar.sha1
deleted file mode 100644
index a1620ba9c7708..0000000000000
--- a/server/licenses/lucene-queryparser-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e97fe1c0d102edb8d6e1c01454992fd2b8d80ae0
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..754e4ea20765f
--- /dev/null
+++ b/server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+a11d7f56a9e78dc8e61f85b9b54ad94d73583bb3
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.11.0.jar.sha1 b/server/licenses/lucene-sandbox-9.11.0.jar.sha1
deleted file mode 100644
index 0dc193f054973..0000000000000
--- a/server/licenses/lucene-sandbox-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e46b790744bd9118ccc053f70235364213312a5
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..08c2bc48ae85b
--- /dev/null
+++ b/server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+75352855bcc052abfba821f878a27fd2b328fb1c
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.11.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.11.0.jar.sha1
deleted file mode 100644
index 9d3a8d2857db6..0000000000000
--- a/server/licenses/lucene-spatial-extras-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-079ca5aaf544a3acde84b8b88423ace6dedc23eb
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..5e0b7196f48c2
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+299be103216d67ca092bef177642b275224e77a6
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.11.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.11.0.jar.sha1
deleted file mode 100644
index fd5ff875a0113..0000000000000
--- a/server/licenses/lucene-spatial3d-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-564558818d70fc384db5b36fbc8a0ab27b107609
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..c79b34adea5e2
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+29b4a76cd0bdabe0e067063831e661dedac6e503
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.11.0.jar.sha1 b/server/licenses/lucene-suggest-9.11.0.jar.sha1
deleted file mode 100644
index 2fa96e97f307a..0000000000000
--- a/server/licenses/lucene-suggest-9.11.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aa345db9b6caaf881e7890ea5b8911357d592167
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1
new file mode 100644
index 0000000000000..8d5334f0c4619
--- /dev/null
+++ b/server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1
@@ -0,0 +1 @@
+597edb659e9ea93398a816e6837da7d47ef53873
\ No newline at end of file

From 823ce6819ffe6b6ca4451c5bb28f44b95990d569 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 18 Jun 2024 10:18:25 -0700
Subject: [PATCH 014/113] [Tiered Cache] Use ConcurrentHashMap explicitly in
 IndicesRequestCache (#14409)

Signed-off-by: Kiran Prakash <awskiran@amazon.com>
---
 .../indices/IndicesRequestCacheIT.java        |  2 +-
 .../indices/IndicesRequestCache.java          | 10 +++----
 .../indices/IndicesRequestCacheTests.java     | 26 ++++++++++++-------
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 299652e4f07a9..0383aca2de33f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -1168,7 +1168,7 @@ public void testCacheCleanupAfterIndexDeletion() throws Exception {
         }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
     }
 
-    // when staleness threshold is lower than staleness, it should clean the cache from all indices having stale keys
+    // when staleness threshold is lower than staleness, it should clean cache from all indices having stale keys
     public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
         int cacheCleanIntervalInMillis = 10;
         String node = internalCluster().startNode(
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 06cd77a34fe0b..93946fa11de13 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -81,6 +81,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -506,7 +507,7 @@ public int hashCode() {
      * */
     class IndicesRequestCacheCleanupManager implements Closeable {
         private final Set<CleanupKey> keysToClean;
-        private final ConcurrentMap<ShardId, ConcurrentMap<String, Integer>> cleanupKeyToCountMap;
+        private final ConcurrentHashMap<ShardId, ConcurrentHashMap<String, Integer>> cleanupKeyToCountMap;
         private final AtomicInteger staleKeysCount;
         private volatile double stalenessThreshold;
         private final IndicesRequestCacheCleaner cacheCleaner;
@@ -514,7 +515,7 @@ class IndicesRequestCacheCleanupManager implements Closeable {
         IndicesRequestCacheCleanupManager(ThreadPool threadpool, TimeValue cleanInterval, double stalenessThreshold) {
             this.stalenessThreshold = stalenessThreshold;
             this.keysToClean = ConcurrentCollections.newConcurrentSet();
-            this.cleanupKeyToCountMap = ConcurrentCollections.newConcurrentMap();
+            this.cleanupKeyToCountMap = new ConcurrentHashMap<>();
             this.staleKeysCount = new AtomicInteger(0);
             this.cacheCleaner = new IndicesRequestCacheCleaner(this, threadpool, cleanInterval);
             threadpool.schedule(cacheCleaner, cleanInterval, ThreadPool.Names.SAME);
@@ -572,8 +573,7 @@ private void updateStaleCountOnCacheInsert(CleanupKey cleanupKey) {
 
         // pkg-private for testing
         void addToCleanupKeyToCountMap(ShardId shardId, String readerCacheKeyId) {
-            cleanupKeyToCountMap.computeIfAbsent(shardId, k -> ConcurrentCollections.newConcurrentMap())
-                .merge(readerCacheKeyId, 1, Integer::sum);
+            cleanupKeyToCountMap.computeIfAbsent(shardId, k -> new ConcurrentHashMap<>()).merge(readerCacheKeyId, 1, Integer::sum);
         }
 
         /**
@@ -831,7 +831,7 @@ public void close() {
         }
 
         // for testing
-        ConcurrentMap<ShardId, ConcurrentMap<String, Integer>> getCleanupKeyToCountMap() {
+        ConcurrentHashMap<ShardId, ConcurrentHashMap<String, Integer>> getCleanupKeyToCountMap() {
             return cleanupKeyToCountMap;
         }
 
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 205712d388cd1..10688de3ab0ae 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -101,7 +101,6 @@
 import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -491,7 +490,8 @@ public void testStaleCount_OnRemovalNotificationOfStaleKey_DecrementsStaleCount(
             indexShard.hashCode()
         );
         // test the mapping
-        ConcurrentMap<ShardId, ConcurrentMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
+        ConcurrentHashMap<ShardId, ConcurrentHashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager
+            .getCleanupKeyToCountMap();
         // shard id should exist
         assertTrue(cleanupKeyToCountMap.containsKey(shardId));
         // reader CacheKeyId should NOT exist
@@ -554,7 +554,8 @@ public void testStaleCount_OnRemovalNotificationOfNonStaleKey_DoesNotDecrementsS
         );
 
         // test the mapping
-        ConcurrentMap<ShardId, ConcurrentMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
+        ConcurrentHashMap<ShardId, ConcurrentHashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager
+            .getCleanupKeyToCountMap();
         // shard id should exist
         assertTrue(cleanupKeyToCountMap.containsKey(shardId));
         // reader CacheKeyId should NOT exist
@@ -722,7 +723,8 @@ public void testCleanupKeyToCountMapAreSetAppropriately() throws Exception {
         cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
         assertEquals(1, cache.count());
         // test the mappings
-        ConcurrentMap<ShardId, ConcurrentMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
+        ConcurrentHashMap<ShardId, ConcurrentHashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager
+            .getCleanupKeyToCountMap();
         assertEquals(1, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(reader)));
 
         cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
@@ -796,7 +798,7 @@ public void testCleanupKeyToCountMapAreSetAppropriately() throws Exception {
     }
 
     // test adding to cleanupKeyToCountMap with multiple threads
-    public void testAddToCleanupKeyToCountMap() throws Exception {
+    public void testAddingToCleanupKeyToCountMapWorksAppropriatelyWithMultipleThreads() throws Exception {
         threadPool = getThreadPool();
         Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "51%").build();
         cache = getIndicesRequestCache(settings);
@@ -804,7 +806,7 @@ public void testAddToCleanupKeyToCountMap() throws Exception {
         int numberOfThreads = 10;
         int numberOfIterations = 1000;
         Phaser phaser = new Phaser(numberOfThreads + 1); // +1 for the main thread
-        AtomicBoolean exceptionDetected = new AtomicBoolean(false);
+        AtomicBoolean concurrentModificationExceptionDetected = new AtomicBoolean(false);
 
         ExecutorService executorService = Executors.newFixedThreadPool(numberOfThreads);
 
@@ -817,7 +819,7 @@ public void testAddToCleanupKeyToCountMap() throws Exception {
                     }
                 } catch (ConcurrentModificationException e) {
                     logger.error("ConcurrentModificationException detected in thread : " + e.getMessage());
-                    exceptionDetected.set(true); // Set flag if exception is detected
+                    concurrentModificationExceptionDetected.set(true); // Set flag if exception is detected
                 }
             });
         }
@@ -836,13 +838,17 @@ public void testAddToCleanupKeyToCountMap() throws Exception {
                 }
             } catch (ConcurrentModificationException e) {
                 logger.error("ConcurrentModificationException detected in main thread : " + e.getMessage());
-                exceptionDetected.set(true); // Set flag if exception is detected
+                concurrentModificationExceptionDetected.set(true); // Set flag if exception is detected
             }
         });
 
         executorService.shutdown();
-        executorService.awaitTermination(60, TimeUnit.SECONDS);
-        assertFalse(exceptionDetected.get());
+        assertTrue(executorService.awaitTermination(60, TimeUnit.SECONDS));
+        assertEquals(
+            numberOfThreads * numberOfIterations,
+            cache.cacheCleanupManager.getCleanupKeyToCountMap().get(indexShard.shardId()).size()
+        );
+        assertFalse(concurrentModificationExceptionDetected.get());
     }
 
     private IndicesRequestCache getIndicesRequestCache(Settings settings) {

From 120678d9b5e1ec611303f4dc5b3ce9b96fe21531 Mon Sep 17 00:00:00 2001
From: Heemin Kim <heemin@amazon.com>
Date: Tue, 18 Jun 2024 10:31:53 -0700
Subject: [PATCH 015/113] Switch to iterative version of WKT format parser
 (#14086)

Signed-off-by: Heemin Kim <heemin@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../geometry/utils/WellKnownText.java         | 69 +++++++++++++++++--
 .../geometry/GeometryCollectionTests.java     | 30 ++++++++
 3 files changed, 94 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 529a5ce57ddf3..dc85bd4f85ffd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
+- Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
 
 ### Security
 
diff --git a/libs/geo/src/main/java/org/opensearch/geometry/utils/WellKnownText.java b/libs/geo/src/main/java/org/opensearch/geometry/utils/WellKnownText.java
index ed1d63e6d4fef..8ad135b8bc1ca 100644
--- a/libs/geo/src/main/java/org/opensearch/geometry/utils/WellKnownText.java
+++ b/libs/geo/src/main/java/org/opensearch/geometry/utils/WellKnownText.java
@@ -49,8 +49,10 @@
 import java.io.StreamTokenizer;
 import java.io.StringReader;
 import java.text.ParseException;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Deque;
 import java.util.List;
 import java.util.Locale;
 
@@ -67,6 +69,7 @@ public class WellKnownText {
     public static final String RPAREN = ")";
     public static final String COMMA = ",";
     public static final String NAN = "NaN";
+    public static final int MAX_DEPTH_OF_GEO_COLLECTION = 1000;
 
     private final String NUMBER = "<NUMBER>";
     private final String EOF = "END-OF-STREAM";
@@ -278,6 +281,16 @@ public Geometry fromWKT(String wkt) throws IOException, ParseException {
      */
     private Geometry parseGeometry(StreamTokenizer stream) throws IOException, ParseException {
         final String type = nextWord(stream).toLowerCase(Locale.ROOT);
+        switch (type) {
+            case "geometrycollection":
+                return parseGeometryCollection(stream);
+            default:
+                return parseSimpleGeometry(stream, type);
+        }
+    }
+
+    private Geometry parseSimpleGeometry(StreamTokenizer stream, String type) throws IOException, ParseException {
+        assert "geometrycollection".equals(type) == false;
         switch (type) {
             case "point":
                 return parsePoint(stream);
@@ -294,7 +307,7 @@ private Geometry parseGeometry(StreamTokenizer stream) throws IOException, Parse
             case "bbox":
                 return parseBBox(stream);
             case "geometrycollection":
-                return parseGeometryCollection(stream);
+                throw new IllegalStateException("Unexpected type: geometrycollection");
             case "circle": // Not part of the standard, but we need it for internal serialization
                 return parseCircle(stream);
         }
@@ -305,12 +318,56 @@ private GeometryCollection<Geometry> parseGeometryCollection(StreamTokenizer str
         if (nextEmptyOrOpen(stream).equals(EMPTY)) {
             return GeometryCollection.EMPTY;
         }
-        List<Geometry> shapes = new ArrayList<>();
-        shapes.add(parseGeometry(stream));
-        while (nextCloserOrComma(stream).equals(COMMA)) {
-            shapes.add(parseGeometry(stream));
+
+        List<Geometry> topLevelShapes = new ArrayList<>();
+        Deque<List<Geometry>> deque = new ArrayDeque<>();
+        deque.push(topLevelShapes);
+        boolean isFirstIteration = true;
+        List<Geometry> currentLevelShapes = null;
+        while (!deque.isEmpty()) {
+            List<Geometry> previousShapes = deque.pop();
+            if (currentLevelShapes != null) {
+                previousShapes.add(new GeometryCollection<>(currentLevelShapes));
+            }
+            currentLevelShapes = previousShapes;
+
+            if (isFirstIteration == true) {
+                isFirstIteration = false;
+            } else {
+                if (nextCloserOrComma(stream).equals(COMMA) == false) {
+                    // Done with current level, continue with parent level
+                    continue;
+                }
+            }
+            while (true) {
+                final String type = nextWord(stream).toLowerCase(Locale.ROOT);
+                if (type.equals("geometrycollection")) {
+                    if (nextEmptyOrOpen(stream).equals(EMPTY) == false) {
+                        // GEOMETRYCOLLECTION() -> 1 depth, GEOMETRYCOLLECTION(GEOMETRYCOLLECTION()) -> 2 depth
+                        // When parsing the top level geometry collection, the queue size is zero.
+                        // When max depth is 1, we don't want to push any sub geometry collection in the queue.
+                        // Therefore, we subtract 2 from max depth.
+                        if (deque.size() >= MAX_DEPTH_OF_GEO_COLLECTION - 2) {
+                            throw new IllegalArgumentException(
+                                "a geometry collection with a depth greater than " + MAX_DEPTH_OF_GEO_COLLECTION + " is not supported"
+                            );
+                        }
+                        deque.push(currentLevelShapes);
+                        currentLevelShapes = new ArrayList<>();
+                        continue;
+                    }
+                    currentLevelShapes.add(GeometryCollection.EMPTY);
+                } else {
+                    currentLevelShapes.add(parseSimpleGeometry(stream, type));
+                }
+
+                if (nextCloserOrComma(stream).equals(COMMA) == false) {
+                    break;
+                }
+            }
         }
-        return new GeometryCollection<>(shapes);
+
+        return new GeometryCollection<>(topLevelShapes);
     }
 
     private Point parsePoint(StreamTokenizer stream) throws IOException, ParseException {
diff --git a/libs/geo/src/test/java/org/opensearch/geometry/GeometryCollectionTests.java b/libs/geo/src/test/java/org/opensearch/geometry/GeometryCollectionTests.java
index 631b6456a77da..cd8bb8f585966 100644
--- a/libs/geo/src/test/java/org/opensearch/geometry/GeometryCollectionTests.java
+++ b/libs/geo/src/test/java/org/opensearch/geometry/GeometryCollectionTests.java
@@ -62,6 +62,11 @@ public void testBasicSerialization() throws IOException, ParseException {
 
         assertEquals("GEOMETRYCOLLECTION EMPTY", wkt.toWKT(GeometryCollection.EMPTY));
         assertEquals(GeometryCollection.EMPTY, wkt.fromWKT("GEOMETRYCOLLECTION EMPTY)"));
+
+        assertEquals(
+            new GeometryCollection<Geometry>(Arrays.asList(GeometryCollection.EMPTY)),
+            wkt.fromWKT("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION EMPTY)")
+        );
     }
 
     @SuppressWarnings("ConstantConditions")
@@ -86,4 +91,29 @@ public void testInitValidation() {
 
         new StandardValidator(true).validate(new GeometryCollection<Geometry>(Collections.singletonList(new Point(20, 10, 30))));
     }
+
+    public void testDeeplyNestedGeometryCollection() throws IOException, ParseException {
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        StringBuilder validGeometryCollectionHead = new StringBuilder("GEOMETRYCOLLECTION");
+        StringBuilder validGeometryCollectionTail = new StringBuilder(" EMPTY");
+        for (int i = 0; i < WellKnownText.MAX_DEPTH_OF_GEO_COLLECTION - 1; i++) {
+            validGeometryCollectionHead.append(" (GEOMETRYCOLLECTION");
+            validGeometryCollectionTail.append(")");
+        }
+        // Expect no exception
+        wkt.fromWKT(validGeometryCollectionHead.append(validGeometryCollectionTail).toString());
+
+        StringBuilder invalidGeometryCollectionHead = new StringBuilder("GEOMETRYCOLLECTION");
+        StringBuilder invalidGeometryCollectionTail = new StringBuilder(" EMPTY");
+        for (int i = 0; i < WellKnownText.MAX_DEPTH_OF_GEO_COLLECTION; i++) {
+            invalidGeometryCollectionHead.append(" (GEOMETRYCOLLECTION");
+            invalidGeometryCollectionTail.append(")");
+        }
+
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> wkt.fromWKT(invalidGeometryCollectionHead.append(invalidGeometryCollectionTail).toString())
+        );
+        assertEquals("a geometry collection with a depth greater than 1000 is not supported", ex.getMessage());
+    }
 }

From 802f2e6e4b21f27ddc6c01e7fc6f6cdcd69138d3 Mon Sep 17 00:00:00 2001
From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com>
Date: Tue, 18 Jun 2024 23:11:42 +0530
Subject: [PATCH 016/113] Fix flaky test
 RecoveryFromGatewayIT.testMultipleReplicaShardAssignmentWithDelayedAllocationAndDifferentNodeStartTimeInBatchMode
 (#14424)

Signed-off-by: Swetha Guptha <gupthasg@amazon.com>
Co-authored-by: Swetha Guptha <gupthasg@amazon.com>
---
 .../gateway/RecoveryFromGatewayIT.java        | 44 ++++++++-----------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java
index fc0a574c191b1..6296608c64d37 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java
@@ -34,7 +34,6 @@
 
 import org.apache.lucene.index.CorruptIndexException;
 import org.opensearch.Version;
-import org.opensearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
 import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
 import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.opensearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
@@ -101,6 +100,8 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -883,17 +884,20 @@ public void testMultipleReplicaShardAssignmentWithDelayedAllocationAndDifferentN
         assertEquals(YELLOW, health.getStatus());
         assertEquals(2, health.getUnassignedShards());
         // shard should be unassigned because of Allocation_Delayed
-        ClusterAllocationExplainResponse allocationExplainResponse = client().admin()
-            .cluster()
-            .prepareAllocationExplain()
-            .setIndex("test")
-            .setShard(0)
-            .setPrimary(false)
-            .get();
-        assertEquals(
-            AllocationDecision.ALLOCATION_DELAYED,
-            allocationExplainResponse.getExplanation().getShardAllocationDecision().getAllocateDecision().getAllocationDecision()
+        BooleanSupplier delayedShardAllocationStatusVerificationSupplier = () -> AllocationDecision.ALLOCATION_DELAYED.equals(
+            client().admin()
+                .cluster()
+                .prepareAllocationExplain()
+                .setIndex("test")
+                .setShard(0)
+                .setPrimary(false)
+                .get()
+                .getExplanation()
+                .getShardAllocationDecision()
+                .getAllocateDecision()
+                .getAllocationDecision()
         );
+        waitUntil(delayedShardAllocationStatusVerificationSupplier, 2, TimeUnit.MINUTES);
 
         logger.info("--> restarting the node 1");
         internalCluster().startDataOnlyNode(
@@ -903,26 +907,16 @@ public void testMultipleReplicaShardAssignmentWithDelayedAllocationAndDifferentN
         assertTrue(clusterRerouteResponse.isAcknowledged());
         ensureStableCluster(6);
         waitUntil(
-            () -> client().admin().cluster().health(Requests.clusterHealthRequest().timeout("5m")).actionGet().getInitializingShards() == 0
+            () -> client().admin().cluster().health(Requests.clusterHealthRequest().timeout("5m")).actionGet().getActiveShards() == 3,
+            2,
+            TimeUnit.MINUTES
         );
-
         health = client().admin().cluster().health(Requests.clusterHealthRequest().timeout("5m")).actionGet();
         assertFalse(health.isTimedOut());
         assertEquals(YELLOW, health.getStatus());
         assertEquals(1, health.getUnassignedShards());
         assertEquals(1, health.getDelayedUnassignedShards());
-        allocationExplainResponse = client().admin()
-            .cluster()
-            .prepareAllocationExplain()
-            .setIndex("test")
-            .setShard(0)
-            .setPrimary(false)
-            .get();
-        assertEquals(
-            AllocationDecision.ALLOCATION_DELAYED,
-            allocationExplainResponse.getExplanation().getShardAllocationDecision().getAllocateDecision().getAllocationDecision()
-        );
-
+        waitUntil(delayedShardAllocationStatusVerificationSupplier, 2, TimeUnit.MINUTES);
         logger.info("--> restarting the node 0");
         internalCluster().startDataOnlyNode(
             Settings.builder().put("node.name", nodesWithReplicaShards.get(1)).put(replicaNode1DataPathSettings).build()

From c5c6024da6d53c145189ad898515ae4874fe7fce Mon Sep 17 00:00:00 2001
From: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
Date: Tue, 18 Jun 2024 11:36:12 -0700
Subject: [PATCH 017/113] Set INDICES_MAX_CLAUSE_COUNT dynamically (#13568)

---------

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
---
 CHANGELOG.md                                  |  1 +
 .../search/query/QueryStringIT.java           |  4 +-
 .../search/query/SimpleQueryStringIT.java     | 51 ++++++++++++++++++-
 .../common/settings/ClusterSettings.java      |  3 +-
 .../index/search/QueryParserHelper.java       |  4 +-
 .../org/opensearch/search/SearchModule.java   |  9 ----
 .../org/opensearch/search/SearchService.java  | 13 +++++
 7 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dc85bd4f85ffd..2daacf507c469 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
 
 ### Changed
+- Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568))
 
 ### Deprecated
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
index c43a9c23661ea..8841638328ea4 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
@@ -45,7 +45,7 @@
 import org.opensearch.index.query.QueryStringQueryBuilder;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
-import org.opensearch.search.SearchModule;
+import org.opensearch.search.SearchService;
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -101,7 +101,7 @@ public void setup() throws Exception {
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
             .put(super.nodeSettings(nodeOrdinal))
-            .put(SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT)
+            .put(SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT)
             .build();
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
index cae543506f919..f9ccdbd62de1c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
@@ -57,7 +57,7 @@
 import org.opensearch.plugins.Plugin;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
-import org.opensearch.search.SearchModule;
+import org.opensearch.search.SearchService;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.junit.BeforeClass;
@@ -79,6 +79,7 @@
 import static org.opensearch.index.query.QueryBuilders.simpleQueryStringQuery;
 import static org.opensearch.index.query.QueryBuilders.termQuery;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
+import static org.opensearch.search.SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING;
 import static org.opensearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures;
@@ -122,7 +123,7 @@ public static void createRandomClusterSetting() {
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
             .put(super.nodeSettings(nodeOrdinal))
-            .put(SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT)
+            .put(SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT)
             .build();
     }
 
@@ -720,6 +721,52 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
         assertHits(response.getHits(), "1");
     }
 
+    public void testDynamicClauseCountUpdate() throws Exception {
+        client().prepareIndex("testdynamic").setId("1").setSource("field", "foo bar baz").get();
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put(INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT - 1))
+        );
+        refresh();
+        StringBuilder sb = new StringBuilder("foo");
+
+        // create clause_count + 1 clauses to hit error
+        for (int i = 0; i <= CLUSTER_MAX_CLAUSE_COUNT; i++) {
+            sb.append(" OR foo" + i);
+        }
+
+        QueryStringQueryBuilder qb = queryStringQuery(sb.toString()).field("field");
+
+        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> {
+            client().prepareSearch("testdynamic").setQuery(qb).get();
+        });
+
+        assert (e.getDetailedMessage().contains("maxClauseCount is set to " + (CLUSTER_MAX_CLAUSE_COUNT - 1)));
+
+        // increase clause count by 2
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put(INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT + 2))
+        );
+
+        Thread.sleep(1);
+
+        SearchResponse response = client().prepareSearch("testdynamic").setQuery(qb).get();
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "1");
+
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().putNull(INDICES_MAX_CLAUSE_COUNT_SETTING.getKey()))
+        );
+    }
+
     private void assertHits(SearchHits hits, String... ids) {
         assertThat(hits.getTotalHits().value, equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 7ea04acf00415..233a8d732d178 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -150,7 +150,6 @@
 import org.opensearch.repositories.fs.FsRepository;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.script.ScriptService;
-import org.opensearch.search.SearchModule;
 import org.opensearch.search.SearchService;
 import org.opensearch.search.aggregations.MultiBucketConsumerService;
 import org.opensearch.search.backpressure.settings.NodeDuressSettings;
@@ -540,6 +539,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SearchService.MAX_OPEN_PIT_CONTEXT,
                 SearchService.MAX_PIT_KEEPALIVE_SETTING,
                 SearchService.MAX_AGGREGATION_REWRITE_FILTERS,
+                SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING,
                 SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD,
                 CreatePitController.PIT_INIT_KEEP_ALIVE,
                 Node.WRITE_PORTS_FILE_SETTING,
@@ -590,7 +590,6 @@ public void apply(Settings value, Settings current, Settings previous) {
                 ResourceWatcherService.RELOAD_INTERVAL_HIGH,
                 ResourceWatcherService.RELOAD_INTERVAL_MEDIUM,
                 ResourceWatcherService.RELOAD_INTERVAL_LOW,
-                SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING,
                 ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING,
                 FastVectorHighlighter.SETTING_TV_HIGHLIGHT_MULTI_VALUE,
                 Node.BREAKER_TYPE_KEY,
diff --git a/server/src/main/java/org/opensearch/index/search/QueryParserHelper.java b/server/src/main/java/org/opensearch/index/search/QueryParserHelper.java
index bae58c0ce1ebf..06f450f090e63 100644
--- a/server/src/main/java/org/opensearch/index/search/QueryParserHelper.java
+++ b/server/src/main/java/org/opensearch/index/search/QueryParserHelper.java
@@ -38,7 +38,7 @@
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.query.QueryShardException;
-import org.opensearch.search.SearchModule;
+import org.opensearch.search.SearchService;
 
 import java.util.Collection;
 import java.util.HashMap;
@@ -180,7 +180,7 @@ static Map<String, Float> resolveMappingField(
     }
 
     static void checkForTooManyFields(int numberOfFields, QueryShardContext context, @Nullable String inputPattern) {
-        Integer limit = SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING.get(context.getIndexSettings().getSettings());
+        int limit = SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING.get(context.getIndexSettings().getSettings());
         if (numberOfFields > limit) {
             StringBuilder errorMsg = new StringBuilder("field expansion ");
             if (inputPattern != null) {
diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java
index 88218896dceae..b463458847a88 100644
--- a/server/src/main/java/org/opensearch/search/SearchModule.java
+++ b/server/src/main/java/org/opensearch/search/SearchModule.java
@@ -37,7 +37,6 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.geo.GeoShapeType;
 import org.opensearch.common.geo.ShapesAvailability;
-import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.ParseFieldRegistry;
 import org.opensearch.core.ParseField;
@@ -302,13 +301,6 @@
  * @opensearch.internal
  */
 public class SearchModule {
-    public static final Setting<Integer> INDICES_MAX_CLAUSE_COUNT_SETTING = Setting.intSetting(
-        "indices.query.bool.max_clause_count",
-        1024,
-        1,
-        Integer.MAX_VALUE,
-        Setting.Property.NodeScope
-    );
 
     private final Map<String, Highlighter> highlighters;
     private final ParseFieldRegistry<MovAvgModel.AbstractModelParser> movingAverageModelParserRegistry = new ParseFieldRegistry<>(
@@ -1094,7 +1086,6 @@ private void registerQueryParsers(List<SearchPlugin> plugins) {
         registerQuery(new QuerySpec<>(MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new, MatchAllQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(QueryStringQueryBuilder.NAME, QueryStringQueryBuilder::new, QueryStringQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(BoostingQueryBuilder.NAME, BoostingQueryBuilder::new, BoostingQueryBuilder::fromXContent));
-        BooleanQuery.setMaxClauseCount(INDICES_MAX_CLAUSE_COUNT_SETTING.get(settings));
         registerQuery(new QuerySpec<>(BoolQueryBuilder.NAME, BoolQueryBuilder::new, BoolQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(TermQueryBuilder.NAME, TermQueryBuilder::new, TermQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(TermsQueryBuilder.NAME, TermsQueryBuilder::new, TermsQueryBuilder::fromXContent));
diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java
index 135af91912e5d..a53a7198c366f 100644
--- a/server/src/main/java/org/opensearch/search/SearchService.java
+++ b/server/src/main/java/org/opensearch/search/SearchService.java
@@ -35,6 +35,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TopDocs;
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.ActionRunnable;
@@ -281,6 +282,15 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         Property.NodeScope
     );
 
+    public static final Setting<Integer> INDICES_MAX_CLAUSE_COUNT_SETTING = Setting.intSetting(
+        "indices.query.bool.max_clause_count",
+        1024,
+        1,
+        Integer.MAX_VALUE,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     public static final Setting<Boolean> CLUSTER_ALLOW_DERIVED_FIELD_SETTING = Setting.boolSetting(
         "search.derived_field.enabled",
         true,
@@ -411,6 +421,9 @@ public SearchService(
         lowLevelCancellation = LOW_LEVEL_CANCELLATION_SETTING.get(settings);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(LOW_LEVEL_CANCELLATION_SETTING, this::setLowLevelCancellation);
 
+        IndexSearcher.setMaxClauseCount(INDICES_MAX_CLAUSE_COUNT_SETTING.get(settings));
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(INDICES_MAX_CLAUSE_COUNT_SETTING, IndexSearcher::setMaxClauseCount);
+
         allowDerivedField = CLUSTER_ALLOW_DERIVED_FIELD_SETTING.get(settings);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(CLUSTER_ALLOW_DERIVED_FIELD_SETTING, this::setAllowDerivedField);
     }

From f8213b8492b213922bea35d22c4317a79786a74f Mon Sep 17 00:00:00 2001
From: Finn <carrofin@amazon.com>
Date: Tue, 18 Jun 2024 13:48:05 -0700
Subject: [PATCH 018/113] Skip ComposeBuild task when docker cli not found
 (#14357)

Signed-off-by: Finn Carroll <carrofin@amazon.com>
---
 .../org/opensearch/gradle/testfixtures/TestFixturesPlugin.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java
index c9e18426966f9..e8772522b19a4 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java
@@ -34,6 +34,7 @@
 import com.avast.gradle.dockercompose.ComposeExtension;
 import com.avast.gradle.dockercompose.DockerComposePlugin;
 import com.avast.gradle.dockercompose.ServiceInfo;
+import com.avast.gradle.dockercompose.tasks.ComposeBuild;
 import com.avast.gradle.dockercompose.tasks.ComposeDown;
 import com.avast.gradle.dockercompose.tasks.ComposePull;
 import com.avast.gradle.dockercompose.tasks.ComposeUp;
@@ -200,6 +201,7 @@ public void execute(Task task) {
         maybeSkipTasks(tasks, dockerSupport, getTaskClass("org.opensearch.gradle.test.RestIntegTestTask"));
         maybeSkipTasks(tasks, dockerSupport, TestingConventionsTasks.class);
         maybeSkipTasks(tasks, dockerSupport, getTaskClass("org.opensearch.gradle.test.AntFixture"));
+        maybeSkipTasks(tasks, dockerSupport, ComposeBuild.class);
         maybeSkipTasks(tasks, dockerSupport, ComposeUp.class);
         maybeSkipTasks(tasks, dockerSupport, ComposePull.class);
         maybeSkipTasks(tasks, dockerSupport, ComposeDown.class);

From f86990db1a1792465e747ffe456ef549e9963496 Mon Sep 17 00:00:00 2001
From: Shubh Sahu <shubhsahu200103@gmail.com>
Date: Wed, 19 Jun 2024 11:40:45 +0530
Subject: [PATCH 019/113] [Remote Migration IT] Using voting config exclusion
 instead of stopping master node (#14433)

Signed-off-by: Shubh Sahu <shubhvs@amazon.com>
---
 .../RemoteMigrationIndexMetadataUpdateIT.java            | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
index 793adef0594fc..6885d37c4aab0 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.remotemigration;
 
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.cluster.metadata.IndexMetadata;
@@ -471,7 +473,6 @@ public void testRemotePathMetadataAddedWithFirstPrimaryMovingToRemote() throws E
      * exclude docrep nodes, assert that remote index path file exists
      * when shards start relocating to the remote nodes.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/13939")
     public void testRemoteIndexPathFileExistsAfterMigration() throws Exception {
         String docrepClusterManager = internalCluster().startClusterManagerOnlyNode();
 
@@ -518,7 +519,11 @@ public void testRemoteIndexPathFileExistsAfterMigration() throws Exception {
                 .isAcknowledged()
         );
 
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(docrepClusterManager));
+        // elect cluster manager with remote-cluster state enabled
+        internalCluster().client()
+            .execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(docrepClusterManager))
+            .get();
+
         internalCluster().validateClusterFormed();
 
         logger.info("---> Excluding docrep nodes from allocation");

From 8aed62e4c56914be1ddc802a118e7fa6039d9dc1 Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Wed, 19 Jun 2024 22:19:20 +0800
Subject: [PATCH 020/113] Make the class CommunityIdProcessor final (#14448)

* Make the class CommunityIdProcessor final

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Update change log

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

---------

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 CHANGELOG.md                                                    | 1 +
 .../java/org/opensearch/ingest/common/CommunityIdProcessor.java | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2daacf507c469..fae8686d1e45d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Changed
 - Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568))
+- Make the class CommunityIdProcessor final ([#14448](https://github.com/opensearch-project/OpenSearch/pull/14448))
 
 ### Deprecated
 
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CommunityIdProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CommunityIdProcessor.java
index c968fb2f6c2da..c84892971c87e 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CommunityIdProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CommunityIdProcessor.java
@@ -29,7 +29,7 @@
  * Processor that generating community id flow hash for the network flow tuples, the algorithm is defined in
  * <a href="https://github.com/corelight/community-id-spec">Community ID Flow Hashing</a>.
  */
-public class CommunityIdProcessor extends AbstractProcessor {
+public final class CommunityIdProcessor extends AbstractProcessor {
     public static final String TYPE = "community_id";
     // the version of the community id flow hashing algorithm
     private static final String COMMUNITY_ID_HASH_VERSION = "1";

From 8e32ed736372aa90db4c0ce3b85888b7b473a337 Mon Sep 17 00:00:00 2001
From: Shivansh Arora <hishiv@amazon.com>
Date: Thu, 20 Jun 2024 00:21:52 +0530
Subject: [PATCH 021/113] Add tests for RemoteGlobalMetadataManager (#14394)

* Add tests for RemoteGlobalMetadataManager

Signed-off-by: Shivansh Arora <hishiv@amazon.com>

* Add TestCapturingListener

Signed-off-by: Shivansh Arora <hishiv@amazon.com>

* Move TestCapturingListener to test/framework

Signed-off-by: Shivansh Arora <hishiv@amazon.com>

* Added javadoc

Signed-off-by: Shivansh Arora <hishiv@amazon.com>

---------

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
---
 .../RemoteGlobalMetadataManagerTests.java     | 532 +++++++++++++++++-
 .../RemoteCoordinationMetadataTests.java      |   2 +-
 .../model/RemoteCustomMetadataTests.java      |   2 +-
 .../model/RemoteGlobalMetadataTests.java      |   2 +-
 ...RemoteHashesOfConsistentSettingsTests.java |   2 +-
 ...RemotePersistentSettingsMetadataTests.java |   2 +-
 .../model/RemoteTemplatesMetadataTests.java   |   2 +-
 .../common/util/TestCapturingListener.java    |  39 ++
 8 files changed, 573 insertions(+), 10 deletions(-)
 create mode 100644 test/framework/src/main/java/org/opensearch/common/util/TestCapturingListener.java

diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
index bd01bc1ab0cdb..c543f986b3e86 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
@@ -9,19 +9,36 @@
 package org.opensearch.gateway.remote;
 
 import org.opensearch.Version;
+import org.opensearch.action.LatchedActionListener;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.DiffableUtils;
+import org.opensearch.cluster.coordination.CoordinationMetadata;
+import org.opensearch.cluster.metadata.DiffableStringMap;
 import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.Metadata.XContentContext;
+import org.opensearch.cluster.metadata.TemplatesMetadata;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.TestCapturingListener;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.compress.Compressor;
 import org.opensearch.core.compress.NoneCompressor;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.gateway.remote.model.RemoteCoordinationMetadata;
+import org.opensearch.gateway.remote.model.RemoteCustomMetadata;
+import org.opensearch.gateway.remote.model.RemoteGlobalMetadata;
+import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings;
+import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata;
+import org.opensearch.gateway.remote.model.RemoteReadResult;
+import org.opensearch.gateway.remote.model.RemoteTemplatesMetadata;
+import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata;
+import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.translog.transfer.BlobStoreTransferService;
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
@@ -32,14 +49,48 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import java.util.function.Function;
 import java.util.stream.Stream;
 
 import static java.util.stream.Collectors.toList;
+import static org.opensearch.cluster.metadata.Metadata.isGlobalStateEquals;
+import static org.opensearch.common.blobstore.stream.write.WritePriority.URGENT;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER;
+import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA;
+import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadataTests.getCoordinationMetadata;
+import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA;
+import static org.opensearch.gateway.remote.model.RemoteCustomMetadataTests.getCustomMetadata;
+import static org.opensearch.gateway.remote.model.RemoteGlobalMetadata.GLOBAL_METADATA;
+import static org.opensearch.gateway.remote.model.RemoteGlobalMetadata.GLOBAL_METADATA_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteGlobalMetadataTests.getGlobalMetadata;
+import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS;
+import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettingsTests.getHashesOfConsistentSettings;
+import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA;
+import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadataTests.getSettings;
+import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA;
+import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadataTests.getTemplatesMetadata;
+import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyIterable;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -47,26 +98,36 @@ public class RemoteGlobalMetadataManagerTests extends OpenSearchTestCase {
     private RemoteGlobalMetadataManager remoteGlobalMetadataManager;
     private ClusterSettings clusterSettings;
     private BlobStoreRepository blobStoreRepository;
+    private BlobStoreTransferService blobStoreTransferService;
+    private Compressor compressor;
+    private NamedXContentRegistry xContentRegistry;
+    private NamedWriteableRegistry namedWriteableRegistry;
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+    private final long METADATA_VERSION = 7331L;
+    private final String CLUSTER_NAME = "test-cluster";
+    private final String CLUSTER_UUID = "test-cluster-uuid";
 
     @Before
     public void setup() {
         clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         blobStoreRepository = mock(BlobStoreRepository.class);
-        BlobStoreTransferService blobStoreTransferService = mock(BlobStoreTransferService.class);
-        NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(
+        blobStoreTransferService = mock(BlobStoreTransferService.class);
+        compressor = new NoneCompressor();
+        xContentRegistry = new NamedXContentRegistry(
             Stream.of(
                 NetworkModule.getNamedXContents().stream(),
                 IndicesModule.getNamedXContents().stream(),
                 ClusterModule.getNamedXWriteables().stream()
             ).flatMap(Function.identity()).collect(toList())
         );
-        Compressor compressor = new NoneCompressor();
+        namedWriteableRegistry = writableRegistry();
+        BlobPath blobPath = new BlobPath();
         when(blobStoreRepository.getCompressor()).thenReturn(compressor);
         when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(xContentRegistry);
+        when(blobStoreRepository.basePath()).thenReturn(blobPath);
         remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(
             clusterSettings,
-            "test-cluster",
+            CLUSTER_NAME,
             blobStoreRepository,
             blobStoreTransferService,
             writableRegistry(),
@@ -96,6 +157,469 @@ public void testGlobalMetadataUploadWaitTimeSetting() {
         assertEquals(globalMetadataUploadTimeout, remoteGlobalMetadataManager.getGlobalMetadataUploadTimeout().seconds());
     }
 
+    public void testGetReadMetadataAsyncAction_CoordinationMetadata() throws Exception {
+        CoordinationMetadata coordinationMetadata = getCoordinationMetadata();
+        String fileName = randomAlphaOfLength(10);
+        RemoteCoordinationMetadata coordinationMetadataForDownload = new RemoteCoordinationMetadata(
+            fileName,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            COORDINATION_METADATA_FORMAT.serialize(coordinationMetadata, fileName, compressor, FORMAT_PARAMS).streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            coordinationMetadataForDownload,
+            COORDINATION_METADATA,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(coordinationMetadata, listener.getResult().getObj());
+        assertEquals(COORDINATION_METADATA, listener.getResult().getComponent());
+        assertEquals(COORDINATION_METADATA, listener.getResult().getComponentName());
+    }
+
+    public void testGetAsyncMetadataWriteAction_CoordinationMetadata() throws Exception {
+        CoordinationMetadata coordinationMetadata = getCoordinationMetadata();
+        RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(
+            coordinationMetadata,
+            METADATA_VERSION,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteGlobalMetadataManager.getAsyncMetadataWriteAction(remoteCoordinationMetadata, new LatchedActionListener<>(listener, latch))
+            .run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+        assertEquals(COORDINATION_METADATA, uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(RemoteClusterStateUtils.encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(GLOBAL_METADATA_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(COORDINATION_METADATA, splitFileName[0]);
+        assertEquals(RemoteStoreUtils.invertLong(METADATA_VERSION), splitFileName[1]);
+        assertEquals(GLOBAL_METADATA_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
+
+    public void testGetReadMetadataAsyncAction_PersistentSettings() throws Exception {
+        Settings settingsMetadata = getSettings();
+        String fileName = randomAlphaOfLength(10);
+        RemotePersistentSettingsMetadata persistentSettings = new RemotePersistentSettingsMetadata(
+            fileName,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            RemotePersistentSettingsMetadata.SETTINGS_METADATA_FORMAT.serialize(settingsMetadata, fileName, compressor, FORMAT_PARAMS)
+                .streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            persistentSettings,
+            SETTING_METADATA,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(settingsMetadata, listener.getResult().getObj());
+        assertEquals(SETTING_METADATA, listener.getResult().getComponent());
+        assertEquals(SETTING_METADATA, listener.getResult().getComponentName());
+    }
+
+    public void testGetAsyncMetadataWriteAction_PersistentSettings() throws Exception {
+        Settings settingsMetadata = getSettings();
+        RemotePersistentSettingsMetadata persistentSettings = new RemotePersistentSettingsMetadata(
+            settingsMetadata,
+            METADATA_VERSION,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataWriteAction(persistentSettings, new LatchedActionListener<>(listener, latch)).run();
+
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+        assertEquals(SETTING_METADATA, uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(RemoteClusterStateUtils.encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(GLOBAL_METADATA_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(SETTING_METADATA, splitFileName[0]);
+        assertEquals(RemoteStoreUtils.invertLong(METADATA_VERSION), splitFileName[1]);
+        assertEquals(GLOBAL_METADATA_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
+
+    public void testGetReadMetadataAsyncAction_TransientSettings() throws Exception {
+        Settings settingsMetadata = getSettings();
+        String fileName = randomAlphaOfLength(10);
+        RemoteTransientSettingsMetadata transientSettings = new RemoteTransientSettingsMetadata(
+            fileName,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            RemoteTransientSettingsMetadata.SETTINGS_METADATA_FORMAT.serialize(settingsMetadata, fileName, compressor, FORMAT_PARAMS)
+                .streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            transientSettings,
+            TRANSIENT_SETTING_METADATA,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(settingsMetadata, listener.getResult().getObj());
+        assertEquals(TRANSIENT_SETTING_METADATA, listener.getResult().getComponent());
+        assertEquals(TRANSIENT_SETTING_METADATA, listener.getResult().getComponentName());
+    }
+
+    public void testGetAsyncMetadataWriteAction_TransientSettings() throws Exception {
+        Settings settingsMetadata = getSettings();
+        RemoteTransientSettingsMetadata transientSettings = new RemoteTransientSettingsMetadata(
+            settingsMetadata,
+            METADATA_VERSION,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataWriteAction(transientSettings, new LatchedActionListener<>(listener, latch)).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+        assertEquals(TRANSIENT_SETTING_METADATA, uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(RemoteClusterStateUtils.encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(GLOBAL_METADATA_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(TRANSIENT_SETTING_METADATA, splitFileName[0]);
+        assertEquals(RemoteStoreUtils.invertLong(METADATA_VERSION), splitFileName[1]);
+        assertEquals(GLOBAL_METADATA_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
+
+    public void testGetReadMetadataAsyncAction_HashesOfConsistentSettings() throws Exception {
+        DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings();
+        String fileName = randomAlphaOfLength(10);
+        RemoteHashesOfConsistentSettings hashesOfConsistentSettingsForDownload = new RemoteHashesOfConsistentSettings(
+            fileName,
+            CLUSTER_UUID,
+            compressor
+        );
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            HASHES_OF_CONSISTENT_SETTINGS_FORMAT.serialize(hashesOfConsistentSettings, fileName, compressor).streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            hashesOfConsistentSettingsForDownload,
+            HASHES_OF_CONSISTENT_SETTINGS,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(hashesOfConsistentSettings, listener.getResult().getObj());
+        assertEquals(HASHES_OF_CONSISTENT_SETTINGS, listener.getResult().getComponent());
+        assertEquals(HASHES_OF_CONSISTENT_SETTINGS, listener.getResult().getComponentName());
+    }
+
+    public void testGetAsyncMetadataWriteAction_HashesOfConsistentSettings() throws Exception {
+        DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings();
+        RemoteHashesOfConsistentSettings hashesOfConsistentSettingsForUpload = new RemoteHashesOfConsistentSettings(
+            hashesOfConsistentSettings,
+            METADATA_VERSION,
+            CLUSTER_UUID,
+            compressor
+        );
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataWriteAction(
+            hashesOfConsistentSettingsForUpload,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+        assertEquals(HASHES_OF_CONSISTENT_SETTINGS, uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(RemoteClusterStateUtils.encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(GLOBAL_METADATA_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(HASHES_OF_CONSISTENT_SETTINGS, splitFileName[0]);
+        assertEquals(RemoteStoreUtils.invertLong(METADATA_VERSION), splitFileName[1]);
+        assertEquals(GLOBAL_METADATA_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
+
+    public void testGetReadMetadataAsyncAction_TemplatesMetadata() throws Exception {
+        TemplatesMetadata templatesMetadata = getTemplatesMetadata();
+        String fileName = randomAlphaOfLength(10);
+        RemoteTemplatesMetadata templatesMetadataForDownload = new RemoteTemplatesMetadata(
+            fileName,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            TEMPLATES_METADATA_FORMAT.serialize(templatesMetadata, fileName, compressor, FORMAT_PARAMS).streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            templatesMetadataForDownload,
+            TEMPLATES_METADATA,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(templatesMetadata, listener.getResult().getObj());
+        assertEquals(TEMPLATES_METADATA, listener.getResult().getComponent());
+        assertEquals(TEMPLATES_METADATA, listener.getResult().getComponentName());
+    }
+
+    public void testGetAsyncMetadataWriteAction_TemplatesMetadata() throws Exception {
+        TemplatesMetadata templatesMetadata = getTemplatesMetadata();
+        RemoteTemplatesMetadata templateMetadataForUpload = new RemoteTemplatesMetadata(
+            templatesMetadata,
+            METADATA_VERSION,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataWriteAction(templateMetadataForUpload, new LatchedActionListener<>(listener, latch))
+            .run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+        assertEquals(TEMPLATES_METADATA, uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(RemoteClusterStateUtils.encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(GLOBAL_METADATA_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(TEMPLATES_METADATA, splitFileName[0]);
+        assertEquals(RemoteStoreUtils.invertLong(METADATA_VERSION), splitFileName[1]);
+        assertEquals(GLOBAL_METADATA_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
+
+    public void testGetReadMetadataAsyncAction_CustomMetadata() throws Exception {
+        Metadata.Custom customMetadata = getCustomMetadata();
+        String fileName = randomAlphaOfLength(10);
+        RemoteCustomMetadata customMetadataForDownload = new RemoteCustomMetadata(
+            fileName,
+            IndexGraveyard.TYPE,
+            CLUSTER_UUID,
+            compressor,
+            namedWriteableRegistry
+        );
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            customMetadataForDownload.customBlobStoreFormat.serialize(customMetadata, fileName, compressor).streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            customMetadataForDownload,
+            IndexGraveyard.TYPE,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(customMetadata, listener.getResult().getObj());
+        assertEquals(CUSTOM_METADATA, listener.getResult().getComponent());
+        assertEquals(IndexGraveyard.TYPE, listener.getResult().getComponentName());
+    }
+
+    public void testGetAsyncMetadataWriteAction_CustomMetadata() throws Exception {
+        Metadata.Custom customMetadata = getCustomMetadata();
+        RemoteCustomMetadata customMetadataForUpload = new RemoteCustomMetadata(
+            customMetadata,
+            IndexGraveyard.TYPE,
+            METADATA_VERSION,
+            CLUSTER_UUID,
+            compressor,
+            namedWriteableRegistry
+        );
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataWriteAction(customMetadataForUpload, new LatchedActionListener<>(listener, latch))
+            .run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+        assertEquals(String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, IndexGraveyard.TYPE), uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(RemoteClusterStateUtils.encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(GLOBAL_METADATA_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, IndexGraveyard.TYPE), splitFileName[0]);
+        assertEquals(RemoteStoreUtils.invertLong(METADATA_VERSION), splitFileName[1]);
+        assertEquals(GLOBAL_METADATA_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
+
+    public void testGetReadMetadataAsyncAction_GlobalMetadata() throws Exception {
+        Metadata metadata = getGlobalMetadata();
+        String fileName = randomAlphaOfLength(10);
+        RemoteGlobalMetadata globalMetadataForDownload = new RemoteGlobalMetadata(fileName, CLUSTER_UUID, compressor, xContentRegistry);
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            GLOBAL_METADATA_FORMAT.serialize(metadata, fileName, compressor, FORMAT_PARAMS).streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            globalMetadataForDownload,
+            GLOBAL_METADATA,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertTrue(isGlobalStateEquals(metadata, (Metadata) listener.getResult().getObj()));
+        assertEquals(GLOBAL_METADATA, listener.getResult().getComponent());
+        assertEquals(GLOBAL_METADATA, listener.getResult().getComponentName());
+    }
+
+    public void testGetReadMetadataAsyncAction_IOException() throws Exception {
+        String fileName = randomAlphaOfLength(10);
+        RemoteCoordinationMetadata coordinationMetadataForDownload = new RemoteCoordinationMetadata(
+            fileName,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        IOException ioException = new IOException("mock test exception");
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenThrow(ioException);
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataReadAction(
+            coordinationMetadataForDownload,
+            COORDINATION_METADATA,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getResult());
+        assertNotNull(listener.getFailure());
+        assertEquals(ioException, listener.getFailure());
+    }
+
+    public void testGetAsyncMetadataWriteAction_IOException() throws Exception {
+        CoordinationMetadata coordinationMetadata = getCoordinationMetadata();
+        RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(
+            coordinationMetadata,
+            METADATA_VERSION,
+            CLUSTER_UUID,
+            compressor,
+            xContentRegistry
+        );
+        IOException ioException = new IOException("mock test exception");
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onFailure(ioException);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        remoteGlobalMetadataManager.getAsyncMetadataWriteAction(remoteCoordinationMetadata, new LatchedActionListener<>(listener, latch))
+            .run();
+        assertNull(listener.getResult());
+        assertNotNull(listener.getFailure());
+        assertTrue(listener.getFailure() instanceof RemoteStateTransferException);
+        assertEquals(ioException, listener.getFailure().getCause());
+    }
+
     public void testGetUpdatedCustoms() {
         Map<String, Metadata.Custom> previousCustoms = Map.of(
             CustomMetadata1.TYPE,
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java
index 9484afe6b7d6c..63d6de05a737c 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java
@@ -229,7 +229,7 @@ public void testSerDe() throws IOException {
         }
     }
 
-    private CoordinationMetadata getCoordinationMetadata() {
+    public static CoordinationMetadata getCoordinationMetadata() {
         return CoordinationMetadata.builder()
             .term(TERM)
             .lastAcceptedConfiguration(new VotingConfiguration(Set.of("node1")))
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java
index 1bce176273270..1e28817be79f2 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java
@@ -232,7 +232,7 @@ public void testSerDe() throws IOException {
         }
     }
 
-    private Custom getCustomMetadata() {
+    public static Custom getCustomMetadata() {
         return IndexGraveyard.builder().addTombstone(new Index("test-index", "3q2423")).build();
     }
 
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java
index 02ddc8ba93071..23de485357547 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java
@@ -180,7 +180,7 @@ public void testSerDe() throws IOException {
         }
     }
 
-    private Metadata getGlobalMetadata() {
+    public static Metadata getGlobalMetadata() {
         return Metadata.builder()
             .templates(
                 TemplatesMetadata.builder()
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java
index d883eabf9fbc9..b931f24f98631 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java
@@ -185,7 +185,7 @@ public void testSerDe() throws IOException {
         }
     }
 
-    private DiffableStringMap getHashesOfConsistentSettings() {
+    public static DiffableStringMap getHashesOfConsistentSettings() {
         Map<String, String> hashesOfConsistentSettings = new HashMap<>();
         hashesOfConsistentSettings.put("secure-setting-key", "secure-setting-value");
         return new DiffableStringMap(hashesOfConsistentSettings);
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTests.java
index 850c18f03fa49..5e4d5d66ca1b7 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTests.java
@@ -224,7 +224,7 @@ public void testSerDe() throws IOException {
         }
     }
 
-    private Settings getSettings() {
+    public static Settings getSettings() {
         return Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build();
     }
 }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java
index b86044003aa55..d7ecd2ad3f44a 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java
@@ -227,7 +227,7 @@ public void testSerDe() throws IOException {
         }
     }
 
-    private TemplatesMetadata getTemplatesMetadata() {
+    public static TemplatesMetadata getTemplatesMetadata() {
         return TemplatesMetadata.builder()
             .put(
                 IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3))
diff --git a/test/framework/src/main/java/org/opensearch/common/util/TestCapturingListener.java b/test/framework/src/main/java/org/opensearch/common/util/TestCapturingListener.java
new file mode 100644
index 0000000000000..a3c8cc15de927
--- /dev/null
+++ b/test/framework/src/main/java/org/opensearch/common/util/TestCapturingListener.java
@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.util;
+
+import org.opensearch.core.action.ActionListener;
+
+/**
+ * A simple implementation of {@link ActionListener} that captures the response and failures used for testing purposes.
+ *
+ * @param <T> the result type
+ */
+public class TestCapturingListener<T> implements ActionListener<T> {
+    private T result;
+    private Exception failure;
+
+    @Override
+    public void onResponse(T result) {
+        this.result = result;
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        this.failure = e;
+    }
+
+    public T getResult() {
+        return result;
+    }
+
+    public Exception getFailure() {
+        return failure;
+    }
+}

From a8e40448dca97428ec9241a85289c581c33e1e4c Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Wed, 19 Jun 2024 16:20:26 -0400
Subject: [PATCH 022/113] Bump OpenTelemetry to 1.39.0 (#14457)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                           |  1 +
 buildSrc/version.properties                            |  4 ++--
 plugins/telemetry-otel/build.gradle                    | 10 ++++------
 .../licenses/opentelemetry-api-1.36.0.jar.sha1         |  1 -
 .../licenses/opentelemetry-api-1.39.0.jar.sha1         |  1 +
 .../opentelemetry-api-incubator-1.39.0-alpha.jar.sha1  |  1 +
 ...NSE.txt => opentelemetry-api-incubator-LICENSE.txt} |  0
 ...TICE.txt => opentelemetry-api-incubator-NOTICE.txt} |  0
 .../licenses/opentelemetry-context-1.36.0.jar.sha1     |  1 -
 .../licenses/opentelemetry-context-1.39.0.jar.sha1     |  1 +
 .../opentelemetry-exporter-common-1.36.0.jar.sha1      |  1 -
 .../opentelemetry-exporter-common-1.39.0.jar.sha1      |  1 +
 .../opentelemetry-exporter-logging-1.36.0.jar.sha1     |  1 -
 .../opentelemetry-exporter-logging-1.39.0.jar.sha1     |  1 +
 .../opentelemetry-exporter-otlp-1.36.0.jar.sha1        |  1 -
 .../opentelemetry-exporter-otlp-1.39.0.jar.sha1        |  1 +
 .../opentelemetry-exporter-otlp-common-1.36.0.jar.sha1 |  1 -
 .../opentelemetry-exporter-otlp-common-1.39.0.jar.sha1 |  1 +
 ...pentelemetry-exporter-sender-okhttp-1.36.0.jar.sha1 |  1 -
 ...pentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1 |  1 +
 ...telemetry-extension-incubator-1.36.0-alpha.jar.sha1 |  1 -
 .../licenses/opentelemetry-sdk-1.36.0.jar.sha1         |  1 -
 .../licenses/opentelemetry-sdk-1.39.0.jar.sha1         |  1 +
 .../licenses/opentelemetry-sdk-common-1.36.0.jar.sha1  |  1 -
 .../licenses/opentelemetry-sdk-common-1.39.0.jar.sha1  |  1 +
 .../licenses/opentelemetry-sdk-logs-1.36.0.jar.sha1    |  1 -
 .../licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1    |  1 +
 .../licenses/opentelemetry-sdk-metrics-1.36.0.jar.sha1 |  1 -
 .../licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1 |  1 +
 .../licenses/opentelemetry-sdk-trace-1.36.0.jar.sha1   |  1 -
 .../licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1   |  1 +
 .../opentelemetry-semconv-1.23.1-alpha.jar.sha1        |  1 -
 .../opentelemetry-semconv-1.25.0-alpha.jar.sha1        |  1 +
 33 files changed, 21 insertions(+), 22 deletions(-)
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1
 rename plugins/telemetry-otel/licenses/{opentelemetry-extension-incubator-LICENSE.txt => opentelemetry-api-incubator-LICENSE.txt} (100%)
 rename plugins/telemetry-otel/licenses/{opentelemetry-extension-incubator-NOTICE.txt => opentelemetry-api-incubator-NOTICE.txt} (100%)
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.36.0-alpha.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.36.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.23.1-alpha.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fae8686d1e45d..24a4f6fd1b1f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.37.3 to 9.40 ([#14398](https://github.com/opensearch-project/OpenSearch/pull/14398))
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
 - Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
+- Bump `opentelemetry` from 1.36.0 to 1.39.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457))
 
 ### Changed
 - Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index af421b97e12e4..e9aa32ea9a4f5 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -74,5 +74,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.36.0
-opentelemetrysemconv  = 1.23.1-alpha
+opentelemetry         = 1.39.0
+opentelemetrysemconv  = 1.25.0-alpha
diff --git a/plugins/telemetry-otel/build.gradle b/plugins/telemetry-otel/build.gradle
index 735cbd92b691a..66d172e3dc7f3 100644
--- a/plugins/telemetry-otel/build.gradle
+++ b/plugins/telemetry-otel/build.gradle
@@ -37,7 +37,7 @@ dependencies {
   runtimeOnly "com.squareup.okhttp3:okhttp:4.11.0"
   runtimeOnly "com.squareup.okio:okio-jvm:3.5.0"
   runtimeOnly "io.opentelemetry:opentelemetry-exporter-sender-okhttp:${versions.opentelemetry}"
-  api "io.opentelemetry:opentelemetry-extension-incubator:${versions.opentelemetry}-alpha"
+  api "io.opentelemetry:opentelemetry-api-incubator:${versions.opentelemetry}-alpha"
   testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${versions.opentelemetry}"
 }
 
@@ -48,7 +48,9 @@ thirdPartyAudit {
    'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerIndexField',
    'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerLimitField',
    'io.opentelemetry.internal.shaded.jctools.util.UnsafeAccess',
-   'io.opentelemetry.internal.shaded.jctools.util.UnsafeRefArrayAccess'
+   'io.opentelemetry.internal.shaded.jctools.util.UnsafeRefArrayAccess',
+   'io.opentelemetry.exporter.internal.marshal.UnsafeAccess',
+   'io.opentelemetry.exporter.internal.marshal.UnsafeAccess$UnsafeHolder'
   )
 
   ignoreMissingClasses(
@@ -78,10 +80,6 @@ thirdPartyAudit {
     'org.conscrypt.ConscryptHostnameVerifier',
     'org.openjsse.javax.net.ssl.SSLParameters',
     'org.openjsse.javax.net.ssl.SSLSocket',
-    'io.opentelemetry.api.events.EventBuilder',
-    'io.opentelemetry.api.events.EventEmitter',
-    'io.opentelemetry.api.events.EventEmitterBuilder',
-    'io.opentelemetry.api.events.EventEmitterProvider',
     'io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties',
     'io.opentelemetry.sdk.autoconfigure.spi.logs.ConfigurableLogRecordExporterProvider',
     'io.opentelemetry.sdk.autoconfigure.spi.metrics.ConfigurableMetricExporterProvider',
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.36.0.jar.sha1
deleted file mode 100644
index b577500d71e1d..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-api-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-59470f4aa3a9207f21936461b8fdcb36d46455ab
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..415fe8f3d8aaa
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1
@@ -0,0 +1 @@
+396b89a66526bd5694ad3bef4604b876177e0b44
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1
new file mode 100644
index 0000000000000..9c3c9f43d153c
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1
@@ -0,0 +1 @@
+1a1fd96155e1b58726300bbf8457630713035e51
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-LICENSE.txt
similarity index 100%
rename from plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-LICENSE.txt
rename to plugins/telemetry-otel/licenses/opentelemetry-api-incubator-LICENSE.txt
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-NOTICE.txt
similarity index 100%
rename from plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-NOTICE.txt
rename to plugins/telemetry-otel/licenses/opentelemetry-api-incubator-NOTICE.txt
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.36.0.jar.sha1
deleted file mode 100644
index d3156577248d5..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-context-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8850bc4c65d0fd22ff987b4683206ec4e69f2689
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..115d4ccb1f34b
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1
@@ -0,0 +1 @@
+f0601fb1c06f661afeffbc73a1dbe29797b2f13b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.36.0.jar.sha1
deleted file mode 100644
index f176b21d12dc4..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8d1cb823ab18fa871a1549e7c522bf28f2b3d8fe
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..a10b92995becd
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1
@@ -0,0 +1 @@
+570d71e39e36fe2caad142557bde0c11fcdb3b92
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.36.0.jar.sha1
deleted file mode 100644
index cd25e0ab9f294..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bc045cae89ff6f18071760f6e4659dd880e88a1b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..f43393104296a
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1
@@ -0,0 +1 @@
+f5b528f8d6f8531836eabba698979516964b24ed
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.36.0.jar.sha1
deleted file mode 100644
index fabb394f9c2e0..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5ee49902ba884d6c3e48499a9311a624396d9630
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..5adba2ba0f342
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1
@@ -0,0 +1 @@
+04fc0e4983253ea58430c3d24b6b3c5c95f84dc9
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.36.0.jar.sha1
deleted file mode 100644
index 378ba4d43dcd1..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2706e3b883d2bcd1a6b3e0bb4118ffbd7820550b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..ea9c293f25025
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1
@@ -0,0 +1 @@
+a2b8571e36b11c3153d31ec87ec69cc168af8036
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.36.0.jar.sha1
deleted file mode 100644
index a3d7e15e1a624..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dcc924787b559278697b74dbc5bb6d046b236ef6
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..dcf23f16ac89f
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1
@@ -0,0 +1 @@
+1a8947a2e28924ad9374e319150a23837926ca4b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.36.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.36.0-alpha.jar.sha1
deleted file mode 100644
index 71ab3e184db9e..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.36.0-alpha.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d58f7c669e371f6ff61b705770af9a3c1f31df52
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.36.0.jar.sha1
deleted file mode 100644
index c9a75d1b4350a..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4056d1b562b4da7720817d8af15d1d3ccdf4b776
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..f603af04d8012
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1
@@ -0,0 +1 @@
+ba9afdf3ef1ea51e42999fd68c959e3ceb219399
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.36.0.jar.sha1
deleted file mode 100644
index c31584f59c0d8..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-11d6f8c7b029efcb5c6c449cadef155b781afb78
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..f9419f6ccfbee
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1
@@ -0,0 +1 @@
+fb8168627bf0059445f61081eaa47c4ab787fc2e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.36.0.jar.sha1
deleted file mode 100644
index a134bb06ec635..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-98e94479db1e68c4779efc44bf6b4fca83e98b54
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..63269f239eacd
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1
@@ -0,0 +1 @@
+b6b45155399bc9fa563945f3e3a77416d7165948
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.36.0.jar.sha1
deleted file mode 100644
index d146241f52f29..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4f8f5d30c3eeede7b2260d979d9f403cfa381c3d
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..f18c8259c1adc
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1
@@ -0,0 +1 @@
+522d46926cc06a4c18829da7e4c4340bdf5673c3
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.36.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.36.0.jar.sha1
deleted file mode 100644
index 802761e38846c..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.36.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e3068cbaedfac6a28c6483923982b2efb861d3f4
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1
new file mode 100644
index 0000000000000..03b81424f46d5
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1
@@ -0,0 +1 @@
+0b72722a5bbea5f46319bf08b2caed5b8f987a92
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.23.1-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.23.1-alpha.jar.sha1
deleted file mode 100644
index e730c83af905e..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.23.1-alpha.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-218e361772670212a46be5940010222d68e66f2a
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1
new file mode 100644
index 0000000000000..7cf8e7e8ede28
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1
@@ -0,0 +1 @@
+76b3d4ca0a8f20b27c1590ceece54f0c7fb5857e
\ No newline at end of file

From bd56456baab283570da9b423152125124d482db3 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Wed, 19 Jun 2024 15:36:59 -0700
Subject: [PATCH 023/113] Fix Indices Request Cache ITs from flaking and timing
 out (#14369)

Signed-off-by: Kiran Prakash <awskiran@amazon.com>
---
 .../indices/IndicesRequestCacheCleanupIT.java | 746 ++++++++++++++++++
 .../indices/IndicesRequestCacheIT.java        | 631 +--------------
 2 files changed, 781 insertions(+), 596 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java
new file mode 100644
index 0000000000000..6f698c5816278
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java
@@ -0,0 +1,746 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.action.admin.cluster.node.stats.NodeStats;
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
+import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.IndexNotFoundException;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.MergePolicyProvider;
+import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.InternalSettingsPlugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.hamcrest.OpenSearchAssertions;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
+import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.equalTo;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
+public class IndicesRequestCacheCleanupIT extends OpenSearchIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(InternalSettingsPlugin.class);
+    }
+
+    public void testCacheWithInvalidation() throws Exception {
+        Client client = client();
+        String index = "index";
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate(index)
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
+                )
+                .get()
+        );
+        indexRandom(false, client.prepareIndex(index).setSource("k", "hello"));
+        ensureSearchable(index);
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        forceMerge(client, index);
+        SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        assertSearchResponse(resp);
+        OpenSearchAssertions.assertAllSuccessful(resp);
+        assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
+
+        assertCacheState(client, index, 0, 1);
+        // Index but don't refresh
+        indexRandom(false, client.prepareIndex(index).setSource("k", "hello2"));
+        resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        assertSearchResponse(resp);
+        // Should expect hit as here as refresh didn't happen
+        assertCacheState(client, index, 1, 1);
+
+        // assert segment counts stay the same
+        assertEquals(1, getSegmentCount(client, index));
+        // Explicit refresh would invalidate cache
+        refreshAndWaitForReplication();
+        // Hit same query again
+        resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        assertSearchResponse(resp);
+        // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
+        assertCacheState(client, index, 1, 2);
+    }
+
+    // calling cache clear api, when staleness threshold is lower than staleness, it should clean the stale keys from cache
+    public void testCacheClearAPIRemovesStaleKeysWhenStalenessThresholdIsLow() throws Exception {
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    // setting intentionally high to avoid cache cleaner interfering
+                    TimeValue.timeValueMillis(300)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(index2);
+        client.admin().indices().clearCache(clearIndicesCacheRequest).actionGet();
+
+        // assert segment counts stay the same
+        assertEquals(1, getSegmentCount(client, index1));
+        assertEquals(1, getSegmentCount(client, index2));
+        // cache cleaner should have cleaned up the stale key from index 2
+        assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+        // cache cleaner should NOT have cleaned from index 1
+        assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+    }
+
+    // when staleness threshold is lower than staleness, it should clean the stale keys from cache
+    public void testStaleKeysCleanupWithLowThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // create 1 stale key
+        indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello"));
+        forceMerge(client, index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // assert segment counts stay the same
+            assertEquals(1, getSegmentCount(client, index1));
+            assertEquals(2, getSegmentCount(client, index2));
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+    }
+
+    // when staleness threshold is equal to staleness, it should clean the stale keys from cache
+    public void testCacheCleanupOnEqualStalenessAndThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.33)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // create 1 stale key
+        indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello"));
+        forceMerge(client, index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // assert segment counts stay the same
+            assertEquals(1, getSegmentCount(client, index1));
+            assertEquals(2, getSegmentCount(client, index2));
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is higher than staleness, it should NOT clean the cache
+    public void testCacheCleanupSkipsWithHighStalenessThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // assert segment counts stay the same
+            assertEquals(1, getSegmentCount(client, index1));
+            assertEquals(1, getSegmentCount(client, index2));
+            // cache cleaner should NOT have cleaned up the stale key from index 2
+            assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is explicitly set to 0, cache cleaner regularly cleans up stale keys.
+    public void testCacheCleanupOnZeroStalenessThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 50;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create 10 index1 cache entries
+        for (int i = 1; i <= 10; i++) {
+            long cacheSizeBefore = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+            createCacheEntry(client, index1, "hello" + i);
+            assertCacheState(client, index1, 0, i);
+            long cacheSizeAfter = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+            assertTrue(cacheSizeAfter > cacheSizeBefore);
+        }
+
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // create 1 stale key
+        indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello"));
+        forceMerge(client, index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // assert segment counts stay the same
+            assertEquals(1, getSegmentCount(client, index1));
+            assertEquals(2, getSegmentCount(client, index2));
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is not explicitly set, cache cleaner regularly cleans up stale keys
+    public void testStaleKeysRemovalWithoutExplicitThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        String index1 = "index1";
+        String index2 = "index2";
+        Client client = client(node);
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello"));
+        forceMerge(client, index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            assertEquals(1, getSegmentCount(client, index1));
+            assertEquals(2, getSegmentCount(client, index2));
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when cache cleaner interval setting is not set, cache cleaner is configured appropriately with the fall-back setting
+    public void testCacheCleanupWithDefaultSettings() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder().put(INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY, TimeValue.timeValueMillis(cacheCleanIntervalInMillis))
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // create 1 stale key
+        indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello"));
+        forceMerge(client, index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // assert segment counts stay the same
+            assertEquals(1, getSegmentCount(client, index1));
+            assertEquals(2, getSegmentCount(client, index2));
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // staleness threshold updates flows through to the cache cleaner
+    public void testDynamicStalenessThresholdUpdate() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        assertTrue(getRequestCacheStats(client, index1).getMemorySizeInBytes() > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > 0);
+
+        // create 1 stale key
+        indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello"));
+        forceMerge(client, index2);
+        assertBusy(() -> {
+            // cache cleaner should NOT have cleaned up the stale key from index 2
+            assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+
+        // Update indices.requests.cache.cleanup.staleness_threshold to "10%"
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), 0.10));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        assertBusy(() -> {
+            // assert segment counts stay the same
+            assertEquals(1, getSegmentCount(client, index1));
+            assertEquals(2, getSegmentCount(client, index2));
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // staleness threshold dynamic updates should throw exceptions on invalid input
+    public void testInvalidStalenessThresholdUpdateThrowsException() throws Exception {
+        // Update indices.requests.cache.cleanup.staleness_threshold to "10%" with illegal argument
+        assertThrows("Ratio should be in [0-1.0]", IllegalArgumentException.class, () -> {
+            ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+            updateSettingsRequest.persistentSettings(
+                Settings.builder().put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 10)
+            );
+            client().admin().cluster().updateSettings(updateSettingsRequest).actionGet();
+        });
+    }
+
+    // closing the Index after caching will clean up from Indices Request Cache
+    public void testCacheClearanceAfterIndexClosure() throws Exception {
+        int cacheCleanIntervalInMillis = 100;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index = "index";
+        setupIndex(client, index);
+
+        // assert there are no entries in the cache for index
+        assertEquals(0, getRequestCacheStats(client, index).getMemorySizeInBytes());
+        // assert there are no entries in the cache from other indices in the node
+        assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        // create first cache entry in index
+        createCacheEntry(client, index, "hello");
+        assertCacheState(client, index, 0, 1);
+        assertTrue(getRequestCacheStats(client, index).getMemorySizeInBytes() > 0);
+        assertTrue(getNodeCacheStats(client).getMemorySizeInBytes() > 0);
+
+        // close index
+        assertAcked(client.admin().indices().prepareClose(index));
+        // request cache stats cannot be access since Index should be closed
+        try {
+            getRequestCacheStats(client, index);
+        } catch (Exception e) {
+            assert (e instanceof IndexClosedException);
+        }
+        // sleep until cache cleaner would have cleaned up the stale key from index
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale keys from index
+            assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // deleting the Index after caching will clean up from Indices Request Cache
+    public void testCacheCleanupAfterIndexDeletion() throws Exception {
+        int cacheCleanIntervalInMillis = 100;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index = "index";
+        setupIndex(client, index);
+
+        // assert there are no entries in the cache for index
+        assertEquals(0, getRequestCacheStats(client, index).getMemorySizeInBytes());
+        // assert there are no entries in the cache from other indices in the node
+        assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        // create first cache entry in index
+        createCacheEntry(client, index, "hello");
+        assertCacheState(client, index, 0, 1);
+        assertTrue(getRequestCacheStats(client, index).getMemorySizeInBytes() > 0);
+        assertTrue(getNodeCacheStats(client).getMemorySizeInBytes() > 0);
+
+        // delete index
+        assertAcked(client.admin().indices().prepareDelete(index));
+        // request cache stats cannot be access since Index should be deleted
+        try {
+            getRequestCacheStats(client, index);
+        } catch (Exception e) {
+            assert (e instanceof IndexNotFoundException);
+        }
+
+        // sleep until cache cleaner would have cleaned up the stale key from index
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale keys from index
+            assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is lower than staleness, it should clean the cache from all indices having stale keys
+    public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
+        int cacheCleanIntervalInMillis = 10;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // assert cache is empty for index1
+        assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1With1Entries = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1With1Entries > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long memorySizeForIndex1With2Entries = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1With2Entries > memorySizeForIndex1With1Entries);
+
+        // assert cache is empty for index2
+        assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // invalidate the cache for index1
+        indexRandom(false, client.prepareIndex(index1).setId("1").setSource("d", "hello"));
+        forceMerge(client, index1);
+        // Assert cache is cleared up
+        assertBusy(
+            () -> { assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes()); },
+            cacheCleanIntervalInMillis * 2,
+            TimeUnit.MILLISECONDS
+        );
+
+        // invalidate the cache for index2
+        indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello"));
+        forceMerge(client, index2);
+
+        // create another cache entry in index 1 same as memorySizeForIndex1With1Entries, this should not be cleaned up.
+        createCacheEntry(client, index1, "hello");
+
+        // sleep until cache cleaner would have cleaned up the stale key from index2
+        assertBusy(() -> {
+            // assert segment counts stay the same
+            assertEquals(2, getSegmentCount(client, index1));
+            assertEquals(2, getSegmentCount(client, index2));
+            // cache cleaner should have cleaned up the stale key from index2 and hence cache should be empty
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should have only cleaned up the stale entities for index1
+            long currentMemorySizeInBytesForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+            // assert the memory size of index1 to only contain 1 entry added after flushAndRefresh
+            assertEquals(memorySizeForIndex1With1Entries, currentMemorySizeInBytesForIndex1);
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    private void setupIndex(Client client, String index) throws Exception {
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate(index)
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
+                        // Disable background segment merges invalidating the cache
+                        .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false)
+                )
+                .get()
+        );
+        indexRandom(false, client.prepareIndex(index).setSource("k", "hello"));
+        indexRandom(false, client.prepareIndex(index).setSource("k", "there"));
+        ensureSearchable(index);
+        forceMerge(client, index);
+    }
+
+    private int getSegmentCount(Client client, String indexName) {
+        return client.admin()
+            .indices()
+            .segments(new IndicesSegmentsRequest(indexName))
+            .actionGet()
+            .getIndices()
+            .get(indexName)
+            .getShards()
+            .get(0)
+            .getShards()[0].getSegments()
+            .size();
+    }
+
+    private void forceMerge(Client client, String index) {
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+        refreshAndWaitForReplication();
+    }
+
+    private void createCacheEntry(Client client, String index, String value) {
+        SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", value)).get();
+        assertSearchResponse(resp);
+        OpenSearchAssertions.assertAllSuccessful(resp);
+    }
+
+    private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, index);
+        // Check the hit count and miss count together so if they are not
+        // correct we can see both values
+        assertEquals(
+            Arrays.asList(expectedHits, expectedMisses, 0L),
+            Arrays.asList(requestCacheStats.getHitCount(), requestCacheStats.getMissCount(), requestCacheStats.getEvictions())
+        );
+
+    }
+
+    private static RequestCacheStats getRequestCacheStats(Client client, String index) {
+        return client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal().getRequestCache();
+    }
+
+    private static RequestCacheStats getNodeCacheStats(Client client) {
+        NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().execute().actionGet();
+        for (NodeStats stat : stats.getNodes()) {
+            if (stat.getNode().isDataNode()) {
+                return stat.getIndices().getRequestCache();
+            }
+        }
+        return null;
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 0383aca2de33f..09d5c208a8756 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -37,7 +37,6 @@
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
@@ -55,7 +54,7 @@
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.env.NodeEnvironment;
-import org.opensearch.index.IndexNotFoundException;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
@@ -75,13 +74,10 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING;
-import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
-import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateRange;
@@ -126,6 +122,8 @@ public void testCacheAggs() throws Exception {
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(SETTING_NUMBER_OF_SHARDS, 1)
                         .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
                 )
                 .get()
         );
@@ -135,6 +133,8 @@ public void testCacheAggs() throws Exception {
             client.prepareIndex(index).setSource("f", "2014-05-13T00:00:00.000Z")
         );
         ensureSearchable(index);
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        forceMerge(client, index);
 
         // This is not a random example: serialization with time zones writes shared strings
         // which used to not work well with the query cache because of the handles stream output
@@ -197,6 +197,8 @@ public void testQueryRewrite() throws Exception {
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5)
                         .put("index.number_of_routing_shards", 5)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
                 )
                 .get()
         );
@@ -214,12 +216,8 @@ public void testQueryRewrite() throws Exception {
         );
         ensureSearchable(index);
         assertCacheState(client, index, 0, 0);
-
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
-        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
-        refreshAndWaitForReplication();
-        ensureSearchable(index);
+        forceMerge(client, index);
 
         assertCacheState(client, index, 0, 0);
 
@@ -268,6 +266,8 @@ public void testQueryRewriteMissingValues() throws Exception {
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
                 )
                 .get()
         );
@@ -287,10 +287,7 @@ public void testQueryRewriteMissingValues() throws Exception {
         assertCacheState(client, index, 0, 0);
 
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
-        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
-        refreshAndWaitForReplication();
-        ensureSearchable(index);
+        forceMerge(client, index);
 
         assertCacheState(client, index, 0, 0);
 
@@ -335,6 +332,8 @@ public void testQueryRewriteDates() throws Exception {
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
                 )
                 .get()
         );
@@ -354,10 +353,7 @@ public void testQueryRewriteDates() throws Exception {
         assertCacheState(client, index, 0, 0);
 
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
-        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
-        refreshAndWaitForReplication();
-        ensureSearchable(index);
+        forceMerge(client, index);
 
         assertCacheState(client, index, 0, 0);
 
@@ -399,6 +395,8 @@ public void testQueryRewriteDatesWithNow() throws Exception {
             .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            // Disable index refreshing to avoid cache being invalidated mid-test
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
             .build();
         assertAcked(client.admin().indices().prepareCreate("index-1").setMapping("d", "type=date").setSettings(settings).get());
         assertAcked(client.admin().indices().prepareCreate("index-2").setMapping("d", "type=date").setSettings(settings).get());
@@ -480,6 +478,7 @@ public void testCanCache() throws Exception {
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
             .put("index.number_of_routing_shards", 2)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
             .build();
         String index = "index";
         assertAcked(client.admin().indices().prepareCreate(index).setMapping("s", "type=date").setSettings(settings).get());
@@ -499,10 +498,7 @@ public void testCanCache() throws Exception {
         assertCacheState(client, index, 0, 0);
 
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
-        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
-        refreshAndWaitForReplication();
-        ensureSearchable(index);
+        forceMerge(client, index);
 
         assertCacheState(client, index, 0, 0);
 
@@ -644,11 +640,15 @@ public void testProfileDisableCache() throws Exception {
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
                 )
                 .get()
         );
         indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
         ensureSearchable(index);
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        forceMerge(client, index);
 
         int expectedHits = 0;
         int expectedMisses = 0;
@@ -673,556 +673,6 @@ public void testProfileDisableCache() throws Exception {
         }
     }
 
-    public void testCacheWithInvalidation() throws Exception {
-        Client client = client();
-        String index = "index";
-        assertAcked(
-            client.admin()
-                .indices()
-                .prepareCreate(index)
-                .setMapping("k", "type=keyword")
-                .setSettings(
-                    Settings.builder()
-                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                        .put("index.refresh_interval", -1)
-                )
-                .get()
-        );
-        indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
-        ensureSearchable(index);
-        SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
-        assertSearchResponse(resp);
-        OpenSearchAssertions.assertAllSuccessful(resp);
-        assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
-
-        assertCacheState(client, index, 0, 1);
-        // Index but don't refresh
-        indexRandom(false, client.prepareIndex(index).setSource("k", "hello2"));
-        resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
-        assertSearchResponse(resp);
-        // Should expect hit as here as refresh didn't happen
-        assertCacheState(client, index, 1, 1);
-
-        // Explicit refresh would invalidate cache
-        refreshAndWaitForReplication();
-        // Hit same query again
-        resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
-        assertSearchResponse(resp);
-        // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
-        assertCacheState(client, index, 1, 2);
-    }
-
-    // calling cache clear api, when staleness threshold is lower than staleness, it should clean the stale keys from cache
-    public void testCacheClearAPIRemovesStaleKeysWhenStalenessThresholdIsLow() throws Exception {
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    // setting intentionally high to avoid cache cleaner interfering
-                    TimeValue.timeValueMillis(300)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1 > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(index2);
-        client.admin().indices().clearCache(clearIndicesCacheRequest).actionGet();
-
-        // cache cleaner should have cleaned up the stale key from index 2
-        assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-        // cache cleaner should NOT have cleaned from index 1
-        assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-    }
-
-    // when staleness threshold is lower than staleness, it should clean the stale keys from cache
-    public void testStaleKeysCleanupWithLowThreshold() throws Exception {
-        int cacheCleanIntervalInMillis = 1;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1 > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index2);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale key from index 2
-            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-    }
-
-    // when staleness threshold is equal to staleness, it should clean the stale keys from cache
-    public void testCacheCleanupOnEqualStalenessAndThreshold() throws Exception {
-        int cacheCleanIntervalInMillis = 1;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.33)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1 > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index2);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale key from index 2
-            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // when staleness threshold is higher than staleness, it should NOT clean the cache
-    public void testCacheCleanupSkipsWithHighStalenessThreshold() throws Exception {
-        int cacheCleanIntervalInMillis = 1;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1 > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index2);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-        assertBusy(() -> {
-            // cache cleaner should NOT have cleaned up the stale key from index 2
-            assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // when staleness threshold is explicitly set to 0, cache cleaner regularly cleans up stale keys.
-    public void testCacheCleanupOnZeroStalenessThreshold() throws Exception {
-        int cacheCleanIntervalInMillis = 50;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create 10 index1 cache entries
-        for (int i = 1; i <= 10; i++) {
-            long cacheSizeBefore = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-            createCacheEntry(client, index1, "hello" + i);
-            assertCacheState(client, index1, 0, i);
-            long cacheSizeAfter = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-            assertTrue(cacheSizeAfter > cacheSizeBefore);
-        }
-
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index2);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale key from index 2
-            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // when staleness threshold is not explicitly set, cache cleaner regularly cleans up stale keys
-    public void testStaleKeysRemovalWithoutExplicitThreshold() throws Exception {
-        int cacheCleanIntervalInMillis = 1;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        String index1 = "index1";
-        String index2 = "index2";
-        Client client = client(node);
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1 > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index2);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale key from index 2
-            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // when cache cleaner interval setting is not set, cache cleaner is configured appropriately with the fall-back setting
-    public void testCacheCleanupWithDefaultSettings() throws Exception {
-        int cacheCleanIntervalInMillis = 1;
-        String node = internalCluster().startNode(
-            Settings.builder().put(INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY, TimeValue.timeValueMillis(cacheCleanIntervalInMillis))
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1 > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index2);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale key from index 2
-            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // staleness threshold updates flows through to the cache cleaner
-    public void testDynamicStalenessThresholdUpdate() throws Exception {
-        int cacheCleanIntervalInMillis = 1;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1 > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        assertTrue(getRequestCacheStats(client, index1).getMemorySizeInBytes() > memorySizeForIndex1);
-
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(finalMemorySizeForIndex1 > 0);
-
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index2);
-        assertBusy(() -> {
-            // cache cleaner should NOT have cleaned up the stale key from index 2
-            assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-
-        // Update indices.requests.cache.cleanup.staleness_threshold to "10%"
-        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
-        updateSettingsRequest.persistentSettings(Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), 0.10));
-        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
-
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale key from index 2
-            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // staleness threshold dynamic updates should throw exceptions on invalid input
-    public void testInvalidStalenessThresholdUpdateThrowsException() throws Exception {
-        // Update indices.requests.cache.cleanup.staleness_threshold to "10%" with illegal argument
-        assertThrows("Ratio should be in [0-1.0]", IllegalArgumentException.class, () -> {
-            ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
-            updateSettingsRequest.persistentSettings(
-                Settings.builder().put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 10)
-            );
-            client().admin().cluster().updateSettings(updateSettingsRequest).actionGet();
-        });
-    }
-
-    // closing the Index after caching will clean up from Indices Request Cache
-    public void testCacheClearanceAfterIndexClosure() throws Exception {
-        int cacheCleanIntervalInMillis = 100;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index = "index";
-        setupIndex(client, index);
-
-        // assert there are no entries in the cache for index
-        assertEquals(0, getRequestCacheStats(client, index).getMemorySizeInBytes());
-        // assert there are no entries in the cache from other indices in the node
-        assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        // create first cache entry in index
-        createCacheEntry(client, index, "hello");
-        assertCacheState(client, index, 0, 1);
-        assertTrue(getRequestCacheStats(client, index).getMemorySizeInBytes() > 0);
-        assertTrue(getNodeCacheStats(client).getMemorySizeInBytes() > 0);
-
-        // close index
-        assertAcked(client.admin().indices().prepareClose(index));
-        // request cache stats cannot be access since Index should be closed
-        try {
-            getRequestCacheStats(client, index);
-        } catch (Exception e) {
-            assert (e instanceof IndexClosedException);
-        }
-        // sleep until cache cleaner would have cleaned up the stale key from index
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale keys from index
-            assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // deleting the Index after caching will clean up from Indices Request Cache
-    public void testCacheCleanupAfterIndexDeletion() throws Exception {
-        int cacheCleanIntervalInMillis = 100;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index = "index";
-        setupIndex(client, index);
-
-        // assert there are no entries in the cache for index
-        assertEquals(0, getRequestCacheStats(client, index).getMemorySizeInBytes());
-        // assert there are no entries in the cache from other indices in the node
-        assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        // create first cache entry in index
-        createCacheEntry(client, index, "hello");
-        assertCacheState(client, index, 0, 1);
-        assertTrue(getRequestCacheStats(client, index).getMemorySizeInBytes() > 0);
-        assertTrue(getNodeCacheStats(client).getMemorySizeInBytes() > 0);
-
-        // delete index
-        assertAcked(client.admin().indices().prepareDelete(index));
-        // request cache stats cannot be access since Index should be deleted
-        try {
-            getRequestCacheStats(client, index);
-        } catch (Exception e) {
-            assert (e instanceof IndexNotFoundException);
-        }
-
-        // sleep until cache cleaner would have cleaned up the stale key from index
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale keys from index
-            assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
-    // when staleness threshold is lower than staleness, it should clean cache from all indices having stale keys
-    public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
-        int cacheCleanIntervalInMillis = 10;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        String index2 = "index2";
-        setupIndex(client, index1);
-        setupIndex(client, index2);
-
-        // assert cache is empty for index1
-        assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        long memorySizeForIndex1With1Entries = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1With1Entries > 0);
-
-        // create second cache entry in index1
-        createCacheEntry(client, index1, "there");
-        assertCacheState(client, index1, 0, 2);
-        long memorySizeForIndex1With2Entries = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-        assertTrue(memorySizeForIndex1With2Entries > memorySizeForIndex1With1Entries);
-
-        // assert cache is empty for index2
-        assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-        // create first cache entry in index2
-        createCacheEntry(client, index2, "hello");
-        assertCacheState(client, index2, 0, 1);
-        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-
-        // force refresh both index1 and index2
-        flushAndRefresh(index1, index2);
-        // create another cache entry in index 1 same as memorySizeForIndex1With1Entries, this should not be cleaned up.
-        createCacheEntry(client, index1, "hello");
-        // sleep until cache cleaner would have cleaned up the stale key from index2
-        assertBusy(() -> {
-            // cache cleaner should have cleaned up the stale key from index2 and hence cache should be empty
-            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
-            // cache cleaner should have only cleaned up the stale entities for index1
-            long currentMemorySizeInBytesForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
-            // assert the memory size of index1 to only contain 1 entry added after flushAndRefresh
-            assertEquals(memorySizeForIndex1With1Entries, currentMemorySizeInBytesForIndex1);
-            // cache for index1 should not be empty since there was an item cached after flushAndRefresh
-            assertTrue(currentMemorySizeInBytesForIndex1 > 0);
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
-    }
-
     public void testDeleteAndCreateSameIndexShardOnSameNode() throws Exception {
         String node_1 = internalCluster().startNode(Settings.builder().build());
         Client client = client(node_1);
@@ -1238,7 +688,12 @@ public void testDeleteAndCreateSameIndexShardOnSameNode() throws Exception {
         logger.info("Creating an index: {} with 2 shards", indexName);
         createIndex(
             indexName,
-            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                // Disable index refreshing to avoid cache being invalidated mid-test
+                .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
+                .build()
         );
 
         ensureGreen(indexName);
@@ -1246,6 +701,9 @@ public void testDeleteAndCreateSameIndexShardOnSameNode() throws Exception {
         logger.info("Writing few docs and searching those which will cache items in RequestCache");
         indexRandom(true, client.prepareIndex(indexName).setSource("k", "hello"));
         indexRandom(true, client.prepareIndex(indexName).setSource("y", "hello again"));
+        ensureSearchable(indexName);
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        forceMerge(client, indexName);
         SearchResponse resp = client.prepareSearch(indexName).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         resp = client.prepareSearch(indexName).setRequestCache(true).setQuery(QueryBuilders.termQuery("y", "hello")).get();
@@ -1318,29 +776,10 @@ private Path[] shardDirectory(String server, Index index, int shard) {
         return paths;
     }
 
-    private void setupIndex(Client client, String index) throws Exception {
-        assertAcked(
-            client.admin()
-                .indices()
-                .prepareCreate(index)
-                .setMapping("k", "type=keyword")
-                .setSettings(
-                    Settings.builder()
-                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                )
-                .get()
-        );
-        indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
-        indexRandom(true, client.prepareIndex(index).setSource("k", "there"));
-        ensureSearchable(index);
-    }
-
-    private void createCacheEntry(Client client, String index, String value) {
-        SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", value)).get();
-        assertSearchResponse(resp);
-        OpenSearchAssertions.assertAllSuccessful(resp);
+    private void forceMerge(Client client, String index) {
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+        refreshAndWaitForReplication();
     }
 
     private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {

From 57fb50b22bf30148a632bd4c5e78bde53116f00f Mon Sep 17 00:00:00 2001
From: bowenlan-amzn <bowenlan23@gmail.com>
Date: Wed, 19 Jun 2024 15:48:48 -0700
Subject: [PATCH 024/113] Apply the date histogram rewrite optimization to
 range aggregation (#13865)

* Refactor the ranges representation

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* Refactor try fast filter

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* Main work finished; left the handling of different numeric data types

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* buildRanges accepts field type

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* first working draft probably

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* add change log

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* accommodate geo distance agg

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* Fix test

support all numeric types
minus one on the upper range

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* [Refactor] range is lower inclusive, right exclusive

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* adding test

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* Adding test and refactor

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* refactor

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* add test

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* add test and update the compare logic in tree traversal

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* fix test, add random test

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* refactor to address comments

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* small potential performance update

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* fix precommit

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* refactor

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* refactor

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* set refresh_interval to -1

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* address comment

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* address comment

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* address comment

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* Fix test

To understand fully about the double and bigdecimal usage in scaled float field will take more time.

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

---------

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 .../index/mapper/ScaledFloatFieldMapper.java  |  18 +-
 .../test/search.aggregation/40_range.yml      | 139 ++++++
 .../index/mapper/DateFieldMapper.java         |   9 +-
 .../index/mapper/NumberFieldMapper.java       |  87 +++-
 .../index/mapper/NumericPointEncoder.java     |  16 +
 .../bucket/FastFilterRewriteHelper.java       | 470 +++++++++++-------
 .../bucket/composite/CompositeAggregator.java |  19 +-
 .../AutoDateHistogramAggregator.java          |  17 +-
 .../histogram/DateHistogramAggregator.java    |  17 +-
 .../range/AbstractRangeAggregatorFactory.java |   3 +-
 .../range/GeoDistanceAggregatorSupplier.java  |   4 +-
 .../GeoDistanceRangeAggregatorFactory.java    |   9 +-
 .../bucket/range/RangeAggregator.java         |  37 +-
 .../bucket/range/RangeAggregatorSupplier.java |   4 +-
 .../DateHistogramAggregatorTests.java         |   2 +-
 .../bucket/range/RangeAggregatorTests.java    | 282 ++++++++++-
 17 files changed, 902 insertions(+), 232 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/NumericPointEncoder.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 24a4f6fd1b1f1..8c3e63e36bc82 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 - Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724))
 - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
+- Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java
index 400d867296e5f..3115dce6c10a5 100644
--- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java
@@ -35,6 +35,7 @@
 import com.fasterxml.jackson.core.JsonParseException;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
@@ -165,7 +166,7 @@ public ScaledFloatFieldMapper build(BuilderContext context) {
 
     public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings()));
 
-    public static final class ScaledFloatFieldType extends SimpleMappedFieldType {
+    public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder {
 
         private final double scalingFactor;
         private final Double nullValue;
@@ -188,6 +189,21 @@ public ScaledFloatFieldType(String name, double scalingFactor) {
             this(name, true, false, true, Collections.emptyMap(), scalingFactor, null);
         }
 
+        @Override
+        public byte[] encodePoint(Number value) {
+            assert value instanceof Double;
+            double doubleValue = (Double) value;
+            byte[] point = new byte[Long.BYTES];
+            if (doubleValue == Double.POSITIVE_INFINITY) {
+                LongPoint.encodeDimension(Long.MAX_VALUE, point, 0);
+            } else if (doubleValue == Double.NEGATIVE_INFINITY) {
+                LongPoint.encodeDimension(Long.MIN_VALUE, point, 0);
+            } else {
+                LongPoint.encodeDimension(Math.round(scale(value)), point, 0);
+            }
+            return point;
+        }
+
         public double getScalingFactor() {
             return scalingFactor;
         }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
index 7d887d56ae8fe..2fd926276d0b4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
@@ -14,6 +14,9 @@ setup:
                 date:
                   type: date
                   format: epoch_second
+                scaled_field:
+                  type: scaled_float
+                  scaling_factor: 100
 
   - do:
       cluster.health:
@@ -528,3 +531,139 @@ setup:
   - is_false:  aggregations.unsigned_long_range.buckets.2.to
 
   - match: { aggregations.unsigned_long_range.buckets.2.doc_count: 0 }
+
+---
+"Double range profiler shows filter rewrite info":
+  - skip:
+      version: " - 2.99.99"
+      reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.15.0)
+
+  - do:
+      indices.create:
+        index: test_profile
+        body:
+          settings:
+            number_of_replicas: 0
+            refresh_interval: -1
+          mappings:
+            properties:
+              ip:
+                type: ip
+              double:
+                type: double
+              date:
+                type: date
+                format: epoch_second
+
+  - do:
+      bulk:
+        index: test_profile
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"double" : 42}'
+          - '{"index": {}}'
+          - '{"double" : 100}'
+          - '{"index": {}}'
+          - '{"double" : 50}'
+
+  - do:
+      search:
+        index: test_profile
+        body:
+          size: 0
+          profile: true
+          aggs:
+            double_range:
+              range:
+                field: double
+                ranges:
+                  - to: 50
+                  - from: 50
+                    to: 150
+                  - from: 150
+
+  - length: { aggregations.double_range.buckets: 3 }
+
+  - match: { aggregations.double_range.buckets.0.key: "*-50.0" }
+  - is_false: aggregations.double_range.buckets.0.from
+  - match: { aggregations.double_range.buckets.0.to: 50.0 }
+  - match: { aggregations.double_range.buckets.0.doc_count: 1 }
+  - match: { aggregations.double_range.buckets.1.key: "50.0-150.0" }
+  - match: { aggregations.double_range.buckets.1.from: 50.0 }
+  - match: { aggregations.double_range.buckets.1.to: 150.0 }
+  - match: { aggregations.double_range.buckets.1.doc_count: 2 }
+  - match: { aggregations.double_range.buckets.2.key: "150.0-*" }
+  - match: { aggregations.double_range.buckets.2.from: 150.0 }
+  - is_false:  aggregations.double_range.buckets.2.to
+  - match: { aggregations.double_range.buckets.2.doc_count: 0 }
+
+  - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }
+
+---
+"Scaled Float Range Aggregation":
+  - do:
+      index:
+        index: test
+        id: 1
+        body: { "scaled_field": 1 }
+
+  - do:
+      index:
+        index: test
+        id: 2
+        body: { "scaled_field": 1.53 }
+
+  - do:
+      index:
+        index: test
+        id: 3
+        body: { "scaled_field": -2.1 }
+
+  - do:
+      index:
+        index: test
+        id: 4
+        body: { "scaled_field": 1.53 }
+
+  - do:
+      indices.refresh: { }
+
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            my_range:
+              range:
+                field: scaled_field
+                ranges:
+                  - to: 0
+                  - from: 0
+                    to: 1
+                  - from: 1
+                    to: 1.5
+                  - from: 1.5
+
+  - length: { aggregations.my_range.buckets: 4 }
+
+  - match: { aggregations.my_range.buckets.0.key: "*-0.0" }
+  - is_false: aggregations.my_range.buckets.0.from
+  - match: { aggregations.my_range.buckets.0.to: 0.0 }
+  - match: { aggregations.my_range.buckets.0.doc_count: 1 }
+  - match: { aggregations.my_range.buckets.1.key: "0.0-1.0" }
+  - match: { aggregations.my_range.buckets.1.from: 0.0 }
+  - match: { aggregations.my_range.buckets.1.to: 1.0 }
+  - match: { aggregations.my_range.buckets.1.doc_count: 0 }
+  - match: { aggregations.my_range.buckets.2.key: "1.0-1.5" }
+  - match: { aggregations.my_range.buckets.2.from: 1.0 }
+  - match: { aggregations.my_range.buckets.2.to: 1.5 }
+  - match: { aggregations.my_range.buckets.2.doc_count: 1 }
+  - match: { aggregations.my_range.buckets.3.key: "1.5-*" }
+  - match: { aggregations.my_range.buckets.3.from: 1.5 }
+  - is_false:  aggregations.my_range.buckets.3.to
+  - match: { aggregations.my_range.buckets.3.doc_count: 2 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
index b7ee3bb8ca3e3..cf8703209fb37 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
@@ -348,7 +348,7 @@ public DateFieldMapper build(BuilderContext context) {
      *
      * @opensearch.internal
      */
-    public static final class DateFieldType extends MappedFieldType {
+    public static final class DateFieldType extends MappedFieldType implements NumericPointEncoder {
         protected final DateFormatter dateTimeFormatter;
         protected final DateMathParser dateMathParser;
         protected final Resolution resolution;
@@ -549,6 +549,13 @@ public static long parseToLong(
             return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone));
         }
 
+        @Override
+        public byte[] encodePoint(Number value) {
+            byte[] point = new byte[Long.BYTES];
+            LongPoint.encodeDimension(value.longValue(), point, 0);
+            return point;
+        }
+
         @Override
         public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) {
             failIfNotIndexedAndNoDocValues();
diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
index eb3a99b0e0388..25e5f9970795f 100644
--- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
@@ -171,7 +171,7 @@ public NumberFieldMapper build(BuilderContext context) {
      *
      * @opensearch.internal
      */
-    public enum NumberType {
+    public enum NumberType implements NumericPointEncoder {
         HALF_FLOAT("half_float", NumericType.HALF_FLOAT) {
             @Override
             public Float parse(Object value, boolean coerce) {
@@ -194,6 +194,13 @@ public Number parsePoint(byte[] value) {
                 return HalfFloatPoint.decodeDimension(value, 0);
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[HalfFloatPoint.BYTES];
+                HalfFloatPoint.encodeDimension(value.floatValue(), point, 0);
+                return point;
+            }
+
             @Override
             public Float parse(XContentParser parser, boolean coerce) throws IOException {
                 float parsed = parser.floatValue(coerce);
@@ -331,6 +338,13 @@ public Number parsePoint(byte[] value) {
                 return FloatPoint.decodeDimension(value, 0);
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[Float.BYTES];
+                FloatPoint.encodeDimension(value.floatValue(), point, 0);
+                return point;
+            }
+
             @Override
             public Float parse(XContentParser parser, boolean coerce) throws IOException {
                 float parsed = parser.floatValue(coerce);
@@ -457,6 +471,13 @@ public Number parsePoint(byte[] value) {
                 return DoublePoint.decodeDimension(value, 0);
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[Double.BYTES];
+                DoublePoint.encodeDimension(value.doubleValue(), point, 0);
+                return point;
+            }
+
             @Override
             public Double parse(XContentParser parser, boolean coerce) throws IOException {
                 double parsed = parser.doubleValue(coerce);
@@ -582,6 +603,13 @@ public Number parsePoint(byte[] value) {
                 return INTEGER.parsePoint(value).byteValue();
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[Integer.BYTES];
+                IntPoint.encodeDimension(value.intValue(), point, 0);
+                return point;
+            }
+
             @Override
             public Short parse(XContentParser parser, boolean coerce) throws IOException {
                 int value = parser.intValue(coerce);
@@ -654,6 +682,13 @@ public Number parsePoint(byte[] value) {
                 return INTEGER.parsePoint(value).shortValue();
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[Integer.BYTES];
+                IntPoint.encodeDimension(value.intValue(), point, 0);
+                return point;
+            }
+
             @Override
             public Short parse(XContentParser parser, boolean coerce) throws IOException {
                 return parser.shortValue(coerce);
@@ -722,6 +757,13 @@ public Number parsePoint(byte[] value) {
                 return IntPoint.decodeDimension(value, 0);
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[Integer.BYTES];
+                IntPoint.encodeDimension(value.intValue(), point, 0);
+                return point;
+            }
+
             @Override
             public Integer parse(XContentParser parser, boolean coerce) throws IOException {
                 return parser.intValue(coerce);
@@ -868,6 +910,13 @@ public Number parsePoint(byte[] value) {
                 return LongPoint.decodeDimension(value, 0);
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[Long.BYTES];
+                LongPoint.encodeDimension(value.longValue(), point, 0);
+                return point;
+            }
+
             @Override
             public Long parse(XContentParser parser, boolean coerce) throws IOException {
                 return parser.longValue(coerce);
@@ -988,6 +1037,13 @@ public Number parsePoint(byte[] value) {
                 return BigIntegerPoint.decodeDimension(value, 0);
             }
 
+            @Override
+            public byte[] encodePoint(Number value) {
+                byte[] point = new byte[BigIntegerPoint.BYTES];
+                BigIntegerPoint.encodeDimension(objectToUnsignedLong(value, false, true), point, 0);
+                return point;
+            }
+
             @Override
             public BigInteger parse(XContentParser parser, boolean coerce) throws IOException {
                 return parser.bigIntegerValue(coerce);
@@ -1215,16 +1271,30 @@ public static long objectToLong(Object value, boolean coerce) {
             return Numbers.toLong(stringValue, coerce);
         }
 
+        public static BigInteger objectToUnsignedLong(Object value, boolean coerce) {
+            return objectToUnsignedLong(value, coerce, false);
+        }
+
         /**
-         * Converts and Object to a {@code long} by checking it against known
+         * Converts an Object to a {@code BigInteger} by checking it against known
          * types and checking its range.
+         *
+         * @param lenientBound if true, use MIN or MAX if the value is out of bound
          */
-        public static BigInteger objectToUnsignedLong(Object value, boolean coerce) {
+        public static BigInteger objectToUnsignedLong(Object value, boolean coerce, boolean lenientBound) {
             if (value instanceof Long) {
                 return Numbers.toUnsignedBigInteger(((Long) value).longValue());
             }
 
             double doubleValue = objectToDouble(value);
+            if (lenientBound) {
+                if (doubleValue < Numbers.MIN_UNSIGNED_LONG_VALUE.doubleValue()) {
+                    return Numbers.MIN_UNSIGNED_LONG_VALUE;
+                }
+                if (doubleValue > Numbers.MAX_UNSIGNED_LONG_VALUE.doubleValue()) {
+                    return Numbers.MAX_UNSIGNED_LONG_VALUE;
+                }
+            }
             if (doubleValue < Numbers.MIN_UNSIGNED_LONG_VALUE.doubleValue()
                 || doubleValue > Numbers.MAX_UNSIGNED_LONG_VALUE.doubleValue()) {
                 throw new IllegalArgumentException("Value [" + value + "] is out of range for an unsigned long");
@@ -1349,7 +1419,7 @@ public static Query unsignedLongRangeQuery(
      *
      * @opensearch.internal
      */
-    public static class NumberFieldType extends SimpleMappedFieldType {
+    public static class NumberFieldType extends SimpleMappedFieldType implements NumericPointEncoder {
 
         private final NumberType type;
         private final boolean coerce;
@@ -1394,6 +1464,10 @@ public String typeName() {
             return type.name;
         }
 
+        public NumberType numberType() {
+            return type;
+        }
+
         public NumericType numericType() {
             return type.numericType();
         }
@@ -1501,6 +1575,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
         public Number parsePoint(byte[] value) {
             return type.parsePoint(value);
         }
+
+        @Override
+        public byte[] encodePoint(Number value) {
+            return type.encodePoint(value);
+        }
     }
 
     private final NumberType type;
diff --git a/server/src/main/java/org/opensearch/index/mapper/NumericPointEncoder.java b/server/src/main/java/org/opensearch/index/mapper/NumericPointEncoder.java
new file mode 100644
index 0000000000000..be746a5526594
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/NumericPointEncoder.java
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+/**
+ * Interface for encoding a point value
+ */
+public interface NumericPointEncoder {
+    byte[] encodePoint(Number value);
+}
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
index c8ce39a52f869..2ab003fb94e33 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
@@ -32,24 +33,26 @@
 import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.NumericPointEncoder;
 import org.opensearch.index.query.DateRangeIncludingNowQuery;
 import org.opensearch.search.aggregations.bucket.composite.CompositeAggregator;
 import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceConfig;
 import org.opensearch.search.aggregations.bucket.composite.RoundingValuesSource;
 import org.opensearch.search.aggregations.bucket.histogram.LongBounds;
+import org.opensearch.search.aggregations.bucket.range.RangeAggregator.Range;
+import org.opensearch.search.aggregations.support.ValuesSource;
+import org.opensearch.search.aggregations.support.ValuesSourceConfig;
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.OptionalLong;
 import java.util.function.BiConsumer;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 
+import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG;
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 /**
@@ -187,8 +190,8 @@ public static class FastFilterContext {
         private AggregationType aggregationType;
         private final SearchContext context;
 
-        private String fieldName;
-        private long[][] ranges;
+        private MappedFieldType fieldType;
+        private Ranges ranges;
 
         // debug info related fields
         public int leaf;
@@ -196,12 +199,13 @@ public static class FastFilterContext {
         public int segments;
         public int optimizedSegments;
 
-        public void setFieldName(String fieldName) {
-            this.fieldName = fieldName;
+        public FastFilterContext(SearchContext context) {
+            this.context = context;
         }
 
-        public FastFilterContext(SearchContext context) {
+        public FastFilterContext(SearchContext context, AggregationType aggregationType) {
             this.context = context;
+            this.aggregationType = aggregationType;
         }
 
         public AggregationType getAggregationType() {
@@ -221,23 +225,87 @@ public boolean isRewriteable(final Object parent, final int subAggLength) {
             return rewriteable;
         }
 
-        public void buildRanges() throws IOException {
+        public void buildRanges(MappedFieldType fieldType) throws IOException {
             assert ranges == null : "Ranges should only be built once at shard level, but they are already built";
-            this.ranges = this.aggregationType.buildRanges(context);
+            this.fieldType = fieldType;
+            this.ranges = this.aggregationType.buildRanges(context, fieldType);
             if (ranges != null) {
                 logger.debug("Ranges built for shard {}", context.indexShard().shardId());
                 rangesBuiltAtShardLevel = true;
             }
         }
 
-        public long[][] buildRanges(LeafReaderContext leaf) throws IOException {
-            long[][] ranges = this.aggregationType.buildRanges(leaf, context);
+        private Ranges buildRanges(LeafReaderContext leaf) throws IOException {
+            Ranges ranges = this.aggregationType.buildRanges(leaf, context, fieldType);
             if (ranges != null) {
                 logger.debug("Ranges built for shard {} segment {}", context.indexShard().shardId(), leaf.ord);
             }
             return ranges;
         }
 
+        /**
+         * Try to populate the bucket doc counts for aggregation
+         * <p>
+         * Usage: invoked at segment level — in getLeafCollector of aggregator
+         *
+         * @param bucketOrd bucket ordinal producer
+         * @param incrementDocCount consume the doc_count results for certain ordinal
+         */
+        public boolean tryFastFilterAggregation(
+            final LeafReaderContext ctx,
+            final BiConsumer<Long, Long> incrementDocCount,
+            final Function<Object, Long> bucketOrd
+        ) throws IOException {
+            this.segments++;
+            if (!this.rewriteable) {
+                return false;
+            }
+
+            if (ctx.reader().hasDeletions()) return false;
+
+            PointValues values = ctx.reader().getPointValues(this.fieldType.name());
+            if (values == null) return false;
+            // only proceed if every document corresponds to exactly one point
+            if (values.getDocCount() != values.size()) return false;
+
+            NumericDocValues docCountValues = DocValues.getNumeric(ctx.reader(), DocCountFieldMapper.NAME);
+            if (docCountValues.nextDoc() != NO_MORE_DOCS) {
+                logger.debug(
+                    "Shard {} segment {} has at least one document with _doc_count field, skip fast filter optimization",
+                    this.context.indexShard().shardId(),
+                    ctx.ord
+                );
+                return false;
+            }
+
+            // even if no ranges built at shard level, we can still perform the optimization
+            // when functionally match-all at segment level
+            if (!this.rangesBuiltAtShardLevel && !segmentMatchAll(this.context, ctx)) {
+                return false;
+            }
+
+            Ranges ranges = this.ranges;
+            if (ranges == null) {
+                logger.debug(
+                    "Shard {} segment {} functionally match all documents. Build the fast filter",
+                    this.context.indexShard().shardId(),
+                    ctx.ord
+                );
+                ranges = this.buildRanges(ctx);
+                if (ranges == null) {
+                    return false;
+                }
+            }
+
+            DebugInfo debugInfo = this.aggregationType.tryFastFilterAggregation(values, ranges, incrementDocCount, bucketOrd);
+            this.consumeDebugInfo(debugInfo);
+
+            this.optimizedSegments++;
+            logger.debug("Fast filter optimization applied to shard {} segment {}", this.context.indexShard().shardId(), ctx.ord);
+            logger.debug("crossed leaf nodes: {}, inner nodes: {}", this.leaf, this.inner);
+            return true;
+        }
+
         private void consumeDebugInfo(DebugInfo debug) {
             leaf += debug.leaf;
             inner += debug.inner;
@@ -250,9 +318,16 @@ private void consumeDebugInfo(DebugInfo debug) {
     interface AggregationType {
         boolean isRewriteable(Object parent, int subAggLength);
 
-        long[][] buildRanges(SearchContext ctx) throws IOException;
+        Ranges buildRanges(SearchContext ctx, MappedFieldType fieldType) throws IOException;
 
-        long[][] buildRanges(LeafReaderContext leaf, SearchContext ctx) throws IOException;
+        Ranges buildRanges(LeafReaderContext leaf, SearchContext ctx, MappedFieldType fieldType) throws IOException;
+
+        DebugInfo tryFastFilterAggregation(
+            PointValues values,
+            Ranges ranges,
+            BiConsumer<Long, Long> incrementDocCount,
+            Function<Object, Long> bucketOrd
+        ) throws IOException;
     }
 
     /**
@@ -286,13 +361,20 @@ public boolean isRewriteable(Object parent, int subAggLength) {
         }
 
         @Override
-        public long[][] buildRanges(SearchContext context) throws IOException {
+        public Ranges buildRanges(SearchContext context, MappedFieldType fieldType) throws IOException {
             long[] bounds = getDateHistoAggBounds(context, fieldType.name());
             logger.debug("Bounds are {} for shard {}", bounds, context.indexShard().shardId());
             return buildRanges(context, bounds);
         }
 
-        private long[][] buildRanges(SearchContext context, long[] bounds) throws IOException {
+        @Override
+        public Ranges buildRanges(LeafReaderContext leaf, SearchContext context, MappedFieldType fieldType) throws IOException {
+            long[] bounds = getSegmentBounds(leaf, fieldType.name());
+            logger.debug("Bounds are {} for shard {} segment {}", bounds, context.indexShard().shardId(), leaf.ord);
+            return buildRanges(context, bounds);
+        }
+
+        private Ranges buildRanges(SearchContext context, long[] bounds) throws IOException {
             bounds = processHardBounds(bounds);
             if (bounds == null) {
                 return null;
@@ -319,13 +401,6 @@ private long[][] buildRanges(SearchContext context, long[] bounds) throws IOExce
             );
         }
 
-        @Override
-        public long[][] buildRanges(LeafReaderContext leaf, SearchContext context) throws IOException {
-            long[] bounds = getSegmentBounds(leaf, fieldType.name());
-            logger.debug("Bounds are {} for shard {} segment {}", bounds, context.indexShard().shardId(), leaf.ord);
-            return buildRanges(context, bounds);
-        }
-
         protected abstract Rounding getRounding(final long low, final long high);
 
         protected abstract Rounding.Prepared getRoundingPrepared();
@@ -354,86 +429,118 @@ public DateFieldMapper.DateFieldType getFieldType() {
             assert fieldType instanceof DateFieldMapper.DateFieldType;
             return (DateFieldMapper.DateFieldType) fieldType;
         }
-    }
 
-    public static boolean isCompositeAggRewriteable(CompositeValuesSourceConfig[] sourceConfigs) {
-        return sourceConfigs.length == 1 && sourceConfigs[0].valuesSource() instanceof RoundingValuesSource;
-    }
+        @Override
+        public DebugInfo tryFastFilterAggregation(
+            PointValues values,
+            Ranges ranges,
+            BiConsumer<Long, Long> incrementDocCount,
+            Function<Object, Long> bucketOrd
+        ) throws IOException {
+            int size = Integer.MAX_VALUE;
+            if (this instanceof CompositeAggregator.CompositeAggregationType) {
+                size = ((CompositeAggregator.CompositeAggregationType) this).getSize();
+            }
+
+            DateFieldMapper.DateFieldType fieldType = getFieldType();
+            BiConsumer<Integer, Integer> incrementFunc = (activeIndex, docCount) -> {
+                long rangeStart = LongPoint.decodeDimension(ranges.lowers[activeIndex], 0);
+                rangeStart = fieldType.convertNanosToMillis(rangeStart);
+                long ord = getBucketOrd(bucketOrd.apply(rangeStart));
+                incrementDocCount.accept(ord, (long) docCount);
+            };
 
-    public static long getBucketOrd(long bucketOrd) {
-        if (bucketOrd < 0) { // already seen
-            bucketOrd = -1 - bucketOrd;
+            return multiRangesTraverse(values.getPointTree(), ranges, incrementFunc, size);
         }
 
-        return bucketOrd;
+        private static long getBucketOrd(long bucketOrd) {
+            if (bucketOrd < 0) { // already seen
+                bucketOrd = -1 - bucketOrd;
+            }
+
+            return bucketOrd;
+        }
     }
 
     /**
-     * Try to get the bucket doc counts for the date histogram aggregation
-     * <p>
-     * Usage: invoked at segment level — in getLeafCollector of aggregator
-     *
-     * @param incrementDocCount takes in the bucket key value and the bucket count
+     * For range aggregation
      */
-    public static boolean tryFastFilterAggregation(
-        final LeafReaderContext ctx,
-        FastFilterContext fastFilterContext,
-        final BiConsumer<Long, Integer> incrementDocCount
-    ) throws IOException {
-        fastFilterContext.segments++;
-        if (!fastFilterContext.rewriteable) {
-            return false;
-        }
+    public static class RangeAggregationType implements AggregationType {
 
-        if (ctx.reader().hasDeletions()) return false;
+        private final ValuesSourceConfig config;
+        private final Range[] ranges;
 
-        PointValues values = ctx.reader().getPointValues(fastFilterContext.fieldName);
-        if (values == null) return false;
-        // only proceed if every document corresponds to exactly one point
-        if (values.getDocCount() != values.size()) return false;
-
-        NumericDocValues docCountValues = DocValues.getNumeric(ctx.reader(), DocCountFieldMapper.NAME);
-        if (docCountValues.nextDoc() != NO_MORE_DOCS) {
-            logger.debug(
-                "Shard {} segment {} has at least one document with _doc_count field, skip fast filter optimization",
-                fastFilterContext.context.indexShard().shardId(),
-                ctx.ord
-            );
-            return false;
+        public RangeAggregationType(ValuesSourceConfig config, Range[] ranges) {
+            this.config = config;
+            this.ranges = ranges;
         }
 
-        // even if no ranges built at shard level, we can still perform the optimization
-        // when functionally match-all at segment level
-        if (!fastFilterContext.rangesBuiltAtShardLevel && !segmentMatchAll(fastFilterContext.context, ctx)) {
+        @Override
+        public boolean isRewriteable(Object parent, int subAggLength) {
+            if (config.fieldType() == null) return false;
+            MappedFieldType fieldType = config.fieldType();
+            if (fieldType.isSearchable() == false || !(fieldType instanceof NumericPointEncoder)) return false;
+
+            if (parent == null && subAggLength == 0 && config.script() == null && config.missing() == null) {
+                if (config.getValuesSource() instanceof ValuesSource.Numeric.FieldData) {
+                    // ranges are already sorted by from and then to
+                    // we want ranges not overlapping with each other
+                    double prevTo = ranges[0].getTo();
+                    for (int i = 1; i < ranges.length; i++) {
+                        if (prevTo > ranges[i].getFrom()) {
+                            return false;
+                        }
+                        prevTo = ranges[i].getTo();
+                    }
+                    return true;
+                }
+            }
             return false;
         }
-        long[][] ranges = fastFilterContext.ranges;
-        if (ranges == null) {
-            logger.debug(
-                "Shard {} segment {} functionally match all documents. Build the fast filter",
-                fastFilterContext.context.indexShard().shardId(),
-                ctx.ord
-            );
-            ranges = fastFilterContext.buildRanges(ctx);
-            if (ranges == null) {
-                return false;
+
+        @Override
+        public Ranges buildRanges(SearchContext context, MappedFieldType fieldType) {
+            assert fieldType instanceof NumericPointEncoder;
+            NumericPointEncoder numericPointEncoder = (NumericPointEncoder) fieldType;
+            byte[][] lowers = new byte[ranges.length][];
+            byte[][] uppers = new byte[ranges.length][];
+            for (int i = 0; i < ranges.length; i++) {
+                double rangeMin = ranges[i].getFrom();
+                double rangeMax = ranges[i].getTo();
+                byte[] lower = numericPointEncoder.encodePoint(rangeMin);
+                byte[] upper = numericPointEncoder.encodePoint(rangeMax);
+                lowers[i] = lower;
+                uppers[i] = upper;
             }
+
+            return new Ranges(lowers, uppers);
         }
 
-        final AggregationType aggregationType = fastFilterContext.aggregationType;
-        assert aggregationType instanceof AbstractDateHistogramAggregationType;
-        final DateFieldMapper.DateFieldType fieldType = ((AbstractDateHistogramAggregationType) aggregationType).getFieldType();
-        int size = Integer.MAX_VALUE;
-        if (aggregationType instanceof CompositeAggregator.CompositeAggregationType) {
-            size = ((CompositeAggregator.CompositeAggregationType) aggregationType).getSize();
+        @Override
+        public Ranges buildRanges(LeafReaderContext leaf, SearchContext ctx, MappedFieldType fieldType) {
+            throw new UnsupportedOperationException("Range aggregation should not build ranges at segment level");
+        }
+
+        @Override
+        public DebugInfo tryFastFilterAggregation(
+            PointValues values,
+            Ranges ranges,
+            BiConsumer<Long, Long> incrementDocCount,
+            Function<Object, Long> bucketOrd
+        ) throws IOException {
+            int size = Integer.MAX_VALUE;
+
+            BiConsumer<Integer, Integer> incrementFunc = (activeIndex, docCount) -> {
+                long ord = bucketOrd.apply(activeIndex);
+                incrementDocCount.accept(ord, (long) docCount);
+            };
+
+            return multiRangesTraverse(values.getPointTree(), ranges, incrementFunc, size);
         }
-        DebugInfo debugInfo = multiRangesTraverse(values.getPointTree(), ranges, incrementDocCount, fieldType, size);
-        fastFilterContext.consumeDebugInfo(debugInfo);
+    }
 
-        fastFilterContext.optimizedSegments++;
-        logger.debug("Fast filter optimization applied to shard {} segment {}", fastFilterContext.context.indexShard().shardId(), ctx.ord);
-        logger.debug("crossed leaf nodes: {}, inner nodes: {}", fastFilterContext.leaf, fastFilterContext.inner);
-        return true;
+    public static boolean isCompositeAggRewriteable(CompositeValuesSourceConfig[] sourceConfigs) {
+        return sourceConfigs.length == 1 && sourceConfigs[0].valuesSource() instanceof RoundingValuesSource;
     }
 
     private static boolean segmentMatchAll(SearchContext ctx, LeafReaderContext leafCtx) throws IOException {
@@ -445,7 +552,7 @@ private static boolean segmentMatchAll(SearchContext ctx, LeafReaderContext leaf
      * Creates the date ranges from date histo aggregations using its interval,
      * and min/max boundaries
      */
-    private static long[][] createRangesFromAgg(
+    private static Ranges createRangesFromAgg(
         final SearchContext context,
         final DateFieldMapper.DateFieldType fieldType,
         final long interval,
@@ -481,9 +588,8 @@ private static long[][] createRangesFromAgg(
                 long lower = i == 0 ? low : fieldType.convertRoundedMillisToNanos(roundedLow);
                 roundedLow = preparedRounding.round(roundedLow + interval);
 
-                // Subtract -1 if the minimum is roundedLow as roundedLow itself
-                // is included in the next bucket
-                long upper = i + 1 == bucketCount ? high : fieldType.convertRoundedMillisToNanos(roundedLow) - 1;
+                // plus one on high value because upper bound is exclusive, but high value exists
+                long upper = i + 1 == bucketCount ? high + 1 : fieldType.convertRoundedMillisToNanos(roundedLow);
 
                 ranges[i][0] = lower;
                 ranges[i][1] = upper;
@@ -491,7 +597,16 @@ private static long[][] createRangesFromAgg(
             }
         }
 
-        return ranges;
+        byte[][] lowers = new byte[ranges.length][];
+        byte[][] uppers = new byte[ranges.length][];
+        for (int i = 0; i < ranges.length; i++) {
+            byte[] lower = LONG.encodePoint(ranges[i][0]);
+            byte[] max = LONG.encodePoint(ranges[i][1]);
+            lowers[i] = lower;
+            uppers[i] = max;
+        }
+
+        return new Ranges(lowers, uppers);
     }
 
     /**
@@ -499,39 +614,18 @@ private static long[][] createRangesFromAgg(
      */
     private static DebugInfo multiRangesTraverse(
         final PointValues.PointTree tree,
-        final long[][] ranges,
-        final BiConsumer<Long, Integer> incrementDocCount,
-        final DateFieldMapper.DateFieldType fieldType,
+        final Ranges ranges,
+        final BiConsumer<Integer, Integer> incrementDocCount,
         final int maxNumNonZeroRanges
     ) throws IOException {
-        // ranges are connected and in ascending order
-        Iterator<long[]> rangeIter = Arrays.stream(ranges).iterator();
-        long[] activeRange = rangeIter.next();
-
-        // make sure the first range at least crosses the min value of the tree
         DebugInfo debugInfo = new DebugInfo();
-        if (activeRange[0] > NumericUtils.sortableBytesToLong(tree.getMaxPackedValue(), 0)) {
+        int activeIndex = ranges.firstRangeIndex(tree.getMinPackedValue(), tree.getMaxPackedValue());
+        if (activeIndex < 0) {
             logger.debug("No ranges match the query, skip the fast filter optimization");
             return debugInfo;
         }
-        while (activeRange[1] < NumericUtils.sortableBytesToLong(tree.getMinPackedValue(), 0)) {
-            if (!rangeIter.hasNext()) {
-                logger.debug("No ranges match the query, skip the fast filter optimization");
-                return debugInfo;
-            }
-            activeRange = rangeIter.next();
-        }
-
-        RangeCollectorForPointTree collector = new RangeCollectorForPointTree(
-            incrementDocCount,
-            fieldType,
-            rangeIter,
-            maxNumNonZeroRanges,
-            activeRange
-        );
-
-        final ArrayUtil.ByteArrayComparator comparator = ArrayUtil.getUnsignedComparator(8);
-        PointValues.IntersectVisitor visitor = getIntersectVisitor(collector, comparator);
+        RangeCollectorForPointTree collector = new RangeCollectorForPointTree(incrementDocCount, maxNumNonZeroRanges, ranges, activeIndex);
+        PointValues.IntersectVisitor visitor = getIntersectVisitor(collector);
         try {
             intersectWithRanges(visitor, tree, collector, debugInfo);
         } catch (CollectionTerminatedException e) {
@@ -542,6 +636,49 @@ private static DebugInfo multiRangesTraverse(
         return debugInfo;
     }
 
+    private static class Ranges {
+        byte[][] lowers; // inclusive
+        byte[][] uppers; // exclusive
+        int size;
+        int byteLen;
+        static ArrayUtil.ByteArrayComparator comparator;
+
+        Ranges(byte[][] lowers, byte[][] uppers) {
+            this.lowers = lowers;
+            this.uppers = uppers;
+            assert lowers.length == uppers.length;
+            this.size = lowers.length;
+            this.byteLen = lowers[0].length;
+            comparator = ArrayUtil.getUnsignedComparator(byteLen);
+        }
+
+        public int firstRangeIndex(byte[] globalMin, byte[] globalMax) {
+            if (compareByteValue(lowers[0], globalMax) > 0) {
+                return -1;
+            }
+            int i = 0;
+            while (compareByteValue(uppers[i], globalMin) <= 0) {
+                i++;
+                if (i >= size) {
+                    return -1;
+                }
+            }
+            return i;
+        }
+
+        public static int compareByteValue(byte[] value1, byte[] value2) {
+            return comparator.compare(value1, 0, value2, 0);
+        }
+
+        public static boolean withinLowerBound(byte[] value, byte[] lowerBound) {
+            return compareByteValue(value, lowerBound) >= 0;
+        }
+
+        public static boolean withinUpperBound(byte[] value, byte[] upperBound) {
+            return compareByteValue(value, upperBound) < 0;
+        }
+    }
+
     private static void intersectWithRanges(
         PointValues.IntersectVisitor visitor,
         PointValues.PointTree pointTree,
@@ -570,10 +707,7 @@ private static void intersectWithRanges(
         }
     }
 
-    private static PointValues.IntersectVisitor getIntersectVisitor(
-        RangeCollectorForPointTree collector,
-        ArrayUtil.ByteArrayComparator comparator
-    ) {
+    private static PointValues.IntersectVisitor getIntersectVisitor(RangeCollectorForPointTree collector) {
         return new PointValues.IntersectVisitor() {
             @Override
             public void visit(int docID) throws IOException {
@@ -591,86 +725,67 @@ public void visit(int docID, byte[] packedValue) throws IOException {
             @Override
             public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOException {
                 visitPoints(packedValue, () -> {
-                    for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
+                    for (int doc = iterator.nextDoc(); doc != NO_MORE_DOCS; doc = iterator.nextDoc()) {
                         collector.count();
                     }
                 });
             }
 
             private void visitPoints(byte[] packedValue, CheckedRunnable<IOException> collect) throws IOException {
-                if (comparator.compare(packedValue, 0, collector.activeRangeAsByteArray[1], 0) > 0) {
-                    // need to move to next range
+                if (!collector.withinUpperBound(packedValue)) {
                     collector.finalizePreviousRange();
-                    if (collector.iterateRangeEnd(packedValue, this::compareByteValue)) {
+                    if (collector.iterateRangeEnd(packedValue)) {
                         throw new CollectionTerminatedException();
                     }
                 }
 
-                if (pointCompare(collector.activeRangeAsByteArray[0], collector.activeRangeAsByteArray[1], packedValue)) {
+                if (collector.withinRange(packedValue)) {
                     collect.run();
                 }
             }
 
-            private boolean pointCompare(byte[] lower, byte[] upper, byte[] packedValue) {
-                if (compareByteValue(packedValue, lower) < 0) {
-                    return false;
-                }
-                return compareByteValue(packedValue, upper) <= 0;
-            }
-
-            private int compareByteValue(byte[] value1, byte[] value2) {
-                return comparator.compare(value1, 0, value2, 0);
-            }
-
             @Override
             public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
-                byte[] rangeMin = collector.activeRangeAsByteArray[0];
-                byte[] rangeMax = collector.activeRangeAsByteArray[1];
-
-                if (compareByteValue(rangeMax, minPackedValue) < 0) {
+                // try to find the first range that may collect values from this cell
+                if (!collector.withinUpperBound(minPackedValue)) {
                     collector.finalizePreviousRange();
-                    if (collector.iterateRangeEnd(minPackedValue, this::compareByteValue)) {
+                    if (collector.iterateRangeEnd(minPackedValue)) {
                         throw new CollectionTerminatedException();
                     }
-                    // compare the next range with this node's min max again
-                    // new rangeMin = previous rangeMax + 1 <= min
-                    rangeMax = collector.activeRangeAsByteArray[1];
                 }
-
-                if (compareByteValue(rangeMin, minPackedValue) > 0 || compareByteValue(rangeMax, maxPackedValue) < 0) {
-                    return PointValues.Relation.CELL_CROSSES_QUERY;
-                } else {
+                // after the loop, min < upper
+                // cell could be outside [min max] lower
+                if (!collector.withinLowerBound(maxPackedValue)) {
+                    return PointValues.Relation.CELL_OUTSIDE_QUERY;
+                }
+                if (collector.withinRange(minPackedValue) && collector.withinRange(maxPackedValue)) {
                     return PointValues.Relation.CELL_INSIDE_QUERY;
                 }
+                return PointValues.Relation.CELL_CROSSES_QUERY;
             }
         };
     }
 
     private static class RangeCollectorForPointTree {
-        private final BiConsumer<Long, Integer> incrementDocCount;
-        private final DateFieldMapper.DateFieldType fieldType;
+        private final BiConsumer<Integer, Integer> incrementRangeDocCount;
         private int counter = 0;
 
-        private long[] activeRange;
-        private byte[][] activeRangeAsByteArray;
-        private final Iterator<long[]> rangeIter;
+        private final Ranges ranges;
+        private int activeIndex;
 
         private int visitedRange = 0;
         private final int maxNumNonZeroRange;
 
         public RangeCollectorForPointTree(
-            BiConsumer<Long, Integer> incrementDocCount,
-            DateFieldMapper.DateFieldType fieldType,
-            Iterator<long[]> rangeIter,
+            BiConsumer<Integer, Integer> incrementRangeDocCount,
             int maxNumNonZeroRange,
-            long[] activeRange
+            Ranges ranges,
+            int activeIndex
         ) {
-            this.incrementDocCount = incrementDocCount;
-            this.fieldType = fieldType;
-            this.rangeIter = rangeIter;
+            this.incrementRangeDocCount = incrementRangeDocCount;
             this.maxNumNonZeroRange = maxNumNonZeroRange;
-            this.activeRange = activeRange;
-            this.activeRangeAsByteArray = activeRangeAsByteArray();
+            this.ranges = ranges;
+            this.activeIndex = activeIndex;
         }
 
         private void count() {
@@ -683,9 +798,7 @@ private void countNode(int count) {
 
         private void finalizePreviousRange() {
             if (counter > 0) {
-                logger.debug("finalize previous range: {}", activeRange[0]);
-                logger.debug("counter: {}", counter);
-                incrementDocCount.accept(fieldType.convertNanosToMillis(activeRange[0]), counter);
+                incrementRangeDocCount.accept(activeIndex, counter);
                 counter = 0;
             }
         }
@@ -693,29 +806,34 @@ private void finalizePreviousRange() {
         /**
          * @return true when iterator exhausted or collect enough non-zero ranges
          */
-        private boolean iterateRangeEnd(byte[] value, BiFunction<byte[], byte[], Integer> comparator) {
+        private boolean iterateRangeEnd(byte[] value) {
             // the new value may not be contiguous to the previous one
             // so try to find the first next range that cross the new value
-            while (comparator.apply(activeRangeAsByteArray[1], value) < 0) {
-                if (!rangeIter.hasNext()) {
+            while (!withinUpperBound(value)) {
+                if (++activeIndex >= ranges.size) {
                     return true;
                 }
-                activeRange = rangeIter.next();
-                activeRangeAsByteArray = activeRangeAsByteArray();
             }
             visitedRange++;
             return visitedRange > maxNumNonZeroRange;
         }
 
-        private byte[][] activeRangeAsByteArray() {
-            byte[] lower = new byte[8];
-            byte[] upper = new byte[8];
-            NumericUtils.longToSortableBytes(activeRange[0], lower, 0);
-            NumericUtils.longToSortableBytes(activeRange[1], upper, 0);
-            return new byte[][] { lower, upper };
+        private boolean withinLowerBound(byte[] value) {
+            return Ranges.withinLowerBound(value, ranges.lowers[activeIndex]);
+        }
+
+        private boolean withinUpperBound(byte[] value) {
+            return Ranges.withinUpperBound(value, ranges.uppers[activeIndex]);
+        }
+
+        private boolean withinRange(byte[] value) {
+            return withinLowerBound(value) && withinUpperBound(value);
         }
     }
 
+    /**
+     * Contains debug info of BKD traversal to show in profile
+     */
     private static class DebugInfo {
         private int leaf = 0; // leaf node visited
         private int inner = 0; // inner node visited
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
index 3713d8f83990d..bfb484dcf478d 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -74,6 +74,7 @@
 import org.opensearch.search.aggregations.MultiBucketConsumerService;
 import org.opensearch.search.aggregations.bucket.BucketsAggregator;
 import org.opensearch.search.aggregations.bucket.FastFilterRewriteHelper;
+import org.opensearch.search.aggregations.bucket.FastFilterRewriteHelper.AbstractDateHistogramAggregationType;
 import org.opensearch.search.aggregations.bucket.missing.MissingOrder;
 import org.opensearch.search.aggregations.bucket.terms.LongKeyedBucketOrds;
 import org.opensearch.search.internal.SearchContext;
@@ -166,21 +167,22 @@ public final class CompositeAggregator extends BucketsAggregator {
         this.rawAfterKey = rawAfterKey;
 
         fastFilterContext = new FastFilterRewriteHelper.FastFilterContext(context);
-        if (!FastFilterRewriteHelper.isCompositeAggRewriteable(sourceConfigs)) return;
+        if (!FastFilterRewriteHelper.isCompositeAggRewriteable(sourceConfigs)) {
+            return;
+        }
         fastFilterContext.setAggregationType(new CompositeAggregationType());
         if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
             // bucketOrds is used for saving date histogram results
             bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), CardinalityUpperBound.ONE);
             preparedRounding = ((CompositeAggregationType) fastFilterContext.getAggregationType()).getRoundingPrepared();
-            fastFilterContext.setFieldName(sourceConfigs[0].fieldType().name());
-            fastFilterContext.buildRanges();
+            fastFilterContext.buildRanges(sourceConfigs[0].fieldType());
         }
     }
 
     /**
      * Currently the filter rewrite is only supported for date histograms
      */
-    public class CompositeAggregationType extends FastFilterRewriteHelper.AbstractDateHistogramAggregationType {
+    public class CompositeAggregationType extends AbstractDateHistogramAggregationType {
         private final RoundingValuesSource valuesSource;
         private long afterKey = -1L;
 
@@ -549,13 +551,10 @@ private void processLeafFromQuery(LeafReaderContext ctx, Sort indexSortPrefix) t
 
     @Override
     protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
-        boolean optimized = FastFilterRewriteHelper.tryFastFilterAggregation(
+        boolean optimized = fastFilterContext.tryFastFilterAggregation(
             ctx,
-            fastFilterContext,
-            (key, count) -> incrementBucketDocCount(
-                FastFilterRewriteHelper.getBucketOrd(bucketOrds.add(0, preparedRounding.round(key))),
-                count
-            )
+            this::incrementBucketDocCount,
+            (key) -> bucketOrds.add(0, preparedRounding.round((long) key))
         );
         if (optimized) throw new CollectionTerminatedException();
 
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
index f326426800909..d13d575a9d696 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
@@ -64,6 +64,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.LongToIntFunction;
@@ -157,8 +158,8 @@ private AutoDateHistogramAggregator(
         this.roundingPreparer = roundingPreparer;
         this.preparedRounding = prepareRounding(0);
 
-        fastFilterContext = new FastFilterRewriteHelper.FastFilterContext(context);
-        fastFilterContext.setAggregationType(
+        fastFilterContext = new FastFilterRewriteHelper.FastFilterContext(
+            context,
             new AutoHistogramAggregationType(
                 valuesSourceConfig.fieldType(),
                 valuesSourceConfig.missing() != null,
@@ -166,8 +167,7 @@ private AutoDateHistogramAggregator(
             )
         );
         if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
-            fastFilterContext.setFieldName(valuesSourceConfig.fieldType().name());
-            fastFilterContext.buildRanges();
+            fastFilterContext.buildRanges(Objects.requireNonNull(valuesSourceConfig.fieldType()));
         }
     }
 
@@ -236,13 +236,10 @@ public final LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBuc
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
 
-        boolean optimized = FastFilterRewriteHelper.tryFastFilterAggregation(
+        boolean optimized = fastFilterContext.tryFastFilterAggregation(
             ctx,
-            fastFilterContext,
-            (key, count) -> incrementBucketDocCount(
-                FastFilterRewriteHelper.getBucketOrd(getBucketOrds().add(0, preparedRounding.round(key))),
-                count
-            )
+            this::incrementBucketDocCount,
+            (key) -> getBucketOrds().add(0, preparedRounding.round((long) key))
         );
         if (optimized) throw new CollectionTerminatedException();
 
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index dd4ee9196fd62..4b84797c18922 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -58,6 +58,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.BiConsumer;
 
 /**
@@ -116,8 +117,8 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg
 
         bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), cardinality);
 
-        fastFilterContext = new FastFilterRewriteHelper.FastFilterContext(context);
-        fastFilterContext.setAggregationType(
+        fastFilterContext = new FastFilterRewriteHelper.FastFilterContext(
+            context,
             new DateHistogramAggregationType(
                 valuesSourceConfig.fieldType(),
                 valuesSourceConfig.missing() != null,
@@ -126,8 +127,7 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg
             )
         );
         if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
-            fastFilterContext.setFieldName(valuesSourceConfig.fieldType().name());
-            fastFilterContext.buildRanges();
+            fastFilterContext.buildRanges(Objects.requireNonNull(valuesSourceConfig.fieldType()));
         }
     }
 
@@ -162,13 +162,10 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
 
-        boolean optimized = FastFilterRewriteHelper.tryFastFilterAggregation(
+        boolean optimized = fastFilterContext.tryFastFilterAggregation(
             ctx,
-            fastFilterContext,
-            (key, count) -> incrementBucketDocCount(
-                FastFilterRewriteHelper.getBucketOrd(bucketOrds.add(0, preparedRounding.round(key))),
-                count
-            )
+            this::incrementBucketDocCount,
+            (key) -> bucketOrds.add(0, preparedRounding.round((long) key))
         );
         if (optimized) throw new CollectionTerminatedException();
 
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
index 41f2768eb7544..fd334638a0c1f 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
@@ -119,7 +119,8 @@ protected Aggregator doCreateInternal(
                 searchContext,
                 parent,
                 cardinality,
-                metadata
+                metadata,
+                config
             );
     }
 
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregatorSupplier.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregatorSupplier.java
index c4a9efda18bda..d72c817c4515b 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregatorSupplier.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregatorSupplier.java
@@ -39,6 +39,7 @@
 import org.opensearch.search.aggregations.AggregatorFactories;
 import org.opensearch.search.aggregations.CardinalityUpperBound;
 import org.opensearch.search.aggregations.support.ValuesSource;
+import org.opensearch.search.aggregations.support.ValuesSourceConfig;
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -64,6 +65,7 @@ Aggregator build(
         SearchContext context,
         Aggregator parent,
         CardinalityUpperBound cardinality,
-        Map<String, Object> metadata
+        Map<String, Object> metadata,
+        ValuesSourceConfig valuesSourceConfig
     ) throws IOException;
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
index 728f43094cf7e..f9e966deb3cc9 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
@@ -82,7 +82,8 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
                 context,
                 parent,
                 cardinality,
-                metadata) -> {
+                metadata,
+                config) -> {
                 DistanceSource distanceSource = new DistanceSource((ValuesSource.GeoPoint) valuesSource, distanceType, origin, units);
                 return new RangeAggregator(
                     name,
@@ -95,7 +96,8 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
                     context,
                     parent,
                     cardinality,
-                    metadata
+                    metadata,
+                    config
                 );
             },
             true
@@ -168,7 +170,8 @@ protected Aggregator doCreateInternal(
                 searchContext,
                 parent,
                 cardinality,
-                metadata
+                metadata,
+                config
             );
     }
 
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java
index b56b817b8177b..2ba2b06514de1 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -32,6 +32,7 @@
 package org.opensearch.search.aggregations.bucket.range;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.ScoreMode;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -54,7 +55,9 @@
 import org.opensearch.search.aggregations.LeafBucketCollectorBase;
 import org.opensearch.search.aggregations.NonCollectingAggregator;
 import org.opensearch.search.aggregations.bucket.BucketsAggregator;
+import org.opensearch.search.aggregations.bucket.FastFilterRewriteHelper;
 import org.opensearch.search.aggregations.support.ValuesSource;
+import org.opensearch.search.aggregations.support.ValuesSourceConfig;
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -62,6 +65,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.BiConsumer;
 
 import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
@@ -245,6 +249,8 @@ public boolean equals(Object obj) {
 
     final double[] maxTo;
 
+    private final FastFilterRewriteHelper.FastFilterContext fastFilterContext;
+
     public RangeAggregator(
         String name,
         AggregatorFactories factories,
@@ -256,17 +262,16 @@ public RangeAggregator(
         SearchContext context,
         Aggregator parent,
         CardinalityUpperBound cardinality,
-        Map<String, Object> metadata
+        Map<String, Object> metadata,
+        ValuesSourceConfig config
     ) throws IOException {
-
         super(name, factories, context, parent, cardinality.multiply(ranges.length), metadata);
         assert valuesSource != null;
         this.valuesSource = valuesSource;
         this.format = format;
         this.keyed = keyed;
         this.rangeFactory = rangeFactory;
-
-        this.ranges = ranges;
+        this.ranges = ranges; // already sorted by the range.from and range.to
 
         maxTo = new double[this.ranges.length];
         maxTo[0] = this.ranges[0].to;
@@ -274,6 +279,13 @@ public RangeAggregator(
             maxTo[i] = Math.max(this.ranges[i].to, maxTo[i - 1]);
         }
 
+        fastFilterContext = new FastFilterRewriteHelper.FastFilterContext(
+            context,
+            new FastFilterRewriteHelper.RangeAggregationType(config, ranges)
+        );
+        if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
+            fastFilterContext.buildRanges(Objects.requireNonNull(config.fieldType()));
+        }
     }
 
     @Override
@@ -286,6 +298,13 @@ public ScoreMode scoreMode() {
 
     @Override
     public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
+        boolean optimized = fastFilterContext.tryFastFilterAggregation(
+            ctx,
+            this::incrementBucketDocCount,
+            (activeIndex) -> subBucketOrdinal(0, (int) activeIndex)
+        );
+        if (optimized) throw new CollectionTerminatedException();
+
         final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx);
         return new LeafBucketCollectorBase(sub, values) {
             @Override
@@ -430,4 +449,14 @@ public InternalAggregation buildEmptyAggregation() {
         }
     }
 
+    @Override
+    public void collectDebugInfo(BiConsumer<String, Object> add) {
+        super.collectDebugInfo(add);
+        if (fastFilterContext.optimizedSegments > 0) {
+            add.accept("optimized_segments", fastFilterContext.optimizedSegments);
+            add.accept("unoptimized_segments", fastFilterContext.segments - fastFilterContext.optimizedSegments);
+            add.accept("leaf_visited", fastFilterContext.leaf);
+            add.accept("inner_visited", fastFilterContext.inner);
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java
index de9b979a56107..02b0c2e612d57 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java
@@ -36,6 +36,7 @@
 import org.opensearch.search.aggregations.AggregatorFactories;
 import org.opensearch.search.aggregations.CardinalityUpperBound;
 import org.opensearch.search.aggregations.support.ValuesSource;
+import org.opensearch.search.aggregations.support.ValuesSourceConfig;
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -58,6 +59,7 @@ Aggregator build(
         SearchContext context,
         Aggregator parent,
         CardinalityUpperBound cardinality,
-        Map<String, Object> metadata
+        Map<String, Object> metadata,
+        ValuesSourceConfig config
     ) throws IOException;
 }
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
index cf95999ec5086..f6e06cce6e233 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -1614,7 +1614,7 @@ public void testMultiRangeTraversalNotApplicable() throws IOException {
             },
             true,
             collectCount -> assertTrue(collectCount > 0),
-            true
+            false
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java
index dd7ae915c3b45..7e796b684e869 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java
@@ -37,29 +37,44 @@
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.common.CheckedConsumer;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
 import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.KeywordFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
-import org.opensearch.index.mapper.NumberFieldMapper;
+import org.opensearch.index.mapper.NumberFieldMapper.NumberFieldType;
+import org.opensearch.index.mapper.NumberFieldMapper.NumberType;
+import org.opensearch.search.aggregations.AggregationBuilder;
 import org.opensearch.search.aggregations.AggregatorTestCase;
 import org.opensearch.search.aggregations.CardinalityUpperBound;
+import org.opensearch.search.aggregations.InternalAggregation;
+import org.opensearch.search.aggregations.MultiBucketConsumerService;
+import org.opensearch.search.aggregations.pipeline.PipelineAggregator;
 import org.opensearch.search.aggregations.support.AggregationInspectionHelper;
 
 import java.io.IOException;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 
 import static java.util.Collections.singleton;
+import static org.opensearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 import static org.hamcrest.Matchers.equalTo;
 
 public class RangeAggregatorTests extends AggregatorTestCase {
@@ -199,7 +214,7 @@ public void testMissingDateWithNumberField() throws IOException {
             .addRange(-2d, 5d)
             .missing("1979-01-01T00:00:00");
 
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldType(NUMBER_FIELD_NAME, NumberType.INTEGER);
 
         expectThrows(NumberFormatException.class, () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
             iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
@@ -212,7 +227,7 @@ public void testUnmappedWithMissingNumber() throws IOException {
             .addRange(-2d, 5d)
             .missing(0L);
 
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldType(NUMBER_FIELD_NAME, NumberType.INTEGER);
 
         testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
             iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
@@ -230,7 +245,7 @@ public void testUnmappedWithMissingDate() throws IOException {
             .addRange(-2d, 5d)
             .missing("2020-02-13T10:11:12");
 
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldType(NUMBER_FIELD_NAME, NumberType.INTEGER);
 
         expectThrows(NumberFormatException.class, () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
             iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
@@ -257,7 +272,7 @@ public void testBadMissingField() {
             .addRange(-2d, 5d)
             .missing("bogus");
 
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldType(NUMBER_FIELD_NAME, NumberType.INTEGER);
 
         expectThrows(NumberFormatException.class, () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
             iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
@@ -270,7 +285,7 @@ public void testUnmappedWithBadMissingField() {
             .addRange(-2d, 5d)
             .missing("bogus");
 
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldType(NUMBER_FIELD_NAME, NumberType.INTEGER);
 
         expectThrows(NumberFormatException.class, () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
             iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
@@ -305,12 +320,185 @@ public void testSubAggCollectsFromManyBucketsIfManyRanges() throws IOException {
         });
     }
 
+    public void testOverlappingRanges() throws IOException {
+        testRewriteOptimizationCase(
+            new NumberFieldType(NumberType.DOUBLE.typeName(), NumberType.DOUBLE),
+            new double[][] { { 1, 2 }, { 1, 1.5 }, { 0, 0.5 } },
+            new MatchAllDocsQuery(),
+            new Number[] { 0.1, 1.1, 2.1 },
+            range -> {
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertEquals(3, ranges.size());
+                assertEquals("0.0-0.5", ranges.get(0).getKeyAsString());
+                assertEquals(1, ranges.get(0).getDocCount());
+                assertEquals("1.0-1.5", ranges.get(1).getKeyAsString());
+                assertEquals(1, ranges.get(1).getDocCount());
+                assertEquals("1.0-2.0", ranges.get(2).getKeyAsString());
+                assertEquals(1, ranges.get(2).getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            false
+        );
+    }
+
+    /**
+     * @return Map [lower, upper) TO data points
+     */
+    private Map<double[], double[]> buildRandomRanges(double[][] possibleRanges) {
+        Map<double[], double[]> dataSet = new LinkedHashMap<>();
+        for (double[] range : possibleRanges) {
+            double lower = randomDoubleBetween(range[0], range[1], true);
+            double upper = randomDoubleBetween(range[0], range[1], true);
+            if (lower > upper) {
+                double d = lower;
+                lower = upper;
+                upper = d;
+            }
+
+            int dataNumber = randomInt(200);
+            double[] data = new double[dataNumber];
+            for (int i = 0; i < dataNumber; i++) {
+                data[i] = randomDoubleBetween(lower, upper, true);
+            }
+            dataSet.put(new double[] { lower, upper }, data);
+        }
+
+        return dataSet;
+    }
+
+    public void testRandomRanges() throws IOException {
+        Map<double[], double[]> dataSet = buildRandomRanges(new double[][] { { 0, 100 }, { 200, 1000 }, { 1000, 3000 } });
+
+        int size = dataSet.size();
+        double[][] ranges = new double[size][];
+        int[] expected = new int[size];
+        List<Number> dataPoints = new LinkedList<>();
+
+        int i = 0;
+        for (Map.Entry<double[], double[]> entry : dataSet.entrySet()) {
+            ranges[i] = entry.getKey();
+            expected[i] = entry.getValue().length;
+            for (double dataPoint : entry.getValue()) {
+                dataPoints.add(dataPoint);
+            }
+            i++;
+        }
+
+        testRewriteOptimizationCase(
+            new NumberFieldType(NumberType.DOUBLE.typeName(), NumberType.DOUBLE),
+            ranges,
+            new MatchAllDocsQuery(),
+            dataPoints.toArray(new Number[0]),
+            range -> {
+                List<? extends InternalRange.Bucket> rangeBuckets = range.getBuckets();
+                assertEquals(size, rangeBuckets.size());
+                for (int j = 0; j < rangeBuckets.size(); j++) {
+                    assertEquals(expected[j], rangeBuckets.get(j).getDocCount());
+                }
+            },
+            true
+        );
+    }
+
+    public void testDoubleType() throws IOException {
+        testRewriteOptimizationCase(
+            new NumberFieldType(NumberType.DOUBLE.typeName(), NumberType.DOUBLE),
+            new double[][] { { 1, 2 }, { 2, 3 } },
+            new MatchAllDocsQuery(),
+            new Number[] { 0.1, 1.1, 2.1 },
+            range -> {
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertEquals(2, ranges.size());
+                assertEquals("1.0-2.0", ranges.get(0).getKeyAsString());
+                assertEquals(1, ranges.get(0).getDocCount());
+                assertEquals("2.0-3.0", ranges.get(1).getKeyAsString());
+                assertEquals(1, ranges.get(1).getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            true
+        );
+    }
+
+    public void testHalfFloatType() throws IOException {
+        testRewriteOptimizationCase(
+            new NumberFieldType(NumberType.HALF_FLOAT.typeName(), NumberType.HALF_FLOAT),
+            new double[][] { { 1, 2 }, { 2, 3 } },
+            new MatchAllDocsQuery(),
+            new Number[] { 0.1, 1.1, 2.1 },
+            range -> {
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertEquals(2, ranges.size());
+                assertEquals("1.0-2.0", ranges.get(0).getKeyAsString());
+                assertEquals(1, ranges.get(0).getDocCount());
+                assertEquals("2.0-3.0", ranges.get(1).getKeyAsString());
+                assertEquals(1, ranges.get(1).getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            true
+        );
+    }
+
+    public void testFloatType() throws IOException {
+        testRewriteOptimizationCase(
+            new NumberFieldType(NumberType.FLOAT.typeName(), NumberType.FLOAT),
+            new double[][] { { 1, 2 }, { 2, 3 } },
+            new MatchAllDocsQuery(),
+            new Number[] { 0.1, 1.1, 2.1 },
+            range -> {
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertEquals(2, ranges.size());
+                assertEquals("1.0-2.0", ranges.get(0).getKeyAsString());
+                assertEquals(1, ranges.get(0).getDocCount());
+                assertEquals("2.0-3.0", ranges.get(1).getKeyAsString());
+                assertEquals(1, ranges.get(1).getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            true
+        );
+    }
+
+    public void testUnsignedLongType() throws IOException {
+        testRewriteOptimizationCase(
+            new NumberFieldType(NumberType.UNSIGNED_LONG.typeName(), NumberType.UNSIGNED_LONG),
+            new double[][] { { 1, 2 }, { 2, 3 } },
+            new MatchAllDocsQuery(),
+            new Number[] { 0, 1, 2 },
+            range -> {
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertEquals(2, ranges.size());
+                assertEquals("1.0-2.0", ranges.get(0).getKeyAsString());
+                assertEquals(1, ranges.get(0).getDocCount());
+                assertEquals("2.0-3.0", ranges.get(1).getKeyAsString());
+                assertEquals(1, ranges.get(1).getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            true
+        );
+
+        testRewriteOptimizationCase(
+            new NumberFieldType(NumberType.UNSIGNED_LONG.typeName(), NumberType.UNSIGNED_LONG),
+            new double[][] { { Double.NEGATIVE_INFINITY, 1 }, { 2, Double.POSITIVE_INFINITY } },
+            new MatchAllDocsQuery(),
+            new Number[] { 0, 1, 2 },
+            range -> {
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertEquals(2, ranges.size());
+                assertEquals("*-1.0", ranges.get(0).getKeyAsString());
+                assertEquals(1, ranges.get(0).getDocCount());
+                assertEquals("2.0-*", ranges.get(1).getKeyAsString());
+                assertEquals(1, ranges.get(1).getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            true
+        );
+    }
+
     private void testCase(
         Query query,
         CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
         Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify
     ) throws IOException {
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldType(NUMBER_FIELD_NAME, NumberType.INTEGER);
         RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg");
         aggregationBuilder.field(NUMBER_FIELD_NAME);
         aggregationBuilder.addRange(0d, 5d);
@@ -323,9 +511,9 @@ private void simpleTestCase(
         Query query,
         Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify
     ) throws IOException {
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldType(NUMBER_FIELD_NAME, NumberType.INTEGER);
 
-        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+        testCase(aggregationBuilder, query, iw -> {
             iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7)));
             iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 2)));
             iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 3)));
@@ -354,8 +542,84 @@ private void testCase(
                     fieldType
                 );
                 verify.accept(agg);
+            }
+        }
+    }
 
+    private void testRewriteOptimizationCase(
+        NumberFieldType fieldType,
+        double[][] ranges,
+        Query query,
+        Number[] dataPoints,
+        Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify,
+        boolean optimized
+    ) throws IOException {
+        NumberType numberType = fieldType.numberType();
+        String fieldName = numberType.typeName();
+
+        try (Directory directory = newDirectory()) {
+            try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec()))) {
+                for (Number dataPoint : dataPoints) {
+                    indexWriter.addDocument(numberType.createFields(fieldName, dataPoint, true, true, false));
+                }
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("range").field(fieldName);
+                for (double[] range : ranges) {
+                    aggregationBuilder.addRange(range[0], range[1]);
+                }
+
+                CountingAggregator aggregator = createCountingAggregator(query, aggregationBuilder, indexSearcher, fieldType);
+                aggregator.preCollection();
+                indexSearcher.search(query, aggregator);
+                aggregator.postCollection();
+
+                MultiBucketConsumerService.MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer(
+                    Integer.MAX_VALUE,
+                    new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)
+                );
+                InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                    aggregator.context().bigArrays(),
+                    getMockScriptService(),
+                    reduceBucketConsumer,
+                    PipelineAggregator.PipelineTree.EMPTY
+                );
+                InternalRange topLevel = (InternalRange) aggregator.buildTopLevel();
+                InternalRange agg = (InternalRange) topLevel.reduce(Collections.singletonList(topLevel), context);
+                doAssertReducedMultiBucketConsumer(agg, reduceBucketConsumer);
+
+                verify.accept(agg);
+
+                if (optimized) {
+                    assertEquals(0, aggregator.getCollectCount().get());
+                } else {
+                    assertTrue(aggregator.getCollectCount().get() > 0);
+                }
             }
         }
     }
+
+    protected CountingAggregator createCountingAggregator(
+        Query query,
+        AggregationBuilder builder,
+        IndexSearcher searcher,
+        MappedFieldType... fieldTypes
+    ) throws IOException {
+        return new CountingAggregator(
+            new AtomicInteger(),
+            createAggregator(
+                query,
+                builder,
+                searcher,
+                new MultiBucketConsumerService.MultiBucketConsumer(
+                    DEFAULT_MAX_BUCKETS,
+                    new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)
+                ),
+                fieldTypes
+            )
+        );
+    }
 }

From f80f3cdc18b7dbdcbcc372527a6bcbc71e405739 Mon Sep 17 00:00:00 2001
From: Suraj Singh <surajrider@gmail.com>
Date: Wed, 19 Jun 2024 18:38:03 -0500
Subject: [PATCH 025/113] Update @dreamer-89 maintainership status (#14461)

Signed-off-by: Suraj Singh <singhnjb@amazon.com>
---
 .github/CODEOWNERS | 4 ++--
 MAINTAINERS.md     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index b5b0a815b02b2..7b685fcc932ea 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -11,7 +11,7 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
 
 /modules/transport-netty4/ @peternied
 
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index 6855281a488ca..b014da1157678 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -26,7 +26,6 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Sarat Vemulapalli        | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |
 | Shweta Thareja           | [shwetathareja](https://github.com/shwetathareja)       | Amazon      |
 | Sorabh Hamirwasia        | [sohami](https://github.com/sohami)                     | Amazon      |
-| Suraj Singh              | [dreamer-89](https://github.com/dreamer-89)             | Amazon      |
 | Tianli Feng              | [tlfeng](https://github.com/tlfeng)                     | Amazon      |
 | Vacha Shah               | [VachaShah](https://github.com/VachaShah)               | Amazon      |
 
@@ -41,3 +40,4 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Himanshu Setia        | [setiah](https://github.com/setiah)       | Amazon      |
 | Ryan Bogan            | [ryanbogan](https://github.com/ryanbogan) | Amazon      |
 | Rabi Panda            | [adnapibar](https://github.com/adnapibar) | Independent |
+| Suraj Singh              | [dreamer-89](https://github.com/dreamer-89)             | Amazon      |

From 9729a92c9b60ba95165f12baf3735813b4ea855e Mon Sep 17 00:00:00 2001
From: Andrew Ross <andrross@amazon.com>
Date: Wed, 19 Jun 2024 18:37:46 -0700
Subject: [PATCH 026/113] Move maintainer tlfeng to emeritus (#14466)

---
 .github/CODEOWNERS |  4 ++--
 MAINTAINERS.md     | 22 +++++++++++-----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 7b685fcc932ea..5a2d08756c49f 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -11,7 +11,7 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
 /modules/transport-netty4/ @peternied
 
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index b014da1157678..91b57a4cbc74e 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -26,18 +26,18 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Sarat Vemulapalli        | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |
 | Shweta Thareja           | [shwetathareja](https://github.com/shwetathareja)       | Amazon      |
 | Sorabh Hamirwasia        | [sohami](https://github.com/sohami)                     | Amazon      |
-| Tianli Feng              | [tlfeng](https://github.com/tlfeng)                     | Amazon      |
 | Vacha Shah               | [VachaShah](https://github.com/VachaShah)               | Amazon      |
 
 ## Emeritus
 
-| Maintainer            | GitHub ID                                 | Affiliation |
-| --------------------- | ----------------------------------------- | ----------- |
-| Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon      |
-| Xue Zhou              | [xuezhou25](https://github.com/xuezhou25) | Amazon      |
-| Kartik Ganesh         | [kartg](https://github.com/kartg)         | Amazon      |
-| Abbas Hussain         | [abbashus](https://github.com/abbashus)   | Meta        |
-| Himanshu Setia        | [setiah](https://github.com/setiah)       | Amazon      |
-| Ryan Bogan            | [ryanbogan](https://github.com/ryanbogan) | Amazon      |
-| Rabi Panda            | [adnapibar](https://github.com/adnapibar) | Independent |
-| Suraj Singh              | [dreamer-89](https://github.com/dreamer-89)             | Amazon      |
+| Maintainer             | GitHub ID                                   | Affiliation |
+| ---------------------- |-------------------------------------------- | ----------- |
+| Megha Sai Kavikondala  | [meghasaik](https://github.com/meghasaik)   | Amazon      |
+| Xue Zhou               | [xuezhou25](https://github.com/xuezhou25)   | Amazon      |
+| Kartik Ganesh          | [kartg](https://github.com/kartg)           | Amazon      |
+| Abbas Hussain          | [abbashus](https://github.com/abbashus)     | Meta        |
+| Himanshu Setia         | [setiah](https://github.com/setiah)         | Amazon      |
+| Ryan Bogan             | [ryanbogan](https://github.com/ryanbogan)   | Amazon      |
+| Rabi Panda             | [adnapibar](https://github.com/adnapibar)   | Independent |
+| Tianli Feng            | [tlfeng](https://github.com/tlfeng)         | Amazon      |
+| Suraj Singh            | [dreamer-89](https://github.com/dreamer-89) | Amazon      |

From e59d77a8a3334981f9da1bdd14aed70b3a936e9f Mon Sep 17 00:00:00 2001
From: Andrew Ross <andrross@amazon.com>
Date: Thu, 20 Jun 2024 05:56:22 -0700
Subject: [PATCH 027/113] Add more details to testing bad practices (#14455)

These are a few cases I have seen that have resulted in flaky tests. I
would love to see more details added here so that this can be used as a
sort of checklist when writing, reviewing, or trying to fix tests.

Signed-off-by: Andrew Ross <andrross@amazon.com>
---
 TESTING.md | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/TESTING.md b/TESTING.md
index 80fc2412d736b..3928b2fc93afa 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -33,6 +33,9 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran
   - [Bad practices](#bad-practices)
     - [Use randomized-testing for coverage](#use-randomized-testing-for-coverage)
     - [Abuse randomization in multi-threaded tests](#abuse-randomization-in-multi-threaded-tests)
+    - [Use `Thread.sleep`](#use-threadsleep)
+    - [Expect a specific segment topology](#expect-a-specific-segment-topology)
+    - [Leave environment in an unstable state after test](#leave-environment-in-an-unstable-state-after-test)
 - [Test coverage analysis](#test-coverage-analysis)
 - [Building with extra plugins](#building-with-extra-plugins)
 - [Environment misc](#environment-misc)
@@ -455,7 +458,7 @@ Unit tests are the preferred way to test some functionality: most of the time th
 
 The reason why `OpenSearchSingleNodeTestCase` exists is that all our components used to be very hard to set up in isolation, which had led us to having a number of integration tests but close to no unit tests. `OpenSearchSingleNodeTestCase` is a workaround for this issue which provides an easy way to spin up a node and get access to components that are hard to instantiate like `IndicesService`. Whenever practical, you should prefer unit tests.
 
-Finally, if the the functionality under test needs to be run in a cluster, there are two test classes to consider:
+Finally, if the functionality under test needs to be run in a cluster, there are two test classes to consider:
   * `OpenSearchRestTestCase` will connect to an external cluster. This is a good option if the tests cases don't rely on a specific configuration of the test cluster. A test cluster is set up as part of the Gradle task running integration tests, and test cases using this class can connect to it. The configuration of the cluster is provided in the Gradle files.
   * `OpenSearchIntegTestCase` will create a local cluster as part of each test case. The configuration of the cluster is controlled by the test class. This is a good option if different tests cases depend on different cluster configurations, as it would be impractical (and limit parallelization) to keep re-configuring (and re-starting) the external cluster for each test case. A good example of when this class might come in handy is for testing security features, where different cluster configurations are needed to fully test each one.
 
@@ -477,6 +480,27 @@ However, it should not be used for coverage. For instance if you are testing a p
 
 Multi-threaded tests are often not reproducible due to the fact that there is no guarantee on the order in which operations occur across threads. Adding randomization to the mix usually makes things worse and should be done with care.
 
+### Use `Thread.sleep`
+
+`Thread.sleep()` is almost always a bad idea because it is very difficult to know that you've waited long enough. Using primitives like `waitUntil` or `assertBusy`, which use Thread.sleep internally, is okay to wait for a specific condition. However, it is almost always better to instrument your code with concurrency primitives like a `CountDownLatch` that will allow you to deterministically wait for a specific condition, without waiting longer than necessary that will happen with a polling approach used by `assertBusy`.
+
+Example:
+- [PrimaryShardAllocatorIT](https://github.com/opensearch-project/OpenSearch/blob/7ffcd6500e0bd5956cef5c289ee66d9f99d533fc/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java#L208-L235): This test is using two latches: one to wait for a recovery to start and one to block that recovery so that it can deterministically test things that happen during a recovery.
+
+### Expect a specific segment topology
+
+By design, OpenSearch integration tests will vary how the merge policy works because in almost all scenarios you should not depend on a specific segment topology (in the real world your code will see a huge diversity of indexing workloads with OpenSearch merging things in the background all the time!). If you do in fact need to care about the segment topology (e.g. for testing statistics that might vary slightly depending on number of segments), then you must take care to ensure that segment topology is deterministic by doing things like disabling background refreshes, force merging after indexing data, etc.
+
+Example:
+- [SegmentReplicationResizeRequestIT](https://github.com/opensearch-project/OpenSearch/blob/f715ee1a485e550802accc1c2e3d8101208d4f0b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationResizeRequestIT.java#L102-L109): This test disables refreshes to prevent interfering with the segment replication behavior under test.
+
+### Leave environment in an unstable state after test
+
+The default test case will ensure that no open file handles or running threads are left after tear down. You must ensure that all resources are cleaned up at the end of each test case, or else the cleanup may end up racing with the tear down logic in the base test class in a way that is very difficult to reproduce.
+
+Example:
+- [AwarenessAttributeDecommissionIT](https://github.com/opensearch-project/OpenSearch/blob/main/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/AwarenessAttributeDecommissionIT.java#L951): Recommissions any decommissioned nodes at the end of the test to ensure the after-test checks succeed.
+
 # Test coverage analysis
 
 The code coverage report can be generated through Gradle with [JaCoCo plugin](https://docs.gradle.org/current/userguide/jacoco_plugin.html).

From 761e207f9f50a482461a049e8e9da287eeddf83f Mon Sep 17 00:00:00 2001
From: kkewwei <kkewwei@163.com>
Date: Thu, 20 Jun 2024 22:13:23 +0800
Subject: [PATCH 028/113] Fix the computed max shards of cluster to avoid int
 overflow (#14155)

Signed-off-by: kkewwei <kkewwei@163.com>
---
 CHANGELOG.md                                  |  1 +
 .../indices/ShardLimitValidator.java          |  7 ++++---
 .../indices/ShardLimitValidatorTests.java     | 19 +++++++++++++++++++
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c3e63e36bc82..cfeb02d5a7b06 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
+- Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java b/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java
index e345b613eebbd..94e91e2d4c3ac 100644
--- a/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java
+++ b/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java
@@ -261,14 +261,15 @@ static Optional<String> checkShardLimit(
             return Optional.empty();
         }
 
+        int computedMaxShards = (int) Math.min(Integer.MAX_VALUE, (long) maxShardsPerNodeSetting * nodeCount);
         int maxShardsInCluster = maxShardsPerClusterSetting;
         if (maxShardsInCluster == -1) {
-            maxShardsInCluster = maxShardsPerNodeSetting * nodeCount;
+            maxShardsInCluster = computedMaxShards;
         } else {
-            maxShardsInCluster = Math.min(maxShardsInCluster, maxShardsPerNodeSetting * nodeCount);
+            maxShardsInCluster = Math.min(maxShardsInCluster, computedMaxShards);
         }
 
-        int currentOpenShards = state.getMetadata().getTotalOpenIndexShards();
+        long currentOpenShards = state.getMetadata().getTotalOpenIndexShards();
         if ((currentOpenShards + newShards) > maxShardsInCluster) {
             String errorMessage = "this action would add ["
                 + newShards
diff --git a/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java b/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java
index 040632ea3ed8d..0b1ec8fd85ae5 100644
--- a/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java
+++ b/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java
@@ -214,6 +214,25 @@ public void testNonSystemIndexCreationFailsWithMaxShardLimitOnCluster() {
         );
     }
 
+    public void testComputedMaxShardsOfClusterIntOverFlow() {
+        final int maxShardLimitPerNode = 500_000_000;
+        ClusterState state = createClusterForShardLimitTest(15, 1, 1);
+        Optional<String> errorMessage = ShardLimitValidator.checkShardLimit(2, state, maxShardLimitPerNode, -1);
+        assertFalse(errorMessage.isPresent());
+
+        errorMessage = ShardLimitValidator.checkShardLimit(Integer.MAX_VALUE - 1, state, maxShardLimitPerNode, -1);
+        assertEquals(
+            "this action would add ["
+                + (Integer.MAX_VALUE - 1)
+                + "] total shards, but this cluster currently has ["
+                + 2
+                + "]/["
+                + Integer.MAX_VALUE
+                + "] maximum shards open",
+            errorMessage.get()
+        );
+    }
+
     public void testNonSystemIndexCreationPassesWithMaxShardLimitOnCluster() {
         final int maxShardLimitOnCluster = 5;
         Settings limitOnlySettings = Settings.builder()

From d2c08b320b49920e142ec70fa66621305fd4ccf8 Mon Sep 17 00:00:00 2001
From: Skyring100 <106502383+Skyring100@users.noreply.github.com>
Date: Thu, 20 Jun 2024 07:29:16 -0700
Subject: [PATCH 029/113] Optimize UnsignedLong range queries to convert to
 MatchNoDocsQuery when lower > upper bounds (#14416)

* Added check for lower > upper at end of function

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* Fixed mistake of using < operator on BigInteger, now using compareTo

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* Fixed simple mistake of flipping > operator

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* Fixed space formatting

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* Updated CHANGELOG.md

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* Issue number linked in CHANGELOG.md

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* doTestDocValueRangeQueries now accepts MatchNoDocsQuery alongside IndexOrDocValuesQuery

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* dotestdoTestDocValueRangeQueries only checks indexQuery and randomAccessQuery only when query is type IndexIndexOrDocValuesQuery

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* Ran gradlew spotlessApply to fix import formatting issues

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

* Imported Matchers.either method instead of entire Matchers class

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>

---------

Signed-off-by: Skyring100 <106502383+Skyring100@users.noreply.github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                             | 1 +
 .../org/opensearch/index/mapper/NumberFieldMapper.java   | 3 +++
 .../opensearch/index/mapper/NumberFieldTypeTests.java    | 9 ++++++---
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cfeb02d5a7b06..6d14f55d56209 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `opentelemetry` from 1.36.0 to 1.39.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457))
 
 ### Changed
+- unsignedLongRangeQuery now returns MatchNoDocsQuery if the lower bounds are greater than the upper bounds ([#14416](https://github.com/opensearch-project/OpenSearch/pull/14416))
 - Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568))
 - Make the class CommunityIdProcessor final ([#14448](https://github.com/opensearch-project/OpenSearch/pull/14448))
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
index 25e5f9970795f..27e62c3746a8e 100644
--- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
@@ -1410,6 +1410,9 @@ public static Query unsignedLongRangeQuery(
                     u = u.subtract(BigInteger.ONE);
                 }
             }
+            if (l.compareTo(u) > 0) {
+                return new MatchNoDocsQuery();
+            }
             return builder.apply(l, u);
         }
     }
diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java
index af852b12e7a30..96487db6dd512 100644
--- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java
@@ -86,6 +86,7 @@
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -673,9 +674,11 @@ public void doTestDocValueRangeQueries(NumberType type, Supplier<Number> valueSu
                 true,
                 MOCK_QSC
             );
-            assertThat(query, instanceOf(IndexOrDocValuesQuery.class));
-            IndexOrDocValuesQuery indexOrDvQuery = (IndexOrDocValuesQuery) query;
-            assertEquals(searcher.count(indexOrDvQuery.getIndexQuery()), searcher.count(indexOrDvQuery.getRandomAccessQuery()));
+            assertThat(query, either(instanceOf(IndexOrDocValuesQuery.class)).or(instanceOf(MatchNoDocsQuery.class)));
+            if (query instanceof IndexOrDocValuesQuery) {
+                IndexOrDocValuesQuery indexOrDvQuery = (IndexOrDocValuesQuery) query;
+                assertEquals(searcher.count(indexOrDvQuery.getIndexQuery()), searcher.count(indexOrDvQuery.getRandomAccessQuery()));
+            }
         }
         reader.close();
         dir.close();

From b8c78196438897132f6819460ebb7d4222b39297 Mon Sep 17 00:00:00 2001
From: rayshrey <121871912+rayshrey@users.noreply.github.com>
Date: Thu, 20 Jun 2024 22:53:18 +0530
Subject: [PATCH 030/113] [Writable Warm] Composite Directory implementation
 and integrating it with FileCache (#12782)

* Composite Directory POC

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Refactor TransferManager interface to RemoteStoreFileTrackerAdapter

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Implement block level fetch for Composite Directory

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Removed CACHE state from FileTracker

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Fixes after latest pull

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Add new setting for warm, remove store type setting, FileTracker and RemoteStoreFileTrackerAdapter, CompositeDirectoryFactory and update Composite Directory implementation

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Modify TransferManager - replace BlobContainer with Functional Interface to fetch an InputStream instead

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Reuse OnDemandBlockSnapshotIndexInput instead of OnDemandBlockCompositeIndexInput

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Modify constructors to avoid breaking public api contract and code review fixes

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Add experimental annotations for newly created classes and review comment fixes

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Use ref count as a temporary measure to prevent file from eviction until uploaded to Remote

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Remove method level locks

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Handle tmp file deletion

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Nit fixes

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Handle delete and close in Composite Directory, log current state of FileCache and correct it's clear method and modify unit and integration tests as per review comments

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Refactor usages of WRITEABLE_REMOTE_INDEX_SETTING to TIERED_REMOTE_INDEX_SETTING

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Add tests for FileCachedIndexInput and review comment fixes

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Add additional IT for feature flag disabled

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Move setting for Partial Locality type behind Feature Flag, fix bug for ref count via cloneMap in FullFileCachedIndexInput and other review fixes

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Minor test and nit fixes

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Add javadocs for FullFileCachedIndexInput

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

* Minor precommit fixes

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>

---------

Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../remotestore/WritableWarmIT.java           | 160 ++++++++
 .../common/settings/IndexScopedSettings.java  |   6 +-
 .../org/opensearch/index/IndexModule.java     |  78 +++-
 .../org/opensearch/index/IndexService.java    |  96 ++++-
 .../org/opensearch/index/IndexSettings.java   |  12 +
 .../shard/RemoteStoreRefreshListener.java     |   5 +
 .../store/CloseableFilterIndexOutput.java     |  53 +++
 .../index/store/CompositeDirectory.java       | 344 ++++++++++++++++++
 .../index/store/RemoteDirectory.java          |  26 +-
 .../RemoteSnapshotDirectoryFactory.java       |   2 +-
 .../file/OnDemandBlockSnapshotIndexInput.java |  14 +-
 .../filecache/CachedFullFileIndexInput.java   |  74 ++++
 .../store/remote/filecache/FileCache.java     |  11 +
 .../filecache/FileCachedIndexInput.java       |   4 +-
 .../filecache/FullFileCachedIndexInput.java   | 108 ++++++
 .../store/remote/utils/BlockIOContext.java    |  53 +++
 .../store/remote/utils/FileTypeUtils.java     |  40 ++
 .../store/remote/utils/TransferManager.java   |  33 +-
 .../store/remote/utils/cache/LRUCache.java    |  26 +-
 .../remote/utils/cache/SegmentedCache.java    |  13 +
 .../opensearch/indices/IndicesService.java    |  73 +++-
 .../main/java/org/opensearch/node/Node.java   |  11 +-
 .../BaseRemoteSegmentStoreDirectoryTests.java | 178 +++++++++
 .../index/store/CompositeDirectoryTests.java  | 202 ++++++++++
 .../index/store/RemoteDirectoryTests.java     |  13 +
 .../RemoteSegmentStoreDirectoryTests.java     | 153 +-------
 .../OnDemandBlockSnapshotIndexInputTests.java |   4 +-
 .../filecache/FileCachedIndexInputTests.java  |  79 ++++
 .../FullFileCachedIndexInputTests.java        |  79 ++++
 ...ansferManagerBlobContainerReaderTests.java |  43 +++
 ...sferManagerRemoteDirectoryReaderTests.java |  51 +++
 ...ests.java => TransferManagerTestCase.java} |  37 +-
 .../utils/cache/RefCountedCacheTestCase.java  |   9 +
 .../test/OpenSearchIntegTestCase.java         |   6 +-
 35 files changed, 1889 insertions(+), 208 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java
 create mode 100644 server/src/main/java/org/opensearch/index/store/CloseableFilterIndexOutput.java
 create mode 100644 server/src/main/java/org/opensearch/index/store/CompositeDirectory.java
 create mode 100644 server/src/main/java/org/opensearch/index/store/remote/filecache/CachedFullFileIndexInput.java
 create mode 100644 server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java
 create mode 100644 server/src/main/java/org/opensearch/index/store/remote/utils/BlockIOContext.java
 create mode 100644 server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java
 create mode 100644 server/src/test/java/org/opensearch/index/store/BaseRemoteSegmentStoreDirectoryTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java
 rename server/src/test/java/org/opensearch/index/store/remote/utils/{TransferManagerTests.java => TransferManagerTestCase.java} (87%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6d14f55d56209..47244ba9afbb1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724))
 - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
+- [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java
new file mode 100644
index 0000000000000..a51bd6b20fff0
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java
@@ -0,0 +1,160 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotestore;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.opensearch.action.admin.indices.get.GetIndexRequest;
+import org.opensearch.action.admin.indices.get.GetIndexResponse;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsException;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.index.IndexModule;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.index.store.CompositeDirectory;
+import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
+import org.opensearch.index.store.remote.filecache.FileCache;
+import org.opensearch.index.store.remote.utils.FileTypeUtils;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.node.Node;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
+
+@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
+// Uncomment the below line to enable trace level logs for this test for better debugging
+// @TestLogging(reason = "Getting trace logs from composite directory package", value = "org.opensearch.index.store:TRACE")
+public class WritableWarmIT extends RemoteStoreBaseIntegTestCase {
+
+    protected static final String INDEX_NAME = "test-idx-1";
+    protected static final int NUM_DOCS_IN_BULK = 1000;
+
+    /*
+    Disabling MockFSIndexStore plugin as the MockFSDirectoryFactory wraps the FSDirectory over a OpenSearchMockDirectoryWrapper which extends FilterDirectory (whereas FSDirectory extends BaseDirectory)
+    As a result of this wrapping the local directory of Composite Directory does not satisfy the assertion that local directory must be of type FSDirectory
+     */
+    @Override
+    protected boolean addMockIndexStorePlugin() {
+        return false;
+    }
+
+    @Override
+    protected Settings featureFlagSettings() {
+        Settings.Builder featureSettings = Settings.builder();
+        featureSettings.put(FeatureFlags.TIERED_REMOTE_INDEX, true);
+        return featureSettings.build();
+    }
+
+    public void testWritableWarmFeatureFlagDisabled() {
+        Settings clusterSettings = Settings.builder().put(super.nodeSettings(0)).put(FeatureFlags.TIERED_REMOTE_INDEX, false).build();
+        InternalTestCluster internalTestCluster = internalCluster();
+        internalTestCluster.startClusterManagerOnlyNode(clusterSettings);
+        internalTestCluster.startDataOnlyNode(clusterSettings);
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name())
+            .build();
+
+        try {
+            prepareCreate(INDEX_NAME).setSettings(indexSettings).get();
+            fail("Should have thrown Exception as setting should not be registered if Feature Flag is Disabled");
+        } catch (SettingsException ex) {
+            assertEquals(
+                "unknown setting ["
+                    + IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey()
+                    + "] please check that any required plugins are installed, or check the "
+                    + "breaking changes documentation for removed settings",
+                ex.getMessage()
+            );
+        }
+    }
+
+    public void testWritableWarmBasic() throws Exception {
+        InternalTestCluster internalTestCluster = internalCluster();
+        internalTestCluster.startClusterManagerOnlyNode();
+        internalTestCluster.startDataOnlyNode();
+        Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name())
+            .build();
+        assertAcked(client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).get());
+
+        // Verify from the cluster settings if the data locality is partial
+        GetIndexResponse getIndexResponse = client().admin()
+            .indices()
+            .getIndex(new GetIndexRequest().indices(INDEX_NAME).includeDefaults(true))
+            .get();
+        Settings indexSettings = getIndexResponse.settings().get(INDEX_NAME);
+        assertEquals(IndexModule.DataLocalityType.PARTIAL.name(), indexSettings.get(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey()));
+
+        // Ingesting some docs
+        indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK);
+        flushAndRefresh(INDEX_NAME);
+
+        // ensuring cluster is green after performing force-merge
+        ensureGreen();
+
+        SearchResponse searchResponse = client().prepareSearch(INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get();
+        // Asserting that search returns same number of docs as ingested
+        assertHitCount(searchResponse, NUM_DOCS_IN_BULK);
+
+        // Ingesting docs again before force merge
+        indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK);
+        flushAndRefresh(INDEX_NAME);
+
+        FileCache fileCache = internalTestCluster.getDataNodeInstance(Node.class).fileCache();
+        IndexShard shard = internalTestCluster.getDataNodeInstance(IndicesService.class)
+            .indexService(resolveIndex(INDEX_NAME))
+            .getShardOrNull(0);
+        Directory directory = (((FilterDirectory) (((FilterDirectory) (shard.store().directory())).getDelegate())).getDelegate());
+
+        // Force merging the index
+        Set<String> filesBeforeMerge = new HashSet<>(Arrays.asList(directory.listAll()));
+        client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).get();
+        flushAndRefresh(INDEX_NAME);
+        Set<String> filesAfterMerge = new HashSet<>(Arrays.asList(directory.listAll()));
+
+        Set<String> filesFromPreviousGenStillPresent = filesBeforeMerge.stream()
+            .filter(filesAfterMerge::contains)
+            .filter(file -> !FileTypeUtils.isLockFile(file))
+            .filter(file -> !FileTypeUtils.isSegmentsFile(file))
+            .collect(Collectors.toUnmodifiableSet());
+
+        // Asserting that after merge all the files from previous gen are no more part of the directory
+        assertTrue(filesFromPreviousGenStillPresent.isEmpty());
+
+        // Asserting that files from previous gen are not present in File Cache as well
+        filesBeforeMerge.stream()
+            .filter(file -> !FileTypeUtils.isLockFile(file))
+            .filter(file -> !FileTypeUtils.isSegmentsFile(file))
+            .forEach(file -> assertNull(fileCache.get(((CompositeDirectory) directory).getFilePath(file))));
+
+        // Deleting the index (so that ref count drops to zero for all the files) and then pruning the cache to clear it to avoid any file
+        // leaks
+        assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get());
+        fileCache.prune();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index 6fe8dec9c21b1..1488f5d30b4ba 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -41,6 +41,7 @@
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.settings.Setting.Property;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.IndexSortConfig;
@@ -260,7 +261,10 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
      * is ready for production release, the feature flag can be removed, and the
      * setting should be moved to {@link #BUILT_IN_INDEX_SETTINGS}.
      */
-    public static final Map<String, List<Setting>> FEATURE_FLAGGED_INDEX_SETTINGS = Map.of();
+    public static final Map<String, List<Setting>> FEATURE_FLAGGED_INDEX_SETTINGS = Map.of(
+        FeatureFlags.TIERED_REMOTE_INDEX,
+        List.of(IndexModule.INDEX_STORE_LOCALITY_SETTING)
+    );
 
     public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);
 
diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
index 3c4cb4fd596c1..4c494a6b35153 100644
--- a/server/src/main/java/org/opensearch/index/IndexModule.java
+++ b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -107,6 +107,8 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import static org.apache.logging.log4j.util.Strings.toRootUpperCase;
+
 /**
  * IndexModule represents the central extension point for index level custom implementations like:
  * <ul>
@@ -141,6 +143,17 @@ public final class IndexModule {
         Property.NodeScope
     );
 
+    /**
+     * Index setting which used to determine how the data is cached locally fully or partially
+     */
+    public static final Setting<DataLocalityType> INDEX_STORE_LOCALITY_SETTING = new Setting<>(
+        "index.store.data_locality",
+        DataLocalityType.FULL.name(),
+        DataLocalityType::getValueOf,
+        Property.IndexScope,
+        Property.NodeScope
+    );
+
     public static final Setting<String> INDEX_RECOVERY_TYPE_SETTING = new Setting<>(
         "index.recovery.type",
         "",
@@ -297,6 +310,7 @@ public Iterator<Setting<?>> settings() {
     private final AtomicBoolean frozen = new AtomicBoolean(false);
     private final BooleanSupplier allowExpensiveQueries;
     private final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories;
+    private final FileCache fileCache;
 
     /**
      * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins
@@ -315,7 +329,8 @@ public IndexModule(
         final Map<String, IndexStorePlugin.DirectoryFactory> directoryFactories,
         final BooleanSupplier allowExpensiveQueries,
         final IndexNameExpressionResolver expressionResolver,
-        final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories
+        final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories,
+        final FileCache fileCache
     ) {
         this.indexSettings = indexSettings;
         this.analysisRegistry = analysisRegistry;
@@ -327,6 +342,30 @@ public IndexModule(
         this.allowExpensiveQueries = allowExpensiveQueries;
         this.expressionResolver = expressionResolver;
         this.recoveryStateFactories = recoveryStateFactories;
+        this.fileCache = fileCache;
+    }
+
+    public IndexModule(
+        final IndexSettings indexSettings,
+        final AnalysisRegistry analysisRegistry,
+        final EngineFactory engineFactory,
+        final EngineConfigFactory engineConfigFactory,
+        final Map<String, IndexStorePlugin.DirectoryFactory> directoryFactories,
+        final BooleanSupplier allowExpensiveQueries,
+        final IndexNameExpressionResolver expressionResolver,
+        final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories
+    ) {
+        this(
+            indexSettings,
+            analysisRegistry,
+            engineFactory,
+            engineConfigFactory,
+            directoryFactories,
+            allowExpensiveQueries,
+            expressionResolver,
+            recoveryStateFactories,
+            null
+        );
     }
 
     /**
@@ -577,6 +616,40 @@ public boolean match(Settings settings) {
         }
     }
 
+    /**
+     * Indicates the locality of the data - whether it will be cached fully or partially
+     */
+    public enum DataLocalityType {
+        /**
+         * Indicates that all the data will be cached locally
+         */
+        FULL,
+        /**
+         * Indicates that only a subset of the data will be cached locally
+         */
+        PARTIAL;
+
+        private static final Map<String, DataLocalityType> LOCALITY_TYPES;
+
+        static {
+            final Map<String, DataLocalityType> localityTypes = new HashMap<>(values().length);
+            for (final DataLocalityType dataLocalityType : values()) {
+                localityTypes.put(dataLocalityType.name(), dataLocalityType);
+            }
+            LOCALITY_TYPES = Collections.unmodifiableMap(localityTypes);
+        }
+
+        public static DataLocalityType getValueOf(final String localityType) {
+            Objects.requireNonNull(localityType, "No locality type given.");
+            final String localityTypeName = toRootUpperCase(localityType.trim());
+            final DataLocalityType type = LOCALITY_TYPES.get(localityTypeName);
+            if (type != null) {
+                return type;
+            }
+            throw new IllegalArgumentException("Unknown locality type constant [" + localityType + "].");
+        }
+    }
+
     public static Type defaultStoreType(final boolean allowMmap) {
         if (allowMmap && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
             return Type.HYBRIDFS;
@@ -665,7 +738,8 @@ public IndexService newIndexService(
                 translogFactorySupplier,
                 clusterDefaultRefreshIntervalSupplier,
                 recoverySettings,
-                remoteStoreSettings
+                remoteStoreSettings,
+                fileCache
             );
             success = true;
             return indexService;
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
index e501d7eff3f81..a7849bcf80474 100644
--- a/server/src/main/java/org/opensearch/index/IndexService.java
+++ b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -55,6 +55,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.util.concurrent.AbstractAsyncTask;
 import org.opensearch.common.util.concurrent.AbstractRunnable;
 import org.opensearch.common.util.io.IOUtils;
@@ -91,8 +92,10 @@
 import org.opensearch.index.shard.ShardNotInPrimaryModeException;
 import org.opensearch.index.shard.ShardPath;
 import org.opensearch.index.similarity.SimilarityService;
+import org.opensearch.index.store.CompositeDirectory;
 import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory;
 import org.opensearch.index.store.Store;
+import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.TranslogFactory;
 import org.opensearch.indices.RemoteStoreSettings;
@@ -188,6 +191,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier;
     private final RecoverySettings recoverySettings;
     private final RemoteStoreSettings remoteStoreSettings;
+    private final FileCache fileCache;
 
     public IndexService(
         IndexSettings indexSettings,
@@ -223,7 +227,8 @@ public IndexService(
         BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
         RecoverySettings recoverySettings,
-        RemoteStoreSettings remoteStoreSettings
+        RemoteStoreSettings remoteStoreSettings,
+        FileCache fileCache
     ) {
         super(indexSettings);
         this.allowExpensiveQueries = allowExpensiveQueries;
@@ -301,9 +306,85 @@ public IndexService(
         this.translogFactorySupplier = translogFactorySupplier;
         this.recoverySettings = recoverySettings;
         this.remoteStoreSettings = remoteStoreSettings;
+        this.fileCache = fileCache;
         updateFsyncTaskIfNecessary();
     }
 
+    public IndexService(
+        IndexSettings indexSettings,
+        IndexCreationContext indexCreationContext,
+        NodeEnvironment nodeEnv,
+        NamedXContentRegistry xContentRegistry,
+        SimilarityService similarityService,
+        ShardStoreDeleter shardStoreDeleter,
+        IndexAnalyzers indexAnalyzers,
+        EngineFactory engineFactory,
+        EngineConfigFactory engineConfigFactory,
+        CircuitBreakerService circuitBreakerService,
+        BigArrays bigArrays,
+        ThreadPool threadPool,
+        ScriptService scriptService,
+        ClusterService clusterService,
+        Client client,
+        QueryCache queryCache,
+        IndexStorePlugin.DirectoryFactory directoryFactory,
+        IndexStorePlugin.DirectoryFactory remoteDirectoryFactory,
+        IndexEventListener eventListener,
+        Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> wrapperFactory,
+        MapperRegistry mapperRegistry,
+        IndicesFieldDataCache indicesFieldDataCache,
+        List<SearchOperationListener> searchOperationListeners,
+        List<IndexingOperationListener> indexingOperationListeners,
+        NamedWriteableRegistry namedWriteableRegistry,
+        BooleanSupplier idFieldDataEnabled,
+        BooleanSupplier allowExpensiveQueries,
+        IndexNameExpressionResolver expressionResolver,
+        ValuesSourceRegistry valuesSourceRegistry,
+        IndexStorePlugin.RecoveryStateFactory recoveryStateFactory,
+        BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
+        Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
+        RecoverySettings recoverySettings,
+        RemoteStoreSettings remoteStoreSettings
+    ) {
+        this(
+            indexSettings,
+            indexCreationContext,
+            nodeEnv,
+            xContentRegistry,
+            similarityService,
+            shardStoreDeleter,
+            indexAnalyzers,
+            engineFactory,
+            engineConfigFactory,
+            circuitBreakerService,
+            bigArrays,
+            threadPool,
+            scriptService,
+            clusterService,
+            client,
+            queryCache,
+            directoryFactory,
+            remoteDirectoryFactory,
+            eventListener,
+            wrapperFactory,
+            mapperRegistry,
+            indicesFieldDataCache,
+            searchOperationListeners,
+            indexingOperationListeners,
+            namedWriteableRegistry,
+            idFieldDataEnabled,
+            allowExpensiveQueries,
+            expressionResolver,
+            valuesSourceRegistry,
+            recoveryStateFactory,
+            translogFactorySupplier,
+            clusterDefaultRefreshIntervalSupplier,
+            recoverySettings,
+            remoteStoreSettings,
+            null
+        );
+    }
+
     static boolean needsMapperService(IndexSettings indexSettings, IndexCreationContext indexCreationContext) {
         return false == (indexSettings.getIndexMetadata().getState() == IndexMetadata.State.CLOSE
             && indexCreationContext == IndexCreationContext.CREATE_INDEX); // metadata verification needs a mapper service
@@ -495,9 +576,9 @@ public synchronized IndexShard createShard(
                 }
             };
             Store remoteStore = null;
+            Directory remoteDirectory = null;
             boolean seedRemote = false;
             if (targetNode.isRemoteStoreNode()) {
-                final Directory remoteDirectory;
                 if (this.indexSettings.isRemoteStoreEnabled()) {
                     remoteDirectory = remoteDirectoryFactory.newDirectory(this.indexSettings, path);
                 } else {
@@ -530,7 +611,16 @@ public synchronized IndexShard createShard(
                 }
             }
 
-            Directory directory = directoryFactory.newDirectory(this.indexSettings, path);
+            Directory directory = null;
+            if (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX_SETTING) &&
+            // TODO : Need to remove this check after support for hot indices is added in Composite Directory
+                this.indexSettings.isStoreLocalityPartial()) {
+                Directory localDirectory = directoryFactory.newDirectory(this.indexSettings, path);
+                directory = new CompositeDirectory(localDirectory, remoteDirectory, fileCache);
+            } else {
+                directory = directoryFactory.newDirectory(this.indexSettings, path);
+            }
+
             store = new Store(
                 shardId,
                 this.indexSettings,
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 613e93698d683..96458ecc49ddc 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -733,6 +733,7 @@ public static IndexMergePolicy fromString(String text) {
     private final int numberOfShards;
     private final ReplicationType replicationType;
     private final boolean isRemoteStoreEnabled;
+    private final boolean isStoreLocalityPartial;
     private volatile TimeValue remoteTranslogUploadBufferInterval;
     private final String remoteStoreTranslogRepository;
     private final String remoteStoreRepository;
@@ -934,6 +935,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null);
         replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings);
         isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false);
+        isStoreLocalityPartial = settings.get(
+            IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey(),
+            IndexModule.DataLocalityType.FULL.toString()
+        ).equalsIgnoreCase(IndexModule.DataLocalityType.PARTIAL.toString());
         remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY);
         remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
         remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY);
@@ -1294,6 +1299,13 @@ public String getRemoteStoreTranslogRepository() {
         return remoteStoreTranslogRepository;
     }
 
+    /**
+     * Returns true if the store locality is partial
+     */
+    public boolean isStoreLocalityPartial() {
+        return isStoreLocalityPartial;
+    }
+
     /**
      * Returns true if this is remote/searchable snapshot
      */
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index 8773b37aa7d4c..3db5e0b7f4b0c 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -30,6 +30,7 @@
 import org.opensearch.index.engine.InternalEngine;
 import org.opensearch.index.remote.RemoteSegmentTransferTracker;
 import org.opensearch.index.seqno.SequenceNumbers;
+import org.opensearch.index.store.CompositeDirectory;
 import org.opensearch.index.store.RemoteSegmentStoreDirectory;
 import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata;
 import org.opensearch.index.translog.Translog;
@@ -438,6 +439,7 @@ private void uploadNewSegments(
         logger.debug("Effective new segments files to upload {}", filteredFiles);
         ActionListener<Collection<Void>> mappedListener = ActionListener.map(listener, resp -> null);
         GroupedActionListener<Void> batchUploadListener = new GroupedActionListener<>(mappedListener, filteredFiles.size());
+        Directory directory = ((FilterDirectory) (((FilterDirectory) storeDirectory).getDelegate())).getDelegate();
 
         for (String src : filteredFiles) {
             // Initializing listener here to ensure that the stats increment operations are thread-safe
@@ -445,6 +447,9 @@ private void uploadNewSegments(
             ActionListener<Void> aggregatedListener = ActionListener.wrap(resp -> {
                 statsListener.onSuccess(src);
                 batchUploadListener.onResponse(resp);
+                if (directory instanceof CompositeDirectory) {
+                    ((CompositeDirectory) directory).afterSyncToRemote(src);
+                }
             }, ex -> {
                 logger.warn(() -> new ParameterizedMessage("Exception: [{}] while uploading segment files", ex), ex);
                 if (ex instanceof CorruptIndexException) {
diff --git a/server/src/main/java/org/opensearch/index/store/CloseableFilterIndexOutput.java b/server/src/main/java/org/opensearch/index/store/CloseableFilterIndexOutput.java
new file mode 100644
index 0000000000000..291f714369a74
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/store/CloseableFilterIndexOutput.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store;
+
+import org.apache.lucene.store.IndexOutput;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.lucene.store.FilterIndexOutput;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * FilterIndexOutput which takes in an additional FunctionalInterface as a parameter to perform required operations once the IndexOutput is closed
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CloseableFilterIndexOutput extends FilterIndexOutput {
+
+    /**
+     * Functional Interface which takes the name of the file as input on which the required operations are to be performed
+     */
+    @FunctionalInterface
+    public interface OnCloseListener {
+        void onClose(String name) throws IOException;
+    }
+
+    private final OnCloseListener onCloseListener;
+    private final String fileName;
+    private final AtomicBoolean isClosed;
+
+    public CloseableFilterIndexOutput(IndexOutput out, String fileName, OnCloseListener onCloseListener) {
+        super("CloseableFilterIndexOutput for file " + fileName, out);
+        this.fileName = fileName;
+        this.onCloseListener = onCloseListener;
+        this.isClosed = new AtomicBoolean(false);
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (isClosed.get() == false) {
+            super.close();
+            onCloseListener.onClose(fileName);
+            isClosed.set(true);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java
new file mode 100644
index 0000000000000..eb89c86ae687f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java
@@ -0,0 +1,344 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.Version;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.lucene.store.InputStreamIndexInput;
+import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
+import org.opensearch.index.store.remote.file.OnDemandBlockSnapshotIndexInput;
+import org.opensearch.index.store.remote.filecache.CachedFullFileIndexInput;
+import org.opensearch.index.store.remote.filecache.CachedIndexInput;
+import org.opensearch.index.store.remote.filecache.FileCache;
+import org.opensearch.index.store.remote.utils.BlockIOContext;
+import org.opensearch.index.store.remote.utils.FileTypeUtils;
+import org.opensearch.index.store.remote.utils.TransferManager;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Composite Directory will contain both local and remote directory
+ * Consumers of Composite directory need not worry whether file is in local or remote
+ * All such abstractions will be handled by the Composite directory itself
+ * Implements all required methods by Directory abstraction
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CompositeDirectory extends FilterDirectory {
+    private static final Logger logger = LogManager.getLogger(CompositeDirectory.class);
+    private final FSDirectory localDirectory;
+    private final RemoteSegmentStoreDirectory remoteDirectory;
+    private final FileCache fileCache;
+    private final TransferManager transferManager;
+
+    /**
+     * Constructor to initialise the composite directory
+     * @param localDirectory corresponding to the local FSDirectory
+     * @param remoteDirectory corresponding to the remote directory
+     * @param fileCache used to cache the remote files locally
+     */
+    public CompositeDirectory(Directory localDirectory, Directory remoteDirectory, FileCache fileCache) {
+        super(localDirectory);
+        validate(localDirectory, remoteDirectory, fileCache);
+        this.localDirectory = (FSDirectory) localDirectory;
+        this.remoteDirectory = (RemoteSegmentStoreDirectory) remoteDirectory;
+        this.fileCache = fileCache;
+        transferManager = new TransferManager(
+            (name, position, length) -> new InputStreamIndexInput(
+                remoteDirectory.openInput(name, new BlockIOContext(IOContext.DEFAULT, position, length)),
+                length
+            ),
+            fileCache
+        );
+    }
+
+    /**
+     * Returns names of all files stored in this directory in sorted order
+     * Does not include locally stored block files (having _block_ in their names) and files pending deletion
+     *
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public String[] listAll() throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: listAll() called", this::toString);
+        String[] localFiles = localDirectory.listAll();
+        Set<String> allFiles = new HashSet<>(Arrays.asList(localFiles));
+        String[] remoteFiles = getRemoteFiles();
+        allFiles.addAll(Arrays.asList(remoteFiles));
+        logger.trace("Composite Directory[{}]: Local Directory files - {}", this::toString, () -> Arrays.toString(localFiles));
+        logger.trace("Composite Directory[{}]: Remote Directory files - {}", this::toString, () -> Arrays.toString(remoteFiles));
+        Set<String> nonBlockLuceneFiles = allFiles.stream()
+            .filter(file -> !FileTypeUtils.isBlockFile(file))
+            .collect(Collectors.toUnmodifiableSet());
+        String[] files = new String[nonBlockLuceneFiles.size()];
+        nonBlockLuceneFiles.toArray(files);
+        Arrays.sort(files);
+        logger.trace("Composite Directory[{}]: listAll() returns : {}", this::toString, () -> Arrays.toString(files));
+        return files;
+    }
+
+    /**
+     * Removes an existing file in the directory.
+     * Currently deleting only from local directory as files from remote should not be deleted as that is taken care by garbage collection logic of remote directory
+     * @param name the name of an existing file.
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public void deleteFile(String name) throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: deleteFile() called {}", this::toString, () -> name);
+        if (FileTypeUtils.isTempFile(name)) {
+            localDirectory.deleteFile(name);
+        } else if (Arrays.asList(listAll()).contains(name) == false) {
+            throw new NoSuchFileException("File " + name + " not found in directory");
+        } else {
+            fileCache.remove(getFilePath(name));
+        }
+    }
+
+    /**
+     * Returns the byte length of a file in the directory.
+     * Throws {@link NoSuchFileException} or {@link FileNotFoundException} in case file is not present locally and in remote as well
+     * @param name the name of an existing file.
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public long fileLength(String name) throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: fileLength() called {}", this::toString, () -> name);
+        long fileLength;
+        Path key = getFilePath(name);
+        if (FileTypeUtils.isTempFile(name) || fileCache.get(key) != null) {
+            try {
+                fileLength = localDirectory.fileLength(name);
+                logger.trace(
+                    "Composite Directory[{}]: fileLength of {} fetched from Local - {}",
+                    this::toString,
+                    () -> name,
+                    () -> fileLength
+                );
+            } finally {
+                fileCache.decRef(key);
+            }
+        } else {
+            fileLength = remoteDirectory.fileLength(name);
+            logger.trace(
+                "Composite Directory[{}]: fileLength of {} fetched from Remote - {}",
+                this::toString,
+                () -> name,
+                () -> fileLength
+            );
+        }
+        return fileLength;
+    }
+
+    /**
+     * Creates a new, empty file in the directory and returns an {@link IndexOutput} instance for
+     * appending data to this file.
+     * @param name the name of the file to create.
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public IndexOutput createOutput(String name, IOContext context) throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: createOutput() called {}", this::toString, () -> name);
+        // The CloseableFilterIndexOutput will ensure that the file is added to FileCache once write is completed on this file
+        return new CloseableFilterIndexOutput(localDirectory.createOutput(name, context), name, this::cacheFile);
+    }
+
+    /**
+     * Ensures that any writes to these files are moved to stable storage (made durable).
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public void sync(Collection<String> names) throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: sync() called {}", this::toString, () -> names);
+        Collection<String> remoteFiles = Arrays.asList(getRemoteFiles());
+        Collection<String> filesToSync = names.stream().filter(name -> remoteFiles.contains(name) == false).collect(Collectors.toList());
+        logger.trace("Composite Directory[{}]: Synced files : {}", this::toString, () -> filesToSync);
+        localDirectory.sync(filesToSync);
+    }
+
+    /**
+     * Renames {@code source} file to {@code dest} file where {@code dest} must not already exist in
+     * the directory.
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public void rename(String source, String dest) throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: rename() called : source-{}, dest-{}", this::toString, () -> source, () -> dest);
+        localDirectory.rename(source, dest);
+        fileCache.remove(getFilePath(source));
+        cacheFile(dest);
+    }
+
+    /**
+     * Opens a stream for reading an existing file.
+     * Check whether the file is present locally or in remote and return the IndexInput accordingly
+     * @param name the name of an existing file.
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public IndexInput openInput(String name, IOContext context) throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: openInput() called {}", this::toString, () -> name);
+        // We aren't tracking temporary files (created via createTempOutput) currently in FileCache as these are created and then deleted
+        // within a very short span of time
+        // We will be reading them directory from the local directory
+        if (FileTypeUtils.isTempFile(name)) {
+            return localDirectory.openInput(name, context);
+        }
+        // Return directly from the FileCache (via TransferManager) if complete file is present
+        Path key = getFilePath(name);
+        CachedIndexInput indexInput = fileCache.get(key);
+        if (indexInput != null) {
+            logger.trace("Composite Directory[{}]: Complete file {} found in FileCache", this::toString, () -> name);
+            try {
+                return indexInput.getIndexInput().clone();
+            } finally {
+                fileCache.decRef(key);
+            }
+        }
+        // If file has been uploaded to the Remote Store, fetch it from the Remote Store in blocks via OnDemandCompositeBlockIndexInput
+        else {
+            logger.trace(
+                "Composite Directory[{}]: Complete file {} not in FileCache, to be fetched in Blocks from Remote",
+                this::toString,
+                () -> name
+            );
+            RemoteSegmentStoreDirectory.UploadedSegmentMetadata uploadedSegmentMetadata = remoteDirectory.getSegmentsUploadedToRemoteStore()
+                .get(name);
+            if (uploadedSegmentMetadata == null) {
+                throw new NoSuchFileException("File " + name + " not found in directory");
+            }
+            // TODO : Refactor FileInfo and OnDemandBlockSnapshotIndexInput to more generic names as they are not Remote Snapshot specific
+            BlobStoreIndexShardSnapshot.FileInfo fileInfo = new BlobStoreIndexShardSnapshot.FileInfo(
+                name,
+                new StoreFileMetadata(name, uploadedSegmentMetadata.getLength(), uploadedSegmentMetadata.getChecksum(), Version.LATEST),
+                null
+            );
+            return new OnDemandBlockSnapshotIndexInput(fileInfo, localDirectory, transferManager);
+        }
+    }
+
+    /**
+     * Closing the local directory here
+     * @throws IOException in case of I/O error
+     */
+    @Override
+    public void close() throws IOException {
+        ensureOpen();
+        logger.trace("Composite Directory[{}]: close() called", this::toString);
+        localDirectory.close();
+    }
+
+    @Override
+    public String toString() {
+        return "Composite Directory @ " + Integer.toHexString(hashCode());
+    }
+
+    /**
+     * Function to perform operations once files have been uploaded to Remote Store
+     * Currently deleting the local files here, as once uploaded to Remote, local files become eligible for eviction from FileCache
+     * @param file : recent files which have been successfully uploaded to Remote Store
+     */
+    public void afterSyncToRemote(String file) {
+        ensureOpen();
+        /*
+        Decrementing the refCount here for the path so that it becomes eligible for eviction
+        This is a temporary solution until pinning support is added
+        TODO - Unpin the files here from FileCache so that they become eligible for eviction, once pinning/unpinning support is added in FileCache
+        Uncomment the below commented line(to remove the file from cache once uploaded) to test block based functionality
+         */
+        logger.trace(
+            "Composite Directory[{}]: File {} uploaded to Remote Store and now can be eligible for eviction in FileCache",
+            this::toString,
+            () -> file
+        );
+        fileCache.decRef(getFilePath(file));
+        // fileCache.remove(getFilePath(fileName));
+    }
+
+    // Visibility public since we need it in IT tests
+    public Path getFilePath(String name) {
+        return localDirectory.getDirectory().resolve(name);
+    }
+
+    /**
+     * Basic validations for Composite Directory parameters (null checks and instance type checks)
+     *
+     * Note: Currently Composite Directory only supports local directory to be of type FSDirectory
+     * The reason is that FileCache currently has it key type as Path
+     * Composite Directory currently uses FSDirectory's getDirectory() method to fetch and use the Path for operating on FileCache
+     * TODO : Refactor FileCache to have key in form of String instead of Path. Once that is done we can remove this assertion
+     */
+    private void validate(Directory localDirectory, Directory remoteDirectory, FileCache fileCache) {
+        if (localDirectory == null || remoteDirectory == null) throw new IllegalStateException(
+            "Local and remote directory cannot be null for Composite Directory"
+        );
+        if (fileCache == null) throw new IllegalStateException(
+            "File Cache not initialized on this Node, cannot create Composite Directory without FileCache"
+        );
+        if (localDirectory instanceof FSDirectory == false) throw new IllegalStateException(
+            "For Composite Directory, local directory must be of type FSDirectory"
+        );
+        if (remoteDirectory instanceof RemoteSegmentStoreDirectory == false) throw new IllegalStateException(
+            "For Composite Directory, remote directory must be of type RemoteSegmentStoreDirectory"
+        );
+    }
+
+    /**
+     * Return the list of files present in Remote
+     */
+    private String[] getRemoteFiles() throws IOException {
+        String[] remoteFiles;
+        try {
+            remoteFiles = remoteDirectory.listAll();
+        } catch (NullPointerException e) {
+            /*
+            We can encounter NPE when no data has been uploaded to remote store yet and as a result the metadata is empty
+            Empty metadata means that there are no files currently in remote, hence returning an empty list in this scenario
+            TODO : Catch the NPE in listAll of RemoteSegmentStoreDirectory itself instead of catching here
+             */
+            remoteFiles = new String[0];
+        }
+        return remoteFiles;
+    }
+
+    private void cacheFile(String name) throws IOException {
+        Path filePath = getFilePath(name);
+        // put will increase the refCount for the path, making sure it is not evicted, will decrease the ref after it is uploaded to Remote
+        // so that it can be evicted after that
+        // this is just a temporary solution, will pin the file once support for that is added in FileCache
+        // TODO : Pin the above filePath in the file cache once pinning support is added so that it cannot be evicted unless it has been
+        // successfully uploaded to Remote
+        fileCache.put(filePath, new CachedFullFileIndexInput(fileCache, filePath, localDirectory.openInput(name, IOContext.DEFAULT)));
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
index 99f78130ad3ef..c61fae74c0bc0 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
@@ -28,9 +28,11 @@
 import org.opensearch.common.blobstore.transfer.RemoteTransferContainer;
 import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream;
 import org.opensearch.common.blobstore.transfer.stream.OffsetRangeInputStream;
+import org.opensearch.common.lucene.store.ByteArrayIndexInput;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.index.store.exception.ChecksumCombinationException;
+import org.opensearch.index.store.remote.utils.BlockIOContext;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -203,10 +205,14 @@ public IndexInput openInput(String name, IOContext context) throws IOException {
     public IndexInput openInput(String name, long fileLength, IOContext context) throws IOException {
         InputStream inputStream = null;
         try {
-            inputStream = blobContainer.readBlob(name);
-            return new RemoteIndexInput(name, downloadRateLimiter.apply(inputStream), fileLength);
+            if (context instanceof BlockIOContext) {
+                return getBlockInput(name, fileLength, (BlockIOContext) context);
+            } else {
+                inputStream = blobContainer.readBlob(name);
+                return new RemoteIndexInput(name, downloadRateLimiter.apply(inputStream), fileLength);
+            }
         } catch (Exception e) {
-            // Incase the RemoteIndexInput creation fails, close the input stream to avoid file handler leak.
+            // In case the RemoteIndexInput creation fails, close the input stream to avoid file handler leak.
             if (inputStream != null) {
                 try {
                     inputStream.close();
@@ -434,4 +440,18 @@ private long calculateChecksumOfChecksum(Directory directory, String file) throw
             }
         }
     }
+
+    private IndexInput getBlockInput(String name, long fileLength, BlockIOContext blockIOContext) throws IOException {
+        long position = blockIOContext.getBlockStart();
+        long length = blockIOContext.getBlockSize();
+        if (position < 0 || length < 0 || (position + length > fileLength)) {
+            throw new IllegalArgumentException("Invalid values of block start and size");
+        }
+        byte[] bytes;
+        try (InputStream inputStream = blobContainer.readBlob(name, position, length)) {
+            // TODO - Explore how we can buffer small chunks of data instead of having the whole 8MB block in memory
+            bytes = downloadRateLimiter.apply(inputStream).readAllBytes();
+        }
+        return new ByteArrayIndexInput(name, bytes);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java
index 7cfa738e75e52..177f0526e7571 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java
@@ -94,7 +94,7 @@ private Future<RemoteSnapshotDirectory> createRemoteSnapshotDirectoryFromSnapsho
             assert indexShardSnapshot instanceof BlobStoreIndexShardSnapshot
                 : "indexShardSnapshot should be an instance of BlobStoreIndexShardSnapshot";
             final BlobStoreIndexShardSnapshot snapshot = (BlobStoreIndexShardSnapshot) indexShardSnapshot;
-            TransferManager transferManager = new TransferManager(blobContainer, remoteStoreFileCache);
+            TransferManager transferManager = new TransferManager(blobContainer::readBlob, remoteStoreFileCache);
             return new RemoteSnapshotDirectory(snapshot, localStoreDir, transferManager);
         });
     }
diff --git a/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java b/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java
index 8097fd08da50a..ad56127394779 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.index.store.remote.file;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.IndexInput;
 import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
@@ -26,6 +28,7 @@
  * @opensearch.internal
  */
 public class OnDemandBlockSnapshotIndexInput extends OnDemandBlockIndexInput {
+    private static final Logger logger = LogManager.getLogger(OnDemandBlockSnapshotIndexInput.class);
     /**
      * Where this class fetches IndexInput parts from
      */
@@ -133,10 +136,19 @@ protected OnDemandBlockSnapshotIndexInput buildSlice(String sliceDescription, lo
 
     @Override
     protected IndexInput fetchBlock(int blockId) throws IOException {
-        final String blockFileName = fileName + "." + blockId;
+        logger.trace("fetchBlock called with blockId -> {}", blockId);
+        final String blockFileName = fileName + "_block_" + blockId;
 
         final long blockStart = getBlockStart(blockId);
         final long blockEnd = blockStart + getActualBlockSize(blockId);
+        logger.trace(
+            "File: {} , Block File: {} , BlockStart: {} , BlockEnd: {} , OriginalFileSize: {}",
+            fileName,
+            blockFileName,
+            blockStart,
+            blockEnd,
+            originalFileSize
+        );
 
         // Block may be present on multiple chunks of a file, so we need
         // to fetch each chunk/blob part separately to fetch an entire block.
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/CachedFullFileIndexInput.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/CachedFullFileIndexInput.java
new file mode 100644
index 0000000000000..286739cb6cd90
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/CachedFullFileIndexInput.java
@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.filecache;
+
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.IndexInput;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Implementation of the CachedIndexInput for full files which takes in an IndexInput as parameter
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CachedFullFileIndexInput implements CachedIndexInput {
+    private final FileCache fileCache;
+    private final Path path;
+    private final FullFileCachedIndexInput fullFileCachedIndexInput;
+    private final AtomicBoolean isClosed;
+
+    /**
+     * Constructor - takes IndexInput as parameter
+     */
+    public CachedFullFileIndexInput(FileCache fileCache, Path path, IndexInput indexInput) {
+        this.fileCache = fileCache;
+        this.path = path;
+        fullFileCachedIndexInput = new FullFileCachedIndexInput(fileCache, path, indexInput);
+        isClosed = new AtomicBoolean(false);
+    }
+
+    /**
+     * Returns the wrapped indexInput
+     */
+    @Override
+    public IndexInput getIndexInput() {
+        if (isClosed.get()) throw new AlreadyClosedException("Index input is already closed");
+        return fullFileCachedIndexInput;
+    }
+
+    /**
+     * Returns the length of the wrapped indexInput
+     */
+    @Override
+    public long length() {
+        return fullFileCachedIndexInput.length();
+    }
+
+    /**
+     * Checks if the wrapped indexInput is closed
+     */
+    @Override
+    public boolean isClosed() {
+        return isClosed.get();
+    }
+
+    /**
+     * Closes the wrapped indexInput
+     */
+    @Override
+    public void close() throws Exception {
+        if (!isClosed.getAndSet(true)) {
+            fullFileCachedIndexInput.close();
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java
index e61e5ecd4084a..a8aa33a977cb8 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.index.store.remote.filecache;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.IndexInput;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.breaker.CircuitBreaker;
@@ -47,6 +49,7 @@
  */
 @PublicApi(since = "2.7.0")
 public class FileCache implements RefCountedCache<Path, CachedIndexInput> {
+    private static final Logger logger = LogManager.getLogger(FileCache.class);
     private final SegmentedCache<Path, CachedIndexInput> theCache;
 
     private final CircuitBreaker circuitBreaker;
@@ -139,6 +142,14 @@ public CacheStats stats() {
         return theCache.stats();
     }
 
+    // To be used only for debugging purposes
+    public void logCurrentState() {
+        logger.trace("CURRENT STATE OF FILE CACHE \n");
+        CacheUsage cacheUsage = theCache.usage();
+        logger.trace("Total Usage: " + cacheUsage.usage() + " , Active Usage: " + cacheUsage.activeUsage());
+        theCache.logCurrentState();
+    }
+
     /**
      * Ensures that the PARENT breaker is not tripped when an entry is added to the cache
      * @param filePath the path key for which entry is added
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInput.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInput.java
index 7d7c40be3a833..ab6f5f931da0f 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInput.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInput.java
@@ -39,9 +39,9 @@ public class FileCachedIndexInput extends IndexInput implements RandomAccessInpu
     protected IndexInput luceneIndexInput;
 
     /** indicates if this IndexInput instance is a clone or not */
-    private final boolean isClone;
+    protected final boolean isClone;
 
-    private volatile boolean closed = false;
+    protected volatile boolean closed = false;
 
     public FileCachedIndexInput(FileCache cache, Path filePath, IndexInput underlyingIndexInput) {
         this(cache, filePath, underlyingIndexInput, false);
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java
new file mode 100644
index 0000000000000..9383c53d6d830
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java
@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.filecache;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.IndexInput;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Extension of {@link FileCachedIndexInput} for full files for handling clones and slices
+ * We maintain a clone map so that we can close them when the parent IndexInput is closed so that ref count is properly maintained in file cache
+ * Closing of clones explicitly is needed as Lucene does not guarantee that it will close the clones
+ * https://github.com/apache/lucene/blob/8340b01c3cc229f33584ce2178b07b8984daa6a9/lucene/core/src/java/org/apache/lucene/store/IndexInput.java#L32-L33
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class FullFileCachedIndexInput extends FileCachedIndexInput {
+    private static final Logger logger = LogManager.getLogger(FullFileCachedIndexInput.class);
+    private final Set<FullFileCachedIndexInput> clones;
+
+    public FullFileCachedIndexInput(FileCache cache, Path filePath, IndexInput underlyingIndexInput) {
+        this(cache, filePath, underlyingIndexInput, false);
+    }
+
+    public FullFileCachedIndexInput(FileCache cache, Path filePath, IndexInput underlyingIndexInput, boolean isClone) {
+        super(cache, filePath, underlyingIndexInput, isClone);
+        clones = new HashSet<>();
+    }
+
+    /**
+     * Clones the index input and returns the clone
+     * Increase the ref count whenever the index input is cloned and add it to the clone map as well
+     */
+    @Override
+    public FullFileCachedIndexInput clone() {
+        FullFileCachedIndexInput clonedIndexInput = new FullFileCachedIndexInput(cache, filePath, luceneIndexInput.clone(), true);
+        cache.incRef(filePath);
+        clones.add(clonedIndexInput);
+        return clonedIndexInput;
+    }
+
+    /**
+     * Clones the index input and returns the slice
+     * Increase the ref count whenever the index input is sliced and add it to the clone map as well
+     */
+    @Override
+    public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+        if (offset < 0 || length < 0 || offset + length > this.length()) {
+            throw new IllegalArgumentException(
+                "slice() "
+                    + sliceDescription
+                    + " out of bounds: offset="
+                    + offset
+                    + ",length="
+                    + length
+                    + ",fileLength="
+                    + this.length()
+                    + ": "
+                    + this
+            );
+        }
+        IndexInput slicedLuceneIndexInput = luceneIndexInput.slice(sliceDescription, offset, length);
+        FullFileCachedIndexInput slicedIndexInput = new FullFileCachedIndexInput(cache, filePath, slicedLuceneIndexInput, true);
+        clones.add(slicedIndexInput);
+        cache.incRef(filePath);
+        return slicedIndexInput;
+    }
+
+    /**
+     * Closes the index input and it's clones as well
+     */
+    @Override
+    public void close() throws IOException {
+        if (!closed) {
+            if (isClone) {
+                cache.decRef(filePath);
+            }
+            clones.forEach(indexInput -> {
+                try {
+                    indexInput.close();
+                } catch (Exception e) {
+                    logger.trace("Exception while closing clone - {}", e.getMessage());
+                }
+            });
+            try {
+                luceneIndexInput.close();
+            } catch (AlreadyClosedException e) {
+                logger.trace("FullFileCachedIndexInput already closed");
+            }
+            luceneIndexInput = null;
+            clones.clear();
+            closed = true;
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/BlockIOContext.java b/server/src/main/java/org/opensearch/index/store/remote/utils/BlockIOContext.java
new file mode 100644
index 0000000000000..a78dd85d6f194
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/store/remote/utils/BlockIOContext.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.utils;
+
+import org.apache.lucene.store.IOContext;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * BlockIOContext is an extension of IOContext which can be used to pass block related information to the openInput() method of any directory
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class BlockIOContext extends IOContext {
+
+    private long blockStart;
+    private long blockSize;
+
+    /**
+     * Constructor to initialise BlockIOContext with block related information
+     */
+    public BlockIOContext(IOContext ctx, long blockStart, long blockSize) {
+        super(ctx.context);
+        verifyBlockStartAndSize(blockStart, blockSize);
+        this.blockStart = blockStart;
+        this.blockSize = blockSize;
+    }
+
+    /**
+     * Getter for blockStart
+     */
+    public long getBlockStart() {
+        return blockStart;
+    }
+
+    /**
+     * Getter for blockSize
+     */
+    public long getBlockSize() {
+        return blockSize;
+    }
+
+    private void verifyBlockStartAndSize(long blockStart, long blockSize) {
+        if (blockStart < 0) throw new IllegalArgumentException("blockStart must be greater than or equal to 0");
+        if (blockSize <= 0) throw new IllegalArgumentException(("blockSize must be greater than 0"));
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java b/server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java
new file mode 100644
index 0000000000000..e78480bd500ee
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.utils;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Utility class for checking file types
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class FileTypeUtils {
+
+    public static boolean isTempFile(String name) {
+        return name.endsWith(".tmp");
+    }
+
+    public static boolean isBlockFile(String name) {
+        return name.contains("_block_");
+    }
+
+    public static boolean isExtraFSFile(String name) {
+        return name.startsWith("extra");
+    }
+
+    public static boolean isLockFile(String name) {
+        return name.endsWith(".lock");
+    }
+
+    public static boolean isSegmentsFile(String name) {
+        return name.startsWith("segments_");
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java
index 98cad7bfadb09..df26f2f0925f6 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java
@@ -12,7 +12,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
-import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.index.store.remote.filecache.CachedIndexInput;
 import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.index.store.remote.filecache.FileCachedIndexInput;
@@ -39,11 +38,19 @@
 public class TransferManager {
     private static final Logger logger = LogManager.getLogger(TransferManager.class);
 
-    private final BlobContainer blobContainer;
+    /**
+     * Functional interface to get an InputStream for a file at a certain offset and size
+     */
+    @FunctionalInterface
+    public interface StreamReader {
+        InputStream read(String name, long position, long length) throws IOException;
+    }
+
+    private final StreamReader streamReader;
     private final FileCache fileCache;
 
-    public TransferManager(final BlobContainer blobContainer, final FileCache fileCache) {
-        this.blobContainer = blobContainer;
+    public TransferManager(final StreamReader streamReader, final FileCache fileCache) {
+        this.streamReader = streamReader;
         this.fileCache = fileCache;
     }
 
@@ -55,12 +62,15 @@ public TransferManager(final BlobContainer blobContainer, final FileCache fileCa
     public IndexInput fetchBlob(BlobFetchRequest blobFetchRequest) throws IOException {
 
         final Path key = blobFetchRequest.getFilePath();
+        logger.trace("fetchBlob called for {}", key.toString());
 
         final CachedIndexInput cacheEntry = fileCache.compute(key, (path, cachedIndexInput) -> {
             if (cachedIndexInput == null || cachedIndexInput.isClosed()) {
+                logger.trace("Transfer Manager - IndexInput closed or not in cache");
                 // Doesn't exist or is closed, either way create a new one
-                return new DelayedCreationCachedIndexInput(fileCache, blobContainer, blobFetchRequest);
+                return new DelayedCreationCachedIndexInput(fileCache, streamReader, blobFetchRequest);
             } else {
+                logger.trace("Transfer Manager - Already in cache");
                 // already in the cache and ready to be used (open)
                 return cachedIndexInput;
             }
@@ -77,7 +87,7 @@ public IndexInput fetchBlob(BlobFetchRequest blobFetchRequest) throws IOExceptio
     }
 
     @SuppressWarnings("removal")
-    private static FileCachedIndexInput createIndexInput(FileCache fileCache, BlobContainer blobContainer, BlobFetchRequest request) {
+    private static FileCachedIndexInput createIndexInput(FileCache fileCache, StreamReader streamReader, BlobFetchRequest request) {
         // We need to do a privileged action here in order to fetch from remote
         // and write to the local file cache in case this is invoked as a side
         // effect of a plugin (such as a scripted search) that doesn't have the
@@ -85,13 +95,14 @@ private static FileCachedIndexInput createIndexInput(FileCache fileCache, BlobCo
         return AccessController.doPrivileged((PrivilegedAction<FileCachedIndexInput>) () -> {
             try {
                 if (Files.exists(request.getFilePath()) == false) {
+                    logger.trace("Fetching from Remote in createIndexInput of Transfer Manager");
                     try (
                         OutputStream fileOutputStream = Files.newOutputStream(request.getFilePath());
                         OutputStream localFileOutputStream = new BufferedOutputStream(fileOutputStream)
                     ) {
                         for (BlobFetchRequest.BlobPart blobPart : request.blobParts()) {
                             try (
-                                InputStream snapshotFileInputStream = blobContainer.readBlob(
+                                InputStream snapshotFileInputStream = streamReader.read(
                                     blobPart.getBlobName(),
                                     blobPart.getPosition(),
                                     blobPart.getLength()
@@ -119,15 +130,15 @@ private static FileCachedIndexInput createIndexInput(FileCache fileCache, BlobCo
      */
     private static class DelayedCreationCachedIndexInput implements CachedIndexInput {
         private final FileCache fileCache;
-        private final BlobContainer blobContainer;
+        private final StreamReader streamReader;
         private final BlobFetchRequest request;
         private final CompletableFuture<IndexInput> result = new CompletableFuture<>();
         private final AtomicBoolean isStarted = new AtomicBoolean(false);
         private final AtomicBoolean isClosed = new AtomicBoolean(false);
 
-        private DelayedCreationCachedIndexInput(FileCache fileCache, BlobContainer blobContainer, BlobFetchRequest request) {
+        private DelayedCreationCachedIndexInput(FileCache fileCache, StreamReader streamReader, BlobFetchRequest request) {
             this.fileCache = fileCache;
-            this.blobContainer = blobContainer;
+            this.streamReader = streamReader;
             this.request = request;
         }
 
@@ -139,7 +150,7 @@ public IndexInput getIndexInput() throws IOException {
             if (isStarted.getAndSet(true) == false) {
                 // We're the first one here, need to download the block
                 try {
-                    result.complete(createIndexInput(fileCache, blobContainer, request));
+                    result.complete(createIndexInput(fileCache, streamReader, request));
                 } catch (Exception e) {
                     result.completeExceptionally(e);
                     fileCache.remove(request.getFilePath());
diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java
index 03d03711f914a..7f7d42e8fbce8 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.index.store.remote.utils.cache;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
@@ -19,6 +21,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.BiFunction;
@@ -43,6 +46,7 @@
  * @opensearch.internal
  */
 class LRUCache<K, V> implements RefCountedCache<K, V> {
+    private static final Logger logger = LogManager.getLogger(LRUCache.class);
     private final long capacity;
 
     private final HashMap<K, Node<K, V>> data;
@@ -192,8 +196,10 @@ public void clear() {
             usage = 0L;
             activeUsage = 0L;
             lru.clear();
-            for (Node<K, V> node : data.values()) {
-                data.remove(node.key);
+            final Iterator<Node<K, V>> iterator = data.values().iterator();
+            while (iterator.hasNext()) {
+                Node<K, V> node = iterator.next();
+                iterator.remove();
                 statsCounter.recordRemoval(node.weight);
                 listener.onRemoval(new RemovalNotification<>(node.key, node.value, RemovalReason.EXPLICIT));
             }
@@ -300,6 +306,22 @@ public CacheStats stats() {
         }
     }
 
+    // To be used only for debugging purposes
+    public void logCurrentState() {
+        lock.lock();
+        try {
+            String allFiles = "\n";
+            for (Map.Entry<K, Node<K, V>> entry : data.entrySet()) {
+                String path = entry.getKey().toString();
+                String file = path.substring(path.lastIndexOf('/'));
+                allFiles += file + " [RefCount: " + entry.getValue().refCount + " , Weight: " + entry.getValue().weight + " ]\n";
+            }
+            logger.trace("Cache entries : " + allFiles);
+        } finally {
+            lock.unlock();
+        }
+    }
+
     private void addNode(K key, V value) {
         final long weight = weigher.weightOf(value);
         Node<K, V> newNode = new Node<>(key, value, weight);
diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java
index d3eb03df37e1b..2ea7ea8dbee12 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.index.store.remote.utils.cache;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.Weigher;
@@ -25,6 +27,7 @@
  * @opensearch.internal
  */
 public class SegmentedCache<K, V> implements RefCountedCache<K, V> {
+    private static final Logger logger = LogManager.getLogger(SegmentedCache.class);
 
     private static final int HASH_BITS = 0x7fffffff;
 
@@ -183,6 +186,16 @@ public CacheStats stats() {
         return new CacheStats(hitCount, missCount, removeCount, removeWeight, replaceCount, evictionCount, evictionWeight);
     }
 
+    // To be used only for debugging purposes
+    public void logCurrentState() {
+        int i = 0;
+        for (RefCountedCache<K, V> cache : table) {
+            logger.trace("SegmentedCache " + i);
+            ((LRUCache<K, V>) cache).logCurrentState();
+            i++;
+        }
+    }
+
     enum SingletonWeigher implements Weigher<Object> {
         INSTANCE;
 
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index 73b4cf3259d5b..a7d879fc06981 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -136,6 +136,7 @@
 import org.opensearch.index.shard.IndexingOperationListener;
 import org.opensearch.index.shard.IndexingStats;
 import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats;
+import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.index.translog.InternalTranslogFactory;
 import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory;
 import org.opensearch.index.translog.TranslogFactory;
@@ -354,6 +355,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier;
     private volatile TimeValue clusterDefaultRefreshInterval;
     private final SearchRequestStats searchRequestStats;
+    private final FileCache fileCache;
 
     @Override
     protected void doStart() {
@@ -388,7 +390,8 @@ public IndicesService(
         @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
         RecoverySettings recoverySettings,
         CacheService cacheService,
-        RemoteStoreSettings remoteStoreSettings
+        RemoteStoreSettings remoteStoreSettings,
+        FileCache fileCache
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -495,6 +498,68 @@ protected void closeInternal() {
             .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate);
         this.recoverySettings = recoverySettings;
         this.remoteStoreSettings = remoteStoreSettings;
+        this.fileCache = fileCache;
+    }
+
+    public IndicesService(
+        Settings settings,
+        PluginsService pluginsService,
+        NodeEnvironment nodeEnv,
+        NamedXContentRegistry xContentRegistry,
+        AnalysisRegistry analysisRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        MapperRegistry mapperRegistry,
+        NamedWriteableRegistry namedWriteableRegistry,
+        ThreadPool threadPool,
+        IndexScopedSettings indexScopedSettings,
+        CircuitBreakerService circuitBreakerService,
+        BigArrays bigArrays,
+        ScriptService scriptService,
+        ClusterService clusterService,
+        Client client,
+        MetaStateService metaStateService,
+        Collection<Function<IndexSettings, Optional<EngineFactory>>> engineFactoryProviders,
+        Map<String, IndexStorePlugin.DirectoryFactory> directoryFactories,
+        ValuesSourceRegistry valuesSourceRegistry,
+        Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories,
+        IndexStorePlugin.DirectoryFactory remoteDirectoryFactory,
+        Supplier<RepositoriesService> repositoriesServiceSupplier,
+        SearchRequestStats searchRequestStats,
+        @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
+        RecoverySettings recoverySettings,
+        CacheService cacheService,
+        RemoteStoreSettings remoteStoreSettings
+    ) {
+        this(
+            settings,
+            pluginsService,
+            nodeEnv,
+            xContentRegistry,
+            analysisRegistry,
+            indexNameExpressionResolver,
+            mapperRegistry,
+            namedWriteableRegistry,
+            threadPool,
+            indexScopedSettings,
+            circuitBreakerService,
+            bigArrays,
+            scriptService,
+            clusterService,
+            client,
+            metaStateService,
+            engineFactoryProviders,
+            directoryFactories,
+            valuesSourceRegistry,
+            recoveryStateFactories,
+            remoteDirectoryFactory,
+            repositoriesServiceSupplier,
+            searchRequestStats,
+            remoteStoreStatsTrackerFactory,
+            recoverySettings,
+            cacheService,
+            remoteStoreSettings,
+            null
+        );
     }
 
     /**
@@ -873,7 +938,8 @@ private synchronized IndexService createIndexService(
             directoryFactories,
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
-            recoveryStateFactories
+            recoveryStateFactories,
+            fileCache
         );
         for (IndexingOperationListener operationListener : indexingOperationListeners) {
             indexModule.addIndexOperationListener(operationListener);
@@ -963,7 +1029,8 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe
             directoryFactories,
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
-            recoveryStateFactories
+            recoveryStateFactories,
+            fileCache
         );
         pluginsService.onIndexModule(indexModule);
         return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService);
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 397949525a3ec..a91dce4ece126 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -869,7 +869,8 @@ protected Node(
                 remoteStoreStatsTrackerFactory,
                 recoverySettings,
                 cacheService,
-                remoteStoreSettings
+                remoteStoreSettings,
+                fileCache
             );
 
             final IngestService ingestService = new IngestService(
@@ -2004,7 +2005,8 @@ DiscoveryNode getNode() {
      * Else it configures the size to 80% of available capacity for a dedicated search node, if not explicitly defined.
      */
     private void initializeFileCache(Settings settings, CircuitBreaker circuitBreaker) throws IOException {
-        if (DiscoveryNode.isSearchNode(settings)) {
+        boolean isWritableRemoteIndexEnabled = FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX_SETTING);
+        if (DiscoveryNode.isSearchNode(settings) || isWritableRemoteIndexEnabled) {
             NodeEnvironment.NodePath fileCacheNodePath = nodeEnvironment.fileCacheNodePath();
             long capacity = NODE_SEARCH_CACHE_SIZE_SETTING.get(settings).getBytes();
             FsInfo.Path info = ExceptionsHelper.catchAsRuntimeException(() -> FsProbe.getFSInfo(fileCacheNodePath));
@@ -2013,7 +2015,10 @@ private void initializeFileCache(Settings settings, CircuitBreaker circuitBreake
             // Initialize default values for cache if NODE_SEARCH_CACHE_SIZE_SETTING is not set.
             if (capacity == 0) {
                 // If node is not a dedicated search node without configuration, prevent cache initialization
-                if (DiscoveryNode.getRolesFromSettings(settings).stream().anyMatch(role -> !DiscoveryNodeRole.SEARCH_ROLE.equals(role))) {
+                if (!isWritableRemoteIndexEnabled
+                    && DiscoveryNode.getRolesFromSettings(settings)
+                        .stream()
+                        .anyMatch(role -> !DiscoveryNodeRole.SEARCH_ROLE.equals(role))) {
                     throw new SettingsException(
                         "Unable to initialize the "
                             + DiscoveryNodeRole.SEARCH_ROLE.roleName()
diff --git a/server/src/test/java/org/opensearch/index/store/BaseRemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/BaseRemoteSegmentStoreDirectoryTests.java
new file mode 100644
index 0000000000000..ff9b62a341deb
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/store/BaseRemoteSegmentStoreDirectoryTests.java
@@ -0,0 +1,178 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store;
+
+import org.apache.lucene.index.SegmentInfos;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
+import org.opensearch.index.engine.NRTReplicationEngineFactory;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.index.shard.IndexShardTestCase;
+import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
+import static org.opensearch.index.store.RemoteSegmentStoreDirectory.METADATA_FILES_TO_FETCH;
+import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes;
+import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class BaseRemoteSegmentStoreDirectoryTests extends IndexShardTestCase {
+
+    protected RemoteDirectory remoteDataDirectory;
+    protected RemoteDirectory remoteMetadataDirectory;
+    protected RemoteStoreMetadataLockManager mdLockManager;
+    protected RemoteSegmentStoreDirectory remoteSegmentStoreDirectory;
+    protected TestUploadListener testUploadTracker;
+    protected IndexShard indexShard;
+    protected SegmentInfos segmentInfos;
+    protected ThreadPool threadPool;
+
+    protected final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
+        12,
+        23,
+        34,
+        1,
+        1,
+        "node-1"
+    );
+
+    protected final String metadataFilenameDup = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
+        12,
+        23,
+        34,
+        2,
+        1,
+        "node-2"
+    );
+    protected final String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
+        12,
+        13,
+        34,
+        1,
+        1,
+        "node-1"
+    );
+    protected final String metadataFilename3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
+        10,
+        38,
+        34,
+        1,
+        1,
+        "node-1"
+    );
+    protected final String metadataFilename4 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
+        10,
+        36,
+        34,
+        1,
+        1,
+        "node-1"
+    );
+
+    public void setupRemoteSegmentStoreDirectory() throws IOException {
+        remoteDataDirectory = mock(RemoteDirectory.class);
+        remoteMetadataDirectory = mock(RemoteDirectory.class);
+        mdLockManager = mock(RemoteStoreMetadataLockManager.class);
+        threadPool = mock(ThreadPool.class);
+        testUploadTracker = new TestUploadListener();
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT)
+            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .build();
+        ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService();
+
+        indexShard = newStartedShard(false, indexSettings, new NRTReplicationEngineFactory());
+        remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory(
+            remoteDataDirectory,
+            remoteMetadataDirectory,
+            mdLockManager,
+            threadPool,
+            indexShard.shardId()
+        );
+        try (Store store = indexShard.store()) {
+            segmentInfos = store.readLastCommittedSegmentsInfo();
+        }
+
+        when(threadPool.executor(ThreadPool.Names.REMOTE_PURGE)).thenReturn(executorService);
+        when(threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY)).thenReturn(executorService);
+        when(threadPool.executor(ThreadPool.Names.SAME)).thenReturn(executorService);
+    }
+
+    protected Map<String, Map<String, String>> populateMetadata() throws IOException {
+        List<String> metadataFiles = new ArrayList<>();
+
+        metadataFiles.add(metadataFilename);
+        metadataFiles.add(metadataFilename2);
+        metadataFiles.add(metadataFilename3);
+
+        when(
+            remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
+                RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
+                METADATA_FILES_TO_FETCH
+            )
+        ).thenReturn(List.of(metadataFilename));
+        when(
+            remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
+                RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
+                Integer.MAX_VALUE
+            )
+        ).thenReturn(metadataFiles);
+
+        Map<String, Map<String, String>> metadataFilenameContentMapping = Map.of(
+            metadataFilename,
+            getDummyMetadata("_0", 1),
+            metadataFilename2,
+            getDummyMetadata("_0", 1),
+            metadataFilename3,
+            getDummyMetadata("_0", 1)
+        );
+
+        when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenAnswer(
+            I -> createMetadataFileBytes(
+                metadataFilenameContentMapping.get(metadataFilename),
+                indexShard.getLatestReplicationCheckpoint(),
+                segmentInfos
+            )
+        );
+        when(remoteMetadataDirectory.getBlobStream(metadataFilename2)).thenAnswer(
+            I -> createMetadataFileBytes(
+                metadataFilenameContentMapping.get(metadataFilename2),
+                indexShard.getLatestReplicationCheckpoint(),
+                segmentInfos
+            )
+        );
+        when(remoteMetadataDirectory.getBlobStream(metadataFilename3)).thenAnswer(
+            I -> createMetadataFileBytes(
+                metadataFilenameContentMapping.get(metadataFilename3),
+                indexShard.getLatestReplicationCheckpoint(),
+                segmentInfos
+            )
+        );
+
+        return metadataFilenameContentMapping;
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        indexShard.close("test tearDown", true, false);
+        super.tearDown();
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java
new file mode 100644
index 0000000000000..d5628cfab9ee7
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java
@@ -0,0 +1,202 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
+import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
+import org.opensearch.index.store.remote.file.OnDemandBlockSnapshotIndexInput;
+import org.opensearch.index.store.remote.filecache.FileCache;
+import org.opensearch.index.store.remote.filecache.FileCacheFactory;
+import org.opensearch.index.store.remote.filecache.FileCachedIndexInput;
+import org.opensearch.index.store.remote.utils.FileTypeUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class)
+public class CompositeDirectoryTests extends BaseRemoteSegmentStoreDirectoryTests {
+    private FileCache fileCache;
+    private FSDirectory localDirectory;
+    private CompositeDirectory compositeDirectory;
+
+    private final static String[] LOCAL_FILES = new String[] { "_1.cfe", "_2.cfe", "_0.cfe_block_7", "_0.cfs_block_7", "temp_file.tmp" };
+    private final static String FILE_PRESENT_LOCALLY = "_1.cfe";
+    private final static String FILE_PRESENT_IN_REMOTE_ONLY = "_0.si";
+    private final static String NON_EXISTENT_FILE = "non_existent_file";
+    private final static String NEW_FILE = "new_file";
+    private final static String TEMP_FILE = "temp_file.tmp";
+    private final static int FILE_CACHE_CAPACITY = 10000;
+
+    @Before
+    public void setup() throws IOException {
+        setupRemoteSegmentStoreDirectory();
+        populateMetadata();
+        remoteSegmentStoreDirectory.init();
+        localDirectory = FSDirectory.open(createTempDir());
+        removeExtraFSFiles();
+        fileCache = FileCacheFactory.createConcurrentLRUFileCache(FILE_CACHE_CAPACITY, new NoopCircuitBreaker(CircuitBreaker.REQUEST));
+        compositeDirectory = new CompositeDirectory(localDirectory, remoteSegmentStoreDirectory, fileCache);
+        addFilesToDirectory(LOCAL_FILES);
+    }
+
+    public void testListAll() throws IOException {
+        String[] actualFileNames = compositeDirectory.listAll();
+        String[] expectedFileNames = new String[] { "_0.cfe", "_0.cfs", "_0.si", "_1.cfe", "_2.cfe", "segments_1", "temp_file.tmp" };
+        assertArrayEquals(expectedFileNames, actualFileNames);
+    }
+
+    public void testDeleteFile() throws IOException {
+        assertTrue(existsInCompositeDirectory(FILE_PRESENT_LOCALLY));
+        // Delete the file and assert that it no more is a part of the directory
+        compositeDirectory.deleteFile(FILE_PRESENT_LOCALLY);
+        assertFalse(existsInCompositeDirectory(FILE_PRESENT_LOCALLY));
+        // Reading deleted file from directory should result in NoSuchFileException
+        assertThrows(NoSuchFileException.class, () -> compositeDirectory.openInput(FILE_PRESENT_LOCALLY, IOContext.DEFAULT));
+    }
+
+    public void testFileLength() throws IOException {
+        // File present locally
+        assertTrue(existsInLocalDirectory(FILE_PRESENT_LOCALLY));
+        assertFalse(existsInRemoteDirectory(FILE_PRESENT_LOCALLY));
+        assertEquals(compositeDirectory.fileLength(FILE_PRESENT_LOCALLY), localDirectory.fileLength(FILE_PRESENT_LOCALLY));
+
+        // File not present locally - present in Remote
+        assertFalse(existsInLocalDirectory(FILE_PRESENT_IN_REMOTE_ONLY));
+        assertTrue(existsInRemoteDirectory(FILE_PRESENT_IN_REMOTE_ONLY));
+        assertEquals(
+            compositeDirectory.fileLength(FILE_PRESENT_IN_REMOTE_ONLY),
+            remoteSegmentStoreDirectory.fileLength(FILE_PRESENT_IN_REMOTE_ONLY)
+        );
+
+        // File not present in both local and remote
+        assertFalse(Arrays.asList(compositeDirectory.listAll()).contains(NON_EXISTENT_FILE));
+        assertThrows(NoSuchFileException.class, () -> compositeDirectory.fileLength(NON_EXISTENT_FILE));
+    }
+
+    public void testCreateOutput() throws IOException {
+        try (IndexOutput indexOutput = compositeDirectory.createOutput(NEW_FILE, IOContext.DEFAULT)) {
+            // File not present in FileCache until the indexOutput is Closed
+            assertNull(fileCache.get(localDirectory.getDirectory().resolve(NEW_FILE)));
+        }
+        // File present in FileCache after the indexOutput is Closed
+        assertNotNull(fileCache.get(localDirectory.getDirectory().resolve(NEW_FILE)));
+    }
+
+    public void testSync() throws IOException {
+        // All the files in the below list are present either locally or on remote, so sync should work as expected
+        Collection<String> names = List.of("_0.cfe", "_0.cfs", "_0.si", "_1.cfe", "_2.cfe", "segments_1");
+        compositeDirectory.sync(names);
+        // Below list contains a non-existent file, hence will throw an error
+        Collection<String> names1 = List.of("_0.cfe", "_0.cfs", "_0.si", "_1.cfe", "_2.cfe", "segments_1", "non_existent_file");
+        assertThrows(NoSuchFileException.class, () -> compositeDirectory.sync(names1));
+    }
+
+    public void testRename() throws IOException {
+        // Rename should work as expected for file present in directory
+        assertTrue(existsInCompositeDirectory(FILE_PRESENT_LOCALLY));
+        compositeDirectory.rename(FILE_PRESENT_LOCALLY, "_1_new.cfe");
+        // Should throw error for file not present
+        assertThrows(NoSuchFileException.class, () -> compositeDirectory.rename(NON_EXISTENT_FILE, "_1_new.cfe"));
+    }
+
+    public void testOpenInput() throws IOException {
+        // File not present in Directory
+        assertFalse(existsInCompositeDirectory(NON_EXISTENT_FILE));
+        assertThrows(NoSuchFileException.class, () -> compositeDirectory.openInput(NON_EXISTENT_FILE, IOContext.DEFAULT));
+
+        // Temp file, read directly form local directory
+        assertTrue(existsInLocalDirectory(TEMP_FILE) && FileTypeUtils.isTempFile(TEMP_FILE));
+        assertEquals(
+            compositeDirectory.openInput(TEMP_FILE, IOContext.DEFAULT).toString(),
+            localDirectory.openInput(TEMP_FILE, IOContext.DEFAULT).toString()
+        );
+
+        // File present in file cache
+        assertNotNull(fileCache.get(getFilePath(FILE_PRESENT_LOCALLY)));
+        assertTrue(compositeDirectory.openInput(FILE_PRESENT_LOCALLY, IOContext.DEFAULT) instanceof FileCachedIndexInput);
+
+        // File present in Remote
+        assertFalse(existsInLocalDirectory(FILE_PRESENT_IN_REMOTE_ONLY));
+        assertTrue(existsInRemoteDirectory(FILE_PRESENT_IN_REMOTE_ONLY));
+        assertTrue(compositeDirectory.openInput(FILE_PRESENT_IN_REMOTE_ONLY, IOContext.DEFAULT) instanceof OnDemandBlockSnapshotIndexInput);
+    }
+
+    public void testClose() throws IOException {
+        // Similar to delete, when close is called existing openInput should be able to function properly but new requests should not be
+        // served
+        IndexInput indexInput = compositeDirectory.openInput(FILE_PRESENT_LOCALLY, IOContext.DEFAULT);
+        compositeDirectory.close();
+        // Any operations after close will throw AlreadyClosedException
+        assertThrows(AlreadyClosedException.class, () -> compositeDirectory.openInput(FILE_PRESENT_LOCALLY, IOContext.DEFAULT));
+        // Existing open IndexInputs will be served
+        indexInput.getFilePointer();
+        indexInput.close();
+        assertThrows(RuntimeException.class, indexInput::getFilePointer);
+        assertThrows(AlreadyClosedException.class, () -> compositeDirectory.close());
+    }
+
+    public void testAfterSyncToRemote() throws IOException {
+        // File will be present locally until uploaded to Remote
+        assertTrue(existsInLocalDirectory(FILE_PRESENT_LOCALLY));
+        compositeDirectory.afterSyncToRemote(FILE_PRESENT_LOCALLY);
+        fileCache.prune();
+        // After uploading to Remote, refCount will be decreased by 1 making it 0 and will be evicted if cache is pruned
+        assertFalse(existsInLocalDirectory(FILE_PRESENT_LOCALLY));
+        // Asserting file is not present in FileCache
+        assertNull(fileCache.get(getFilePath(FILE_PRESENT_LOCALLY)));
+    }
+
+    private void addFilesToDirectory(String[] files) throws IOException {
+        for (String file : files) {
+            IndexOutput indexOutput = compositeDirectory.createOutput(file, IOContext.DEFAULT);
+            indexOutput.close();
+        }
+    }
+
+    private void removeExtraFSFiles() throws IOException {
+        HashSet<String> allFiles = new HashSet<>(Arrays.asList(localDirectory.listAll()));
+        allFiles.stream().filter(FileTypeUtils::isExtraFSFile).forEach(file -> {
+            try {
+                localDirectory.deleteFile(file);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    private boolean existsInLocalDirectory(String name) throws IOException {
+        return Arrays.asList(localDirectory.listAll()).contains(name);
+    }
+
+    private boolean existsInRemoteDirectory(String name) throws IOException {
+        return Arrays.asList(remoteSegmentStoreDirectory.listAll()).contains(name);
+    }
+
+    private boolean existsInCompositeDirectory(String name) throws IOException {
+        return Arrays.asList(compositeDirectory.listAll()).contains(name);
+    }
+
+    private Path getFilePath(String name) {
+        return localDirectory.getDirectory().resolve(name);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
index ee81369725e6f..ed79a2b0bd8e4 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
@@ -20,7 +20,9 @@
 import org.opensearch.common.blobstore.BlobMetadata;
 import org.opensearch.common.blobstore.stream.write.WriteContext;
 import org.opensearch.common.blobstore.support.PlainBlobMetadata;
+import org.opensearch.common.lucene.store.ByteArrayIndexInput;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.index.store.remote.utils.BlockIOContext;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
@@ -216,6 +218,17 @@ public void testOpenInput() throws IOException {
         assertTrue(indexInput instanceof RemoteIndexInput);
         assertEquals(100, indexInput.length());
         verify(blobContainer).listBlobsByPrefixInSortedOrder("segment_1", 1, LEXICOGRAPHIC);
+
+        BlockIOContext blockIOContextInvalidValues = new BlockIOContext(IOContext.DEFAULT, 10, 1000);
+        assertThrows(IllegalArgumentException.class, () -> remoteDirectory.openInput("segment_1", blockIOContextInvalidValues));
+
+        BlockIOContext blockIOContext = new BlockIOContext(IOContext.DEFAULT, 10, 50);
+        when(blobContainer.readBlob("segment_1", 10, 50)).thenReturn(mockInputStream);
+        byte[] bytes = new byte[(int) blockIOContext.getBlockSize()];
+        when(mockInputStream.readAllBytes()).thenReturn(bytes);
+        indexInput = remoteDirectory.openInput("segment_1", blockIOContext);
+        assertTrue(indexInput instanceof ByteArrayIndexInput);
+        assertEquals(blockIOContext.getBlockSize(), indexInput.length());
     }
 
     public void testOpenInputWithLength() throws IOException {
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
index 567199cf64cd8..574c5bf620474 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
@@ -10,7 +10,6 @@
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexFormatTooNewException;
@@ -23,34 +22,25 @@
 import org.apache.lucene.store.OutputStreamIndexOutput;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.Version;
-import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
 import org.opensearch.common.blobstore.stream.write.WriteContext;
 import org.opensearch.common.io.VersionedCodecStreamWrapper;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.store.ByteArrayIndexInput;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.index.engine.NRTReplicationEngineFactory;
 import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
 import org.opensearch.index.remote.RemoteStoreEnums.PathType;
 import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteStoreUtils;
-import org.opensearch.index.shard.IndexShard;
-import org.opensearch.index.shard.IndexShardTestCase;
-import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager;
 import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata;
 import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler;
-import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.MockLogAppender;
 import org.opensearch.test.junit.annotations.TestLogging;
 import org.opensearch.threadpool.ThreadPool;
-import org.junit.After;
 import org.junit.Before;
 
 import java.io.ByteArrayInputStream;
@@ -64,7 +54,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -87,95 +76,11 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class RemoteSegmentStoreDirectoryTests extends IndexShardTestCase {
-    private static final Logger logger = LogManager.getLogger(RemoteSegmentStoreDirectoryTests.class);
-    private RemoteDirectory remoteDataDirectory;
-    private RemoteDirectory remoteMetadataDirectory;
-    private RemoteStoreMetadataLockManager mdLockManager;
-
-    private RemoteSegmentStoreDirectory remoteSegmentStoreDirectory;
-    private TestUploadListener testUploadTracker;
-    private IndexShard indexShard;
-    private SegmentInfos segmentInfos;
-    private ThreadPool threadPool;
-
-    private final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
-        12,
-        23,
-        34,
-        1,
-        1,
-        "node-1"
-    );
-
-    private final String metadataFilenameDup = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
-        12,
-        23,
-        34,
-        2,
-        1,
-        "node-2"
-    );
-    private final String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
-        12,
-        13,
-        34,
-        1,
-        1,
-        "node-1"
-    );
-    private final String metadataFilename3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
-        10,
-        38,
-        34,
-        1,
-        1,
-        "node-1"
-    );
-    private final String metadataFilename4 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
-        10,
-        36,
-        34,
-        1,
-        1,
-        "node-1"
-    );
+public class RemoteSegmentStoreDirectoryTests extends BaseRemoteSegmentStoreDirectoryTests {
 
     @Before
     public void setup() throws IOException {
-        remoteDataDirectory = mock(RemoteDirectory.class);
-        remoteMetadataDirectory = mock(RemoteDirectory.class);
-        mdLockManager = mock(RemoteStoreMetadataLockManager.class);
-        threadPool = mock(ThreadPool.class);
-        testUploadTracker = new TestUploadListener();
-
-        Settings indexSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT)
-            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
-            .build();
-        ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService();
-
-        indexShard = newStartedShard(false, indexSettings, new NRTReplicationEngineFactory());
-        remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory(
-            remoteDataDirectory,
-            remoteMetadataDirectory,
-            mdLockManager,
-            threadPool,
-            indexShard.shardId()
-        );
-        try (Store store = indexShard.store()) {
-            segmentInfos = store.readLastCommittedSegmentsInfo();
-        }
-
-        when(threadPool.executor(ThreadPool.Names.REMOTE_PURGE)).thenReturn(executorService);
-        when(threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY)).thenReturn(executorService);
-        when(threadPool.executor(ThreadPool.Names.SAME)).thenReturn(executorService);
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        indexShard.close("test tearDown", true, false);
-        super.tearDown();
+        setupRemoteSegmentStoreDirectory();
     }
 
     public void testUploadedSegmentMetadataToString() {
@@ -256,60 +161,6 @@ public void testInitMultipleMetadataFile() throws IOException {
         assertThrows(IllegalStateException.class, () -> remoteSegmentStoreDirectory.init());
     }
 
-    private Map<String, Map<String, String>> populateMetadata() throws IOException {
-        List<String> metadataFiles = new ArrayList<>();
-
-        metadataFiles.add(metadataFilename);
-        metadataFiles.add(metadataFilename2);
-        metadataFiles.add(metadataFilename3);
-
-        when(
-            remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
-                RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
-                METADATA_FILES_TO_FETCH
-            )
-        ).thenReturn(List.of(metadataFilename));
-        when(
-            remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
-                RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
-                Integer.MAX_VALUE
-            )
-        ).thenReturn(metadataFiles);
-
-        Map<String, Map<String, String>> metadataFilenameContentMapping = Map.of(
-            metadataFilename,
-            getDummyMetadata("_0", 1),
-            metadataFilename2,
-            getDummyMetadata("_0", 1),
-            metadataFilename3,
-            getDummyMetadata("_0", 1)
-        );
-
-        when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenAnswer(
-            I -> createMetadataFileBytes(
-                metadataFilenameContentMapping.get(metadataFilename),
-                indexShard.getLatestReplicationCheckpoint(),
-                segmentInfos
-            )
-        );
-        when(remoteMetadataDirectory.getBlobStream(metadataFilename2)).thenAnswer(
-            I -> createMetadataFileBytes(
-                metadataFilenameContentMapping.get(metadataFilename2),
-                indexShard.getLatestReplicationCheckpoint(),
-                segmentInfos
-            )
-        );
-        when(remoteMetadataDirectory.getBlobStream(metadataFilename3)).thenAnswer(
-            I -> createMetadataFileBytes(
-                metadataFilenameContentMapping.get(metadataFilename3),
-                indexShard.getLatestReplicationCheckpoint(),
-                segmentInfos
-            )
-        );
-
-        return metadataFilenameContentMapping;
-    }
-
     public void testInit() throws IOException {
         populateMetadata();
 
diff --git a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java
index a135802c5f49c..c7d0cc0c5b96e 100644
--- a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java
+++ b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java
@@ -207,7 +207,7 @@ private void initBlockFiles(int blockSize, FSDirectory fsDirectory) {
             // write 48, -80 alternatively
             for (int i = 0; i < numOfBlocks; i++) {
                 // create normal blocks
-                String blockName = BLOCK_FILE_PREFIX + "." + i;
+                String blockName = BLOCK_FILE_PREFIX + "_block_" + i;
                 IndexOutput output = fsDirectory.createOutput(blockName, null);
                 // since block size is always even number, safe to do division
                 for (int j = 0; j < blockSize / 2; j++) {
@@ -221,7 +221,7 @@ private void initBlockFiles(int blockSize, FSDirectory fsDirectory) {
 
             if (numOfBlocks > 1 && sizeOfLastBlock != 0) {
                 // create last block
-                String lastBlockName = BLOCK_FILE_PREFIX + "." + numOfBlocks;
+                String lastBlockName = BLOCK_FILE_PREFIX + "_block_" + numOfBlocks;
                 IndexOutput output = fsDirectory.createOutput(lastBlockName, null);
                 for (int i = 0; i < sizeOfLastBlock; i++) {
                     if ((i & 1) == 0) {
diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java
new file mode 100644
index 0000000000000..258bc2db4c5d0
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java
@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.filecache;
+
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public class FileCachedIndexInputTests extends OpenSearchTestCase {
+
+    protected FileCache fileCache;
+    protected Path filePath;
+    protected IndexInput underlyingIndexInput;
+    private FileCachedIndexInput fileCachedIndexInput;
+
+    protected static final int FILE_CACHE_CAPACITY = 1000;
+    protected static final String TEST_FILE = "test_file";
+    protected static final String SLICE_DESC = "slice_description";
+
+    @Before
+    public void setup() throws IOException {
+        Path basePath = createTempDir("FileCachedIndexInputTests");
+        FSDirectory fsDirectory = FSDirectory.open(basePath);
+        IndexOutput indexOutput = fsDirectory.createOutput(TEST_FILE, IOContext.DEFAULT);
+        // Writing to the file so that it's size is not zero
+        indexOutput.writeInt(100);
+        indexOutput.close();
+        filePath = basePath.resolve(TEST_FILE);
+        underlyingIndexInput = fsDirectory.openInput(TEST_FILE, IOContext.DEFAULT);
+        fileCache = FileCacheFactory.createConcurrentLRUFileCache(FILE_CACHE_CAPACITY, new NoopCircuitBreaker(CircuitBreaker.REQUEST));
+    }
+
+    protected void setupIndexInputAndAddToFileCache() {
+        fileCachedIndexInput = new FileCachedIndexInput(fileCache, filePath, underlyingIndexInput);
+        fileCache.put(filePath, new CachedFullFileIndexInput(fileCache, filePath, fileCachedIndexInput));
+    }
+
+    public void testClone() throws IOException {
+        setupIndexInputAndAddToFileCache();
+
+        // Since the file ia already in cache and has refCount 1, activeUsage and totalUsage will be same
+        assertTrue(isActiveAndTotalUsageSame());
+
+        // Decrementing the refCount explicitly on the file which will make it inactive (as refCount will drop to 0)
+        fileCache.decRef(filePath);
+        assertFalse(isActiveAndTotalUsageSame());
+
+        // After cloning the refCount will increase again and activeUsage and totalUsage will be same again
+        FileCachedIndexInput clonedFileCachedIndexInput = fileCachedIndexInput.clone();
+        assertTrue(isActiveAndTotalUsageSame());
+
+        // Closing the clone will again decrease the refCount making it 0
+        clonedFileCachedIndexInput.close();
+        assertFalse(isActiveAndTotalUsageSame());
+    }
+
+    public void testSlice() throws IOException {
+        setupIndexInputAndAddToFileCache();
+        assertThrows(UnsupportedOperationException.class, () -> fileCachedIndexInput.slice(SLICE_DESC, 10, 100));
+    }
+
+    protected boolean isActiveAndTotalUsageSame() {
+        return fileCache.usage().activeUsage() == fileCache.usage().usage();
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java
new file mode 100644
index 0000000000000..7fb7a03584e20
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java
@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.filecache;
+
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.IndexInput;
+
+import java.io.IOException;
+
+public class FullFileCachedIndexInputTests extends FileCachedIndexInputTests {
+    private FullFileCachedIndexInput fullFileCachedIndexInput;
+
+    @Override
+    protected void setupIndexInputAndAddToFileCache() {
+        fullFileCachedIndexInput = new FullFileCachedIndexInput(fileCache, filePath, underlyingIndexInput);
+        fileCache.put(filePath, new CachedFullFileIndexInput(fileCache, filePath, fullFileCachedIndexInput));
+    }
+
+    @Override
+    public void testClone() throws IOException {
+        setupIndexInputAndAddToFileCache();
+
+        // Since the file is already in cache and has refCount 1, activeUsage and totalUsage will be same
+        assertTrue(isActiveAndTotalUsageSame());
+
+        // Getting the file cache entry (which wil increase the ref count, hence doing dec ref immediately afterwards)
+        CachedIndexInput cachedIndexInput = fileCache.get(filePath);
+        fileCache.decRef(filePath);
+
+        // Decrementing the refCount explicitly on the file which will make it inactive (as refCount will drop to 0)
+        fileCache.decRef(filePath);
+        assertFalse(isActiveAndTotalUsageSame());
+
+        // After cloning the refCount will increase again and activeUsage and totalUsage will be same again
+        FileCachedIndexInput clonedFileCachedIndexInput1 = fullFileCachedIndexInput.clone();
+        FileCachedIndexInput clonedFileCachedIndexInput2 = clonedFileCachedIndexInput1.clone();
+        FileCachedIndexInput clonedFileCachedIndexInput3 = clonedFileCachedIndexInput2.clone();
+        assertTrue(isActiveAndTotalUsageSame());
+
+        // closing the first level clone will close all subsequent level clones and reduce ref count to 0
+        clonedFileCachedIndexInput1.close();
+        assertFalse(isActiveAndTotalUsageSame());
+
+        fileCache.prune();
+
+        // since the file cache entry was evicted the corresponding CachedIndexInput will be closed and will throw exception when trying to
+        // read the index input
+        assertThrows(AlreadyClosedException.class, cachedIndexInput::getIndexInput);
+    }
+
+    @Override
+    public void testSlice() throws IOException {
+        setupIndexInputAndAddToFileCache();
+
+        // Throw IllegalArgumentException if offset is negative
+        assertThrows(IllegalArgumentException.class, () -> fullFileCachedIndexInput.slice(SLICE_DESC, -1, 10));
+
+        // Throw IllegalArgumentException if length is negative
+        assertThrows(IllegalArgumentException.class, () -> fullFileCachedIndexInput.slice(SLICE_DESC, 5, -1));
+
+        // Decrementing the refCount explicitly on the file which will make it inactive (as refCount will drop to 0)
+        fileCache.decRef(filePath);
+        assertFalse(isActiveAndTotalUsageSame());
+
+        // Creating a slice will increase the refCount
+        IndexInput slicedFileCachedIndexInput = fullFileCachedIndexInput.slice(SLICE_DESC, 1, 2);
+        assertTrue(isActiveAndTotalUsageSame());
+
+        // Closing the parent will close all the slices as well decreasing the refCount to 0
+        fullFileCachedIndexInput.close();
+        assertFalse(isActiveAndTotalUsageSame());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java
new file mode 100644
index 0000000000000..24e57cde3ffda
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.utils;
+
+import org.opensearch.common.blobstore.BlobContainer;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+
+public class TransferManagerBlobContainerReaderTests extends TransferManagerTestCase {
+    private BlobContainer blobContainer;
+
+    @Override
+    protected void initializeTransferManager() throws IOException {
+        blobContainer = mock(BlobContainer.class);
+        doAnswer(i -> new ByteArrayInputStream(createData())).when(blobContainer).readBlob(eq("blob"), anyLong(), anyLong());
+        transferManager = new TransferManager(blobContainer::readBlob, fileCache);
+    }
+
+    protected void mockExceptionWhileReading() throws IOException {
+        doThrow(new IOException("Expected test exception")).when(blobContainer).readBlob(eq("failure-blob"), anyLong(), anyLong());
+    }
+
+    protected void mockWaitForLatchReader(CountDownLatch latch) throws IOException {
+        doAnswer(i -> {
+            latch.await();
+            return new ByteArrayInputStream(createData());
+        }).when(blobContainer).readBlob(eq("blocking-blob"), anyLong(), anyLong());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java
new file mode 100644
index 0000000000000..e777a287bf10f
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.store.remote.utils;
+
+import org.apache.lucene.store.IOContext;
+import org.opensearch.common.lucene.store.ByteArrayIndexInput;
+import org.opensearch.common.lucene.store.InputStreamIndexInput;
+import org.opensearch.index.store.RemoteDirectory;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+
+public class TransferManagerRemoteDirectoryReaderTests extends TransferManagerTestCase {
+    private RemoteDirectory remoteDirectory;
+
+    @Override
+    protected void initializeTransferManager() throws IOException {
+        remoteDirectory = mock(RemoteDirectory.class);
+        doAnswer(i -> new ByteArrayIndexInput("blob", createData())).when(remoteDirectory).openInput(eq("blob"), any());
+        transferManager = new TransferManager(
+            (name, position, length) -> new InputStreamIndexInput(
+                remoteDirectory.openInput(name, new BlockIOContext(IOContext.DEFAULT, position, length)),
+                length
+            ),
+            fileCache
+        );
+    }
+
+    protected void mockExceptionWhileReading() throws IOException {
+        doThrow(new IOException("Expected test exception")).when(remoteDirectory).openInput(eq("failure-blob"), any());
+    }
+
+    protected void mockWaitForLatchReader(CountDownLatch latch) throws IOException {
+        doAnswer(i -> {
+            latch.await();
+            return new ByteArrayIndexInput("blocking-blob", createData());
+        }).when(remoteDirectory).openInput(eq("blocking-blob"), any());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java
similarity index 87%
rename from server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java
rename to server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java
index 7ae3944eb6944..810a4c336fdf7 100644
--- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.SimpleFSLockFactory;
-import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.core.common.breaker.CircuitBreaker;
 import org.opensearch.core.common.breaker.NoopCircuitBreaker;
 import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
@@ -24,7 +23,6 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -36,31 +34,23 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
 
 @ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class)
-public class TransferManagerTests extends OpenSearchTestCase {
-    private static final int EIGHT_MB = 1024 * 1024 * 8;
-    private final FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(
+public abstract class TransferManagerTestCase extends OpenSearchTestCase {
+    protected static final int EIGHT_MB = 1024 * 1024 * 8;
+    protected final FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(
         EIGHT_MB * 2,
         1,
         new NoopCircuitBreaker(CircuitBreaker.REQUEST)
     );
-    private MMapDirectory directory;
-    private BlobContainer blobContainer;
-    private TransferManager transferManager;
+    protected MMapDirectory directory;
+    protected TransferManager transferManager;
 
     @Before
     public void setUp() throws Exception {
         super.setUp();
         directory = new MMapDirectory(createTempDir(), SimpleFSLockFactory.INSTANCE);
-        blobContainer = mock(BlobContainer.class);
-        doAnswer(i -> new ByteArrayInputStream(createData())).when(blobContainer).readBlob(eq("blob"), anyLong(), anyLong());
-        transferManager = new TransferManager(blobContainer, fileCache);
+        initializeTransferManager();
     }
 
     @After
@@ -68,7 +58,7 @@ public void tearDown() throws Exception {
         super.tearDown();
     }
 
-    private static byte[] createData() {
+    protected static byte[] createData() {
         final byte[] data = new byte[EIGHT_MB];
         data[EIGHT_MB - 1] = 7;
         return data;
@@ -162,7 +152,7 @@ public void testUsageExceedsCapacity() throws Exception {
     }
 
     public void testDownloadFails() throws Exception {
-        doThrow(new IOException("Expected test exception")).when(blobContainer).readBlob(eq("failure-blob"), anyLong(), anyLong());
+        mockExceptionWhileReading();
         List<BlobFetchRequest.BlobPart> blobParts = new ArrayList<>();
         blobParts.add(new BlobFetchRequest.BlobPart("failure-blob", 0, EIGHT_MB));
         expectThrows(
@@ -177,10 +167,7 @@ public void testFetchesToDifferentBlobsDoNotBlockOnEachOther() throws Exception
         // Mock a call for a blob that will block until the latch is released,
         // then start the fetch for that blob on a separate thread
         final CountDownLatch latch = new CountDownLatch(1);
-        doAnswer(i -> {
-            latch.await();
-            return new ByteArrayInputStream(createData());
-        }).when(blobContainer).readBlob(eq("blocking-blob"), anyLong(), anyLong());
+        mockWaitForLatchReader(latch);
         List<BlobFetchRequest.BlobPart> blobParts = new ArrayList<>();
         blobParts.add(new BlobFetchRequest.BlobPart("blocking-blob", 0, EIGHT_MB));
 
@@ -206,6 +193,12 @@ public void testFetchesToDifferentBlobsDoNotBlockOnEachOther() throws Exception
         assertFalse(blockingThread.isAlive());
     }
 
+    protected abstract void initializeTransferManager() throws IOException;
+
+    protected abstract void mockExceptionWhileReading() throws IOException;
+
+    protected abstract void mockWaitForLatchReader(CountDownLatch latch) throws IOException;
+
     private IndexInput fetchBlobWithName(String blobname) throws IOException {
         List<BlobFetchRequest.BlobPart> blobParts = new ArrayList<>();
         blobParts.add(new BlobFetchRequest.BlobPart("blob", 0, EIGHT_MB));
diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/cache/RefCountedCacheTestCase.java b/server/src/test/java/org/opensearch/index/store/remote/utils/cache/RefCountedCacheTestCase.java
index b11740b53e11f..97e9fb288136d 100644
--- a/server/src/test/java/org/opensearch/index/store/remote/utils/cache/RefCountedCacheTestCase.java
+++ b/server/src/test/java/org/opensearch/index/store/remote/utils/cache/RefCountedCacheTestCase.java
@@ -211,6 +211,15 @@ public void testComputeStats() {
         assertEquals(1, refCountedCache.stats().removeCount());
     }
 
+    public void testClear() {
+        refCountedCache.put("1", 10L);
+        refCountedCache.put("2", 10L);
+        refCountedCache.put("3", 10L);
+        assertEquals(30L, refCountedCache.usage().usage());
+        refCountedCache.clear();
+        assertEquals(0L, refCountedCache.usage().usage());
+    }
+
     private void assertUsage(long usage, long activeUsage) {
         assertEquals(usage, refCountedCache.usage().usage());
         assertEquals(activeUsage, refCountedCache.usage().activeUsage());
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index 0eca08a7678ae..71ab56c98312a 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -2077,6 +2077,10 @@ protected boolean addMockTransportService() {
         return true;
     }
 
+    protected boolean addMockIndexStorePlugin() {
+        return true;
+    }
+
     /** Returns {@code true} iff this test cluster should use a dummy http transport */
     protected boolean addMockHttpTransport() {
         return true;
@@ -2119,7 +2123,7 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
             if (randomBoolean() && addMockTransportService()) {
                 mocks.add(MockTransportService.TestPlugin.class);
             }
-            if (randomBoolean()) {
+            if (randomBoolean() && addMockIndexStorePlugin()) {
                 mocks.add(MockFSIndexStore.TestPlugin.class);
             }
             if (randomBoolean()) {

From 9675c4f6ec0d412993ef361bce44a8b789bff27b Mon Sep 17 00:00:00 2001
From: Thomas Farr <tsfarr@amazon.com>
Date: Fri, 21 Jun 2024 08:32:54 +1200
Subject: [PATCH 031/113] Correct RHLC searchTemplate & mtermVectors endpoints
 to have leading slash (#14465)

Signed-off-by: Thomas Farr <tsfarr@amazon.com>
---
 CHANGELOG.md                                              | 1 +
 .../java/org/opensearch/client/RequestConverters.java     | 5 ++---
 .../test/java/org/opensearch/client/IndicesClientIT.java  | 8 ++++----
 .../src/test/java/org/opensearch/client/RankEvalIT.java   | 2 +-
 .../org/opensearch/client/RequestConvertersTests.java     | 6 +++---
 .../src/test/java/org/opensearch/client/SearchIT.java     | 4 ++--
 .../client/documentation/SearchDocumentationIT.java       | 2 +-
 .../documentation/SnapshotClientDocumentationIT.java      | 2 +-
 8 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 47244ba9afbb1..6138e6dd969b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
 - Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
+- Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
 
 ### Security
 
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
index cf01febbd33cd..3546776fa3617 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
@@ -540,7 +540,7 @@ static Request searchTemplate(SearchTemplateRequest searchTemplateRequest) throw
         Request request;
 
         if (searchTemplateRequest.isSimulate()) {
-            request = new Request(HttpGet.METHOD_NAME, "_render/template");
+            request = new Request(HttpGet.METHOD_NAME, "/_render/template");
         } else {
             SearchRequest searchRequest = searchTemplateRequest.getRequest();
             String endpoint = endpoint(searchRequest.indices(), "_search/template");
@@ -803,8 +803,7 @@ static Request termVectors(TermVectorsRequest tvrequest) throws IOException {
     }
 
     static Request mtermVectors(MultiTermVectorsRequest mtvrequest) throws IOException {
-        String endpoint = "_mtermvectors";
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Request request = new Request(HttpGet.METHOD_NAME, "/_mtermvectors");
         request.setEntity(createEntity(mtvrequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java
index c8eafb88b7495..ccdcc21f0fc8b 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java
@@ -701,7 +701,7 @@ public void testOpenExistingIndex() throws IOException {
         closeIndex(index);
         ResponseException exception = expectThrows(
             ResponseException.class,
-            () -> client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search"))
+            () -> client().performRequest(new Request(HttpGet.METHOD_NAME, "/" + index + "/_search"))
         );
         assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
         assertThat(exception.getMessage().contains(index), equalTo(true));
@@ -714,7 +714,7 @@ public void testOpenExistingIndex() throws IOException {
         );
         assertTrue(openIndexResponse.isAcknowledged());
 
-        Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search"));
+        Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, "/" + index + "/_search"));
         assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
     }
 
@@ -771,7 +771,7 @@ public void testCloseExistingIndex() throws IOException {
 
             ResponseException exception = expectThrows(
                 ResponseException.class,
-                () -> client().performRequest(new Request(HttpGet.METHOD_NAME, indexResult.getIndex() + "/_search"))
+                () -> client().performRequest(new Request(HttpGet.METHOD_NAME, "/" + indexResult.getIndex() + "/_search"))
             );
             assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
             assertThat(exception.getMessage().contains(indexResult.getIndex()), equalTo(true));
@@ -1270,7 +1270,7 @@ public void testGetAliasesNonExistentIndexOrAlias() throws IOException {
             assertThat(getAliasesResponse.getException(), nullValue());
         }
         createIndex(index, Settings.EMPTY);
-        client().performRequest(new Request(HttpPut.METHOD_NAME, index + "/_alias/" + alias));
+        client().performRequest(new Request(HttpPut.METHOD_NAME, "/" + index + "/_alias/" + alias));
         {
             GetAliasesRequest getAliasesRequest = new GetAliasesRequest().indices(index, "non_existent_index");
             GetAliasesResponse getAliasesResponse = execute(
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java
index 47add92ecaccd..01fdd489aa7d8 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java
@@ -121,7 +121,7 @@ public void testRankEvalRequest() throws IOException {
         }
 
         // now try this when test2 is closed
-        client().performRequest(new Request("POST", "index2/_close"));
+        client().performRequest(new Request("POST", "/index2/_close"));
         rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
     }
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java
index 084d754275dec..38f5d9302440e 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java
@@ -1399,7 +1399,7 @@ public void testRenderSearchTemplate() throws Exception {
 
         // Verify that the resulting REST request looks as expected.
         Request request = RequestConverters.searchTemplate(searchTemplateRequest);
-        String endpoint = "_render/template";
+        String endpoint = "/_render/template";
 
         assertEquals(HttpGet.METHOD_NAME, request.getMethod());
         assertEquals(endpoint, request.getEndpoint());
@@ -1565,7 +1565,7 @@ public void testMultiTermVectors() throws IOException {
 
         Request request = RequestConverters.mtermVectors(mtvRequest);
         assertEquals(HttpGet.METHOD_NAME, request.getMethod());
-        assertEquals("_mtermvectors", request.getEndpoint());
+        assertEquals("/_mtermvectors", request.getEndpoint());
         assertToXContentBody(mtvRequest, request.getEntity());
     }
 
@@ -1585,7 +1585,7 @@ public void testMultiTermVectorsWithType() throws IOException {
 
         Request request = RequestConverters.mtermVectors(mtvRequest);
         assertEquals(HttpGet.METHOD_NAME, request.getMethod());
-        assertEquals("_mtermvectors", request.getEndpoint());
+        assertEquals("/_mtermvectors", request.getEndpoint());
         assertToXContentBody(mtvRequest, request.getEntity());
     }
 
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
index c1f1cbf1d0e91..d10dc3df43ee5 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
@@ -727,7 +727,7 @@ public void testSearchWithSuggest() throws IOException {
     }
 
     public void testSearchWithWeirdScriptFields() throws Exception {
-        Request doc = new Request("PUT", "test/_doc/1");
+        Request doc = new Request("PUT", "/test/_doc/1");
         doc.setJsonEntity("{\"field\":\"value\"}");
         client().performRequest(doc);
         client().performRequest(new Request("POST", "/test/_refresh"));
@@ -774,7 +774,7 @@ public void testSearchWithWeirdScriptFields() throws Exception {
     public void testSearchWithDerivedFields() throws Exception {
         // Just testing DerivedField definition from SearchSourceBuilder derivedField()
         // We are not testing the full functionality here
-        Request doc = new Request("PUT", "test/_doc/1");
+        Request doc = new Request("PUT", "/test/_doc/1");
         doc.setJsonEntity("{\"field\":\"value\"}");
         client().performRequest(doc);
         client().performRequest(new Request("POST", "/test/_refresh"));
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java
index bf0f70304168e..326dde54cfb61 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java
@@ -998,7 +998,7 @@ public void onFailure(Exception e) {
 
     protected void registerQueryScript(RestClient restClient) throws IOException {
         // tag::register-script
-        Request scriptRequest = new Request("POST", "_scripts/title_search");
+        Request scriptRequest = new Request("POST", "/_scripts/title_search");
         scriptRequest.setJsonEntity(
             "{" +
             "  \"script\": {" +
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java
index d0015db044843..6949bc382bfe8 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java
@@ -827,7 +827,7 @@ private void createTestIndex() throws IOException {
     }
 
     private void createTestSnapshots() throws IOException {
-        Request createSnapshot = new Request("put", String.format(Locale.ROOT, "_snapshot/%s/%s", repositoryName, snapshotName));
+        Request createSnapshot = new Request("put", String.format(Locale.ROOT, "/_snapshot/%s/%s", repositoryName, snapshotName));
         createSnapshot.addParameter("wait_for_completion", "true");
         createSnapshot.setJsonEntity("{\"indices\":\"" + indexName + "\"}");
         Response response = highLevelClient().getLowLevelClient().performRequest(createSnapshot);

From e8b7913926a9428a0a737569e6df44cfeff3defb Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 20 Jun 2024 14:17:23 -0700
Subject: [PATCH 032/113] Update IndicesRequestCacheCleanupIT.java (#14478)

Signed-off-by: Kiran Prakash <awskiran@amazon.com>
---
 .../indices/IndicesRequestCacheCleanupIT.java | 48 +++++++------------
 1 file changed, 17 insertions(+), 31 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java
index 6f698c5816278..988ea99130b13 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java
@@ -66,6 +66,8 @@
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
 public class IndicesRequestCacheCleanupIT extends OpenSearchIntegTestCase {
 
+    private static final long MAX_ITERATIONS = 5;
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(InternalSettingsPlugin.class);
@@ -74,23 +76,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     public void testCacheWithInvalidation() throws Exception {
         Client client = client();
         String index = "index";
-        assertAcked(
-            client.admin()
-                .indices()
-                .prepareCreate(index)
-                .setMapping("k", "type=keyword")
-                .setSettings(
-                    Settings.builder()
-                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                        .put("index.refresh_interval", -1)
-                        // Disable index refreshing to avoid cache being invalidated mid-test
-                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
-                )
-                .get()
-        );
-        indexRandom(false, client.prepareIndex(index).setSource("k", "hello"));
+        setupIndex(client, index);
         ensureSearchable(index);
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
         forceMerge(client, index);
@@ -125,8 +111,8 @@ public void testCacheClearAPIRemovesStaleKeysWhenStalenessThresholdIsLow() throw
                 .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
                 .put(
                     IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    // setting intentionally high to avoid cache cleaner interfering
-                    TimeValue.timeValueMillis(300)
+                    // Set interval much larger than test timeout to effectively disable it
+                    TimeValue.timeValueDays(1)
                 )
         );
         Client client = client(node);
@@ -210,7 +196,7 @@ public void testStaleKeysCleanupWithLowThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
         // sleep until cache cleaner would have cleaned up the stale key from index 2
     }
 
@@ -260,7 +246,7 @@ public void testCacheCleanupOnEqualStalenessAndThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is higher than staleness, it should NOT clean the cache
@@ -308,7 +294,7 @@ public void testCacheCleanupSkipsWithHighStalenessThreshold() throws Exception {
             assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is explicitly set to 0, cache cleaner regularly cleans up stale keys.
@@ -356,7 +342,7 @@ public void testCacheCleanupOnZeroStalenessThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is not explicitly set, cache cleaner regularly cleans up stale keys
@@ -403,7 +389,7 @@ public void testStaleKeysRemovalWithoutExplicitThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when cache cleaner interval setting is not set, cache cleaner is configured appropriately with the fall-back setting
@@ -447,7 +433,7 @@ public void testCacheCleanupWithDefaultSettings() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // staleness threshold updates flows through to the cache cleaner
@@ -490,7 +476,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception {
         assertBusy(() -> {
             // cache cleaner should NOT have cleaned up the stale key from index 2
             assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
 
         // Update indices.requests.cache.cleanup.staleness_threshold to "10%"
         ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
@@ -505,7 +491,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // staleness threshold dynamic updates should throw exceptions on invalid input
@@ -557,7 +543,7 @@ public void testCacheClearanceAfterIndexClosure() throws Exception {
         assertBusy(() -> {
             // cache cleaner should have cleaned up the stale keys from index
             assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // deleting the Index after caching will clean up from Indices Request Cache
@@ -598,7 +584,7 @@ public void testCacheCleanupAfterIndexDeletion() throws Exception {
         assertBusy(() -> {
             // cache cleaner should have cleaned up the stale keys from index
             assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is lower than staleness, it should clean the cache from all indices having stale keys
@@ -645,7 +631,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
         // Assert cache is cleared up
         assertBusy(
             () -> { assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes()); },
-            cacheCleanIntervalInMillis * 2,
+            cacheCleanIntervalInMillis * MAX_ITERATIONS,
             TimeUnit.MILLISECONDS
         );
 
@@ -667,7 +653,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
             long currentMemorySizeInBytesForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
             // assert the memory size of index1 to only contain 1 entry added after flushAndRefresh
             assertEquals(memorySizeForIndex1With1Entries, currentMemorySizeInBytesForIndex1);
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     private void setupIndex(Client client, String index) throws Exception {

From 3a516cf55f901076f00a590bb1e7ffe281b0c3c4 Mon Sep 17 00:00:00 2001
From: Shivansh Arora <hishiv@amazon.com>
Date: Fri, 21 Jun 2024 08:48:16 +0530
Subject: [PATCH 033/113] Fix component name for ClusterState custom (#14477)

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
---
 .../opensearch/gateway/remote/RemoteClusterStateService.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 4a1c9c8615e39..ef14adeb42b68 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -1160,7 +1160,8 @@ private ClusterState readClusterStateInParallel(
         for (Map.Entry<String, UploadedMetadataAttribute> entry : clusterStateCustomToRead.entrySet()) {
             asyncMetadataReadActions.add(
                 remoteClusterStateAttributesManager.getAsyncMetadataReadAction(
-                    CLUSTER_STATE_CUSTOM,
+                    // pass component name as cluster-state-custom--<custom_name>, so that we can interpret it later
+                    String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, entry.getKey()),
                     new RemoteClusterStateCustoms(
                         entry.getValue().getUploadedFilename(),
                         entry.getValue().getAttributeName(),

From 3106ea5d83eff5f7f6ee9565147eba0bd4cad34c Mon Sep 17 00:00:00 2001
From: Daniil Roman <danroman17397@gmail.com>
Date: Fri, 21 Jun 2024 15:17:38 +0200
Subject: [PATCH 034/113] Adjusting TESTING.md for a single test run (#14441)

* Adjusting TESTING.md for a single test run

Signed-off-by: Daniil Roman <daniilroman.cv@gmail.com>

* Adjusting TESTING.md for a single test run

Signed-off-by: Daniil Roman <danroman17397@gmail.com>

* Update TESTING.md

Co-authored-by: Andriy Redko <drreta@gmail.com>
Signed-off-by: Daniil Roman <danroman17397@gmail.com>

* Adjusting TESTING.md for a single test run

Signed-off-by: Daniil Roman <danroman17397@gmail.com>

* Adjusting TESTING.md for a single test run

Signed-off-by: Daniil Roman <danroman17397@gmail.com>

---------

Signed-off-by: Daniil Roman <daniilroman.cv@gmail.com>
Signed-off-by: Daniil Roman <danroman17397@gmail.com>
Co-authored-by: Andriy Redko <drreta@gmail.com>
---
 TESTING.md | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/TESTING.md b/TESTING.md
index 3928b2fc93afa..e8416f61be7e1 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -91,21 +91,23 @@ This will instruct all JVMs (including any that run cli tools such as creating t
 
 ## Test case filtering
 
--   `tests.class` is a class-filtering shell-like glob pattern
--   `tests.method` is a method-filtering glob pattern.
+To be able to run a single test you need to specify the module where you're running the tests from.
+
+Example: `./gradlew server:test --tests "*.ReplicaShardBatchAllocatorTests.testNoAsyncFetchData"`
 
 Run a single test case (variants)
 
-    ./gradlew test -Dtests.class=org.opensearch.package.ClassName
-    ./gradlew test "-Dtests.class=*.ClassName"
+    ./gradlew module:test --tests org.opensearch.package.ClassName
+    ./gradlew module:test --tests org.opensearch.package.ClassName.testName
+    ./gradlew module:test --tests "*.ClassName"
 
 Run all tests in a package and its sub-packages
 
-    ./gradlew test "-Dtests.class=org.opensearch.package.*"
+    ./gradlew module:test --tests "org.opensearch.package.*"
 
 Run any test methods that contain *esi* (e.g.: .r*esi*ze.)
 
-    ./gradlew test "-Dtests.method=*esi*"
+    ./gradlew module:test --tests "*esi*"
 
 Run all tests that are waiting for a bugfix (disabled by default)
 

From 568c193dd817b76b2f0ca4647bc4e01908db709d Mon Sep 17 00:00:00 2001
From: panguixin <panguixin@bytedance.com>
Date: Fri, 21 Jun 2024 22:54:48 +0800
Subject: [PATCH 035/113] Write shard level metadata blob when snapshotting
 searchable snapshot indexes (#13190)

* fix snapshot status

Signed-off-by: panguixin <panguixin@bytedance.com>

* add change log

Signed-off-by: panguixin <panguixin@bytedance.com>

* Fix spotless violations

Signed-off-by: Andrew Ross <andrross@amazon.com>

---------

Signed-off-by: panguixin <panguixin@bytedance.com>
Signed-off-by: Andrew Ross <andrross@amazon.com>
Co-authored-by: Andrew Ross <andrross@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../snapshots/SearchableSnapshotIT.java       | 55 ++++++++-----
 .../blobstore/BlobStoreRepository.java        |  9 ++-
 .../snapshots/SnapshotShardsService.java      | 80 +++++++++----------
 .../blobstore/BlobStoreTestUtil.java          | 32 +++-----
 5 files changed, 93 insertions(+), 84 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6138e6dd969b1..390850b2d21bf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
 - Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
 - Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
+- Write shard level metadata blob when snapshotting searchable snapshot indexes ([#13190](https://github.com/opensearch-project/OpenSearch/pull/13190))
 
 ### Security
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
index b41dd99ff6d40..2440a3c64e956 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
@@ -17,6 +17,7 @@
 import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
@@ -53,11 +54,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
@@ -132,21 +135,24 @@ public void testCreateSearchableSnapshot() throws Exception {
 
     public void testSnapshottingSearchableSnapshots() throws Exception {
         final String repoName = "test-repo";
+        final String initSnapName = "initial-snapshot";
         final String indexName = "test-idx";
+        final String repeatSnapNamePrefix = "test-repeated-snap-";
+        final String repeatIndexNamePrefix = indexName + "-copy-";
         final Client client = client();
 
         // create an index, add data, snapshot it, then delete it
         internalCluster().ensureAtLeastNumDataNodes(1);
         createIndexWithDocsAndEnsureGreen(0, 100, indexName);
         createRepositoryWithSettings(null, repoName);
-        takeSnapshot(client, "initial-snapshot", repoName, indexName);
+        takeSnapshot(client, initSnapName, repoName, indexName);
         deleteIndicesAndEnsureGreen(client, indexName);
 
         // restore the index as a searchable snapshot
         internalCluster().ensureAtLeastNumSearchNodes(1);
         client.admin()
             .cluster()
-            .prepareRestoreSnapshot(repoName, "initial-snapshot")
+            .prepareRestoreSnapshot(repoName, initSnapName)
             .setRenamePattern("(.+)")
             .setRenameReplacement("$1-copy-0")
             .setStorageType(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT)
@@ -159,7 +165,7 @@ public void testSnapshottingSearchableSnapshots() throws Exception {
 
         // Test that the searchable snapshot index can continue to be snapshotted and restored
         for (int i = 0; i < 4; i++) {
-            final String repeatedSnapshotName = "test-repeated-snap-" + i;
+            final String repeatedSnapshotName = repeatSnapNamePrefix + i;
             takeSnapshot(client, repeatedSnapshotName, repoName);
             deleteIndicesAndEnsureGreen(client, "_all");
             client.admin()
@@ -181,21 +187,34 @@ public void testSnapshottingSearchableSnapshots() throws Exception {
         final Map<String, List<String>> snapshotInfoMap = response.getSnapshots()
             .stream()
             .collect(Collectors.toMap(s -> s.snapshotId().getName(), SnapshotInfo::indices));
-        assertEquals(
-            Map.of(
-                "initial-snapshot",
-                List.of("test-idx"),
-                "test-repeated-snap-0",
-                List.of("test-idx-copy-0"),
-                "test-repeated-snap-1",
-                List.of("test-idx-copy-1"),
-                "test-repeated-snap-2",
-                List.of("test-idx-copy-2"),
-                "test-repeated-snap-3",
-                List.of("test-idx-copy-3")
-            ),
-            snapshotInfoMap
-        );
+        final Map<String, List<String>> expect = new HashMap<>();
+        expect.put(initSnapName, List.of(indexName));
+        IntStream.range(0, 4).forEach(i -> expect.put(repeatSnapNamePrefix + i, List.of(repeatIndexNamePrefix + i)));
+        assertEquals(expect, snapshotInfoMap);
+
+        String[] snapNames = new String[5];
+        IntStream.range(0, 4).forEach(i -> snapNames[i] = repeatSnapNamePrefix + i);
+        snapNames[4] = initSnapName;
+        SnapshotsStatusResponse snapshotsStatusResponse = client.admin()
+            .cluster()
+            .prepareSnapshotStatus(repoName)
+            .addSnapshots(snapNames)
+            .execute()
+            .actionGet();
+        snapshotsStatusResponse.getSnapshots().forEach(s -> {
+            String snapName = s.getSnapshot().getSnapshotId().getName();
+            assertEquals(1, s.getIndices().size());
+            assertEquals(1, s.getShards().size());
+            if (snapName.equals("initial-snapshot")) {
+                assertNotNull(s.getIndices().get("test-idx"));
+                assertTrue(s.getShards().get(0).getStats().getTotalFileCount() > 0);
+            } else {
+                assertTrue(snapName.startsWith(repeatSnapNamePrefix));
+                assertEquals(1, s.getIndices().size());
+                assertNotNull(s.getIndices().get(repeatIndexNamePrefix + snapName.substring(repeatSnapNamePrefix.length())));
+                assertEquals(0L, s.getShards().get(0).getStats().getTotalFileCount());
+            }
+        });
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index c41e97d278dd5..53c44f743c781 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -2801,9 +2801,12 @@ public void snapshotShard(
             long indexIncrementalSize = 0;
             long indexTotalFileSize = 0;
             final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
-            // If we did not find a set of files that is equal to the current commit we determine the files to upload by comparing files
-            // in the commit with files already in the repository
-            if (filesFromSegmentInfos == null) {
+            if (store.indexSettings().isRemoteSnapshot()) {
+                // If the source of the data is another remote snapshot (i.e. searchable snapshot) then no need to snapshot the shard
+                indexCommitPointFiles = List.of();
+            } else if (filesFromSegmentInfos == null) {
+                // If we did not find a set of files that is equal to the current commit we determine the files to upload by comparing files
+                // in the commit with files already in the repository
                 indexCommitPointFiles = new ArrayList<>();
                 final Collection<String> fileNames;
                 final Store.MetadataSnapshot metadataFromStore;
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java
index 89f1ea142336e..8da36bbb8d4bd 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java
@@ -276,53 +276,47 @@ private void startNewShards(SnapshotsInProgress.Entry entry, Map<ShardId, IndexS
                 final IndexShardSnapshotStatus snapshotStatus = shardEntry.getValue();
                 final IndexId indexId = indicesMap.get(shardId.getIndexName());
                 assert indexId != null;
-                if (isRemoteSnapshot(shardId)) {
-                    // If the source of the data is another remote snapshot (i.e. searchable snapshot)
-                    // then no need to snapshot the shard and can immediately notify success.
-                    notifySuccessfulSnapshotShard(snapshot, shardId, snapshotStatus.generation());
-                } else {
-                    snapshot(
-                        shardId,
-                        snapshot,
-                        indexId,
-                        entry.userMetadata(),
-                        snapshotStatus,
-                        entry.version(),
-                        entry.remoteStoreIndexShallowCopy(),
-                        new ActionListener<>() {
-                            @Override
-                            public void onResponse(String newGeneration) {
-                                assert newGeneration != null;
-                                assert newGeneration.equals(snapshotStatus.generation());
-                                if (logger.isDebugEnabled()) {
-                                    final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy();
-                                    logger.debug(
-                                        "snapshot [{}] completed to [{}] with [{}] at generation [{}]",
-                                        snapshot,
-                                        snapshot.getRepository(),
-                                        lastSnapshotStatus,
-                                        snapshotStatus.generation()
-                                    );
-                                }
-                                notifySuccessfulSnapshotShard(snapshot, shardId, newGeneration);
+                snapshot(
+                    shardId,
+                    snapshot,
+                    indexId,
+                    entry.userMetadata(),
+                    snapshotStatus,
+                    entry.version(),
+                    entry.remoteStoreIndexShallowCopy(),
+                    new ActionListener<>() {
+                        @Override
+                        public void onResponse(String newGeneration) {
+                            assert newGeneration != null;
+                            assert newGeneration.equals(snapshotStatus.generation());
+                            if (logger.isDebugEnabled()) {
+                                final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy();
+                                logger.debug(
+                                    "snapshot [{}] completed to [{}] with [{}] at generation [{}]",
+                                    snapshot,
+                                    snapshot.getRepository(),
+                                    lastSnapshotStatus,
+                                    snapshotStatus.generation()
+                                );
                             }
+                            notifySuccessfulSnapshotShard(snapshot, shardId, newGeneration);
+                        }
 
-                            @Override
-                            public void onFailure(Exception e) {
-                                final String failure;
-                                if (e instanceof AbortedSnapshotException) {
-                                    failure = "aborted";
-                                    logger.debug(() -> new ParameterizedMessage("[{}][{}] aborted shard snapshot", shardId, snapshot), e);
-                                } else {
-                                    failure = summarizeFailure(e);
-                                    logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to snapshot shard", shardId, snapshot), e);
-                                }
-                                snapshotStatus.moveToFailed(threadPool.absoluteTimeInMillis(), failure);
-                                notifyFailedSnapshotShard(snapshot, shardId, failure);
+                        @Override
+                        public void onFailure(Exception e) {
+                            final String failure;
+                            if (e instanceof AbortedSnapshotException) {
+                                failure = "aborted";
+                                logger.debug(() -> new ParameterizedMessage("[{}][{}] aborted shard snapshot", shardId, snapshot), e);
+                            } else {
+                                failure = summarizeFailure(e);
+                                logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to snapshot shard", shardId, snapshot), e);
                             }
+                            snapshotStatus.moveToFailed(threadPool.absoluteTimeInMillis(), failure);
+                            notifyFailedSnapshotShard(snapshot, shardId, failure);
                         }
-                    );
-                }
+                    }
+                );
             }
         });
     }
diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java
index f55eb72b7aa28..32f445bf24a41 100644
--- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java
+++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java
@@ -90,7 +90,6 @@
 
 import static org.opensearch.test.OpenSearchTestCase.buildNewFakeTransportAddress;
 import static org.opensearch.test.OpenSearchTestCase.randomIntBetween;
-import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasKey;
@@ -143,7 +142,7 @@ public static void assertConsistency(BlobStoreRepository repository, Executor ex
                 }
                 assertIndexUUIDs(repository, repositoryData);
                 assertSnapshotUUIDs(repository, repositoryData);
-                assertShardIndexGenerations(repository, blobContainer, repositoryData);
+                assertShardIndexGenerations(blobContainer, repositoryData);
                 return null;
             } catch (AssertionError e) {
                 return e;
@@ -167,8 +166,7 @@ private static void assertIndexGenerations(BlobContainer repoRoot, long latestGe
         assertTrue(indexGenerations.length <= 2);
     }
 
-    private static void assertShardIndexGenerations(BlobStoreRepository repository, BlobContainer repoRoot, RepositoryData repositoryData)
-        throws IOException {
+    private static void assertShardIndexGenerations(BlobContainer repoRoot, RepositoryData repositoryData) throws IOException {
         final ShardGenerations shardGenerations = repositoryData.shardGenerations();
         final BlobContainer indicesContainer = repoRoot.children().get("indices");
         for (IndexId index : shardGenerations.indices()) {
@@ -176,22 +174,16 @@ private static void assertShardIndexGenerations(BlobStoreRepository repository,
             if (gens.isEmpty() == false) {
                 final BlobContainer indexContainer = indicesContainer.children().get(index.getId());
                 final Map<String, BlobContainer> shardContainers = indexContainer.children();
-                if (isRemoteSnapshot(repository, repositoryData, index)) {
-                    // If the source of the data is another snapshot (i.e. searchable snapshot)
-                    // then assert that there is no shard data (because it exists in the source snapshot)
-                    assertThat(shardContainers, anEmptyMap());
-                } else {
-                    for (int i = 0; i < gens.size(); i++) {
-                        final String generation = gens.get(i);
-                        assertThat(generation, not(ShardGenerations.DELETED_SHARD_GEN));
-                        if (generation != null && generation.equals(ShardGenerations.NEW_SHARD_GEN) == false) {
-                            final String shardId = Integer.toString(i);
-                            assertThat(shardContainers, hasKey(shardId));
-                            assertThat(
-                                shardContainers.get(shardId).listBlobsByPrefix(BlobStoreRepository.INDEX_FILE_PREFIX),
-                                hasKey(BlobStoreRepository.INDEX_FILE_PREFIX + generation)
-                            );
-                        }
+                for (int i = 0; i < gens.size(); i++) {
+                    final String generation = gens.get(i);
+                    assertThat(generation, not(ShardGenerations.DELETED_SHARD_GEN));
+                    if (generation != null && generation.equals(ShardGenerations.NEW_SHARD_GEN) == false) {
+                        final String shardId = Integer.toString(i);
+                        assertThat(shardContainers, hasKey(shardId));
+                        assertThat(
+                            shardContainers.get(shardId).listBlobsByPrefix(BlobStoreRepository.INDEX_FILE_PREFIX),
+                            hasKey(BlobStoreRepository.INDEX_FILE_PREFIX + generation)
+                        );
                     }
                 }
             }

From bcccedbba41df6e69cd73a118e4106c3a09f47ca Mon Sep 17 00:00:00 2001
From: Kaushal Kumar <ravi.kaushal97@gmail.com>
Date: Fri, 21 Jun 2024 09:06:41 -0700
Subject: [PATCH 036/113] Bug/sbp cancellation (#13474)

* change cancellation logic to fix disparity bw trackers and resource duress

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add additional tests for searchBackpressureService and refactor code

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add enumMap instead of list for tracking taskResourceUsageTrackets

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add nodeNotInDuress test for nodeDuressTrackers class

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address comments

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add entry in CHANGELOG

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address comments

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address comments

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* remove wildcard import

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* streamline imports

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address comments

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add additional test case to test the circuit breaker for SBP logic

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add missing javadoc to resourece type enum

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add javadoc to a method

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* fix javadoc warnings

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* fix javadoc warnings

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

---------

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 .../org/opensearch/search/ResourceType.java   |  41 +++
 .../SearchBackpressureService.java            | 241 ++++++++------
 .../stats/SearchShardTaskStats.java           |   2 +-
 .../backpressure/stats/SearchTaskStats.java   |   2 +-
 .../trackers/CpuUsageTracker.java             |  45 +--
 .../trackers/ElapsedTimeTracker.java          |  49 +--
 .../trackers/HeapUsageTracker.java            |  65 ++--
 .../trackers/NodeDuressTracker.java           |  41 ---
 .../trackers/NodeDuressTrackers.java          |  83 +++++
 .../trackers/TaskResourceUsageTracker.java    |  63 ----
 .../trackers/TaskResourceUsageTrackers.java   | 148 +++++++++
 .../opensearch/tasks/TaskCancellation.java    |  15 +
 .../SearchBackpressureServiceTests.java       | 308 +++++++++++++++---
 .../stats/SearchShardTaskStatsTests.java      |   2 +-
 .../stats/SearchTaskStatsTests.java           |   2 +-
 .../trackers/CpuUsageTrackerTests.java        |   6 +-
 .../trackers/ElapsedTimeTrackerTests.java     |   4 +-
 .../trackers/HeapUsageTrackerTests.java       |  21 +-
 .../trackers/NodeDuressTrackerTests.java      |  15 +-
 .../trackers/NodeDuressTrackersTests.java     |  84 +++++
 .../tasks/TaskCancellationTests.java          |   4 +-
 .../SearchBackpressureTestHelpers.java        |   8 +-
 23 files changed, 915 insertions(+), 335 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/search/ResourceType.java
 delete mode 100644 server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTracker.java
 create mode 100644 server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java
 delete mode 100644 server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java
 create mode 100644 server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java
 create mode 100644 server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 390850b2d21bf..41d9c4f4f3905 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix bug in SBP cancellation logic ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13474))
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
 - Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
diff --git a/server/src/main/java/org/opensearch/search/ResourceType.java b/server/src/main/java/org/opensearch/search/ResourceType.java
new file mode 100644
index 0000000000000..5bbcd7de1c2ce
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/ResourceType.java
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search;
+
+/**
+ * Enum to hold the resource type
+ */
+public enum ResourceType {
+    CPU("cpu"),
+    JVM("jvm");
+
+    private final String name;
+
+    ResourceType(String name) {
+        this.name = name;
+    }
+
+    /**
+     * The string match here is case-sensitive
+     * @param s name matching the resource type name
+     * @return a {@link ResourceType}
+     */
+    public static ResourceType fromName(String s) {
+        for (ResourceType resourceType : values()) {
+            if (resourceType.getName().equals(s)) {
+                return resourceType;
+            }
+        }
+        throw new IllegalArgumentException("Unknown resource type: [" + s + "]");
+    }
+
+    private String getName() {
+        return name;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index ebf9623eb367a..3e8ed3070e4ef 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -18,6 +18,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.monitor.jvm.JvmStats;
 import org.opensearch.monitor.process.ProcessProbe;
+import org.opensearch.search.ResourceType;
 import org.opensearch.search.backpressure.settings.SearchBackpressureMode;
 import org.opensearch.search.backpressure.settings.SearchBackpressureSettings;
 import org.opensearch.search.backpressure.settings.SearchShardTaskSettings;
@@ -28,9 +29,11 @@
 import org.opensearch.search.backpressure.trackers.CpuUsageTracker;
 import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker;
 import org.opensearch.search.backpressure.trackers.HeapUsageTracker;
-import org.opensearch.search.backpressure.trackers.NodeDuressTracker;
-import org.opensearch.search.backpressure.trackers.TaskResourceUsageTracker;
+import org.opensearch.search.backpressure.trackers.NodeDuressTrackers;
+import org.opensearch.search.backpressure.trackers.NodeDuressTrackers.NodeDuressTracker;
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.tasks.CancellableTask;
 import org.opensearch.tasks.SearchBackpressureTask;
 import org.opensearch.tasks.Task;
@@ -44,11 +47,13 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.DoubleSupplier;
+import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
@@ -62,7 +67,14 @@
  */
 public class SearchBackpressureService extends AbstractLifecycleComponent implements TaskCompletionListener {
     private static final Logger logger = LogManager.getLogger(SearchBackpressureService.class);
-
+    private static final Map<TaskResourceUsageTrackerType, Function<NodeDuressTrackers, Boolean>> trackerApplyConditions = Map.of(
+        TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
+        (nodeDuressTrackers) -> nodeDuressTrackers.isResourceInDuress(ResourceType.CPU),
+        TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER,
+        (nodeDuressTrackers) -> isHeapTrackingSupported() && nodeDuressTrackers.isResourceInDuress(ResourceType.JVM),
+        TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER,
+        (nodeDuressTrackers) -> true
+    );
     private volatile Scheduler.Cancellable scheduledFuture;
 
     private final SearchBackpressureSettings settings;
@@ -70,8 +82,8 @@ public class SearchBackpressureService extends AbstractLifecycleComponent implem
     private final ThreadPool threadPool;
     private final LongSupplier timeNanosSupplier;
 
-    private final List<NodeDuressTracker> nodeDuressTrackers;
-    private final Map<Class<? extends SearchBackpressureTask>, List<TaskResourceUsageTracker>> taskTrackers;
+    private final NodeDuressTrackers nodeDuressTrackers;
+    private final Map<Class<? extends SearchBackpressureTask>, TaskResourceUsageTrackers> taskTrackers;
 
     private final Map<Class<? extends SearchBackpressureTask>, SearchBackpressureState> searchBackpressureStates;
     private final TaskManager taskManager;
@@ -82,19 +94,26 @@ public SearchBackpressureService(
         ThreadPool threadPool,
         TaskManager taskManager
     ) {
-        this(
-            settings,
-            taskResourceTrackingService,
-            threadPool,
-            System::nanoTime,
-            List.of(
-                new NodeDuressTracker(
-                    () -> ProcessProbe.getInstance().getProcessCpuPercent() / 100.0 >= settings.getNodeDuressSettings().getCpuThreshold()
-                ),
-                new NodeDuressTracker(
-                    () -> JvmStats.jvmStats().getMem().getHeapUsedPercent() / 100.0 >= settings.getNodeDuressSettings().getHeapThreshold()
-                )
-            ),
+        this(settings, taskResourceTrackingService, threadPool, System::nanoTime, new NodeDuressTrackers(new EnumMap<>(ResourceType.class) {
+            {
+                put(
+                    ResourceType.CPU,
+                    new NodeDuressTracker(
+                        () -> ProcessProbe.getInstance().getProcessCpuPercent() / 100.0 >= settings.getNodeDuressSettings()
+                            .getCpuThreshold(),
+                        () -> settings.getNodeDuressSettings().getNumSuccessiveBreaches()
+                    )
+                );
+                put(
+                    ResourceType.JVM,
+                    new NodeDuressTracker(
+                        () -> JvmStats.jvmStats().getMem().getHeapUsedPercent() / 100.0 >= settings.getNodeDuressSettings()
+                            .getHeapThreshold(),
+                        () -> settings.getNodeDuressSettings().getNumSuccessiveBreaches()
+                    )
+                );
+            }
+        }),
             getTrackers(
                 settings.getSearchTaskSettings()::getCpuTimeNanosThreshold,
                 settings.getSearchTaskSettings()::getHeapVarianceThreshold,
@@ -117,14 +136,14 @@ public SearchBackpressureService(
         );
     }
 
-    public SearchBackpressureService(
+    SearchBackpressureService(
         SearchBackpressureSettings settings,
         TaskResourceTrackingService taskResourceTrackingService,
         ThreadPool threadPool,
         LongSupplier timeNanosSupplier,
-        List<NodeDuressTracker> nodeDuressTrackers,
-        List<TaskResourceUsageTracker> searchTaskTrackers,
-        List<TaskResourceUsageTracker> searchShardTaskTrackers,
+        NodeDuressTrackers nodeDuressTrackers,
+        TaskResourceUsageTrackers searchTaskTrackers,
+        TaskResourceUsageTrackers searchShardTaskTrackers,
         TaskManager taskManager
     ) {
         this.settings = settings;
@@ -163,40 +182,48 @@ void doRun() {
             return;
         }
 
-        if (isNodeInDuress() == false) {
+        if (nodeDuressTrackers.isNodeInDuress() == false) {
             return;
         }
 
         List<CancellableTask> searchTasks = getTaskByType(SearchTask.class);
         List<CancellableTask> searchShardTasks = getTaskByType(SearchShardTask.class);
-        List<CancellableTask> cancellableTasks = new ArrayList<>();
+
+        boolean isHeapUsageDominatedBySearchTasks = isHeapUsageDominatedBySearch(
+            searchTasks,
+            getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold()
+        );
+        boolean isHeapUsageDominatedBySearchShardTasks = isHeapUsageDominatedBySearch(
+            searchShardTasks,
+            getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold()
+        );
+        final Map<Class<? extends SearchBackpressureTask>, List<CancellableTask>> cancellableTasks = Map.of(
+            SearchTask.class,
+            isHeapUsageDominatedBySearchTasks ? searchTasks : Collections.emptyList(),
+            SearchShardTask.class,
+            isHeapUsageDominatedBySearchShardTasks ? searchShardTasks : Collections.emptyList()
+        );
 
         // Force-refresh usage stats of these tasks before making a cancellation decision.
         taskResourceTrackingService.refreshResourceStats(searchTasks.toArray(new Task[0]));
         taskResourceTrackingService.refreshResourceStats(searchShardTasks.toArray(new Task[0]));
 
-        // Check if increase in heap usage is due to SearchTasks
-        if (HeapUsageTracker.isHeapUsageDominatedBySearch(
-            searchTasks,
-            getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold()
-        )) {
-            cancellableTasks.addAll(searchTasks);
-        }
+        List<TaskCancellation> taskCancellations = new ArrayList<>();
 
-        // Check if increase in heap usage is due to SearchShardTasks
-        if (HeapUsageTracker.isHeapUsageDominatedBySearch(
-            searchShardTasks,
-            getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold()
-        )) {
-            cancellableTasks.addAll(searchShardTasks);
+        for (TaskResourceUsageTrackerType trackerType : TaskResourceUsageTrackerType.values()) {
+            if (shouldApply(trackerType)) {
+                addResourceTrackerBasedCancellations(trackerType, taskCancellations, cancellableTasks);
+            }
         }
 
-        // none of the task type is breaching the heap usage thresholds and hence we do not cancel any tasks
-        if (cancellableTasks.isEmpty()) {
-            return;
-        }
+        // Since these cancellations might be duplicate due to multiple trackers causing cancellation for same task
+        // We need to merge them
+        taskCancellations = mergeTaskCancellations(taskCancellations).stream()
+            .map(this::addSBPStateUpdateCallback)
+            .filter(TaskCancellation::isEligibleForCancellation)
+            .collect(Collectors.toList());
 
-        for (TaskCancellation taskCancellation : getTaskCancellations(cancellableTasks)) {
+        for (TaskCancellation taskCancellation : taskCancellations) {
             logger.warn(
                 "[{} mode] cancelling task [{}] due to high resource consumption [{}]",
                 mode.getName(),
@@ -226,6 +253,66 @@ void doRun() {
         }
     }
 
+    /**
+     * Had to define this method to help mock this static method to test the scenario where SearchTraffic should not be
+     * penalised when not breaching the threshold
+     * @param searchTasks inFlight co-ordinator requests
+     * @param threshold   miniumum  jvm allocated bytes ratio w.r.t. available heap
+     * @return a boolean value based on whether the threshold is breached
+     */
+    boolean isHeapUsageDominatedBySearch(List<CancellableTask> searchTasks, double threshold) {
+        return HeapUsageTracker.isHeapUsageDominatedBySearch(searchTasks, threshold);
+    }
+
+    private TaskCancellation addSBPStateUpdateCallback(TaskCancellation taskCancellation) {
+        CancellableTask task = taskCancellation.getTask();
+        Runnable toAddCancellationCallbackForSBPState = searchBackpressureStates.get(SearchShardTask.class)::incrementCancellationCount;
+        if (task instanceof SearchTask) {
+            toAddCancellationCallbackForSBPState = searchBackpressureStates.get(SearchTask.class)::incrementCancellationCount;
+        }
+        List<Runnable> newOnCancelCallbacks = new ArrayList<>(taskCancellation.getOnCancelCallbacks());
+        newOnCancelCallbacks.add(toAddCancellationCallbackForSBPState);
+        return new TaskCancellation(task, taskCancellation.getReasons(), newOnCancelCallbacks);
+    }
+
+    private boolean shouldApply(TaskResourceUsageTrackerType trackerType) {
+        return trackerApplyConditions.get(trackerType).apply(nodeDuressTrackers);
+    }
+
+    private List<TaskCancellation> addResourceTrackerBasedCancellations(
+        TaskResourceUsageTrackerType type,
+        List<TaskCancellation> taskCancellations,
+        Map<Class<? extends SearchBackpressureTask>, List<CancellableTask>> cancellableTasks
+    ) {
+        for (Map.Entry<Class<? extends SearchBackpressureTask>, TaskResourceUsageTrackers> taskResourceUsageTrackers : taskTrackers
+            .entrySet()) {
+            final Optional<TaskResourceUsageTracker> taskResourceUsageTracker = taskResourceUsageTrackers.getValue().getTracker(type);
+            final Class<? extends SearchBackpressureTask> taskType = taskResourceUsageTrackers.getKey();
+
+            taskResourceUsageTracker.ifPresent(
+                tracker -> taskCancellations.addAll(tracker.getTaskCancellations(cancellableTasks.get(taskType)))
+            );
+        }
+
+        return taskCancellations;
+    }
+
+    /**
+     * Method to reduce the taskCancellations into unique bunch
+     * @param taskCancellations all task cancellations
+     * @return unique task cancellations
+     */
+    private List<TaskCancellation> mergeTaskCancellations(final List<TaskCancellation> taskCancellations) {
+        final Map<Long, TaskCancellation> uniqueTaskCancellations = new HashMap<>();
+
+        for (TaskCancellation taskCancellation : taskCancellations) {
+            final long taskId = taskCancellation.getTask().getId();
+            uniqueTaskCancellations.put(taskId, uniqueTaskCancellations.getOrDefault(taskId, taskCancellation).merge(taskCancellation));
+        }
+
+        return new ArrayList<>(uniqueTaskCancellations.values());
+    }
+
     /**
      * Given a task, returns the type of the task
      */
@@ -243,16 +330,7 @@ Class<? extends SearchBackpressureTask> getTaskType(Task task) {
      * Returns true if the node is in duress consecutively for the past 'n' observations.
      */
     boolean isNodeInDuress() {
-        boolean isNodeInDuress = false;
-        int numSuccessiveBreaches = getSettings().getNodeDuressSettings().getNumSuccessiveBreaches();
-
-        for (NodeDuressTracker tracker : nodeDuressTrackers) {
-            if (tracker.check() >= numSuccessiveBreaches) {
-                isNodeInDuress = true;  // not breaking the loop so that each tracker's streak gets updated.
-            }
-        }
-
-        return isNodeInDuress;
+        return nodeDuressTrackers.isNodeInDuress();
     }
 
     /*
@@ -271,39 +349,6 @@ <T extends CancellableTask & SearchBackpressureTask> List<CancellableTask> getTa
             .collect(Collectors.toUnmodifiableList());
     }
 
-    /**
-     * Returns a TaskCancellation wrapper containing the list of reasons (possibly zero), along with an overall
-     * cancellation score for the given task. Cancelling a task with a higher score has better chance of recovering the
-     * node from duress.
-     */
-    TaskCancellation getTaskCancellation(CancellableTask task) {
-        List<TaskCancellation.Reason> reasons = new ArrayList<>();
-        List<Runnable> callbacks = new ArrayList<>();
-        Class<? extends SearchBackpressureTask> taskType = getTaskType(task);
-        List<TaskResourceUsageTracker> trackers = taskTrackers.get(taskType);
-        for (TaskResourceUsageTracker tracker : trackers) {
-            Optional<TaskCancellation.Reason> reason = tracker.checkAndMaybeGetCancellationReason(task);
-            if (reason.isPresent()) {
-                callbacks.add(tracker::incrementCancellations);
-                reasons.add(reason.get());
-            }
-        }
-        callbacks.add(searchBackpressureStates.get(taskType)::incrementCancellationCount);
-
-        return new TaskCancellation(task, reasons, callbacks);
-    }
-
-    /**
-     * Returns a list of TaskCancellations sorted by descending order of their cancellation scores.
-     */
-    List<TaskCancellation> getTaskCancellations(List<? extends CancellableTask> tasks) {
-        return tasks.stream()
-            .map(this::getTaskCancellation)
-            .filter(TaskCancellation::isEligibleForCancellation)
-            .sorted(Comparator.reverseOrder())
-            .collect(Collectors.toUnmodifiableList());
-    }
-
     SearchBackpressureSettings getSettings() {
         return settings;
     }
@@ -315,7 +360,7 @@ SearchBackpressureState getSearchBackpressureState(Class<? extends SearchBackpre
     /**
      * Given the threshold suppliers, returns the list of applicable trackers
      */
-    public static List<TaskResourceUsageTracker> getTrackers(
+    public static TaskResourceUsageTrackers getTrackers(
         LongSupplier cpuThresholdSupplier,
         DoubleSupplier heapVarianceSupplier,
         DoubleSupplier heapPercentThresholdSupplier,
@@ -324,23 +369,27 @@ public static List<TaskResourceUsageTracker> getTrackers(
         ClusterSettings clusterSettings,
         Setting<Integer> windowSizeSetting
     ) {
-        List<TaskResourceUsageTracker> trackers = new ArrayList<>();
-        trackers.add(new CpuUsageTracker(cpuThresholdSupplier));
+        TaskResourceUsageTrackers trackers = new TaskResourceUsageTrackers();
+        trackers.addTracker(new CpuUsageTracker(cpuThresholdSupplier), TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
         if (isHeapTrackingSupported()) {
-            trackers.add(
+            trackers.addTracker(
                 new HeapUsageTracker(
                     heapVarianceSupplier,
                     heapPercentThresholdSupplier,
                     heapMovingAverageWindowSize,
                     clusterSettings,
                     windowSizeSetting
-                )
+                ),
+                TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER
             );
         } else {
             logger.warn("heap size couldn't be determined");
         }
-        trackers.add(new ElapsedTimeTracker(ElapsedTimeNanosSupplier, System::nanoTime));
-        return Collections.unmodifiableList(trackers);
+        trackers.addTracker(
+            new ElapsedTimeTracker(ElapsedTimeNanosSupplier, System::nanoTime),
+            TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER
+        );
+        return trackers;
     }
 
     @Override
@@ -360,8 +409,8 @@ public void onTaskCompleted(Task task) {
         }
 
         List<Exception> exceptions = new ArrayList<>();
-        List<TaskResourceUsageTracker> trackers = taskTrackers.get(taskType);
-        for (TaskResourceUsageTracker tracker : trackers) {
+        TaskResourceUsageTrackers trackers = taskTrackers.get(taskType);
+        for (TaskResourceUsageTracker tracker : trackers.all()) {
             try {
                 tracker.update(task);
             } catch (Exception e) {
@@ -401,6 +450,7 @@ public SearchBackpressureStats nodeStats() {
             searchBackpressureStates.get(SearchTask.class).getLimitReachedCount(),
             searchBackpressureStates.get(SearchTask.class).getCompletionCount(),
             taskTrackers.get(SearchTask.class)
+                .all()
                 .stream()
                 .collect(Collectors.toUnmodifiableMap(t -> TaskResourceUsageTrackerType.fromName(t.name()), t -> t.stats(searchTasks)))
         );
@@ -410,6 +460,7 @@ public SearchBackpressureStats nodeStats() {
             searchBackpressureStates.get(SearchShardTask.class).getLimitReachedCount(),
             searchBackpressureStates.get(SearchShardTask.class).getCompletionCount(),
             taskTrackers.get(SearchShardTask.class)
+                .all()
                 .stream()
                 .collect(Collectors.toUnmodifiableMap(t -> TaskResourceUsageTrackerType.fromName(t.name()), t -> t.stats(searchShardTasks)))
         );
diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java
index ffe97d125b27a..be714271c8919 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java
@@ -18,8 +18,8 @@
 import org.opensearch.search.backpressure.trackers.CpuUsageTracker;
 import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker;
 import org.opensearch.search.backpressure.trackers.HeapUsageTracker;
-import org.opensearch.search.backpressure.trackers.TaskResourceUsageTracker;
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 
 import java.io.IOException;
 import java.util.Map;
diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java
index a7f9b4e3d004f..0f5f409b15def 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java
@@ -18,8 +18,8 @@
 import org.opensearch.search.backpressure.trackers.CpuUsageTracker;
 import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker;
 import org.opensearch.search.backpressure.trackers.HeapUsageTracker;
-import org.opensearch.search.backpressure.trackers.TaskResourceUsageTracker;
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 
 import java.io.IOException;
 import java.util.Map;
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java
index 507953cb4a20e..a303b625f4b59 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java
@@ -12,6 +12,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskCancellation;
 
@@ -34,7 +35,30 @@ public class CpuUsageTracker extends TaskResourceUsageTracker {
     private final LongSupplier thresholdSupplier;
 
     public CpuUsageTracker(LongSupplier thresholdSupplier) {
+        this(thresholdSupplier, (task) -> {
+            long usage = task.getTotalResourceStats().getCpuTimeInNanos();
+            long threshold = thresholdSupplier.getAsLong();
+
+            if (usage < threshold) {
+                return Optional.empty();
+            }
+
+            return Optional.of(
+                new TaskCancellation.Reason(
+                    "cpu usage exceeded ["
+                        + new TimeValue(usage, TimeUnit.NANOSECONDS)
+                        + " >= "
+                        + new TimeValue(threshold, TimeUnit.NANOSECONDS)
+                        + "]",
+                    1  // TODO: fine-tune the cancellation score/weight
+                )
+            );
+        });
+    }
+
+    public CpuUsageTracker(LongSupplier thresholdSupplier, ResourceUsageBreachEvaluator resourceUsageBreachEvaluator) {
         this.thresholdSupplier = thresholdSupplier;
+        this.resourceUsageBreachEvaluator = resourceUsageBreachEvaluator;
     }
 
     @Override
@@ -42,27 +66,6 @@ public String name() {
         return CPU_USAGE_TRACKER.getName();
     }
 
-    @Override
-    public Optional<TaskCancellation.Reason> checkAndMaybeGetCancellationReason(Task task) {
-        long usage = task.getTotalResourceStats().getCpuTimeInNanos();
-        long threshold = thresholdSupplier.getAsLong();
-
-        if (usage < threshold) {
-            return Optional.empty();
-        }
-
-        return Optional.of(
-            new TaskCancellation.Reason(
-                "cpu usage exceeded ["
-                    + new TimeValue(usage, TimeUnit.NANOSECONDS)
-                    + " >= "
-                    + new TimeValue(threshold, TimeUnit.NANOSECONDS)
-                    + "]",
-                1  // TODO: fine-tune the cancellation score/weight
-            )
-        );
-    }
-
     @Override
     public TaskResourceUsageTracker.Stats stats(List<? extends Task> activeTasks) {
         long currentMax = activeTasks.stream().mapToLong(t -> t.getTotalResourceStats().getCpuTimeInNanos()).max().orElse(0);
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java
index f1e8abe7e3230..216947315cd2d 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java
@@ -12,6 +12,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskCancellation;
 
@@ -34,8 +35,35 @@ public class ElapsedTimeTracker extends TaskResourceUsageTracker {
     private final LongSupplier timeNanosSupplier;
 
     public ElapsedTimeTracker(LongSupplier thresholdSupplier, LongSupplier timeNanosSupplier) {
+        this(thresholdSupplier, timeNanosSupplier, (Task task) -> {
+            long usage = timeNanosSupplier.getAsLong() - task.getStartTimeNanos();
+            long threshold = thresholdSupplier.getAsLong();
+
+            if (usage < threshold) {
+                return Optional.empty();
+            }
+
+            return Optional.of(
+                new TaskCancellation.Reason(
+                    "elapsed time exceeded ["
+                        + new TimeValue(usage, TimeUnit.NANOSECONDS)
+                        + " >= "
+                        + new TimeValue(threshold, TimeUnit.NANOSECONDS)
+                        + "]",
+                    1  // TODO: fine-tune the cancellation score/weight
+                )
+            );
+        });
+    }
+
+    public ElapsedTimeTracker(
+        LongSupplier thresholdSupplier,
+        LongSupplier timeNanosSupplier,
+        ResourceUsageBreachEvaluator resourceUsageBreachEvaluator
+    ) {
         this.thresholdSupplier = thresholdSupplier;
         this.timeNanosSupplier = timeNanosSupplier;
+        this.resourceUsageBreachEvaluator = resourceUsageBreachEvaluator;
     }
 
     @Override
@@ -43,27 +71,6 @@ public String name() {
         return ELAPSED_TIME_TRACKER.getName();
     }
 
-    @Override
-    public Optional<TaskCancellation.Reason> checkAndMaybeGetCancellationReason(Task task) {
-        long usage = timeNanosSupplier.getAsLong() - task.getStartTimeNanos();
-        long threshold = thresholdSupplier.getAsLong();
-
-        if (usage < threshold) {
-            return Optional.empty();
-        }
-
-        return Optional.of(
-            new TaskCancellation.Reason(
-                "elapsed time exceeded ["
-                    + new TimeValue(usage, TimeUnit.NANOSECONDS)
-                    + " >= "
-                    + new TimeValue(threshold, TimeUnit.NANOSECONDS)
-                    + "]",
-                1  // TODO: fine-tune the cancellation score/weight
-            )
-        );
-    }
-
     @Override
     public TaskResourceUsageTracker.Stats stats(List<? extends Task> activeTasks) {
         long now = timeNanosSupplier.getAsLong();
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java
index 56b9f947f6e37..c69de8ce21f89 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java
@@ -18,6 +18,7 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.monitor.jvm.JvmStats;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.tasks.CancellableTask;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskCancellation;
@@ -55,6 +56,43 @@ public HeapUsageTracker(
         this.heapPercentThresholdSupplier = heapPercentThresholdSupplier;
         this.movingAverageReference = new AtomicReference<>(new MovingAverage(heapMovingAverageWindowSize));
         clusterSettings.addSettingsUpdateConsumer(windowSizeSetting, this::updateWindowSize);
+        setDefaultResourceUsageBreachEvaluator();
+    }
+
+    /**
+     * Had to refactor this method out of the constructor as we can't pass a lambda which references a member variable in constructor
+     * error: cannot reference movingAverageReference before supertype constructor has been called
+     */
+    private void setDefaultResourceUsageBreachEvaluator() {
+        this.resourceUsageBreachEvaluator = (task) -> {
+            MovingAverage movingAverage = movingAverageReference.get();
+
+            // There haven't been enough measurements.
+            if (movingAverage.isReady() == false) {
+                return Optional.empty();
+            }
+
+            double currentUsage = task.getTotalResourceStats().getMemoryInBytes();
+            double averageUsage = movingAverage.getAverage();
+            double variance = heapVarianceSupplier.getAsDouble();
+            double allowedUsage = averageUsage * variance;
+            double threshold = heapPercentThresholdSupplier.getAsDouble() * HEAP_SIZE_BYTES;
+
+            if (isHeapTrackingSupported() == false || currentUsage < threshold || currentUsage < allowedUsage) {
+                return Optional.empty();
+            }
+
+            return Optional.of(
+                new TaskCancellation.Reason(
+                    "heap usage exceeded ["
+                        + new ByteSizeValue((long) currentUsage)
+                        + " >= "
+                        + new ByteSizeValue((long) allowedUsage)
+                        + "]",
+                    (int) (currentUsage / averageUsage)  // TODO: fine-tune the cancellation score/weight
+                )
+            );
+        };
     }
 
     @Override
@@ -67,33 +105,6 @@ public void update(Task task) {
         movingAverageReference.get().record(task.getTotalResourceStats().getMemoryInBytes());
     }
 
-    @Override
-    public Optional<TaskCancellation.Reason> checkAndMaybeGetCancellationReason(Task task) {
-        MovingAverage movingAverage = movingAverageReference.get();
-
-        // There haven't been enough measurements.
-        if (movingAverage.isReady() == false) {
-            return Optional.empty();
-        }
-
-        double currentUsage = task.getTotalResourceStats().getMemoryInBytes();
-        double averageUsage = movingAverage.getAverage();
-        double variance = heapVarianceSupplier.getAsDouble();
-        double allowedUsage = averageUsage * variance;
-        double threshold = heapPercentThresholdSupplier.getAsDouble() * HEAP_SIZE_BYTES;
-
-        if (isHeapTrackingSupported() == false || currentUsage < threshold || currentUsage < allowedUsage) {
-            return Optional.empty();
-        }
-
-        return Optional.of(
-            new TaskCancellation.Reason(
-                "heap usage exceeded [" + new ByteSizeValue((long) currentUsage) + " >= " + new ByteSizeValue((long) allowedUsage) + "]",
-                (int) (currentUsage / averageUsage)  // TODO: fine-tune the cancellation score/weight
-            )
-        );
-    }
-
     private void updateWindowSize(int heapMovingAverageWindowSize) {
         this.movingAverageReference.set(new MovingAverage(heapMovingAverageWindowSize));
     }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTracker.java
deleted file mode 100644
index 8e35c724a8fef..0000000000000
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTracker.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.search.backpressure.trackers;
-
-import org.opensearch.common.util.Streak;
-
-import java.util.function.BooleanSupplier;
-
-/**
- * NodeDuressTracker is used to check if the node is in duress.
- *
- * @opensearch.internal
- */
-public class NodeDuressTracker {
-    /**
-     * Tracks the number of consecutive breaches.
-     */
-    private final Streak breaches = new Streak();
-
-    /**
-     * Predicate that returns true when the node is in duress.
-     */
-    private final BooleanSupplier isNodeInDuress;
-
-    public NodeDuressTracker(BooleanSupplier isNodeInDuress) {
-        this.isNodeInDuress = isNodeInDuress;
-    }
-
-    /**
-     * Evaluates the predicate and returns the number of consecutive breaches.
-     */
-    public int check() {
-        return breaches.record(isNodeInDuress.getAsBoolean());
-    }
-}
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java
new file mode 100644
index 0000000000000..ae60a82fc2816
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java
@@ -0,0 +1,83 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.trackers;
+
+import org.opensearch.common.util.Streak;
+import org.opensearch.search.ResourceType;
+
+import java.util.Map;
+import java.util.function.BooleanSupplier;
+import java.util.function.IntSupplier;
+
+/**
+ * NodeDuressTrackers is used to check if the node is in duress based on various resources.
+ *
+ * @opensearch.internal
+ */
+public class NodeDuressTrackers {
+    private final Map<ResourceType, NodeDuressTracker> duressTrackers;
+
+    public NodeDuressTrackers(Map<ResourceType, NodeDuressTracker> duressTrackers) {
+        this.duressTrackers = duressTrackers;
+    }
+
+    /**
+     * Method to check the {@link ResourceType} in duress
+     * @return Boolean
+     */
+    public boolean isResourceInDuress(ResourceType resourceType) {
+        return duressTrackers.get(resourceType).test();
+    }
+
+    /**
+     * Method to evaluate whether the node is in duress or not
+     * @return true if node is in duress because of either system resource
+     */
+    public boolean isNodeInDuress() {
+        for (ResourceType resourceType : ResourceType.values()) {
+            if (isResourceInDuress(resourceType)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * NodeDuressTracker is used to check if the node is in duress
+     * @opensearch.internal
+     */
+    public static class NodeDuressTracker {
+        /**
+         * Tracks the number of consecutive breaches.
+         */
+        private final Streak breaches = new Streak();
+
+        /**
+         * Predicate that returns true when the node is in duress.
+         */
+        private final BooleanSupplier isNodeInDuress;
+
+        /**
+         * Predicate that returns the max number of breaches allowed for this resource before we mark it as in duress
+         */
+        private final IntSupplier maxBreachAllowedSupplier;
+
+        public NodeDuressTracker(BooleanSupplier isNodeInDuress, IntSupplier maxBreachAllowedSupplier) {
+            this.isNodeInDuress = isNodeInDuress;
+            this.maxBreachAllowedSupplier = maxBreachAllowedSupplier;
+        }
+
+        /**
+         * Returns true if the node is in duress consecutively for the past 'n' observations.
+         */
+        public boolean test() {
+            return breaches.record(isNodeInDuress.getAsBoolean()) >= maxBreachAllowedSupplier.getAsInt();
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java
deleted file mode 100644
index ce15e9e9b6622..0000000000000
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.search.backpressure.trackers;
-
-import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.core.xcontent.ToXContentObject;
-import org.opensearch.tasks.Task;
-import org.opensearch.tasks.TaskCancellation;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * TaskResourceUsageTracker is used to track completions and cancellations of search related tasks.
- *
- * @opensearch.internal
- */
-public abstract class TaskResourceUsageTracker {
-    /**
-     * Counts the number of cancellations made due to this tracker.
-     */
-    private final AtomicLong cancellations = new AtomicLong();
-
-    public long incrementCancellations() {
-        return cancellations.incrementAndGet();
-    }
-
-    public long getCancellations() {
-        return cancellations.get();
-    }
-
-    /**
-     * Returns a unique name for this tracker.
-     */
-    public abstract String name();
-
-    /**
-     * Notifies the tracker to update its state when a task execution completes.
-     */
-    public void update(Task task) {}
-
-    /**
-     * Returns the cancellation reason for the given task, if it's eligible for cancellation.
-     */
-    public abstract Optional<TaskCancellation.Reason> checkAndMaybeGetCancellationReason(Task task);
-
-    /**
-     * Returns the tracker's state for tasks as seen in the stats API.
-     */
-    public abstract Stats stats(List<? extends Task> activeTasks);
-
-    /**
-     * Represents the tracker's state as seen in the stats API.
-     */
-    public interface Stats extends ToXContentObject, Writeable {}
-}
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java
new file mode 100644
index 0000000000000..3b0072288681c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java
@@ -0,0 +1,148 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.trackers;
+
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.tasks.CancellableTask;
+import org.opensearch.tasks.Task;
+import org.opensearch.tasks.TaskCancellation;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+/**
+ * TaskResourceUsageTrackers is used to hold all the {@link TaskResourceUsageTracker} objects.
+ *
+ * @opensearch.internal
+ */
+public class TaskResourceUsageTrackers {
+    private final EnumMap<TaskResourceUsageTrackerType, TaskResourceUsageTracker> all;
+
+    public TaskResourceUsageTrackers() {
+        all = new EnumMap<>(TaskResourceUsageTrackerType.class);
+    }
+
+    /**
+     * adds the tracker for the TrackerType
+     * @param tracker is {@link TaskResourceUsageTracker} implementation which will be added
+     * @param trackerType is {@link TaskResourceUsageTrackerType} which depicts the implementation type
+     */
+    public void addTracker(final TaskResourceUsageTracker tracker, final TaskResourceUsageTrackerType trackerType) {
+        all.put(trackerType, tracker);
+    }
+
+    /**
+     * getter for tracker for a {@link TaskResourceUsageTrackerType}
+     * @param type for which the implementation is returned
+     * @return the {@link TaskResourceUsageTrackerType}
+     */
+    public Optional<TaskResourceUsageTracker> getTracker(TaskResourceUsageTrackerType type) {
+        return Optional.ofNullable(all.get(type));
+    }
+
+    /**
+     * Method to access all available {@link TaskResourceUsageTracker}
+     * @return all enabled and available {@link TaskResourceUsageTracker}s
+     */
+    public List<TaskResourceUsageTracker> all() {
+        return new ArrayList<>(all.values());
+    }
+
+    /**
+     * TaskResourceUsageTracker is used to track completions and cancellations of search related tasks.
+     * @opensearch.internal
+     */
+    public static abstract class TaskResourceUsageTracker {
+        /**
+         * Counts the number of cancellations made due to this tracker.
+         */
+        private final AtomicLong cancellations = new AtomicLong();
+        protected ResourceUsageBreachEvaluator resourceUsageBreachEvaluator;
+
+        /**
+         * for test purposes only
+         * @param resourceUsageBreachEvaluator which suggests whether a task should be cancelled or not
+         */
+        public void setResourceUsageBreachEvaluator(final ResourceUsageBreachEvaluator resourceUsageBreachEvaluator) {
+            this.resourceUsageBreachEvaluator = resourceUsageBreachEvaluator;
+        }
+
+        public long incrementCancellations() {
+            return cancellations.incrementAndGet();
+        }
+
+        public long getCancellations() {
+            return cancellations.get();
+        }
+
+        /**
+         * Returns a unique name for this tracker.
+         */
+        public abstract String name();
+
+        /**
+         * Notifies the tracker to update its state when a task execution completes.
+         */
+        public void update(Task task) {}
+
+        /**
+         * Returns the cancellation reason for the given task, if it's eligible for cancellation.
+         */
+        public Optional<TaskCancellation.Reason> checkAndMaybeGetCancellationReason(Task task) {
+            return resourceUsageBreachEvaluator.evaluate(task);
+        }
+
+        /**
+         * Returns the tracker's state for tasks as seen in the stats API.
+         */
+        public abstract Stats stats(List<? extends Task> activeTasks);
+
+        /**
+         * Method to get taskCancellations due to this tracker for the given {@link CancellableTask} tasks
+         * @param tasks cancellation eligible tasks due to node duress and search traffic threshold breach
+         * @return the list of tasks which are breaching task level thresholds for this {@link TaskResourceUsageTracker}
+         */
+        public List<TaskCancellation> getTaskCancellations(List<CancellableTask> tasks) {
+            return tasks.stream()
+                .map(task -> this.getTaskCancellation(task, List.of(this::incrementCancellations)))
+                .filter(TaskCancellation::isEligibleForCancellation)
+                .collect(Collectors.toList());
+        }
+
+        private TaskCancellation getTaskCancellation(final CancellableTask task, final List<Runnable> cancellationCallback) {
+            Optional<TaskCancellation.Reason> reason = checkAndMaybeGetCancellationReason(task);
+            List<TaskCancellation.Reason> reasons = new ArrayList<>();
+            reason.ifPresent(reasons::add);
+
+            return new TaskCancellation(task, reasons, cancellationCallback);
+        }
+
+        /**
+         * Represents the tracker's state as seen in the stats API.
+         */
+        public interface Stats extends ToXContentObject, Writeable {}
+
+        /**
+         * This interface carries the logic to decide whether a task should be cancelled or not
+         */
+        public interface ResourceUsageBreachEvaluator {
+            /**
+             * evaluates whether the task is eligible for cancellation based on {@link TaskResourceUsageTracker} implementation
+             * @param task is input to this method on which the cancellation evaluation is performed
+             * @return a {@link TaskCancellation.Reason} why this task should be cancelled otherwise empty
+             */
+            public Optional<TaskCancellation.Reason> evaluate(final Task task);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellation.java b/server/src/main/java/org/opensearch/tasks/TaskCancellation.java
index 2d152e513f197..872f5b79bb205 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskCancellation.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskCancellation.java
@@ -42,10 +42,25 @@ public List<Reason> getReasons() {
         return reasons;
     }
 
+    public List<Runnable> getOnCancelCallbacks() {
+        return onCancelCallbacks;
+    }
+
     public String getReasonString() {
         return reasons.stream().map(Reason::getMessage).collect(Collectors.joining(", "));
     }
 
+    public TaskCancellation merge(final TaskCancellation other) {
+        if (other == this) {
+            return this;
+        }
+        final List<Reason> newReasons = new ArrayList<>(reasons);
+        newReasons.addAll(other.getReasons());
+        final List<Runnable> newOnCancelCallbacks = new ArrayList<>(onCancelCallbacks);
+        newOnCancelCallbacks.addAll(other.onCancelCallbacks);
+        return new TaskCancellation(task, newReasons, newOnCancelCallbacks);
+    }
+
     /**
      * Cancels the task and invokes all onCancelCallbacks.
      */
diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
index 9778798b706f4..43df482fcc2ae 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
@@ -16,6 +16,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.search.ResourceType;
 import org.opensearch.search.backpressure.settings.SearchBackpressureMode;
 import org.opensearch.search.backpressure.settings.SearchBackpressureSettings;
 import org.opensearch.search.backpressure.settings.SearchShardTaskSettings;
@@ -23,9 +24,11 @@
 import org.opensearch.search.backpressure.stats.SearchBackpressureStats;
 import org.opensearch.search.backpressure.stats.SearchShardTaskStats;
 import org.opensearch.search.backpressure.stats.SearchTaskStats;
-import org.opensearch.search.backpressure.trackers.NodeDuressTracker;
-import org.opensearch.search.backpressure.trackers.TaskResourceUsageTracker;
+import org.opensearch.search.backpressure.trackers.NodeDuressTrackers;
+import org.opensearch.search.backpressure.trackers.NodeDuressTrackers.NodeDuressTracker;
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.tasks.CancellableTask;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskCancellation;
@@ -42,6 +45,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -52,10 +56,14 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.LongSupplier;
 
+import static org.opensearch.search.ResourceType.CPU;
+import static org.opensearch.search.ResourceType.JVM;
 import static org.opensearch.search.backpressure.SearchBackpressureTestHelpers.createMockTaskWithResourceStats;
 import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyDouble;
+import static org.mockito.Mockito.anyList;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
@@ -89,8 +97,15 @@ public void testIsNodeInDuress() {
 
         AtomicReference<Double> cpuUsage = new AtomicReference<>();
         AtomicReference<Double> heapUsage = new AtomicReference<>();
-        NodeDuressTracker cpuUsageTracker = new NodeDuressTracker(() -> cpuUsage.get() >= 0.5);
-        NodeDuressTracker heapUsageTracker = new NodeDuressTracker(() -> heapUsage.get() >= 0.5);
+        NodeDuressTracker cpuUsageTracker = new NodeDuressTracker(() -> cpuUsage.get() >= 0.5, () -> 3);
+        NodeDuressTracker heapUsageTracker = new NodeDuressTracker(() -> heapUsage.get() >= 0.5, () -> 3);
+
+        EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.JVM, heapUsageTracker);
+                put(ResourceType.CPU, cpuUsageTracker);
+            }
+        };
 
         SearchBackpressureSettings settings = new SearchBackpressureSettings(
             Settings.EMPTY,
@@ -102,9 +117,9 @@ public void testIsNodeInDuress() {
             mockTaskResourceTrackingService,
             threadPool,
             System::nanoTime,
-            List.of(cpuUsageTracker, heapUsageTracker),
-            Collections.emptyList(),
-            Collections.emptyList(),
+            new NodeDuressTrackers(duressTrackers),
+            new TaskResourceUsageTrackers(),
+            new TaskResourceUsageTrackers(),
             taskManager
         );
 
@@ -132,6 +147,8 @@ public void testTrackerStateUpdateOnSearchTaskCompletion() {
         TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
         LongSupplier mockTimeNanosSupplier = () -> TimeUnit.SECONDS.toNanos(1234);
         TaskResourceUsageTracker mockTaskResourceUsageTracker = mock(TaskResourceUsageTracker.class);
+        TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
+        taskResourceUsageTrackers.addTracker(mockTaskResourceUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
 
         SearchBackpressureSettings settings = new SearchBackpressureSettings(
             Settings.EMPTY,
@@ -143,15 +160,15 @@ public void testTrackerStateUpdateOnSearchTaskCompletion() {
             mockTaskResourceTrackingService,
             threadPool,
             mockTimeNanosSupplier,
-            Collections.emptyList(),
-            List.of(mockTaskResourceUsageTracker),
-            Collections.emptyList(),
+            new NodeDuressTrackers(new EnumMap<>(ResourceType.class)),
+            taskResourceUsageTrackers,
+            new TaskResourceUsageTrackers(),
             taskManager
         );
 
         for (int i = 0; i < 100; i++) {
             // service.onTaskCompleted(new SearchTask(1, "test", "test", () -> "Test", TaskId.EMPTY_TASK_ID, new HashMap<>()));
-            service.onTaskCompleted(createMockTaskWithResourceStats(SearchTask.class, 100, 200));
+            service.onTaskCompleted(createMockTaskWithResourceStats(SearchTask.class, 100, 200, i));
         }
         assertEquals(100, service.getSearchBackpressureState(SearchTask.class).getCompletionCount());
         verify(mockTaskResourceUsageTracker, times(100)).update(any());
@@ -161,6 +178,8 @@ public void testTrackerStateUpdateOnSearchShardTaskCompletion() {
         TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
         LongSupplier mockTimeNanosSupplier = () -> TimeUnit.SECONDS.toNanos(1234);
         TaskResourceUsageTracker mockTaskResourceUsageTracker = mock(TaskResourceUsageTracker.class);
+        TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
+        taskResourceUsageTrackers.addTracker(mockTaskResourceUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
 
         SearchBackpressureSettings settings = new SearchBackpressureSettings(
             Settings.EMPTY,
@@ -172,16 +191,16 @@ public void testTrackerStateUpdateOnSearchShardTaskCompletion() {
             mockTaskResourceTrackingService,
             threadPool,
             mockTimeNanosSupplier,
-            Collections.emptyList(),
-            Collections.emptyList(),
-            List.of(mockTaskResourceUsageTracker),
+            new NodeDuressTrackers(new EnumMap<>(ResourceType.class)),
+            new TaskResourceUsageTrackers(),
+            taskResourceUsageTrackers,
             taskManager
         );
 
         // Record task completions to update the tracker state. Tasks other than SearchTask & SearchShardTask are ignored.
-        service.onTaskCompleted(createMockTaskWithResourceStats(CancellableTask.class, 100, 200));
+        service.onTaskCompleted(createMockTaskWithResourceStats(CancellableTask.class, 100, 200, 101));
         for (int i = 0; i < 100; i++) {
-            service.onTaskCompleted(createMockTaskWithResourceStats(SearchShardTask.class, 100, 200));
+            service.onTaskCompleted(createMockTaskWithResourceStats(SearchShardTask.class, 100, 200, i));
         }
         assertEquals(100, service.getSearchBackpressureState(SearchShardTask.class).getCompletionCount());
         verify(mockTaskResourceUsageTracker, times(100)).update(any());
@@ -192,21 +211,41 @@ public void testSearchTaskInFlightCancellation() {
         TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
         AtomicLong mockTime = new AtomicLong(0);
         LongSupplier mockTimeNanosSupplier = mockTime::get;
-        NodeDuressTracker mockNodeDuressTracker = new NodeDuressTracker(() -> true);
+        NodeDuressTracker mockNodeDuressTracker = new NodeDuressTracker(() -> true, () -> 3);
 
-        TaskResourceUsageTracker mockTaskResourceUsageTracker = getMockedTaskResourceUsageTracker();
+        TaskResourceUsageTracker mockTaskResourceUsageTracker = getMockedTaskResourceUsageTracker(
+            TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
+            (task) -> {
+                if (task.getTotalResourceStats().getCpuTimeInNanos() < 300) {
+                    return Optional.empty();
+                }
+
+                return Optional.of(new TaskCancellation.Reason("limits exceeded", 5));
+            }
+        );
+        TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
+        taskResourceUsageTrackers.addTracker(mockTaskResourceUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
 
         // Mocking 'settings' with predictable rate limiting thresholds.
         SearchBackpressureSettings settings = getBackpressureSettings("enforced", 0.1, 0.003, 5.0);
 
+        NodeDuressTracker heapUsageTracker = new NodeDuressTracker(() -> false, () -> 3);
+
+        EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
+            {
+                put(JVM, heapUsageTracker);
+                put(CPU, mockNodeDuressTracker);
+            }
+        };
+
         SearchBackpressureService service = new SearchBackpressureService(
             settings,
             mockTaskResourceTrackingService,
             threadPool,
             mockTimeNanosSupplier,
-            List.of(mockNodeDuressTracker),
-            List.of(mockTaskResourceUsageTracker),
-            Collections.emptyList(),
+            new NodeDuressTrackers(duressTrackers),
+            taskResourceUsageTrackers,
+            new TaskResourceUsageTrackers(),
             mockTaskManager
         );
 
@@ -225,9 +264,9 @@ public void testSearchTaskInFlightCancellation() {
         Map<Long, Task> activeSearchTasks = new HashMap<>();
         for (long i = 0; i < 75; i++) {
             if (i % 3 == 0) {
-                activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 500, taskHeapUsageBytes));
+                activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 500, taskHeapUsageBytes, i));
             } else {
-                activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, taskHeapUsageBytes));
+                activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, taskHeapUsageBytes, i));
             }
         }
         doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks();
@@ -265,9 +304,28 @@ public void testSearchShardTaskInFlightCancellation() {
         TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
         AtomicLong mockTime = new AtomicLong(0);
         LongSupplier mockTimeNanosSupplier = mockTime::get;
-        NodeDuressTracker mockNodeDuressTracker = new NodeDuressTracker(() -> true);
+        NodeDuressTracker mockNodeDuressTracker = new NodeDuressTracker(() -> true, () -> 3);
 
-        TaskResourceUsageTracker mockTaskResourceUsageTracker = getMockedTaskResourceUsageTracker();
+        EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
+            {
+                put(JVM, new NodeDuressTracker(() -> false, () -> 3));
+                put(CPU, mockNodeDuressTracker);
+            }
+        };
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers);
+
+        TaskResourceUsageTracker mockTaskResourceUsageTracker = getMockedTaskResourceUsageTracker(
+            TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
+            (task) -> {
+                if (task.getTotalResourceStats().getCpuTimeInNanos() < 300) {
+                    return Optional.empty();
+                }
+
+                return Optional.of(new TaskCancellation.Reason("limits exceeded", 5));
+            }
+        );
+        TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
+        taskResourceUsageTrackers.addTracker(mockTaskResourceUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
 
         // Mocking 'settings' with predictable rate limiting thresholds.
         SearchBackpressureSettings settings = getBackpressureSettings("enforced", 0.1, 0.003, 10.0);
@@ -277,9 +335,9 @@ public void testSearchShardTaskInFlightCancellation() {
             mockTaskResourceTrackingService,
             threadPool,
             mockTimeNanosSupplier,
-            List.of(mockNodeDuressTracker),
-            Collections.emptyList(),
-            List.of(mockTaskResourceUsageTracker),
+            nodeDuressTrackers,
+            new TaskResourceUsageTrackers(),
+            taskResourceUsageTrackers,
             mockTaskManager
         );
 
@@ -298,9 +356,9 @@ public void testSearchShardTaskInFlightCancellation() {
         Map<Long, Task> activeSearchShardTasks = new HashMap<>();
         for (long i = 0; i < 75; i++) {
             if (i % 5 == 0) {
-                activeSearchShardTasks.put(i, createMockTaskWithResourceStats(SearchShardTask.class, 500, taskHeapUsageBytes));
+                activeSearchShardTasks.put(i, createMockTaskWithResourceStats(SearchShardTask.class, 500, taskHeapUsageBytes, i));
             } else {
-                activeSearchShardTasks.put(i, createMockTaskWithResourceStats(SearchShardTask.class, 100, taskHeapUsageBytes));
+                activeSearchShardTasks.put(i, createMockTaskWithResourceStats(SearchShardTask.class, 100, taskHeapUsageBytes, i));
             }
         }
         doReturn(activeSearchShardTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks();
@@ -318,7 +376,7 @@ public void testSearchShardTaskInFlightCancellation() {
         // Simulate task completion to replenish some tokens.
         // This will add 2 tokens (task count delta * cancellationRatio) to 'rateLimitPerTaskCompletion'.
         for (int i = 0; i < 20; i++) {
-            service.onTaskCompleted(createMockTaskWithResourceStats(SearchShardTask.class, 100, taskHeapUsageBytes));
+            service.onTaskCompleted(createMockTaskWithResourceStats(SearchShardTask.class, 100, taskHeapUsageBytes, i));
         }
         service.doRun();
         verify(mockTaskManager, times(12)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any());
@@ -335,6 +393,181 @@ public void testSearchShardTaskInFlightCancellation() {
         assertEquals(expectedStats, actualStats);
     }
 
+    public void testNonCancellationOfHeapBasedTasksWhenHeapNotInDuress() {
+        TaskManager mockTaskManager = spy(taskManager);
+        TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
+        AtomicLong mockTime = new AtomicLong(0);
+        LongSupplier mockTimeNanosSupplier = mockTime::get;
+
+        EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
+            {
+                put(JVM, new NodeDuressTracker(() -> false, () -> 3));
+                put(CPU, new NodeDuressTracker(() -> true, () -> 3));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers);
+
+        // Creating heap and cpu usage trackers where heap tracker will always evaluate with reasons to cancel the
+        // tasks but heap based cancellation should not happen because heap is not in duress
+        TaskResourceUsageTracker heapUsageTracker = getMockedTaskResourceUsageTracker(
+            TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER,
+            (task) -> Optional.of(new TaskCancellation.Reason("mem exceeded", 10))
+        );
+        TaskResourceUsageTracker cpuUsageTracker = getMockedTaskResourceUsageTracker(
+            TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
+            (task) -> {
+                if (task.getTotalResourceStats().getCpuTimeInNanos() < 400) {
+                    return Optional.empty();
+                }
+                return Optional.of(new TaskCancellation.Reason("cpu time limit exceeded", 5));
+            }
+        );
+
+        TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
+        taskResourceUsageTrackers.addTracker(cpuUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
+        taskResourceUsageTrackers.addTracker(heapUsageTracker, TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER);
+
+        // Mocking 'settings' with predictable rate limiting thresholds.
+        SearchBackpressureSettings settings = getBackpressureSettings("enforced", 0.1, 0.003, 10.0);
+
+        SearchBackpressureService service = new SearchBackpressureService(
+            settings,
+            mockTaskResourceTrackingService,
+            threadPool,
+            mockTimeNanosSupplier,
+            nodeDuressTrackers,
+            taskResourceUsageTrackers,
+            new TaskResourceUsageTrackers(),
+            mockTaskManager
+        );
+
+        service.doRun();
+        service.doRun();
+
+        SearchTaskSettings searchTaskSettings = mock(SearchTaskSettings.class);
+        // setting the total heap percent threshold to minimum so that circuit does not break in SearchBackpressureService
+        when(searchTaskSettings.getTotalHeapPercentThreshold()).thenReturn(0.0);
+        when(settings.getSearchTaskSettings()).thenReturn(searchTaskSettings);
+
+        // Create a mix of low and high resource usage tasks (60 low + 15 high resource usage tasks).
+        Map<Long, Task> activeSearchTasks = new HashMap<>();
+        for (long i = 0; i < 75; i++) {
+            if (i % 5 == 0) {
+                activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 500, 800, i));
+            } else {
+                activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, 800, i));
+            }
+        }
+        doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks();
+
+        // this will trigger cancellation but these cancellation should only be cpu based
+        service.doRun();
+        verify(mockTaskManager, times(5)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any());
+        assertEquals(5, service.getSearchBackpressureState(SearchTask.class).getCancellationCount());
+        assertEquals(1, service.getSearchBackpressureState(SearchTask.class).getLimitReachedCount());
+
+        SearchBackpressureStats expectedStats = new SearchBackpressureStats(
+            new SearchTaskStats(
+                5,
+                1,
+                0,
+                Map.of(
+                    TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
+                    new MockStats(5),
+                    TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER,
+                    new MockStats(0)
+                )
+            ),
+            new SearchShardTaskStats(0, 0, 0, Collections.emptyMap()),
+            SearchBackpressureMode.ENFORCED
+        );
+
+        SearchBackpressureStats actualStats = service.nodeStats();
+        assertEquals(expectedStats, actualStats);
+    }
+
+    public void testNonCancellationWhenSearchTrafficIsNotQualifyingForCancellation() {
+        TaskManager mockTaskManager = spy(taskManager);
+        TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
+        AtomicLong mockTime = new AtomicLong(0);
+        LongSupplier mockTimeNanosSupplier = mockTime::get;
+
+        EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
+            {
+                put(JVM, new NodeDuressTracker(() -> false, () -> 3));
+                put(CPU, new NodeDuressTracker(() -> true, () -> 3));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers);
+
+        // Creating heap and cpu usage trackers where heap tracker will always evaluate with reasons to cancel the
+        // tasks but heap based cancellation should not happen because heap is not in duress
+        TaskResourceUsageTracker heapUsageTracker = getMockedTaskResourceUsageTracker(
+            TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER,
+            (task) -> Optional.of(new TaskCancellation.Reason("mem exceeded", 10))
+        );
+        TaskResourceUsageTracker cpuUsageTracker = getMockedTaskResourceUsageTracker(
+            TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
+            (task) -> {
+                if (task.getTotalResourceStats().getCpuTimeInNanos() < 400) {
+                    return Optional.empty();
+                }
+                return Optional.of(new TaskCancellation.Reason("cpu time limit exceeded", 5));
+            }
+        );
+
+        TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
+        taskResourceUsageTrackers.addTracker(cpuUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
+        taskResourceUsageTrackers.addTracker(heapUsageTracker, TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER);
+
+        // Mocking 'settings' with predictable rate limiting thresholds.
+        SearchBackpressureSettings settings = getBackpressureSettings("enforced", 0.1, 0.003, 10.0);
+
+        SearchBackpressureService service = spy(
+            new SearchBackpressureService(
+                settings,
+                mockTaskResourceTrackingService,
+                threadPool,
+                mockTimeNanosSupplier,
+                nodeDuressTrackers,
+                taskResourceUsageTrackers,
+                new TaskResourceUsageTrackers(),
+                mockTaskManager
+            )
+        );
+
+        when(service.isHeapUsageDominatedBySearch(anyList(), anyDouble())).thenReturn(false);
+
+        service.doRun();
+        service.doRun();
+
+        SearchTaskSettings searchTaskSettings = mock(SearchTaskSettings.class);
+        // setting the total heap percent threshold to minimum so that circuit does not break in SearchBackpressureService
+        when(searchTaskSettings.getTotalHeapPercentThreshold()).thenReturn(0.0);
+        when(settings.getSearchTaskSettings()).thenReturn(searchTaskSettings);
+
+        // Create a mix of low and high resource usage tasks (60 low + 15 high resource usage tasks).
+        Map<Long, Task> activeSearchTasks = new HashMap<>();
+        for (long i = 0; i < 75; i++) {
+            Class<? extends CancellableTask> taskType = randomBoolean() ? SearchTask.class : SearchShardTask.class;
+            if (i % 5 == 0) {
+                activeSearchTasks.put(i, createMockTaskWithResourceStats(taskType, 500, 800, i));
+            } else {
+                activeSearchTasks.put(i, createMockTaskWithResourceStats(taskType, 100, 800, i));
+            }
+        }
+        doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks();
+
+        // this will trigger cancellation but the cancellation should not happen as the node is not is duress because of search traffic
+        service.doRun();
+
+        verify(mockTaskManager, times(0)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any());
+        assertEquals(0, service.getSearchBackpressureState(SearchTask.class).getCancellationCount());
+        assertEquals(0, service.getSearchBackpressureState(SearchShardTask.class).getCancellationCount());
+    }
+
     private SearchBackpressureSettings getBackpressureSettings(String mode, double ratio, double rate, double burst) {
         return spy(
             new SearchBackpressureSettings(
@@ -344,11 +577,14 @@ private SearchBackpressureSettings getBackpressureSettings(String mode, double r
         );
     }
 
-    private TaskResourceUsageTracker getMockedTaskResourceUsageTracker() {
+    private TaskResourceUsageTracker getMockedTaskResourceUsageTracker(
+        TaskResourceUsageTrackerType type,
+        TaskResourceUsageTracker.ResourceUsageBreachEvaluator evaluator
+    ) {
         return new TaskResourceUsageTracker() {
             @Override
             public String name() {
-                return TaskResourceUsageTrackerType.CPU_USAGE_TRACKER.getName();
+                return type.getName();
             }
 
             @Override
@@ -356,11 +592,7 @@ public void update(Task task) {}
 
             @Override
             public Optional<TaskCancellation.Reason> checkAndMaybeGetCancellationReason(Task task) {
-                if (task.getTotalResourceStats().getCpuTimeInNanos() < 300) {
-                    return Optional.empty();
-                }
-
-                return Optional.of(new TaskCancellation.Reason("limits exceeded", 5));
+                return evaluator.evaluate(task);
             }
 
             @Override
diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java
index f28b82cad30d3..45a44136d41f7 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java
@@ -12,8 +12,8 @@
 import org.opensearch.search.backpressure.trackers.CpuUsageTracker;
 import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker;
 import org.opensearch.search.backpressure.trackers.HeapUsageTracker;
-import org.opensearch.search.backpressure.trackers.TaskResourceUsageTracker;
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.test.AbstractWireSerializingTestCase;
 
 import java.util.Map;
diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java
index cc7aa92826b41..3ac5cfd658fc3 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java
@@ -12,8 +12,8 @@
 import org.opensearch.search.backpressure.trackers.CpuUsageTracker;
 import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker;
 import org.opensearch.search.backpressure.trackers.HeapUsageTracker;
-import org.opensearch.search.backpressure.trackers.TaskResourceUsageTracker;
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.test.AbstractWireSerializingTestCase;
 
 import java.util.Map;
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/CpuUsageTrackerTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/CpuUsageTrackerTests.java
index 8cdcbc7511bd2..0117b0ed71c27 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/trackers/CpuUsageTrackerTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/CpuUsageTrackerTests.java
@@ -33,7 +33,7 @@ public class CpuUsageTrackerTests extends OpenSearchTestCase {
     );
 
     public void testSearchTaskEligibleForCancellation() {
-        Task task = createMockTaskWithResourceStats(SearchTask.class, 100000000, 200);
+        Task task = createMockTaskWithResourceStats(SearchTask.class, 100000000, 200, randomNonNegativeLong());
         CpuUsageTracker tracker = new CpuUsageTracker(mockSettings.getSearchTaskSettings()::getCpuTimeNanosThreshold);
 
         Optional<TaskCancellation.Reason> reason = tracker.checkAndMaybeGetCancellationReason(task);
@@ -43,7 +43,7 @@ public void testSearchTaskEligibleForCancellation() {
     }
 
     public void testSearchShardTaskEligibleForCancellation() {
-        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 200000000, 200);
+        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 200000000, 200, randomNonNegativeLong());
         CpuUsageTracker tracker = new CpuUsageTracker(mockSettings.getSearchShardTaskSettings()::getCpuTimeNanosThreshold);
 
         Optional<TaskCancellation.Reason> reason = tracker.checkAndMaybeGetCancellationReason(task);
@@ -53,7 +53,7 @@ public void testSearchShardTaskEligibleForCancellation() {
     }
 
     public void testNotEligibleForCancellation() {
-        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 5000000, 200);
+        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 5000000, 200, randomNonNegativeLong());
         CpuUsageTracker tracker = new CpuUsageTracker(mockSettings.getSearchShardTaskSettings()::getCpuTimeNanosThreshold);
 
         Optional<TaskCancellation.Reason> reason = tracker.checkAndMaybeGetCancellationReason(task);
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTrackerTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTrackerTests.java
index 921d01e7355a7..514f1b4785aa1 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTrackerTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTrackerTests.java
@@ -47,7 +47,7 @@ public void testSearchTaskEligibleForCancellation() {
     }
 
     public void testSearchShardTaskEligibleForCancellation() {
-        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 1, 0);
+        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 1, 0, randomNonNegativeLong());
         ElapsedTimeTracker tracker = new ElapsedTimeTracker(
             mockSettings.getSearchShardTaskSettings()::getElapsedTimeNanosThreshold,
             () -> 200000000
@@ -60,7 +60,7 @@ public void testSearchShardTaskEligibleForCancellation() {
     }
 
     public void testNotEligibleForCancellation() {
-        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 1, 150000000);
+        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 1, 150000000, randomNonNegativeLong());
         ElapsedTimeTracker tracker = new ElapsedTimeTracker(
             mockSettings.getSearchShardTaskSettings()::getElapsedTimeNanosThreshold,
             () -> 200000000
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/HeapUsageTrackerTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/HeapUsageTrackerTests.java
index 3950d00b0c8b5..1c46305e9fda6 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/trackers/HeapUsageTrackerTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/HeapUsageTrackerTests.java
@@ -58,7 +58,7 @@ public void testSearchTaskEligibleForCancellation() {
                 SearchTaskSettings.SETTING_HEAP_MOVING_AVERAGE_WINDOW_SIZE
             )
         );
-        Task task = createMockTaskWithResourceStats(SearchTask.class, 1, 50);
+        Task task = createMockTaskWithResourceStats(SearchTask.class, 1, 50, randomNonNegativeLong());
 
         // Record enough observations to make the moving average 'ready'.
         for (int i = 0; i < HEAP_MOVING_AVERAGE_WINDOW_SIZE; i++) {
@@ -66,7 +66,7 @@ public void testSearchTaskEligibleForCancellation() {
         }
 
         // Task that has heap usage >= heapBytesThreshold and (movingAverage * heapVariance).
-        task = createMockTaskWithResourceStats(SearchTask.class, 1, 300);
+        task = createMockTaskWithResourceStats(SearchTask.class, 1, 300, randomNonNegativeLong());
         Optional<TaskCancellation.Reason> reason = tracker.checkAndMaybeGetCancellationReason(task);
         assertTrue(reason.isPresent());
         assertEquals(6, reason.get().getCancellationScore());
@@ -88,7 +88,7 @@ public void testSearchShardTaskEligibleForCancellation() {
                 SearchShardTaskSettings.SETTING_HEAP_MOVING_AVERAGE_WINDOW_SIZE
             )
         );
-        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 50);
+        Task task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 50, randomNonNegativeLong());
 
         // Record enough observations to make the moving average 'ready'.
         for (int i = 0; i < HEAP_MOVING_AVERAGE_WINDOW_SIZE; i++) {
@@ -96,7 +96,7 @@ public void testSearchShardTaskEligibleForCancellation() {
         }
 
         // Task that has heap usage >= heapBytesThreshold and (movingAverage * heapVariance).
-        task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 200);
+        task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 200, randomNonNegativeLong());
         Optional<TaskCancellation.Reason> reason = tracker.checkAndMaybeGetCancellationReason(task);
         assertTrue(reason.isPresent());
         assertEquals(4, reason.get().getCancellationScore());
@@ -122,7 +122,7 @@ public void testNotEligibleForCancellation() {
         );
 
         // Task with heap usage < heapBytesThreshold.
-        task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 99);
+        task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 99, randomNonNegativeLong());
 
         // Not enough observations.
         reason = tracker.checkAndMaybeGetCancellationReason(task);
@@ -139,7 +139,12 @@ public void testNotEligibleForCancellation() {
 
         // Task with heap usage between heapBytesThreshold and (movingAverage * heapVariance) should not be cancelled.
         double allowedHeapUsage = 99.0 * 2.0;
-        task = createMockTaskWithResourceStats(SearchShardTask.class, 1, randomLongBetween(99, (long) allowedHeapUsage - 1));
+        task = createMockTaskWithResourceStats(
+            SearchShardTask.class,
+            1,
+            randomLongBetween(99, (long) allowedHeapUsage - 1),
+            randomNonNegativeLong()
+        );
         reason = tracker.checkAndMaybeGetCancellationReason(task);
         assertFalse(reason.isPresent());
     }
@@ -148,12 +153,12 @@ public void testIsHeapUsageDominatedBySearch() {
         assumeTrue("Skip the test if the hardware doesn't support heap usage tracking", HeapUsageTracker.isHeapTrackingSupported());
 
         // task with 1 byte of heap usage so that it does not breach the threshold
-        CancellableTask task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 1);
+        CancellableTask task = createMockTaskWithResourceStats(SearchShardTask.class, 1, 1, randomNonNegativeLong());
         assertFalse(HeapUsageTracker.isHeapUsageDominatedBySearch(List.of(task), 0.5));
 
         long totalHeap = JvmStats.jvmStats().getMem().getHeapMax().getBytes();
         // task with heap usage of [totalHeap - 1] so that it breaches the threshold
-        task = createMockTaskWithResourceStats(SearchShardTask.class, 1, totalHeap - 1);
+        task = createMockTaskWithResourceStats(SearchShardTask.class, 1, totalHeap - 1, randomNonNegativeLong());
         assertTrue(HeapUsageTracker.isHeapUsageDominatedBySearch(List.of(task), 0.5));
     }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackerTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackerTests.java
index 472ba95566523..32aca6ac3230e 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackerTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackerTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.search.backpressure.trackers;
 
+import org.opensearch.search.backpressure.trackers.NodeDuressTrackers.NodeDuressTracker;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.concurrent.atomic.AtomicReference;
@@ -16,20 +17,20 @@ public class NodeDuressTrackerTests extends OpenSearchTestCase {
 
     public void testNodeDuressTracker() {
         AtomicReference<Double> cpuUsage = new AtomicReference<>(0.0);
-        NodeDuressTracker tracker = new NodeDuressTracker(() -> cpuUsage.get() >= 0.5);
+        NodeDuressTracker tracker = new NodeDuressTracker(() -> cpuUsage.get() >= 0.5, () -> 3);
 
         // Node not in duress.
-        assertEquals(0, tracker.check());
+        assertFalse(tracker.test());
 
         // Node in duress; the streak must keep increasing.
         cpuUsage.set(0.7);
-        assertEquals(1, tracker.check());
-        assertEquals(2, tracker.check());
-        assertEquals(3, tracker.check());
+        assertFalse(tracker.test());
+        assertFalse(tracker.test());
+        assertTrue(tracker.test());
 
         // Node not in duress anymore.
         cpuUsage.set(0.3);
-        assertEquals(0, tracker.check());
-        assertEquals(0, tracker.check());
+        assertFalse(tracker.test());
+        assertFalse(tracker.test());
     }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
new file mode 100644
index 0000000000000..2db251ee461db
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
@@ -0,0 +1,84 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.trackers;
+
+import org.opensearch.search.ResourceType;
+import org.opensearch.search.backpressure.trackers.NodeDuressTrackers.NodeDuressTracker;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.EnumMap;
+
+public class NodeDuressTrackersTests extends OpenSearchTestCase {
+
+    public void testNodeNotInDuress() {
+        EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.JVM, new NodeDuressTracker(() -> false, () -> 2));
+                put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 2));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map);
+
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+    }
+
+    public void testNodeInDuressWhenHeapInDuress() {
+        EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.JVM, new NodeDuressTracker(() -> true, () -> 3));
+                put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 1));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map);
+
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+
+        // for the third time it should be in duress
+        assertTrue(nodeDuressTrackers.isNodeInDuress());
+    }
+
+    public void testNodeInDuressWhenCPUInDuress() {
+        EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.JVM, new NodeDuressTracker(() -> false, () -> 1));
+                put(ResourceType.CPU, new NodeDuressTracker(() -> true, () -> 3));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map);
+
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+
+        // for the third time it should be in duress
+        assertTrue(nodeDuressTrackers.isNodeInDuress());
+    }
+
+    public void testNodeInDuressWhenCPUAndHeapInDuress() {
+        EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.JVM, new NodeDuressTracker(() -> true, () -> 3));
+                put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 3));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map);
+
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+
+        // for the third time it should be in duress
+        assertTrue(nodeDuressTrackers.isNodeInDuress());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/tasks/TaskCancellationTests.java b/server/src/test/java/org/opensearch/tasks/TaskCancellationTests.java
index e74f89c905499..f08c12ea258ca 100644
--- a/server/src/test/java/org/opensearch/tasks/TaskCancellationTests.java
+++ b/server/src/test/java/org/opensearch/tasks/TaskCancellationTests.java
@@ -9,7 +9,7 @@
 package org.opensearch.tasks;
 
 import org.opensearch.action.search.SearchShardTask;
-import org.opensearch.search.backpressure.trackers.TaskResourceUsageTracker;
+import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
@@ -69,7 +69,7 @@ public Optional<TaskCancellation.Reason> checkAndMaybeGetCancellationReason(Task
             }
 
             @Override
-            public Stats stats(List<? extends Task> activeTasks) {
+            public TaskResourceUsageTracker.Stats stats(List<? extends Task> activeTasks) {
                 return null;
             }
         };
diff --git a/test/framework/src/main/java/org/opensearch/search/backpressure/SearchBackpressureTestHelpers.java b/test/framework/src/main/java/org/opensearch/search/backpressure/SearchBackpressureTestHelpers.java
index af06b1688dca2..8f31f2a60ea86 100644
--- a/test/framework/src/main/java/org/opensearch/search/backpressure/SearchBackpressureTestHelpers.java
+++ b/test/framework/src/main/java/org/opensearch/search/backpressure/SearchBackpressureTestHelpers.java
@@ -21,19 +21,21 @@
 
 public class SearchBackpressureTestHelpers extends OpenSearchTestCase {
 
-    public static <T extends CancellableTask> T createMockTaskWithResourceStats(Class<T> type, long cpuUsage, long heapUsage) {
-        return createMockTaskWithResourceStats(type, cpuUsage, heapUsage, 0);
+    public static <T extends CancellableTask> T createMockTaskWithResourceStats(Class<T> type, long cpuUsage, long heapUsage, long taskId) {
+        return createMockTaskWithResourceStats(type, cpuUsage, heapUsage, 0, taskId);
     }
 
     public static <T extends CancellableTask> T createMockTaskWithResourceStats(
         Class<T> type,
         long cpuUsage,
         long heapUsage,
-        long startTimeNanos
+        long startTimeNanos,
+        long taskId
     ) {
         T task = mock(type);
         when(task.getTotalResourceStats()).thenReturn(new TaskResourceUsage(cpuUsage, heapUsage));
         when(task.getStartTimeNanos()).thenReturn(startTimeNanos);
+        when(task.getId()).thenReturn(randomNonNegativeLong());
 
         AtomicBoolean isCancelled = new AtomicBoolean(false);
         doAnswer(invocation -> {

From f5dbbb04b96bb8866255e4f967241db8bed16141 Mon Sep 17 00:00:00 2001
From: kkewwei <kkewwei@163.com>
Date: Sat, 22 Jun 2024 01:43:27 +0800
Subject: [PATCH 037/113] Fix aggs result of NestedAggregator with sub
 NestedAggregator (#13324)

Signed-off-by: kkewwei <kkewwei@163.com>
---
 CHANGELOG.md                                  |   1 +
 .../bucket/nested/NestedAggregator.java       |  84 ++++-
 .../bucket/nested/NestedAggregatorTests.java  | 332 +++++++++++++++++-
 .../aggregations/AggregatorTestCase.java      |  16 +
 4 files changed, 399 insertions(+), 34 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 41d9c4f4f3905..c0266d2a6dd4e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
 - Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
 - Write shard level metadata blob when snapshotting searchable snapshot indexes ([#13190](https://github.com/opensearch-project/OpenSearch/pull/13190))
+- Fix aggs result of NestedAggregator with sub NestedAggregator ([#13324](https://github.com/opensearch-project/OpenSearch/pull/13324))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java
index cfa1d32a52501..150efa878f866 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java
@@ -43,8 +43,10 @@
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.util.BitSet;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.lucene.search.Queries;
 import org.opensearch.core.ParseField;
+import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.ObjectMapper;
 import org.opensearch.search.aggregations.Aggregator;
 import org.opensearch.search.aggregations.AggregatorFactories;
@@ -88,12 +90,25 @@ public class NestedAggregator extends BucketsAggregator implements SingleBucketA
     ) throws IOException {
         super(name, factories, context, parent, cardinality, metadata);
 
-        Query parentFilter = parentObjectMapper != null ? parentObjectMapper.nestedTypeFilter() : Queries.newNonNestedFilter();
+        Query parentFilter = isParent(parentObjectMapper, childObjectMapper, context.mapperService())
+            ? parentObjectMapper.nestedTypeFilter()
+            : Queries.newNonNestedFilter();
         this.parentFilter = context.bitsetFilterCache().getBitSetProducer(parentFilter);
         this.childFilter = childObjectMapper.nestedTypeFilter();
         this.collectsFromSingleBucket = cardinality.map(estimate -> estimate < 2);
     }
 
+    private boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) {
+        if (parentObjectMapper == null) {
+            return false;
+        }
+        ObjectMapper parent;
+        do {
+            parent = childObjectMapper.getParentObjectMapper(mapperService);
+        } while (parent != null && parent != parentObjectMapper);
+        return parentObjectMapper == parent;
+    }
+
     @Override
     public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
         IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx);
@@ -107,20 +122,17 @@ public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final L
         if (collectsFromSingleBucket) {
             return new LeafBucketCollectorBase(sub, null) {
                 @Override
-                public void collect(int parentDoc, long bucket) throws IOException {
-                    // if parentDoc is 0 then this means that this parent doesn't have child docs (b/c these appear always before the parent
-                    // doc), so we can skip:
-                    if (parentDoc == 0 || parentDocs == null || childDocs == null) {
+                public void collect(int parentAggDoc, long bucket) throws IOException {
+                    // parentAggDoc can be 0 when aggregation:
+                    if (parentDocs == null || childDocs == null) {
                         return;
                     }
 
-                    final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
-                    int childDocId = childDocs.docID();
-                    if (childDocId <= prevParentDoc) {
-                        childDocId = childDocs.advance(prevParentDoc + 1);
-                    }
+                    Tuple<Integer, Integer> res = getParentAndChildId(parentDocs, childDocs, parentAggDoc);
+                    int currentParentDoc = res.v1();
+                    int childDocId = res.v2();
 
-                    for (; childDocId < parentDoc; childDocId = childDocs.nextDoc()) {
+                    for (; childDocId < currentParentDoc; childDocId = childDocs.nextDoc()) {
                         collectBucket(sub, childDocId, bucket);
                     }
                 }
@@ -130,6 +142,43 @@ public void collect(int parentDoc, long bucket) throws IOException {
         }
     }
 
+    /**
+     * In one case, it's talking about the parent doc (from the Lucene block-join standpoint),
+     * while in the other case, it's talking about a child doc ID (from the block-join standpoint)
+     * from the parent aggregation, where we're trying to aggregate over a sibling of that child.
+     * So, we need to map from that document to its parent, then join to the appropriate sibling.
+     *
+     * @param parentAggDoc the parent aggregation's current doc
+     *                    (which may or may not be a block-level parent doc)
+     * @return a tuple consisting of the current block-level parent doc (the parent of the
+     *         parameter doc), and the next matching child doc (hopefully under this parent)
+     *         for the aggregation (according to the child doc iterator).
+     */
+    static Tuple<Integer, Integer> getParentAndChildId(BitSet parentDocs, DocIdSetIterator childDocs, int parentAggDoc) throws IOException {
+        int currentParentAggDoc;
+        int prevParentDoc = parentDocs.prevSetBit(parentAggDoc);
+        if (prevParentDoc == -1) {
+            currentParentAggDoc = parentDocs.nextSetBit(0);
+        } else if (prevParentDoc == parentAggDoc) {
+            // parentAggDoc is the parent of that child, and is belongs to parentDocs
+            currentParentAggDoc = parentAggDoc;
+            if (currentParentAggDoc == 0) {
+                prevParentDoc = -1;
+            } else {
+                prevParentDoc = parentDocs.prevSetBit(currentParentAggDoc - 1);
+            }
+        } else {
+            // parentAggDoc is the sibling of that child, and it means the block-join parent
+            currentParentAggDoc = parentDocs.nextSetBit(prevParentDoc + 1);
+        }
+
+        int childDocId = childDocs.docID();
+        if (childDocId <= prevParentDoc) {
+            childDocId = childDocs.advance(prevParentDoc + 1);
+        }
+        return Tuple.tuple(currentParentAggDoc, childDocId);
+    }
+
     @Override
     protected void preGetSubLeafCollectors(LeafReaderContext ctx) throws IOException {
         super.preGetSubLeafCollectors(ctx);
@@ -191,9 +240,8 @@ public void setScorer(Scorable scorer) throws IOException {
 
         @Override
         public void collect(int parentDoc, long bucket) throws IOException {
-            // if parentDoc is 0 then this means that this parent doesn't have child docs (b/c these appear always before the parent
-            // doc), so we can skip:
-            if (parentDoc == 0 || parentDocs == null || childDocs == null) {
+            // parentAggDoc can be 0 when aggregation:
+            if (parentDocs == null || childDocs == null) {
                 return;
             }
 
@@ -214,11 +262,9 @@ void processBufferedChildBuckets() throws IOException {
                 return;
             }
 
-            final int prevParentDoc = parentDocs.prevSetBit(currentParentDoc - 1);
-            int childDocId = childDocs.docID();
-            if (childDocId <= prevParentDoc) {
-                childDocId = childDocs.advance(prevParentDoc + 1);
-            }
+            Tuple<Integer, Integer> res = getParentAndChildId(parentDocs, childDocs, currentParentDoc);
+            int currentParentDoc = res.v1();
+            int childDocId = res.v2();
 
             for (; childDocId < currentParentDoc; childDocId = childDocs.nextDoc()) {
                 cachedScorer.doc = childDocId;
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index 406c411494d60..c7fbca538c6ee 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -45,23 +46,36 @@
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
 import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.lucene.search.Queries;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.cache.bitset.BitsetFilterCache;
+import org.opensearch.index.mapper.ContentPath;
 import org.opensearch.index.mapper.IdFieldMapper;
 import org.opensearch.index.mapper.KeywordFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.Mapper;
 import org.opensearch.index.mapper.NestedPathFieldMapper;
 import org.opensearch.index.mapper.NumberFieldMapper;
+import org.opensearch.index.mapper.ObjectMapper;
 import org.opensearch.index.mapper.SeqNoFieldMapper;
 import org.opensearch.index.mapper.Uid;
 import org.opensearch.index.query.MatchAllQueryBuilder;
+import org.opensearch.index.query.NestedQueryBuilder;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.query.TermsQueryBuilder;
+import org.opensearch.index.query.support.NestedScope;
 import org.opensearch.script.MockScriptEngine;
 import org.opensearch.script.Script;
 import org.opensearch.script.ScriptEngine;
@@ -104,20 +118,34 @@
 import java.util.stream.DoubleStream;
 import java.util.stream.LongStream;
 
+import org.mockito.Mockito;
+
 import static java.util.stream.Collectors.toList;
 import static org.opensearch.search.aggregations.AggregationBuilders.max;
 import static org.opensearch.search.aggregations.AggregationBuilders.nested;
+import static org.opensearch.search.aggregations.bucket.nested.NestedAggregator.getParentAndChildId;
+import static org.opensearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 import static org.hamcrest.Matchers.equalTo;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class NestedAggregatorTests extends AggregatorTestCase {
 
     private static final String VALUE_FIELD_NAME = "number";
+    private static final String VALUE_FIELD_NAME2 = "number2";
     private static final String NESTED_OBJECT = "nested_object";
     private static final String NESTED_OBJECT2 = "nested_object2";
     private static final String NESTED_AGG = "nestedAgg";
     private static final String MAX_AGG_NAME = "maxAgg";
     private static final String SUM_AGG_NAME = "sumAgg";
     private static final String INVERSE_SCRIPT = "inverse";
+    private static final String OUT_NESTED = "outNested";
+    private static final String OUT_TERMS = "outTerms";
+    private static final String INNER_NESTED = "innerNested";
+    private static final String INNER_TERMS = "innerTerms";
 
     private static final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
 
@@ -201,17 +229,22 @@ public void testSingleNestingMax() throws IOException {
             }
             try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
                 NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG, NESTED_OBJECT);
-                MaxAggregationBuilder maxAgg = new MaxAggregationBuilder(MAX_AGG_NAME).field(VALUE_FIELD_NAME);
+                MaxAggregationBuilder maxAgg = new MaxAggregationBuilder(MAX_AGG_NAME).field(NESTED_OBJECT + "." + VALUE_FIELD_NAME);
                 nestedBuilder.subAggregation(maxAgg);
-                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG);
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+                    NESTED_OBJECT + "." + VALUE_FIELD_NAME,
+                    NumberFieldMapper.NumberType.LONG
+                );
 
                 InternalNested nested = searchAndReduce(
+                    createIndexSettings(),
                     newSearcher(indexReader, false, true),
                     new MatchAllDocsQuery(),
                     nestedBuilder,
+                    DEFAULT_MAX_BUCKETS,
+                    true,
                     fieldType
                 );
-                assertEquals(expectedNestedDocs, nested.getDocCount());
 
                 assertEquals(NESTED_AGG, nested.getName());
                 assertEquals(expectedNestedDocs, nested.getDocCount());
@@ -240,7 +273,7 @@ public void testDoubleNestingMax() throws IOException {
                     int numNestedDocs = randomIntBetween(0, 20);
                     expectedMaxValue = Math.max(
                         expectedMaxValue,
-                        generateMaxDocs(documents, numNestedDocs, i, NESTED_OBJECT + "." + NESTED_OBJECT2, VALUE_FIELD_NAME)
+                        generateMaxDocs(documents, numNestedDocs, i, NESTED_OBJECT, VALUE_FIELD_NAME)
                     );
                     expectedNestedDocs += numNestedDocs;
 
@@ -253,19 +286,24 @@ public void testDoubleNestingMax() throws IOException {
                 iw.commit();
             }
             try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
-                NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG, NESTED_OBJECT + "." + NESTED_OBJECT2);
-                MaxAggregationBuilder maxAgg = new MaxAggregationBuilder(MAX_AGG_NAME).field(VALUE_FIELD_NAME);
+                NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG, NESTED_OBJECT);
+                MaxAggregationBuilder maxAgg = new MaxAggregationBuilder(MAX_AGG_NAME).field(NESTED_OBJECT + "." + VALUE_FIELD_NAME);
                 nestedBuilder.subAggregation(maxAgg);
 
-                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG);
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+                    NESTED_OBJECT + "." + VALUE_FIELD_NAME,
+                    NumberFieldMapper.NumberType.LONG
+                );
 
                 InternalNested nested = searchAndReduce(
+                    createIndexSettings(),
                     newSearcher(indexReader, false, true),
                     new MatchAllDocsQuery(),
                     nestedBuilder,
+                    DEFAULT_MAX_BUCKETS,
+                    true,
                     fieldType
                 );
-                assertEquals(expectedNestedDocs, nested.getDocCount());
 
                 assertEquals(NESTED_AGG, nested.getName());
                 assertEquals(expectedNestedDocs, nested.getDocCount());
@@ -310,17 +348,22 @@ public void testOrphanedDocs() throws IOException {
             }
             try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
                 NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG, NESTED_OBJECT);
-                SumAggregationBuilder sumAgg = new SumAggregationBuilder(SUM_AGG_NAME).field(VALUE_FIELD_NAME);
+                SumAggregationBuilder sumAgg = new SumAggregationBuilder(SUM_AGG_NAME).field(NESTED_OBJECT + "." + VALUE_FIELD_NAME);
                 nestedBuilder.subAggregation(sumAgg);
-                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG);
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+                    NESTED_OBJECT + "." + VALUE_FIELD_NAME,
+                    NumberFieldMapper.NumberType.LONG
+                );
 
                 InternalNested nested = searchAndReduce(
+                    createIndexSettings(),
                     newSearcher(indexReader, false, true),
                     new MatchAllDocsQuery(),
                     nestedBuilder,
+                    DEFAULT_MAX_BUCKETS,
+                    true,
                     fieldType
                 );
-                assertEquals(expectedNestedDocs, nested.getDocCount());
 
                 assertEquals(NESTED_AGG, nested.getName());
                 assertEquals(expectedNestedDocs, nested.getDocCount());
@@ -747,8 +790,24 @@ public void testFieldAlias() throws IOException {
                     max(MAX_AGG_NAME).field(VALUE_FIELD_NAME + "-alias")
                 );
 
-                InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), agg, fieldType);
-                Nested aliasNested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), aliasAgg, fieldType);
+                InternalNested nested = searchAndReduce(
+                    createIndexSettings(),
+                    newSearcher(indexReader, false, true),
+                    new MatchAllDocsQuery(),
+                    agg,
+                    DEFAULT_MAX_BUCKETS,
+                    true,
+                    fieldType
+                );
+                Nested aliasNested = searchAndReduce(
+                    createIndexSettings(),
+                    newSearcher(indexReader, false, true),
+                    new MatchAllDocsQuery(),
+                    aliasAgg,
+                    DEFAULT_MAX_BUCKETS,
+                    true,
+                    fieldType
+                );
 
                 assertEquals(nested, aliasNested);
                 assertEquals(expectedNestedDocs, nested.getDocCount());
@@ -796,13 +855,15 @@ public void testNestedWithPipeline() throws IOException {
                 MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG);
 
                 InternalNested nested = searchAndReduce(
+                    createIndexSettings(),
                     newSearcher(indexReader, false, true),
                     new MatchAllDocsQuery(),
                     nestedBuilder,
+                    DEFAULT_MAX_BUCKETS,
+                    true,
                     fieldType
                 );
 
-                assertEquals(expectedNestedDocs, nested.getDocCount());
                 assertEquals(NESTED_AGG, nested.getName());
                 assertEquals(expectedNestedDocs, nested.getDocCount());
 
@@ -853,6 +914,238 @@ public void testNestedUnderTerms() throws IOException {
         }, resellersMappedFields());
     }
 
+    public void testBufferingNestedLeafBucketCollector() throws IOException {
+        int numRootDocs = scaledRandomIntBetween(2, 200);
+        int expectedNestedDocs;
+        String[] bucketKeys;
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < numRootDocs; i++) {
+
+                    List<Document> documents = new ArrayList<>();
+                    if (randomBoolean()) {
+                        generateDocument(documents, i, NESTED_OBJECT, VALUE_FIELD_NAME, 1);
+                        generateDocument(documents, i, NESTED_OBJECT2, VALUE_FIELD_NAME2, i);
+                    } else {
+                        generateDocument(documents, i, NESTED_OBJECT2, VALUE_FIELD_NAME2, i);
+                        generateDocument(documents, i, NESTED_OBJECT, VALUE_FIELD_NAME, 1);
+                    }
+                    Document document = new Document();
+                    document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE));
+                    document.add(sequenceIDFields.primaryTerm);
+                    documents.add(document);
+                    iw.addDocuments(documents);
+                }
+                iw.commit();
+            }
+            try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
+                IndexSettings indexSettings = createIndexSettings();
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+                    NESTED_OBJECT + "." + VALUE_FIELD_NAME,
+                    NumberFieldMapper.NumberType.LONG
+                );
+                MappedFieldType fieldType1 = new NumberFieldMapper.NumberFieldType(
+                    NESTED_OBJECT2 + "." + VALUE_FIELD_NAME2,
+                    NumberFieldMapper.NumberType.LONG
+                );
+                QueryShardContext queryShardContext = createQueryShardContext(NESTED_OBJECT2, indexSettings, fieldType1);
+                // query
+                expectedNestedDocs = numRootDocs / 2;
+                bucketKeys = new String[expectedNestedDocs];
+                BytesRef[] values = new BytesRef[numRootDocs / 2];
+                for (int i = 0; i < numRootDocs / 2; i++) {
+                    bucketKeys[i] = "" + (i * 2);
+                    values[i] = new BytesRef(bucketKeys[i]);
+                }
+                TermsQueryBuilder termsQueryBuilder = new TermsQueryBuilder(NESTED_OBJECT2 + "." + VALUE_FIELD_NAME2, (Object[]) values);
+                NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(NESTED_OBJECT2, termsQueryBuilder, ScoreMode.None);
+
+                // out nested aggs
+                NestedAggregationBuilder outNestedBuilder = new NestedAggregationBuilder(OUT_NESTED, NESTED_OBJECT);
+                TermsAggregationBuilder outTermsAggregator = new TermsAggregationBuilder(OUT_TERMS).field(
+                    NESTED_OBJECT + "." + VALUE_FIELD_NAME
+                ).size(100);
+                outNestedBuilder.subAggregation(outTermsAggregator);
+
+                // inner nested aggs
+                NestedAggregationBuilder innerNestedBuilder = new NestedAggregationBuilder(INNER_NESTED, NESTED_OBJECT2);
+                TermsAggregationBuilder innerTermsAggregator = new TermsAggregationBuilder(INNER_TERMS).field(
+                    NESTED_OBJECT2 + "." + VALUE_FIELD_NAME2
+                ).size(100);
+                innerNestedBuilder.subAggregation(innerTermsAggregator);
+                outTermsAggregator.subAggregation(innerNestedBuilder);
+
+                InternalNested nested = searchAndReduce(
+                    indexSettings,
+                    newSearcher(indexReader, false, true),
+                    nestedQueryBuilder.toQuery(queryShardContext),
+                    outNestedBuilder,
+                    DEFAULT_MAX_BUCKETS,
+                    true,
+                    fieldType,
+                    fieldType1
+                );
+
+                assertEquals(OUT_NESTED, nested.getName());
+                assertEquals(expectedNestedDocs, nested.getDocCount());
+
+                LongTerms outTerms = (LongTerms) nested.getProperty(OUT_TERMS);
+                assertEquals(1, outTerms.getBuckets().size());
+
+                InternalNested internalNested = (InternalNested) (((Object[]) outTerms.getProperty(INNER_NESTED))[0]);
+                assertEquals(expectedNestedDocs, internalNested.getDocCount());
+
+                LongTerms innerTerms = (LongTerms) internalNested.getProperty(INNER_TERMS);
+                assertEquals(bucketKeys.length, innerTerms.getBuckets().size());
+                for (int i = 0; i < expectedNestedDocs; i++) {
+                    LongTerms.Bucket bucket = innerTerms.getBuckets().get(i);
+                    assertEquals(bucketKeys[i], bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+            }
+        }
+    }
+
+    private DocIdSetIterator getDocIdSetIterator(int[] value) {
+        int[] bits = new int[value[value.length - 1] + 1];
+        for (int i : value) {
+            bits[i] = 1;
+        }
+        return new DocIdSetIterator() {
+            int index = -1;
+
+            @Override
+            public int docID() {
+                if (index == -1 || index > bits.length || bits[index] != 1) {
+                    return -1;
+                }
+                return index;
+            }
+
+            @Override
+            public int nextDoc() {
+                for (int i = index; i < bits.length; i++) {
+                    if (bits[i] == 1) {
+                        index = i;
+                        return index;
+                    }
+                }
+                index = bits.length;
+                return NO_MORE_DOCS;
+            }
+
+            @Override
+            public int advance(int target) {
+                for (int i = target; i < bits.length; i++) {
+                    if (bits[i] == 1) {
+                        index = i;
+                        return index;
+                    }
+                }
+                index = bits.length;
+                return NO_MORE_DOCS;
+            }
+
+            @Override
+            public long cost() {
+                return bits.length;
+            }
+        };
+    }
+
+    public void testGetParentAndChildId() throws IOException {
+        {
+            // p: parent c: child
+            // [p0], [p1], [c2,p3], [c4,x5,p6], [p7], [p8]
+            BitSet parentDocs = new FixedBitSet(20);
+            parentDocs.set(0);
+            parentDocs.set(1);
+            parentDocs.set(3);
+            parentDocs.set(6);
+            parentDocs.set(7);
+            parentDocs.set(8);
+            DocIdSetIterator childDocs = getDocIdSetIterator(new int[] { 2, 4 });
+
+            Tuple<Integer, Integer> res = getParentAndChildId(parentDocs, childDocs, 0);
+            assertEquals(0, res.v1().intValue());
+            assertEquals(2, res.v2().intValue());
+
+            res = getParentAndChildId(parentDocs, childDocs, 3);
+            assertEquals(3, res.v1().intValue());
+            assertEquals(2, res.v2().intValue());
+
+            res = getParentAndChildId(parentDocs, childDocs, 4);
+            assertEquals(6, res.v1().intValue());
+            assertEquals(4, res.v2().intValue());
+
+            res = getParentAndChildId(parentDocs, childDocs, 8);
+            assertEquals(8, res.v1().intValue());
+            assertEquals(NO_MORE_DOCS, res.v2().intValue());
+        }
+
+        {
+            // p: parent c: child1 d: child2
+            // [p0], [c1,d2,p3], [d4,c5,p6], [c7,d8,p9], [c10,p11]
+            BitSet parentDocs = new FixedBitSet(20);
+            parentDocs.set(0);
+            parentDocs.set(3);
+            parentDocs.set(6);
+            parentDocs.set(9);
+            parentDocs.set(11);
+            {
+                DocIdSetIterator childDocs = getDocIdSetIterator(new int[] { 1, 5, 7, 10 });
+                Tuple<Integer, Integer> res = getParentAndChildId(parentDocs, childDocs, 2);
+                assertEquals(3, res.v1().intValue());
+                assertEquals(1, res.v2().intValue());
+
+                res = getParentAndChildId(parentDocs, childDocs, 4);
+                assertEquals(6, res.v1().intValue());
+                assertEquals(5, res.v2().intValue());
+
+                res = getParentAndChildId(parentDocs, childDocs, 8);
+                assertEquals(9, res.v1().intValue());
+                assertEquals(7, res.v2().intValue());
+            }
+
+            {
+                DocIdSetIterator childDocs = getDocIdSetIterator(new int[] { 2, 4, 8 });
+                Tuple<Integer, Integer> res = getParentAndChildId(parentDocs, childDocs, 1);
+                assertEquals(3, res.v1().intValue());
+                assertEquals(2, res.v2().intValue());
+
+                res = getParentAndChildId(parentDocs, childDocs, 5);
+                assertEquals(6, res.v1().intValue());
+                assertEquals(4, res.v2().intValue());
+
+                res = getParentAndChildId(parentDocs, childDocs, 7);
+                assertEquals(9, res.v1().intValue());
+                assertEquals(8, res.v2().intValue());
+
+                res = getParentAndChildId(parentDocs, childDocs, 10);
+                assertEquals(11, res.v1().intValue());
+                assertEquals(NO_MORE_DOCS, res.v2().intValue());
+            }
+        }
+    }
+
+    protected QueryShardContext createQueryShardContext(String fieldName, IndexSettings indexSettings, MappedFieldType fieldType) {
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.nestedScope()).thenReturn(new NestedScope(indexSettings));
+
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, Mockito.mock(BitsetFilterCache.Listener.class));
+        when(queryShardContext.bitsetFilter(any())).thenReturn(bitsetFilterCache.getBitSetProducer(Queries.newNonNestedFilter()));
+        when(queryShardContext.fieldMapper(anyString())).thenReturn(fieldType);
+        when(queryShardContext.getSearchQuoteAnalyzer(any())).thenCallRealMethod();
+        when(queryShardContext.getSearchAnalyzer(any())).thenCallRealMethod();
+        when(queryShardContext.getIndexSettings()).thenReturn(indexSettings);
+        when(queryShardContext.getObjectMapper(anyString())).thenAnswer(invocation -> {
+            Mapper.BuilderContext context = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath());
+            return new ObjectMapper.Builder<>(fieldName).nested(ObjectMapper.Nested.newNested()).build(context);
+        });
+        when(queryShardContext.allowExpensiveQueries()).thenReturn(true);
+        return queryShardContext;
+    }
+
     public static CheckedConsumer<RandomIndexWriter, IOException> buildResellerData(int numProducts, int numResellers) {
         return iw -> {
             for (int p = 0; p < numProducts; p++) {
@@ -893,13 +1186,22 @@ private static double[] generateDocuments(List<Document> documents, int numNeste
             document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(id)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
             document.add(new Field(NestedPathFieldMapper.NAME, path, NestedPathFieldMapper.Defaults.FIELD_TYPE));
             long value = randomNonNegativeLong() % 10000;
-            document.add(new SortedNumericDocValuesField(fieldName, value));
+            document.add(new SortedNumericDocValuesField(path + "." + fieldName, value));
             documents.add(document);
             values[nested] = value;
         }
         return values;
     }
 
+    private static void generateDocument(List<Document> documents, int id, String path, String fieldName, long vales) {
+        Document document = new Document();
+        document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(id)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+        document.add(new Field(NestedPathFieldMapper.NAME, path, NestedPathFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new SortedNumericDocValuesField(path + "." + fieldName, vales));
+        document.add(new LongPoint(path + "." + fieldName, vales));
+        documents.add(document);
+    }
+
     private List<Document> generateBook(String id, String[] authors, int[] numPages) {
         List<Document> documents = new ArrayList<>();
 
diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
index 50b27ec000615..28323a94af721 100644
--- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
@@ -69,6 +69,7 @@
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
+import org.opensearch.common.lucene.search.Queries;
 import org.opensearch.common.network.NetworkAddress;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.BigArrays;
@@ -533,6 +534,17 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
         return searchAndReduce(createIndexSettings(), searcher, query, builder, maxBucket, fieldTypes);
     }
 
+    protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(
+        IndexSettings indexSettings,
+        IndexSearcher searcher,
+        Query query,
+        AggregationBuilder builder,
+        int maxBucket,
+        MappedFieldType... fieldTypes
+    ) throws IOException {
+        return searchAndReduce(indexSettings, searcher, query, builder, maxBucket, false, fieldTypes);
+    }
+
     /**
      * Collects all documents that match the provided query {@link Query} and
      * returns the reduced {@link InternalAggregation}.
@@ -547,11 +559,15 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
         Query query,
         AggregationBuilder builder,
         int maxBucket,
+        boolean hasNested,
         MappedFieldType... fieldTypes
     ) throws IOException {
         final IndexReaderContext ctx = searcher.getTopReaderContext();
         final PipelineTree pipelines = builder.buildPipelineTree();
         List<InternalAggregation> aggs = new ArrayList<>();
+        if (hasNested) {
+            query = Queries.filtered(query, Queries.newNonNestedFilter());
+        }
         Query rewritten = searcher.rewrite(query);
         MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(
             maxBucket,

From f8e886507950d13e7e7d3f7d9bd44241e5ff83b7 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Fri, 21 Jun 2024 16:32:13 -0400
Subject: [PATCH 038/113] Add ListPitInfo::getKeepAlive() getter (#14495)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                     |  1 +
 .../replication/SegmentReplicationIT.java        |  2 +-
 .../opensearch/search/pit/PitMultiNodeIT.java    | 16 +++++++++++++---
 .../opensearch/action/search/ListPitInfo.java    |  4 ++++
 .../opensearch/action/search/PitTestsUtil.java   |  5 ++++-
 .../search/CreatePitSingleNodeTests.java         | 12 ++++++------
 6 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c0266d2a6dd4e..8589298997f7f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
 - Write shard level metadata blob when snapshotting searchable snapshot indexes ([#13190](https://github.com/opensearch-project/OpenSearch/pull/13190))
 - Fix aggs result of NestedAggregator with sub NestedAggregator ([#13324](https://github.com/opensearch-project/OpenSearch/pull/13324))
+- Add ListPitInfo::getKeepAlive() getter ([#14495](https://github.com/opensearch-project/OpenSearch/pull/14495))
 
 ### Security
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
index 70da3b0e38472..2421a1a507372 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
@@ -1405,7 +1405,7 @@ public void testPitCreatedOnReplica() throws Exception {
             .setPointInTime(new PointInTimeBuilder(pitResponse.getId()).setKeepAlive(TimeValue.timeValueDays(1)))
             .setRequestCache(false)
             .get();
-        PitTestsUtil.assertUsingGetAllPits(client(replica), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(replica), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, INDEX_NAME, 1, client(replica), pitResponse.getId());
 
         List<String> currentFiles = List.of(replicaShard.store().directory().listAll());
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java
index 8bea5ef97fbba..faec3977f94ef 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java
@@ -104,7 +104,7 @@ public void testPit() throws Exception {
         assertEquals(2, searchResponse.getSuccessfulShards());
         assertEquals(2, searchResponse.getTotalShards());
         validatePitStats("index", 2, 2);
-        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, client(), pitResponse.getId());
     }
 
@@ -131,7 +131,12 @@ public void testCreatePitWhileNodeDropWithAllowPartialCreationTrue() throws Exce
             public Settings onNodeStopped(String nodeName) throws Exception {
                 ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
                 CreatePitResponse pitResponse = execute.get();
-                PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+                PitTestsUtil.assertUsingGetAllPits(
+                    client(),
+                    pitResponse.getId(),
+                    pitResponse.getCreationTime(),
+                    TimeValue.timeValueDays(1)
+                );
                 assertSegments(false, "index", 1, client(), pitResponse.getId());
                 assertEquals(1, pitResponse.getSuccessfulShards());
                 assertEquals(2, pitResponse.getTotalShards());
@@ -164,7 +169,12 @@ public Settings onNodeStopped(String nodeName) throws Exception {
                 assertEquals(0, searchResponse.getSkippedShards());
                 assertEquals(2, searchResponse.getTotalShards());
                 validatePitStats("index", 1, 1);
-                PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+                PitTestsUtil.assertUsingGetAllPits(
+                    client(),
+                    pitResponse.getId(),
+                    pitResponse.getCreationTime(),
+                    TimeValue.timeValueDays(1)
+                );
                 return super.onNodeStopped(nodeName);
             }
         });
diff --git a/server/src/main/java/org/opensearch/action/search/ListPitInfo.java b/server/src/main/java/org/opensearch/action/search/ListPitInfo.java
index 7e4ed186dd665..ac321d961679a 100644
--- a/server/src/main/java/org/opensearch/action/search/ListPitInfo.java
+++ b/server/src/main/java/org/opensearch/action/search/ListPitInfo.java
@@ -53,6 +53,10 @@ public long getCreationTime() {
         return creationTime;
     }
 
+    public long getKeepAlive() {
+        return keepAlive;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(pitId);
diff --git a/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java b/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java
index d80e011c04332..6571bdd8a3431 100644
--- a/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java
+++ b/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java
@@ -17,6 +17,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.action.ActionFuture;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.AtomicArray;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.query.IdsQueryBuilder;
@@ -97,7 +98,8 @@ public static String getPitId() {
         return SearchContextId.encode(array.asList(), aliasFilters, version);
     }
 
-    public static void assertUsingGetAllPits(Client client, String id, long creationTime) throws ExecutionException, InterruptedException {
+    public static void assertUsingGetAllPits(Client client, String id, long creationTime, TimeValue keepAlive) throws ExecutionException,
+        InterruptedException {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.local(false);
         clusterStateRequest.clear().nodes(true).routingTable(true).indices("*");
@@ -113,6 +115,7 @@ public static void assertUsingGetAllPits(Client client, String id, long creation
         GetAllPitNodesResponse getPitResponse = execute1.get();
         assertTrue(getPitResponse.getPitInfos().get(0).getPitId().contains(id));
         Assert.assertEquals(getPitResponse.getPitInfos().get(0).getCreationTime(), creationTime);
+        Assert.assertEquals(getPitResponse.getPitInfos().get(0).getKeepAlive(), keepAlive.getMillis());
     }
 
     public static void assertGetAllPitsEmpty(Client client) throws ExecutionException, InterruptedException {
diff --git a/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java b/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java
index 2661873d9498f..f54c5cfbacd46 100644
--- a/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java
+++ b/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java
@@ -76,7 +76,7 @@ public void testCreatePITSuccess() throws ExecutionException, InterruptedExcepti
         request.setIndices(new String[] { "index" });
         ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse pitResponse = execute.get();
-        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, client(), pitResponse.getId());
         client().prepareIndex("index").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse searchResponse = client().prepareSearch("index")
@@ -106,7 +106,7 @@ public void testCreatePITWithMultipleIndicesSuccess() throws ExecutionException,
 
         ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse response = execute.get();
-        PitTestsUtil.assertUsingGetAllPits(client(), response.getId(), response.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), response.getId(), response.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, client(), response.getId());
         assertEquals(4, response.getSuccessfulShards());
         assertEquals(4, service.getActiveContexts());
@@ -127,7 +127,7 @@ public void testCreatePITWithShardReplicasSuccess() throws ExecutionException, I
         request.setIndices(new String[] { "index" });
         ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse pitResponse = execute.get();
-        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, client(), pitResponse.getId());
         client().prepareIndex("index").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse searchResponse = client().prepareSearch("index")
@@ -229,7 +229,7 @@ public void testPitSearchOnCloseIndex() throws ExecutionException, InterruptedEx
         request.setIndices(new String[] { "index" });
         ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse pitResponse = execute.get();
-        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, client(), pitResponse.getId());
         SearchService service = getInstanceFromNode(SearchService.class);
         assertEquals(2, service.getActiveContexts());
@@ -412,7 +412,7 @@ public void testPitAfterUpdateIndex() throws Exception {
         request.setIndices(new String[] { "test" });
         ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse pitResponse = execute.get();
-        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueMinutes(2));
         SearchService service = getInstanceFromNode(SearchService.class);
 
         assertThat(
@@ -570,7 +570,7 @@ public void testConcurrentSearches() throws Exception {
         request.setIndices(new String[] { "index" });
         ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse pitResponse = execute.get();
-        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, client(), pitResponse.getId());
         Thread[] threads = new Thread[5];
         CountDownLatch latch = new CountDownLatch(threads.length);

From 47feca7b9182574659b548269d64d07d284557e9 Mon Sep 17 00:00:00 2001
From: Craig Perkins <cwperx@amazon.com>
Date: Sat, 22 Jun 2024 11:15:45 -0400
Subject: [PATCH 039/113] Update dependency com.azure:azure-identity to 1.13.0,
 com.microsoft.azure:msal4j to 1.15.1,
 com.microsoft.azure:msal4j-persistence-extension to 1.3.0 (#14506)

* Update dependency com.azure:azure-identity to v1.12.2

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add to CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Bump msal4j from 1.14.3 to 1.15.1

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add to CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Address review feedback

Signed-off-by: Craig Perkins <cwperx@amazon.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
---
 CHANGELOG.md                                                | 1 +
 plugins/repository-azure/build.gradle                       | 6 +++---
 .../licenses/azure-identity-1.11.4.jar.sha1                 | 1 -
 .../licenses/azure-identity-1.13.0.jar.sha1                 | 1 +
 plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1    | 1 -
 plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1    | 1 +
 .../licenses/msal4j-persistence-extension-1.2.0.jar.sha1    | 1 -
 .../licenses/msal4j-persistence-extension-1.3.0.jar.sha1    | 1 +
 8 files changed, 7 insertions(+), 6 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-1.3.0.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8589298997f7f..ba1279b4cf458 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
 - Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
 - Bump `opentelemetry` from 1.36.0 to 1.39.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457))
+- Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506))
 
 ### Changed
 - unsignedLongRangeQuery now returns MatchNoDocsQuery if the lower bounds are greater than the upper bounds ([#14416](https://github.com/opensearch-project/OpenSearch/pull/14416))
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 0fd30af71dd0a..f3aa64316b667 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -57,11 +57,11 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
-  api 'com.azure:azure-identity:1.11.4'
+  api 'com.azure:azure-identity:1.13.0'
   // Start of transitive dependencies for azure-identity
-  api 'com.microsoft.azure:msal4j-persistence-extension:1.2.0'
+  api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
-  api 'com.microsoft.azure:msal4j:1.14.3'
+  api 'com.microsoft.azure:msal4j:1.15.1'
   api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
   api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api 'com.nimbusds:content-type:2.3'
diff --git a/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
deleted file mode 100644
index c8d98ba9c8ad2..0000000000000
--- a/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-59b5ce48888f638b80d85ef5aa0e22a265d3dc89
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1
new file mode 100644
index 0000000000000..b59c2a3be5c92
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1
@@ -0,0 +1 @@
+54b44a74636322d06e9dc42d611a9f12a0966790
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
deleted file mode 100644
index 2a6e42e3f2b48..0000000000000
--- a/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-117b28c41bd760f979ed1b6467c5ec491f0d4d60
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1
new file mode 100644
index 0000000000000..797f21d0d4995
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1
@@ -0,0 +1 @@
+cd1daa94b81bd97153536b661c31295f99cbb8e7
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
deleted file mode 100644
index cfcf7548b7694..0000000000000
--- a/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1111a95878de8745ddc9de132df18ebd9ca7024d
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-1.3.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.3.0.jar.sha1
new file mode 100644
index 0000000000000..9c5909e7ff240
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.3.0.jar.sha1
@@ -0,0 +1 @@
+8a8ef1517d27a5b4de1512ef94679bdb59f210b6
\ No newline at end of file

From 2a6855c1bee93ff5fb6ed06f531d8c671792caac Mon Sep 17 00:00:00 2001
From: Ankit Kala <ankikala@amazon.com>
Date: Mon, 24 Jun 2024 21:27:39 +0530
Subject: [PATCH 040/113] Added details for indexing triage meeting (#14518)

Signed-off-by: Ankit Kala <ankikala@amazon.com>
---
 TRIAGING.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/TRIAGING.md b/TRIAGING.md
index 90842cd8e9393..c7c07a8ce30bd 100644
--- a/TRIAGING.md
+++ b/TRIAGING.md
@@ -14,7 +14,7 @@ Each meeting we seek to address all new issues. However, should we run out of ti
 
 ### How do I join a Triage meeting?
 
- Check the [OpenSearch Meetup Group](https://www.meetup.com/opensearch/) for the latest schedule and details for joining each meeting. Each component area has its own meetup series: [Search](https://www.meetup.com/opensearch/events/300929493/), [Storage](https://www.meetup.com/opensearch/events/299907409/), [Cluster Manager](https://www.meetup.com/opensearch/events/301082218/), and [Core](https://www.meetup.com/opensearch/events/301061009/).
+ Check the [OpenSearch Meetup Group](https://www.meetup.com/opensearch/) for the latest schedule and details for joining each meeting. Each component area has its own meetup series: [Search](https://www.meetup.com/opensearch/events/300929493/), [Storage](https://www.meetup.com/opensearch/events/299907409/), [Cluster Manager](https://www.meetup.com/opensearch/events/301082218/), [Indexing](https://www.meetup.com/opensearch/events/301734024/), and [Core](https://www.meetup.com/opensearch/events/301061009/).
 
 After joining the virtual meeting, you can enable your video / voice to join the discussion.  If you do not have a webcam or microphone available, you can still join in via the text chat.
 
@@ -29,9 +29,10 @@ Meeting structure may vary slightly, but the general structure is as follows:
 3. **Announcements:** Any announcements will be made at the beginning of the meeting.
 4. **Review of New Issues:** We start by reviewing all untriaged issues. Each meeting has a label-based search to find relevant issues:
    - [Search](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+label%3A%22Search%22%2C%22Search%3ARemote+Search%22%2C%22Search%3AResiliency%22%2C%22Search%3APerformance%22%2C%22Search%3ARelevance%22%2C%22Search%3AAggregations%22%2C%22Search%3AQuery+Capabilities%22%2C%22Search%3AQuery+Insights%22%2C%22Search%3ASearchable+Snapshots%22%2C%22Search%3AUser+Behavior+Insights%22)
+   - [Indexing](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+label%3A%22Indexing%3AReplication%22%2C%22Indexing%22%2C%22Indexing%3APerformance%22%2C%22Indexing+%26+Search%22%2C)
    - [Storage](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+label%3AStorage%2C%22Storage%3AResiliency%22%2C%22Storage%3APerformance%22%2C%22Storage%3ASnapshots%22%2C%22Storage%3ARemote%22%2C%22Storage%3ADurability%22)
    - [Cluster Manager](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+label%3A%22Cluster+Manager%22%2C%22ClusterManager%3ARemoteState%22)
-   - [Core](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+-label%3A%22Search%22%2C%22Search%3ARemote+Search%22%2C%22Search%3AResiliency%22%2C%22Search%3APerformance%22%2C%22Search%3ARelevance%22%2C%22Search%3AAggregations%22%2C%22Search%3AQuery+Capabilities%22%2C%22Search%3AQuery+Insights%22%2C%22Search%3ASearchable+Snapshots%22%2C%22Search%3AUser+Behavior+Insights%22%2C%22Storage%22%2C%22Storage%3AResiliency%22%2C%22Storage%3APerformance%22%2C%22Storage%3ASnapshots%22%2C%22Storage%3ARemote%22%2C%22Storage%3ADurability%22%2C%22Cluster+Manager%22%2C%22ClusterManager%3ARemoteState%22)
+   - [Core](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+-label%3A%22Search%22%2C%22Search%3ARemote+Search%22%2C%22Search%3AResiliency%22%2C%22Search%3APerformance%22%2C%22Search%3ARelevance%22%2C%22Search%3AAggregations%22%2C%22Search%3AQuery+Capabilities%22%2C%22Search%3AQuery+Insights%22%2C%22Search%3ASearchable+Snapshots%22%2C%22Search%3AUser+Behavior+Insights%22%2C%22Storage%22%2C%22Storage%3AResiliency%22%2C%22Storage%3APerformance%22%2C%22Storage%3ASnapshots%22%2C%22Storage%3ARemote%22%2C%22Storage%3ADurability%22%2C%22Cluster+Manager%22%2C%22ClusterManager%3ARemoteState%22%2C%22Indexing%3AReplication%22%2C%22Indexing%22%2C%22Indexing%3APerformance%22%2C%22Indexing+%26+Search%22)
 5. **Attendee Requests:** An opportunity for any meeting member to request consideration of an issue or pull request.
 6. **Open Discussion:** Attendees can bring up any topics not already covered by filed issues or pull requests.
 7. **Review of Old Untriaged Issues:** Time permitting, each meeting will look at all [untriaged issues older than 14 days](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+created%3A%3C2024-05-20) to prevent issues from falling through the cracks (note the GitHub API does not allow for relative times, so the date in this search must be updated every meeting).

From 1da19d3b5bf4297e286e3fbaacec4704903bfc55 Mon Sep 17 00:00:00 2001
From: panguixin <panguixin@bytedance.com>
Date: Tue, 25 Jun 2024 00:45:12 +0800
Subject: [PATCH 041/113] Fix fs info reporting negative available size
 (#11573)

* fix fs info reporting negative available size

Signed-off-by: panguixin <panguixin@bytedance.com>

* change log

Signed-off-by: panguixin <panguixin@bytedance.com>

* fix test

Signed-off-by: panguixin <panguixin@bytedance.com>

* fix test

Signed-off-by: panguixin <panguixin@bytedance.com>

* spotless

Signed-off-by: panguixin <panguixin@bytedance.com>

---------

Signed-off-by: panguixin <panguixin@bytedance.com>
Signed-off-by: Andrew Ross <andrross@amazon.com>
Co-authored-by: Andrew Ross <andrross@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../org/opensearch/monitor/fs/FsProbe.java    |  4 ++
 .../opensearch/monitor/fs/FsProbeTests.java   | 41 +++++++++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba1279b4cf458..3f5c1c01f8dc0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
 - Write shard level metadata blob when snapshotting searchable snapshot indexes ([#13190](https://github.com/opensearch-project/OpenSearch/pull/13190))
 - Fix aggs result of NestedAggregator with sub NestedAggregator ([#13324](https://github.com/opensearch-project/OpenSearch/pull/13324))
+- Fix fs info reporting negative available size ([#11573](https://github.com/opensearch-project/OpenSearch/pull/11573))
 - Add ListPitInfo::getKeepAlive() getter ([#14495](https://github.com/opensearch-project/OpenSearch/pull/14495))
 
 ### Security
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
index f4731a4a34373..f93cb63ff1f0a 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
@@ -82,6 +82,10 @@ public FsInfo stats(FsInfo previous) throws IOException {
                 paths[i].fileCacheReserved = adjustForHugeFilesystems(dataLocations[i].fileCacheReservedSize.getBytes());
                 paths[i].fileCacheUtilized = adjustForHugeFilesystems(fileCache.usage().usage());
                 paths[i].available -= (paths[i].fileCacheReserved - paths[i].fileCacheUtilized);
+                // occurs if reserved file cache space is occupied by other files, like local indices
+                if (paths[i].available < 0) {
+                    paths[i].available = 0;
+                }
             }
         }
         FsInfo.IoStats ioStats = null;
diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java
index 59a888c665be7..e2e09d5ce63fe 100644
--- a/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java
+++ b/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java
@@ -58,6 +58,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import static org.opensearch.monitor.fs.FsProbe.adjustForHugeFilesystems;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.emptyOrNullString;
 import static org.hamcrest.Matchers.greaterThan;
@@ -162,6 +163,46 @@ public void testFsCacheInfo() throws IOException {
         }
     }
 
+    public void testFsInfoWhenFileCacheOccupied() throws IOException {
+        Settings settings = Settings.builder().putList("node.roles", "search", "data").build();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            // Use the total space as reserved space to simulate the situation where the cache space is occupied
+            final long totalSpace = adjustForHugeFilesystems(env.fileCacheNodePath().fileStore.getTotalSpace());
+            ByteSizeValue gbByteSizeValue = new ByteSizeValue(totalSpace, ByteSizeUnit.BYTES);
+            env.fileCacheNodePath().fileCacheReservedSize = gbByteSizeValue;
+            FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(
+                gbByteSizeValue.getBytes(),
+                16,
+                new NoopCircuitBreaker(CircuitBreaker.REQUEST)
+            );
+
+            FsProbe probe = new FsProbe(env, fileCache);
+            FsInfo stats = probe.stats(null);
+            assertNotNull(stats);
+            assertTrue(stats.getTimestamp() > 0L);
+            FsInfo.Path total = stats.getTotal();
+            assertNotNull(total);
+            assertTrue(total.total > 0L);
+            assertTrue(total.free > 0L);
+            assertTrue(total.fileCacheReserved > 0L);
+
+            for (FsInfo.Path path : stats) {
+                assertNotNull(path);
+                assertFalse(path.getPath().isEmpty());
+                assertFalse(path.getMount().isEmpty());
+                assertFalse(path.getType().isEmpty());
+                assertTrue(path.total > 0L);
+                assertTrue(path.free > 0L);
+
+                if (path.fileCacheReserved > 0L) {
+                    assertEquals(0L, path.available);
+                } else {
+                    assertTrue(path.available > 0L);
+                }
+            }
+        }
+    }
+
     public void testFsInfoOverflow() throws Exception {
         final FsInfo.Path pathStats = new FsInfo.Path(
             "/foo/bar",

From 212efd76637bddebf9dac85a0aa5eadebd9456cb Mon Sep 17 00:00:00 2001
From: Rishabh Maurya <rishabhmaurya05@gmail.com>
Date: Mon, 24 Jun 2024 10:35:04 -0700
Subject: [PATCH 042/113] Fix a race condition in Derived Field parsing from
 search request (#14445)

Signed-off-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
---
 CHANGELOG.md                                  |  1 +
 .../mapper/DefaultDerivedFieldResolver.java   | 28 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f5c1c01f8dc0..55728a58eca03 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
+- Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java b/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java
index c577a4117247b..4dd17703b6f55 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java
@@ -15,6 +15,8 @@
 import org.opensearch.script.Script;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -189,9 +191,10 @@ private void initDerivedFieldTypes(Map<String, Object> derivedFieldsObject, List
 
     private Map<String, DerivedFieldType> getAllDerivedFieldTypeFromObject(Map<String, Object> derivedFieldObject) {
         Map<String, DerivedFieldType> derivedFieldTypes = new HashMap<>();
+        // deep copy of derivedFieldObject is required as DocumentMapperParser modifies the map
         DocumentMapper documentMapper = queryShardContext.getMapperService()
             .documentMapperParser()
-            .parse(DerivedFieldMapper.CONTENT_TYPE, derivedFieldObject);
+            .parse(DerivedFieldMapper.CONTENT_TYPE, (Map) deepCopy(derivedFieldObject));
         if (documentMapper != null && documentMapper.mappers() != null) {
             for (Mapper mapper : documentMapper.mappers()) {
                 if (mapper instanceof DerivedFieldMapper) {
@@ -226,4 +229,27 @@ private DerivedFieldType resolveUsingMappings(String name) {
         }
         return null;
     }
+
+    private static Object deepCopy(Object value) {
+        if (value instanceof Map) {
+            Map<?, ?> mapValue = (Map<?, ?>) value;
+            Map<Object, Object> copy = new HashMap<>(mapValue.size());
+            for (Map.Entry<?, ?> entry : mapValue.entrySet()) {
+                copy.put(entry.getKey(), deepCopy(entry.getValue()));
+            }
+            return copy;
+        } else if (value instanceof List) {
+            List<?> listValue = (List<?>) value;
+            List<Object> copy = new ArrayList<>(listValue.size());
+            for (Object itemValue : listValue) {
+                copy.add(deepCopy(itemValue));
+            }
+            return copy;
+        } else if (value instanceof byte[]) {
+            byte[] bytes = (byte[]) value;
+            return Arrays.copyOf(bytes, bytes.length);
+        } else {
+            return value;
+        }
+    }
 }

From afad5ebd4f1979fc77911bf3c369c74d3b605e3f Mon Sep 17 00:00:00 2001
From: kkewwei <kkewwei@163.com>
Date: Tue, 25 Jun 2024 05:43:34 +0800
Subject: [PATCH 043/113] Fix FuzzyQuery in keyword field when both of index
 and doc_value are true (#14378)

Signed-off-by: kkewwei <kkewwei@163.com>
---
 CHANGELOG.md                                  |  1 +
 .../index/mapper/KeywordFieldMapper.java      |  2 +-
 .../index/mapper/StringFieldType.java         | 30 +++++++++++++++++++
 .../index/query/FuzzyQueryBuilder.java        |  2 +-
 .../index/mapper/KeywordFieldTypeTests.java   |  7 +++--
 5 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 55728a58eca03..cafe9c20e7ff4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix aggs result of NestedAggregator with sub NestedAggregator ([#13324](https://github.com/opensearch-project/OpenSearch/pull/13324))
 - Fix fs info reporting negative available size ([#11573](https://github.com/opensearch-project/OpenSearch/pull/11573))
 - Add ListPitInfo::getKeepAlive() getter ([#14495](https://github.com/opensearch-project/OpenSearch/pull/14495))
+- Fix FuzzyQuery in keyword field will use IndexOrDocValuesQuery when both of index and doc_value are true ([#14378](https://github.com/opensearch-project/OpenSearch/pull/14378))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
index 7f6d9231a37fc..2116ac522b705 100644
--- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
@@ -549,7 +549,7 @@ public Query fuzzyQuery(
                 );
             }
             if (isSearchable() && hasDocValues()) {
-                Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context);
+                Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context);
                 Query dvQuery = super.fuzzyQuery(
                     value,
                     fuzziness,
diff --git a/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java b/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java
index fbfca44c3062a..682ccc13f769d 100644
--- a/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java
@@ -55,6 +55,7 @@
 import java.util.regex.Pattern;
 
 import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
+import static org.apache.lucene.search.FuzzyQuery.defaultRewriteMethod;
 
 /** Base class for {@link MappedFieldType} implementations that use the same
  * representation for internal index terms as the external representation so
@@ -102,6 +103,35 @@ public Query fuzzyQuery(
         );
     }
 
+    @Override
+    public Query fuzzyQuery(
+        Object value,
+        Fuzziness fuzziness,
+        int prefixLength,
+        int maxExpansions,
+        boolean transpositions,
+        MultiTermQuery.RewriteMethod method,
+        QueryShardContext context
+    ) {
+        if (!context.allowExpensiveQueries()) {
+            throw new OpenSearchException(
+                "[fuzzy] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false."
+            );
+        }
+        failIfNotIndexed();
+        if (method == null) {
+            method = defaultRewriteMethod(maxExpansions);
+        }
+        return new FuzzyQuery(
+            new Term(name(), indexedValueForSearch(value)),
+            fuzziness.asDistance(BytesRefs.toString(value)),
+            prefixLength,
+            maxExpansions,
+            transpositions,
+            method
+        );
+    }
+
     @Override
     public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) {
         if (context.allowExpensiveQueries() == false) {
diff --git a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java
index a25a426792e31..93c32bbedcef4 100644
--- a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java
@@ -357,7 +357,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             throw new IllegalStateException("Rewrite first");
         }
         String rewrite = this.rewrite;
-        Query query = fieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context);
+        Query query = fieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, null, context);
         if (query instanceof MultiTermQuery) {
             MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE);
             QueryParsers.setRewriteMethod((MultiTermQuery) query, rewriteMethod);
diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java
index 393c448330142..b10035f54a0c0 100644
--- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java
@@ -263,8 +263,11 @@ public void testRegexpQuery() {
     public void testFuzzyQuery() {
         MappedFieldType ft = new KeywordFieldType("field");
         assertEquals(
-            new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true),
-            ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MOCK_QSC)
+            new IndexOrDocValuesQuery(
+                new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true),
+                new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE)
+            ),
+            ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC)
         );
 
         Query indexExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true);

From 0d01d1755e282763e5ea020112617ba25459863c Mon Sep 17 00:00:00 2001
From: bowenlan-amzn <bowenlan23@gmail.com>
Date: Tue, 25 Jun 2024 08:21:33 -0700
Subject: [PATCH 044/113] Fix flaky test in range aggregation yaml test
 (#14486)

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>
---
 .../rest-api-spec/test/search.aggregation/10_histogram.yml  | 5 +++++
 .../rest-api-spec/test/search.aggregation/230_composite.yml | 6 ++++++
 .../test/search.aggregation/330_auto_date_histogram.yml     | 5 +++++
 .../rest-api-spec/test/search.aggregation/40_range.yml      | 6 ++++++
 4 files changed, 22 insertions(+)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
index 996c2aae8cfe4..a75b1d0eac793 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
@@ -678,6 +678,11 @@ setup:
           - '{"index": {}}'
           - '{"date": "2016-03-01"}'
 
+  - do:
+      indices.forcemerge:
+        index: test_2
+        max_num_segments: 1
+
   - do:
       search:
         index: test_2
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
index 78e2e6858c6ff..ade9eb3eee0dc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
@@ -1101,6 +1101,12 @@ setup:
           - '{"date": "2016-02-01"}'
           - '{"index": {}}'
           - '{"date": "2016-03-01"}'
+
+  - do:
+      indices.forcemerge:
+        index: test_2
+        max_num_segments: 1
+
   - do:
       search:
         index: test_2
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
index fc82517788c91..0897e0bdd894b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
@@ -133,6 +133,11 @@ setup:
           - '{"index": {}}'
           - '{"date": "2020-03-09", "v": 4}'
 
+  - do:
+      indices.forcemerge:
+        index: test_profile
+        max_num_segments: 1
+
   - do:
       search:
         index: test_profile
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
index 2fd926276d0b4..80aad96ce1f6b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
@@ -544,6 +544,7 @@ setup:
         body:
           settings:
             number_of_replicas: 0
+            number_of_shards: 1
             refresh_interval: -1
           mappings:
             properties:
@@ -567,6 +568,11 @@ setup:
           - '{"index": {}}'
           - '{"double" : 50}'
 
+  - do:
+      indices.forcemerge:
+        index: test_profile
+        max_num_segments: 1
+
   - do:
       search:
         index: test_profile

From aa83733c3ffdf6ef1ff98abd9519001e5aa71509 Mon Sep 17 00:00:00 2001
From: Prudhvi Godithi <pgodithi@amazon.com>
Date: Tue, 25 Jun 2024 11:29:41 -0700
Subject: [PATCH 045/113] Use CODECOV_TOKEN (#14536)

Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>
---
 .github/workflows/gradle-check.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
index 2909ee95349ce..89d894403ff1a 100644
--- a/.github/workflows/gradle-check.yml
+++ b/.github/workflows/gradle-check.yml
@@ -113,6 +113,7 @@ jobs:
         if: success()
         uses: codecov/codecov-action@v4
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: ./codeCoverage.xml
 
       - name: Create Comment Success

From 563375de28b16870ab42b9fb4260127598d47d91 Mon Sep 17 00:00:00 2001
From: Sagar <99425694+sgup432@users.noreply.github.com>
Date: Tue, 25 Jun 2024 12:04:17 -0700
Subject: [PATCH 046/113] [Tiered Caching]  Moving query recomputation logic
 outside of write lock (#14187)

* Moving query recompute out of write lock

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* [Tiered Caching] Moving query recomputation logic outside of write lock

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Adding java doc for the completable map

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Changes to call future handler only once per key

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Fixing spotless check

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Added changelog

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Addressing comments

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Fixing gradle fail

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Addressing comments to refactor unit test

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* minor UT refactor

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

---------

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>
Signed-off-by: Sagar <99425694+sgup432@users.noreply.github.com>
Co-authored-by: Sagar Upadhyaya <upasagar@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../common/tier/TieredSpilloverCache.java     |  85 ++++-
 .../tier/TieredSpilloverCacheTests.java       | 339 +++++++++++++++++-
 3 files changed, 406 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cafe9c20e7ff4..f71ba46745ef1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506))
 
 ### Changed
+- [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187))
 - unsignedLongRangeQuery now returns MatchNoDocsQuery if the lower bounds are greater than the upper bounds ([#14416](https://github.com/opensearch-project/OpenSearch/pull/14416))
 - Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568))
 - Make the class CommunityIdProcessor final ([#14448](https://github.com/opensearch-project/OpenSearch/pull/14448))
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
index 63cdbca101f2a..b6d6913a9f8d4 100644
--- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.cache.common.tier;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.cache.common.policy.TookTimePolicy;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.cache.CacheType;
@@ -35,9 +37,13 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.ToLongBiFunction;
@@ -61,6 +67,7 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
 
     // Used to avoid caching stale entries in lower tiers.
     private static final List<RemovalReason> SPILLOVER_REMOVAL_REASONS = List.of(RemovalReason.EVICTED, RemovalReason.CAPACITY);
+    private static final Logger logger = LogManager.getLogger(TieredSpilloverCache.class);
 
     private final ICache<K, V> diskCache;
     private final ICache<K, V> onHeapCache;
@@ -86,6 +93,12 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
     private final Map<ICache<K, V>, TierInfo> caches;
     private final List<Predicate<V>> policies;
 
+    /**
+     * This map is used to handle concurrent requests for same key in computeIfAbsent() to ensure we load the value
+     * only once.
+     */
+    Map<ICacheKey<K>, CompletableFuture<Tuple<ICacheKey<K>, V>>> completableFutureMap = new ConcurrentHashMap<>();
+
     TieredSpilloverCache(Builder<K, V> builder) {
         Objects.requireNonNull(builder.onHeapCacheFactory, "onHeap cache builder can't be null");
         Objects.requireNonNull(builder.diskCacheFactory, "disk cache builder can't be null");
@@ -190,10 +203,7 @@ public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V>
             // Add the value to the onHeap cache. We are calling computeIfAbsent which does another get inside.
             // This is needed as there can be many requests for the same key at the same time and we only want to load
             // the value once.
-            V value = null;
-            try (ReleasableLock ignore = writeLock.acquire()) {
-                value = onHeapCache.computeIfAbsent(key, loader);
-            }
+            V value = compute(key, loader);
             // Handle stats
             if (loader.isLoaded()) {
                 // The value was just computed and added to the cache by this thread. Register a miss for the heap cache, and the disk cache
@@ -222,6 +232,57 @@ public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V>
         return cacheValueTuple.v1();
     }
 
+    private V compute(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
+        // Only one of the threads will succeed putting a future into map for the same key.
+        // Rest will fetch existing future and wait on that to complete.
+        CompletableFuture<Tuple<ICacheKey<K>, V>> future = completableFutureMap.putIfAbsent(key, new CompletableFuture<>());
+        // Handler to handle results post processing. Takes a tuple<key, value> or exception as an input and returns
+        // the value. Also before returning value, puts the value in cache.
+        BiFunction<Tuple<ICacheKey<K>, V>, Throwable, Void> handler = (pair, ex) -> {
+            if (pair != null) {
+                try (ReleasableLock ignore = writeLock.acquire()) {
+                    onHeapCache.put(pair.v1(), pair.v2());
+                } catch (Exception e) {
+                    // TODO: Catch specific exceptions to know whether this resulted from cache or underlying removal
+                    // listeners/stats. Needs better exception handling at underlying layers.For now swallowing
+                    // exception.
+                    logger.warn("Exception occurred while putting item onto heap cache", e);
+                }
+            } else {
+                if (ex != null) {
+                    logger.warn("Exception occurred while trying to compute the value", ex);
+                }
+            }
+            completableFutureMap.remove(key); // Remove key from map as not needed anymore.
+            return null;
+        };
+        V value = null;
+        if (future == null) {
+            future = completableFutureMap.get(key);
+            future.handle(handler);
+            try {
+                value = loader.load(key);
+            } catch (Exception ex) {
+                future.completeExceptionally(ex);
+                throw new ExecutionException(ex);
+            }
+            if (value == null) {
+                NullPointerException npe = new NullPointerException("Loader returned a null value");
+                future.completeExceptionally(npe);
+                throw new ExecutionException(npe);
+            } else {
+                future.complete(new Tuple<>(key, value));
+            }
+        } else {
+            try {
+                value = future.get().v2();
+            } catch (InterruptedException ex) {
+                throw new IllegalStateException(ex);
+            }
+        }
+        return value;
+    }
+
     @Override
     public void invalidate(ICacheKey<K> key) {
         // We are trying to invalidate the key from all caches though it would be present in only of them.
@@ -328,12 +389,22 @@ void handleRemovalFromHeapTier(RemovalNotification<ICacheKey<K>, V> notification
         ICacheKey<K> key = notification.getKey();
         boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason());
         boolean countEvictionTowardsTotal = false; // Don't count this eviction towards the cache's total if it ends up in the disk tier
-        if (caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue())) {
+        boolean exceptionOccurredOnDiskCachePut = false;
+        boolean canCacheOnDisk = caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue());
+        if (canCacheOnDisk) {
             try (ReleasableLock ignore = writeLock.acquire()) {
                 diskCache.put(key, notification.getValue()); // spill over to the disk tier and increment its stats
+            } catch (Exception ex) {
+                // TODO: Catch specific exceptions. Needs better exception handling. We are just swallowing exception
+                // in this case as it shouldn't cause upstream request to fail.
+                logger.warn("Exception occurred while putting item to disk cache", ex);
+                exceptionOccurredOnDiskCachePut = true;
             }
-            updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue());
-        } else {
+            if (!exceptionOccurredOnDiskCachePut) {
+                updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue());
+            }
+        }
+        if (!canCacheOnDisk || exceptionOccurredOnDiskCachePut) {
             // If the value is not going to the disk cache, send this notification to the TSC's removal listener
             // as the value is leaving the TSC entirely
             removalListener.onRemoval(notification);
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
index 54b15f236a418..b9c7bbdb77d3d 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
@@ -44,8 +44,12 @@
 import java.util.UUID;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.Phaser;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.Predicate;
@@ -56,6 +60,10 @@
 import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK;
 import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP;
 import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TieredSpilloverCacheTests extends OpenSearchTestCase {
     static final List<String> dimensionNames = List.of("dim1", "dim2", "dim3");
@@ -408,6 +416,7 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
         assertEquals(onHeapCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         assertEquals(cacheMiss + numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
         assertEquals(diskCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, tieredSpilloverCache.completableFutureMap.size());
     }
 
     public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception {
@@ -802,7 +811,7 @@ public String load(ICacheKey<String> key) {
                     };
                     loadAwareCacheLoaderList.add(loadAwareCacheLoader);
                     phaser.arriveAndAwaitAdvance();
-                    tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader);
+                    assertEquals(value, tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader));
                 } catch (Exception e) {
                     throw new RuntimeException(e);
                 }
@@ -811,7 +820,7 @@ public String load(ICacheKey<String> key) {
             threads[i].start();
         }
         phaser.arriveAndAwaitAdvance();
-        countDownLatch.await(); // Wait for rest of tasks to be cancelled.
+        countDownLatch.await();
         int numberOfTimesKeyLoaded = 0;
         assertEquals(numberOfSameKeys, loadAwareCacheLoaderList.size());
         for (int i = 0; i < loadAwareCacheLoaderList.size(); i++) {
@@ -824,6 +833,215 @@ public String load(ICacheKey<String> key) {
         // We should see only one heap miss, and the rest hits
         assertEquals(1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         assertEquals(numberOfSameKeys - 1, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, tieredSpilloverCache.completableFutureMap.size());
+    }
+
+    public void testComputIfAbsentConcurrentlyWithMultipleKeys() throws Exception {
+        int onHeapCacheSize = randomIntBetween(300, 500);
+        int diskCacheSize = randomIntBetween(600, 700);
+        int keyValueSize = 50;
+
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        Settings settings = Settings.builder()
+            .put(
+                OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                    .getKey(),
+                onHeapCacheSize * keyValueSize + "b"
+            )
+            .build();
+
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            settings,
+            0
+        );
+
+        int iterations = 10;
+        int numberOfKeys = 20;
+        List<ICacheKey<String>> iCacheKeyList = new ArrayList<>();
+        for (int i = 0; i < numberOfKeys; i++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            iCacheKeyList.add(key);
+        }
+        ExecutorService executorService = Executors.newFixedThreadPool(8);
+        CountDownLatch countDownLatch = new CountDownLatch(iterations * numberOfKeys); // To wait for all threads to finish.
+
+        List<LoadAwareCacheLoader<ICacheKey<String>, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
+        for (int j = 0; j < numberOfKeys; j++) {
+            int finalJ = j;
+            for (int i = 0; i < iterations; i++) {
+                executorService.submit(() -> {
+                    try {
+                        LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
+                            boolean isLoaded = false;
+
+                            @Override
+                            public boolean isLoaded() {
+                                return isLoaded;
+                            }
+
+                            @Override
+                            public String load(ICacheKey<String> key) {
+                                isLoaded = true;
+                                return iCacheKeyList.get(finalJ).key;
+                            }
+                        };
+                        loadAwareCacheLoaderList.add(loadAwareCacheLoader);
+                        tieredSpilloverCache.computeIfAbsent(iCacheKeyList.get(finalJ), loadAwareCacheLoader);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    } finally {
+                        countDownLatch.countDown();
+                    }
+                });
+            }
+        }
+        countDownLatch.await();
+        int numberOfTimesKeyLoaded = 0;
+        assertEquals(iterations * numberOfKeys, loadAwareCacheLoaderList.size());
+        for (int i = 0; i < loadAwareCacheLoaderList.size(); i++) {
+            LoadAwareCacheLoader<ICacheKey<String>, String> loader = loadAwareCacheLoaderList.get(i);
+            if (loader.isLoaded()) {
+                numberOfTimesKeyLoaded++;
+            }
+        }
+        assertEquals(numberOfKeys, numberOfTimesKeyLoaded); // It should be loaded only once.
+        // We should see only one heap miss, and the rest hits
+        assertEquals(numberOfKeys, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals((iterations * numberOfKeys) - numberOfKeys, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, tieredSpilloverCache.completableFutureMap.size());
+        executorService.shutdownNow();
+    }
+
+    public void testComputeIfAbsentConcurrentlyAndThrowsException() throws Exception {
+        LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
+            boolean isLoaded = false;
+
+            @Override
+            public boolean isLoaded() {
+                return isLoaded;
+            }
+
+            @Override
+            public String load(ICacheKey<String> key) {
+                throw new RuntimeException("Testing");
+            }
+        };
+        verifyComputeIfAbsentThrowsException(RuntimeException.class, loadAwareCacheLoader, "Testing");
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public void testComputeIfAbsentWithOnHeapCacheThrowingExceptionOnPut() throws Exception {
+        int onHeapCacheSize = randomIntBetween(100, 300);
+        int diskCacheSize = randomIntBetween(200, 400);
+        int keyValueSize = 50;
+
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        Settings settings = Settings.builder()
+            .put(
+                OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                    .getKey(),
+                onHeapCacheSize * keyValueSize + "b"
+            )
+            .build();
+        ICache.Factory onHeapCacheFactory = mock(OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.class);
+        ICache mockOnHeapCache = mock(ICache.class);
+        when(onHeapCacheFactory.create(any(), any(), any())).thenReturn(mockOnHeapCache);
+        doThrow(new RuntimeException("Testing")).when(mockOnHeapCache).put(any(), any());
+        CacheConfig<String, String> cacheConfig = getCacheConfig(keyValueSize, settings, removalListener);
+        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false);
+
+        TieredSpilloverCache<String, String> tieredSpilloverCache = getTieredSpilloverCache(
+            onHeapCacheFactory,
+            mockDiskCacheFactory,
+            cacheConfig,
+            null,
+            removalListener
+        );
+        String value = "";
+        value = tieredSpilloverCache.computeIfAbsent(getICacheKey("test"), new LoadAwareCacheLoader<>() {
+            @Override
+            public boolean isLoaded() {
+                return false;
+            }
+
+            @Override
+            public String load(ICacheKey<String> key) {
+                return "test";
+            }
+        });
+        assertEquals("test", value);
+        assertEquals(0, tieredSpilloverCache.completableFutureMap.size());
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public void testComputeIfAbsentWithDiskCacheThrowingExceptionOnPut() throws Exception {
+        int onHeapCacheSize = 0;
+        int keyValueSize = 50;
+
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        Settings settings = Settings.builder()
+            .put(
+                OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                    .getKey(),
+                onHeapCacheSize * keyValueSize + "b"
+            )
+            .build();
+        ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory();
+        CacheConfig<String, String> cacheConfig = getCacheConfig(keyValueSize, settings, removalListener);
+        ICache.Factory mockDiskCacheFactory = mock(MockDiskCache.MockDiskCacheFactory.class);
+        ICache mockDiskCache = mock(ICache.class);
+        when(mockDiskCacheFactory.create(any(), any(), any())).thenReturn(mockDiskCache);
+        doThrow(new RuntimeException("Test")).when(mockDiskCache).put(any(), any());
+
+        TieredSpilloverCache<String, String> tieredSpilloverCache = getTieredSpilloverCache(
+            onHeapCacheFactory,
+            mockDiskCacheFactory,
+            cacheConfig,
+            null,
+            removalListener
+        );
+
+        String response = "";
+        response = tieredSpilloverCache.computeIfAbsent(getICacheKey("test"), new LoadAwareCacheLoader<>() {
+            @Override
+            public boolean isLoaded() {
+                return false;
+            }
+
+            @Override
+            public String load(ICacheKey<String> key) {
+                return "test";
+            }
+        });
+        ImmutableCacheStats diskStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK);
+
+        assertEquals(0, diskStats.getSizeInBytes());
+        assertEquals(1, removalListener.evictionsMetric.count());
+        assertEquals("test", response);
+        assertEquals(0, tieredSpilloverCache.completableFutureMap.size());
+    }
+
+    public void testComputeIfAbsentConcurrentlyWithLoaderReturningNull() throws Exception {
+        LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
+            boolean isLoaded = false;
+
+            @Override
+            public boolean isLoaded() {
+                return isLoaded;
+            }
+
+            @Override
+            public String load(ICacheKey<String> key) {
+                return null;
+            }
+        };
+        verifyComputeIfAbsentThrowsException(NullPointerException.class, loadAwareCacheLoader, "Loader returned a null value");
     }
 
     public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exception {
@@ -1408,6 +1626,26 @@ public boolean isLoaded() {
         };
     }
 
+    private TieredSpilloverCache<String, String> getTieredSpilloverCache(
+        ICache.Factory onHeapCacheFactory,
+        ICache.Factory mockDiskCacheFactory,
+        CacheConfig<String, String> cacheConfig,
+        List<Predicate<String>> policies,
+        RemovalListener<ICacheKey<String>, String> removalListener
+    ) {
+        TieredSpilloverCache.Builder<String, String> builder = new TieredSpilloverCache.Builder<String, String>().setCacheType(
+            CacheType.INDICES_REQUEST_CACHE
+        )
+            .setRemovalListener(removalListener)
+            .setOnHeapCacheFactory(onHeapCacheFactory)
+            .setDiskCacheFactory(mockDiskCacheFactory)
+            .setCacheConfig(cacheConfig);
+        if (policies != null) {
+            builder.addPolicies(policies);
+        }
+        return builder.build();
+    }
+
     private TieredSpilloverCache<String, String> initializeTieredSpilloverCache(
         int keyValueSize,
         int diskCacheSize,
@@ -1450,17 +1688,34 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
             .build();
         ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize, false);
 
-        TieredSpilloverCache.Builder<String, String> builder = new TieredSpilloverCache.Builder<String, String>().setCacheType(
-            CacheType.INDICES_REQUEST_CACHE
-        )
+        return getTieredSpilloverCache(onHeapCacheFactory, mockDiskCacheFactory, cacheConfig, policies, removalListener);
+    }
+
+    private CacheConfig<String, String> getCacheConfig(
+        int keyValueSize,
+        Settings settings,
+        RemovalListener<ICacheKey<String>, String> removalListener
+    ) {
+        return new CacheConfig.Builder<String, String>().setKeyType(String.class)
+            .setKeyType(String.class)
+            .setWeigher((k, v) -> keyValueSize)
+            .setSettings(settings)
+            .setDimensionNames(dimensionNames)
             .setRemovalListener(removalListener)
-            .setOnHeapCacheFactory(onHeapCacheFactory)
-            .setDiskCacheFactory(mockDiskCacheFactory)
-            .setCacheConfig(cacheConfig);
-        if (policies != null) {
-            builder.addPolicies(policies);
-        }
-        return builder.build();
+            .setKeySerializer(new StringSerializer())
+            .setValueSerializer(new StringSerializer())
+            .setSettings(
+                Settings.builder()
+                    .put(
+                        CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                        TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME
+                    )
+                    .put(FeatureFlags.PLUGGABLE_CACHE, "true")
+                    .put(settings)
+                    .build()
+            )
+            .setClusterSettings(clusterSettings)
+            .build();
     }
 
     // Helper functions for extracting tier aggregated stats.
@@ -1501,6 +1756,66 @@ private ImmutableCacheStats getStatsSnapshotForTier(TieredSpilloverCache<?, ?> t
         return snapshot;
     }
 
+    private void verifyComputeIfAbsentThrowsException(
+        Class<? extends Exception> expectedException,
+        LoadAwareCacheLoader<ICacheKey<String>, String> loader,
+        String expectedExceptionMessage
+    ) throws InterruptedException {
+        int onHeapCacheSize = randomIntBetween(100, 300);
+        int diskCacheSize = randomIntBetween(200, 400);
+        int keyValueSize = 50;
+
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        Settings settings = Settings.builder()
+            .put(
+                OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                    .getKey(),
+                onHeapCacheSize * keyValueSize + "b"
+            )
+            .build();
+
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            settings,
+            0
+        );
+
+        int numberOfSameKeys = randomIntBetween(10, onHeapCacheSize - 1);
+        ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+        String value = UUID.randomUUID().toString();
+        AtomicInteger exceptionCount = new AtomicInteger();
+
+        Thread[] threads = new Thread[numberOfSameKeys];
+        Phaser phaser = new Phaser(numberOfSameKeys + 1);
+        CountDownLatch countDownLatch = new CountDownLatch(numberOfSameKeys); // To wait for all threads to finish.
+
+        for (int i = 0; i < numberOfSameKeys; i++) {
+            threads[i] = new Thread(() -> {
+                try {
+                    phaser.arriveAndAwaitAdvance();
+                    tieredSpilloverCache.computeIfAbsent(key, loader);
+                } catch (Exception e) {
+                    exceptionCount.incrementAndGet();
+                    assertEquals(ExecutionException.class, e.getClass());
+                    assertEquals(expectedException, e.getCause().getClass());
+                    assertEquals(expectedExceptionMessage, e.getCause().getMessage());
+                } finally {
+                    countDownLatch.countDown();
+                }
+            });
+            threads[i].start();
+        }
+        phaser.arriveAndAwaitAdvance();
+        countDownLatch.await(); // Wait for rest of tasks to be cancelled.
+
+        // Verify exception count was equal to number of requests
+        assertEquals(numberOfSameKeys, exceptionCount.get());
+        assertEquals(0, tieredSpilloverCache.completableFutureMap.size());
+    }
+
     private ImmutableCacheStats getTotalStatsSnapshot(TieredSpilloverCache<?, ?> tsc) throws IOException {
         ImmutableCacheStatsHolder cacheStats = tsc.stats(new String[0]);
         return cacheStats.getStatsForDimensionValues(List.of());

From badf851c4883c8ecd4bf47fbad3ca0d3a608f613 Mon Sep 17 00:00:00 2001
From: kkewwei <kkewwei@163.com>
Date: Wed, 26 Jun 2024 03:12:43 +0800
Subject: [PATCH 047/113] Fix Flaky Test
 ClusterRerouteIT.testDelayWithALargeAmountOfShards (#14510)

Signed-off-by: kkewwei kkewwei@163.com

Signed-off-by: kkewwei kkewwei@163.com
Signed-off-by: kkewwei <kkewwei@163.com>
---
 .../cluster/allocation/ClusterRerouteIT.java     |  3 ++-
 .../opensearch/test/OpenSearchIntegTestCase.java | 16 +++++++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java
index dbcb030d8a4f7..f4b5f112f5785 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java
@@ -273,7 +273,8 @@ public void testDelayWithALargeAmountOfShards() throws Exception {
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_1));
 
         // This might run slowly on older hardware
-        ensureGreen(TimeValue.timeValueMinutes(2));
+        // In some case, the shards will be rebalanced back and forth, it seems like a very low probability bug.
+        ensureGreen(TimeValue.timeValueMinutes(2), false);
     }
 
     private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exception {
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index 71ab56c98312a..ca5ddf21710af 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -864,6 +864,10 @@ public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) {
         return ensureColor(ClusterHealthStatus.GREEN, timeout, false, indices);
     }
 
+    public ClusterHealthStatus ensureGreen(TimeValue timeout, boolean waitForNoRelocatingShards, String... indices) {
+        return ensureColor(ClusterHealthStatus.GREEN, timeout, waitForNoRelocatingShards, false, indices);
+    }
+
     /**
      * Ensures the cluster has a yellow state via the cluster health API.
      */
@@ -891,6 +895,16 @@ private ClusterHealthStatus ensureColor(
         TimeValue timeout,
         boolean waitForNoInitializingShards,
         String... indices
+    ) {
+        return ensureColor(clusterHealthStatus, timeout, true, waitForNoInitializingShards, indices);
+    }
+
+    private ClusterHealthStatus ensureColor(
+        ClusterHealthStatus clusterHealthStatus,
+        TimeValue timeout,
+        boolean waitForNoRelocatingShards,
+        boolean waitForNoInitializingShards,
+        String... indices
     ) {
         String color = clusterHealthStatus.name().toLowerCase(Locale.ROOT);
         String method = "ensure" + Strings.capitalize(color);
@@ -899,7 +913,7 @@ private ClusterHealthStatus ensureColor(
             .timeout(timeout)
             .waitForStatus(clusterHealthStatus)
             .waitForEvents(Priority.LANGUID)
-            .waitForNoRelocatingShards(true)
+            .waitForNoRelocatingShards(waitForNoRelocatingShards)
             .waitForNoInitializingShards(waitForNoInitializingShards)
             // We currently often use ensureGreen or ensureYellow to check whether the cluster is back in a good state after shutting down
             // a node. If the node that is stopped is the cluster-manager node, another node will become cluster-manager and publish a

From d320f36ca01ff68f344b54a91a75ef3390824eb7 Mon Sep 17 00:00:00 2001
From: bowenlan-amzn <bowenlan23@gmail.com>
Date: Tue, 25 Jun 2024 12:35:35 -0700
Subject: [PATCH 048/113] Add doc for debugging rest tests (#14491)

* add doc for debugging rest tests

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* Update TESTING.md

Co-authored-by: Marc Handalian <handalm@amazon.com>
Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

* Address comment

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

---------

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>
Co-authored-by: Marc Handalian <handalm@amazon.com>
---
 TESTING.md | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/TESTING.md b/TESTING.md
index e8416f61be7e1..de7ab3eefe2f8 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -17,6 +17,8 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran
   - [Miscellaneous](#miscellaneous)
 - [Running verification tasks](#running-verification-tasks)
 - [Testing the REST layer](#testing-the-rest-layer)
+  - [Running REST Tests Against An External Cluster](#running-rest-tests-against-an-external-cluster)
+  - [Debugging REST Tests](#debugging-rest-tests)
 - [Testing packaging](#testing-packaging)
   - [Testing packaging on Windows](#testing-packaging-on-windows)
   - [Testing VMs are disposable](#testing-vms-are-disposable)
@@ -272,7 +274,18 @@ yamlRestTest’s and javaRestTest’s are easy to identify, since they are found
 
 If in doubt about which command to use, simply run &lt;gradle path&gt;:check
 
-Note that the REST tests, like all the integration tests, can be run against an external cluster by specifying the `tests.cluster` property, which if present needs to contain a comma separated list of nodes to connect to (e.g. localhost:9300).
+## Running REST Tests Against An External Cluster
+
+Note that the REST tests, like all the integration tests, can be run against an external cluster by specifying the following properties `tests.cluster`, `tests.rest.cluster`, `tests.clustername`. Use a comma separated list of node properties for the multi-node cluster.
+
+For example :
+
+    ./gradlew :rest-api-spec:yamlRestTest \
+      -Dtests.cluster=localhost:9200 -Dtests.rest.cluster=localhost:9200 -Dtests.clustername=opensearch
+
+## Debugging REST Tests
+
+You can launch a local OpenSearch cluster in debug mode following [Launching and debugging from an IDE](#launching-and-debugging-from-an-ide), and run your REST tests against that following [Running REST Tests Against An External Cluster](#running-rest-tests-against-an-external-cluster).
 
 # Testing packaging
 

From 0eb39aec6796d9a576e51b186b1cb2474f16f70a Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <peter.alfonsi@gmail.com>
Date: Tue, 25 Jun 2024 13:26:54 -0700
Subject: [PATCH 049/113] Fix flaky DefaultCacheStatsHolderTests (#14462)

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
Co-authored-by: Peter Alfonsi <petealft@amazon.com>
---
 .../stats/DefaultCacheStatsHolderTests.java   | 75 +++++++++++--------
 1 file changed, 42 insertions(+), 33 deletions(-)

diff --git a/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java b/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java
index c6e8252ddf806..8a59dd9d2d105 100644
--- a/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java
@@ -127,49 +127,58 @@ public void testCount() throws Exception {
     }
 
     public void testConcurrentRemoval() throws Exception {
-        List<String> dimensionNames = List.of("dim1", "dim2");
+        List<String> dimensionNames = List.of("A", "B");
         DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
 
         // Create stats for the following dimension sets
-        List<List<String>> populatedStats = List.of(List.of("A1", "B1"), List.of("A2", "B2"), List.of("A2", "B3"));
+        List<List<String>> populatedStats = new ArrayList<>();
+        int numAValues = 10;
+        int numBValues = 2;
+        for (int indexA = 0; indexA < numAValues; indexA++) {
+            for (int indexB = 0; indexB < numBValues; indexB++) {
+                populatedStats.add(List.of("A" + indexA, "B" + indexB));
+            }
+        }
         for (List<String> dims : populatedStats) {
             cacheStatsHolder.incrementHits(dims);
         }
 
-        // Remove (A2, B2) and (A1, B1), before re-adding (A2, B2). At the end we should have stats for (A2, B2) but not (A1, B1).
-
-        Thread[] threads = new Thread[3];
-        CountDownLatch countDownLatch = new CountDownLatch(3);
-        threads[0] = new Thread(() -> {
-            cacheStatsHolder.removeDimensions(List.of("A2", "B2"));
-            countDownLatch.countDown();
-        });
-        threads[1] = new Thread(() -> {
-            cacheStatsHolder.removeDimensions(List.of("A1", "B1"));
-            countDownLatch.countDown();
-        });
-        threads[2] = new Thread(() -> {
-            cacheStatsHolder.incrementMisses(List.of("A2", "B2"));
-            cacheStatsHolder.incrementMisses(List.of("A2", "B3"));
-            countDownLatch.countDown();
-        });
+        // Remove a subset of the dimensions concurrently.
+        // Remove both (A0, B0), and (A0, B1), so we expect the intermediate node for A0 to be null afterwards.
+        // For all the others, remove only the B0 value. Then we expect the intermediate nodes for A1 through A9 to be present
+        // and reflect only the stats for their B1 child.
+
+        Thread[] threads = new Thread[numAValues + 1];
+        for (int i = 0; i < numAValues; i++) {
+            int finalI = i;
+            threads[i] = new Thread(() -> { cacheStatsHolder.removeDimensions(List.of("A" + finalI, "B0")); });
+        }
+        threads[numAValues] = new Thread(() -> { cacheStatsHolder.removeDimensions(List.of("A0", "B1")); });
         for (Thread thread : threads) {
             thread.start();
-            // Add short sleep to ensure threads start their functions in order (so that incrementing doesn't happen before removal)
-            Thread.sleep(1);
         }
-        countDownLatch.await();
-        assertNull(getNode(List.of("A1", "B1"), cacheStatsHolder.getStatsRoot()));
-        assertNull(getNode(List.of("A1"), cacheStatsHolder.getStatsRoot()));
-        assertNotNull(getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot()));
-        assertEquals(
-            new ImmutableCacheStats(0, 1, 0, 0, 0),
-            getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot()).getImmutableStats()
-        );
-        assertEquals(
-            new ImmutableCacheStats(1, 1, 0, 0, 0),
-            getNode(List.of("A2", "B3"), cacheStatsHolder.getStatsRoot()).getImmutableStats()
-        );
+        for (Thread thread : threads) {
+            thread.join();
+        }
+
+        // intermediate node for A0 should be null
+        assertNull(getNode(List.of("A0"), cacheStatsHolder.getStatsRoot()));
+
+        // leaf nodes for all B0 values should be null since they were removed
+        for (int indexA = 0; indexA < numAValues; indexA++) {
+            assertNull(getNode(List.of("A" + indexA, "B0"), cacheStatsHolder.getStatsRoot()));
+        }
+
+        // leaf nodes for all B1 values, except (A0, B1), should not be null as they weren't removed,
+        // and the intermediate nodes A1 through A9 shouldn't be null as they have remaining children
+        for (int indexA = 1; indexA < numAValues; indexA++) {
+            DefaultCacheStatsHolder.Node b1LeafNode = getNode(List.of("A" + indexA, "B1"), cacheStatsHolder.getStatsRoot());
+            assertNotNull(b1LeafNode);
+            assertEquals(new ImmutableCacheStats(1, 0, 0, 0, 0), b1LeafNode.getImmutableStats());
+            DefaultCacheStatsHolder.Node intermediateLevelNode = getNode(List.of("A" + indexA), cacheStatsHolder.getStatsRoot());
+            assertNotNull(intermediateLevelNode);
+            assertEquals(b1LeafNode.getImmutableStats(), intermediateLevelNode.getImmutableStats());
+        }
     }
 
     /**

From 8839904f2ce12ba1a88d395fb9025877f2d5410f Mon Sep 17 00:00:00 2001
From: "opensearch-trigger-bot[bot]"
 <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com>
Date: Wed, 26 Jun 2024 09:24:04 -0400
Subject: [PATCH 050/113] [AUTO] [main] Add bwc version 2.15.1. (#14549)

* Add bwc version 2.15.1

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Fix auto-generated version

Signed-off-by: Andrew Ross <andrross@amazon.com>

---------

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: Andrew Ross <andrross@amazon.com>
Co-authored-by: opensearch-ci-bot <83309141+opensearch-ci-bot@users.noreply.github.com>
Co-authored-by: Andrew Ross <andrross@amazon.com>
---
 .ci/bwcVersions                                     | 1 +
 libs/core/src/main/java/org/opensearch/Version.java | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 1f80ed34d6c10..a738eb54e17f6 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -34,4 +34,5 @@ BWC_VERSION:
   - "2.14.0"
   - "2.14.1"
   - "2.15.0"
+  - "2.15.1"
   - "2.16.0"
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index 0cb2d4f867c12..da43894863432 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -105,6 +105,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_14_1 = new Version(2140199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_15_1 = new Version(2150199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version CURRENT = V_3_0_0;

From 729276f7e8f799ca21ace12bb2767869322ff253 Mon Sep 17 00:00:00 2001
From: Sagar <99425694+sgup432@users.noreply.github.com>
Date: Wed, 26 Jun 2024 11:14:42 -0700
Subject: [PATCH 051/113] Fix flaky test
 TieredSpilloverCacheTests.testComputeIfAbsentConcurrently (#14550)

* Fix flaky test TieredSpilloverCacheTests.testComputeIfAbsentConcurrently

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Addressing comment

Signed-off-by: Sagar Upadhyaya <upasagar@amazon.com>

---------

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>
Signed-off-by: Sagar Upadhyaya <upasagar@amazon.com>
Co-authored-by: Sagar Upadhyaya <upasagar@amazon.com>
---
 .../common/tier/TieredSpilloverCache.java     | 21 ++++++++++++-------
 .../tier/TieredSpilloverCacheTests.java       |  4 ++--
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
index b6d6913a9f8d4..f69c56808b2a1 100644
--- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
@@ -195,7 +195,16 @@ public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V>
         // and it only has to be loaded one time, we should report one miss and the rest hits. But, if we do stats in
         // getValueFromTieredCache(),
         // we will see all misses. Instead, handle stats in computeIfAbsent().
-        Tuple<V, String> cacheValueTuple = getValueFromTieredCache(false).apply(key);
+        Tuple<V, String> cacheValueTuple;
+        CompletableFuture<Tuple<ICacheKey<K>, V>> future = null;
+        try (ReleasableLock ignore = readLock.acquire()) {
+            cacheValueTuple = getValueFromTieredCache(false).apply(key);
+            if (cacheValueTuple == null) {
+                // Only one of the threads will succeed putting a future into map for the same key.
+                // Rest will fetch existing future and wait on that to complete.
+                future = completableFutureMap.putIfAbsent(key, new CompletableFuture<>());
+            }
+        }
         List<String> heapDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_ON_HEAP);
         List<String> diskDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_DISK);
 
@@ -203,7 +212,7 @@ public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V>
             // Add the value to the onHeap cache. We are calling computeIfAbsent which does another get inside.
             // This is needed as there can be many requests for the same key at the same time and we only want to load
             // the value once.
-            V value = compute(key, loader);
+            V value = compute(key, loader, future);
             // Handle stats
             if (loader.isLoaded()) {
                 // The value was just computed and added to the cache by this thread. Register a miss for the heap cache, and the disk cache
@@ -232,10 +241,8 @@ public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V>
         return cacheValueTuple.v1();
     }
 
-    private V compute(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
-        // Only one of the threads will succeed putting a future into map for the same key.
-        // Rest will fetch existing future and wait on that to complete.
-        CompletableFuture<Tuple<ICacheKey<K>, V>> future = completableFutureMap.putIfAbsent(key, new CompletableFuture<>());
+    private V compute(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader, CompletableFuture<Tuple<ICacheKey<K>, V>> future)
+        throws Exception {
         // Handler to handle results post processing. Takes a tuple<key, value> or exception as an input and returns
         // the value. Also before returning value, puts the value in cache.
         BiFunction<Tuple<ICacheKey<K>, V>, Throwable, Void> handler = (pair, ex) -> {
@@ -253,7 +260,7 @@ private V compute(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader
                     logger.warn("Exception occurred while trying to compute the value", ex);
                 }
             }
-            completableFutureMap.remove(key); // Remove key from map as not needed anymore.
+            completableFutureMap.remove(key);// Remove key from map as not needed anymore.
             return null;
         };
         V value = null;
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
index b9c7bbdb77d3d..c6440a1e1797f 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
@@ -760,7 +760,7 @@ public void testInvalidateAll() throws Exception {
     }
 
     public void testComputeIfAbsentConcurrently() throws Exception {
-        int onHeapCacheSize = randomIntBetween(100, 300);
+        int onHeapCacheSize = randomIntBetween(500, 700);
         int diskCacheSize = randomIntBetween(200, 400);
         int keyValueSize = 50;
 
@@ -782,7 +782,7 @@ public void testComputeIfAbsentConcurrently() throws Exception {
             0
         );
 
-        int numberOfSameKeys = randomIntBetween(10, onHeapCacheSize - 1);
+        int numberOfSameKeys = randomIntBetween(400, onHeapCacheSize - 1);
         ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
         String value = UUID.randomUUID().toString();
 

From a99b4949ec8a16f2f47b9c909f66c262ae5605f8 Mon Sep 17 00:00:00 2001
From: Andrew Ross <andrross@amazon.com>
Date: Wed, 26 Jun 2024 12:48:09 -0700
Subject: [PATCH 052/113] Add allowlist setting for ingest-common processors
 (#14479)

Add a new static setting that lets an operator choose specific ingest
processors to enable by name. The behavior is as follows:

- If the allowlist setting is not defined, all installed processors are
  enabled. This is the status quo.
- If the allowlist setting is defined as the empty set, then all processors
  are disabled.
- If the allowlist setting contains the names of valid processors, only those
  processors are enabled.
- If the allowlist setting contains a name of a processor that does not exist,
  then the server will fail to start with an IllegalStateException
  listing which processors were defined in the allowlist but are not
  installed.
- If the allowlist setting is changed between server restarts then any
  ingest pipeline using a now-disabled processor will fail. This is the
  same experience if a pipeline used a processor defined by a plugin but
  then that plugin were to be uninstalled across restarts.

Related to #14439

Signed-off-by: Andrew Ross <andrross@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../common/IngestCommonModulePlugin.java      |  39 ++++++-
 .../common/IngestCommonModulePluginTests.java | 109 ++++++++++++++++++
 3 files changed, 146 insertions(+), 3 deletions(-)
 create mode 100644 modules/ingest-common/src/test/java/org/opensearch/ingest/common/IngestCommonModulePluginTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f71ba46745ef1..b31784d5ac31c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
+- Add allowlist setting for ingest-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java
index 162934efa6778..bf9e9b71b8491 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java
@@ -58,10 +58,20 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 public class IngestCommonModulePlugin extends Plugin implements ActionPlugin, IngestPlugin {
 
+    static final Setting<List<String>> PROCESSORS_ALLOWLIST_SETTING = Setting.listSetting(
+        "ingest.common.processors.allowed",
+        List.of(),
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+
     static final Setting<TimeValue> WATCHDOG_INTERVAL = Setting.timeSetting(
         "ingest.grok.watchdog.interval",
         TimeValue.timeValueSeconds(1),
@@ -77,7 +87,7 @@ public IngestCommonModulePlugin() {}
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-        Map<String, Processor.Factory> processors = new HashMap<>();
+        final Map<String, Processor.Factory> processors = new HashMap<>();
         processors.put(DateProcessor.TYPE, new DateProcessor.Factory(parameters.scriptService));
         processors.put(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService));
         processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService));
@@ -110,7 +120,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
         processors.put(RemoveByPatternProcessor.TYPE, new RemoveByPatternProcessor.Factory());
         processors.put(CommunityIdProcessor.TYPE, new CommunityIdProcessor.Factory());
         processors.put(FingerprintProcessor.TYPE, new FingerprintProcessor.Factory());
-        return Collections.unmodifiableMap(processors);
+        return filterForAllowlistSetting(parameters.env.settings(), processors);
     }
 
     @Override
@@ -133,7 +143,7 @@ public List<RestHandler> getRestHandlers(
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME);
+        return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME, PROCESSORS_ALLOWLIST_SETTING);
     }
 
     private static MatcherWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) {
@@ -147,4 +157,27 @@ private static MatcherWatchdog createGrokThreadWatchdog(Processor.Parameters par
         );
     }
 
+    private Map<String, Processor.Factory> filterForAllowlistSetting(Settings settings, Map<String, Processor.Factory> map) {
+        if (PROCESSORS_ALLOWLIST_SETTING.exists(settings) == false) {
+            return Map.copyOf(map);
+        }
+        final Set<String> allowlist = Set.copyOf(PROCESSORS_ALLOWLIST_SETTING.get(settings));
+        // Assert that no unknown processors are defined in the allowlist
+        final Set<String> unknownAllowlistProcessors = allowlist.stream()
+            .filter(p -> map.containsKey(p) == false)
+            .collect(Collectors.toSet());
+        if (unknownAllowlistProcessors.isEmpty() == false) {
+            throw new IllegalArgumentException(
+                "Processor(s) "
+                    + unknownAllowlistProcessors
+                    + " were defined in ["
+                    + PROCESSORS_ALLOWLIST_SETTING.getKey()
+                    + "] but do not exist"
+            );
+        }
+        return map.entrySet()
+            .stream()
+            .filter(e -> allowlist.contains(e.getKey()))
+            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/IngestCommonModulePluginTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/IngestCommonModulePluginTests.java
new file mode 100644
index 0000000000000..b0c1e0fdbaa63
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/IngestCommonModulePluginTests.java
@@ -0,0 +1,109 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ingest.common;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.env.TestEnvironment;
+import org.opensearch.ingest.Processor;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+public class IngestCommonModulePluginTests extends OpenSearchTestCase {
+
+    public void testAllowlist() throws IOException {
+        runAllowlistTest(List.of());
+        runAllowlistTest(List.of("date"));
+        runAllowlistTest(List.of("set"));
+        runAllowlistTest(List.of("copy", "date"));
+        runAllowlistTest(List.of("date", "set", "copy"));
+    }
+
+    private void runAllowlistTest(List<String> allowlist) throws IOException {
+        final Settings settings = Settings.builder()
+            .putList(IngestCommonModulePlugin.PROCESSORS_ALLOWLIST_SETTING.getKey(), allowlist)
+            .build();
+        try (IngestCommonModulePlugin plugin = new IngestCommonModulePlugin()) {
+            assertEquals(Set.copyOf(allowlist), plugin.getProcessors(createParameters(settings)).keySet());
+        }
+    }
+
+    public void testAllowlistNotSpecified() throws IOException {
+        final Settings.Builder builder = Settings.builder();
+        builder.remove(IngestCommonModulePlugin.PROCESSORS_ALLOWLIST_SETTING.getKey());
+        final Settings settings = builder.build();
+        try (IngestCommonModulePlugin plugin = new IngestCommonModulePlugin()) {
+            final Set<String> expected = Set.of(
+                "append",
+                "urldecode",
+                "sort",
+                "fail",
+                "trim",
+                "set",
+                "fingerprint",
+                "pipeline",
+                "json",
+                "join",
+                "kv",
+                "bytes",
+                "date",
+                "drop",
+                "community_id",
+                "lowercase",
+                "convert",
+                "copy",
+                "gsub",
+                "dot_expander",
+                "rename",
+                "remove_by_pattern",
+                "html_strip",
+                "remove",
+                "csv",
+                "grok",
+                "date_index_name",
+                "foreach",
+                "script",
+                "dissect",
+                "uppercase",
+                "split"
+            );
+            assertEquals(expected, plugin.getProcessors(createParameters(settings)).keySet());
+        }
+    }
+
+    public void testAllowlistHasNonexistentProcessors() throws IOException {
+        final Settings settings = Settings.builder()
+            .putList(IngestCommonModulePlugin.PROCESSORS_ALLOWLIST_SETTING.getKey(), List.of("threeve"))
+            .build();
+        try (IngestCommonModulePlugin plugin = new IngestCommonModulePlugin()) {
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> plugin.getProcessors(createParameters(settings))
+            );
+            assertTrue(e.getMessage(), e.getMessage().contains("threeve"));
+        }
+    }
+
+    private static Processor.Parameters createParameters(Settings settings) {
+        return new Processor.Parameters(
+            TestEnvironment.newEnvironment(Settings.builder().put(settings).put("path.home", "").build()),
+            null,
+            null,
+            null,
+            () -> 0L,
+            (a, b) -> null,
+            null,
+            null,
+            $ -> {},
+            null
+        );
+    }
+}

From 2be25bbfe83800e96752aee0acfd01b50d9c0a68 Mon Sep 17 00:00:00 2001
From: panguixin <panguixin@bytedance.com>
Date: Thu, 27 Jun 2024 07:16:18 +0800
Subject: [PATCH 053/113] Fix file cache initialization (#14004)

* fix file cache initialization

Signed-off-by: panguixin <panguixin@bytedance.com>

* changelog

Signed-off-by: panguixin <panguixin@bytedance.com>

* add test

Signed-off-by: panguixin <panguixin@bytedance.com>

---------

Signed-off-by: panguixin <panguixin@bytedance.com>
---
 CHANGELOG.md                                  |  1 +
 .../snapshots/SearchableSnapshotIT.java       | 25 ++++++
 .../cluster/node/DiscoveryNode.java           |  4 +
 .../remote/utils/cache/SegmentedCache.java    |  4 +-
 .../org/opensearch/monitor/fs/FsProbe.java    | 13 ++-
 .../main/java/org/opensearch/node/Node.java   | 86 ++++++++++++-------
 .../env/NodeRepurposeCommandTests.java        |  2 +-
 .../java/org/opensearch/node/NodeTests.java   |  2 +-
 .../opensearch/test/InternalTestCluster.java  | 10 ++-
 9 files changed, 106 insertions(+), 41 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b31784d5ac31c..3f8fff1db214a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix fs info reporting negative available size ([#11573](https://github.com/opensearch-project/OpenSearch/pull/11573))
 - Add ListPitInfo::getKeepAlive() getter ([#14495](https://github.com/opensearch-project/OpenSearch/pull/14495))
 - Fix FuzzyQuery in keyword field will use IndexOrDocValuesQuery when both of index and doc_value are true ([#14378](https://github.com/opensearch-project/OpenSearch/pull/14378))
+- Fix file cache initialization ([#14004](https://github.com/opensearch-project/OpenSearch/pull/14004))
 
 ### Security
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
index 2440a3c64e956..1c199df4d548e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
@@ -28,6 +28,7 @@
 import org.opensearch.cluster.block.ClusterBlockException;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.routing.GroupShardsIterator;
 import org.opensearch.cluster.routing.ShardIterator;
 import org.opensearch.cluster.routing.ShardRouting;
@@ -35,6 +36,7 @@
 import org.opensearch.common.Priority;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.index.Index;
@@ -65,10 +67,13 @@
 import java.util.stream.StreamSupport;
 
 import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS;
+import static org.opensearch.common.util.FeatureFlags.TIERED_REMOTE_INDEX;
 import static org.opensearch.core.common.util.CollectionUtils.iterableAsArrayList;
 import static org.opensearch.index.store.remote.filecache.FileCacheSettings.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING;
 import static org.opensearch.test.NodeRoles.clusterManagerOnlyNode;
 import static org.opensearch.test.NodeRoles.dataNode;
+import static org.opensearch.test.NodeRoles.onlyRole;
+import static org.opensearch.test.NodeRoles.onlyRoles;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
@@ -1009,6 +1014,26 @@ public void cleanup() throws Exception {
         );
     }
 
+    public void testStartSearchNode() throws Exception {
+        // test start dedicated search node
+        internalCluster().startNode(Settings.builder().put(onlyRole(DiscoveryNodeRole.SEARCH_ROLE)));
+        // test start node without search role
+        internalCluster().startNode(Settings.builder().put(onlyRole(DiscoveryNodeRole.DATA_ROLE)));
+        // test start non-dedicated search node with TIERED_REMOTE_INDEX feature enabled
+        internalCluster().startNode(
+            Settings.builder()
+                .put(onlyRoles(Set.of(DiscoveryNodeRole.SEARCH_ROLE, DiscoveryNodeRole.DATA_ROLE)))
+                .put(TIERED_REMOTE_INDEX, true)
+        );
+        // test start non-dedicated search node
+        assertThrows(
+            SettingsException.class,
+            () -> internalCluster().startNode(
+                Settings.builder().put(onlyRoles(Set.of(DiscoveryNodeRole.SEARCH_ROLE, DiscoveryNodeRole.DATA_ROLE)))
+            )
+        );
+    }
+
     private void assertSearchableSnapshotIndexDirectoryExistence(String nodeName, Index index, boolean exists) throws Exception {
         final Node node = internalCluster().getInstance(Node.class, nodeName);
         final ShardId shardId = new ShardId(index, 0);
diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
index 690621c2e7bca..653f81830ed17 100644
--- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
@@ -130,6 +130,10 @@ public static boolean isSearchNode(Settings settings) {
         return hasRole(settings, DiscoveryNodeRole.SEARCH_ROLE);
     }
 
+    public static boolean isDedicatedSearchNode(Settings settings) {
+        return getRolesFromSettings(settings).stream().allMatch(DiscoveryNodeRole.SEARCH_ROLE::equals);
+    }
+
     private final String nodeName;
     private final String nodeId;
     private final String ephemeralId;
diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java
index 2ea7ea8dbee12..9ff6ddb1fb667 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java
@@ -52,15 +52,15 @@ private static final int ceilingNextPowerOfTwo(int x) {
     private final Weigher<V> weigher;
 
     public SegmentedCache(Builder<K, V> builder) {
-        this.capacity = builder.capacity;
         final int segments = ceilingNextPowerOfTwo(builder.concurrencyLevel);
         this.segmentMask = segments - 1;
         this.table = newSegmentArray(segments);
-        this.perSegmentCapacity = (capacity + (segments - 1)) / segments;
+        this.perSegmentCapacity = (builder.capacity + (segments - 1)) / segments;
         this.weigher = builder.weigher;
         for (int i = 0; i < table.length; i++) {
             table[i] = new LRUCache<>(perSegmentCapacity, builder.listener, builder.weigher);
         }
+        this.capacity = perSegmentCapacity * segments;
     }
 
     @SuppressWarnings("unchecked")
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
index f93cb63ff1f0a..db77ec7628e76 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
@@ -81,7 +81,11 @@ public FsInfo stats(FsInfo previous) throws IOException {
             if (fileCache != null && dataLocations[i].fileCacheReservedSize != ByteSizeValue.ZERO) {
                 paths[i].fileCacheReserved = adjustForHugeFilesystems(dataLocations[i].fileCacheReservedSize.getBytes());
                 paths[i].fileCacheUtilized = adjustForHugeFilesystems(fileCache.usage().usage());
-                paths[i].available -= (paths[i].fileCacheReserved - paths[i].fileCacheUtilized);
+                // fileCacheFree will be less than zero if the cache being over-subscribed
+                long fileCacheFree = paths[i].fileCacheReserved - paths[i].fileCacheUtilized;
+                if (fileCacheFree > 0) {
+                    paths[i].available -= fileCacheFree;
+                }
                 // occurs if reserved file cache space is occupied by other files, like local indices
                 if (paths[i].available < 0) {
                     paths[i].available = 0;
@@ -215,4 +219,11 @@ public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException {
         return fsPath;
     }
 
+    public static long getTotalSize(NodePath nodePath) throws IOException {
+        return adjustForHugeFilesystems(nodePath.fileStore.getTotalSpace());
+    }
+
+    public static long getAvailableSize(NodePath nodePath) throws IOException {
+        return adjustForHugeFilesystems(nodePath.fileStore.getUsableSpace());
+    }
 }
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index a91dce4ece126..505c9264d62bb 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -38,6 +38,7 @@
 import org.opensearch.Build;
 import org.opensearch.ExceptionsHelper;
 import org.opensearch.OpenSearchException;
+import org.opensearch.OpenSearchParseException;
 import org.opensearch.OpenSearchTimeoutException;
 import org.opensearch.Version;
 import org.opensearch.action.ActionModule;
@@ -108,6 +109,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.settings.SettingsModule;
+import org.opensearch.common.unit.RatioValue;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.FeatureFlags;
@@ -176,7 +178,6 @@
 import org.opensearch.ingest.IngestService;
 import org.opensearch.monitor.MonitorService;
 import org.opensearch.monitor.fs.FsHealthService;
-import org.opensearch.monitor.fs.FsInfo;
 import org.opensearch.monitor.fs.FsProbe;
 import org.opensearch.monitor.jvm.JvmInfo;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
@@ -372,9 +373,12 @@ public class Node implements Closeable {
         }
     }, Setting.Property.NodeScope);
 
-    public static final Setting<ByteSizeValue> NODE_SEARCH_CACHE_SIZE_SETTING = Setting.byteSizeSetting(
+    private static final String ZERO = "0";
+
+    public static final Setting<String> NODE_SEARCH_CACHE_SIZE_SETTING = new Setting<>(
         "node.search.cache.size",
-        ByteSizeValue.ZERO,
+        s -> (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX_SETTING) || DiscoveryNode.isDedicatedSearchNode(s)) ? "80%" : ZERO,
+        Node::validateFileCacheSize,
         Property.NodeScope
     );
 
@@ -2002,43 +2006,59 @@ DiscoveryNode getNode() {
      * Initializes the search cache with a defined capacity.
      * The capacity of the cache is based on user configuration for {@link Node#NODE_SEARCH_CACHE_SIZE_SETTING}.
      * If the user doesn't configure the cache size, it fails if the node is a data + search node.
-     * Else it configures the size to 80% of available capacity for a dedicated search node, if not explicitly defined.
+     * Else it configures the size to 80% of total capacity for a dedicated search node, if not explicitly defined.
      */
     private void initializeFileCache(Settings settings, CircuitBreaker circuitBreaker) throws IOException {
         boolean isWritableRemoteIndexEnabled = FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX_SETTING);
-        if (DiscoveryNode.isSearchNode(settings) || isWritableRemoteIndexEnabled) {
-            NodeEnvironment.NodePath fileCacheNodePath = nodeEnvironment.fileCacheNodePath();
-            long capacity = NODE_SEARCH_CACHE_SIZE_SETTING.get(settings).getBytes();
-            FsInfo.Path info = ExceptionsHelper.catchAsRuntimeException(() -> FsProbe.getFSInfo(fileCacheNodePath));
-            long availableCapacity = info.getAvailable().getBytes();
-
-            // Initialize default values for cache if NODE_SEARCH_CACHE_SIZE_SETTING is not set.
-            if (capacity == 0) {
-                // If node is not a dedicated search node without configuration, prevent cache initialization
-                if (!isWritableRemoteIndexEnabled
-                    && DiscoveryNode.getRolesFromSettings(settings)
-                        .stream()
-                        .anyMatch(role -> !DiscoveryNodeRole.SEARCH_ROLE.equals(role))) {
-                    throw new SettingsException(
-                        "Unable to initialize the "
-                            + DiscoveryNodeRole.SEARCH_ROLE.roleName()
-                            + "-"
-                            + DiscoveryNodeRole.DATA_ROLE.roleName()
-                            + " node: Missing value for configuration "
-                            + NODE_SEARCH_CACHE_SIZE_SETTING.getKey()
-                    );
-                } else {
-                    capacity = 80 * availableCapacity / 100;
-                }
+        if (DiscoveryNode.isSearchNode(settings) == false && isWritableRemoteIndexEnabled == false) {
+            return;
+        }
+
+        String capacityRaw = NODE_SEARCH_CACHE_SIZE_SETTING.get(settings);
+        logger.info("cache size [{}]", capacityRaw);
+        if (capacityRaw.equals(ZERO)) {
+            throw new SettingsException(
+                "Unable to initialize the "
+                    + DiscoveryNodeRole.SEARCH_ROLE.roleName()
+                    + "-"
+                    + DiscoveryNodeRole.DATA_ROLE.roleName()
+                    + " node: Missing value for configuration "
+                    + NODE_SEARCH_CACHE_SIZE_SETTING.getKey()
+            );
+        }
+
+        NodeEnvironment.NodePath fileCacheNodePath = nodeEnvironment.fileCacheNodePath();
+        long totalSpace = ExceptionsHelper.catchAsRuntimeException(() -> FsProbe.getTotalSize(fileCacheNodePath));
+        long capacity = calculateFileCacheSize(capacityRaw, totalSpace);
+        if (capacity <= 0 || totalSpace <= capacity) {
+            throw new SettingsException("Cache size must be larger than zero and less than total capacity");
+        }
+
+        this.fileCache = FileCacheFactory.createConcurrentLRUFileCache(capacity, circuitBreaker);
+        fileCacheNodePath.fileCacheReservedSize = new ByteSizeValue(this.fileCache.capacity(), ByteSizeUnit.BYTES);
+        List<Path> fileCacheDataPaths = collectFileCacheDataPath(fileCacheNodePath);
+        this.fileCache.restoreFromDirectory(fileCacheDataPaths);
+    }
+
+    private static long calculateFileCacheSize(String capacityRaw, long totalSpace) {
+        try {
+            RatioValue ratioValue = RatioValue.parseRatioValue(capacityRaw);
+            return Math.round(totalSpace * ratioValue.getAsRatio());
+        } catch (OpenSearchParseException e) {
+            try {
+                return ByteSizeValue.parseBytesSizeValue(capacityRaw, NODE_SEARCH_CACHE_SIZE_SETTING.getKey()).getBytes();
+            } catch (OpenSearchParseException ex) {
+                ex.addSuppressed(e);
+                throw ex;
             }
-            capacity = Math.min(capacity, availableCapacity);
-            fileCacheNodePath.fileCacheReservedSize = new ByteSizeValue(capacity, ByteSizeUnit.BYTES);
-            this.fileCache = FileCacheFactory.createConcurrentLRUFileCache(capacity, circuitBreaker);
-            List<Path> fileCacheDataPaths = collectFileCacheDataPath(fileCacheNodePath);
-            this.fileCache.restoreFromDirectory(fileCacheDataPaths);
         }
     }
 
+    private static String validateFileCacheSize(String capacityRaw) {
+        calculateFileCacheSize(capacityRaw, 0L);
+        return capacityRaw;
+    }
+
     /**
      * Returns the {@link FileCache} instance for remote search node
      * Note: Visible for testing
diff --git a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java
index 2a3525143c01f..d2d6fdc387dfe 100644
--- a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java
+++ b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java
@@ -95,7 +95,7 @@ public void createNodePaths() throws IOException {
         dataClusterManagerSettings = buildEnvSettings(Settings.EMPTY);
         Settings defaultSearchSettings = Settings.builder()
             .put(dataClusterManagerSettings)
-            .put(NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(16, ByteSizeUnit.GB))
+            .put(NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(16, ByteSizeUnit.GB).toString())
             .build();
 
         searchNoDataNoClusterManagerSettings = onlyRole(dataClusterManagerSettings, DiscoveryNodeRole.SEARCH_ROLE);
diff --git a/server/src/test/java/org/opensearch/node/NodeTests.java b/server/src/test/java/org/opensearch/node/NodeTests.java
index f44cc352cd330..0093091f61a1c 100644
--- a/server/src/test/java/org/opensearch/node/NodeTests.java
+++ b/server/src/test/java/org/opensearch/node/NodeTests.java
@@ -380,7 +380,7 @@ public void testCreateWithFileCache() throws Exception {
         List<Class<? extends Plugin>> plugins = basePlugins();
         ByteSizeValue cacheSize = new ByteSizeValue(16, ByteSizeUnit.GB);
         Settings searchRoleSettingsWithConfig = baseSettings().put(searchRoleSettings)
-            .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize)
+            .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize.toString())
             .build();
         Settings onlySearchRoleSettings = Settings.builder()
             .put(searchRoleSettingsWithConfig)
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index ca80c65e58522..ec88002317284 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -165,6 +165,7 @@
 import static org.opensearch.test.NodeRoles.onlyRoles;
 import static org.opensearch.test.NodeRoles.removeRoles;
 import static org.opensearch.test.OpenSearchTestCase.assertBusy;
+import static org.opensearch.test.OpenSearchTestCase.randomBoolean;
 import static org.opensearch.test.OpenSearchTestCase.randomFrom;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -216,7 +217,8 @@ public final class InternalTestCluster extends TestCluster {
         nodeAndClient.node.settings()
     );
 
-    private static final ByteSizeValue DEFAULT_SEARCH_CACHE_SIZE = new ByteSizeValue(2, ByteSizeUnit.GB);
+    private static final String DEFAULT_SEARCH_CACHE_SIZE_BYTES = "2gb";
+    private static final String DEFAULT_SEARCH_CACHE_SIZE_PERCENT = "5%";
 
     public static final int DEFAULT_LOW_NUM_CLUSTER_MANAGER_NODES = 1;
     public static final int DEFAULT_HIGH_NUM_CLUSTER_MANAGER_NODES = 3;
@@ -700,8 +702,10 @@ public synchronized void ensureAtLeastNumSearchAndDataNodes(int n) {
             logger.info("increasing cluster size from {} to {}", size, n);
             Set<DiscoveryNodeRole> searchAndDataRoles = Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.SEARCH_ROLE);
             Settings settings = Settings.builder()
-                .put(Settings.EMPTY)
-                .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), DEFAULT_SEARCH_CACHE_SIZE)
+                .put(
+                    Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(),
+                    randomBoolean() ? DEFAULT_SEARCH_CACHE_SIZE_PERCENT : DEFAULT_SEARCH_CACHE_SIZE_BYTES
+                )
                 .build();
             startNodes(n - size, Settings.builder().put(onlyRoles(settings, searchAndDataRoles)).build());
             validateClusterFormed();

From bb9819c3ed4d319e67f683d5fedd23184b39b85f Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Thu, 27 Jun 2024 07:48:42 +0530
Subject: [PATCH 054/113] Add Ashish Singh as maintainer (#14567)

Signed-off-by: Bukhtawar Khan <bukhtawa@amazon.com>
---
 .github/CODEOWNERS | 2 +-
 MAINTAINERS.md     | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 5a2d08756c49f..8d69e98220b69 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index 91b57a4cbc74e..3298ceb15463c 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -9,6 +9,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Anas Alkouz              | [anasalkouz](https://github.com/anasalkouz)             | Amazon      |
 | Andrew Ross              | [andrross](https://github.com/andrross)                 | Amazon      |
 | Andriy Redko             | [reta](https://github.com/reta)                         | Aiven       |
+| Ashish Singh             | [ashking94](https://github.com/ashking94)               | Amazon      |
 | Bukhtawar Khan           | [Bukhtawar](https://github.com/Bukhtawar)               | Amazon      |
 | Charlotte Henkle         | [CEHENKLE](https://github.com/CEHENKLE)                 | Amazon      |
 | Dan Widdis               | [dbwiddis](https://github.com/dbwiddis)                 | Amazon      |

From 391dee23aa2e8a808821389c379499e91ee2d550 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Thu, 27 Jun 2024 12:16:16 -0400
Subject: [PATCH 055/113] Allow @InternalApi annotation on classes not meant to
 be constructed outside of the OpenSearch core (#14575)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                  |  1 +
 .../ApiAnnotationProcessorTests.java          | 13 ++++++++++++
 .../processor/InternalApiAnnotated.java       |  4 ++--
 ...licApiConstructorAnnotatedInternalApi.java | 21 +++++++++++++++++++
 4 files changed, 37 insertions(+), 2 deletions(-)
 create mode 100644 libs/common/src/test/resources/org/opensearch/common/annotation/processor/PublicApiConstructorAnnotatedInternalApi.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f8fff1db214a..e01dfed0e585e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - unsignedLongRangeQuery now returns MatchNoDocsQuery if the lower bounds are greater than the upper bounds ([#14416](https://github.com/opensearch-project/OpenSearch/pull/14416))
 - Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568))
 - Make the class CommunityIdProcessor final ([#14448](https://github.com/opensearch-project/OpenSearch/pull/14448))
+- Allow @InternalApi annotation on classes not meant to be constructed outside of the OpenSearch core ([#14575](https://github.com/opensearch-project/OpenSearch/pull/14575))
 
 ### Deprecated
 
diff --git a/libs/common/src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java b/libs/common/src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java
index 8d8a4c7895339..52162e3df0c1c 100644
--- a/libs/common/src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java
+++ b/libs/common/src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java
@@ -473,4 +473,17 @@ public void testPublicApiWithProtectedInterface() {
 
         assertThat(failure.diagnotics(), not(hasItem(matching(Diagnostic.Kind.ERROR))));
     }
+
+    /**
+     * The constructor arguments have relaxed semantics at the moment: those could be not annotated or be annotated as {@link InternalApi}
+     */
+    public void testPublicApiConstructorAnnotatedInternalApi() {
+        final CompilerResult result = compile("PublicApiConstructorAnnotatedInternalApi.java", "NotAnnotated.java");
+        assertThat(result, instanceOf(Failure.class));
+
+        final Failure failure = (Failure) result;
+        assertThat(failure.diagnotics(), hasSize(2));
+
+        assertThat(failure.diagnotics(), not(hasItem(matching(Diagnostic.Kind.ERROR))));
+    }
 }
diff --git a/libs/common/src/test/resources/org/opensearch/common/annotation/processor/InternalApiAnnotated.java b/libs/common/src/test/resources/org/opensearch/common/annotation/processor/InternalApiAnnotated.java
index 9996ba8b736aa..b0b542e127285 100644
--- a/libs/common/src/test/resources/org/opensearch/common/annotation/processor/InternalApiAnnotated.java
+++ b/libs/common/src/test/resources/org/opensearch/common/annotation/processor/InternalApiAnnotated.java
@@ -8,9 +8,9 @@
 
 package org.opensearch.common.annotation.processor;
 
-import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.annotation.InternalApi;
 
-@PublicApi(since = "1.0.0")
+@InternalApi
 public class InternalApiAnnotated {
 
 }
diff --git a/libs/common/src/test/resources/org/opensearch/common/annotation/processor/PublicApiConstructorAnnotatedInternalApi.java b/libs/common/src/test/resources/org/opensearch/common/annotation/processor/PublicApiConstructorAnnotatedInternalApi.java
new file mode 100644
index 0000000000000..d355a6b770391
--- /dev/null
+++ b/libs/common/src/test/resources/org/opensearch/common/annotation/processor/PublicApiConstructorAnnotatedInternalApi.java
@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.annotation.processor;
+
+import org.opensearch.common.annotation.InternalApi;
+import org.opensearch.common.annotation.PublicApi;
+
+@PublicApi(since = "1.0.0")
+public class PublicApiConstructorAnnotatedInternalApi {
+    /**
+     * The constructors have relaxed semantics at the moment: those could be not annotated or be annotated as {@link InternalApi}
+     */
+    @InternalApi
+    public PublicApiConstructorAnnotatedInternalApi(NotAnnotated arg) {}
+}

From f70fd71c388fe1903cb1b1efa2675f2428199aa8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 27 Jun 2024 12:17:51 -0400
Subject: [PATCH 056/113] Bump com.azure:azure-storage-common from 12.21.2 to
 12.25.1 in /plugins/repository-azure (#14517)

* Bump com.azure:azure-storage-common in /plugins/repository-azure

Bumps [com.azure:azure-storage-common](https://github.com/Azure/azure-sdk-for-java) from 12.21.2 to 12.25.1.
- [Release notes](https://github.com/Azure/azure-sdk-for-java/releases)
- [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-storage-common_12.21.2...azure-storage-blob_12.25.1)

---
updated-dependencies:
- dependency-name: com.azure:azure-storage-common
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                                   | 1 +
 plugins/repository-azure/build.gradle                          | 2 +-
 .../licenses/azure-storage-common-12.21.2.jar.sha1             | 1 -
 .../licenses/azure-storage-common-12.25.1.jar.sha1             | 1 +
 .../org/opensearch/repositories/azure/AzureStorageService.java | 3 +++
 .../src/main/plugin-metadata/plugin-security.policy            | 1 +
 6 files changed, 7 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.21.2.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e01dfed0e585e..5a52250906ff6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
 - Bump `opentelemetry` from 1.36.0 to 1.39.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457))
 - Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506))
+- Bump `com.azure:azure-storage-common` from 12.21.2 to 12.25.1 ([#14517](https://github.com/opensearch-project/OpenSearch/pull/14517))
 
 ### Changed
 - [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187))
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index f3aa64316b667..f88d291a8eb4a 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -47,7 +47,7 @@ dependencies {
   api 'com.azure:azure-core:1.49.1'
   api 'com.azure:azure-json:1.1.0'
   api 'com.azure:azure-xml:1.0.0'
-  api 'com.azure:azure-storage-common:12.21.2'
+  api 'com.azure:azure-storage-common:12.25.1'
   api 'com.azure:azure-core-http-netty:1.15.1'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.21.2.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.21.2.jar.sha1
deleted file mode 100644
index b3c73774764df..0000000000000
--- a/plugins/repository-azure/licenses/azure-storage-common-12.21.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d2676d4fc40a501bd5d0437b8d2bfb9926022bea
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1
new file mode 100644
index 0000000000000..822a60d81ca27
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1
@@ -0,0 +1 @@
+96e2df76ce9a8fa084ae289bb59295d565f2b8d5
\ No newline at end of file
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
index f39ed185d8b35..4f30247f0af08 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
@@ -141,6 +141,9 @@ public Void run() {
         // - https://github.com/Azure/azure-sdk-for-java/pull/25004
         // - https://github.com/Azure/azure-sdk-for-java/pull/24374
         Configuration.getGlobalConfiguration().put("AZURE_JACKSON_ADAPTER_USE_ACCESS_HELPER", "true");
+        // See please:
+        // - https://github.com/Azure/azure-sdk-for-java/issues/37464
+        Configuration.getGlobalConfiguration().put("AZURE_ENABLE_SHUTDOWN_HOOK_WITH_PRIVILEGE", "true");
     }
 
     public AzureStorageService(Settings settings) {
diff --git a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
index e8fbe35ebab1d..eedcfd98da150 100644
--- a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
@@ -38,6 +38,7 @@ grant {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   permission java.lang.RuntimePermission "setContextClassLoader";
+  permission java.lang.RuntimePermission "shutdownHooks";
 
   // azure client set Authenticator for proxy username/password
   permission java.net.NetPermission "setDefaultAuthenticator";

From d9e9944670ffc02dfef6c6a5e50dabd14779b67a Mon Sep 17 00:00:00 2001
From: Andrew Ross <andrross@amazon.com>
Date: Thu, 27 Jun 2024 12:14:38 -0500
Subject: [PATCH 057/113] Add allowlist setting for search-pipeline-common
 processors (#14562)

Add a new static setting that lets an operator choose specific search
pipeline processors to enable by name. The behavior is as follows:

- If the allowlist setting is not defined, all installed processors are
  enabled. This is the status quo.
- If the allowlist setting is defined as the empty set, then all processors
  are disabled.
- If the allowlist setting contains the names of valid processors, only those
  processors are enabled.
- If the allowlist setting contains a name of a processor that does not exist,
  then the server will fail to start with an IllegalStateException
  listing which processors were defined in the allowlist but are not
  installed.
- If the allowlist setting is changed between server restarts then any
  ingest pipeline using a now-disabled processor will fail. This is the
  same experience if a pipeline used a processor defined by a plugin but
  then that plugin were to be uninstalled across restarts.

A distinct setting exists for each of request, response, and search phase
results processors.

Related to #14439

Signed-off-by: Andrew Ross <andrross@amazon.com>
---
 CHANGELOG.md                                  |   2 +-
 .../common/IngestCommonModulePlugin.java      |   2 +-
 .../SearchPipelineCommonModulePlugin.java     | 102 ++++++++++++++---
 ...SearchPipelineCommonModulePluginTests.java | 106 ++++++++++++++++++
 4 files changed, 196 insertions(+), 16 deletions(-)
 create mode 100644 modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePluginTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5a52250906ff6..c6b2d815750f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
-- Add allowlist setting for ingest-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
+- Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java
index bf9e9b71b8491..5b2db9ff940e7 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/IngestCommonModulePlugin.java
@@ -165,7 +165,7 @@ private Map<String, Processor.Factory> filterForAllowlistSetting(Settings settin
         // Assert that no unknown processors are defined in the allowlist
         final Set<String> unknownAllowlistProcessors = allowlist.stream()
             .filter(p -> map.containsKey(p) == false)
-            .collect(Collectors.toSet());
+            .collect(Collectors.toUnmodifiableSet());
         if (unknownAllowlistProcessors.isEmpty() == false) {
             throw new IllegalArgumentException(
                 "Processor(s) "
diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java
index 5378a6721efb2..1574621a8200e 100644
--- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java
+++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java
@@ -8,24 +8,61 @@
 
 package org.opensearch.search.pipeline.common;
 
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.SearchPipelinePlugin;
 import org.opensearch.search.pipeline.Processor;
+import org.opensearch.search.pipeline.SearchPhaseResultsProcessor;
 import org.opensearch.search.pipeline.SearchRequestProcessor;
 import org.opensearch.search.pipeline.SearchResponseProcessor;
 
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Plugin providing common search request/response processors for use in search pipelines.
  */
 public class SearchPipelineCommonModulePlugin extends Plugin implements SearchPipelinePlugin {
 
+    static final Setting<List<String>> REQUEST_PROCESSORS_ALLOWLIST_SETTING = Setting.listSetting(
+        "search.pipeline.common.request.processors.allowed",
+        List.of(),
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+
+    static final Setting<List<String>> RESPONSE_PROCESSORS_ALLOWLIST_SETTING = Setting.listSetting(
+        "search.pipeline.common.response.processors.allowed",
+        List.of(),
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+
+    static final Setting<List<String>> SEARCH_PHASE_RESULTS_PROCESSORS_ALLOWLIST_SETTING = Setting.listSetting(
+        "search.pipeline.common.search.phase.results.processors.allowed",
+        List.of(),
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+
     /**
      * No constructor needed, but build complains if we don't have a constructor with JavaDoc.
      */
     public SearchPipelineCommonModulePlugin() {}
 
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(
+            REQUEST_PROCESSORS_ALLOWLIST_SETTING,
+            RESPONSE_PROCESSORS_ALLOWLIST_SETTING,
+            SEARCH_PHASE_RESULTS_PROCESSORS_ALLOWLIST_SETTING
+        );
+    }
+
     /**
      * Returns a map of processor factories.
      *
@@ -34,25 +71,62 @@ public SearchPipelineCommonModulePlugin() {}
      */
     @Override
     public Map<String, Processor.Factory<SearchRequestProcessor>> getRequestProcessors(Parameters parameters) {
-        return Map.of(
-            FilterQueryRequestProcessor.TYPE,
-            new FilterQueryRequestProcessor.Factory(parameters.namedXContentRegistry),
-            ScriptRequestProcessor.TYPE,
-            new ScriptRequestProcessor.Factory(parameters.scriptService),
-            OversampleRequestProcessor.TYPE,
-            new OversampleRequestProcessor.Factory()
+        return filterForAllowlistSetting(
+            REQUEST_PROCESSORS_ALLOWLIST_SETTING,
+            parameters.env.settings(),
+            Map.of(
+                FilterQueryRequestProcessor.TYPE,
+                new FilterQueryRequestProcessor.Factory(parameters.namedXContentRegistry),
+                ScriptRequestProcessor.TYPE,
+                new ScriptRequestProcessor.Factory(parameters.scriptService),
+                OversampleRequestProcessor.TYPE,
+                new OversampleRequestProcessor.Factory()
+            )
         );
     }
 
     @Override
     public Map<String, Processor.Factory<SearchResponseProcessor>> getResponseProcessors(Parameters parameters) {
-        return Map.of(
-            RenameFieldResponseProcessor.TYPE,
-            new RenameFieldResponseProcessor.Factory(),
-            TruncateHitsResponseProcessor.TYPE,
-            new TruncateHitsResponseProcessor.Factory(),
-            CollapseResponseProcessor.TYPE,
-            new CollapseResponseProcessor.Factory()
+        return filterForAllowlistSetting(
+            RESPONSE_PROCESSORS_ALLOWLIST_SETTING,
+            parameters.env.settings(),
+            Map.of(
+                RenameFieldResponseProcessor.TYPE,
+                new RenameFieldResponseProcessor.Factory(),
+                TruncateHitsResponseProcessor.TYPE,
+                new TruncateHitsResponseProcessor.Factory(),
+                CollapseResponseProcessor.TYPE,
+                new CollapseResponseProcessor.Factory()
+            )
         );
     }
+
+    @Override
+    public Map<String, Processor.Factory<SearchPhaseResultsProcessor>> getSearchPhaseResultsProcessors(Parameters parameters) {
+        return filterForAllowlistSetting(SEARCH_PHASE_RESULTS_PROCESSORS_ALLOWLIST_SETTING, parameters.env.settings(), Map.of());
+    }
+
+    private <T extends Processor> Map<String, Processor.Factory<T>> filterForAllowlistSetting(
+        Setting<List<String>> allowlistSetting,
+        Settings settings,
+        Map<String, Processor.Factory<T>> map
+    ) {
+        if (allowlistSetting.exists(settings) == false) {
+            return Map.copyOf(map);
+        }
+        final Set<String> allowlist = Set.copyOf(allowlistSetting.get(settings));
+        // Assert that no unknown processors are defined in the allowlist
+        final Set<String> unknownAllowlistProcessors = allowlist.stream()
+            .filter(p -> map.containsKey(p) == false)
+            .collect(Collectors.toUnmodifiableSet());
+        if (unknownAllowlistProcessors.isEmpty() == false) {
+            throw new IllegalArgumentException(
+                "Processor(s) " + unknownAllowlistProcessors + " were defined in [" + allowlistSetting.getKey() + "] but do not exist"
+            );
+        }
+        return map.entrySet()
+            .stream()
+            .filter(e -> allowlist.contains(e.getKey()))
+            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
 }
diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePluginTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePluginTests.java
new file mode 100644
index 0000000000000..519468ebe17ff
--- /dev/null
+++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePluginTests.java
@@ -0,0 +1,106 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.pipeline.common;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.env.TestEnvironment;
+import org.opensearch.plugins.SearchPipelinePlugin;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiFunction;
+
+public class SearchPipelineCommonModulePluginTests extends OpenSearchTestCase {
+
+    public void testRequestProcessorAllowlist() throws IOException {
+        final String key = SearchPipelineCommonModulePlugin.REQUEST_PROCESSORS_ALLOWLIST_SETTING.getKey();
+        runAllowlistTest(key, List.of(), SearchPipelineCommonModulePlugin::getRequestProcessors);
+        runAllowlistTest(key, List.of("filter_query"), SearchPipelineCommonModulePlugin::getRequestProcessors);
+        runAllowlistTest(key, List.of("script"), SearchPipelineCommonModulePlugin::getRequestProcessors);
+        runAllowlistTest(key, List.of("oversample", "script"), SearchPipelineCommonModulePlugin::getRequestProcessors);
+        runAllowlistTest(key, List.of("filter_query", "script", "oversample"), SearchPipelineCommonModulePlugin::getRequestProcessors);
+
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> runAllowlistTest(key, List.of("foo"), SearchPipelineCommonModulePlugin::getRequestProcessors)
+        );
+        assertTrue(e.getMessage(), e.getMessage().contains("foo"));
+    }
+
+    public void testResponseProcessorAllowlist() throws IOException {
+        final String key = SearchPipelineCommonModulePlugin.RESPONSE_PROCESSORS_ALLOWLIST_SETTING.getKey();
+        runAllowlistTest(key, List.of(), SearchPipelineCommonModulePlugin::getResponseProcessors);
+        runAllowlistTest(key, List.of("rename_field"), SearchPipelineCommonModulePlugin::getResponseProcessors);
+        runAllowlistTest(key, List.of("truncate_hits"), SearchPipelineCommonModulePlugin::getResponseProcessors);
+        runAllowlistTest(key, List.of("collapse", "truncate_hits"), SearchPipelineCommonModulePlugin::getResponseProcessors);
+        runAllowlistTest(
+            key,
+            List.of("rename_field", "truncate_hits", "collapse"),
+            SearchPipelineCommonModulePlugin::getResponseProcessors
+        );
+
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> runAllowlistTest(key, List.of("foo"), SearchPipelineCommonModulePlugin::getResponseProcessors)
+        );
+        assertTrue(e.getMessage(), e.getMessage().contains("foo"));
+    }
+
+    public void testSearchPhaseResultsProcessorAllowlist() throws IOException {
+        final String key = SearchPipelineCommonModulePlugin.SEARCH_PHASE_RESULTS_PROCESSORS_ALLOWLIST_SETTING.getKey();
+        runAllowlistTest(key, List.of(), SearchPipelineCommonModulePlugin::getSearchPhaseResultsProcessors);
+
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> runAllowlistTest(key, List.of("foo"), SearchPipelineCommonModulePlugin::getSearchPhaseResultsProcessors)
+        );
+        assertTrue(e.getMessage(), e.getMessage().contains("foo"));
+    }
+
+    private void runAllowlistTest(
+        String settingKey,
+        List<String> allowlist,
+        BiFunction<SearchPipelineCommonModulePlugin, SearchPipelinePlugin.Parameters, Map<String, ?>> function
+    ) throws IOException {
+        final Settings settings = Settings.builder().putList(settingKey, allowlist).build();
+        try (SearchPipelineCommonModulePlugin plugin = new SearchPipelineCommonModulePlugin()) {
+            assertEquals(Set.copyOf(allowlist), function.apply(plugin, createParameters(settings)).keySet());
+        }
+    }
+
+    public void testAllowlistNotSpecified() throws IOException {
+        final Settings settings = Settings.EMPTY;
+        try (SearchPipelineCommonModulePlugin plugin = new SearchPipelineCommonModulePlugin()) {
+            assertEquals(Set.of("oversample", "filter_query", "script"), plugin.getRequestProcessors(createParameters(settings)).keySet());
+            assertEquals(
+                Set.of("rename_field", "truncate_hits", "collapse"),
+                plugin.getResponseProcessors(createParameters(settings)).keySet()
+            );
+            assertEquals(Set.of(), plugin.getSearchPhaseResultsProcessors(createParameters(settings)).keySet());
+        }
+    }
+
+    private static SearchPipelinePlugin.Parameters createParameters(Settings settings) {
+        return new SearchPipelinePlugin.Parameters(
+            TestEnvironment.newEnvironment(Settings.builder().put(settings).put("path.home", "").build()),
+            null,
+            null,
+            null,
+            () -> 0L,
+            (a, b) -> null,
+            null,
+            null,
+            $ -> {},
+            null
+        );
+    }
+}

From 243e8db04edb1339dc308b877c3fd26bdc92acc9 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Thu, 27 Jun 2024 15:03:02 -0400
Subject: [PATCH 058/113] Bump Apache Lucene to 9.11.1 (#14576) (#14581)

(cherry picked from commit 0095fd1a44b583a7457b4d4578cdf6a0ed1fd2f3)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 buildSrc/version.properties                                     | 2 +-
 libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 | 1 +
 libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1 | 1 -
 libs/core/src/main/java/org/opensearch/Version.java             | 2 +-
 .../lucene-expressions-9.12.0-snapshot-847316d.jar.sha1         | 1 +
 .../lucene-expressions-9.12.0-snapshot-c896995.jar.sha1         | 1 -
 .../lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1        | 1 +
 .../lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1        | 1 -
 .../lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1   | 1 +
 .../lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1   | 1 -
 .../lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1       | 1 +
 .../lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1       | 1 -
 .../lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1   | 1 +
 .../lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1   | 1 -
 .../lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1    | 1 +
 .../lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1    | 1 -
 .../lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1    | 1 +
 .../lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1    | 1 -
 .../lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1 | 1 +
 .../lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1 | 1 -
 .../lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1     | 1 +
 .../lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1     | 1 -
 .../lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1     | 1 +
 .../lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1     | 1 -
 server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1    | 1 +
 server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1    | 1 -
 .../licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1   | 1 +
 .../licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1   | 1 -
 .../lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1         | 1 +
 .../lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1         | 1 -
 server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1    | 1 +
 server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1    | 1 -
 server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1  | 1 +
 server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1  | 1 -
 server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1    | 1 +
 server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1    | 1 -
 server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1 | 1 +
 server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1 | 1 -
 .../lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1         | 1 +
 .../lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1         | 1 -
 server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1 | 1 +
 server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1 | 1 -
 .../lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1      | 1 +
 .../lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1      | 1 -
 .../licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1  | 1 +
 .../licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1  | 1 -
 server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1 | 1 +
 server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1 | 1 -
 48 files changed, 25 insertions(+), 25 deletions(-)
 create mode 100644 libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1
 create mode 100644 server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1
 delete mode 100644 server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index e9aa32ea9a4f5..a99bd4801b7f3 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.12.0-snapshot-c896995
+lucene            = 9.12.0-snapshot-847316d
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9
diff --git a/libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 b/libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..e3fd1708ea428
--- /dev/null
+++ b/libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+51ff4940eb1024184bbaa5dae39695d2392c5bab
\ No newline at end of file
diff --git a/libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1 b/libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 299283562fddc..0000000000000
--- a/libs/core/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-826b328c37ea7f27c05d685db03bf8d2b00457ff
\ No newline at end of file
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index da43894863432..b647a92d6708a 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -106,7 +106,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_14_1 = new Version(2140199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_1 = new Version(2150199, org.apache.lucene.util.Version.LUCENE_9_10_0);
-    public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_0);
+    public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version CURRENT = V_3_0_0;
 
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..83dd8e657bdd5
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+b866103bbaca4141c152deca9252bd137026dafc
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 6d8d3be59f945..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9f0321cf2d34fca3f1f9334fdfee2b79d9d27444
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..80e254ed3d098
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+04436942995a4952ce5654126dfb767d6335674e
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 696803bf63b46..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e6314f36fb29e208d58c0470f14269c9c36996ba
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..3baed2a6e660b
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+85918e24fc3bf63fcd953807ab2eb3fa55c987c2
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 7a12077d7fc62..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-77fbf1e37af79715f28f66d8cc5b50af2982fc54
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..4e9327112d412
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+15e425e9cc0ab9d65fac3c919199a24dfa3631eb
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index efed62c7e5e5b..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a7a4e9c6004c72782e1002e1dcfaf4fbab7887d8
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..7e7e9fe5b22b4
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+3d16c18348e7d4a00cb83100c43f3e21239d224e
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index f2020abcb8ef7..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-42ac148a3769d6eb880d7f184d1917bad48ca303
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..98e0ecc9cbb89
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+2ef6d9dffc6816d3cd04a54fe1ee43e13f850a37
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index b64e4061311e5..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-adf2a25339ac8722647f8196288c1f5056bbf0de
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..ef675f2b9702e
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+e72b2262f5393d9ff255fb901297d4e6790e9102
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index f56e7fc5df766..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a689e3af2015b21b7b4f41a1206b50c44519b6f7
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..d8bbac27fd360
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+416ac44b2e76592c9e85338798cae93c3cf5475e
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 30732e3c4a688..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c875f7706ee81b1fb0b3443767a8c9c52f30abc5
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..f1249066d10f2
--- /dev/null
+++ b/server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+7e282aab7388efc911348f1eacd90e661580dda7
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 4b545e061c52f..0000000000000
--- a/server/licenses/lucene-analysis-common-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-73696492c6e59972974cd91e03ad9464e6b5bfcd
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..ac50c5e110a72
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+69e59ba4bed4c58836d2727d72b7f0095d2dcb92
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index ae4ffb2b1800b..0000000000000
--- a/server/licenses/lucene-backward-codecs-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3cbb29ecc873e8c880a6f32e739655551708dbcf
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..e3fd1708ea428
--- /dev/null
+++ b/server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+51ff4940eb1024184bbaa5dae39695d2392c5bab
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 299283562fddc..0000000000000
--- a/server/licenses/lucene-core-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-826b328c37ea7f27c05d685db03bf8d2b00457ff
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..cc5bf5bfd8ec0
--- /dev/null
+++ b/server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+5847a7d47f13ecb7f039fb9adf6f3b8e4bddde77
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index b0268c98167d3..0000000000000
--- a/server/licenses/lucene-grouping-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a3a7003dc83197523e830f058a3748dbea96cab7
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..eb14059d2cd8c
--- /dev/null
+++ b/server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+7cc0a26777a479f06fbcfae7abc23e784e1a00dc
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index d87927364b5a8..0000000000000
--- a/server/licenses/lucene-highlighter-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-00eb386915c3cffa9efcef2dc4c406f8a6776afe
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..b87170c39c78c
--- /dev/null
+++ b/server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+9cd99401c826d910da3c2beab8e42f1af8be6ea4
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 25a95546ab544..0000000000000
--- a/server/licenses/lucene-join-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bb1fc572da7d473bf39672fd8ac323b15a1ffff0
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..de591dd659cb5
--- /dev/null
+++ b/server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+cfee136ecbc3df7adc38b38e020dca5e61c22773
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index a0b3fd812561c..0000000000000
--- a/server/licenses/lucene-memory-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-05ebfcef0435f4870859a19c93020e24398bb939
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..1a999bb9c6686
--- /dev/null
+++ b/server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+afbc5adf93d4eb1a1b109ad828d1968bf16ef292
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 1e2cc97c37257..0000000000000
--- a/server/licenses/lucene-misc-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d5747ed1be242b59aa36b0c32b0d3bd26b1d8fb8
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..783a26551ae8c
--- /dev/null
+++ b/server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+16907c36f6adb8ba8f260e05738c66afb37c72d3
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 31d4fe2886fc1..0000000000000
--- a/server/licenses/lucene-queries-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fb6678d7fe035e55c545450682b67be49457ef1b
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..b3e9e4de96174
--- /dev/null
+++ b/server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+72baa9bddcf2efb71ffb695f1e9f548699ec13a0
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 754e4ea20765f..0000000000000
--- a/server/licenses/lucene-queryparser-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a11d7f56a9e78dc8e61f85b9b54ad94d73583bb3
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..2aefa435b1e9a
--- /dev/null
+++ b/server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+dd3c63066f583d90b563ebaa6fbe61c603403acb
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 08c2bc48ae85b..0000000000000
--- a/server/licenses/lucene-sandbox-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-75352855bcc052abfba821f878a27fd2b328fb1c
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..d27112c6db6ab
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+69b99530e0b05251c12863bee6a9325cafd5fdaa
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 5e0b7196f48c2..0000000000000
--- a/server/licenses/lucene-spatial-extras-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-299be103216d67ca092bef177642b275224e77a6
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..29423ac0ababd
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+a67d193b4b08790169db7cf005a2429991260287
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index c79b34adea5e2..0000000000000
--- a/server/licenses/lucene-spatial3d-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-29b4a76cd0bdabe0e067063831e661dedac6e503
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1
new file mode 100644
index 0000000000000..6ce1f639ccbb7
--- /dev/null
+++ b/server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1
@@ -0,0 +1 @@
+7a1625ae39071ccbfb3af11df5a74291758f4b47
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1 b/server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1
deleted file mode 100644
index 8d5334f0c4619..0000000000000
--- a/server/licenses/lucene-suggest-9.12.0-snapshot-c896995.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-597edb659e9ea93398a816e6837da7d47ef53873
\ No newline at end of file

From a34270d31df6e6ebd85f7e3b52513b80494f6bcd Mon Sep 17 00:00:00 2001
From: Shivansh Arora <hishiv@amazon.com>
Date: Fri, 28 Jun 2024 14:46:34 +0530
Subject: [PATCH 059/113] Add unittests for RemoteClusterStateAttributesManager
 (#14427)

* Add unittests for RemoteClusterStateAttributesManager

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
---
 .../model/RemoteClusterStateCustoms.java      |   2 +-
 ...oteClusterStateAttributesManagerTests.java | 276 +++++++++++++++---
 2 files changed, 231 insertions(+), 47 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java
index f384908bc6b65..affbc7ba66cb8 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java
@@ -34,12 +34,12 @@
  */
 public class RemoteClusterStateCustoms extends AbstractRemoteWritableBlobEntity<Custom> {
     public static final String CLUSTER_STATE_CUSTOM = "cluster-state-custom";
+    public final ChecksumWritableBlobStoreFormat<ClusterState.Custom> clusterStateCustomsFormat;
 
     private long stateVersion;
     private final String customType;
     private ClusterState.Custom custom;
     private final NamedWriteableRegistry namedWriteableRegistry;
-    private final ChecksumWritableBlobStoreFormat<ClusterState.Custom> clusterStateCustomsFormat;
 
     public RemoteClusterStateCustoms(
         final ClusterState.Custom custom,
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
index 41e1546ead164..fe9ed57fa77b8 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
@@ -17,9 +17,10 @@
 import org.opensearch.cluster.DiffableUtils;
 import org.opensearch.cluster.block.ClusterBlocks;
 import org.opensearch.cluster.node.DiscoveryNodes;
-import org.opensearch.common.CheckedRunnable;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.TestCapturingListener;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -28,6 +29,7 @@
 import org.opensearch.core.compress.NoneCompressor;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.gateway.remote.model.RemoteClusterBlocks;
+import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms;
 import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes;
 import org.opensearch.gateway.remote.model.RemoteReadResult;
 import org.opensearch.index.translog.transfer.BlobStoreTransferService;
@@ -36,46 +38,63 @@
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicReference;
 
-import static java.util.Collections.emptyList;
+import static org.opensearch.common.blobstore.stream.write.WritePriority.URGENT;
+import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE;
+import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION;
 import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.encodeString;
 import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS;
 import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS_FORMAT;
 import static org.opensearch.gateway.remote.model.RemoteClusterBlocksTests.randomClusterBlocks;
+import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM;
+import static org.opensearch.gateway.remote.model.RemoteClusterStateCustomsTests.getClusterStateCustom;
 import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES_FORMAT;
 import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodesTests.getDiscoveryNodes;
+import static org.opensearch.index.remote.RemoteStoreUtils.invertLong;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyIterable;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class RemoteClusterStateAttributesManagerTests extends OpenSearchTestCase {
     private RemoteClusterStateAttributesManager remoteClusterStateAttributesManager;
     private BlobStoreTransferService blobStoreTransferService;
-    private BlobStoreRepository blobStoreRepository;
     private Compressor compressor;
-    private ThreadPool threadPool = new TestThreadPool(RemoteClusterStateAttributesManagerTests.class.getName());
+    private final ThreadPool threadPool = new TestThreadPool(RemoteClusterStateAttributesManagerTests.class.getName());
+    private final long VERSION = 7331L;
+    private NamedWriteableRegistry namedWriteableRegistry;
+    private final String CLUSTER_NAME = "test-cluster";
+    private final String CLUSTER_UUID = "test-cluster-uuid";
 
     @Before
     public void setup() throws Exception {
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(emptyList());
-        blobStoreRepository = mock(BlobStoreRepository.class);
+        namedWriteableRegistry = writableRegistry();
+        BlobStoreRepository blobStoreRepository = mock(BlobStoreRepository.class);
+        when(blobStoreRepository.basePath()).thenReturn(new BlobPath());
         blobStoreTransferService = mock(BlobStoreTransferService.class);
         compressor = new NoneCompressor();
 
         remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(
-            "test-cluster",
+            CLUSTER_NAME,
             blobStoreRepository,
             blobStoreTransferService,
             writableRegistry(),
@@ -89,7 +108,40 @@ public void tearDown() throws Exception {
         threadPool.shutdown();
     }
 
-    public void testGetAsyncMetadataReadAction_DiscoveryNodes() throws IOException {
+    public void testGetAsyncMetadataWriteAction_DiscoveryNodes() throws IOException, InterruptedException {
+        DiscoveryNodes discoveryNodes = getDiscoveryNodes();
+        RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(discoveryNodes, VERSION, CLUSTER_UUID, compressor);
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        final CountDownLatch latch = new CountDownLatch(1);
+        final TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        remoteClusterStateAttributesManager.getAsyncMetadataWriteAction(
+            DISCOVERY_NODES,
+            remoteDiscoveryNodes,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(DISCOVERY_NODES, listener.getResult().getComponent());
+        String uploadedFileName = listener.getResult().getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(RemoteClusterStateUtils.encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(DISCOVERY_NODES, splitFileName[0]);
+        assertEquals(invertLong(VERSION), splitFileName[1]);
+        assertEquals(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
+
+    public void testGetAsyncMetadataReadAction_DiscoveryNodes() throws IOException, InterruptedException {
         DiscoveryNodes discoveryNodes = getDiscoveryNodes();
         String fileName = randomAlphaOfLength(10);
         when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
@@ -97,29 +149,57 @@ public void testGetAsyncMetadataReadAction_DiscoveryNodes() throws IOException {
         );
         RemoteDiscoveryNodes remoteObjForDownload = new RemoteDiscoveryNodes(fileName, "cluster-uuid", compressor);
         CountDownLatch latch = new CountDownLatch(1);
-        AtomicReference<DiscoveryNodes> readDiscoveryNodes = new AtomicReference<>();
-        LatchedActionListener<RemoteReadResult> assertingListener = new LatchedActionListener<>(
-            ActionListener.wrap(response -> readDiscoveryNodes.set((DiscoveryNodes) response.getObj()), Assert::assertNull),
-            latch
-        );
-        CheckedRunnable<IOException> runnable = remoteClusterStateAttributesManager.getAsyncMetadataReadAction(
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        remoteClusterStateAttributesManager.getAsyncMetadataReadAction(
             DISCOVERY_NODES,
             remoteObjForDownload,
-            assertingListener
-        );
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(CLUSTER_STATE_ATTRIBUTE, listener.getResult().getComponent());
+        assertEquals(DISCOVERY_NODES, listener.getResult().getComponentName());
+        DiscoveryNodes readDiscoveryNodes = (DiscoveryNodes) listener.getResult().getObj();
+        assertEquals(discoveryNodes.getSize(), readDiscoveryNodes.getSize());
+        discoveryNodes.getNodes().forEach((nodeId, node) -> assertEquals(readDiscoveryNodes.get(nodeId), node));
+        assertEquals(discoveryNodes.getClusterManagerNodeId(), readDiscoveryNodes.getClusterManagerNodeId());
+    }
 
-        try {
-            runnable.run();
-            latch.await();
-            assertEquals(discoveryNodes.getSize(), readDiscoveryNodes.get().getSize());
-            discoveryNodes.getNodes().forEach((nodeId, node) -> assertEquals(readDiscoveryNodes.get().get(nodeId), node));
-            assertEquals(discoveryNodes.getClusterManagerNodeId(), readDiscoveryNodes.get().getClusterManagerNodeId());
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
+    public void testGetAsyncMetadataWriteAction_ClusterBlocks() throws IOException, InterruptedException {
+        ClusterBlocks clusterBlocks = randomClusterBlocks();
+        RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(clusterBlocks, VERSION, CLUSTER_UUID, compressor);
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        final CountDownLatch latch = new CountDownLatch(1);
+        final TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        remoteClusterStateAttributesManager.getAsyncMetadataWriteAction(
+            CLUSTER_BLOCKS,
+            remoteClusterBlocks,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(CLUSTER_BLOCKS, listener.getResult().getComponent());
+        String uploadedFileName = listener.getResult().getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(CLUSTER_BLOCKS, splitFileName[0]);
+        assertEquals(invertLong(VERSION), splitFileName[1]);
+        assertEquals(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
     }
 
-    public void testGetAsyncMetadataReadAction_ClusterBlocks() throws IOException {
+    public void testGetAsyncMetadataReadAction_ClusterBlocks() throws IOException, InterruptedException {
         ClusterBlocks clusterBlocks = randomClusterBlocks();
         String fileName = randomAlphaOfLength(10);
         when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
@@ -127,29 +207,133 @@ public void testGetAsyncMetadataReadAction_ClusterBlocks() throws IOException {
         );
         RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(fileName, "cluster-uuid", compressor);
         CountDownLatch latch = new CountDownLatch(1);
-        AtomicReference<ClusterBlocks> readClusterBlocks = new AtomicReference<>();
-        LatchedActionListener<RemoteReadResult> assertingListener = new LatchedActionListener<>(
-            ActionListener.wrap(response -> readClusterBlocks.set((ClusterBlocks) response.getObj()), Assert::assertNull),
-            latch
-        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
 
-        CheckedRunnable<IOException> runnable = remoteClusterStateAttributesManager.getAsyncMetadataReadAction(
+        remoteClusterStateAttributesManager.getAsyncMetadataReadAction(
             CLUSTER_BLOCKS,
             remoteClusterBlocks,
-            assertingListener
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(CLUSTER_STATE_ATTRIBUTE, listener.getResult().getComponent());
+        assertEquals(CLUSTER_BLOCKS, listener.getResult().getComponentName());
+        ClusterBlocks readClusterBlocks = (ClusterBlocks) listener.getResult().getObj();
+        assertEquals(clusterBlocks.global(), readClusterBlocks.global());
+        assertEquals(clusterBlocks.indices().keySet(), readClusterBlocks.indices().keySet());
+        for (String index : clusterBlocks.indices().keySet()) {
+            assertEquals(clusterBlocks.indices().get(index), readClusterBlocks.indices().get(index));
+        }
+    }
+
+    public void testGetAsyncMetadataWriteAction_Custom() throws IOException, InterruptedException {
+        Custom custom = getClusterStateCustom();
+        RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(
+            custom,
+            custom.getWriteableName(),
+            VERSION,
+            CLUSTER_UUID,
+            compressor,
+            namedWriteableRegistry
         );
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+        final TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        final CountDownLatch latch = new CountDownLatch(1);
+        remoteClusterStateAttributesManager.getAsyncMetadataWriteAction(
+            CLUSTER_STATE_CUSTOM,
+            remoteClusterStateCustoms,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, custom.getWriteableName()), listener.getResult().getComponent());
+        String uploadedFileName = listener.getResult().getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(5, pathTokens.length);
+        assertEquals(encodeString(CLUSTER_NAME), pathTokens[0]);
+        assertEquals(CLUSTER_STATE_PATH_TOKEN, pathTokens[1]);
+        assertEquals(CLUSTER_UUID, pathTokens[2]);
+        assertEquals(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN, pathTokens[3]);
+        String[] splitFileName = pathTokens[4].split(DELIMITER);
+        assertEquals(4, splitFileName.length);
+        assertEquals(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, custom.getWriteableName()), splitFileName[0]);
+        assertEquals(invertLong(VERSION), splitFileName[1]);
+        assertEquals(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION, Integer.parseInt(splitFileName[3]));
+    }
 
-        try {
-            runnable.run();
-            latch.await();
-            assertEquals(clusterBlocks.global(), readClusterBlocks.get().global());
-            assertEquals(clusterBlocks.indices().keySet(), readClusterBlocks.get().indices().keySet());
-            for (String index : clusterBlocks.indices().keySet()) {
-                assertEquals(clusterBlocks.indices().get(index), readClusterBlocks.get().indices().get(index));
-            }
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
+    public void testGetAsyncMetadataReadAction_Custom() throws IOException, InterruptedException {
+        Custom custom = getClusterStateCustom();
+        String fileName = randomAlphaOfLength(10);
+        RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(
+            fileName,
+            custom.getWriteableName(),
+            CLUSTER_UUID,
+            compressor,
+            namedWriteableRegistry
+        );
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            remoteClusterStateCustoms.clusterStateCustomsFormat.serialize(custom, fileName, compressor).streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> capturingListener = new TestCapturingListener<>();
+        final CountDownLatch latch = new CountDownLatch(1);
+        remoteClusterStateAttributesManager.getAsyncMetadataReadAction(
+            CLUSTER_STATE_CUSTOM,
+            remoteClusterStateCustoms,
+            new LatchedActionListener<>(capturingListener, latch)
+        ).run();
+        latch.await();
+        assertNull(capturingListener.getFailure());
+        assertNotNull(capturingListener.getResult());
+        assertEquals(custom, capturingListener.getResult().getObj());
+        assertEquals(CLUSTER_STATE_ATTRIBUTE, capturingListener.getResult().getComponent());
+        assertEquals(CLUSTER_STATE_CUSTOM, capturingListener.getResult().getComponentName());
+    }
+
+    public void testGetAsyncMetadataWriteAction_Exception() throws IOException, InterruptedException {
+        DiscoveryNodes discoveryNodes = getDiscoveryNodes();
+        RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(discoveryNodes, VERSION, CLUSTER_UUID, compressor);
+
+        IOException ioException = new IOException("mock test exception");
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onFailure(ioException);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), anyIterable(), anyString(), eq(URGENT), any(ActionListener.class));
+
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> capturingListener = new TestCapturingListener<>();
+        final CountDownLatch latch = new CountDownLatch(1);
+        remoteClusterStateAttributesManager.getAsyncMetadataWriteAction(
+            DISCOVERY_NODES,
+            remoteDiscoveryNodes,
+            new LatchedActionListener<>(capturingListener, latch)
+        ).run();
+        latch.await();
+        assertNull(capturingListener.getResult());
+        assertTrue(capturingListener.getFailure() instanceof RemoteStateTransferException);
+        assertEquals(ioException, capturingListener.getFailure().getCause());
+    }
+
+    public void testGetAsyncMetadataReadAction_Exception() throws IOException, InterruptedException {
+        String fileName = randomAlphaOfLength(10);
+        RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(fileName, CLUSTER_UUID, compressor);
+        Exception ioException = new IOException("mock test exception");
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenThrow(ioException);
+        CountDownLatch latch = new CountDownLatch(1);
+        TestCapturingListener<RemoteReadResult> capturingListener = new TestCapturingListener<>();
+        remoteClusterStateAttributesManager.getAsyncMetadataReadAction(
+            DISCOVERY_NODES,
+            remoteDiscoveryNodes,
+            new LatchedActionListener<>(capturingListener, latch)
+        ).run();
+        latch.await();
+        assertNull(capturingListener.getResult());
+        assertEquals(ioException, capturingListener.getFailure());
     }
 
     public void testGetUpdatedCustoms() {

From 8ad199dac020c91c80180b4f8042f65c9994a81a Mon Sep 17 00:00:00 2001
From: Ashish Singh <ssashish@amazon.com>
Date: Fri, 28 Jun 2024 15:30:28 +0530
Subject: [PATCH 060/113] Add Ashish Singh to codeowners (#14592)

Signed-off-by: Ashish Singh <ssashish@amazon.com>
---
 .github/CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 8d69e98220b69..8ceecb3abb4a2 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -11,7 +11,7 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+* @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
 /modules/transport-netty4/ @peternied
 

From 8e493f313eba21ce9946f2604c214bb66840d7f3 Mon Sep 17 00:00:00 2001
From: Liyun Xiu <xiliyun@amazon.com>
Date: Fri, 28 Jun 2024 12:02:49 -0700
Subject: [PATCH 061/113] Add batching processor base type
 AbstractBatchingProcessor (#14554)

Signed-off-by: Liyun Xiu <xiliyun@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../ingest/AbstractBatchingProcessor.java     | 136 +++++++++++++++
 .../AbstractBatchingProcessorTests.java       | 160 ++++++++++++++++++
 3 files changed, 297 insertions(+)
 create mode 100644 server/src/main/java/org/opensearch/ingest/AbstractBatchingProcessor.java
 create mode 100644 server/src/test/java/org/opensearch/ingest/AbstractBatchingProcessorTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c6b2d815750f9..8835032785430 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
+- Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
 - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
 
diff --git a/server/src/main/java/org/opensearch/ingest/AbstractBatchingProcessor.java b/server/src/main/java/org/opensearch/ingest/AbstractBatchingProcessor.java
new file mode 100644
index 0000000000000..55413b9bbdad1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ingest/AbstractBatchingProcessor.java
@@ -0,0 +1,136 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ingest;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+
+import static org.opensearch.ingest.ConfigurationUtils.newConfigurationException;
+
+/**
+ * Abstract base class for batch processors.
+ *
+ * @opensearch.internal
+ */
+public abstract class AbstractBatchingProcessor extends AbstractProcessor {
+
+    public static final String BATCH_SIZE_FIELD = "batch_size";
+    private static final int DEFAULT_BATCH_SIZE = 1;
+    protected final int batchSize;
+
+    protected AbstractBatchingProcessor(String tag, String description, int batchSize) {
+        super(tag, description);
+        this.batchSize = batchSize;
+    }
+
+    /**
+     * Internal logic to process batched documents, must be implemented by concrete batch processors.
+     *
+     * @param ingestDocumentWrappers {@link List} of {@link IngestDocumentWrapper} to be processed.
+     * @param handler                {@link Consumer} to be called with the results of the processing.
+     */
+    protected abstract void subBatchExecute(
+        List<IngestDocumentWrapper> ingestDocumentWrappers,
+        Consumer<List<IngestDocumentWrapper>> handler
+    );
+
+    @Override
+    public void batchExecute(List<IngestDocumentWrapper> ingestDocumentWrappers, Consumer<List<IngestDocumentWrapper>> handler) {
+        if (ingestDocumentWrappers.isEmpty()) {
+            handler.accept(Collections.emptyList());
+            return;
+        }
+
+        // if batch size is larger than document size, send one batch
+        if (this.batchSize >= ingestDocumentWrappers.size()) {
+            subBatchExecute(ingestDocumentWrappers, handler);
+            return;
+        }
+
+        // split documents into multiple batches and send each batch to batch processors
+        List<List<IngestDocumentWrapper>> batches = cutBatches(ingestDocumentWrappers);
+        int size = ingestDocumentWrappers.size();
+        AtomicInteger counter = new AtomicInteger(size);
+        List<IngestDocumentWrapper> allResults = Collections.synchronizedList(new ArrayList<>());
+        for (List<IngestDocumentWrapper> batch : batches) {
+            this.subBatchExecute(batch, batchResults -> {
+                allResults.addAll(batchResults);
+                if (counter.addAndGet(-batchResults.size()) == 0) {
+                    handler.accept(allResults);
+                }
+                assert counter.get() >= 0 : "counter is negative";
+            });
+        }
+    }
+
+    private List<List<IngestDocumentWrapper>> cutBatches(List<IngestDocumentWrapper> ingestDocumentWrappers) {
+        List<List<IngestDocumentWrapper>> batches = new ArrayList<>();
+        for (int i = 0; i < ingestDocumentWrappers.size(); i += this.batchSize) {
+            batches.add(ingestDocumentWrappers.subList(i, Math.min(i + this.batchSize, ingestDocumentWrappers.size())));
+        }
+        return batches;
+    }
+
+    /**
+     * Factory class for creating {@link AbstractBatchingProcessor} instances.
+     *
+     * @opensearch.internal
+     */
+    public abstract static class Factory implements Processor.Factory {
+        final String processorType;
+
+        protected Factory(String processorType) {
+            this.processorType = processorType;
+        }
+
+        /**
+         * Creates a new processor instance.
+         *
+         * @param processorFactories The processor factories.
+         * @param tag                 The processor tag.
+         * @param description         The processor description.
+         * @param config              The processor configuration.
+         * @return The new AbstractBatchProcessor instance.
+         * @throws Exception If the processor could not be created.
+         */
+        @Override
+        public AbstractBatchingProcessor create(
+            Map<String, Processor.Factory> processorFactories,
+            String tag,
+            String description,
+            Map<String, Object> config
+        ) throws Exception {
+            int batchSize = ConfigurationUtils.readIntProperty(this.processorType, tag, config, BATCH_SIZE_FIELD, DEFAULT_BATCH_SIZE);
+            if (batchSize < 1) {
+                throw newConfigurationException(this.processorType, tag, BATCH_SIZE_FIELD, "batch size must be a positive integer");
+            }
+            return newProcessor(tag, description, batchSize, config);
+        }
+
+        /**
+         * Returns a new processor instance.
+         *
+         * @param tag tag of the processor
+         * @param description description of the processor
+         * @param batchSize batch size of the processor
+         * @param config configuration of the processor
+         * @return a new batch processor instance
+         */
+        protected abstract AbstractBatchingProcessor newProcessor(
+            String tag,
+            String description,
+            int batchSize,
+            Map<String, Object> config
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ingest/AbstractBatchingProcessorTests.java b/server/src/test/java/org/opensearch/ingest/AbstractBatchingProcessorTests.java
new file mode 100644
index 0000000000000..54fc30cb5befa
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ingest/AbstractBatchingProcessorTests.java
@@ -0,0 +1,160 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ingest;
+
+import org.opensearch.OpenSearchParseException;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+public class AbstractBatchingProcessorTests extends OpenSearchTestCase {
+
+    public void testBatchExecute_emptyInput() {
+        DummyProcessor processor = new DummyProcessor(3);
+        Consumer<List<IngestDocumentWrapper>> handler = (results) -> assertTrue(results.isEmpty());
+        processor.batchExecute(Collections.emptyList(), handler);
+        assertTrue(processor.getSubBatches().isEmpty());
+    }
+
+    public void testBatchExecute_singleBatchSize() {
+        DummyProcessor processor = new DummyProcessor(3);
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3)
+        );
+        List<IngestDocumentWrapper> resultList = new ArrayList<>();
+        processor.batchExecute(wrapperList, resultList::addAll);
+        assertEquals(wrapperList, resultList);
+        assertEquals(1, processor.getSubBatches().size());
+        assertEquals(wrapperList, processor.getSubBatches().get(0));
+    }
+
+    public void testBatchExecute_multipleBatches() {
+        DummyProcessor processor = new DummyProcessor(2);
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3),
+            IngestDocumentPreparer.createIngestDocumentWrapper(4),
+            IngestDocumentPreparer.createIngestDocumentWrapper(5)
+        );
+        List<IngestDocumentWrapper> resultList = new ArrayList<>();
+        processor.batchExecute(wrapperList, resultList::addAll);
+        assertEquals(wrapperList, resultList);
+        assertEquals(3, processor.getSubBatches().size());
+        assertEquals(wrapperList.subList(0, 2), processor.getSubBatches().get(0));
+        assertEquals(wrapperList.subList(2, 4), processor.getSubBatches().get(1));
+        assertEquals(wrapperList.subList(4, 5), processor.getSubBatches().get(2));
+    }
+
+    public void testBatchExecute_randomBatches() {
+        int batchSize = randomIntBetween(2, 32);
+        int docCount = randomIntBetween(2, 32);
+        DummyProcessor processor = new DummyProcessor(batchSize);
+        List<IngestDocumentWrapper> wrapperList = new ArrayList<>();
+        for (int i = 0; i < docCount; ++i) {
+            wrapperList.add(IngestDocumentPreparer.createIngestDocumentWrapper(i));
+        }
+        List<IngestDocumentWrapper> resultList = new ArrayList<>();
+        processor.batchExecute(wrapperList, resultList::addAll);
+        assertEquals(wrapperList, resultList);
+        assertEquals(docCount / batchSize + (docCount % batchSize == 0 ? 0 : 1), processor.getSubBatches().size());
+    }
+
+    public void testBatchExecute_defaultBatchSize() {
+        DummyProcessor processor = new DummyProcessor(1);
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3)
+        );
+        List<IngestDocumentWrapper> resultList = new ArrayList<>();
+        processor.batchExecute(wrapperList, resultList::addAll);
+        assertEquals(wrapperList, resultList);
+        assertEquals(3, processor.getSubBatches().size());
+        assertEquals(wrapperList.subList(0, 1), processor.getSubBatches().get(0));
+        assertEquals(wrapperList.subList(1, 2), processor.getSubBatches().get(1));
+        assertEquals(wrapperList.subList(2, 3), processor.getSubBatches().get(2));
+    }
+
+    public void testFactory_invalidBatchSize() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("batch_size", 0);
+        DummyProcessor.DummyProcessorFactory factory = new DummyProcessor.DummyProcessorFactory("DummyProcessor");
+        OpenSearchParseException exception = assertThrows(OpenSearchParseException.class, () -> factory.create(config));
+        assertEquals("[batch_size] batch size must be a positive integer", exception.getMessage());
+    }
+
+    public void testFactory_defaultBatchSize() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        DummyProcessor.DummyProcessorFactory factory = new DummyProcessor.DummyProcessorFactory("DummyProcessor");
+        DummyProcessor processor = (DummyProcessor) factory.create(config);
+        assertEquals(1, processor.batchSize);
+    }
+
+    public void testFactory_callNewProcessor() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("batch_size", 3);
+        DummyProcessor.DummyProcessorFactory factory = new DummyProcessor.DummyProcessorFactory("DummyProcessor");
+        DummyProcessor processor = (DummyProcessor) factory.create(config);
+        assertEquals(3, processor.batchSize);
+    }
+
+    static class DummyProcessor extends AbstractBatchingProcessor {
+        private List<List<IngestDocumentWrapper>> subBatches = new ArrayList<>();
+
+        public List<List<IngestDocumentWrapper>> getSubBatches() {
+            return subBatches;
+        }
+
+        protected DummyProcessor(int batchSize) {
+            super("tag", "description", batchSize);
+        }
+
+        @Override
+        public void subBatchExecute(List<IngestDocumentWrapper> ingestDocumentWrappers, Consumer<List<IngestDocumentWrapper>> handler) {
+            subBatches.add(ingestDocumentWrappers);
+            handler.accept(ingestDocumentWrappers);
+        }
+
+        @Override
+        public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
+            return ingestDocument;
+        }
+
+        @Override
+        public String getType() {
+            return null;
+        }
+
+        public static class DummyProcessorFactory extends Factory {
+
+            protected DummyProcessorFactory(String processorType) {
+                super(processorType);
+            }
+
+            public AbstractBatchingProcessor create(Map<String, Object> config) throws Exception {
+                final Map<String, org.opensearch.ingest.Processor.Factory> processorFactories = new HashMap<>();
+                return super.create(processorFactories, "tag", "description", config);
+            }
+
+            @Override
+            protected AbstractBatchingProcessor newProcessor(String tag, String description, int batchSize, Map<String, Object> config) {
+                return new DummyProcessor(batchSize);
+            }
+        }
+    }
+}

From 5c8623f15f1fbec40328f05f53814404e3438ff7 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Fri, 28 Jun 2024 17:01:43 -0400
Subject: [PATCH 062/113] Add @InternalApi annotation to japicmp exclusions
 (#14597)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md        | 1 +
 server/build.gradle | 1 +
 2 files changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8835032785430..c7cfbba928da9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568))
 - Make the class CommunityIdProcessor final ([#14448](https://github.com/opensearch-project/OpenSearch/pull/14448))
 - Allow @InternalApi annotation on classes not meant to be constructed outside of the OpenSearch core ([#14575](https://github.com/opensearch-project/OpenSearch/pull/14575))
+- Add @InternalApi annotation to japicmp exclusions ([#14597](https://github.com/opensearch-project/OpenSearch/pull/14597))
 
 ### Deprecated
 
diff --git a/server/build.gradle b/server/build.gradle
index b8a99facbf964..429af5d0ac258 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -409,6 +409,7 @@ tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) {
     failOnModification = true
     ignoreMissingClasses = true
     annotationIncludes = ['@org.opensearch.common.annotation.PublicApi', '@org.opensearch.common.annotation.DeprecatedApi']
+    annotationExcludes = ['@org.opensearch.common.annotation.InternalApi']
     txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt")
     htmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.html")
     dependsOn downloadJapicmpCompareTarget

From c71fd4a2f2e6d5d7d9f2f304c573180027af8f44 Mon Sep 17 00:00:00 2001
From: Vatsal <36672090+imvtsl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 22:16:47 -0700
Subject: [PATCH 063/113] =?UTF-8?q?Fix=20issue=2014519:Parsing=20a=20GetRe?=
 =?UTF-8?q?sult=20returns=20NPE=20if=20found=20field=20is=20mis=E2=80=A6?=
 =?UTF-8?q?=20(#14552)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Fix issue 14519:Parsing a GetResult returns NPE if found field is missing. Signed-off-by: Vatsal <vatsal.v.anand@gmail.com>

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>

* Fix issue 14519:Parsing a GetResult returns NPE if found field is missing. Signed-off-by: Vatsal <vatsal.v.anand@gmail.com>

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>

* Fix issue 14519:Fix wildcart import. Signed-off-by: Vatsal <vatsal.v.anand@gmail.com>

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>

* Fix issue 14519:Fix wildcart import. Signed-off-by: Vatsal <vatsal.v.anand@gmail.com>

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>

* Fix issue 14519:Fix spotless issues. Signed-off-by: Vatsal <vatsal.v.anand@gmail.com>

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>

* Fix issue 14519:update changelog

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>

---------

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>
Signed-off-by: Daniel Widdis <widdis@gmail.com>
Co-authored-by: Daniel Widdis <widdis@gmail.com>
---
 CHANGELOG.md                                  |  1 +
 .../org/opensearch/index/get/GetResult.java   | 10 ++++++++++
 .../opensearch/index/get/GetResultTests.java  | 20 +++++++++++++++++++
 3 files changed, 31 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c7cfbba928da9..e9470d9bb4727 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -52,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add ListPitInfo::getKeepAlive() getter ([#14495](https://github.com/opensearch-project/OpenSearch/pull/14495))
 - Fix FuzzyQuery in keyword field will use IndexOrDocValuesQuery when both of index and doc_value are true ([#14378](https://github.com/opensearch-project/OpenSearch/pull/14378))
 - Fix file cache initialization ([#14004](https://github.com/opensearch-project/OpenSearch/pull/14004))
+- Handle NPE in GetResult if "found" field is missing ([#14552](https://github.com/opensearch-project/OpenSearch/pull/14552))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/index/get/GetResult.java b/server/src/main/java/org/opensearch/index/get/GetResult.java
index c0dd1cd2ecb30..27a2826f71e19 100644
--- a/server/src/main/java/org/opensearch/index/get/GetResult.java
+++ b/server/src/main/java/org/opensearch/index/get/GetResult.java
@@ -37,6 +37,7 @@
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.document.DocumentField;
 import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.core.common.ParsingException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -56,6 +57,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
@@ -398,6 +400,14 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
                 }
             }
         }
+
+        if (found == null) {
+            throw new ParsingException(
+                parser.getTokenLocation(),
+                String.format(Locale.ROOT, "Missing required field [%s]", GetResult.FOUND)
+            );
+        }
+
         return new GetResult(index, id, seqNo, primaryTerm, version, found, source, documentFields, metaFields);
     }
 
diff --git a/server/src/test/java/org/opensearch/index/get/GetResultTests.java b/server/src/test/java/org/opensearch/index/get/GetResultTests.java
index 64b14744a40d2..2001bb84454cd 100644
--- a/server/src/test/java/org/opensearch/index/get/GetResultTests.java
+++ b/server/src/test/java/org/opensearch/index/get/GetResultTests.java
@@ -35,12 +35,16 @@
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.document.DocumentField;
 import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.common.ParsingException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.mapper.IdFieldMapper;
@@ -220,6 +224,22 @@ public void testEqualsAndHashcode() {
         );
     }
 
+    public void testFomXContentEmbeddedFoundParsingException() throws IOException {
+        String json = "{\"_index\":\"foo\",\"_id\":\"bar\"}";
+        try (
+            XContentParser parser = JsonXContent.jsonXContent.createParser(
+                NamedXContentRegistry.EMPTY,
+                LoggingDeprecationHandler.INSTANCE,
+                json
+            )
+        ) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+            ParsingException parsingException = assertThrows(ParsingException.class, () -> GetResult.fromXContentEmbedded(parser));
+            assertEquals("Missing required field [found]", parsingException.getMessage());
+        }
+
+    }
+
     public static GetResult copyGetResult(GetResult getResult) {
         return new GetResult(
             getResult.getIndex(),

From 6267e94b00d62c3fba80abf02ea584c85ea3aad0 Mon Sep 17 00:00:00 2001
From: Bharathwaj G <bharath78910@gmail.com>
Date: Mon, 1 Jul 2024 14:38:03 +0530
Subject: [PATCH 064/113]  Star tree mapping changes (#14605)

* Star tree mapping changes with feature flag
---------
Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
---
 distribution/src/config/opensearch.yml        |   4 +
 .../index/mapper/StarTreeMapperIT.java        | 440 ++++++++++
 .../metadata/MetadataCreateIndexService.java  |   5 +
 .../metadata/MetadataMappingService.java      |  12 +-
 .../common/settings/ClusterSettings.java      |   6 +-
 .../common/settings/FeatureFlagSettings.java  |   3 +-
 .../common/settings/IndexScopedSettings.java  |  10 +
 .../opensearch/common/util/FeatureFlags.java  |  10 +-
 .../org/opensearch/index/IndexModule.java     |  10 +-
 .../org/opensearch/index/IndexService.java    |  11 +-
 .../CompositeIndexSettings.java               |  55 ++
 .../CompositeIndexValidator.java              |  46 ++
 .../datacube/DateDimension.java               |  72 ++
 .../compositeindex/datacube/Dimension.java    |  22 +
 .../datacube/DimensionFactory.java            |  99 +++
 .../index/compositeindex/datacube/Metric.java |  65 ++
 .../compositeindex/datacube/MetricStat.java   |  44 +
 .../datacube/NumericDimension.java            |  57 ++
 .../compositeindex/datacube/package-info.java |  11 +
 .../datacube/startree/StarTreeField.java      |  94 +++
 .../startree/StarTreeFieldConfiguration.java  | 108 +++
 .../startree/StarTreeIndexSettings.java       | 116 +++
 .../datacube/startree/StarTreeValidator.java  |  94 +++
 .../datacube/startree/package-info.java       |  11 +
 .../index/compositeindex/package-info.java    |  13 +
 .../mapper/CompositeDataCubeFieldType.java    |  56 ++
 .../mapper/CompositeMappedFieldType.java      |  75 ++
 .../org/opensearch/index/mapper/Mapper.java   |   5 +
 .../index/mapper/MapperService.java           |  17 +
 .../opensearch/index/mapper/ObjectMapper.java | 106 ++-
 .../index/mapper/RootObjectMapper.java        |  15 +-
 .../index/mapper/StarTreeMapper.java          | 406 +++++++++
 .../org/opensearch/indices/IndicesModule.java |   2 +
 .../opensearch/indices/IndicesService.java    |  17 +-
 .../main/java/org/opensearch/node/Node.java   |   5 +-
 .../index/mapper/ObjectMapperTests.java       |  73 ++
 .../index/mapper/StarTreeMapperTests.java     | 767 ++++++++++++++++++
 .../index/mapper/MapperTestCase.java          |   2 +-
 .../aggregations/AggregatorTestCase.java      |   2 +
 39 files changed, 2951 insertions(+), 15 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeValidator.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java
 create mode 100644 server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java

diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
index 10bab9b3fce92..4115601f62ada 100644
--- a/distribution/src/config/opensearch.yml
+++ b/distribution/src/config/opensearch.yml
@@ -125,3 +125,7 @@ ${path.logs}
 # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting.
 #
 #opensearch.experimental.feature.pluggable.caching.enabled: false
+#
+# Gates the functionality of star tree index, which improves the performance of search aggregations.
+#
+#opensearch.experimental.feature.composite_index.star_tree.enabled: true
diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
new file mode 100644
index 0000000000000..8e5193b650868
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
@@ -0,0 +1,440 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.Rounding;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
+import org.opensearch.index.compositeindex.datacube.DateDimension;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+/**
+ * Integration tests for star tree mapper
+ */
+public class StarTreeMapperIT extends OpenSearchIntegTestCase {
+    private static final String TEST_INDEX = "test";
+
+    private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean keywordDim) {
+        try {
+            return jsonBuilder().startObject()
+                .startObject("composite")
+                .startObject("startree-1")
+                .field("type", "star_tree")
+                .startObject("config")
+                .startArray("ordered_dimensions")
+                .startObject()
+                .field("name", "timestamp")
+                .endObject()
+                .startObject()
+                .field("name", getDim(invalidDim, keywordDim))
+                .endObject()
+                .endArray()
+                .startArray("metrics")
+                .startObject()
+                .field("name", getDim(invalidMetric, false))
+                .endObject()
+                .endArray()
+                .endObject()
+                .endObject()
+                .endObject()
+                .startObject("properties")
+                .startObject("timestamp")
+                .field("type", "date")
+                .endObject()
+                .startObject("numeric_dv")
+                .field("type", "integer")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("numeric")
+                .field("type", "integer")
+                .field("doc_values", false)
+                .endObject()
+                .startObject("keyword_dv")
+                .field("type", "keyword")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("keyword")
+                .field("type", "keyword")
+                .field("doc_values", false)
+                .endObject()
+                .endObject()
+                .endObject();
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    private static XContentBuilder createMaxDimTestMapping() {
+        try {
+            return jsonBuilder().startObject()
+                .startObject("composite")
+                .startObject("startree-1")
+                .field("type", "star_tree")
+                .startObject("config")
+                .startArray("ordered_dimensions")
+                .startObject()
+                .field("name", "timestamp")
+                .startArray("calendar_intervals")
+                .value("day")
+                .value("month")
+                .endArray()
+                .endObject()
+                .startObject()
+                .field("name", "dim2")
+                .endObject()
+                .startObject()
+                .field("name", "dim3")
+                .endObject()
+                .endArray()
+                .startArray("metrics")
+                .startObject()
+                .field("name", "dim2")
+                .endObject()
+                .endArray()
+                .endObject()
+                .endObject()
+                .endObject()
+                .startObject("properties")
+                .startObject("timestamp")
+                .field("type", "date")
+                .endObject()
+                .startObject("dim2")
+                .field("type", "integer")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("dim3")
+                .field("type", "integer")
+                .field("doc_values", true)
+                .endObject()
+                .endObject()
+                .endObject();
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) {
+        try {
+            return jsonBuilder().startObject()
+                .startObject("properties")
+                .startObject("timestamp")
+                .field("type", "date")
+                .endObject()
+                .startObject("numeric_dv")
+                .field("type", "integer")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("numeric")
+                .field("type", "integer")
+                .field("doc_values", false)
+                .endObject()
+                .startObject("keyword_dv")
+                .field("type", "keyword")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("keyword")
+                .field("type", "keyword")
+                .field("doc_values", false)
+                .endObject()
+                .endObject()
+                .endObject();
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolean sameStarTree) {
+        try {
+            return jsonBuilder().startObject()
+                .startObject("composite")
+                .startObject(sameStarTree ? "startree-1" : "startree-2")
+                .field("type", "star_tree")
+                .startObject("config")
+                .startArray("ordered_dimensions")
+                .startObject()
+                .field("name", "timestamp")
+                .endObject()
+                .startObject()
+                .field("name", changeDim ? "numeric_new" : getDim(false, false))
+                .endObject()
+                .endArray()
+                .startArray("metrics")
+                .startObject()
+                .field("name", getDim(false, false))
+                .endObject()
+                .endArray()
+                .endObject()
+                .endObject()
+                .endObject()
+                .startObject("properties")
+                .startObject("timestamp")
+                .field("type", "date")
+                .endObject()
+                .startObject("numeric_dv")
+                .field("type", "integer")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("numeric")
+                .field("type", "integer")
+                .field("doc_values", false)
+                .endObject()
+                .startObject("numeric_new")
+                .field("type", "integer")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("keyword_dv")
+                .field("type", "keyword")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("keyword")
+                .field("type", "keyword")
+                .field("doc_values", false)
+                .endObject()
+                .endObject()
+                .endObject();
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    private static String getDim(boolean hasDocValues, boolean isKeyword) {
+        if (hasDocValues) {
+            return "numeric";
+        } else if (isKeyword) {
+            return "keyword";
+        }
+        return "numeric_dv";
+    }
+
+    @Override
+    protected Settings featureFlagSettings() {
+        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.STAR_TREE_INDEX, "true").build();
+    }
+
+    @Before
+    public final void setupNodeSettings() {
+        Settings request = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), true).build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
+    }
+
+    public void testValidCompositeIndex() {
+        prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get();
+        Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
+        for (IndicesService service : dataNodeInstances) {
+            final Index index = resolveIndex("test");
+            if (service.hasIndex(index)) {
+                IndexService indexService = service.indexService(index);
+                Set<CompositeMappedFieldType> fts = indexService.mapperService().getCompositeFieldTypes();
+
+                for (CompositeMappedFieldType ft : fts) {
+                    assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType);
+                    StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft;
+                    assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
+                    assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
+                    DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
+                    List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
+                        Rounding.DateTimeUnit.MINUTES_OF_HOUR,
+                        Rounding.DateTimeUnit.HOUR_OF_DAY
+                    );
+                    assertEquals(expectedTimeUnits, dateDim.getIntervals());
+                    assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
+                    assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
+                    List<MetricStat> expectedMetrics = Arrays.asList(
+                        MetricStat.AVG,
+                        MetricStat.COUNT,
+                        MetricStat.SUM,
+                        MetricStat.MAX,
+                        MetricStat.MIN
+                    );
+                    assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
+                    assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
+                    assertEquals(
+                        StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP,
+                        starTreeFieldType.getStarTreeConfig().getBuildMode()
+                    );
+                    assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
+                }
+            }
+        }
+    }
+
+    public void testUpdateIndexWithAdditionOfStarTree() {
+        prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get();
+
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutMapping(TEST_INDEX).setSource(createUpdateTestMapping(false, false)).get()
+        );
+        assertEquals("Index cannot have more than [1] star tree fields", ex.getMessage());
+    }
+
+    public void testUpdateIndexWithNewerStarTree() {
+        prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree(false, false, false)).get();
+
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutMapping(TEST_INDEX).setSource(createUpdateTestMapping(false, false)).get()
+        );
+        assertEquals(
+            "Composite fields must be specified during index creation, addition of new composite fields during update is not supported",
+            ex.getMessage()
+        );
+    }
+
+    public void testUpdateIndexWhenMappingIsDifferent() {
+        prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get();
+
+        // update some field in the mapping
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutMapping(TEST_INDEX).setSource(createUpdateTestMapping(true, true)).get()
+        );
+        assertTrue(ex.getMessage().contains("Cannot update parameter [config] from"));
+    }
+
+    public void testUpdateIndexWhenMappingIsSame() {
+        prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get();
+
+        // update some field in the mapping
+        AcknowledgedResponse putMappingResponse = client().admin()
+            .indices()
+            .preparePutMapping(TEST_INDEX)
+            .setSource(createMinimalTestMapping(false, false, false))
+            .get();
+        assertAcked(putMappingResponse);
+
+        Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
+        for (IndicesService service : dataNodeInstances) {
+            final Index index = resolveIndex("test");
+            if (service.hasIndex(index)) {
+                IndexService indexService = service.indexService(index);
+                Set<CompositeMappedFieldType> fts = indexService.mapperService().getCompositeFieldTypes();
+
+                for (CompositeMappedFieldType ft : fts) {
+                    assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType);
+                    StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft;
+                    assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
+                    assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
+                    DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
+                    List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
+                        Rounding.DateTimeUnit.MINUTES_OF_HOUR,
+                        Rounding.DateTimeUnit.HOUR_OF_DAY
+                    );
+                    assertEquals(expectedTimeUnits, dateDim.getIntervals());
+                    assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
+                    assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
+                    List<MetricStat> expectedMetrics = Arrays.asList(
+                        MetricStat.AVG,
+                        MetricStat.COUNT,
+                        MetricStat.SUM,
+                        MetricStat.MAX,
+                        MetricStat.MIN
+                    );
+                    assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
+                    assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
+                    assertEquals(
+                        StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP,
+                        starTreeFieldType.getStarTreeConfig().getBuildMode()
+                    );
+                    assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
+                }
+            }
+        }
+    }
+
+    public void testInvalidDimCompositeIndex() {
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(true, false, false)).get()
+        );
+        assertEquals(
+            "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of star tree field",
+            ex.getMessage()
+        );
+    }
+
+    public void testMaxDimsCompositeIndex() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping())
+                .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2))
+                .get()
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: ordered_dimensions cannot have more than 2 dimensions for star tree field [startree-1]",
+            ex.getMessage()
+        );
+    }
+
+    public void testMaxCalendarIntervalsCompositeIndex() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping())
+                .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1))
+                .get()
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]",
+            ex.getMessage()
+        );
+    }
+
+    public void testUnsupportedDim() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, true)).get()
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: unsupported field type associated with dimension [keyword] as part of star tree field [startree-1]",
+            ex.getMessage()
+        );
+    }
+
+    public void testInvalidMetric() {
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, true, false)).get()
+        );
+        assertEquals(
+            "Aggregations not supported for the metrics field [numeric] with field type [integer] as part of star tree field",
+            ex.getMessage()
+        );
+    }
+
+    @After
+    public final void cleanupNodeSettings() {
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().putNull("*"))
+                .setTransientSettings(Settings.builder().putNull("*"))
+        );
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 16edec112f123..7973745ce84b3 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -85,6 +85,7 @@
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
+import org.opensearch.index.compositeindex.CompositeIndexValidator;
 import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
@@ -1318,6 +1319,10 @@ private static void updateIndexMappingsAndBuildSortOrder(
             }
         }
 
+        if (mapperService.isCompositeIndexPresent()) {
+            CompositeIndexValidator.validate(mapperService, indexService.getCompositeIndexSettings(), indexService.getIndexSettings());
+        }
+
         if (sourceMetadata == null) {
             // now that the mapping is merged we can validate the index sort.
             // we cannot validate for index shrinking since the mapping is empty
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java
index 1406287149e8d..43894db86c512 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java
@@ -55,6 +55,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.index.Index;
 import org.opensearch.index.IndexService;
+import org.opensearch.index.compositeindex.CompositeIndexValidator;
 import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
@@ -282,6 +283,7 @@ private ClusterState applyRequest(
                     // first, simulate: just call merge and ignore the result
                     existingMapper.merge(newMapper.mapping(), MergeReason.MAPPING_UPDATE);
                 }
+
             }
             Metadata.Builder builder = Metadata.builder(metadata);
             boolean updated = false;
@@ -291,7 +293,7 @@ private ClusterState applyRequest(
                 // we use the exact same indexService and metadata we used to validate above here to actually apply the update
                 final Index index = indexMetadata.getIndex();
                 final MapperService mapperService = indexMapperServices.get(index);
-
+                boolean isCompositeFieldPresent = !mapperService.getCompositeFieldTypes().isEmpty();
                 CompressedXContent existingSource = null;
                 DocumentMapper existingMapper = mapperService.documentMapper();
                 if (existingMapper != null) {
@@ -302,6 +304,14 @@ private ClusterState applyRequest(
                     mappingUpdateSource,
                     MergeReason.MAPPING_UPDATE
                 );
+
+                CompositeIndexValidator.validate(
+                    mapperService,
+                    indicesService.getCompositeIndexSettings(),
+                    mapperService.getIndexSettings(),
+                    isCompositeFieldPresent
+                );
+
                 CompressedXContent updatedSource = mergedMapper.mappingSource();
 
                 if (existingSource != null) {
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 233a8d732d178..5dcf23ae52294 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -115,6 +115,7 @@
 import org.opensearch.index.ShardIndexingPressureMemoryManager;
 import org.opensearch.index.ShardIndexingPressureSettings;
 import org.opensearch.index.ShardIndexingPressureStore;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.remote.RemoteStorePressureSettings;
 import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
 import org.opensearch.index.store.remote.filecache.FileCacheSettings;
@@ -754,7 +755,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
                 RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA,
-                SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING
+                SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING,
+
+                // Composite index settings
+                CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
index 238df1bd90113..b6166f5d3cce1 100644
--- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
@@ -37,6 +37,7 @@ protected FeatureFlagSettings(
         FeatureFlags.TIERED_REMOTE_INDEX_SETTING,
         FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         FeatureFlags.PLUGGABLE_CACHE_SETTING,
-        FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING
+        FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING,
+        FeatureFlags.STAR_TREE_INDEX_SETTING
     );
 }
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index 1488f5d30b4ba..ca2c4dab6102b 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -52,6 +52,7 @@
 import org.opensearch.index.SearchSlowLog;
 import org.opensearch.index.TieredMergePolicyProvider;
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.fielddata.IndexFieldDataService;
 import org.opensearch.index.mapper.FieldMapper;
@@ -239,6 +240,15 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 // Settings for concurrent segment search
                 IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING,
                 IndexSettings.ALLOW_DERIVED_FIELDS,
+
+                // Settings for star tree index
+                StarTreeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS,
+                StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING,
+                StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING,
+                StarTreeIndexSettings.DEFAULT_METRICS_LIST,
+                StarTreeIndexSettings.DEFAULT_DATE_INTERVALS,
+                StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING,
+
                 // validate that built-in similarities don't get redefined
                 Setting.groupSetting("index.similarity.", (s) -> {
                     Map<String, Settings> groups = s.getAsGroups();
diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
index 6c6e2f2d600f0..ceb2559a0e16c 100644
--- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
+++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -100,6 +100,13 @@ public class FeatureFlags {
         Property.NodeScope
     );
 
+    /**
+     * Gates the functionality of star tree index, which improves the performance of search
+     * aggregations.
+     */
+    public static final String STAR_TREE_INDEX = "opensearch.experimental.feature.composite_index.star_tree.enabled";
+    public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope);
+
     private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
         REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         EXTENSIONS_SETTING,
@@ -108,7 +115,8 @@ public class FeatureFlags {
         DATETIME_FORMATTER_CACHING_SETTING,
         TIERED_REMOTE_INDEX_SETTING,
         PLUGGABLE_CACHE_SETTING,
-        REMOTE_PUBLICATION_EXPERIMENTAL_SETTING
+        REMOTE_PUBLICATION_EXPERIMENTAL_SETTING,
+        STAR_TREE_INDEX_SETTING
     );
     /**
      * Should store the settings from opensearch.yml.
diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
index 4c494a6b35153..09b904394ee09 100644
--- a/server/src/main/java/org/opensearch/index/IndexModule.java
+++ b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -66,6 +66,7 @@
 import org.opensearch.index.cache.query.DisabledQueryCache;
 import org.opensearch.index.cache.query.IndexQueryCache;
 import org.opensearch.index.cache.query.QueryCache;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.engine.EngineConfigFactory;
 import org.opensearch.index.engine.EngineFactory;
@@ -311,6 +312,7 @@ public Iterator<Setting<?>> settings() {
     private final BooleanSupplier allowExpensiveQueries;
     private final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories;
     private final FileCache fileCache;
+    private final CompositeIndexSettings compositeIndexSettings;
 
     /**
      * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins
@@ -330,7 +332,8 @@ public IndexModule(
         final BooleanSupplier allowExpensiveQueries,
         final IndexNameExpressionResolver expressionResolver,
         final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories,
-        final FileCache fileCache
+        final FileCache fileCache,
+        final CompositeIndexSettings compositeIndexSettings
     ) {
         this.indexSettings = indexSettings;
         this.analysisRegistry = analysisRegistry;
@@ -343,6 +346,7 @@ public IndexModule(
         this.expressionResolver = expressionResolver;
         this.recoveryStateFactories = recoveryStateFactories;
         this.fileCache = fileCache;
+        this.compositeIndexSettings = compositeIndexSettings;
     }
 
     public IndexModule(
@@ -364,6 +368,7 @@ public IndexModule(
             allowExpensiveQueries,
             expressionResolver,
             recoveryStateFactories,
+            null,
             null
         );
     }
@@ -739,7 +744,8 @@ public IndexService newIndexService(
                 clusterDefaultRefreshIntervalSupplier,
                 recoverySettings,
                 remoteStoreSettings,
-                fileCache
+                fileCache,
+                compositeIndexSettings
             );
             success = true;
             return indexService;
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
index a7849bcf80474..1c0db0095bb98 100644
--- a/server/src/main/java/org/opensearch/index/IndexService.java
+++ b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -73,6 +73,7 @@
 import org.opensearch.index.cache.IndexCache;
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
 import org.opensearch.index.cache.query.QueryCache;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.engine.EngineConfigFactory;
 import org.opensearch.index.engine.EngineFactory;
@@ -192,6 +193,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final RecoverySettings recoverySettings;
     private final RemoteStoreSettings remoteStoreSettings;
     private final FileCache fileCache;
+    private final CompositeIndexSettings compositeIndexSettings;
 
     public IndexService(
         IndexSettings indexSettings,
@@ -228,7 +230,8 @@ public IndexService(
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
         RecoverySettings recoverySettings,
         RemoteStoreSettings remoteStoreSettings,
-        FileCache fileCache
+        FileCache fileCache,
+        CompositeIndexSettings compositeIndexSettings
     ) {
         super(indexSettings);
         this.allowExpensiveQueries = allowExpensiveQueries;
@@ -306,6 +309,7 @@ public IndexService(
         this.translogFactorySupplier = translogFactorySupplier;
         this.recoverySettings = recoverySettings;
         this.remoteStoreSettings = remoteStoreSettings;
+        this.compositeIndexSettings = compositeIndexSettings;
         this.fileCache = fileCache;
         updateFsyncTaskIfNecessary();
     }
@@ -381,6 +385,7 @@ public IndexService(
             clusterDefaultRefreshIntervalSupplier,
             recoverySettings,
             remoteStoreSettings,
+            null,
             null
         );
     }
@@ -1110,6 +1115,10 @@ private void rescheduleRefreshTasks() {
         }
     }
 
+    public CompositeIndexSettings getCompositeIndexSettings() {
+        return compositeIndexSettings;
+    }
+
     /**
      * Shard Store Deleter Interface
      *
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java
new file mode 100644
index 0000000000000..014dd22426a10
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java
@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+
+/**
+ * Cluster level settings for composite indices
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CompositeIndexSettings {
+    public static final Setting<Boolean> STAR_TREE_INDEX_ENABLED_SETTING = Setting.boolSetting(
+        "indices.composite_index.star_tree.enabled",
+        false,
+        value -> {
+            if (FeatureFlags.isEnabled(FeatureFlags.STAR_TREE_INDEX_SETTING) == false && value == true) {
+                throw new IllegalArgumentException(
+                    "star tree index is under an experimental feature and can be activated only by enabling "
+                        + FeatureFlags.STAR_TREE_INDEX_SETTING.getKey()
+                        + " feature flag in the JVM options"
+                );
+            }
+        },
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    private volatile boolean starTreeIndexCreationEnabled;
+
+    public CompositeIndexSettings(Settings settings, ClusterSettings clusterSettings) {
+        this.starTreeIndexCreationEnabled = STAR_TREE_INDEX_ENABLED_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(STAR_TREE_INDEX_ENABLED_SETTING, this::starTreeIndexCreationEnabled);
+
+    }
+
+    private void starTreeIndexCreationEnabled(boolean value) {
+        this.starTreeIndexCreationEnabled = value;
+    }
+
+    public boolean isStarTreeIndexCreationEnabled() {
+        return starTreeIndexCreationEnabled;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java
new file mode 100644
index 0000000000000..995352e3ce6a5
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeValidator;
+import org.opensearch.index.mapper.MapperService;
+
+import java.util.Locale;
+
+/**
+ * Validation for composite indices as part of mappings
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CompositeIndexValidator {
+
+    public static void validate(MapperService mapperService, CompositeIndexSettings compositeIndexSettings, IndexSettings indexSettings) {
+        StarTreeValidator.validate(mapperService, compositeIndexSettings, indexSettings);
+    }
+
+    public static void validate(
+        MapperService mapperService,
+        CompositeIndexSettings compositeIndexSettings,
+        IndexSettings indexSettings,
+        boolean isCompositeFieldPresent
+    ) {
+        if (!isCompositeFieldPresent && mapperService.isCompositeIndexPresent()) {
+            throw new IllegalArgumentException(
+                String.format(
+                    Locale.ROOT,
+                    "Composite fields must be specified during index creation, addition of new composite fields during update is not supported"
+                )
+            );
+        }
+        StarTreeValidator.validate(mapperService, compositeIndexSettings, indexSettings);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java
new file mode 100644
index 0000000000000..074016db2aed7
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.common.Rounding;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.mapper.CompositeDataCubeFieldType;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Date dimension class
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class DateDimension implements Dimension {
+    private final List<Rounding.DateTimeUnit> calendarIntervals;
+    public static final String CALENDAR_INTERVALS = "calendar_intervals";
+    public static final String DATE = "date";
+    private final String field;
+
+    public DateDimension(String field, List<Rounding.DateTimeUnit> calendarIntervals) {
+        this.field = field;
+        this.calendarIntervals = calendarIntervals;
+    }
+
+    public List<Rounding.DateTimeUnit> getIntervals() {
+        return calendarIntervals;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(CompositeDataCubeFieldType.NAME, this.getField());
+        builder.field(CompositeDataCubeFieldType.TYPE, DATE);
+        builder.startArray(CALENDAR_INTERVALS);
+        for (Rounding.DateTimeUnit interval : calendarIntervals) {
+            builder.value(interval.shortName());
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DateDimension that = (DateDimension) o;
+        return Objects.equals(field, that.getField()) && Objects.equals(calendarIntervals, that.calendarIntervals);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, calendarIntervals);
+    }
+
+    @Override
+    public String getField() {
+        return field;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java
new file mode 100644
index 0000000000000..0151a474579be
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.xcontent.ToXContent;
+
+/**
+ * Base interface for data-cube dimensions
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface Dimension extends ToXContent {
+    String getField();
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java
new file mode 100644
index 0000000000000..6a09e947217f5
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java
@@ -0,0 +1,99 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.common.Rounding;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.xcontent.support.XContentMapValues;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
+import org.opensearch.index.mapper.DateFieldMapper;
+import org.opensearch.index.mapper.Mapper;
+import org.opensearch.index.mapper.NumberFieldMapper;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS;
+
+/**
+ * Dimension factory class mainly used to parse and create dimension from the mappings
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class DimensionFactory {
+    public static Dimension parseAndCreateDimension(
+        String name,
+        String type,
+        Map<String, Object> dimensionMap,
+        Mapper.TypeParser.ParserContext c
+    ) {
+        switch (type) {
+            case DateDimension.DATE:
+                return parseAndCreateDateDimension(name, dimensionMap, c);
+            case NumericDimension.NUMERIC:
+                return new NumericDimension(name);
+            default:
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, "unsupported field type associated with dimension [%s] as part of star tree field", name)
+                );
+        }
+    }
+
+    public static Dimension parseAndCreateDimension(
+        String name,
+        Mapper.Builder builder,
+        Map<String, Object> dimensionMap,
+        Mapper.TypeParser.ParserContext c
+    ) {
+        if (builder instanceof DateFieldMapper.Builder) {
+            return parseAndCreateDateDimension(name, dimensionMap, c);
+        } else if (builder instanceof NumberFieldMapper.Builder) {
+            return new NumericDimension(name);
+        }
+        throw new IllegalArgumentException(
+            String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name)
+        );
+    }
+
+    private static DateDimension parseAndCreateDateDimension(
+        String name,
+        Map<String, Object> dimensionMap,
+        Mapper.TypeParser.ParserContext c
+    ) {
+        List<Rounding.DateTimeUnit> calendarIntervals = new ArrayList<>();
+        List<String> intervalStrings = XContentMapValues.extractRawValues(CALENDAR_INTERVALS, dimensionMap)
+            .stream()
+            .map(Object::toString)
+            .collect(Collectors.toList());
+        if (intervalStrings == null || intervalStrings.isEmpty()) {
+            calendarIntervals = StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings());
+        } else {
+            if (intervalStrings.size() > StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.get(c.getSettings())) {
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "At most [%s] calendar intervals are allowed in dimension [%s]",
+                        StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.get(c.getSettings()),
+                        name
+                    )
+                );
+            }
+            for (String interval : intervalStrings) {
+                calendarIntervals.add(StarTreeIndexSettings.getTimeUnit(interval));
+            }
+            calendarIntervals = new ArrayList<>(calendarIntervals);
+        }
+        dimensionMap.remove(CALENDAR_INTERVALS);
+        return new DateDimension(name, calendarIntervals);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java
new file mode 100644
index 0000000000000..9accb0201170a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Holds details of metrics field as part of composite field
+ */
+@ExperimentalApi
+public class Metric implements ToXContent {
+    private final String field;
+    private final List<MetricStat> metrics;
+
+    public Metric(String field, List<MetricStat> metrics) {
+        this.field = field;
+        this.metrics = metrics;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public List<MetricStat> getMetrics() {
+        return metrics;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("name", field);
+        builder.startArray("stats");
+        for (MetricStat metricType : metrics) {
+            builder.value(metricType.getTypeName());
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Metric metric = (Metric) o;
+        return Objects.equals(field, metric.field) && Objects.equals(metrics, metric.metrics);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, metrics);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java
new file mode 100644
index 0000000000000..fbde296b15f7e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java
@@ -0,0 +1,44 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Supported metric types for composite index
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public enum MetricStat {
+    COUNT("count"),
+    AVG("avg"),
+    SUM("sum"),
+    MIN("min"),
+    MAX("max");
+
+    private final String typeName;
+
+    MetricStat(String typeName) {
+        this.typeName = typeName;
+    }
+
+    public String getTypeName() {
+        return typeName;
+    }
+
+    public static MetricStat fromTypeName(String typeName) {
+        for (MetricStat metric : MetricStat.values()) {
+            if (metric.getTypeName().equalsIgnoreCase(typeName)) {
+                return metric;
+            }
+        }
+        throw new IllegalArgumentException("Invalid metric stat: " + typeName);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java
new file mode 100644
index 0000000000000..9c25ef5b25503
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.mapper.CompositeDataCubeFieldType;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Composite index numeric dimension class
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class NumericDimension implements Dimension {
+    public static final String NUMERIC = "numeric";
+    private final String field;
+
+    public NumericDimension(String field) {
+        this.field = field;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(CompositeDataCubeFieldType.NAME, field);
+        builder.field(CompositeDataCubeFieldType.TYPE, NUMERIC);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NumericDimension dimension = (NumericDimension) o;
+        return Objects.equals(field, dimension.getField());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java
new file mode 100644
index 0000000000000..320876ea937bf
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java
@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/**
+ * Core classes for handling data cube indices such as star tree index.
+ */
+package org.opensearch.index.compositeindex.datacube;
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java
new file mode 100644
index 0000000000000..922ddcbea4fe2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java
@@ -0,0 +1,94 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Star tree field which contains dimensions, metrics and specs
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeField implements ToXContent {
+    private final String name;
+    private final List<Dimension> dimensionsOrder;
+    private final List<Metric> metrics;
+    private final StarTreeFieldConfiguration starTreeConfig;
+
+    public StarTreeField(String name, List<Dimension> dimensions, List<Metric> metrics, StarTreeFieldConfiguration starTreeConfig) {
+        this.name = name;
+        this.dimensionsOrder = dimensions;
+        this.metrics = metrics;
+        this.starTreeConfig = starTreeConfig;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public List<Dimension> getDimensionsOrder() {
+        return dimensionsOrder;
+    }
+
+    public List<Metric> getMetrics() {
+        return metrics;
+    }
+
+    public StarTreeFieldConfiguration getStarTreeConfig() {
+        return starTreeConfig;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("name", name);
+        if (dimensionsOrder != null && !dimensionsOrder.isEmpty()) {
+            builder.startArray("ordered_dimensions");
+            for (Dimension dimension : dimensionsOrder) {
+                dimension.toXContent(builder, params);
+            }
+            builder.endArray();
+        }
+        if (metrics != null && !metrics.isEmpty()) {
+            builder.startArray("metrics");
+            for (Metric metric : metrics) {
+                metric.toXContent(builder, params);
+            }
+            builder.endArray();
+        }
+        starTreeConfig.toXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        StarTreeField that = (StarTreeField) o;
+        return Objects.equals(name, that.name)
+            && Objects.equals(dimensionsOrder, that.dimensionsOrder)
+            && Objects.equals(metrics, that.metrics)
+            && Objects.equals(starTreeConfig, that.starTreeConfig);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, dimensionsOrder, metrics, starTreeConfig);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java
new file mode 100644
index 0000000000000..755c064c2c60a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java
@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Star tree index specific configuration
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeFieldConfiguration implements ToXContent {
+
+    private final AtomicInteger maxLeafDocs = new AtomicInteger();
+    private final Set<String> skipStarNodeCreationInDims;
+    private final StarTreeBuildMode buildMode;
+
+    public StarTreeFieldConfiguration(int maxLeafDocs, Set<String> skipStarNodeCreationInDims, StarTreeBuildMode buildMode) {
+        this.maxLeafDocs.set(maxLeafDocs);
+        this.skipStarNodeCreationInDims = skipStarNodeCreationInDims;
+        this.buildMode = buildMode;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        // build mode is internal and not part of user mappings config, hence not added as part of toXContent
+        builder.field("max_leaf_docs", maxLeafDocs.get());
+        builder.startArray("skip_star_node_creation_for_dimensions");
+        for (String dim : skipStarNodeCreationInDims) {
+            builder.value(dim);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    /**
+     * Star tree build mode using which sorting and aggregations are performed during index creation.
+     *
+     * @opensearch.experimental
+     */
+    @ExperimentalApi
+    public enum StarTreeBuildMode {
+        // TODO : remove onheap support unless this proves useful
+        ON_HEAP("onheap"),
+        OFF_HEAP("offheap");
+
+        private final String typeName;
+
+        StarTreeBuildMode(String typeName) {
+            this.typeName = typeName;
+        }
+
+        public String getTypeName() {
+            return typeName;
+        }
+
+        public static StarTreeBuildMode fromTypeName(String typeName) {
+            for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) {
+                if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) {
+                    return starTreeBuildMode;
+                }
+            }
+            throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName));
+        }
+    }
+
+    public int maxLeafDocs() {
+        return maxLeafDocs.get();
+    }
+
+    public StarTreeBuildMode getBuildMode() {
+        return buildMode;
+    }
+
+    public Set<String> getSkipStarNodeCreationInDims() {
+        return skipStarNodeCreationInDims;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        StarTreeFieldConfiguration that = (StarTreeFieldConfiguration) o;
+        return Objects.equals(maxLeafDocs.get(), that.maxLeafDocs.get())
+            && Objects.equals(skipStarNodeCreationInDims, that.skipStarNodeCreationInDims)
+            && buildMode == that.buildMode;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(maxLeafDocs.get(), skipStarNodeCreationInDims, buildMode);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java
new file mode 100644
index 0000000000000..a2ac545be3cc9
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java
@@ -0,0 +1,116 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree;
+
+import org.opensearch.common.Rounding;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Index settings for star tree fields. The settings are final as right now
+ * there is no support for update of star tree mapping.
+ *
+ * @opensearch.experimental
+ */
+public class StarTreeIndexSettings {
+
+    public static int STAR_TREE_MAX_DIMENSIONS_DEFAULT = 10;
+    /**
+     * This setting determines the max number of star tree fields that can be part of composite index mapping. For each
+     * star tree field, we will generate associated star tree index.
+     */
+    public static final Setting<Integer> STAR_TREE_MAX_FIELDS_SETTING = Setting.intSetting(
+        "index.composite_index.star_tree.max_fields",
+        1,
+        1,
+        1,
+        Setting.Property.IndexScope,
+        Setting.Property.Final
+    );
+
+    /**
+     * This setting determines the max number of dimensions that can be part of star tree index field. Number of
+     * dimensions and associated cardinality has direct effect of star tree index size and query performance.
+     */
+    public static final Setting<Integer> STAR_TREE_MAX_DIMENSIONS_SETTING = Setting.intSetting(
+        "index.composite_index.star_tree.field.max_dimensions",
+        STAR_TREE_MAX_DIMENSIONS_DEFAULT,
+        2,
+        10,
+        Setting.Property.IndexScope,
+        Setting.Property.Final
+    );
+
+    /**
+     * This setting determines the max number of date intervals that can be part of star tree date field.
+     */
+    public static final Setting<Integer> STAR_TREE_MAX_DATE_INTERVALS_SETTING = Setting.intSetting(
+        "index.composite_index.star_tree.field.max_date_intervals",
+        3,
+        1,
+        3,
+        Setting.Property.IndexScope,
+        Setting.Property.Final
+    );
+
+    /**
+     * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and
+     * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa
+     * <p>
+     * We can remove this later or change it to an enum based constant setting.
+     *
+     * @opensearch.experimental
+     */
+    public static final Setting<Integer> STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting(
+        "index.composite_index.star_tree.default.max_leaf_docs",
+        10000,
+        1,
+        Setting.Property.IndexScope,
+        Setting.Property.Final
+    );
+
+    /**
+     * Default intervals for date dimension as part of star tree fields
+     */
+    public static final Setting<List<Rounding.DateTimeUnit>> DEFAULT_DATE_INTERVALS = Setting.listSetting(
+        "index.composite_index.star_tree.field.default.date_intervals",
+        Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()),
+        StarTreeIndexSettings::getTimeUnit,
+        Setting.Property.IndexScope,
+        Setting.Property.Final
+    );
+
+    /**
+     * Default metrics for metrics as part of star tree fields
+     */
+    public static final Setting<List<MetricStat>> DEFAULT_METRICS_LIST = Setting.listSetting(
+        "index.composite_index.star_tree.field.default.metrics",
+        Arrays.asList(
+            MetricStat.AVG.toString(),
+            MetricStat.COUNT.toString(),
+            MetricStat.SUM.toString(),
+            MetricStat.MAX.toString(),
+            MetricStat.MIN.toString()
+        ),
+        MetricStat::fromTypeName,
+        Setting.Property.IndexScope,
+        Setting.Property.Final
+    );
+
+    public static Rounding.DateTimeUnit getTimeUnit(String expression) {
+        if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) {
+            throw new IllegalArgumentException("unknown calendar intervals specified in star tree index mapping");
+        }
+        return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeValidator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeValidator.java
new file mode 100644
index 0000000000000..cbed46604681d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeValidator.java
@@ -0,0 +1,94 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.mapper.CompositeMappedFieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.StarTreeMapper;
+
+import java.util.Locale;
+import java.util.Set;
+
+/**
+ * Validations for star tree fields as part of mappings
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeValidator {
+    public static void validate(MapperService mapperService, CompositeIndexSettings compositeIndexSettings, IndexSettings indexSettings) {
+        Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes();
+        if (compositeFieldTypes.size() > StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(indexSettings.getSettings())) {
+            throw new IllegalArgumentException(
+                String.format(
+                    Locale.ROOT,
+                    "Index cannot have more than [%s] star tree fields",
+                    StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(indexSettings.getSettings())
+                )
+            );
+        }
+        for (CompositeMappedFieldType compositeFieldType : compositeFieldTypes) {
+            if (!(compositeFieldType instanceof StarTreeMapper.StarTreeFieldType)) {
+                continue;
+            }
+            if (!compositeIndexSettings.isStarTreeIndexCreationEnabled()) {
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "star tree index cannot be created, enable it using [%s] setting",
+                        CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey()
+                    )
+                );
+            }
+            StarTreeMapper.StarTreeFieldType dataCubeFieldType = (StarTreeMapper.StarTreeFieldType) compositeFieldType;
+            for (Dimension dim : dataCubeFieldType.getDimensions()) {
+                MappedFieldType ft = mapperService.fieldType(dim.getField());
+                if (ft == null) {
+                    throw new IllegalArgumentException(
+                        String.format(Locale.ROOT, "unknown dimension field [%s] as part of star tree field", dim.getField())
+                    );
+                }
+                if (ft.isAggregatable() == false) {
+                    throw new IllegalArgumentException(
+                        String.format(
+                            Locale.ROOT,
+                            "Aggregations not supported for the dimension field [%s] with field type [%s] as part of star tree field",
+                            dim.getField(),
+                            ft.typeName()
+                        )
+                    );
+                }
+            }
+            for (Metric metric : dataCubeFieldType.getMetrics()) {
+                MappedFieldType ft = mapperService.fieldType(metric.getField());
+                if (ft == null) {
+                    throw new IllegalArgumentException(
+                        String.format(Locale.ROOT, "unknown metric field [%s] as part of star tree field", metric.getField())
+                    );
+                }
+                if (ft.isAggregatable() == false) {
+                    throw new IllegalArgumentException(
+                        String.format(
+                            Locale.ROOT,
+                            "Aggregations not supported for the metrics field [%s] with field type [%s] as part of star tree field",
+                            metric.getField(),
+                            ft.typeName()
+                        )
+                    );
+                }
+            }
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java
new file mode 100644
index 0000000000000..4f4e670478e2f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java
@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/**
+ * Core classes for handling star tree index.
+ */
+package org.opensearch.index.compositeindex.datacube.startree;
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java
new file mode 100644
index 0000000000000..59f18efec26b1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Core classes for handling composite indices.
+ * @opensearch.experimental
+ */
+package org.opensearch.index.compositeindex;
diff --git a/server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java b/server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java
new file mode 100644
index 0000000000000..baf6442f0c08c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java
@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Base class for multi field data cube fields
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public abstract class CompositeDataCubeFieldType extends CompositeMappedFieldType {
+    public static final String NAME = "name";
+    public static final String TYPE = "type";
+    private final List<Dimension> dimensions;
+    private final List<Metric> metrics;
+
+    public CompositeDataCubeFieldType(String name, List<Dimension> dims, List<Metric> metrics, CompositeFieldType type) {
+        super(name, getFields(dims, metrics), type);
+        this.dimensions = dims;
+        this.metrics = metrics;
+    }
+
+    private static List<String> getFields(List<Dimension> dims, List<Metric> metrics) {
+        Set<String> fields = new HashSet<>();
+        for (Dimension dim : dims) {
+            fields.add(dim.getField());
+        }
+        for (Metric metric : metrics) {
+            fields.add(metric.getField());
+        }
+        return new ArrayList<>(fields);
+    }
+
+    public List<Dimension> getDimensions() {
+        return dimensions;
+    }
+
+    public List<Metric> getMetrics() {
+        return metrics;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java
new file mode 100644
index 0000000000000..f52ce29a86dd2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java
@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Base class for composite field types
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public abstract class CompositeMappedFieldType extends MappedFieldType {
+    private final List<String> fields;
+    private final CompositeFieldType type;
+
+    public CompositeMappedFieldType(
+        String name,
+        boolean isIndexed,
+        boolean isStored,
+        boolean hasDocValues,
+        TextSearchInfo textSearchInfo,
+        Map<String, String> meta,
+        List<String> fields,
+        CompositeFieldType type
+    ) {
+        super(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta);
+        this.fields = fields;
+        this.type = type;
+    }
+
+    public CompositeMappedFieldType(String name, List<String> fields, CompositeFieldType type) {
+        this(name, false, false, false, TextSearchInfo.NONE, Collections.emptyMap(), fields, type);
+    }
+
+    /**
+     * Supported composite field types
+     */
+    public enum CompositeFieldType {
+        STAR_TREE("star_tree");
+
+        private final String name;
+
+        CompositeFieldType(String name) {
+            this.name = name;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public static CompositeFieldType fromName(String name) {
+            for (CompositeFieldType metric : CompositeFieldType.values()) {
+                if (metric.getName().equalsIgnoreCase(name)) {
+                    return metric;
+                }
+            }
+            throw new IllegalArgumentException("Invalid metric stat: " + name);
+        }
+    }
+
+    public List<String> fields() {
+        return fields;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java
index bd5d3f15c0706..46a5050d4fc18 100644
--- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java
@@ -253,6 +253,11 @@ public boolean isWithinMultiField() {
         }
 
         Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;
+
+        default Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext parserContext, ObjectMapper.Builder objBuilder)
+            throws MapperParsingException {
+            throw new UnsupportedOperationException("should not be invoked");
+        }
     }
 
     private final String simpleName;
diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java
index a1f3894c9f14c..c2e7411a3b47a 100644
--- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java
@@ -650,6 +650,23 @@ public Iterable<MappedFieldType> fieldTypes() {
         return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes();
     }
 
+    public boolean isCompositeIndexPresent() {
+        return this.mapper != null && !getCompositeFieldTypes().isEmpty();
+    }
+
+    public Set<CompositeMappedFieldType> getCompositeFieldTypes() {
+        Set<CompositeMappedFieldType> compositeMappedFieldTypes = new HashSet<>();
+        if (this.mapper == null) {
+            return Collections.emptySet();
+        }
+        for (MappedFieldType type : this.mapper.fieldTypes()) {
+            if (type instanceof CompositeMappedFieldType) {
+                compositeMappedFieldTypes.add((CompositeMappedFieldType) type);
+            }
+        }
+        return compositeMappedFieldTypes;
+    }
+
     public ObjectMapper getObjectMapper(String name) {
         return this.mapper == null ? null : this.mapper.objectMappers().get(name);
     }
diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
index 92ffdb60e6cde..be3adfe8b2c4e 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
@@ -42,9 +42,11 @@
 import org.opensearch.common.collect.CopyOnWriteHashMap;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.support.XContentMapValues;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
 import org.opensearch.index.mapper.MapperService.MergeReason;
 
 import java.io.IOException;
@@ -176,6 +178,7 @@ public void setIncludeInRoot(boolean value) {
      * @opensearch.internal
      */
     @SuppressWarnings("rawtypes")
+    @PublicApi(since = "1.0.0")
     public static class Builder<T extends Builder> extends Mapper.Builder<T> {
 
         protected Explicit<Boolean> enabled = new Explicit<>(true, false);
@@ -262,14 +265,25 @@ public static class TypeParser implements Mapper.TypeParser {
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             ObjectMapper.Builder builder = new Builder(name);
             parseNested(name, node, builder, parserContext);
+            Object compositeField = null;
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
-                if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)) {
+                if (fieldName.equals("composite")) {
+                    compositeField = fieldNode;
                     iterator.remove();
+                } else {
+                    if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)) {
+                        iterator.remove();
+                    }
                 }
             }
+            // Important : Composite field is made up of 2 or more source fields of the index, so this must be called
+            // after parsing all other properties
+            if (compositeField != null) {
+                parseCompositeField(builder, (Map<String, Object>) compositeField, parserContext);
+            }
             return builder;
         }
 
@@ -407,6 +421,96 @@ protected static void parseDerived(ObjectMapper.Builder objBuilder, Map<String,
             );
         }
 
+        /**
+         * Contains logic to parse 'composite' section of mappings. This should be called only after
+         * parsing normal properties as this checks if the given fields are also source fields of the index.
+         */
+        protected static void parseCompositeField(
+            ObjectMapper.Builder objBuilder,
+            Map<String, Object> compositeNode,
+            ParserContext parserContext
+        ) {
+            if (!FeatureFlags.isEnabled(FeatureFlags.STAR_TREE_INDEX_SETTING)) {
+                throw new IllegalArgumentException(
+                    "star tree index is under an experimental feature and can be activated only by enabling "
+                        + FeatureFlags.STAR_TREE_INDEX_SETTING.getKey()
+                        + " feature flag in the JVM options"
+                );
+            }
+            Iterator<Map.Entry<String, Object>> iterator = compositeNode.entrySet().iterator();
+            if (compositeNode.size() > StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(parserContext.getSettings())) {
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "Composite fields cannot have more than [%s] fields",
+                        StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(parserContext.getSettings())
+                    )
+                );
+            }
+            while (iterator.hasNext()) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String fieldName = entry.getKey();
+                // Should accept empty arrays, as a work around for when the
+                // user can't provide an empty Map. (PHP for example)
+                boolean isEmptyList = entry.getValue() instanceof List && ((List<?>) entry.getValue()).isEmpty();
+                if (entry.getValue() instanceof Map) {
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> propNode = (Map<String, Object>) entry.getValue();
+                    String type;
+                    Object typeNode = propNode.get("type");
+                    if (typeNode != null) {
+                        type = typeNode.toString();
+                    } else {
+                        // lets see if we can derive this...
+                        throw new MapperParsingException("No type specified for field [" + fieldName + "]");
+                    }
+                    Mapper.TypeParser typeParser = getSupportedCompositeTypeParser(type, parserContext);
+                    if (typeParser == null) {
+                        throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]");
+                    }
+                    String[] fieldNameParts = fieldName.split("\\.");
+                    // field name is just ".", which is invalid
+                    if (fieldNameParts.length < 1) {
+                        throw new MapperParsingException("Invalid field name " + fieldName);
+                    }
+                    String realFieldName = fieldNameParts[fieldNameParts.length - 1];
+                    Mapper.Builder<?> fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext, objBuilder);
+                    for (int i = fieldNameParts.length - 2; i >= 0; --i) {
+                        ObjectMapper.Builder<?> intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]);
+                        intermediate.add(fieldBuilder);
+                        fieldBuilder = intermediate;
+                    }
+                    objBuilder.add(fieldBuilder);
+                    propNode.remove("type");
+                    DocumentMapperParser.checkNoRemainingFields(fieldName, propNode, parserContext.indexVersionCreated());
+                    iterator.remove();
+                } else if (isEmptyList) {
+                    iterator.remove();
+                } else {
+                    throw new MapperParsingException(
+                        "Expected map for property [fields] on field [" + fieldName + "] but got a " + fieldName.getClass()
+                    );
+                }
+            }
+
+            DocumentMapperParser.checkNoRemainingFields(
+                compositeNode,
+                parserContext.indexVersionCreated(),
+                "DocType mapping definition has unsupported parameters: "
+            );
+        }
+
+        private static Mapper.TypeParser getSupportedCompositeTypeParser(String type, ParserContext parserContext) {
+            switch (type) {
+                case StarTreeMapper.CONTENT_TYPE:
+                    return parserContext.typeParser(type);
+                default:
+                    throw new IllegalArgumentException(
+                        String.format(Locale.ROOT, "Type [%s] isn't supported in composite field context.", type)
+                    );
+            }
+        }
+
         protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
             Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator();
             while (iterator.hasNext()) {
diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java
index 9504e6eafc046..e06e5be4633f9 100644
--- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java
@@ -177,15 +177,26 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
             RootObjectMapper.Builder builder = new Builder(name);
             Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
+            Object compositeField = null;
             while (iterator.hasNext()) {
                 Map.Entry<String, Object> entry = iterator.next();
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
-                if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
-                    || processField(builder, fieldName, fieldNode, parserContext)) {
+                if (fieldName.equals("composite")) {
+                    compositeField = fieldNode;
                     iterator.remove();
+                } else {
+                    if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
+                        || processField(builder, fieldName, fieldNode, parserContext)) {
+                        iterator.remove();
+                    }
                 }
             }
+            // Important : Composite field is made up of 2 or more source properties of the index, so this must be called
+            // after parsing all other properties
+            if (compositeField != null) {
+                parseCompositeField(builder, (Map<String, Object>) compositeField, parserContext);
+            }
             return builder;
         }
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java
new file mode 100644
index 0000000000000..d2debe762e9be
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java
@@ -0,0 +1,406 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.search.Query;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.xcontent.support.XContentMapValues;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.DimensionFactory;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.search.lookup.SearchLookup;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A field mapper for star tree fields
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeMapper extends ParametrizedFieldMapper {
+    public static final String CONTENT_TYPE = "star_tree";
+    public static final String CONFIG = "config";
+    public static final String MAX_LEAF_DOCS = "max_leaf_docs";
+    public static final String SKIP_STAR_NODE_IN_DIMS = "skip_star_node_creation_for_dimensions";
+    public static final String BUILD_MODE = "build_mode";
+    public static final String ORDERED_DIMENSIONS = "ordered_dimensions";
+    public static final String METRICS = "metrics";
+    public static final String STATS = "stats";
+
+    @Override
+    public ParametrizedFieldMapper.Builder getMergeBuilder() {
+        return new Builder(simpleName(), objBuilder).init(this);
+
+    }
+
+    /**
+     * Builder for the star tree field mapper
+     *
+     * @opensearch.internal
+     */
+    public static class Builder extends ParametrizedFieldMapper.Builder {
+        private ObjectMapper.Builder objbuilder;
+        private static final Set<Class<? extends Mapper.Builder>> ALLOWED_DIMENSION_MAPPER_BUILDERS = Set.of(
+            NumberFieldMapper.Builder.class,
+            DateFieldMapper.Builder.class
+        );
+        private static final Set<Class<? extends Mapper.Builder>> ALLOWED_METRIC_MAPPER_BUILDERS = Set.of(NumberFieldMapper.Builder.class);
+
+        @SuppressWarnings("unchecked")
+        private final Parameter<StarTreeField> config = new Parameter<>(CONFIG, false, () -> null, (name, context, nodeObj) -> {
+            if (nodeObj instanceof Map<?, ?>) {
+                Map<String, Object> paramMap = (Map<String, Object>) nodeObj;
+                int maxLeafDocs = XContentMapValues.nodeIntegerValue(
+                    paramMap.get(MAX_LEAF_DOCS),
+                    StarTreeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS.get(context.getSettings())
+                );
+                if (maxLeafDocs < 1) {
+                    throw new IllegalArgumentException(
+                        String.format(Locale.ROOT, "%s [%s] must be greater than 0", MAX_LEAF_DOCS, maxLeafDocs)
+                    );
+                }
+                paramMap.remove(MAX_LEAF_DOCS);
+                Set<String> skipStarInDims = new LinkedHashSet<>(
+                    List.of(XContentMapValues.nodeStringArrayValue(paramMap.getOrDefault(SKIP_STAR_NODE_IN_DIMS, new ArrayList<String>())))
+                );
+                paramMap.remove(SKIP_STAR_NODE_IN_DIMS);
+                // TODO : change this to off heap once off heap gets implemented
+                StarTreeFieldConfiguration.StarTreeBuildMode buildMode = StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP;
+
+                List<Dimension> dimensions = buildDimensions(name, paramMap, context);
+                paramMap.remove(ORDERED_DIMENSIONS);
+                List<Metric> metrics = buildMetrics(name, paramMap, context);
+                paramMap.remove(METRICS);
+                paramMap.remove(CompositeDataCubeFieldType.NAME);
+                for (String dim : skipStarInDims) {
+                    if (dimensions.stream().filter(d -> d.getField().equals(dim)).findAny().isEmpty()) {
+                        throw new IllegalArgumentException(
+                            String.format(
+                                Locale.ROOT,
+                                "[%s] in skip_star_node_creation_for_dimensions should be part of ordered_dimensions",
+                                dim
+                            )
+                        );
+                    }
+                }
+                StarTreeFieldConfiguration spec = new StarTreeFieldConfiguration(maxLeafDocs, skipStarInDims, buildMode);
+                DocumentMapperParser.checkNoRemainingFields(
+                    paramMap,
+                    context.indexVersionCreated(),
+                    "Star tree mapping definition has unsupported parameters: "
+                );
+                return new StarTreeField(this.name, dimensions, metrics, spec);
+
+            } else {
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, "unable to parse config for star tree field [%s]", this.name)
+                );
+            }
+        }, m -> toType(m).starTreeField);
+
+        /**
+         * Build dimensions from mapping
+         */
+        @SuppressWarnings("unchecked")
+        private List<Dimension> buildDimensions(String fieldName, Map<String, Object> map, Mapper.TypeParser.ParserContext context) {
+            Object dims = XContentMapValues.extractValue("ordered_dimensions", map);
+            if (dims == null) {
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, "ordered_dimensions is required for star tree field [%s]", fieldName)
+                );
+            }
+            List<Dimension> dimensions = new LinkedList<>();
+            if (dims instanceof List<?>) {
+                List<Object> dimList = (List<Object>) dims;
+                if (dimList.size() > context.getSettings()
+                    .getAsInt(
+                        StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(),
+                        StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_DEFAULT
+                    )) {
+                    throw new IllegalArgumentException(
+                        String.format(
+                            Locale.ROOT,
+                            "ordered_dimensions cannot have more than %s dimensions for star tree field [%s]",
+                            context.getSettings()
+                                .getAsInt(
+                                    StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(),
+                                    StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_DEFAULT
+                                ),
+                            fieldName
+                        )
+                    );
+                }
+                if (dimList.size() < 2) {
+                    throw new IllegalArgumentException(
+                        String.format(Locale.ROOT, "Atleast two dimensions are required to build star tree index field [%s]", fieldName)
+                    );
+                }
+                for (Object dim : dimList) {
+                    dimensions.add(getDimension(fieldName, dim, context));
+                }
+            } else {
+                throw new MapperParsingException(
+                    String.format(Locale.ROOT, "unable to parse ordered_dimensions for star tree field [%s]", fieldName)
+                );
+            }
+            return dimensions;
+        }
+
+        /**
+         * Get dimension based on mapping
+         */
+        @SuppressWarnings("unchecked")
+        private Dimension getDimension(String fieldName, Object dimensionMapping, Mapper.TypeParser.ParserContext context) {
+            Dimension dimension;
+            Map<String, Object> dimensionMap = (Map<String, Object>) dimensionMapping;
+            String name = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.NAME, dimensionMap);
+            dimensionMap.remove(CompositeDataCubeFieldType.NAME);
+            if (this.objbuilder == null || this.objbuilder.mappersBuilders == null) {
+                String type = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.TYPE, dimensionMap);
+                dimensionMap.remove(CompositeDataCubeFieldType.TYPE);
+                if (type == null) {
+                    throw new MapperParsingException(
+                        String.format(Locale.ROOT, "unable to parse ordered_dimensions for star tree field [%s]", fieldName)
+                    );
+                }
+                return DimensionFactory.parseAndCreateDimension(name, type, dimensionMap, context);
+            } else {
+                Optional<Mapper.Builder> dimBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders);
+                if (dimBuilder.isEmpty()) {
+                    throw new IllegalArgumentException(String.format(Locale.ROOT, "unknown dimension field [%s]", name));
+                }
+                if (!isBuilderAllowedForDimension(dimBuilder.get())) {
+                    throw new IllegalArgumentException(
+                        String.format(
+                            Locale.ROOT,
+                            "unsupported field type associated with dimension [%s] as part of star tree field [%s]",
+                            name,
+                            fieldName
+                        )
+                    );
+                }
+                dimension = DimensionFactory.parseAndCreateDimension(name, dimBuilder.get(), dimensionMap, context);
+            }
+            DocumentMapperParser.checkNoRemainingFields(
+                dimensionMap,
+                context.indexVersionCreated(),
+                "Star tree mapping definition has unsupported parameters: "
+            );
+            return dimension;
+        }
+
+        /**
+         * Build metrics from mapping
+         */
+        @SuppressWarnings("unchecked")
+        private List<Metric> buildMetrics(String fieldName, Map<String, Object> map, Mapper.TypeParser.ParserContext context) {
+            List<Metric> metrics = new LinkedList<>();
+            Object metricsFromInput = XContentMapValues.extractValue(METRICS, map);
+            if (metricsFromInput == null) {
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, "metrics section is required for star tree field [%s]", fieldName)
+                );
+            }
+            if (metricsFromInput instanceof List<?>) {
+                List<?> metricsList = (List<?>) metricsFromInput;
+                for (Object metric : metricsList) {
+                    Map<String, Object> metricMap = (Map<String, Object>) metric;
+                    String name = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.NAME, metricMap);
+                    metricMap.remove(CompositeDataCubeFieldType.NAME);
+                    if (objbuilder == null || objbuilder.mappersBuilders == null) {
+                        metrics.add(getMetric(name, metricMap, context));
+                    } else {
+                        Optional<Mapper.Builder> meticBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders);
+                        if (meticBuilder.isEmpty()) {
+                            throw new IllegalArgumentException(String.format(Locale.ROOT, "unknown metric field [%s]", name));
+                        }
+                        if (!isBuilderAllowedForMetric(meticBuilder.get())) {
+                            throw new IllegalArgumentException(
+                                String.format(Locale.ROOT, "non-numeric field type is associated with star tree metric [%s]", this.name)
+                            );
+                        }
+                        metrics.add(getMetric(name, metricMap, context));
+                        DocumentMapperParser.checkNoRemainingFields(
+                            metricMap,
+                            context.indexVersionCreated(),
+                            "Star tree mapping definition has unsupported parameters: "
+                        );
+                    }
+                }
+            } else {
+                throw new MapperParsingException(String.format(Locale.ROOT, "unable to parse metrics for star tree field [%s]", this.name));
+            }
+
+            return metrics;
+        }
+
+        @SuppressWarnings("unchecked")
+        private Metric getMetric(String name, Map<String, Object> metric, Mapper.TypeParser.ParserContext context) {
+            List<MetricStat> metricTypes;
+            List<String> metricStrings = XContentMapValues.extractRawValues(STATS, metric)
+                .stream()
+                .map(Object::toString)
+                .collect(Collectors.toList());
+            metric.remove(STATS);
+            if (metricStrings.isEmpty()) {
+                metricTypes = new ArrayList<>(StarTreeIndexSettings.DEFAULT_METRICS_LIST.get(context.getSettings()));
+            } else {
+                Set<MetricStat> metricSet = new LinkedHashSet<>();
+                for (String metricString : metricStrings) {
+                    metricSet.add(MetricStat.fromTypeName(metricString));
+                }
+                metricTypes = new ArrayList<>(metricSet);
+            }
+            return new Metric(name, metricTypes);
+        }
+
+        @Override
+        protected List<Parameter<?>> getParameters() {
+            return List.of(config);
+        }
+
+        private static boolean isBuilderAllowedForDimension(Mapper.Builder builder) {
+            return ALLOWED_DIMENSION_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder));
+        }
+
+        private static boolean isBuilderAllowedForMetric(Mapper.Builder builder) {
+            return ALLOWED_METRIC_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder));
+        }
+
+        private Optional<Mapper.Builder> findMapperBuilderByName(String field, List<Mapper.Builder> mappersBuilders) {
+            return mappersBuilders.stream().filter(builder -> builder.name().equals(field)).findFirst();
+        }
+
+        public Builder(String name, ObjectMapper.Builder objBuilder) {
+            super(name);
+            this.objbuilder = objBuilder;
+        }
+
+        @Override
+        public ParametrizedFieldMapper build(BuilderContext context) {
+            StarTreeFieldType type = new StarTreeFieldType(name, this.config.get());
+            return new StarTreeMapper(name, type, this, objbuilder);
+        }
+    }
+
+    private static StarTreeMapper toType(FieldMapper in) {
+        return (StarTreeMapper) in;
+    }
+
+    /**
+     * Concrete parse for star tree type
+     *
+     * @opensearch.internal
+     */
+    public static class TypeParser implements Mapper.TypeParser {
+
+        /**
+         * default constructor of VectorFieldMapper.TypeParser
+         */
+        public TypeParser() {}
+
+        @Override
+        public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext context) throws MapperParsingException {
+            Builder builder = new StarTreeMapper.Builder(name, null);
+            builder.parse(name, context, node);
+            return builder;
+        }
+
+        @Override
+        public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext context, ObjectMapper.Builder objBuilder)
+            throws MapperParsingException {
+            Builder builder = new StarTreeMapper.Builder(name, objBuilder);
+            builder.parse(name, context, node);
+            return builder;
+        }
+    }
+
+    private final StarTreeField starTreeField;
+
+    private final ObjectMapper.Builder objBuilder;
+
+    protected StarTreeMapper(String simpleName, StarTreeFieldType type, Builder builder, ObjectMapper.Builder objbuilder) {
+        super(simpleName, type, MultiFields.empty(), CopyTo.empty());
+        this.starTreeField = builder.config.get();
+        this.objBuilder = objbuilder;
+    }
+
+    @Override
+    public StarTreeFieldType fieldType() {
+        return (StarTreeFieldType) super.fieldType();
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context) {
+        throw new MapperParsingException(
+            String.format(
+                Locale.ROOT,
+                "Field [%s] is a star tree field and cannot be added inside a document. Use the index API request parameters.",
+                name()
+            )
+        );
+    }
+
+    /**
+     * Star tree mapped field type containing dimensions, metrics, star tree specs
+     *
+     * @opensearch.experimental
+     */
+    @ExperimentalApi
+    public static final class StarTreeFieldType extends CompositeDataCubeFieldType {
+
+        private final StarTreeFieldConfiguration starTreeConfig;
+
+        public StarTreeFieldType(String name, StarTreeField starTreeField) {
+            super(name, starTreeField.getDimensionsOrder(), starTreeField.getMetrics(), CompositeFieldType.STAR_TREE);
+            this.starTreeConfig = starTreeField.getStarTreeConfig();
+        }
+
+        public StarTreeFieldConfiguration getStarTreeConfig() {
+            return starTreeConfig;
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
+            // TODO : evaluate later
+            throw new UnsupportedOperationException("Cannot fetch values for star tree field [" + name() + "].");
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            // TODO : evaluate later
+            throw new UnsupportedOperationException("Cannot perform terms query on star tree field [" + name() + "].");
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java
index 033b163bb0d67..f7e52ce9fc1ae 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java
@@ -70,6 +70,7 @@
 import org.opensearch.index.mapper.RoutingFieldMapper;
 import org.opensearch.index.mapper.SeqNoFieldMapper;
 import org.opensearch.index.mapper.SourceFieldMapper;
+import org.opensearch.index.mapper.StarTreeMapper;
 import org.opensearch.index.mapper.TextFieldMapper;
 import org.opensearch.index.mapper.VersionFieldMapper;
 import org.opensearch.index.mapper.WildcardFieldMapper;
@@ -174,6 +175,7 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
         mappers.put(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser());
         mappers.put(DerivedFieldMapper.CONTENT_TYPE, DerivedFieldMapper.PARSER);
         mappers.put(WildcardFieldMapper.CONTENT_TYPE, WildcardFieldMapper.PARSER);
+        mappers.put(StarTreeMapper.CONTENT_TYPE, new StarTreeMapper.TypeParser());
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
             for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index a7d879fc06981..902ca95643625 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -106,6 +106,7 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.AnalysisRegistry;
 import org.opensearch.index.cache.request.ShardRequestCache;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.engine.CommitStats;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.engine.EngineConfigFactory;
@@ -356,6 +357,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private volatile TimeValue clusterDefaultRefreshInterval;
     private final SearchRequestStats searchRequestStats;
     private final FileCache fileCache;
+    private final CompositeIndexSettings compositeIndexSettings;
 
     @Override
     protected void doStart() {
@@ -391,7 +393,8 @@ public IndicesService(
         RecoverySettings recoverySettings,
         CacheService cacheService,
         RemoteStoreSettings remoteStoreSettings,
-        FileCache fileCache
+        FileCache fileCache,
+        CompositeIndexSettings compositeIndexSettings
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -498,6 +501,7 @@ protected void closeInternal() {
             .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate);
         this.recoverySettings = recoverySettings;
         this.remoteStoreSettings = remoteStoreSettings;
+        this.compositeIndexSettings = compositeIndexSettings;
         this.fileCache = fileCache;
     }
 
@@ -558,6 +562,7 @@ public IndicesService(
             recoverySettings,
             cacheService,
             remoteStoreSettings,
+            null,
             null
         );
     }
@@ -939,7 +944,8 @@ private synchronized IndexService createIndexService(
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
             recoveryStateFactories,
-            fileCache
+            fileCache,
+            compositeIndexSettings
         );
         for (IndexingOperationListener operationListener : indexingOperationListeners) {
             indexModule.addIndexOperationListener(operationListener);
@@ -1030,7 +1036,8 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
             recoveryStateFactories,
-            fileCache
+            fileCache,
+            compositeIndexSettings
         );
         pluginsService.onIndexModule(indexModule);
         return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService);
@@ -2098,4 +2105,8 @@ private TimeValue getClusterDefaultRefreshInterval() {
     public RemoteStoreSettings getRemoteStoreSettings() {
         return this.remoteStoreSettings;
     }
+
+    public CompositeIndexSettings getCompositeIndexSettings() {
+        return this.compositeIndexSettings;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 505c9264d62bb..85ef547e27787 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -149,6 +149,7 @@
 import org.opensearch.index.IndexingPressureService;
 import org.opensearch.index.SegmentReplicationStatsTracker;
 import org.opensearch.index.analysis.AnalysisRegistry;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.engine.EngineFactory;
 import org.opensearch.index.recovery.RemoteStoreRestoreService;
 import org.opensearch.index.remote.RemoteIndexPathUploader;
@@ -834,6 +835,7 @@ protected Node(
             final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings());
 
             final RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, settingsModule.getClusterSettings());
+            final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(settings, settingsModule.getClusterSettings());
 
             final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(
                 repositoriesServiceReference::get,
@@ -874,7 +876,8 @@ protected Node(
                 recoverySettings,
                 cacheService,
                 remoteStoreSettings,
-                fileCache
+                fileCache,
+                compositeIndexSettings
             );
 
             final IngestService ingestService = new IngestService(
diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
index b10a7d8155056..504bc622ec12e 100644
--- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
@@ -33,6 +33,8 @@
 package org.opensearch.index.mapper;
 
 import org.opensearch.common.compress.CompressedXContent;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
@@ -46,6 +48,7 @@
 import java.io.IOException;
 import java.util.Collection;
 
+import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX;
 import static org.hamcrest.Matchers.containsString;
 
 public class ObjectMapperTests extends OpenSearchSingleNodeTestCase {
@@ -487,6 +490,76 @@ public void testDerivedFields() throws Exception {
         assertEquals("date", mapper.typeName());
     }
 
+    public void testCompositeFields() throws Exception {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("tweet")
+            .startObject("composite")
+            .startObject("startree")
+            .field("type", "star_tree")
+            .startObject("config")
+            .startArray("ordered_dimensions")
+            .startObject()
+            .field("name", "@timestamp")
+            .endObject()
+            .startObject()
+            .field("name", "status")
+            .endObject()
+            .endArray()
+            .startArray("metrics")
+            .startObject()
+            .field("name", "status")
+            .endObject()
+            .startObject()
+            .field("name", "metric_field")
+            .endObject()
+            .endArray()
+            .endObject()
+            .endObject()
+            .endObject()
+            .startObject("properties")
+            .startObject("@timestamp")
+            .field("type", "date")
+            .endObject()
+            .startObject("status")
+            .field("type", "integer")
+            .endObject()
+            .startObject("metric_field")
+            .field("type", "integer")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .toString();
+
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> createIndex("invalid").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping))
+        );
+        assertEquals(
+            "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.star_tree.enabled feature flag in the JVM options",
+            ex.getMessage()
+        );
+
+        final Settings starTreeEnabledSettings = Settings.builder().put(STAR_TREE_INDEX, "true").build();
+        FeatureFlags.initializeFeatureFlags(starTreeEnabledSettings);
+
+        DocumentMapper documentMapper = createIndex("test").mapperService()
+            .documentMapperParser()
+            .parse("tweet", new CompressedXContent(mapping));
+
+        Mapper mapper = documentMapper.root().getMapper("startree");
+        assertTrue(mapper instanceof StarTreeMapper);
+        StarTreeMapper starTreeMapper = (StarTreeMapper) mapper;
+        assertEquals("star_tree", starTreeMapper.fieldType().typeName());
+        // Check that field in properties was parsed correctly as well
+        mapper = documentMapper.root().getMapper("@timestamp");
+        assertNotNull(mapper);
+        assertEquals("date", mapper.typeName());
+
+        FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java
new file mode 100644
index 0000000000000..3144b1b007924
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java
@@ -0,0 +1,767 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.Rounding;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
+import org.opensearch.index.compositeindex.CompositeIndexValidator;
+import org.opensearch.index.compositeindex.datacube.DateDimension;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.NumericDimension;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Tests for {@link StarTreeMapper}.
+ */
+public class StarTreeMapperTests extends MapperTestCase {
+
+    @Before
+    public void setup() {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.STAR_TREE_INDEX, true).build());
+    }
+
+    @After
+    public void teardown() {
+        FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
+    }
+
+    public void testValidStarTree() throws IOException {
+        MapperService mapperService = createMapperService(getExpandedMapping("status", "size"));
+        Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes();
+        for (CompositeMappedFieldType type : compositeFieldTypes) {
+            StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type;
+            assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField());
+            assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
+            DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
+            List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
+                Rounding.DateTimeUnit.DAY_OF_MONTH,
+                Rounding.DateTimeUnit.MONTH_OF_YEAR
+            );
+            assertEquals(expectedTimeUnits, dateDim.getIntervals());
+            assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
+            assertEquals("size", starTreeFieldType.getMetrics().get(0).getField());
+            List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.SUM, MetricStat.AVG);
+            assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
+            assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
+            assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
+            assertEquals(
+                new HashSet<>(Arrays.asList("@timestamp", "status")),
+                starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()
+            );
+        }
+    }
+
+    public void testValidStarTreeDefaults() throws IOException {
+        MapperService mapperService = createMapperService(getMinMapping());
+        Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes();
+        for (CompositeMappedFieldType type : compositeFieldTypes) {
+            StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type;
+            assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField());
+            assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
+            DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
+            List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
+                Rounding.DateTimeUnit.MINUTES_OF_HOUR,
+                Rounding.DateTimeUnit.HOUR_OF_DAY
+            );
+            assertEquals(expectedTimeUnits, dateDim.getIntervals());
+            assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
+            assertEquals("status", starTreeFieldType.getMetrics().get(0).getField());
+            List<MetricStat> expectedMetrics = Arrays.asList(
+                MetricStat.AVG,
+                MetricStat.COUNT,
+                MetricStat.SUM,
+                MetricStat.MAX,
+                MetricStat.MIN
+            );
+            assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
+            assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
+            assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
+            assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
+        }
+    }
+
+    public void testInvalidDim() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getExpandedMapping("invalid", "size"))
+        );
+        assertEquals("Failed to parse mapping [_doc]: unknown dimension field [invalid]", ex.getMessage());
+    }
+
+    public void testInvalidMetric() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getExpandedMapping("status", "invalid"))
+        );
+        assertEquals("Failed to parse mapping [_doc]: unknown metric field [invalid]", ex.getMessage());
+    }
+
+    public void testNoMetrics() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getMinMapping(false, true, false, false))
+        );
+        assertThat(
+            ex.getMessage(),
+            containsString("Failed to parse mapping [_doc]: metrics section is required for star tree field [startree]")
+        );
+    }
+
+    public void testInvalidParam() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getInvalidMapping(false, false, false, false, true))
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: Star tree mapping definition has unsupported parameters:  [invalid : {invalid=invalid}]",
+            ex.getMessage()
+        );
+    }
+
+    public void testNoDims() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getMinMapping(true, false, false, false))
+        );
+        assertThat(
+            ex.getMessage(),
+            containsString("Failed to parse mapping [_doc]: ordered_dimensions is required for star tree field [startree]")
+        );
+    }
+
+    public void testMissingDims() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getMinMapping(false, false, true, false))
+        );
+        assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [@timestamp]"));
+    }
+
+    public void testMissingMetrics() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getMinMapping(false, false, false, true))
+        );
+        assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown metric field [metric_field]"));
+    }
+
+    public void testInvalidMetricType() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getInvalidMapping(false, false, false, true))
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: non-numeric field type is associated with star tree metric [startree]",
+            ex.getMessage()
+        );
+    }
+
+    public void testInvalidDimType() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getInvalidMapping(false, false, true, false))
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: unsupported field type associated with dimension [@timestamp] as part of star tree field [startree]",
+            ex.getMessage()
+        );
+    }
+
+    public void testInvalidSkipDim() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getInvalidMapping(false, true, false, false))
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: [invalid] in skip_star_node_creation_for_dimensions should be part of ordered_dimensions",
+            ex.getMessage()
+        );
+    }
+
+    public void testInvalidSingleDim() {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getInvalidMapping(true, false, false, false))
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: Atleast two dimensions are required to build star tree index field [startree]",
+            ex.getMessage()
+        );
+    }
+
+    public void testMetric() {
+        List<MetricStat> m1 = new ArrayList<>();
+        m1.add(MetricStat.MAX);
+        Metric metric1 = new Metric("name", m1);
+        Metric metric2 = new Metric("name", m1);
+        assertEquals(metric1, metric2);
+        List<MetricStat> m2 = new ArrayList<>();
+        m2.add(MetricStat.MAX);
+        m2.add(MetricStat.COUNT);
+        metric2 = new Metric("name", m2);
+        assertNotEquals(metric1, metric2);
+
+        assertEquals(MetricStat.COUNT, MetricStat.fromTypeName("count"));
+        assertEquals(MetricStat.MAX, MetricStat.fromTypeName("max"));
+        assertEquals(MetricStat.MIN, MetricStat.fromTypeName("min"));
+        assertEquals(MetricStat.SUM, MetricStat.fromTypeName("sum"));
+        assertEquals(MetricStat.AVG, MetricStat.fromTypeName("avg"));
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> MetricStat.fromTypeName("invalid"));
+        assertEquals("Invalid metric stat: invalid", ex.getMessage());
+    }
+
+    public void testDimensions() {
+        List<Rounding.DateTimeUnit> d1CalendarIntervals = new ArrayList<>();
+        d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY);
+        DateDimension d1 = new DateDimension("name", d1CalendarIntervals);
+        DateDimension d2 = new DateDimension("name", d1CalendarIntervals);
+        assertEquals(d1, d2);
+        d2 = new DateDimension("name1", d1CalendarIntervals);
+        assertNotEquals(d1, d2);
+        List<Rounding.DateTimeUnit> d2CalendarIntervals = new ArrayList<>();
+        d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY);
+        d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY);
+        d2 = new DateDimension("name", d2CalendarIntervals);
+        assertNotEquals(d1, d2);
+        NumericDimension n1 = new NumericDimension("name");
+        NumericDimension n2 = new NumericDimension("name");
+        assertEquals(n1, n2);
+        n2 = new NumericDimension("name1");
+        assertNotEquals(n1, n2);
+    }
+
+    public void testStarTreeField() {
+        List<MetricStat> m1 = new ArrayList<>();
+        m1.add(MetricStat.MAX);
+        Metric metric1 = new Metric("name", m1);
+        List<Rounding.DateTimeUnit> d1CalendarIntervals = new ArrayList<>();
+        d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY);
+        DateDimension d1 = new DateDimension("name", d1CalendarIntervals);
+        NumericDimension n1 = new NumericDimension("numeric");
+        NumericDimension n2 = new NumericDimension("name1");
+
+        List<Metric> metrics = List.of(metric1);
+        List<Dimension> dims = List.of(d1, n2);
+        StarTreeFieldConfiguration config = new StarTreeFieldConfiguration(
+            100,
+            Set.of("name"),
+            StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP
+        );
+
+        StarTreeField field1 = new StarTreeField("starTree", dims, metrics, config);
+        StarTreeField field2 = new StarTreeField("starTree", dims, metrics, config);
+        assertEquals(field1, field2);
+
+        dims = List.of(d1, n2, n1);
+        field2 = new StarTreeField("starTree", dims, metrics, config);
+        assertNotEquals(field1, field2);
+
+        dims = List.of(d1, n2);
+        metrics = List.of(metric1, metric1);
+        field2 = new StarTreeField("starTree", dims, metrics, config);
+        assertNotEquals(field1, field2);
+
+        dims = List.of(d1, n2);
+        metrics = List.of(metric1);
+        StarTreeFieldConfiguration config1 = new StarTreeFieldConfiguration(
+            1000,
+            Set.of("name"),
+            StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP
+        );
+        field2 = new StarTreeField("starTree", dims, metrics, config1);
+        assertNotEquals(field1, field2);
+
+        config1 = new StarTreeFieldConfiguration(100, Set.of("name", "field2"), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP);
+        field2 = new StarTreeField("starTree", dims, metrics, config1);
+        assertNotEquals(field1, field2);
+
+        config1 = new StarTreeFieldConfiguration(100, Set.of("name"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP);
+        field2 = new StarTreeField("starTree", dims, metrics, config1);
+        assertNotEquals(field1, field2);
+
+        field2 = new StarTreeField("starTree", dims, metrics, config);
+        assertEquals(field1, field2);
+    }
+
+    public void testValidations() throws IOException {
+        MapperService mapperService = createMapperService(getExpandedMapping("status", "size"));
+        Settings settings = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), true).build();
+        CompositeIndexSettings enabledCompositeIndexSettings = new CompositeIndexSettings(
+            settings,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        CompositeIndexValidator.validate(mapperService, enabledCompositeIndexSettings, mapperService.getIndexSettings());
+        settings = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), false).build();
+        CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(
+            settings,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        MapperService finalMapperService = mapperService;
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> CompositeIndexValidator.validate(finalMapperService, compositeIndexSettings, finalMapperService.getIndexSettings())
+        );
+        assertEquals(
+            "star tree index cannot be created, enable it using [indices.composite_index.star_tree.enabled] setting",
+            ex.getMessage()
+        );
+
+        MapperService mapperServiceInvalid = createMapperService(getInvalidMappingWithDv(false, false, false, true));
+        ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> CompositeIndexValidator.validate(
+                mapperServiceInvalid,
+                enabledCompositeIndexSettings,
+                mapperServiceInvalid.getIndexSettings()
+            )
+        );
+        assertEquals(
+            "Aggregations not supported for the metrics field [metric_field] with field type [integer] as part of star tree field",
+            ex.getMessage()
+        );
+
+        MapperService mapperServiceInvalidDim = createMapperService(getInvalidMappingWithDv(false, false, true, false));
+        ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> CompositeIndexValidator.validate(
+                mapperServiceInvalidDim,
+                enabledCompositeIndexSettings,
+                mapperServiceInvalidDim.getIndexSettings()
+            )
+        );
+        assertEquals(
+            "Aggregations not supported for the dimension field [@timestamp] with field type [date] as part of star tree field",
+            ex.getMessage()
+        );
+
+        MapperParsingException mapperParsingExceptionex = expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(getMinMappingWith2StarTrees())
+        );
+        assertEquals(
+            "Failed to parse mapping [_doc]: Composite fields cannot have more than [1] fields",
+            mapperParsingExceptionex.getMessage()
+        );
+    }
+
+    private XContentBuilder getExpandedMapping(String dim, String metric) throws IOException {
+        return topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("startree");
+            b.field("type", "star_tree");
+            b.startObject("config");
+            b.field("max_leaf_docs", 100);
+            b.startArray("skip_star_node_creation_for_dimensions");
+            {
+                b.value("@timestamp");
+                b.value("status");
+            }
+            b.endArray();
+            b.startArray("ordered_dimensions");
+            b.startObject();
+            b.field("name", "@timestamp");
+            b.startArray("calendar_intervals");
+            b.value("day");
+            b.value("month");
+            b.endArray();
+            b.endObject();
+            b.startObject();
+            b.field("name", dim);
+            b.endObject();
+            b.endArray();
+            b.startArray("metrics");
+            b.startObject();
+            b.field("name", metric);
+            b.startArray("stats");
+            b.value("sum");
+            b.value("avg");
+            b.endArray();
+            b.endObject();
+            b.endArray();
+            b.endObject();
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("@timestamp");
+            b.field("type", "date");
+            b.endObject();
+            b.startObject("status");
+            b.field("type", "integer");
+            b.endObject();
+            b.startObject("size");
+            b.field("type", "integer");
+            b.endObject();
+            b.endObject();
+        });
+    }
+
+    private XContentBuilder getMinMapping() throws IOException {
+        return getMinMapping(false, false, false, false);
+    }
+
+    private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetrics, boolean missingDim, boolean missingMetric)
+        throws IOException {
+        return topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("startree");
+            b.field("type", "star_tree");
+            b.startObject("config");
+            if (!isEmptyDims) {
+                b.startArray("ordered_dimensions");
+                b.startObject();
+                b.field("name", "@timestamp");
+                b.endObject();
+                b.startObject();
+                b.field("name", "status");
+                b.endObject();
+                b.endArray();
+            }
+            if (!isEmptyMetrics) {
+                b.startArray("metrics");
+                b.startObject();
+                b.field("name", "status");
+                b.endObject();
+                b.startObject();
+                b.field("name", "metric_field");
+                b.endObject();
+                b.endArray();
+            }
+            b.endObject();
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            if (!missingDim) {
+                b.startObject("@timestamp");
+                b.field("type", "date");
+                b.endObject();
+            }
+            b.startObject("status");
+            b.field("type", "integer");
+            b.endObject();
+            if (!missingMetric) {
+                b.startObject("metric_field");
+                b.field("type", "integer");
+                b.endObject();
+            }
+            b.endObject();
+        });
+    }
+
+    private XContentBuilder getMinMappingWith2StarTrees() throws IOException {
+        return topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("startree");
+            b.field("type", "star_tree");
+            b.startObject("config");
+
+            b.startArray("ordered_dimensions");
+            b.startObject();
+            b.field("name", "@timestamp");
+            b.endObject();
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.endArray();
+
+            b.startArray("metrics");
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.startObject();
+            b.field("name", "metric_field");
+            b.endObject();
+            b.endArray();
+
+            b.endObject();
+            b.endObject();
+
+            b.startObject("startree1");
+            b.field("type", "star_tree");
+            b.startObject("config");
+
+            b.startArray("ordered_dimensions");
+            b.startObject();
+            b.field("name", "@timestamp");
+            b.endObject();
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.endArray();
+
+            b.startArray("metrics");
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.startObject();
+            b.field("name", "metric_field");
+            b.endObject();
+            b.endArray();
+
+            b.endObject();
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("@timestamp");
+            b.field("type", "date");
+            b.endObject();
+            b.startObject("status");
+            b.field("type", "integer");
+            b.endObject();
+            b.startObject("metric_field");
+            b.field("type", "integer");
+            b.endObject();
+
+            b.endObject();
+        });
+    }
+
+    private XContentBuilder getInvalidMapping(
+        boolean singleDim,
+        boolean invalidSkipDims,
+        boolean invalidDimType,
+        boolean invalidMetricType,
+        boolean invalidParam
+    ) throws IOException {
+        return topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("startree");
+            b.field("type", "star_tree");
+            b.startObject("config");
+
+            b.startArray("skip_star_node_creation_for_dimensions");
+            {
+                if (invalidSkipDims) {
+                    b.value("invalid");
+                }
+                b.value("status");
+            }
+            b.endArray();
+            if (invalidParam) {
+                b.startObject("invalid");
+                b.field("invalid", "invalid");
+                b.endObject();
+            }
+            b.startArray("ordered_dimensions");
+            if (!singleDim) {
+                b.startObject();
+                b.field("name", "@timestamp");
+                b.endObject();
+            }
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.endArray();
+            b.startArray("metrics");
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.startObject();
+            b.field("name", "metric_field");
+            b.endObject();
+            b.endArray();
+            b.endObject();
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("@timestamp");
+            if (!invalidDimType) {
+                b.field("type", "date");
+            } else {
+                b.field("type", "keyword");
+            }
+            b.endObject();
+
+            b.startObject("status");
+            b.field("type", "integer");
+            b.endObject();
+            b.startObject("metric_field");
+            if (invalidMetricType) {
+                b.field("type", "date");
+            } else {
+                b.field("type", "integer");
+            }
+            b.endObject();
+            b.endObject();
+        });
+    }
+
+    private XContentBuilder getInvalidMappingWithDv(
+        boolean singleDim,
+        boolean invalidSkipDims,
+        boolean invalidDimType,
+        boolean invalidMetricType
+    ) throws IOException {
+        return topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("startree");
+            b.field("type", "star_tree");
+            b.startObject("config");
+
+            b.startArray("skip_star_node_creation_for_dimensions");
+            {
+                if (invalidSkipDims) {
+                    b.value("invalid");
+                }
+                b.value("status");
+            }
+            b.endArray();
+            b.startArray("ordered_dimensions");
+            if (!singleDim) {
+                b.startObject();
+                b.field("name", "@timestamp");
+                b.endObject();
+            }
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.endArray();
+            b.startArray("metrics");
+            b.startObject();
+            b.field("name", "status");
+            b.endObject();
+            b.startObject();
+            b.field("name", "metric_field");
+            b.endObject();
+            b.endArray();
+            b.endObject();
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("@timestamp");
+            if (!invalidDimType) {
+                b.field("type", "date");
+                b.field("doc_values", "true");
+            } else {
+                b.field("type", "date");
+                b.field("doc_values", "false");
+            }
+            b.endObject();
+
+            b.startObject("status");
+            b.field("type", "integer");
+            b.endObject();
+            b.startObject("metric_field");
+            if (invalidMetricType) {
+                b.field("type", "integer");
+                b.field("doc_values", "false");
+            } else {
+                b.field("type", "integer");
+                b.field("doc_values", "true");
+            }
+            b.endObject();
+            b.endObject();
+        });
+    }
+
+    private XContentBuilder getInvalidMapping(boolean singleDim, boolean invalidSkipDims, boolean invalidDimType, boolean invalidMetricType)
+        throws IOException {
+        return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false);
+    }
+
+    protected boolean supportsOrIgnoresBoost() {
+        return false;
+    }
+
+    protected boolean supportsMeta() {
+        return false;
+    }
+
+    @Override
+    protected void assertExistsQuery(MapperService mapperService) {}
+
+    // Overriding fieldMapping to make it create composite mappings by default.
+    // This way, the parent tests are checking the right behavior for this Mapper.
+    @Override
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+        return topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("startree");
+            buildField.accept(b);
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("size");
+            b.field("type", "integer");
+            b.endObject();
+            b.startObject("status");
+            b.field("type", "integer");
+            b.endObject();
+            b.endObject();
+        });
+    }
+
+    @Override
+    public void testEmptyName() {
+        MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("");
+            minimalMapping(b);
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("size");
+            b.field("type", "integer");
+            b.endObject();
+            b.startObject("status");
+            b.field("type", "integer");
+            b.endObject();
+            b.endObject();
+        })));
+        assertThat(e.getMessage(), containsString("name cannot be empty string"));
+        assertParseMinimalWarnings();
+    }
+
+    @Override
+    protected void minimalMapping(XContentBuilder b) throws IOException {
+        b.field("type", "star_tree");
+        b.startObject("config");
+        b.startArray("ordered_dimensions");
+        b.startObject();
+        b.field("name", "size");
+        b.endObject();
+        b.startObject();
+        b.field("name", "status");
+        b.endObject();
+        b.endArray();
+        b.startArray("metrics");
+        b.startObject();
+        b.field("name", "status");
+        b.endObject();
+        b.endArray();
+        b.endObject();
+    }
+
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {}
+
+    @Override
+    protected void registerParameters(ParameterChecker checker) throws IOException {
+
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java
index dc5954907a4fa..01a4005255f29 100644
--- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java
@@ -174,7 +174,7 @@ protected static void assertNoDocValuesField(ParseContext.Document doc, String f
         }
     }
 
-    public final void testEmptyName() {
+    public void testEmptyName() {
         MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping(b -> {
             b.startObject("");
             minimalMapping(b);
diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
index 28323a94af721..544fb100a17bf 100644
--- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
@@ -115,6 +115,7 @@
 import org.opensearch.index.mapper.ObjectMapper.Nested;
 import org.opensearch.index.mapper.RangeFieldMapper;
 import org.opensearch.index.mapper.RangeType;
+import org.opensearch.index.mapper.StarTreeMapper;
 import org.opensearch.index.mapper.TextFieldMapper;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.shard.IndexShard;
@@ -201,6 +202,7 @@ public abstract class AggregatorTestCase extends OpenSearchTestCase {
         denylist.add(CompletionFieldMapper.CONTENT_TYPE); // TODO support completion
         denylist.add(FieldAliasMapper.CONTENT_TYPE); // TODO support alias
         denylist.add(DerivedFieldMapper.CONTENT_TYPE); // TODO support derived fields
+        denylist.add(StarTreeMapper.CONTENT_TYPE); // TODO evaluate support for star tree fields
         TYPE_TEST_DENYLIST = denylist;
     }
 

From 8904557e835082b85267eccbfd78daf27ba06256 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Jul 2024 18:51:29 -0400
Subject: [PATCH 065/113] Bump com.microsoft.azure:msal4j from 1.15.1 to 1.16.0
 in /plugins/repository-azure (#14610)

* Bump com.microsoft.azure:msal4j in /plugins/repository-azure

Bumps [com.microsoft.azure:msal4j](https://github.com/AzureAD/microsoft-authentication-library-for-java) from 1.15.1 to 1.16.0.
- [Release notes](https://github.com/AzureAD/microsoft-authentication-library-for-java/releases)
- [Changelog](https://github.com/AzureAD/microsoft-authentication-library-for-java/blob/dev/changelog.txt)
- [Commits](https://github.com/AzureAD/microsoft-authentication-library-for-java/compare/v1.15.1...v1.16.0)

---
updated-dependencies:
- dependency-name: com.microsoft.azure:msal4j
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                             | 1 +
 plugins/repository-azure/build.gradle                    | 2 +-
 plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1 | 1 -
 plugins/repository-azure/licenses/msal4j-1.16.0.jar.sha1 | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/msal4j-1.16.0.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e9470d9bb4727..a8e8f120ddd2f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `opentelemetry` from 1.36.0 to 1.39.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457))
 - Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506))
 - Bump `com.azure:azure-storage-common` from 12.21.2 to 12.25.1 ([#14517](https://github.com/opensearch-project/OpenSearch/pull/14517))
+- Bump `com.microsoft.azure:msal4j` from 1.15.1 to 1.16.0 ([#14610](https://github.com/opensearch-project/OpenSearch/pull/14610))
 
 ### Changed
 - [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187))
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index f88d291a8eb4a..13b711019ff2a 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -61,7 +61,7 @@ dependencies {
   // Start of transitive dependencies for azure-identity
   api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
-  api 'com.microsoft.azure:msal4j:1.15.1'
+  api 'com.microsoft.azure:msal4j:1.16.0'
   api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
   api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api 'com.nimbusds:content-type:2.3'
diff --git a/plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1
deleted file mode 100644
index 797f21d0d4995..0000000000000
--- a/plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cd1daa94b81bd97153536b661c31295f99cbb8e7
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-1.16.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.16.0.jar.sha1
new file mode 100644
index 0000000000000..29fe5022a1570
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-1.16.0.jar.sha1
@@ -0,0 +1 @@
+708a0a986ed091054f1c08866712e5b41aec6700
\ No newline at end of file

From f9512db4e4f773b16384649bfd4006b6865006b1 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <peter.alfonsi@gmail.com>
Date: Mon, 1 Jul 2024 15:54:39 -0700
Subject: [PATCH 066/113] [Bugfix] Fix ICacheKeySerializerTests flakiness
 (#14564)

* Fix testInvalidInput flakiness

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

* Addressed andrross's comment

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

* rerun security check

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

---------

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
Co-authored-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/serializer/ICacheKeySerializerTests.java  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/server/src/test/java/org/opensearch/common/cache/serializer/ICacheKeySerializerTests.java b/server/src/test/java/org/opensearch/common/cache/serializer/ICacheKeySerializerTests.java
index 7713fdf1d0adc..4b0fc3d2a7366 100644
--- a/server/src/test/java/org/opensearch/common/cache/serializer/ICacheKeySerializerTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/serializer/ICacheKeySerializerTests.java
@@ -43,10 +43,9 @@ public void testInvalidInput() throws Exception {
         ICacheKeySerializer<BytesReference> serializer = new ICacheKeySerializer<>(keySer);
 
         Random rand = Randomness.get();
-        byte[] randomInput = new byte[1000];
-        rand.nextBytes(randomInput);
-
-        assertThrows(OpenSearchException.class, () -> serializer.deserialize(randomInput));
+        // The first thing the serializer reads is a VInt for the number of dimensions.
+        // This is an invalid input for StreamInput.readVInt(), so we are guaranteed to have an exception
+        assertThrows(OpenSearchException.class, () -> serializer.deserialize(new byte[] { -1, -1, -1, -1, -1 }));
     }
 
     public void testDimNumbers() throws Exception {

From f1f4f89e4ccc9bc3c30717a0c4645b673c6f88ca Mon Sep 17 00:00:00 2001
From: Vatsal <36672090+imvtsl@users.noreply.github.com>
Date: Tue, 2 Jul 2024 09:40:14 -0700
Subject: [PATCH 067/113] Correct typo in method name (#14621)

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>
---
 .../src/test/java/org/opensearch/index/get/GetResultTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/test/java/org/opensearch/index/get/GetResultTests.java b/server/src/test/java/org/opensearch/index/get/GetResultTests.java
index 2001bb84454cd..ef8c48f2753c7 100644
--- a/server/src/test/java/org/opensearch/index/get/GetResultTests.java
+++ b/server/src/test/java/org/opensearch/index/get/GetResultTests.java
@@ -224,7 +224,7 @@ public void testEqualsAndHashcode() {
         );
     }
 
-    public void testFomXContentEmbeddedFoundParsingException() throws IOException {
+    public void testFromXContentEmbeddedFoundParsingException() throws IOException {
         String json = "{\"_index\":\"foo\",\"_id\":\"bar\"}";
         try (
             XContentParser parser = JsonXContent.jsonXContent.createParser(

From 0742453ecc9b4f36ce72218d18d844baa9defe4e Mon Sep 17 00:00:00 2001
From: Siddhant Deshmukh <deshsid@amazon.com>
Date: Tue, 2 Jul 2024 13:18:39 -0700
Subject: [PATCH 068/113] Refactoring FilterPath.parse by using an iterative
 approach instead of recursion. (#14200)

* Refactor FilterPath parse function (#12067)
Signed-off-by: Robin Friedmann <robinfriedmann.rf@gmail.com>

* Implement unit tests for FilterPathTests (#12067)
Signed-off-by: Robin Friedmann <robinfriedmann.rf@gmail.com>

* Write warn log if Filter is empty; Add comments (#12067)
Signed-off-by: Robin Friedmann <robinfriedmann.rf@gmail.com>

* Add changelog

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Remove unnecessary log statement

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Remove unused logger

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Spotless apply

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Remove incorrect changelog

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

---------

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>
Co-authored-by: Robin Friedmann <robinfriedmann.rf@gmail.com>
---
 CHANGELOG.md                                  |  1 +
 .../core/xcontent/filtering/FilterPath.java   | 30 ++++++++-----------
 .../xcontent/filtering/FilterPathTests.java   | 17 +++++++++++
 3 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a8e8f120ddd2f..14804bbd5974a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix FuzzyQuery in keyword field will use IndexOrDocValuesQuery when both of index and doc_value are true ([#14378](https://github.com/opensearch-project/OpenSearch/pull/14378))
 - Fix file cache initialization ([#14004](https://github.com/opensearch-project/OpenSearch/pull/14004))
 - Handle NPE in GetResult if "found" field is missing ([#14552](https://github.com/opensearch-project/OpenSearch/pull/14552))
+- Refactoring FilterPath.parse by using an iterative approach ([#14200](https://github.com/opensearch-project/OpenSearch/pull/14200))
 
 ### Security
 
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/filtering/FilterPath.java b/libs/core/src/main/java/org/opensearch/core/xcontent/filtering/FilterPath.java
index 5389538a8c7dd..b8da9787165f8 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/filtering/FilterPath.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/filtering/FilterPath.java
@@ -46,7 +46,6 @@
 public class FilterPath {
 
     static final FilterPath EMPTY = new FilterPath();
-
     private final String filter;
     private final String segment;
     private final FilterPath next;
@@ -99,32 +98,29 @@ public static FilterPath[] compile(Set<String> filters) {
 
         List<FilterPath> paths = new ArrayList<>();
         for (String filter : filters) {
-            if (filter != null) {
+            if (filter != null && !filter.isEmpty()) {
                 filter = filter.trim();
                 if (filter.length() > 0) {
-                    paths.add(parse(filter, filter));
+                    paths.add(parse(filter));
                 }
             }
         }
         return paths.toArray(new FilterPath[0]);
     }
 
-    private static FilterPath parse(final String filter, final String segment) {
-        int end = segment.length();
-
-        for (int i = 0; i < end;) {
-            char c = segment.charAt(i);
+    private static FilterPath parse(final String filter) {
+        // Split the filter into segments using a regex
+        // that avoids splitting escaped dots.
+        String[] segments = filter.split("(?<!\\\\)\\.");
+        FilterPath next = EMPTY;
 
-            if (c == '.') {
-                String current = segment.substring(0, i).replaceAll("\\\\.", ".");
-                return new FilterPath(filter, current, parse(filter, segment.substring(i + 1)));
-            }
-            ++i;
-            if ((c == '\\') && (i < end) && (segment.charAt(i) == '.')) {
-                ++i;
-            }
+        for (int i = segments.length - 1; i >= 0; i--) {
+            // Replace escaped dots with actual dots in the current segment.
+            String segment = segments[i].replaceAll("\\\\.", ".");
+            next = new FilterPath(filter, segment, next);
         }
-        return new FilterPath(filter, segment.replaceAll("\\\\.", "."), EMPTY);
+
+        return next;
     }
 
     @Override
diff --git a/libs/core/src/test/java/org/opensearch/core/xcontent/filtering/FilterPathTests.java b/libs/core/src/test/java/org/opensearch/core/xcontent/filtering/FilterPathTests.java
index 0c5a17b70a956..d3191609f6119 100644
--- a/libs/core/src/test/java/org/opensearch/core/xcontent/filtering/FilterPathTests.java
+++ b/libs/core/src/test/java/org/opensearch/core/xcontent/filtering/FilterPathTests.java
@@ -35,6 +35,7 @@
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.util.HashSet;
 import java.util.Set;
 
 import static java.util.Collections.singleton;
@@ -369,4 +370,20 @@ public void testMultipleFilterPaths() {
         assertThat(filterPath.getSegment(), is(emptyString()));
         assertSame(filterPath, FilterPath.EMPTY);
     }
+
+    public void testCompileWithEmptyString() {
+        Set<String> filters = new HashSet<>();
+        filters.add("");
+        FilterPath[] filterPaths = FilterPath.compile(filters);
+        assertNotNull(filterPaths);
+        assertEquals(0, filterPaths.length);
+    }
+
+    public void testCompileWithNull() {
+        Set<String> filters = new HashSet<>();
+        filters.add(null);
+        FilterPath[] filterPaths = FilterPath.compile(filters);
+        assertNotNull(filterPaths);
+        assertEquals(0, filterPaths.length);
+    }
 }

From e82b432940be34941c999dd1fe5cdc9fed06f02b Mon Sep 17 00:00:00 2001
From: rishavz_sagar <rishavsagar4b1@gmail.com>
Date: Wed, 3 Jul 2024 15:25:51 +0530
Subject: [PATCH 069/113] Removing String format in
 RemoteStoreMigrationAllocationDecider to optimise performance(#14612)

Signed-off-by: RS146BIJAY <rishavsagar4b1@gmail.com>
---
 ...RemoteStoreMigrationAllocationDecider.java | 34 +++++++++----------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
index 4fc5fff805663..67fe4ea1dcb1b 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
@@ -44,8 +44,6 @@
 import org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
 import org.opensearch.node.remotestore.RemoteStoreNodeService.Direction;
 
-import java.util.Locale;
-
 /**
  * A new allocation decider for migration of document replication clusters to remote store backed clusters:
  * - For STRICT compatibility mode, the decision is always YES
@@ -101,7 +99,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
         if (migrationDirection.equals(Direction.NONE)) {
             // remote backed indices on docrep nodes and non remote backed indices on remote nodes are not allowed
             boolean isNoDecision = remoteSettingsBackedIndex ^ targetNode.isRemoteStoreNode();
-            String reason = String.format(Locale.ROOT, " for %sremote store backed index", remoteSettingsBackedIndex ? "" : "non ");
+            String reason = " for " + (remoteSettingsBackedIndex ? "" : "non ") + "remote store backed index";
             return allocation.decision(
                 isNoDecision ? Decision.NO : Decision.YES,
                 NAME,
@@ -114,11 +112,9 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             // check for remote store backed indices
             if (remoteSettingsBackedIndex && targetNode.isRemoteStoreNode() == false) {
                 // allocations and relocations must be to a remote node
-                String reason = String.format(
-                    Locale.ROOT,
-                    " because a remote store backed index's shard copy can only be %s to a remote node",
-                    ((shardRouting.assignedToNode() == false) ? "allocated" : "relocated")
-                );
+                String reason = new StringBuilder(" because a remote store backed index's shard copy can only be ").append(
+                    (shardRouting.assignedToNode() == false) ? "allocated" : "relocated"
+                ).append(" to a remote node").toString();
                 return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason));
             }
 
@@ -168,16 +164,18 @@ private Decision replicaShardDecision(ShardRouting replicaShardRouting, Discover
 
     // get detailed reason for the decision
     private String getDecisionDetails(boolean isYes, ShardRouting shardRouting, DiscoveryNode targetNode, String reason) {
-        return String.format(
-            Locale.ROOT,
-            "[%s migration_direction]: %s shard copy %s be %s to a %s node%s",
-            migrationDirection.direction,
-            (shardRouting.primary() ? "primary" : "replica"),
-            (isYes ? "can" : "can not"),
-            ((shardRouting.assignedToNode() == false) ? "allocated" : "relocated"),
-            (targetNode.isRemoteStoreNode() ? "remote" : "non-remote"),
-            reason
-        );
+        return new StringBuilder("[").append(migrationDirection.direction)
+            .append(" migration_direction]: ")
+            .append(shardRouting.primary() ? "primary" : "replica")
+            .append(" shard copy ")
+            .append(isYes ? "can" : "can not")
+            .append(" be ")
+            .append((shardRouting.assignedToNode() == false) ? "allocated" : "relocated")
+            .append(" to a ")
+            .append(targetNode.isRemoteStoreNode() ? "remote" : "non-remote")
+            .append(" node")
+            .append(reason)
+            .toString();
     }
 
 }

From 501a7024d18f7c7bc135b5e686835f0bcdf00d30 Mon Sep 17 00:00:00 2001
From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com>
Date: Wed, 3 Jul 2024 22:33:54 +0530
Subject: [PATCH 070/113] Clear templates before Adding; Use
 NamedWriteableAwareStreamInput for RemoteCustomMetadata; Correct the check
 for deciding upload of HashesOfConsistentSettings (#14513)

* Clear templates before Adding; Use NamedWriteableAwareStreamInput for RemoteCustomMetadata
* Correct the check for deciding upload of hashes of consistent settings

Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
---
 .../opensearch/cluster/metadata/Metadata.java |   1 +
 .../remote/RemoteClusterStateService.java     |   5 +-
 .../remote/model/RemoteCustomMetadata.java    |   5 +-
 .../cluster/metadata/MetadataTests.java       |  27 ++++
 .../RemoteClusterStateServiceTests.java       | 125 +++++++++++++++++-
 .../model/RemoteCustomMetadataTests.java      |  87 +++++++++++-
 6 files changed, 239 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
index a0ef8de07fbf2..e3f63b1c27b83 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
@@ -1287,6 +1287,7 @@ public Builder templates(Map<String, IndexTemplateMetadata> templates) {
         }
 
         public Builder templates(TemplatesMetadata templatesMetadata) {
+            this.templates.clear();
             this.templates.putAll(templatesMetadata.getTemplates());
             return this;
         }
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index ef14adeb42b68..74abe9cd257b4 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -356,7 +356,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
             && clusterState.getNodes().delta(previousClusterState.getNodes()).hasChanges();
         final boolean updateClusterBlocks = isPublicationEnabled && !clusterState.blocks().equals(previousClusterState.blocks());
         final boolean updateHashesOfConsistentSettings = isPublicationEnabled
-            || Metadata.isHashesOfConsistentSettingsEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
+            && Metadata.isHashesOfConsistentSettingsEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
 
         uploadedMetadataResults = writeMetadataInParallel(
             clusterState,
@@ -476,7 +476,8 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
         return manifestDetails;
     }
 
-    private UploadedMetadataResults writeMetadataInParallel(
+    // package private for testing
+    UploadedMetadataResults writeMetadataInParallel(
         ClusterState clusterState,
         List<IndexMetadata> indexToUpload,
         Map<String, IndexMetadata> prevIndexMetadataByName,
diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java
index 4c7069ee8be9e..ec5dfbec820d4 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java
@@ -12,6 +12,7 @@
 import org.opensearch.common.io.Streams;
 import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity;
 import org.opensearch.common.remote.BlobPathParameters;
+import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.compress.Compressor;
@@ -122,6 +123,8 @@ public UploadedMetadata getUploadedMetadata() {
 
     public static Custom readFrom(StreamInput streamInput, NamedWriteableRegistry namedWriteableRegistry, String customType)
         throws IOException {
-        return namedWriteableRegistry.getReader(Custom.class, customType).read(streamInput);
+        try (StreamInput in = new NamedWriteableAwareStreamInput(streamInput, namedWriteableRegistry)) {
+            return namedWriteableRegistry.getReader(Custom.class, customType).read(in);
+        }
     }
 }
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java
index 618fcb923bc60..a434a713f330b 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java
@@ -1482,6 +1482,33 @@ public void testIsSegmentReplicationDisabled() {
         assertFalse(metadata.isSegmentReplicationEnabled(indexName));
     }
 
+    public void testTemplatesMetadata() {
+        TemplatesMetadata templatesMetadata1 = TemplatesMetadata.builder()
+            .put(
+                IndexTemplateMetadata.builder("template_1")
+                    .patterns(Arrays.asList("bar-*", "foo-*"))
+                    .settings(Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build())
+                    .build()
+            )
+            .build();
+        Metadata metadata1 = Metadata.builder().templates(templatesMetadata1).build();
+        assertThat(metadata1.templates(), is(templatesMetadata1.getTemplates()));
+
+        TemplatesMetadata templatesMetadata2 = TemplatesMetadata.builder()
+            .put(
+                IndexTemplateMetadata.builder("template_2")
+                    .patterns(Arrays.asList("bar-*", "foo-*"))
+                    .settings(Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build())
+                    .build()
+            )
+            .build();
+
+        Metadata metadata2 = Metadata.builder(metadata1).templates(templatesMetadata2).build();
+
+        assertThat(metadata2.templates(), is(templatesMetadata2.getTemplates()));
+
+    }
+
     public static Metadata randomMetadata() {
         Metadata.Builder md = Metadata.builder()
             .put(buildIndexMetadata("index", "alias", randomBoolean() ? null : randomBoolean()).build(), randomBoolean())
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index c8fd982fec1e1..d983a4d8c4027 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -20,6 +20,7 @@
 import org.opensearch.cluster.metadata.IndexTemplateMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.TemplatesMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService;
@@ -92,6 +93,7 @@
 
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 import static java.util.stream.Collectors.toList;
 import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL;
@@ -111,6 +113,7 @@
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
@@ -118,6 +121,7 @@
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -518,11 +522,13 @@ public void testWriteIncrementalMetadataSuccess() throws IOException {
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(Collections.emptyList()).build();
 
         remoteClusterStateService.start();
-        final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata(
+        final RemoteClusterStateService rcssSpy = Mockito.spy(remoteClusterStateService);
+        final RemoteClusterStateManifestInfo manifestInfo = rcssSpy.writeIncrementalMetadata(
             previousClusterState,
             clusterState,
             previousManifest
-        ).getClusterMetadataManifest();
+        );
+        final ClusterMetadataManifest manifest = manifestInfo.getClusterMetadataManifest();
         final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename__2");
         final List<UploadedIndexMetadata> indices = List.of(uploadedIndexMetadata);
 
@@ -535,6 +541,24 @@ public void testWriteIncrementalMetadataSuccess() throws IOException {
             .previousClusterUUID("prev-cluster-uuid")
             .build();
 
+        Mockito.verify(rcssSpy)
+            .writeMetadataInParallel(
+                eq(clusterState),
+                eq(new ArrayList<IndexMetadata>(clusterState.metadata().indices().values())),
+                eq(Collections.singletonMap(indices.get(0).getIndexName(), null)),
+                eq(clusterState.metadata().customs()),
+                eq(true),
+                eq(true),
+                eq(true),
+                eq(false),
+                eq(false),
+                eq(false),
+                eq(Collections.emptyMap()),
+                eq(false),
+                eq(Collections.emptyList())
+            );
+
+        assertThat(manifestInfo.getManifestFileName(), notNullValue());
         assertThat(manifest.getIndices().size(), is(1));
         assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName()));
         assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID()));
@@ -543,6 +567,95 @@ public void testWriteIncrementalMetadataSuccess() throws IOException {
         assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion()));
         assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
+        assertThat(manifest.getHashesOfConsistentSettings(), nullValue());
+        assertThat(manifest.getDiscoveryNodesMetadata(), nullValue());
+        assertThat(manifest.getClusterBlocksMetadata(), nullValue());
+        assertThat(manifest.getClusterStateCustomMap(), anEmptyMap());
+        assertThat(manifest.getTransientSettingsMetadata(), nullValue());
+        assertThat(manifest.getTemplatesMetadata(), notNullValue());
+        assertThat(manifest.getCoordinationMetadata(), notNullValue());
+        assertThat(manifest.getCustomMetadataMap().size(), is(2));
+        assertThat(manifest.getIndicesRouting().size(), is(0));
+    }
+
+    public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws IOException {
+        publicationEnabled = true;
+        Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, publicationEnabled).build();
+        FeatureFlags.initializeFeatureFlags(nodeSettings);
+        remoteClusterStateService = new RemoteClusterStateService(
+            "test-node-id",
+            repositoriesServiceSupplier,
+            settings,
+            clusterService,
+            () -> 0L,
+            threadPool,
+            List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)),
+            writableRegistry()
+        );
+        final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        mockBlobStoreObjects();
+        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
+        final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
+            .build();
+
+        final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(Collections.emptyList()).build();
+
+        remoteClusterStateService.start();
+        final RemoteClusterStateService rcssSpy = Mockito.spy(remoteClusterStateService);
+        final RemoteClusterStateManifestInfo manifestInfo = rcssSpy.writeIncrementalMetadata(
+            previousClusterState,
+            clusterState,
+            previousManifest
+        );
+        final ClusterMetadataManifest manifest = manifestInfo.getClusterMetadataManifest();
+        final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename__2");
+        final List<UploadedIndexMetadata> indices = List.of(uploadedIndexMetadata);
+
+        final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
+            .indices(indices)
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .stateUUID("state-uuid")
+            .clusterUUID("cluster-uuid")
+            .previousClusterUUID("prev-cluster-uuid")
+            .build();
+
+        Mockito.verify(rcssSpy)
+            .writeMetadataInParallel(
+                eq(clusterState),
+                eq(new ArrayList<IndexMetadata>(clusterState.metadata().indices().values())),
+                eq(Collections.singletonMap(indices.get(0).getIndexName(), null)),
+                eq(clusterState.metadata().customs()),
+                eq(true),
+                eq(true),
+                eq(true),
+                eq(true),
+                eq(false),
+                eq(false),
+                eq(Collections.emptyMap()),
+                eq(true),
+                Mockito.anyList()
+            );
+
+        assertThat(manifestInfo.getManifestFileName(), notNullValue());
+        assertThat(manifest.getIndices().size(), is(1));
+        assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName()));
+        assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID()));
+        assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue());
+        assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm()));
+        assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion()));
+        assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
+        assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
+        assertThat(manifest.getHashesOfConsistentSettings(), notNullValue());
+        assertThat(manifest.getDiscoveryNodesMetadata(), notNullValue());
+        assertThat(manifest.getClusterBlocksMetadata(), nullValue());
+        assertThat(manifest.getClusterStateCustomMap(), anEmptyMap());
+        assertThat(manifest.getTransientSettingsMetadata(), nullValue());
+        assertThat(manifest.getTemplatesMetadata(), notNullValue());
+        assertThat(manifest.getCoordinationMetadata(), notNullValue());
+        assertThat(manifest.getCustomMetadataMap().size(), is(2));
+        assertThat(manifest.getIndicesRouting().size(), is(1));
     }
 
     /*
@@ -2012,7 +2125,9 @@ static ClusterState.Builder generateClusterStateWithOneIndex() {
             .build();
         final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
         final Settings settings = Settings.builder().put("mock-settings", true).build();
-        final TemplatesMetadata templatesMetadata = TemplatesMetadata.EMPTY_METADATA;
+        final TemplatesMetadata templatesMetadata = TemplatesMetadata.builder()
+            .put(IndexTemplateMetadata.builder("template1").settings(idxSettings).patterns(List.of("test*")).build())
+            .build();
         final CustomMetadata1 customMetadata1 = new CustomMetadata1("custom-metadata-1");
         return ClusterState.builder(ClusterName.DEFAULT)
             .version(1L)
@@ -2025,6 +2140,7 @@ static ClusterState.Builder generateClusterStateWithOneIndex() {
                     .coordinationMetadata(coordinationMetadata)
                     .persistentSettings(settings)
                     .templates(templatesMetadata)
+                    .hashesOfConsistentSettings(Map.of("key1", "value1", "key2", "value2"))
                     .putCustom(customMetadata1.getWriteableName(), customMetadata1)
                     .build()
             )
@@ -2032,7 +2148,8 @@ static ClusterState.Builder generateClusterStateWithOneIndex() {
     }
 
     static DiscoveryNodes nodesWithLocalNodeClusterManager() {
-        return DiscoveryNodes.builder().clusterManagerNodeId("cluster-manager-id").localNodeId("cluster-manager-id").build();
+        final DiscoveryNode localNode = new DiscoveryNode("cluster-manager-id", buildNewFakeTransportAddress(), Version.CURRENT);
+        return DiscoveryNodes.builder().clusterManagerNodeId("cluster-manager-id").localNodeId("cluster-manager-id").add(localNode).build();
     }
 
     private static class CustomMetadata1 extends TestCustomMetadata {
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java
index 1e28817be79f2..60cceb205f43d 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.gateway.remote.model;
 
+import org.opensearch.Version;
+import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.Metadata.Custom;
 import org.opensearch.common.blobstore.BlobPath;
@@ -16,13 +18,20 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.compress.Compressor;
 import org.opensearch.core.compress.NoneCompressor;
 import org.opensearch.core.index.Index;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata;
 import org.opensearch.gateway.remote.RemoteClusterStateUtils;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.translog.transfer.BlobStoreTransferService;
+import org.opensearch.persistent.PersistentTaskParams;
+import org.opensearch.persistent.PersistentTasksCustomMetadata;
+import org.opensearch.persistent.PersistentTasksCustomMetadata.Assignment;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
@@ -33,6 +42,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Objects;
 
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION;
 import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER;
@@ -216,24 +226,93 @@ public void testGetUploadedMetadata() throws IOException {
 
     public void testSerDe() throws IOException {
         Custom customMetadata = getCustomMetadata();
+        verifySerDe(customMetadata, IndexGraveyard.TYPE);
+    }
+
+    public void testSerDeForPersistentTasks() throws IOException {
+        Custom customMetadata = getPersistentTasksMetadata();
+        verifySerDe(customMetadata, PersistentTasksCustomMetadata.TYPE);
+    }
+
+    private void verifySerDe(Custom objectToUpload, String objectType) throws IOException {
         RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(
-            customMetadata,
-            IndexGraveyard.TYPE,
+            objectToUpload,
+            objectType,
             METADATA_VERSION,
             clusterUUID,
             compressor,
-            namedWriteableRegistry
+            customWritableRegistry()
         );
         try (InputStream inputStream = remoteObjectForUpload.serialize()) {
             remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath());
             assertThat(inputStream.available(), greaterThan(0));
             Custom readCustomMetadata = remoteObjectForUpload.deserialize(inputStream);
-            assertThat(readCustomMetadata, is(customMetadata));
+            assertThat(readCustomMetadata, is(objectToUpload));
         }
     }
 
+    private NamedWriteableRegistry customWritableRegistry() {
+        List<Entry> entries = ClusterModule.getNamedWriteables();
+        entries.add(new Entry(PersistentTaskParams.class, TestPersistentTaskParams.PARAM_NAME, TestPersistentTaskParams::new));
+        return new NamedWriteableRegistry(entries);
+    }
+
     public static Custom getCustomMetadata() {
         return IndexGraveyard.builder().addTombstone(new Index("test-index", "3q2423")).build();
     }
 
+    private static Custom getPersistentTasksMetadata() {
+        return PersistentTasksCustomMetadata.builder()
+            .addTask("_task_1", "testTaskName", new TestPersistentTaskParams("task param data"), new Assignment(null, "_reason"))
+            .build();
+    }
+
+    public static class TestPersistentTaskParams implements PersistentTaskParams {
+
+        private static final String PARAM_NAME = "testTaskName";
+
+        private final String data;
+
+        public TestPersistentTaskParams(String data) {
+            this.data = data;
+        }
+
+        public TestPersistentTaskParams(StreamInput in) throws IOException {
+            this(in.readString());
+        }
+
+        @Override
+        public String getWriteableName() {
+            return PARAM_NAME;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.V_2_13_0;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(data);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject().field("data_field", data);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            TestPersistentTaskParams that = (TestPersistentTaskParams) o;
+            return Objects.equals(data, that.data);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(data);
+        }
+    }
+
 }

From 58d1164f74e921a26b7a73b6185b38cc87bbc7a9 Mon Sep 17 00:00:00 2001
From: rishavz_sagar <rishavsagar4b1@gmail.com>
Date: Thu, 4 Jul 2024 11:50:47 +0530
Subject: [PATCH 071/113] Improve reroute performance by optimising
 List.removeAll in LocalShardsBalancer to filter remote search shard from
 relocation decision (#14613)

Signed-off-by: RS146BIJAY <rishavsagar4b1@gmail.com>
---
 .../allocator/LocalShardsBalancer.java        | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
index ec25d041bda43..6978c988fd648 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
@@ -32,7 +32,6 @@
 import org.opensearch.gateway.PriorityComparator;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -41,7 +40,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
@@ -779,15 +777,16 @@ void allocateUnassigned() {
          * if we allocate for instance (0, R, IDX1) we move the second replica to the secondary array and proceed with
          * the next replica. If we could not find a node to allocate (0,R,IDX1) we move all it's replicas to ignoreUnassigned.
          */
-        ShardRouting[] unassignedShards = unassigned.drain();
-        List<ShardRouting> allUnassignedShards = Arrays.stream(unassignedShards).collect(Collectors.toList());
-        List<ShardRouting> localUnassignedShards = allUnassignedShards.stream()
-            .filter(shard -> RoutingPool.LOCAL_ONLY.equals(RoutingPool.getShardPool(shard, allocation)))
-            .collect(Collectors.toList());
-        allUnassignedShards.removeAll(localUnassignedShards);
-        allUnassignedShards.forEach(shard -> routingNodes.unassigned().add(shard));
-        unassignedShards = localUnassignedShards.toArray(new ShardRouting[0]);
-        ShardRouting[] primary = unassignedShards;
+        List<ShardRouting> primaryList = new ArrayList<>();
+        for (ShardRouting shard : unassigned.drain()) {
+            if (RoutingPool.LOCAL_ONLY.equals(RoutingPool.getShardPool(shard, allocation))) {
+                primaryList.add(shard);
+            } else {
+                routingNodes.unassigned().add(shard);
+            }
+        }
+
+        ShardRouting[] primary = primaryList.toArray(new ShardRouting[0]);
         ShardRouting[] secondary = new ShardRouting[primary.length];
         int secondaryLength = 0;
         int primaryLength = primary.length;

From 74230b76a255d52b4aca711734864f7b6e4b314c Mon Sep 17 00:00:00 2001
From: Sachin Kale <sachinpkale@gmail.com>
Date: Fri, 5 Jul 2024 09:16:37 +0530
Subject: [PATCH 072/113] Fix assertion failure while deleting remote backed
 index (#14601)

Signed-off-by: Sachin Kale <kalsac@amazon.com>
---
 .../RemoteMigrationIndexMetadataUpdateIT.java   |  2 --
 .../opensearch/remotestore/RemoteStoreIT.java   | 16 +++++++++++++++-
 .../java/org/opensearch/index/IndexService.java | 17 +++++++++++++++--
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
index 6885d37c4aab0..216c104dfecc1 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
@@ -275,7 +275,6 @@ initalMetadataVersion < internalCluster().client()
      * After shard relocation completes, shuts down the docrep nodes and asserts remote
      * index settings are applied even when the index is in YELLOW state
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/13737")
     public void testIndexSettingsUpdatedEvenForMisconfiguredReplicas() throws Exception {
         internalCluster().startClusterManagerOnlyNode();
 
@@ -332,7 +331,6 @@ public void testIndexSettingsUpdatedEvenForMisconfiguredReplicas() throws Except
      * After shard relocation completes, restarts the docrep node holding extra replica shard copy
      * and asserts remote index settings are applied as soon as the docrep replica copy is unassigned
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/13871")
     public void testIndexSettingsUpdatedWhenDocrepNodeIsRestarted() throws Exception {
         internalCluster().startClusterManagerOnlyNode();
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
index 96d6338e5913b..194dce5f4a57a 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
@@ -65,7 +65,6 @@
 import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
 import static org.opensearch.index.shard.IndexShardTestCase.getTranslog;
 import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
-import static org.opensearch.test.OpenSearchTestCase.getShardLevelBlobPath;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.comparesEqualTo;
@@ -133,6 +132,21 @@ private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throw
         );
     }
 
+    public void testRemoteStoreIndexCreationAndDeletionWithReferencedStore() throws InterruptedException, ExecutionException {
+        String dataNode = internalCluster().startNodes(1).get(0);
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(0));
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        IndexShard indexShard = getIndexShard(dataNode, INDEX_NAME);
+
+        // Simulating a condition where store is already in use by increasing ref count, this helps in testing index
+        // deletion when refresh is in-progress.
+        indexShard.store().incRef();
+        assertAcked(client().admin().indices().prepareDelete(INDEX_NAME));
+        indexShard.store().decRef();
+    }
+
     public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataFlush() throws Exception {
         testPeerRecovery(1, true);
     }
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
index 1c0db0095bb98..12b02d3dbd6fa 100644
--- a/server/src/main/java/org/opensearch/index/IndexService.java
+++ b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -602,7 +602,21 @@ public synchronized IndexShard createShard(
                         this.indexSettings.getRemoteStorePathStrategy()
                     );
                 }
-                remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, lock, Store.OnClose.EMPTY, path);
+                // When an instance of Store is created, a shardlock is created which is released on closing the instance of store.
+                // Currently, we create 2 instances of store for remote store backed indices: store and remoteStore.
+                // As there can be only one shardlock acquired for a given shard, the lock is shared between store and remoteStore.
+                // This creates an issue when we are deleting the index as it results in closing both store and remoteStore.
+                // Sample test failure: https://github.com/opensearch-project/OpenSearch/issues/13871
+                // The following method provides ShardLock that is not maintained by NodeEnvironment.
+                // As part of https://github.com/opensearch-project/OpenSearch/issues/13075, we want to move away from keeping 2
+                // store instances.
+                ShardLock remoteStoreLock = new ShardLock(shardId) {
+                    @Override
+                    protected void closeInternal() {
+                        // Do nothing for shard lock on remote store
+                    }
+                };
+                remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, remoteStoreLock, Store.OnClose.EMPTY, path);
             } else {
                 // Disallow shards with remote store based settings to be created on non-remote store enabled nodes
                 // Even though we have `RemoteStoreMigrationAllocationDecider` in place to prevent something like this from happening at the
@@ -625,7 +639,6 @@ public synchronized IndexShard createShard(
             } else {
                 directory = directoryFactory.newDirectory(this.indexSettings, path);
             }
-
             store = new Store(
                 shardId,
                 this.indexSettings,

From f14b5c8c1f52acb4cfe0c088fed09ea075743d69 Mon Sep 17 00:00:00 2001
From: Craig Perkins <cwperx@amazon.com>
Date: Fri, 5 Jul 2024 11:59:40 -0400
Subject: [PATCH 073/113] Allow system index warning in
 OpenSearchRestTestCase.refreshAllIndices (#14635)

* Allow system index warning

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add to CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Address code review comments

Signed-off-by: Craig Perkins <cwperx@amazon.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
---
 CHANGELOG.md                                         |  1 +
 .../opensearch/test/rest/OpenSearchRestTestCase.java | 12 ++++++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 14804bbd5974a..1f7d1ea5b3d19 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Make the class CommunityIdProcessor final ([#14448](https://github.com/opensearch-project/OpenSearch/pull/14448))
 - Allow @InternalApi annotation on classes not meant to be constructed outside of the OpenSearch core ([#14575](https://github.com/opensearch-project/OpenSearch/pull/14575))
 - Add @InternalApi annotation to japicmp exclusions ([#14597](https://github.com/opensearch-project/OpenSearch/pull/14597))
+- Allow system index warning in OpenSearchRestTestCase.refreshAllIndices ([#14635](https://github.com/opensearch-project/OpenSearch/pull/14635))
 
 ### Deprecated
 
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java
index b7c31685bafa6..8c612d258f183 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java
@@ -708,11 +708,15 @@ protected void refreshAllIndices() throws IOException {
         requestOptions.setWarningsHandler(warnings -> {
             if (warnings.isEmpty()) {
                 return false;
-            } else if (warnings.size() > 1) {
-                return true;
-            } else {
-                return warnings.get(0).startsWith("this request accesses system indices:") == false;
             }
+            boolean allSystemIndexWarnings = true;
+            for (String warning : warnings) {
+                if (!warning.startsWith("this request accesses system indices:")) {
+                    allSystemIndexWarnings = false;
+                    break;
+                }
+            }
+            return !allSystemIndexWarnings;
         });
         refreshRequest.setOptions(requestOptions);
         client().performRequest(refreshRequest);

From f0c2fa6138f2b6c7a5eb5d9501fb8782271a3cd3 Mon Sep 17 00:00:00 2001
From: Bharathwaj G <bharath78910@gmail.com>
Date: Mon, 8 Jul 2024 19:31:58 +0530
Subject: [PATCH 074/113] Star tree codec changes (#14514)

---------
Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
---
 .../opensearch/index/codec/CodecService.java  |  16 ++-
 .../codec/composite/Composite99Codec.java     |  57 +++++++++
 .../composite/Composite99DocValuesFormat.java |  64 ++++++++++
 .../composite/Composite99DocValuesReader.java |  89 ++++++++++++++
 .../composite/Composite99DocValuesWriter.java | 114 ++++++++++++++++++
 .../composite/CompositeCodecFactory.java      |  42 +++++++
 .../codec/composite/CompositeIndexReader.java |  34 ++++++
 .../codec/composite/CompositeIndexValues.java |  21 ++++
 .../datacube/startree/StarTreeValues.java     |  35 ++++++
 .../datacube/startree/package-info.java       |  12 ++
 .../index/codec/composite/package-info.java   |  12 ++
 .../mapper/CompositeMappedFieldType.java      |   3 +
 .../index/mapper/MapperService.java           |   9 ++
 .../services/org.apache.lucene.codecs.Codec   |   1 +
 .../opensearch/index/codec/CodecTests.java    |  47 ++++++++
 .../StarTreeDocValuesFormatTests.java         | 110 +++++++++++++++++
 16 files changed, 662 insertions(+), 4 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesFormat.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/package-info.java
 create mode 100644 server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
 create mode 100644 server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java

diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java
index 67f38536a0d11..59fafdf1ba74e 100644
--- a/server/src/main/java/org/opensearch/index/codec/CodecService.java
+++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java
@@ -39,6 +39,7 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.IndexSettings;
+import org.opensearch.index.codec.composite.CompositeCodecFactory;
 import org.opensearch.index.mapper.MapperService;
 
 import java.util.Map;
@@ -63,6 +64,7 @@ public class CodecService {
      * the raw unfiltered lucene default. useful for testing
      */
     public static final String LUCENE_DEFAULT_CODEC = "lucene_default";
+    private final CompositeCodecFactory compositeCodecFactory = new CompositeCodecFactory();
 
     public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
@@ -73,10 +75,16 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
             codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Mode.BEST_COMPRESSION));
             codecs.put(ZLIB, new Lucene99Codec(Mode.BEST_COMPRESSION));
         } else {
-            codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
-            codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
-            codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
-            codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
+            // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
+            // We can still support all the compression codecs when composite index is present
+            if (mapperService.isCompositeIndexPresent()) {
+                codecs.putAll(compositeCodecFactory.getCompositeIndexCodecs(mapperService, logger));
+            } else {
+                codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
+                codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
+                codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
+                codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
+            }
         }
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java
new file mode 100644
index 0000000000000..de04944e67cd2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
+import org.opensearch.index.mapper.MapperService;
+
+/**
+ *  Extends the Codec to support new file formats for composite indices eg: star tree index
+ *  based on the mappings.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class Composite99Codec extends FilterCodec {
+    public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite99Codec";
+    private final MapperService mapperService;
+
+    // needed for SPI - this is used in reader path
+    public Composite99Codec() {
+        this(COMPOSITE_INDEX_CODEC_NAME, new Lucene99Codec(), null);
+    }
+
+    public Composite99Codec(Lucene99Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
+        this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
+    }
+
+    /**
+     * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
+     * this ctor.
+     *
+     * @param name name of the codec
+     * @param delegate codec delegate
+     * @param mapperService mapper service instance
+     */
+    protected Composite99Codec(String name, Codec delegate, MapperService mapperService) {
+        super(name, delegate);
+        this.mapperService = mapperService;
+    }
+
+    @Override
+    public DocValuesFormat docValuesFormat() {
+        return new Composite99DocValuesFormat(mapperService);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesFormat.java
new file mode 100644
index 0000000000000..216ed4f68f333
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesFormat.java
@@ -0,0 +1,64 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.mapper.MapperService;
+
+import java.io.IOException;
+
+/**
+ * DocValues format to handle composite indices
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class Composite99DocValuesFormat extends DocValuesFormat {
+    /**
+     * Creates a new docvalues format.
+     *
+     * <p>The provided name will be written into the index segment in some configurations (such as
+     * when using {@code PerFieldDocValuesFormat}): in such configurations, for the segment to be read
+     * this class should be registered with Java's SPI mechanism (registered in META-INF/ of your jar
+     * file, etc).
+     */
+    private final DocValuesFormat delegate;
+    private final MapperService mapperService;
+
+    // needed for SPI
+    public Composite99DocValuesFormat() {
+        this(new Lucene90DocValuesFormat(), null);
+    }
+
+    public Composite99DocValuesFormat(MapperService mapperService) {
+        this(new Lucene90DocValuesFormat(), mapperService);
+    }
+
+    public Composite99DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) {
+        super(delegate.getName());
+        this.delegate = delegate;
+        this.mapperService = mapperService;
+    }
+
+    @Override
+    public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+        return new Composite99DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService);
+    }
+
+    @Override
+    public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
+        return new Composite99DocValuesReader(delegate.fieldsProducer(state), state);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java
new file mode 100644
index 0000000000000..82c844088cfd4
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java
@@ -0,0 +1,89 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.mapper.CompositeMappedFieldType;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Reader for star tree index and star tree doc values from the segments
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class Composite99DocValuesReader extends DocValuesProducer implements CompositeIndexReader {
+    private DocValuesProducer delegate;
+
+    public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState state) throws IOException {
+        this.delegate = producer;
+        // TODO : read star tree files
+    }
+
+    @Override
+    public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+        return delegate.getNumeric(field);
+    }
+
+    @Override
+    public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+        return delegate.getBinary(field);
+    }
+
+    @Override
+    public SortedDocValues getSorted(FieldInfo field) throws IOException {
+        return delegate.getSorted(field);
+    }
+
+    @Override
+    public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+        return delegate.getSortedNumeric(field);
+    }
+
+    @Override
+    public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+        return delegate.getSortedSet(field);
+    }
+
+    @Override
+    public void checkIntegrity() throws IOException {
+        delegate.checkIntegrity();
+        // Todo : check integrity of composite index related [star tree] files
+    }
+
+    @Override
+    public void close() throws IOException {
+        delegate.close();
+        // Todo: close composite index related files [star tree] files
+    }
+
+    @Override
+    public List<String> getCompositeIndexFields() {
+        // todo : read from file formats and get the field names.
+        throw new UnsupportedOperationException();
+
+    }
+
+    @Override
+    public CompositeIndexValues getCompositeIndexValues(String field, CompositeMappedFieldType.CompositeFieldType fieldType)
+        throws IOException {
+        // TODO : read compositeIndexValues [starTreeValues] from star tree files
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java
new file mode 100644
index 0000000000000..75bbf78dbdad2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java
@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.mapper.CompositeMappedFieldType;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.StarTreeMapper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * This class write the star tree index and star tree doc values
+ * based on the doc values structures of the original index
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class Composite99DocValuesWriter extends DocValuesConsumer {
+    private final DocValuesConsumer delegate;
+    private final SegmentWriteState state;
+    private final MapperService mapperService;
+    AtomicReference<MergeState> mergeState = new AtomicReference<>();
+    private final Set<CompositeMappedFieldType> compositeMappedFieldTypes;
+    private final Set<String> compositeFieldSet;
+
+    private final Map<String, DocValuesProducer> fieldProducerMap = new HashMap<>();
+
+    public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) {
+
+        this.delegate = delegate;
+        this.state = segmentWriteState;
+        this.mapperService = mapperService;
+        this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes();
+        compositeFieldSet = new HashSet<>();
+        for (CompositeMappedFieldType type : compositeMappedFieldTypes) {
+            compositeFieldSet.addAll(type.fields());
+        }
+    }
+
+    @Override
+    public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        delegate.addNumericField(field, valuesProducer);
+    }
+
+    @Override
+    public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        delegate.addBinaryField(field, valuesProducer);
+    }
+
+    @Override
+    public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        delegate.addSortedField(field, valuesProducer);
+    }
+
+    @Override
+    public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        delegate.addSortedNumericField(field, valuesProducer);
+        // Perform this only during flush flow
+        if (mergeState.get() == null) {
+            createCompositeIndicesIfPossible(valuesProducer, field);
+        }
+    }
+
+    @Override
+    public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        delegate.addSortedSetField(field, valuesProducer);
+    }
+
+    @Override
+    public void close() throws IOException {
+        delegate.close();
+    }
+
+    private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, FieldInfo field) throws IOException {
+        if (compositeFieldSet.isEmpty()) return;
+        if (compositeFieldSet.contains(field.name)) {
+            fieldProducerMap.put(field.name, valuesProducer);
+            compositeFieldSet.remove(field.name);
+        }
+        // we have all the required fields to build composite fields
+        if (compositeFieldSet.isEmpty()) {
+            for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) {
+                if (mappedType instanceof StarTreeMapper.StarTreeFieldType) {
+                    // TODO : Call StarTree builder
+                }
+            }
+        }
+    }
+
+    @Override
+    public void merge(MergeState mergeState) throws IOException {
+        this.mergeState.compareAndSet(null, mergeState);
+        super.merge(mergeState);
+        // TODO : handle merge star tree
+        // mergeStarTreeFields(mergeState);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java
new file mode 100644
index 0000000000000..3acedc6a27d7f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.mapper.MapperService;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.opensearch.index.codec.CodecService.BEST_COMPRESSION_CODEC;
+import static org.opensearch.index.codec.CodecService.DEFAULT_CODEC;
+import static org.opensearch.index.codec.CodecService.LZ4;
+import static org.opensearch.index.codec.CodecService.ZLIB;
+
+/**
+ * Factory class to return the latest composite codec for all the modes
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CompositeCodecFactory {
+    public CompositeCodecFactory() {}
+
+    public Map<String, Codec> getCompositeIndexCodecs(MapperService mapperService, Logger logger) {
+        Map<String, Codec> codecs = new HashMap<>();
+        codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(ZLIB, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        return codecs;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java
new file mode 100644
index 0000000000000..d02438b75377d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java
@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.mapper.CompositeMappedFieldType;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Interface that abstracts the functionality to read composite index structures from the segment
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface CompositeIndexReader {
+    /**
+     * Get list of composite index fields from the segment
+     *
+     */
+    List<String> getCompositeIndexFields();
+
+    /**
+     * Get composite index values based on the field name and the field type
+     */
+    CompositeIndexValues getCompositeIndexValues(String field, CompositeMappedFieldType.CompositeFieldType fieldType) throws IOException;
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java
new file mode 100644
index 0000000000000..f8848aceab343
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java
@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Interface for composite index values
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface CompositeIndexValues {
+    CompositeIndexValues getValues();
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java
new file mode 100644
index 0000000000000..2a5b96ce2620a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite.datacube.startree;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.codec.composite.CompositeIndexValues;
+
+import java.util.List;
+
+/**
+ * Concrete class that holds the star tree associated values from the segment
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeValues implements CompositeIndexValues {
+    private final List<String> dimensionsOrder;
+
+    // TODO : come up with full set of vales such as dimensions and metrics doc values + star tree
+    public StarTreeValues(List<String> dimensionsOrder) {
+        super();
+        this.dimensionsOrder = List.copyOf(dimensionsOrder);
+    }
+
+    @Override
+    public CompositeIndexValues getValues() {
+        return this;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java
new file mode 100644
index 0000000000000..67808ad51289a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ *  classes responsible for handling all star tree structures and operations as part of codec
+ */
+package org.opensearch.index.codec.composite.datacube.startree;
diff --git a/server/src/main/java/org/opensearch/index/codec/composite/package-info.java b/server/src/main/java/org/opensearch/index/codec/composite/package-info.java
new file mode 100644
index 0000000000000..5d15e99c00975
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/codec/composite/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ *  classes responsible for handling all composite index codecs and operations
+ */
+package org.opensearch.index.codec.composite;
diff --git a/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java
index f52ce29a86dd2..e067e70621304 100644
--- a/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java
+++ b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java
@@ -45,7 +45,10 @@ public CompositeMappedFieldType(String name, List<String> fields, CompositeField
 
     /**
      * Supported composite field types
+     *
+     * @opensearch.experimental
      */
+    @ExperimentalApi
     public enum CompositeFieldType {
         STAR_TREE("star_tree");
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java
index c2e7411a3b47a..530a3092a5aa7 100644
--- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java
@@ -226,6 +226,8 @@ public enum MergeReason {
 
     private final BooleanSupplier idFieldDataEnabled;
 
+    private volatile Set<CompositeMappedFieldType> compositeMappedFieldTypes;
+
     public MapperService(
         IndexSettings indexSettings,
         IndexAnalyzers indexAnalyzers,
@@ -542,6 +544,9 @@ private synchronized Map<String, DocumentMapper> internalMerge(DocumentMapper ma
         }
 
         assert results.values().stream().allMatch(this::assertSerialization);
+
+        // initialize composite fields post merge
+        this.compositeMappedFieldTypes = getCompositeFieldTypesFromMapper();
         return results;
     }
 
@@ -655,6 +660,10 @@ public boolean isCompositeIndexPresent() {
     }
 
     public Set<CompositeMappedFieldType> getCompositeFieldTypes() {
+        return compositeMappedFieldTypes;
+    }
+
+    private Set<CompositeMappedFieldType> getCompositeFieldTypesFromMapper() {
         Set<CompositeMappedFieldType> compositeMappedFieldTypes = new HashSet<>();
         if (this.mapper == null) {
             return Collections.emptySet();
diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
new file mode 100644
index 0000000000000..e030a813373c1
--- /dev/null
+++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -0,0 +1 @@
+org.opensearch.index.codec.composite.Composite99Codec
diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java
index b31edd79411d0..7146b7dc51753 100644
--- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java
+++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java
@@ -48,6 +48,7 @@
 import org.opensearch.env.Environment;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.IndexAnalyzers;
+import org.opensearch.index.codec.composite.Composite99Codec;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.similarity.SimilarityService;
@@ -59,6 +60,8 @@
 import java.io.IOException;
 import java.util.Collections;
 
+import org.mockito.Mockito;
+
 import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -76,23 +79,52 @@ public void testDefault() throws Exception {
         assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec);
     }
 
+    public void testDefaultWithCompositeIndex() throws Exception {
+        Codec codec = createCodecService(false, true).codec("default");
+        assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec);
+        assert codec instanceof Composite99Codec;
+    }
+
     public void testBestCompression() throws Exception {
         Codec codec = createCodecService(false).codec("best_compression");
         assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec);
     }
 
+    public void testBestCompressionWithCompositeIndex() throws Exception {
+        Codec codec = createCodecService(false, true).codec("best_compression");
+        assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec);
+        assert codec instanceof Composite99Codec;
+    }
+
     public void testLZ4() throws Exception {
         Codec codec = createCodecService(false).codec("lz4");
         assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec);
         assert codec instanceof PerFieldMappingPostingFormatCodec;
     }
 
+    public void testLZ4WithCompositeIndex() throws Exception {
+        Codec codec = createCodecService(false, true).codec("lz4");
+        assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec);
+        assert codec instanceof Composite99Codec;
+    }
+
     public void testZlib() throws Exception {
         Codec codec = createCodecService(false).codec("zlib");
         assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec);
         assert codec instanceof PerFieldMappingPostingFormatCodec;
     }
 
+    public void testZlibWithCompositeIndex() throws Exception {
+        Codec codec = createCodecService(false, true).codec("zlib");
+        assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec);
+        assert codec instanceof Composite99Codec;
+    }
+
+    public void testResolveDefaultCodecsWithCompositeIndex() throws Exception {
+        CodecService codecService = createCodecService(false, true);
+        assertThat(codecService.codec("default"), instanceOf(Composite99Codec.class));
+    }
+
     public void testBestCompressionWithCompressionLevel() {
         final Settings settings = Settings.builder()
             .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6))
@@ -150,10 +182,17 @@ private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Co
     }
 
     private CodecService createCodecService(boolean isMapperServiceNull) throws IOException {
+        return createCodecService(isMapperServiceNull, false);
+    }
+
+    private CodecService createCodecService(boolean isMapperServiceNull, boolean isCompositeIndexPresent) throws IOException {
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
         if (isMapperServiceNull) {
             return new CodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test"));
         }
+        if (isCompositeIndexPresent) {
+            return buildCodecServiceWithCompositeIndex(nodeSettings);
+        }
         return buildCodecService(nodeSettings);
     }
 
@@ -176,6 +215,14 @@ private CodecService buildCodecService(Settings nodeSettings) throws IOException
         return new CodecService(service, indexSettings, LogManager.getLogger("test"));
     }
 
+    private CodecService buildCodecServiceWithCompositeIndex(Settings nodeSettings) throws IOException {
+
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
+        MapperService service = Mockito.mock(MapperService.class);
+        Mockito.when(service.isCompositeIndexPresent()).thenReturn(true);
+        return new CodecService(service, indexSettings, LogManager.getLogger("test"));
+    }
+
     private SegmentReader getSegmentReader(Codec codec) throws IOException {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(null);
diff --git a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java
new file mode 100644
index 0000000000000..6c6d26656e4de
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java
@@ -0,0 +1,110 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite.datacube.startree;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.opensearch.common.Rounding;
+import org.opensearch.index.codec.composite.Composite99Codec;
+import org.opensearch.index.compositeindex.datacube.DateDimension;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.NumericDimension;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.StarTreeMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.mockito.Mockito;
+
+/**
+ * Star tree doc values Lucene tests
+ */
+@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
+public class StarTreeDocValuesFormatTests extends BaseDocValuesFormatTestCase {
+    @Override
+    protected Codec getCodec() {
+        MapperService service = Mockito.mock(MapperService.class);
+        Mockito.when(service.getCompositeFieldTypes()).thenReturn(Set.of(getStarTreeFieldType()));
+        final Logger testLogger = LogManager.getLogger(StarTreeDocValuesFormatTests.class);
+        return new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, service, testLogger);
+    }
+
+    private StarTreeMapper.StarTreeFieldType getStarTreeFieldType() {
+        List<MetricStat> m1 = new ArrayList<>();
+        m1.add(MetricStat.MAX);
+        Metric metric = new Metric("sndv", m1);
+        List<Rounding.DateTimeUnit> d1CalendarIntervals = new ArrayList<>();
+        d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY);
+        StarTreeField starTreeField = getStarTreeField(d1CalendarIntervals, metric);
+
+        return new StarTreeMapper.StarTreeFieldType("star_tree", starTreeField);
+    }
+
+    private static StarTreeField getStarTreeField(List<Rounding.DateTimeUnit> d1CalendarIntervals, Metric metric1) {
+        DateDimension d1 = new DateDimension("field", d1CalendarIntervals);
+        NumericDimension d2 = new NumericDimension("dv");
+
+        List<Metric> metrics = List.of(metric1);
+        List<Dimension> dims = List.of(d1, d2);
+        StarTreeFieldConfiguration config = new StarTreeFieldConfiguration(
+            100,
+            Collections.emptySet(),
+            StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP
+        );
+
+        return new StarTreeField("starTree", dims, metrics, config);
+    }
+
+    public void testStarTreeDocValues() throws IOException {
+        Directory directory = newDirectory();
+        IndexWriterConfig conf = newIndexWriterConfig(null);
+        conf.setMergePolicy(newLogMergePolicy());
+        RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
+        Document doc = new Document();
+        doc.add(new SortedNumericDocValuesField("sndv", 1));
+        doc.add(new SortedNumericDocValuesField("dv", 1));
+        doc.add(new SortedNumericDocValuesField("field", 1));
+        iw.addDocument(doc);
+        doc.add(new SortedNumericDocValuesField("sndv", 1));
+        doc.add(new SortedNumericDocValuesField("dv", 1));
+        doc.add(new SortedNumericDocValuesField("field", 1));
+        iw.addDocument(doc);
+        iw.forceMerge(1);
+        doc.add(new SortedNumericDocValuesField("sndv", 2));
+        doc.add(new SortedNumericDocValuesField("dv", 2));
+        doc.add(new SortedNumericDocValuesField("field", 2));
+        iw.addDocument(doc);
+        doc.add(new SortedNumericDocValuesField("sndv", 2));
+        doc.add(new SortedNumericDocValuesField("dv", 2));
+        doc.add(new SortedNumericDocValuesField("field", 2));
+        iw.addDocument(doc);
+        iw.forceMerge(1);
+        iw.close();
+
+        // TODO : validate star tree structures that got created
+        directory.close();
+    }
+}

From ed1d85216014aeb861a18645557f52a4b4ca7694 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 8 Jul 2024 10:19:29 -0400
Subject: [PATCH 075/113] Bump com.github.spullara.mustache.java:compiler from
 0.9.13 to 0.9.14 in /modules/lang-mustache (#14672)

* Bump com.github.spullara.mustache.java:compiler

Bumps [com.github.spullara.mustache.java:compiler](https://github.com/spullara/mustache.java) from 0.9.13 to 0.9.14.
- [Commits](https://github.com/spullara/mustache.java/compare/mustache.java-0.9.13...mustache.java-0.9.14)

---
updated-dependencies:
- dependency-name: com.github.spullara.mustache.java:compiler
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                            | 1 +
 modules/lang-mustache/build.gradle                      | 2 +-
 modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1 | 1 -
 modules/lang-mustache/licenses/compiler-0.9.14.jar.sha1 | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1
 create mode 100644 modules/lang-mustache/licenses/compiler-0.9.14.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f7d1ea5b3d19..96b0d49c7e6ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506))
 - Bump `com.azure:azure-storage-common` from 12.21.2 to 12.25.1 ([#14517](https://github.com/opensearch-project/OpenSearch/pull/14517))
 - Bump `com.microsoft.azure:msal4j` from 1.15.1 to 1.16.0 ([#14610](https://github.com/opensearch-project/OpenSearch/pull/14610))
+- Bump `com.github.spullara.mustache.java:compiler` from 0.9.13 to 0.9.14 ([#14672](https://github.com/opensearch-project/OpenSearch/pull/14672))
 
 ### Changed
 - [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187))
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index bcf5c07ea8c64..a836124f94b41 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -38,7 +38,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api "com.github.spullara.mustache.java:compiler:0.9.13"
+  api "com.github.spullara.mustache.java:compiler:0.9.14"
 }
 
 restResources {
diff --git a/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1
deleted file mode 100644
index 70d53aac260eb..0000000000000
--- a/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-60666500a7dce7a5d3e17c09b46ea6f037192bd5
\ No newline at end of file
diff --git a/modules/lang-mustache/licenses/compiler-0.9.14.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.14.jar.sha1
new file mode 100644
index 0000000000000..29069ac90817a
--- /dev/null
+++ b/modules/lang-mustache/licenses/compiler-0.9.14.jar.sha1
@@ -0,0 +1 @@
+e6df8b5aabb80d6eb6d8fef312a56d66b7659ba6
\ No newline at end of file

From 41fa0855faf1913b0f5334398578cd6e2307d853 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 8 Jul 2024 11:29:48 -0400
Subject: [PATCH 076/113] Bump net.minidev:accessors-smart from 2.5.0 to 2.5.1
 in /plugins/repository-azure (#14673)

* Bump net.minidev:accessors-smart in /plugins/repository-azure

Bumps [net.minidev:accessors-smart](https://github.com/netplex/json-smart-v2) from 2.5.0 to 2.5.1.
- [Release notes](https://github.com/netplex/json-smart-v2/releases)
- [Commits](https://github.com/netplex/json-smart-v2/compare/2.5.0...2.5.1)

---
updated-dependencies:
- dependency-name: net.minidev:accessors-smart
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                                    | 1 +
 plugins/repository-azure/build.gradle                           | 2 +-
 .../repository-azure/licenses/accessors-smart-2.5.0.jar.sha1    | 1 -
 .../repository-azure/licenses/accessors-smart-2.5.1.jar.sha1    | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/accessors-smart-2.5.1.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96b0d49c7e6ad..cfb12ee853a8f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.azure:azure-storage-common` from 12.21.2 to 12.25.1 ([#14517](https://github.com/opensearch-project/OpenSearch/pull/14517))
 - Bump `com.microsoft.azure:msal4j` from 1.15.1 to 1.16.0 ([#14610](https://github.com/opensearch-project/OpenSearch/pull/14610))
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.13 to 0.9.14 ([#14672](https://github.com/opensearch-project/OpenSearch/pull/14672))
+- Bump `net.minidev:accessors-smart` from 2.5.0 to 2.5.1 ([#14673](https://github.com/opensearch-project/OpenSearch/pull/14673))
 
 ### Changed
 - [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187))
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 13b711019ff2a..0f822a02e05d8 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -69,7 +69,7 @@ dependencies {
   // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,
   // selected the higher version which is 2.5.0
   api 'net.minidev:json-smart:2.5.0'
-  api 'net.minidev:accessors-smart:2.5.0'
+  api 'net.minidev:accessors-smart:2.5.1'
   api "org.ow2.asm:asm:${versions.asm}"
   // End of transitive dependencies for azure-identity
   api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
diff --git a/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
deleted file mode 100644
index 1578c94fcdc7b..0000000000000
--- a/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aca011492dfe9c26f4e0659028a4fe0970829dd8
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/accessors-smart-2.5.1.jar.sha1 b/plugins/repository-azure/licenses/accessors-smart-2.5.1.jar.sha1
new file mode 100644
index 0000000000000..8f7452437323d
--- /dev/null
+++ b/plugins/repository-azure/licenses/accessors-smart-2.5.1.jar.sha1
@@ -0,0 +1 @@
+19b820261eb2e7de7d5bde11d1c06e4501dd7e5f
\ No newline at end of file

From 0684342b9e3711118614f38fb0648d8e6428477e Mon Sep 17 00:00:00 2001
From: Kaushal Kumar <ravi.kaushal97@gmail.com>
Date: Mon, 8 Jul 2024 14:39:39 -0700
Subject: [PATCH 077/113] Adding QueryGroup schema (#13669)

* rebase with opensearch/main

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add resourceLimitGroupId propagation logic from coordinator to data nodes

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add sandbox schema

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add resourceLimitGroupTests

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add resourceLimitGroupMetadata tests

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* run spotlessApply

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add mode field in ResourceLimitGroup schema

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* fix breaking testcases

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add task cancellation skeleton

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add multitenant labels in searchSource builder

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* write custom xcontent parser for ResourceLimitGroup

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* remove unrelated changes

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* remove non-existing import fro cluster settings

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* remove non releated changes

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add _id as the resourceLimitGroup key

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add change to register resource limit group metadata

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add updatedAt in resource limit group

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* rename resourceLimitGroup to queryGroup

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address the comments on PR

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* rename the mode member var to resiliency mode

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address comments

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add change in CHANGELOG

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add tests for custom namedWritable QueryGroupMetadata

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* structure resourceLimits into an object

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add QueryGroup.toXContent test case

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* fix precommit errors

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* fix precommit errors

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* fix assemble errors

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* fix checkstyle errors

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address comments

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

---------

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 .../org/opensearch/cluster/ClusterModule.java |   3 +
 .../opensearch/cluster/metadata/Metadata.java |  19 ++
 .../cluster/metadata/QueryGroup.java          | 317 ++++++++++++++++++
 .../cluster/metadata/QueryGroupMetadata.java  | 185 ++++++++++
 .../org/opensearch/search/ResourceType.java   |  14 +-
 .../SearchBackpressureService.java            |   4 +-
 .../cluster/ClusterModuleTests.java           |  10 +
 .../metadata/QueryGroupMetadataTests.java     |  93 +++++
 .../cluster/metadata/QueryGroupTests.java     | 158 +++++++++
 .../SearchBackpressureServiceTests.java       |  12 +-
 .../trackers/NodeDuressTrackersTests.java     |   8 +-
 12 files changed, 810 insertions(+), 14 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/QueryGroupMetadata.java
 create mode 100644 server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java
 create mode 100644 server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cfb12ee853a8f..4d0990db31d20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
+- [Workload Management] Add QueryGroup schema ([13669](https://github.com/opensearch-project/OpenSearch/pull/13669))
 - Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
 - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
index c7fd263bda56a..bb51c42252448 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
@@ -48,6 +48,7 @@
 import org.opensearch.cluster.metadata.MetadataIndexTemplateService;
 import org.opensearch.cluster.metadata.MetadataMappingService;
 import org.opensearch.cluster.metadata.MetadataUpdateSettingsService;
+import org.opensearch.cluster.metadata.QueryGroupMetadata;
 import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.cluster.metadata.ViewMetadata;
 import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
@@ -214,6 +215,8 @@ public static List<Entry> getNamedWriteables() {
             DecommissionAttributeMetadata::new,
             DecommissionAttributeMetadata::readDiffFrom
         );
+
+        registerMetadataCustom(entries, QueryGroupMetadata.TYPE, QueryGroupMetadata::new, QueryGroupMetadata::readDiffFrom);
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
         return entries;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
index e3f63b1c27b83..2a54f6444ffda 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
@@ -1368,6 +1368,25 @@ public Builder removeDataStream(String name) {
             return this;
         }
 
+        public Builder queryGroups(final Map<String, QueryGroup> queryGroups) {
+            this.customs.put(QueryGroupMetadata.TYPE, new QueryGroupMetadata(queryGroups));
+            return this;
+        }
+
+        public Builder put(final QueryGroup queryGroup) {
+            Objects.requireNonNull(queryGroup, "queryGroup should not be null");
+            Map<String, QueryGroup> existing = new HashMap<>(getQueryGroups());
+            existing.put(queryGroup.get_id(), queryGroup);
+            return queryGroups(existing);
+        }
+
+        private Map<String, QueryGroup> getQueryGroups() {
+            return Optional.ofNullable(this.customs.get(QueryGroupMetadata.TYPE))
+                .map(o -> (QueryGroupMetadata) o)
+                .map(QueryGroupMetadata::queryGroups)
+                .orElse(Collections.emptyMap());
+        }
+
         private Map<String, View> getViews() {
             return Optional.ofNullable(customs.get(ViewMetadata.TYPE))
                 .map(o -> (ViewMetadata) o)
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java
new file mode 100644
index 0000000000000..beaab198073df
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java
@@ -0,0 +1,317 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.metadata;
+
+import org.opensearch.cluster.AbstractDiffable;
+import org.opensearch.cluster.Diff;
+import org.opensearch.common.UUIDs;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.search.ResourceType;
+import org.joda.time.Instant;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Class to define the QueryGroup schema
+ * {
+ *              "_id": "fafjafjkaf9ag8a9ga9g7ag0aagaga",
+ *              "resourceLimits": {
+ *                  "jvm": 0.4
+ *              },
+ *              "resiliency_mode": "enforced",
+ *              "name": "analytics",
+ *              "updatedAt": 4513232415
+ * }
+ */
+@ExperimentalApi
+public class QueryGroup extends AbstractDiffable<QueryGroup> implements ToXContentObject {
+
+    private static final int MAX_CHARS_ALLOWED_IN_NAME = 50;
+    private final String name;
+    private final String _id;
+    private final ResiliencyMode resiliencyMode;
+    // It is an epoch in millis
+    private final long updatedAtInMillis;
+    private final Map<ResourceType, Object> resourceLimits;
+
+    public QueryGroup(String name, ResiliencyMode resiliencyMode, Map<ResourceType, Object> resourceLimits) {
+        this(name, UUIDs.randomBase64UUID(), resiliencyMode, resourceLimits, Instant.now().getMillis());
+    }
+
+    public QueryGroup(String name, String _id, ResiliencyMode resiliencyMode, Map<ResourceType, Object> resourceLimits, long updatedAt) {
+        Objects.requireNonNull(name, "QueryGroup.name can't be null");
+        Objects.requireNonNull(resourceLimits, "QueryGroup.resourceLimits can't be null");
+        Objects.requireNonNull(resiliencyMode, "QueryGroup.resiliencyMode can't be null");
+        Objects.requireNonNull(_id, "QueryGroup._id can't be null");
+
+        if (name.length() > MAX_CHARS_ALLOWED_IN_NAME) {
+            throw new IllegalArgumentException("QueryGroup.name shouldn't be more than 50 chars long");
+        }
+
+        if (resourceLimits.isEmpty()) {
+            throw new IllegalArgumentException("QueryGroup.resourceLimits should at least have 1 resource limit");
+        }
+        validateResourceLimits(resourceLimits);
+        if (!isValid(updatedAt)) {
+            throw new IllegalArgumentException("QueryGroup.updatedAtInMillis is not a valid epoch");
+        }
+
+        this.name = name;
+        this._id = _id;
+        this.resiliencyMode = resiliencyMode;
+        this.resourceLimits = resourceLimits;
+        this.updatedAtInMillis = updatedAt;
+    }
+
+    private static boolean isValid(long updatedAt) {
+        long minValidTimestamp = Instant.ofEpochMilli(0L).getMillis();
+
+        // Use Instant.now() to get the current time in seconds since epoch
+        long currentSeconds = Instant.now().getMillis();
+
+        // Check if the timestamp is within a reasonable range
+        return minValidTimestamp <= updatedAt && updatedAt <= currentSeconds;
+    }
+
+    public QueryGroup(StreamInput in) throws IOException {
+        this(
+            in.readString(),
+            in.readString(),
+            ResiliencyMode.fromName(in.readString()),
+            in.readMap((i) -> ResourceType.fromName(i.readString()), StreamInput::readGenericValue),
+            in.readLong()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(_id);
+        out.writeString(resiliencyMode.getName());
+        out.writeMap(resourceLimits, ResourceType::writeTo, StreamOutput::writeGenericValue);
+        out.writeLong(updatedAtInMillis);
+    }
+
+    private void validateResourceLimits(Map<ResourceType, Object> resourceLimits) {
+        for (Map.Entry<ResourceType, Object> resource : resourceLimits.entrySet()) {
+            Double threshold = (Double) resource.getValue();
+            Objects.requireNonNull(resource.getKey(), "resourceName can't be null");
+            Objects.requireNonNull(threshold, "resource limit threshold for" + resource.getKey().getName() + " : can't be null");
+
+            if (Double.compare(threshold, 1.0) > 0) {
+                throw new IllegalArgumentException("resource value should be less than 1.0");
+            }
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.startObject();
+        builder.field("_id", _id);
+        builder.field("name", name);
+        builder.field("resiliency_mode", resiliencyMode.getName());
+        builder.field("updatedAt", updatedAtInMillis);
+        // write resource limits
+        builder.startObject("resourceLimits");
+        for (ResourceType resourceType : ResourceType.values()) {
+            if (resourceLimits.containsKey(resourceType)) {
+                builder.field(resourceType.getName(), resourceLimits.get(resourceType));
+            }
+        }
+        builder.endObject();
+
+        builder.endObject();
+        return builder;
+    }
+
+    public static QueryGroup fromXContent(final XContentParser parser) throws IOException {
+        if (parser.currentToken() == null) { // fresh parser? move to the first token
+            parser.nextToken();
+        }
+
+        Builder builder = builder();
+
+        XContentParser.Token token = parser.currentToken();
+
+        if (token != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("Expected START_OBJECT token but found [" + parser.currentName() + "]");
+        }
+
+        String fieldName = "";
+        // Map to hold resources
+        final Map<ResourceType, Object> resourceLimits = new HashMap<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (fieldName.equals("_id")) {
+                    builder._id(parser.text());
+                } else if (fieldName.equals("name")) {
+                    builder.name(parser.text());
+                } else if (fieldName.equals("resiliency_mode")) {
+                    builder.mode(parser.text());
+                } else if (fieldName.equals("updatedAt")) {
+                    builder.updatedAt(parser.longValue());
+                } else {
+                    throw new IllegalArgumentException(fieldName + " is not a valid field in QueryGroup");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+
+                if (!fieldName.equals("resourceLimits")) {
+                    throw new IllegalArgumentException(
+                        "QueryGroup.resourceLimits is an object and expected token was { " + " but found " + token
+                    );
+                }
+
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        fieldName = parser.currentName();
+                    } else {
+                        resourceLimits.put(ResourceType.fromName(fieldName), parser.doubleValue());
+                    }
+                }
+
+            }
+        }
+        builder.resourceLimits(resourceLimits);
+        return builder.build();
+    }
+
+    public static Diff<QueryGroup> readDiff(final StreamInput in) throws IOException {
+        return readDiffFrom(QueryGroup::new, in);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        QueryGroup that = (QueryGroup) o;
+        return Objects.equals(name, that.name)
+            && Objects.equals(resourceLimits, that.resourceLimits)
+            && Objects.equals(_id, that._id)
+            && updatedAtInMillis == that.updatedAtInMillis;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, resourceLimits, updatedAtInMillis, _id);
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public ResiliencyMode getResiliencyMode() {
+        return resiliencyMode;
+    }
+
+    public Map<ResourceType, Object> getResourceLimits() {
+        return resourceLimits;
+    }
+
+    public String get_id() {
+        return _id;
+    }
+
+    public long getUpdatedAtInMillis() {
+        return updatedAtInMillis;
+    }
+
+    /**
+     * builder method for the {@link QueryGroup}
+     * @return Builder object
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * This enum models the different QueryGroup resiliency modes
+     * SOFT - means that this query group can consume more than query group resource limits if node is not in duress
+     * ENFORCED - means that it will never breach the assigned limits and will cancel as soon as the limits are breached
+     * MONITOR - it will not cause any cancellation but just log the eligible task cancellations
+     */
+    @ExperimentalApi
+    public enum ResiliencyMode {
+        SOFT("soft"),
+        ENFORCED("enforced"),
+        MONITOR("monitor");
+
+        private final String name;
+
+        ResiliencyMode(String mode) {
+            this.name = mode;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public static ResiliencyMode fromName(String s) {
+            for (ResiliencyMode mode : values()) {
+                if (mode.getName().equalsIgnoreCase(s)) return mode;
+
+            }
+            throw new IllegalArgumentException("Invalid value for QueryGroupMode: " + s);
+        }
+
+    }
+
+    /**
+     * Builder class for {@link QueryGroup}
+     */
+    @ExperimentalApi
+    public static class Builder {
+        private String name;
+        private String _id;
+        private ResiliencyMode resiliencyMode;
+        private long updatedAt;
+        private Map<ResourceType, Object> resourceLimits;
+
+        private Builder() {}
+
+        public Builder name(String name) {
+            this.name = name;
+            return this;
+        }
+
+        public Builder _id(String _id) {
+            this._id = _id;
+            return this;
+        }
+
+        public Builder mode(String mode) {
+            this.resiliencyMode = ResiliencyMode.fromName(mode);
+            return this;
+        }
+
+        public Builder updatedAt(long updatedAt) {
+            this.updatedAt = updatedAt;
+            return this;
+        }
+
+        public Builder resourceLimits(Map<ResourceType, Object> resourceLimits) {
+            this.resourceLimits = resourceLimits;
+            return this;
+        }
+
+        public QueryGroup build() {
+            return new QueryGroup(name, _id, resiliencyMode, resourceLimits, updatedAt);
+        }
+
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroupMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroupMetadata.java
new file mode 100644
index 0000000000000..79732bc505ee2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroupMetadata.java
@@ -0,0 +1,185 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.metadata;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.Diff;
+import org.opensearch.cluster.DiffableUtils;
+import org.opensearch.cluster.NamedDiff;
+import org.opensearch.core.ParseField;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.opensearch.cluster.metadata.Metadata.ALL_CONTEXTS;
+
+/**
+ * This class holds the QueryGroupMetadata
+ * sample schema
+ * {
+ *     "queryGroups": {
+ *         "_id": {
+ *             {@link QueryGroup}
+ *         },
+ *        ...
+ *     }
+ * }
+ */
+public class QueryGroupMetadata implements Metadata.Custom {
+    public static final String TYPE = "queryGroups";
+    private static final ParseField QUERY_GROUP_FIELD = new ParseField("queryGroups");
+
+    private final Map<String, QueryGroup> queryGroups;
+
+    public QueryGroupMetadata(Map<String, QueryGroup> queryGroups) {
+        this.queryGroups = queryGroups;
+    }
+
+    public QueryGroupMetadata(StreamInput in) throws IOException {
+        this.queryGroups = in.readMap(StreamInput::readString, QueryGroup::new);
+    }
+
+    public Map<String, QueryGroup> queryGroups() {
+        return this.queryGroups;
+    }
+
+    /**
+     * Returns the name of the writeable object
+     */
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    /**
+     * The minimal version of the recipient this object can be sent to
+     */
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_3_0_0;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(queryGroups, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        for (Map.Entry<String, QueryGroup> entry : queryGroups.entrySet()) {
+            builder.field(entry.getKey(), entry.getValue());
+        }
+        return builder;
+    }
+
+    public static QueryGroupMetadata fromXContent(XContentParser parser) throws IOException {
+        Map<String, QueryGroup> queryGroupMap = new HashMap<>();
+
+        if (parser.currentToken() == null) {
+            parser.nextToken();
+        }
+
+        if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException(
+                "QueryGroupMetadata.fromXContent was expecting a { token but found : " + parser.currentToken()
+            );
+        }
+        XContentParser.Token token = parser.currentToken();
+        String fieldName = parser.currentName();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else {
+                QueryGroup queryGroup = QueryGroup.fromXContent(parser);
+                queryGroupMap.put(fieldName, queryGroup);
+            }
+        }
+
+        return new QueryGroupMetadata(queryGroupMap);
+    }
+
+    @Override
+    public Diff<Metadata.Custom> diff(final Metadata.Custom previousState) {
+        return new QueryGroupMetadataDiff((QueryGroupMetadata) previousState, this);
+    }
+
+    public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new QueryGroupMetadataDiff(in);
+    }
+
+    @Override
+    public EnumSet<Metadata.XContentContext> context() {
+        return ALL_CONTEXTS;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        QueryGroupMetadata that = (QueryGroupMetadata) o;
+        return Objects.equals(queryGroups, that.queryGroups);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(queryGroups);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(MediaTypeRegistry.JSON, this);
+    }
+
+    /**
+     * QueryGroupMetadataDiff
+     */
+    static class QueryGroupMetadataDiff implements NamedDiff<Metadata.Custom> {
+        final Diff<Map<String, QueryGroup>> dataStreamDiff;
+
+        QueryGroupMetadataDiff(final QueryGroupMetadata before, final QueryGroupMetadata after) {
+            dataStreamDiff = DiffableUtils.diff(before.queryGroups, after.queryGroups, DiffableUtils.getStringKeySerializer());
+        }
+
+        QueryGroupMetadataDiff(final StreamInput in) throws IOException {
+            this.dataStreamDiff = DiffableUtils.readJdkMapDiff(
+                in,
+                DiffableUtils.getStringKeySerializer(),
+                QueryGroup::new,
+                QueryGroup::readDiff
+            );
+        }
+
+        /**
+         * Returns the name of the writeable object
+         */
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            dataStreamDiff.writeTo(out);
+        }
+
+        @Override
+        public Metadata.Custom apply(Metadata.Custom part) {
+            return new QueryGroupMetadata(new HashMap<>(dataStreamDiff.apply(((QueryGroupMetadata) part).queryGroups)));
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/search/ResourceType.java b/server/src/main/java/org/opensearch/search/ResourceType.java
index 5bbcd7de1c2ce..fe5ce4dd2bb50 100644
--- a/server/src/main/java/org/opensearch/search/ResourceType.java
+++ b/server/src/main/java/org/opensearch/search/ResourceType.java
@@ -8,12 +8,18 @@
 
 package org.opensearch.search;
 
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.core.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
 /**
  * Enum to hold the resource type
  */
+@PublicApi(since = "2.x")
 public enum ResourceType {
     CPU("cpu"),
-    JVM("jvm");
+    MEMORY("memory");
 
     private final String name;
 
@@ -35,7 +41,11 @@ public static ResourceType fromName(String s) {
         throw new IllegalArgumentException("Unknown resource type: [" + s + "]");
     }
 
-    private String getName() {
+    public static void writeTo(StreamOutput out, ResourceType resourceType) throws IOException {
+        out.writeString(resourceType.getName());
+    }
+
+    public String getName() {
         return name;
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index 3e8ed3070e4ef..c26c5d63a3573 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -71,7 +71,7 @@ public class SearchBackpressureService extends AbstractLifecycleComponent implem
         TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
         (nodeDuressTrackers) -> nodeDuressTrackers.isResourceInDuress(ResourceType.CPU),
         TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER,
-        (nodeDuressTrackers) -> isHeapTrackingSupported() && nodeDuressTrackers.isResourceInDuress(ResourceType.JVM),
+        (nodeDuressTrackers) -> isHeapTrackingSupported() && nodeDuressTrackers.isResourceInDuress(ResourceType.MEMORY),
         TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER,
         (nodeDuressTrackers) -> true
     );
@@ -105,7 +105,7 @@ public SearchBackpressureService(
                     )
                 );
                 put(
-                    ResourceType.JVM,
+                    ResourceType.MEMORY,
                     new NodeDuressTracker(
                         () -> JvmStats.jvmStats().getMem().getHeapUsedPercent() / 100.0 >= settings.getNodeDuressSettings()
                             .getHeapThreshold(),
diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
index f2d99a51f1c9a..97706927ba857 100644
--- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
@@ -33,6 +33,7 @@
 package org.opensearch.cluster;
 
 import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.metadata.QueryGroupMetadata;
 import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.allocation.ExistingShardsAllocator;
@@ -69,6 +70,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsModule;
 import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.gateway.GatewayAllocator;
 import org.opensearch.plugins.ClusterPlugin;
 import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
@@ -327,6 +329,14 @@ public void testRejectsDuplicateExistingShardsAllocatorName() {
         );
     }
 
+    public void testQueryGroupMetadataRegister() {
+        List<NamedWriteableRegistry.Entry> customEntries = ClusterModule.getNamedWriteables();
+        assertTrue(
+            customEntries.stream()
+                .anyMatch(entry -> entry.categoryClass == Metadata.Custom.class && entry.name.equals(QueryGroupMetadata.TYPE))
+        );
+    }
+
     private static ClusterPlugin existingShardsAllocatorPlugin(final String allocatorName) {
         return new ClusterPlugin() {
             @Override
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java
new file mode 100644
index 0000000000000..d70a9ce5e10cd
--- /dev/null
+++ b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java
@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.metadata;
+
+import org.opensearch.cluster.Diff;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.search.ResourceType;
+import org.opensearch.test.AbstractDiffableSerializationTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.opensearch.cluster.metadata.QueryGroupTests.createRandomQueryGroup;
+
+public class QueryGroupMetadataTests extends AbstractDiffableSerializationTestCase<Metadata.Custom> {
+
+    public void testToXContent() throws IOException {
+        long updatedAt = 1720047207;
+        QueryGroupMetadata queryGroupMetadata = new QueryGroupMetadata(
+            Map.of(
+                "ajakgakg983r92_4242",
+                new QueryGroup(
+                    "test",
+                    "ajakgakg983r92_4242",
+                    QueryGroup.ResiliencyMode.ENFORCED,
+                    Map.of(ResourceType.MEMORY, 0.5),
+                    updatedAt
+                )
+            )
+        );
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        queryGroupMetadata.toXContent(builder, null);
+        builder.endObject();
+        assertEquals(
+            "{\"ajakgakg983r92_4242\":{\"_id\":\"ajakgakg983r92_4242\",\"name\":\"test\",\"resiliency_mode\":\"enforced\",\"updatedAt\":1720047207,\"resourceLimits\":{\"memory\":0.5}}}",
+            builder.toString()
+        );
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Collections.singletonList(
+                new NamedWriteableRegistry.Entry(QueryGroupMetadata.class, QueryGroupMetadata.TYPE, QueryGroupMetadata::new)
+            )
+        );
+    }
+
+    @Override
+    protected Metadata.Custom makeTestChanges(Metadata.Custom testInstance) {
+        final QueryGroup queryGroup = createRandomQueryGroup("asdfakgjwrir23r25");
+        final QueryGroupMetadata queryGroupMetadata = new QueryGroupMetadata(Map.of(queryGroup.get_id(), queryGroup));
+        return queryGroupMetadata;
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<Metadata.Custom>> diffReader() {
+        return QueryGroupMetadata::readDiffFrom;
+    }
+
+    @Override
+    protected Metadata.Custom doParseInstance(XContentParser parser) throws IOException {
+        return QueryGroupMetadata.fromXContent(parser);
+    }
+
+    @Override
+    protected Writeable.Reader<Metadata.Custom> instanceReader() {
+        return QueryGroupMetadata::new;
+    }
+
+    @Override
+    protected QueryGroupMetadata createTestInstance() {
+        return new QueryGroupMetadata(getRandomQueryGroups());
+    }
+
+    private Map<String, QueryGroup> getRandomQueryGroups() {
+        QueryGroup qg1 = createRandomQueryGroup("1243gsgsdgs");
+        QueryGroup qg2 = createRandomQueryGroup("lkajga8080");
+        return Map.of(qg1.get_id(), qg1, qg2.get_id(), qg2);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java
new file mode 100644
index 0000000000000..c564f0778e6f0
--- /dev/null
+++ b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java
@@ -0,0 +1,158 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.metadata;
+
+import org.opensearch.common.UUIDs;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.search.ResourceType;
+import org.opensearch.test.AbstractSerializingTestCase;
+import org.joda.time.Instant;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class QueryGroupTests extends AbstractSerializingTestCase<QueryGroup> {
+
+    private static final List<QueryGroup.ResiliencyMode> allowedModes = List.of(
+        QueryGroup.ResiliencyMode.SOFT,
+        QueryGroup.ResiliencyMode.ENFORCED,
+        QueryGroup.ResiliencyMode.MONITOR
+    );
+
+    static QueryGroup createRandomQueryGroup(String _id) {
+        String name = randomAlphaOfLength(10);
+        Map<ResourceType, Object> resourceLimit = new HashMap<>();
+        resourceLimit.put(ResourceType.MEMORY, randomDoubleBetween(0.0, 0.80, false));
+        return new QueryGroup(name, _id, randomMode(), resourceLimit, Instant.now().getMillis());
+    }
+
+    private static QueryGroup.ResiliencyMode randomMode() {
+        return allowedModes.get(randomIntBetween(0, allowedModes.size() - 1));
+    }
+
+    /**
+     * Parses to a new instance using the provided {@link XContentParser}
+     *
+     * @param parser
+     */
+    @Override
+    protected QueryGroup doParseInstance(XContentParser parser) throws IOException {
+        return QueryGroup.fromXContent(parser);
+    }
+
+    /**
+     * Returns a {@link Writeable.Reader} that can be used to de-serialize the instance
+     */
+    @Override
+    protected Writeable.Reader<QueryGroup> instanceReader() {
+        return QueryGroup::new;
+    }
+
+    /**
+     * Creates a random test instance to use in the tests. This method will be
+     * called multiple times during test execution and should return a different
+     * random instance each time it is called.
+     */
+    @Override
+    protected QueryGroup createTestInstance() {
+        return createRandomQueryGroup("1232sfraeradf_");
+    }
+
+    public void testNullName() {
+        assertThrows(
+            NullPointerException.class,
+            () -> new QueryGroup(null, "_id", randomMode(), Collections.emptyMap(), Instant.now().getMillis())
+        );
+    }
+
+    public void testNullId() {
+        assertThrows(
+            NullPointerException.class,
+            () -> new QueryGroup("Dummy", null, randomMode(), Collections.emptyMap(), Instant.now().getMillis())
+        );
+    }
+
+    public void testNullResourceLimits() {
+        assertThrows(NullPointerException.class, () -> new QueryGroup("analytics", "_id", randomMode(), null, Instant.now().getMillis()));
+    }
+
+    public void testEmptyResourceLimits() {
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new QueryGroup("analytics", "_id", randomMode(), Collections.emptyMap(), Instant.now().getMillis())
+        );
+    }
+
+    public void testIllegalQueryGroupMode() {
+        assertThrows(
+            NullPointerException.class,
+            () -> new QueryGroup("analytics", "_id", null, Map.of(ResourceType.MEMORY, (Object) 0.4), Instant.now().getMillis())
+        );
+    }
+
+    public void testInvalidResourceLimitWhenInvalidSystemResourceValueIsGiven() {
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new QueryGroup(
+                "analytics",
+                "_id",
+                randomMode(),
+                Map.of(ResourceType.MEMORY, (Object) randomDoubleBetween(1.1, 1.8, false)),
+                Instant.now().getMillis()
+            )
+        );
+    }
+
+    public void testValidQueryGroup() {
+        QueryGroup queryGroup = new QueryGroup(
+            "analytics",
+            "_id",
+            randomMode(),
+            Map.of(ResourceType.MEMORY, randomDoubleBetween(0.01, 0.8, false)),
+            Instant.ofEpochMilli(1717187289).getMillis()
+        );
+
+        assertNotNull(queryGroup.getName());
+        assertEquals("analytics", queryGroup.getName());
+        assertNotNull(queryGroup.getResourceLimits());
+        assertFalse(queryGroup.getResourceLimits().isEmpty());
+        assertEquals(1, queryGroup.getResourceLimits().size());
+        assertTrue(allowedModes.contains(queryGroup.getResiliencyMode()));
+        assertEquals(1717187289, queryGroup.getUpdatedAtInMillis());
+    }
+
+    public void testToXContent() throws IOException {
+        long currentTimeInMillis = Instant.now().getMillis();
+        String queryGroupId = UUIDs.randomBase64UUID();
+        QueryGroup queryGroup = new QueryGroup(
+            "TestQueryGroup",
+            queryGroupId,
+            QueryGroup.ResiliencyMode.ENFORCED,
+            Map.of(ResourceType.CPU, 0.30, ResourceType.MEMORY, 0.40),
+            currentTimeInMillis
+        );
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        queryGroup.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals(
+            "{\"_id\":\""
+                + queryGroupId
+                + "\",\"name\":\"TestQueryGroup\",\"resiliency_mode\":\"enforced\",\"updatedAt\":"
+                + currentTimeInMillis
+                + ",\"resourceLimits\":{\"cpu\":0.3,\"memory\":0.4}}",
+            builder.toString()
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
index 43df482fcc2ae..15d0fcd10d701 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
@@ -57,7 +57,7 @@
 import java.util.function.LongSupplier;
 
 import static org.opensearch.search.ResourceType.CPU;
-import static org.opensearch.search.ResourceType.JVM;
+import static org.opensearch.search.ResourceType.MEMORY;
 import static org.opensearch.search.backpressure.SearchBackpressureTestHelpers.createMockTaskWithResourceStats;
 import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -102,7 +102,7 @@ public void testIsNodeInDuress() {
 
         EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
             {
-                put(ResourceType.JVM, heapUsageTracker);
+                put(ResourceType.MEMORY, heapUsageTracker);
                 put(ResourceType.CPU, cpuUsageTracker);
             }
         };
@@ -233,7 +233,7 @@ public void testSearchTaskInFlightCancellation() {
 
         EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
             {
-                put(JVM, heapUsageTracker);
+                put(MEMORY, heapUsageTracker);
                 put(CPU, mockNodeDuressTracker);
             }
         };
@@ -308,7 +308,7 @@ public void testSearchShardTaskInFlightCancellation() {
 
         EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
             {
-                put(JVM, new NodeDuressTracker(() -> false, () -> 3));
+                put(MEMORY, new NodeDuressTracker(() -> false, () -> 3));
                 put(CPU, mockNodeDuressTracker);
             }
         };
@@ -401,7 +401,7 @@ public void testNonCancellationOfHeapBasedTasksWhenHeapNotInDuress() {
 
         EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
             {
-                put(JVM, new NodeDuressTracker(() -> false, () -> 3));
+                put(MEMORY, new NodeDuressTracker(() -> false, () -> 3));
                 put(CPU, new NodeDuressTracker(() -> true, () -> 3));
             }
         };
@@ -495,7 +495,7 @@ public void testNonCancellationWhenSearchTrafficIsNotQualifyingForCancellation()
 
         EnumMap<ResourceType, NodeDuressTracker> duressTrackers = new EnumMap<>(ResourceType.class) {
             {
-                put(JVM, new NodeDuressTracker(() -> false, () -> 3));
+                put(MEMORY, new NodeDuressTracker(() -> false, () -> 3));
                 put(CPU, new NodeDuressTracker(() -> true, () -> 3));
             }
         };
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
index 2db251ee461db..801576bdf89d4 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
@@ -19,7 +19,7 @@ public class NodeDuressTrackersTests extends OpenSearchTestCase {
     public void testNodeNotInDuress() {
         EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
             {
-                put(ResourceType.JVM, new NodeDuressTracker(() -> false, () -> 2));
+                put(ResourceType.MEMORY, new NodeDuressTracker(() -> false, () -> 2));
                 put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 2));
             }
         };
@@ -34,7 +34,7 @@ public void testNodeNotInDuress() {
     public void testNodeInDuressWhenHeapInDuress() {
         EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
             {
-                put(ResourceType.JVM, new NodeDuressTracker(() -> true, () -> 3));
+                put(ResourceType.MEMORY, new NodeDuressTracker(() -> true, () -> 3));
                 put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 1));
             }
         };
@@ -51,7 +51,7 @@ public void testNodeInDuressWhenHeapInDuress() {
     public void testNodeInDuressWhenCPUInDuress() {
         EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
             {
-                put(ResourceType.JVM, new NodeDuressTracker(() -> false, () -> 1));
+                put(ResourceType.MEMORY, new NodeDuressTracker(() -> false, () -> 1));
                 put(ResourceType.CPU, new NodeDuressTracker(() -> true, () -> 3));
             }
         };
@@ -68,7 +68,7 @@ public void testNodeInDuressWhenCPUInDuress() {
     public void testNodeInDuressWhenCPUAndHeapInDuress() {
         EnumMap<ResourceType, NodeDuressTracker> map = new EnumMap<>(ResourceType.class) {
             {
-                put(ResourceType.JVM, new NodeDuressTracker(() -> true, () -> 3));
+                put(ResourceType.MEMORY, new NodeDuressTracker(() -> true, () -> 3));
                 put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 3));
             }
         };

From 51af2e2203abf41f42f41decc05e6a49e6b6d40d Mon Sep 17 00:00:00 2001
From: Shivansh Arora <hishiv@amazon.com>
Date: Tue, 9 Jul 2024 13:18:25 +0530
Subject: [PATCH 078/113] Add UTs for RemoteIndexMetadataManager (#14660)

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
Co-authored-by: Arpit-Bandejiya <abandeji@amazon.com>
---
 .../remote/RemoteIndexMetadataManager.java    |  21 --
 .../RemoteIndexMetadataManagerTests.java      | 190 ++++++++++++++++++
 2 files changed, 190 insertions(+), 21 deletions(-)
 create mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java
index a84161b202a22..c595f19279354 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java
@@ -26,10 +26,7 @@
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
 
 /**
  * A Manager which provides APIs to write and read Index Metadata to remote store
@@ -136,24 +133,6 @@ IndexMetadata getIndexMetadata(ClusterMetadataManifest.UploadedIndexMetadata upl
         }
     }
 
-    /**
-     * Fetch latest index metadata from remote cluster state
-     *
-     * @param clusterMetadataManifest manifest file of cluster
-     * @param clusterUUID             uuid of cluster state to refer to in remote
-     * @return {@code Map<String, IndexMetadata>} latest IndexUUID to IndexMetadata map
-     */
-    Map<String, IndexMetadata> getIndexMetadataMap(String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) {
-        assert Objects.equals(clusterUUID, clusterMetadataManifest.getClusterUUID())
-            : "Corrupt ClusterMetadataManifest found. Cluster UUID mismatch.";
-        Map<String, IndexMetadata> remoteIndexMetadata = new HashMap<>();
-        for (ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata : clusterMetadataManifest.getIndices()) {
-            IndexMetadata indexMetadata = getIndexMetadata(uploadedIndexMetadata, clusterUUID);
-            remoteIndexMetadata.put(uploadedIndexMetadata.getIndexUUID(), indexMetadata);
-        }
-        return remoteIndexMetadata;
-    }
-
     public TimeValue getIndexMetadataUploadTimeout() {
         return this.indexMetadataUploadTimeout;
     }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java
new file mode 100644
index 0000000000000..817fc7b55d09a
--- /dev/null
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java
@@ -0,0 +1,190 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote;
+
+import org.opensearch.Version;
+import org.opensearch.action.LatchedActionListener;
+import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.Nullable;
+import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
+import org.opensearch.common.blobstore.BlobContainer;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.blobstore.BlobStore;
+import org.opensearch.common.blobstore.stream.write.WritePriority;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.TestCapturingListener;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.compress.Compressor;
+import org.opensearch.core.compress.NoneCompressor;
+import org.opensearch.gateway.remote.model.RemoteReadResult;
+import org.opensearch.index.remote.RemoteStoreUtils;
+import org.opensearch.index.translog.transfer.BlobStoreTransferService;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER;
+import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX;
+import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_METADATA_FORMAT;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyIterable;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class RemoteIndexMetadataManagerTests extends OpenSearchTestCase {
+
+    private RemoteIndexMetadataManager remoteIndexMetadataManager;
+    private BlobStoreRepository blobStoreRepository;
+    private BlobStoreTransferService blobStoreTransferService;
+    private Compressor compressor;
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Before
+    public void setup() {
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        blobStoreRepository = mock(BlobStoreRepository.class);
+        BlobPath blobPath = new BlobPath().add("random-path");
+        when((blobStoreRepository.basePath())).thenReturn(blobPath);
+        blobStoreTransferService = mock(BlobStoreTransferService.class);
+        compressor = new NoneCompressor();
+        when(blobStoreRepository.getCompressor()).thenReturn(compressor);
+        remoteIndexMetadataManager = new RemoteIndexMetadataManager(
+            clusterSettings,
+            "test-cluster",
+            blobStoreRepository,
+            blobStoreTransferService,
+            threadPool
+        );
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+    }
+
+    public void testGetAsyncIndexMetadataWriteAction_Success() throws Exception {
+        IndexMetadata indexMetadata = getIndexMetadata(randomAlphaOfLength(10), randomBoolean(), randomAlphaOfLength(10));
+        BlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class);
+        BlobStore blobStore = mock(BlobStore.class);
+        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        String expectedFilePrefix = String.join(DELIMITER, "metadata", RemoteStoreUtils.invertLong(indexMetadata.getVersion()));
+
+        doAnswer((invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        })).when(blobStoreTransferService).uploadBlob(any(), any(), any(), eq(WritePriority.URGENT), any(ActionListener.class));
+
+        remoteIndexMetadataManager.getAsyncIndexMetadataWriteAction(
+            indexMetadata,
+            "cluster-uuid",
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+        assertEquals(INDEX + "--" + indexMetadata.getIndex().getName(), uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(7, pathTokens.length);
+        assertEquals(INDEX, pathTokens[4]);
+        assertEquals(indexMetadata.getIndex().getUUID(), pathTokens[5]);
+        assertTrue(pathTokens[6].startsWith(expectedFilePrefix));
+    }
+
+    public void testGetAsyncIndexMetadataWriteAction_IOFailure() throws Exception {
+        IndexMetadata indexMetadata = getIndexMetadata(randomAlphaOfLength(10), randomBoolean(), randomAlphaOfLength(10));
+        BlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class);
+        BlobStore blobStore = mock(BlobStore.class);
+        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        doAnswer((invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onFailure(new IOException("failure"));
+            return null;
+        })).when(blobStoreTransferService).uploadBlob(any(), any(), any(), eq(WritePriority.URGENT), any(ActionListener.class));
+
+        remoteIndexMetadataManager.getAsyncIndexMetadataWriteAction(
+            indexMetadata,
+            "cluster-uuid",
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getResult());
+        assertNotNull(listener.getFailure());
+        assertTrue(listener.getFailure() instanceof RemoteStateTransferException);
+    }
+
+    public void testGetAsyncIndexMetadataReadAction_Success() throws Exception {
+        IndexMetadata indexMetadata = getIndexMetadata(randomAlphaOfLength(10), randomBoolean(), randomAlphaOfLength(10));
+        String fileName = randomAlphaOfLength(10);
+        fileName = fileName + DELIMITER + '2';
+        when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn(
+            INDEX_METADATA_FORMAT.serialize(indexMetadata, fileName, compressor, FORMAT_PARAMS).streamInput()
+        );
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteIndexMetadataManager.getAsyncIndexMetadataReadAction("cluster-uuid", fileName, new LatchedActionListener<>(listener, latch))
+            .run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        assertEquals(indexMetadata, listener.getResult().getObj());
+    }
+
+    public void testGetAsyncIndexMetadataReadAction_IOFailure() throws Exception {
+        String fileName = randomAlphaOfLength(10);
+        fileName = fileName + DELIMITER + '2';
+        Exception exception = new IOException("testing failure");
+        doThrow(exception).when(blobStoreTransferService).downloadBlob(anyIterable(), anyString());
+        TestCapturingListener<RemoteReadResult> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteIndexMetadataManager.getAsyncIndexMetadataReadAction("cluster-uuid", fileName, new LatchedActionListener<>(listener, latch))
+            .run();
+        latch.await();
+        assertNull(listener.getResult());
+        assertNotNull(listener.getFailure());
+        assertEquals(exception, listener.getFailure());
+    }
+
+    private IndexMetadata getIndexMetadata(String name, @Nullable Boolean writeIndex, String... aliases) {
+        IndexMetadata.Builder builder = IndexMetadata.builder(name)
+            .settings(
+                Settings.builder()
+                    .put("index.version.created", Version.CURRENT.id)
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 1)
+            );
+        for (String alias : aliases) {
+            builder.putAlias(AliasMetadata.builder(alias).writeIndex(writeIndex).build());
+        }
+        return builder.build();
+    }
+}

From 2e639131b3bb88316f53bfeb9262cbcd81606d50 Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Tue, 9 Jul 2024 19:29:44 +0800
Subject: [PATCH 079/113] Fix match_phrase_prefix_query not working on text
 field with multiple values and index_prefixes (#10959)

* Fix match_phrase_prefix_query not working on text field with multiple values and index_prefixes
Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Add more test

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* modify change log

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Fix test failure

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Change the indexAnalyzer used by prefix field

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Skip old version for yaml test

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Optimize some code

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Fix test failure

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Modify yaml test description

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Remove the name parameter for setAnalyzer()

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

---------

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../test/search/190_index_prefix_search.yml   | 52 ++++++++++++++++++-
 .../index/mapper/TextFieldMapper.java         | 28 +++++++---
 .../index/mapper/TextFieldMapperTests.java    | 51 ++++++++++++++++++
 .../index/mapper/TextFieldTypeTests.java      |  1 +
 5 files changed, 124 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d0990db31d20..2cffa99fb66a5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix bug in SBP cancellation logic ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13474))
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
+- Fix match_phrase_prefix_query not working on text field with multiple values and index_prefixes ([#10959](https://github.com/opensearch-project/OpenSearch/pull/10959))
 - Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
 - Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
 - Write shard level metadata blob when snapshotting searchable snapshot indexes ([#13190](https://github.com/opensearch-project/OpenSearch/pull/13190))
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml
index 25d3dd160e031..8b031c132f979 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml
@@ -10,7 +10,12 @@ setup:
                 index_prefixes:
                   min_chars: 2
                   max_chars: 5
-
+              text_with_pos_inc_gap:
+                type: text
+                position_increment_gap: 201
+                index_prefixes:
+                  min_chars: 2
+                  max_chars: 5
   - do:
       index:
           index:  test
@@ -23,6 +28,18 @@ setup:
         id:     2
         body:   { text: sentence with UPPERCASE WORDS }
 
+  - do:
+      index:
+        index:  test
+        id:     3
+        body:   { text: ["foo", "b-12"] }
+
+  - do:
+      index:
+        index:  test
+        id:     4
+        body:   { text_with_pos_inc_gap: ["foo", "b-12"] }
+
   - do:
       indices.refresh:
         index: [test]
@@ -116,3 +133,36 @@ setup:
               ]
 
   - match: {hits.total: 1}
+
+# related issue: https://github.com/opensearch-project/OpenSearch/issues/9203
+---
+"search index prefixes with multiple values":
+  - skip:
+      version: " - 2.99.99"
+      reason: "the bug was fixed in 3.0.0"
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            match_phrase_prefix:
+              text: "b-12"
+
+  - match: {hits.total: 1}
+
+---
+"search index prefixes with multiple values and custom position_increment_gap":
+  - skip:
+      version: " - 2.99.99"
+      reason: "the bug was fixed in 3.0.0"
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            match_phrase_prefix:
+              text_with_pos_inc_gap: "b-12"
+
+  - match: {hits.total: 1}
diff --git a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java
index d0e041e68a81d..ba053a3aeee1d 100644
--- a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java
@@ -448,7 +448,6 @@ protected PrefixFieldMapper buildPrefixMapper(BuilderContext context, FieldType
                 pft.setStoreTermVectorOffsets(true);
             }
             PrefixFieldType prefixFieldType = new PrefixFieldType(tft, fullName + "._index_prefix", indexPrefixes.get());
-            prefixFieldType.setAnalyzer(analyzers.getIndexAnalyzer());
             tft.setPrefixFieldType(prefixFieldType);
             return new PrefixFieldMapper(pft, prefixFieldType);
         }
@@ -522,12 +521,14 @@ private static class PrefixWrappedAnalyzer extends AnalyzerWrapper {
         private final int minChars;
         private final int maxChars;
         private final Analyzer delegate;
+        private final int positionIncrementGap;
 
-        PrefixWrappedAnalyzer(Analyzer delegate, int minChars, int maxChars) {
+        PrefixWrappedAnalyzer(Analyzer delegate, int minChars, int maxChars, int positionIncrementGap) {
             super(delegate.getReuseStrategy());
             this.delegate = delegate;
             this.minChars = minChars;
             this.maxChars = maxChars;
+            this.positionIncrementGap = positionIncrementGap;
         }
 
         @Override
@@ -535,6 +536,11 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
             return delegate;
         }
 
+        @Override
+        public int getPositionIncrementGap(String fieldName) {
+            return positionIncrementGap;
+        }
+
         @Override
         protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
             TokenFilter filter = new EdgeNGramTokenFilter(components.getTokenStream(), minChars, maxChars, false);
@@ -588,17 +594,18 @@ static final class PrefixFieldType extends StringFieldType {
 
         final int minChars;
         final int maxChars;
-        final TextFieldType parentField;
+        final TextFieldType parent;
 
         PrefixFieldType(TextFieldType parentField, String name, PrefixConfig config) {
             this(parentField, name, config.minChars, config.maxChars);
         }
 
-        PrefixFieldType(TextFieldType parentField, String name, int minChars, int maxChars) {
-            super(name, true, false, false, parentField.getTextSearchInfo(), Collections.emptyMap());
+        PrefixFieldType(TextFieldType parent, String name, int minChars, int maxChars) {
+            super(name, true, false, false, parent.getTextSearchInfo(), Collections.emptyMap());
             this.minChars = minChars;
             this.maxChars = maxChars;
-            this.parentField = parentField;
+            this.parent = parent;
+            setAnalyzer(parent.indexAnalyzer());
         }
 
         @Override
@@ -609,8 +616,13 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchL
         }
 
         void setAnalyzer(NamedAnalyzer delegate) {
+            String analyzerName = delegate.name();
             setIndexAnalyzer(
-                new NamedAnalyzer(delegate.name(), AnalyzerScope.INDEX, new PrefixWrappedAnalyzer(delegate.analyzer(), minChars, maxChars))
+                new NamedAnalyzer(
+                    analyzerName,
+                    AnalyzerScope.INDEX,
+                    new PrefixWrappedAnalyzer(delegate.analyzer(), minChars, maxChars, delegate.getPositionIncrementGap(analyzerName))
+                )
             );
         }
 
@@ -639,7 +651,7 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool
             Automaton automaton = Operations.concatenate(automata);
             AutomatonQuery query = AutomatonQueries.createAutomatonQuery(new Term(name(), value + "*"), automaton, method);
             return new BooleanQuery.Builder().add(query, BooleanClause.Occur.SHOULD)
-                .add(new TermQuery(new Term(parentField.name(), value)), BooleanClause.Occur.SHOULD)
+                .add(new TermQuery(new Term(parent.name(), value)), BooleanClause.Occur.SHOULD)
                 .build();
         }
 
diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java
index a22bfa5e845b1..0253caea9759d 100644
--- a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java
@@ -380,6 +380,57 @@ public void testIndexOptions() throws IOException {
         }
     }
 
+    public void testPositionIncrementGapOnIndexPrefixField() throws IOException {
+        // test default position_increment_gap
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "text").field("analyzer", "default").startObject("index_prefixes").endObject())
+        );
+        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.array("field", new String[] { "a", "b 12" })));
+
+        withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), reader -> {
+            TermsEnum terms = getOnlyLeafReader(reader).terms("field").iterator();
+            assertTrue(terms.seekExact(new BytesRef("12")));
+            PostingsEnum postings = terms.postings(null, PostingsEnum.POSITIONS);
+            assertEquals(0, postings.nextDoc());
+            assertEquals(TextFieldMapper.Defaults.POSITION_INCREMENT_GAP + 2, postings.nextPosition());
+        });
+
+        withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), reader -> {
+            TermsEnum terms = getOnlyLeafReader(reader).terms("field._index_prefix").iterator();
+            assertTrue(terms.seekExact(new BytesRef("12")));
+            PostingsEnum postings = terms.postings(null, PostingsEnum.POSITIONS);
+            assertEquals(0, postings.nextDoc());
+            assertEquals(TextFieldMapper.Defaults.POSITION_INCREMENT_GAP + 2, postings.nextPosition());
+        });
+
+        // test custom position_increment_gap
+        final int positionIncrementGap = randomIntBetween(1, 1000);
+        MapperService mapperService2 = createMapperService(
+            fieldMapping(
+                b -> b.field("type", "text")
+                    .field("position_increment_gap", positionIncrementGap)
+                    .field("analyzer", "default")
+                    .startObject("index_prefixes")
+                    .endObject()
+            )
+        );
+        ParsedDocument doc2 = mapperService2.documentMapper().parse(source(b -> b.array("field", new String[] { "a", "b 12" })));
+        withLuceneIndex(mapperService2, iw -> iw.addDocument(doc2.rootDoc()), reader -> {
+            TermsEnum terms = getOnlyLeafReader(reader).terms("field").iterator();
+            assertTrue(terms.seekExact(new BytesRef("12")));
+            PostingsEnum postings = terms.postings(null, PostingsEnum.POSITIONS);
+            assertEquals(0, postings.nextDoc());
+            assertEquals(positionIncrementGap + 2, postings.nextPosition());
+        });
+        withLuceneIndex(mapperService2, iw -> iw.addDocument(doc2.rootDoc()), reader -> {
+            TermsEnum terms = getOnlyLeafReader(reader).terms("field._index_prefix").iterator();
+            assertTrue(terms.seekExact(new BytesRef("12")));
+            PostingsEnum postings = terms.postings(null, PostingsEnum.POSITIONS);
+            assertEquals(0, postings.nextDoc());
+            assertEquals(positionIncrementGap + 2, postings.nextPosition());
+        });
+    }
+
     public void testDefaultPositionIncrementGap() throws IOException {
         MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping));
         ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.array("field", new String[] { "a", "b" })));
diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java
index 9c177bbec61fd..e672f94819541 100644
--- a/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java
@@ -167,6 +167,7 @@ public void testFuzzyQuery() {
 
     public void testIndexPrefixes() {
         TextFieldType ft = createFieldType(true);
+        ft.setIndexAnalyzer(Lucene.STANDARD_ANALYZER);
         ft.setPrefixFieldType(new TextFieldMapper.PrefixFieldType(ft, "field._index_prefix", 2, 10));
 
         Query q = ft.prefixQuery("goin", CONSTANT_SCORE_REWRITE, false, randomMockShardContext());

From 6d0484a75bbb5897f29660ede02ced7c796ba8cf Mon Sep 17 00:00:00 2001
From: rishavz_sagar <rishavsagar4b1@gmail.com>
Date: Tue, 9 Jul 2024 20:20:38 +0530
Subject: [PATCH 080/113] Offline calculation of total shard per node and
 caching it for weight calculation inside LocalShardBalancer (#14675)

Signed-off-by: RS146BIJAY <rishavsagar4b1@gmail.com>
---
 .../allocation/allocator/LocalShardsBalancer.java    | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
index 6978c988fd648..00eb79add9f1d 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
@@ -68,6 +68,7 @@ public class LocalShardsBalancer extends ShardsBalancer {
     private final float avgPrimaryShardsPerNode;
     private final BalancedShardsAllocator.NodeSorter sorter;
     private final Set<RoutingNode> inEligibleTargetNode;
+    private int totalShardCount = 0;
 
     public LocalShardsBalancer(
         Logger logger,
@@ -125,8 +126,7 @@ public float avgPrimaryShardsPerNode() {
      */
     @Override
     public float avgShardsPerNode() {
-        float totalShards = nodes.values().stream().map(BalancedShardsAllocator.ModelNode::numShards).reduce(0, Integer::sum);
-        return totalShards / nodes.size();
+        return totalShardCount / nodes.size();
     }
 
     /**
@@ -598,6 +598,7 @@ void moveShards() {
                 final BalancedShardsAllocator.ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
                 final BalancedShardsAllocator.ModelNode targetNode = nodes.get(moveDecision.getTargetNode().getId());
                 sourceNode.removeShard(shardRouting);
+                --totalShardCount;
                 Tuple<ShardRouting, ShardRouting> relocatingShards = routingNodes.relocateShard(
                     shardRouting,
                     targetNode.getNodeId(),
@@ -605,6 +606,7 @@ void moveShards() {
                     allocation.changes()
                 );
                 targetNode.addShard(relocatingShards.v2());
+                ++totalShardCount;
                 if (logger.isTraceEnabled()) {
                     logger.trace("Moved shard [{}] to node [{}]", shardRouting, targetNode.getRoutingNode());
                 }
@@ -724,6 +726,7 @@ private Map<String, BalancedShardsAllocator.ModelNode> buildModelFromAssigned()
                 /* we skip relocating shards here since we expect an initializing shard with the same id coming in */
                 if (RoutingPool.LOCAL_ONLY.equals(RoutingPool.getShardPool(shard, allocation)) && shard.state() != RELOCATING) {
                     node.addShard(shard);
+                    ++totalShardCount;
                     if (logger.isTraceEnabled()) {
                         logger.trace("Assigned shard [{}] to node [{}]", shard, node.getNodeId());
                     }
@@ -815,6 +818,7 @@ void allocateUnassigned() {
                     );
                     shard = routingNodes.initializeShard(shard, minNode.getNodeId(), null, shardSize, allocation.changes());
                     minNode.addShard(shard);
+                    ++totalShardCount;
                     if (!shard.primary()) {
                         // copy over the same replica shards to the secondary array so they will get allocated
                         // in a subsequent iteration, allowing replicas of other shards to be allocated first
@@ -844,6 +848,7 @@ void allocateUnassigned() {
                             allocation.routingTable()
                         );
                         minNode.addShard(shard.initialize(minNode.getNodeId(), null, shardSize));
+                        ++totalShardCount;
                     } else {
                         if (logger.isTraceEnabled()) {
                             logger.trace("No Node found to assign shard [{}]", shard);
@@ -1011,18 +1016,21 @@ private boolean tryRelocateShard(BalancedShardsAllocator.ModelNode minNode, Bala
                 }
                 final Decision decision = new Decision.Multi().add(allocationDecision).add(rebalanceDecision);
                 maxNode.removeShard(shard);
+                --totalShardCount;
                 long shardSize = allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
 
                 if (decision.type() == Decision.Type.YES) {
                     /* only allocate on the cluster if we are not throttled */
                     logger.debug("Relocate [{}] from [{}] to [{}]", shard, maxNode.getNodeId(), minNode.getNodeId());
                     minNode.addShard(routingNodes.relocateShard(shard, minNode.getNodeId(), shardSize, allocation.changes()).v1());
+                    ++totalShardCount;
                     return true;
                 } else {
                     /* allocate on the model even if throttled */
                     logger.debug("Simulate relocation of [{}] from [{}] to [{}]", shard, maxNode.getNodeId(), minNode.getNodeId());
                     assert decision.type() == Decision.Type.THROTTLE;
                     minNode.addShard(shard.relocate(minNode.getNodeId(), shardSize));
+                    ++totalShardCount;
                     return false;
                 }
             }

From acc46316550ee203851d5c622d3b4724646d3f3e Mon Sep 17 00:00:00 2001
From: Chenyang Ji <cyji@amazon.com>
Date: Tue, 9 Jul 2024 11:14:55 -0700
Subject: [PATCH 081/113] [bug fix] validate lower bound for top n size
 (#14587)

Signed-off-by: Chenyang Ji <cyji@amazon.com>
---
 .../plugin/insights/core/service/TopQueriesService.java   | 8 +++-----
 .../insights/core/service/TopQueriesServiceTests.java     | 4 ++++
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java
index c21b89be4dcca..bbe8b8fc40dac 100644
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java
+++ b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java
@@ -138,17 +138,15 @@ public int getTopNSize() {
      * @param size the wanted top N size
      */
     public void validateTopNSize(final int size) {
-        if (size > QueryInsightsSettings.MAX_N_SIZE) {
+        if (size < 1 || size > QueryInsightsSettings.MAX_N_SIZE) {
             throw new IllegalArgumentException(
                 "Top N size setting for ["
                     + metricType
                     + "]"
-                    + " should be smaller than max top N size ["
+                    + " should be between 1 and "
                     + QueryInsightsSettings.MAX_N_SIZE
-                    + "was ("
+                    + ", was ("
                     + size
-                    + " > "
-                    + QueryInsightsSettings.MAX_N_SIZE
                     + ")"
             );
         }
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java
index 3efd4c86833cc..8478fe1621698 100644
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java
+++ b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java
@@ -78,6 +78,10 @@ public void testValidateTopNSize() {
         assertThrows(IllegalArgumentException.class, () -> { topQueriesService.validateTopNSize(QueryInsightsSettings.MAX_N_SIZE + 1); });
     }
 
+    public void testValidateNegativeTopNSize() {
+        assertThrows(IllegalArgumentException.class, () -> { topQueriesService.validateTopNSize(-1); });
+    }
+
     public void testGetTopQueriesWhenNotEnabled() {
         topQueriesService.setEnabled(false);
         assertThrows(IllegalArgumentException.class, () -> { topQueriesService.getTopQueriesRecords(false); });

From bf56227ad1d679573beffd94f419f7b73cd07188 Mon Sep 17 00:00:00 2001
From: Craig Perkins <cwperx@amazon.com>
Date: Tue, 9 Jul 2024 14:39:24 -0400
Subject: [PATCH 082/113] Create SystemIndexRegistry with helper method
 matchesSystemIndex (#14415)

* Create new extension point in SystemIndexPlugin for a single plugin to get registered system indices

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add to CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* WIP on system indices from IndexNameExpressionResolver

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add test in IndexNameExpressionResolverTests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Remove changes in SystemIndexPlugin

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add method in IndexNameExpressionResolver to get matching system indices

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Show how resolver can be chained to get system indices

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix forbiddenApis check

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Make SystemIndices internal

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Remove unneeded changes

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix CI failures

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix precommit errors

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Use Regex instead of WildcardMatcher

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Address code review feedback

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Allow caller to pass index expressions

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Create SystemIndexRegistry

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Remove singleton limitation

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add javadoc

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add @ExperimentalApi annotation

Signed-off-by: Craig Perkins <cwperx@amazon.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../indices/SystemIndexDescriptor.java        |   4 +-
 .../indices/SystemIndexRegistry.java          | 130 ++++++++++++++++++
 .../org/opensearch/indices/SystemIndices.java |  88 +-----------
 .../main/java/org/opensearch/node/Node.java   |  17 ++-
 .../indices/SystemIndicesTests.java           |  99 ++++++++++++-
 6 files changed, 242 insertions(+), 97 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2cffa99fb66a5..fe8d5d524097e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
 - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
+- Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/main/java/org/opensearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/opensearch/indices/SystemIndexDescriptor.java
index f3592a3561d3a..f3212b1e2fae1 100644
--- a/server/src/main/java/org/opensearch/indices/SystemIndexDescriptor.java
+++ b/server/src/main/java/org/opensearch/indices/SystemIndexDescriptor.java
@@ -33,6 +33,7 @@
 package org.opensearch.indices;
 
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.regex.Regex;
 
 import java.util.Objects;
@@ -40,8 +41,9 @@
 /**
  * Describes a system index. Provides the information required to create and maintain the system index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.16.0")
 public class SystemIndexDescriptor {
     private final String indexPattern;
     private final String description;
diff --git a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
new file mode 100644
index 0000000000000..d9608e220d924
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
@@ -0,0 +1,130 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.regex.Regex;
+import org.opensearch.tasks.TaskResultsService;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static java.util.Collections.unmodifiableMap;
+import static org.opensearch.tasks.TaskResultsService.TASK_INDEX;
+
+/**
+ * This class holds the {@link SystemIndexDescriptor} objects that represent system indices the
+ * node knows about. This class also contains static methods that identify if index expressions match
+ * registered system index patterns
+ *
+ * @opensearch.api
+ */
+@ExperimentalApi
+public class SystemIndexRegistry {
+    private static final SystemIndexDescriptor TASK_INDEX_DESCRIPTOR = new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index");
+    private static final Map<String, Collection<SystemIndexDescriptor>> SERVER_SYSTEM_INDEX_DESCRIPTORS = singletonMap(
+        TaskResultsService.class.getName(),
+        singletonList(TASK_INDEX_DESCRIPTOR)
+    );
+
+    private volatile static String[] SYSTEM_INDEX_PATTERNS = new String[0];
+    volatile static Collection<SystemIndexDescriptor> SYSTEM_INDEX_DESCRIPTORS = Collections.emptyList();
+
+    static void register(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) {
+        final Map<String, Collection<SystemIndexDescriptor>> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
+        checkForOverlappingPatterns(descriptorsMap);
+        List<SystemIndexDescriptor> descriptors = pluginAndModulesDescriptors.values()
+            .stream()
+            .flatMap(Collection::stream)
+            .collect(Collectors.toList());
+        descriptors.add(TASK_INDEX_DESCRIPTOR);
+
+        SYSTEM_INDEX_DESCRIPTORS = descriptors.stream().collect(Collectors.toUnmodifiableList());
+        SYSTEM_INDEX_PATTERNS = descriptors.stream().map(SystemIndexDescriptor::getIndexPattern).toArray(String[]::new);
+    }
+
+    public static List<String> matchesSystemIndexPattern(String... indexExpressions) {
+        return Arrays.stream(indexExpressions)
+            .filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern))
+            .collect(Collectors.toList());
+    }
+
+    /**
+     * Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed
+     * descriptors overlap with any of the other patterns. If any do, throws an exception.
+     *
+     * @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide.
+     * @throws IllegalStateException Thrown if any of the index patterns overlaps with another.
+     */
+    static void checkForOverlappingPatterns(Map<String, Collection<SystemIndexDescriptor>> sourceToDescriptors) {
+        List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToDescriptors.entrySet()
+            .stream()
+            .flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor)))
+            .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message
+            .collect(Collectors.toList());
+
+        // This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the
+        // automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states
+        // per pattern should be low as well. If these assumptions change, this might need to be reworked.
+        sourceDescriptorPair.forEach(descriptorToCheck -> {
+            List<Tuple<String, SystemIndexDescriptor>> descriptorsMatchingThisPattern = sourceDescriptorPair.stream()
+
+                .filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked
+                .filter(d -> overlaps(descriptorToCheck.v2(), d.v2()))
+                .collect(Collectors.toList());
+            if (descriptorsMatchingThisPattern.isEmpty() == false) {
+                throw new IllegalStateException(
+                    "a system index descriptor ["
+                        + descriptorToCheck.v2()
+                        + "] from ["
+                        + descriptorToCheck.v1()
+                        + "] overlaps with other system index descriptors: ["
+                        + descriptorsMatchingThisPattern.stream()
+                            .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]")
+                            .collect(Collectors.joining(", "))
+                );
+            }
+        });
+    }
+
+    private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) {
+        Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern());
+        Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern());
+        return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false;
+    }
+
+    private static Map<String, Collection<SystemIndexDescriptor>> buildSystemIndexDescriptorMap(
+        Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesMap
+    ) {
+        final Map<String, Collection<SystemIndexDescriptor>> map = new HashMap<>(
+            pluginAndModulesMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size()
+        );
+        map.putAll(pluginAndModulesMap);
+        // put the server items last since we expect less of them
+        SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, descriptors) -> {
+            if (map.putIfAbsent(source, descriptors) != null) {
+                throw new IllegalArgumentException(
+                    "plugin or module attempted to define the same source [" + source + "] as a built-in system index"
+                );
+            }
+        });
+        return unmodifiableMap(map);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/SystemIndices.java b/server/src/main/java/org/opensearch/indices/SystemIndices.java
index a85e938c61b7a..bbf58fe91512f 100644
--- a/server/src/main/java/org/opensearch/indices/SystemIndices.java
+++ b/server/src/main/java/org/opensearch/indices/SystemIndices.java
@@ -40,25 +40,15 @@
 import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.opensearch.common.Nullable;
-import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.core.index.Index;
-import org.opensearch.tasks.TaskResultsService;
 
 import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.singletonList;
-import static java.util.Collections.singletonMap;
-import static java.util.Collections.unmodifiableList;
-import static java.util.Collections.unmodifiableMap;
-import static org.opensearch.tasks.TaskResultsService.TASK_INDEX;
-
 /**
  * This class holds the {@link SystemIndexDescriptor} objects that represent system indices the
  * node knows about. Methods for determining if an index should be a system index are also provided
@@ -69,21 +59,11 @@
 public class SystemIndices {
     private static final Logger logger = LogManager.getLogger(SystemIndices.class);
 
-    private static final Map<String, Collection<SystemIndexDescriptor>> SERVER_SYSTEM_INDEX_DESCRIPTORS = singletonMap(
-        TaskResultsService.class.getName(),
-        singletonList(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index"))
-    );
-
     private final CharacterRunAutomaton runAutomaton;
-    private final Collection<SystemIndexDescriptor> systemIndexDescriptors;
 
     public SystemIndices(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) {
-        final Map<String, Collection<SystemIndexDescriptor>> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
-        checkForOverlappingPatterns(descriptorsMap);
-        this.systemIndexDescriptors = unmodifiableList(
-            descriptorsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList())
-        );
-        this.runAutomaton = buildCharacterRunAutomaton(systemIndexDescriptors);
+        SystemIndexRegistry.register(pluginAndModulesDescriptors);
+        this.runAutomaton = buildCharacterRunAutomaton(SystemIndexRegistry.SYSTEM_INDEX_DESCRIPTORS);
     }
 
     /**
@@ -111,7 +91,7 @@ public boolean isSystemIndex(String indexName) {
      * @throws IllegalStateException if multiple descriptors match the name
      */
     public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) {
-        final List<SystemIndexDescriptor> matchingDescriptors = systemIndexDescriptors.stream()
+        final List<SystemIndexDescriptor> matchingDescriptors = SystemIndexRegistry.SYSTEM_INDEX_DESCRIPTORS.stream()
             .filter(descriptor -> descriptor.matchesIndexPattern(name))
             .collect(Collectors.toList());
 
@@ -168,66 +148,4 @@ private static CharacterRunAutomaton buildCharacterRunAutomaton(Collection<Syste
             .reduce(Operations::union);
         return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(Automata.makeEmpty()), Integer.MAX_VALUE));
     }
-
-    /**
-     * Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed
-     * descriptors overlap with any of the other patterns. If any do, throws an exception.
-     *
-     * @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide.
-     * @throws IllegalStateException Thrown if any of the index patterns overlaps with another.
-     */
-    static void checkForOverlappingPatterns(Map<String, Collection<SystemIndexDescriptor>> sourceToDescriptors) {
-        List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToDescriptors.entrySet()
-            .stream()
-            .flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor)))
-            .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message
-            .collect(Collectors.toList());
-
-        // This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the
-        // automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states
-        // per pattern should be low as well. If these assumptions change, this might need to be reworked.
-        sourceDescriptorPair.forEach(descriptorToCheck -> {
-            List<Tuple<String, SystemIndexDescriptor>> descriptorsMatchingThisPattern = sourceDescriptorPair.stream()
-
-                .filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked
-                .filter(d -> overlaps(descriptorToCheck.v2(), d.v2()))
-                .collect(Collectors.toList());
-            if (descriptorsMatchingThisPattern.isEmpty() == false) {
-                throw new IllegalStateException(
-                    "a system index descriptor ["
-                        + descriptorToCheck.v2()
-                        + "] from ["
-                        + descriptorToCheck.v1()
-                        + "] overlaps with other system index descriptors: ["
-                        + descriptorsMatchingThisPattern.stream()
-                            .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]")
-                            .collect(Collectors.joining(", "))
-                );
-            }
-        });
-    }
-
-    private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) {
-        Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern());
-        Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern());
-        return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false;
-    }
-
-    private static Map<String, Collection<SystemIndexDescriptor>> buildSystemIndexDescriptorMap(
-        Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesMap
-    ) {
-        final Map<String, Collection<SystemIndexDescriptor>> map = new HashMap<>(
-            pluginAndModulesMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size()
-        );
-        map.putAll(pluginAndModulesMap);
-        // put the server items last since we expect less of them
-        SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, descriptors) -> {
-            if (map.putIfAbsent(source, descriptors) != null) {
-                throw new IllegalArgumentException(
-                    "plugin or module attempted to define the same source [" + source + "] as a built-in system index"
-                );
-            }
-        });
-        return unmodifiableMap(map);
-    }
 }
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 85ef547e27787..96a716af7f1a1 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -695,6 +695,14 @@ protected Node(
                 repositoriesServiceReference::get,
                 rerouteServiceReference::get
             );
+            final Map<String, Collection<SystemIndexDescriptor>> systemIndexDescriptorMap = Collections.unmodifiableMap(
+                pluginsService.filterPlugins(SystemIndexPlugin.class)
+                    .stream()
+                    .collect(
+                        Collectors.toMap(plugin -> plugin.getClass().getSimpleName(), plugin -> plugin.getSystemIndexDescriptors(settings))
+                    )
+            );
+            final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap);
             final ClusterModule clusterModule = new ClusterModule(
                 settings,
                 clusterService,
@@ -819,15 +827,6 @@ protected Node(
                 .flatMap(m -> m.entrySet().stream())
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
-            final Map<String, Collection<SystemIndexDescriptor>> systemIndexDescriptorMap = Collections.unmodifiableMap(
-                pluginsService.filterPlugins(SystemIndexPlugin.class)
-                    .stream()
-                    .collect(
-                        Collectors.toMap(plugin -> plugin.getClass().getSimpleName(), plugin -> plugin.getSystemIndexDescriptors(settings))
-                    )
-            );
-            final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap);
-
             final RerouteService rerouteService = new BatchedRerouteService(clusterService, clusterModule.getAllocationService()::reroute);
             rerouteServiceReference.set(rerouteService);
             clusterService.setRerouteService(rerouteService);
diff --git a/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java b/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java
index 2b40c01c61242..8ac457c32d53a 100644
--- a/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java
+++ b/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java
@@ -32,12 +32,17 @@
 
 package org.opensearch.indices;
 
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.SystemIndexPlugin;
 import org.opensearch.tasks.TaskResultsService;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
@@ -67,7 +72,7 @@ public void testBasicOverlappingPatterns() {
 
         IllegalStateException exception = expectThrows(
             IllegalStateException.class,
-            () -> SystemIndices.checkForOverlappingPatterns(descriptors)
+            () -> SystemIndexRegistry.checkForOverlappingPatterns(descriptors)
         );
         assertThat(
             exception.getMessage(),
@@ -104,7 +109,7 @@ public void testComplexOverlappingPatterns() {
 
         IllegalStateException exception = expectThrows(
             IllegalStateException.class,
-            () -> SystemIndices.checkForOverlappingPatterns(descriptors)
+            () -> SystemIndexRegistry.checkForOverlappingPatterns(descriptors)
         );
         assertThat(
             exception.getMessage(),
@@ -133,4 +138,94 @@ public void testPluginCannotOverrideBuiltInSystemIndex() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SystemIndices(pluginMap));
         assertThat(e.getMessage(), containsString("plugin or module attempted to define the same source"));
     }
+
+    public void testSystemIndexMatching() {
+        SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
+        SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
+        SystemIndexPlugin plugin3 = new SystemIndexPatternPlugin();
+        SystemIndices pluginSystemIndices = new SystemIndices(
+            Map.of(
+                SystemIndexPlugin1.class.getCanonicalName(),
+                plugin1.getSystemIndexDescriptors(Settings.EMPTY),
+                SystemIndexPlugin2.class.getCanonicalName(),
+                plugin2.getSystemIndexDescriptors(Settings.EMPTY),
+                SystemIndexPatternPlugin.class.getCanonicalName(),
+                plugin3.getSystemIndexDescriptors(Settings.EMPTY)
+            )
+        );
+
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index2"),
+            equalTo(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2))
+        );
+        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index1"), equalTo(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1)));
+        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index2"), equalTo(List.of(SystemIndexPlugin2.SYSTEM_INDEX_2)));
+        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern1"), equalTo(List.of(".system-index-pattern1")));
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern-sub*"),
+            equalTo(List.of(".system-index-pattern-sub*"))
+        );
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern1", ".system-index-pattern2"),
+            equalTo(List.of(".system-index-pattern1", ".system-index-pattern2"))
+        );
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index-pattern1"),
+            equalTo(List.of(".system-index1", ".system-index-pattern1"))
+        );
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index-pattern1", ".not-system"),
+            equalTo(List.of(".system-index1", ".system-index-pattern1"))
+        );
+        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".not-system"), equalTo(Collections.emptyList()));
+    }
+
+    public void testRegisteredSystemIndexExpansion() {
+        SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
+        SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
+        SystemIndices pluginSystemIndices = new SystemIndices(
+            Map.of(
+                SystemIndexPlugin1.class.getCanonicalName(),
+                plugin1.getSystemIndexDescriptors(Settings.EMPTY),
+                SystemIndexPlugin2.class.getCanonicalName(),
+                plugin2.getSystemIndexDescriptors(Settings.EMPTY)
+            )
+        );
+        List<String> systemIndices = SystemIndexRegistry.matchesSystemIndexPattern(
+            SystemIndexPlugin1.SYSTEM_INDEX_1,
+            SystemIndexPlugin2.SYSTEM_INDEX_2
+        );
+        assertEquals(2, systemIndices.size());
+        assertTrue(systemIndices.containsAll(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2)));
+    }
+
+    static final class SystemIndexPlugin1 extends Plugin implements SystemIndexPlugin {
+        public static final String SYSTEM_INDEX_1 = ".system-index1";
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            final SystemIndexDescriptor systemIndexDescriptor = new SystemIndexDescriptor(SYSTEM_INDEX_1, "System index 1");
+            return Collections.singletonList(systemIndexDescriptor);
+        }
+    }
+
+    static final class SystemIndexPlugin2 extends Plugin implements SystemIndexPlugin {
+        public static final String SYSTEM_INDEX_2 = ".system-index2";
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            final SystemIndexDescriptor systemIndexDescriptor = new SystemIndexDescriptor(SYSTEM_INDEX_2, "System index 2");
+            return Collections.singletonList(systemIndexDescriptor);
+        }
+    }
+
+    static final class SystemIndexPatternPlugin extends Plugin implements SystemIndexPlugin {
+        public static final String SYSTEM_INDEX_PATTERN = ".system-index-pattern*";
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            final SystemIndexDescriptor systemIndexDescriptor = new SystemIndexDescriptor(SYSTEM_INDEX_PATTERN, "System index pattern");
+            return Collections.singletonList(systemIndexDescriptor);
+        }
+    }
 }

From 13f04d97e3438b0389d1e851dfd08678ab644361 Mon Sep 17 00:00:00 2001
From: Sandesh Kumar <sandeshkr419@gmail.com>
Date: Tue, 9 Jul 2024 11:49:03 -0700
Subject: [PATCH 083/113] Refactor Grok validate pattern to iterative approach
 (#14206)

* grok validate patterns recusrion to iterative

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

* Add max depth in resolving a pattern to avoid OOM

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

* change path from deque to arraylist

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

* rename queue to stack

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

* Change max depth to 500

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

* typo originPatternName fix

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

* spotless

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

---------

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 .../main/java/org/opensearch/grok/Grok.java   | 122 +++++++++++++-----
 .../java/org/opensearch/grok/GrokTests.java   |  10 ++
 3 files changed, 99 insertions(+), 34 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe8d5d524097e..0f683154c52f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -61,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix file cache initialization ([#14004](https://github.com/opensearch-project/OpenSearch/pull/14004))
 - Handle NPE in GetResult if "found" field is missing ([#14552](https://github.com/opensearch-project/OpenSearch/pull/14552))
 - Refactoring FilterPath.parse by using an iterative approach ([#14200](https://github.com/opensearch-project/OpenSearch/pull/14200))
+- Refactoring Grok.validatePatternBank by using an iterative approach ([#14206](https://github.com/opensearch-project/OpenSearch/pull/14206))
 
 ### Security
 
diff --git a/libs/grok/src/main/java/org/opensearch/grok/Grok.java b/libs/grok/src/main/java/org/opensearch/grok/Grok.java
index 7aa3347ba4f4b..aa5b1a936b99d 100644
--- a/libs/grok/src/main/java/org/opensearch/grok/Grok.java
+++ b/libs/grok/src/main/java/org/opensearch/grok/Grok.java
@@ -37,14 +37,18 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Stack;
+import java.util.Set;
 import java.util.function.Consumer;
 
 import org.jcodings.specific.UTF8Encoding;
@@ -86,6 +90,7 @@ public final class Grok {
         UTF8Encoding.INSTANCE,
         Syntax.DEFAULT
     );
+    private static final int MAX_PATTERN_DEPTH_SIZE = 500;
 
     private static final int MAX_TO_REGEX_ITERATIONS = 100_000; // sanity limit
 
@@ -128,7 +133,7 @@ private Grok(
             expressionBytes.length,
             Option.DEFAULT,
             UTF8Encoding.INSTANCE,
-            message -> logCallBack.accept(message)
+            logCallBack::accept
         );
 
         List<GrokCaptureConfig> captureConfig = new ArrayList<>();
@@ -144,7 +149,7 @@ private Grok(
      */
     private void validatePatternBank() {
         for (String patternName : patternBank.keySet()) {
-            validatePatternBank(patternName, new Stack<>());
+            validatePatternBank(patternName);
         }
     }
 
@@ -156,33 +161,84 @@ private void validatePatternBank() {
      * a reference to another named pattern. This method will navigate to all these named patterns and
      * check for a circular reference.
      */
-    private void validatePatternBank(String patternName, Stack<String> path) {
-        String pattern = patternBank.get(patternName);
-        boolean isSelfReference = pattern.contains("%{" + patternName + "}") || pattern.contains("%{" + patternName + ":");
-        if (isSelfReference) {
-            throwExceptionForCircularReference(patternName, pattern);
-        } else if (path.contains(patternName)) {
-            // current pattern name is already in the path, fetch its predecessor
-            String prevPatternName = path.pop();
-            String prevPattern = patternBank.get(prevPatternName);
-            throwExceptionForCircularReference(prevPatternName, prevPattern, patternName, path);
-        }
-        path.push(patternName);
-        for (int i = pattern.indexOf("%{"); i != -1; i = pattern.indexOf("%{", i + 1)) {
-            int begin = i + 2;
-            int syntaxEndIndex = pattern.indexOf('}', begin);
-            if (syntaxEndIndex == -1) {
-                throw new IllegalArgumentException("Malformed pattern [" + patternName + "][" + pattern + "]");
+    private void validatePatternBank(String initialPatternName) {
+        Deque<Frame> stack = new ArrayDeque<>();
+        Set<String> visitedPatterns = new HashSet<>();
+        Map<String, List<String>> pathMap = new HashMap<>();
+
+        List<String> initialPath = new ArrayList<>();
+        initialPath.add(initialPatternName);
+        pathMap.put(initialPatternName, initialPath);
+        stack.push(new Frame(initialPatternName, initialPath, 0));
+
+        while (!stack.isEmpty()) {
+            Frame frame = stack.peek();
+            String patternName = frame.patternName;
+            List<String> path = frame.path;
+            int startIndex = frame.startIndex;
+            String pattern = patternBank.get(patternName);
+
+            if (visitedPatterns.contains(patternName)) {
+                stack.pop();
+                continue;
+            }
+
+            visitedPatterns.add(patternName);
+            boolean foundDependency = false;
+
+            for (int i = startIndex; i < pattern.length(); i++) {
+                if (pattern.startsWith("%{", i)) {
+                    int begin = i + 2;
+                    int syntaxEndIndex = pattern.indexOf('}', begin);
+                    if (syntaxEndIndex == -1) {
+                        throw new IllegalArgumentException("Malformed pattern [" + patternName + "][" + pattern + "]");
+                    }
+
+                    int semanticNameIndex = pattern.indexOf(':', begin);
+                    int end = semanticNameIndex == -1 ? syntaxEndIndex : Math.min(syntaxEndIndex, semanticNameIndex);
+
+                    String dependsOnPattern = pattern.substring(begin, end);
+
+                    if (dependsOnPattern.equals(patternName)) {
+                        throwExceptionForCircularReference(patternName, pattern);
+                    }
+
+                    if (pathMap.containsKey(dependsOnPattern)) {
+                        throwExceptionForCircularReference(patternName, pattern, dependsOnPattern, path.subList(0, path.size() - 1));
+                    }
+
+                    List<String> newPath = new ArrayList<>(path);
+                    newPath.add(dependsOnPattern);
+                    pathMap.put(dependsOnPattern, newPath);
+
+                    stack.push(new Frame(dependsOnPattern, newPath, 0));
+                    frame.startIndex = i + 1;
+                    foundDependency = true;
+                    break;
+                }
             }
-            int semanticNameIndex = pattern.indexOf(':', begin);
-            int end = syntaxEndIndex;
-            if (semanticNameIndex != -1) {
-                end = Math.min(syntaxEndIndex, semanticNameIndex);
+
+            if (!foundDependency) {
+                pathMap.remove(patternName);
+                stack.pop();
+            }
+
+            if (stack.size() > MAX_PATTERN_DEPTH_SIZE) {
+                throw new IllegalArgumentException("Pattern references exceeded maximum depth of " + MAX_PATTERN_DEPTH_SIZE);
             }
-            String dependsOnPattern = pattern.substring(begin, end);
-            validatePatternBank(dependsOnPattern, path);
         }
-        path.pop();
+    }
+
+    private static class Frame {
+        String patternName;
+        List<String> path;
+        int startIndex;
+
+        Frame(String patternName, List<String> path, int startIndex) {
+            this.patternName = patternName;
+            this.path = path;
+            this.startIndex = startIndex;
+        }
     }
 
     private static void throwExceptionForCircularReference(String patternName, String pattern) {
@@ -192,13 +248,13 @@ private static void throwExceptionForCircularReference(String patternName, Strin
     private static void throwExceptionForCircularReference(
         String patternName,
         String pattern,
-        String originPatterName,
-        Stack<String> path
+        String originPatternName,
+        List<String> path
     ) {
         StringBuilder message = new StringBuilder("circular reference in pattern [");
         message.append(patternName).append("][").append(pattern).append("]");
-        if (originPatterName != null) {
-            message.append(" back to pattern [").append(originPatterName).append("]");
+        if (originPatternName != null) {
+            message.append(" back to pattern [").append(originPatternName).append("]");
         }
         if (path != null && path.size() > 1) {
             message.append(" via patterns [").append(String.join("=>", path)).append("]");
@@ -217,9 +273,7 @@ private String groupMatch(String name, Region region, String pattern) {
             int begin = region.getBeg(number);
             int end = region.getEnd(number);
             return new String(pattern.getBytes(StandardCharsets.UTF_8), begin, end - begin, StandardCharsets.UTF_8);
-        } catch (StringIndexOutOfBoundsException e) {
-            return null;
-        } catch (ValueException e) {
+        } catch (StringIndexOutOfBoundsException | ValueException e) {
             return null;
         }
     }
diff --git a/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java b/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java
index a37689e051c67..8476d541aa46e 100644
--- a/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java
+++ b/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java
@@ -377,6 +377,16 @@ public void testCircularReference() {
             "circular reference in pattern [NAME5][!!!%{NAME1}!!!] back to pattern [NAME1] " + "via patterns [NAME1=>NAME2=>NAME3=>NAME4]",
             e.getMessage()
         );
+
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            Map<String, String> bank = new TreeMap<>();
+            for (int i = 1; i <= 501; i++) {
+                bank.put("NAME" + i, "!!!%{NAME" + (i + 1) + "}!!!");
+            }
+            String pattern = "%{NAME1}";
+            new Grok(bank, pattern, false, logger::warn);
+        });
+        assertEquals("Pattern references exceeded maximum depth of 500", e.getMessage());
     }
 
     public void testMalformedPattern() {

From b8dc46d93711882a8fae8b953f11934d940f12a5 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Tue, 9 Jul 2024 16:15:40 -0400
Subject: [PATCH 084/113] Bump opentelemetry from 1.39.0 to 1.40.0 (#14674)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                                 | 5 +++--
 buildSrc/version.properties                                  | 4 ++--
 .../licenses/opentelemetry-api-1.39.0.jar.sha1               | 1 -
 .../licenses/opentelemetry-api-1.40.0.jar.sha1               | 1 +
 .../opentelemetry-api-incubator-1.39.0-alpha.jar.sha1        | 1 -
 .../opentelemetry-api-incubator-1.40.0-alpha.jar.sha1        | 1 +
 .../licenses/opentelemetry-context-1.39.0.jar.sha1           | 1 -
 .../licenses/opentelemetry-context-1.40.0.jar.sha1           | 1 +
 .../licenses/opentelemetry-exporter-common-1.39.0.jar.sha1   | 1 -
 .../licenses/opentelemetry-exporter-common-1.40.0.jar.sha1   | 1 +
 .../licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1  | 1 -
 .../licenses/opentelemetry-exporter-logging-1.40.0.jar.sha1  | 1 +
 .../licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1     | 1 -
 .../licenses/opentelemetry-exporter-otlp-1.40.0.jar.sha1     | 1 +
 .../opentelemetry-exporter-otlp-common-1.39.0.jar.sha1       | 1 -
 .../opentelemetry-exporter-otlp-common-1.40.0.jar.sha1       | 1 +
 .../opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1     | 1 -
 .../opentelemetry-exporter-sender-okhttp-1.40.0.jar.sha1     | 1 +
 .../licenses/opentelemetry-sdk-1.39.0.jar.sha1               | 1 -
 .../licenses/opentelemetry-sdk-1.40.0.jar.sha1               | 1 +
 .../licenses/opentelemetry-sdk-common-1.39.0.jar.sha1        | 1 -
 .../licenses/opentelemetry-sdk-common-1.40.0.jar.sha1        | 1 +
 .../licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1          | 1 -
 .../licenses/opentelemetry-sdk-logs-1.40.0.jar.sha1          | 1 +
 .../licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1       | 1 -
 .../licenses/opentelemetry-sdk-metrics-1.40.0.jar.sha1       | 1 +
 .../licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1         | 1 -
 .../licenses/opentelemetry-sdk-trace-1.40.0.jar.sha1         | 1 +
 .../licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1     | 1 -
 .../licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1     | 1 +
 30 files changed, 19 insertions(+), 18 deletions(-)
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.40.0-alpha.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.40.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0f683154c52f9..1807fb6d5e00c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,8 +26,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.37.3 to 9.40 ([#14398](https://github.com/opensearch-project/OpenSearch/pull/14398))
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
 - Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
-- Bump `opentelemetry` from 1.36.0 to 1.39.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457))
-- Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506))
+- Bump `opentelemetry` from 1.36.0 to 1.40.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457), [#14674](https://github.com/opensearch-project/OpenSearch/pull/14674))
+- Bump `opentelemetry-semconv` from 1.25.0-alpha to 1.26.0-alpha ([#14674](https://github.com/opensearch-project/OpenSearch/pull/14674))
+- Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14673))
 - Bump `com.azure:azure-storage-common` from 12.21.2 to 12.25.1 ([#14517](https://github.com/opensearch-project/OpenSearch/pull/14517))
 - Bump `com.microsoft.azure:msal4j` from 1.15.1 to 1.16.0 ([#14610](https://github.com/opensearch-project/OpenSearch/pull/14610))
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.13 to 0.9.14 ([#14672](https://github.com/opensearch-project/OpenSearch/pull/14672))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index a99bd4801b7f3..a04fb68f47f55 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -74,5 +74,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.39.0
-opentelemetrysemconv  = 1.25.0-alpha
+opentelemetry         = 1.40.0
+opentelemetrysemconv  = 1.26.0-alpha
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1
deleted file mode 100644
index 415fe8f3d8aaa..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-api-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-396b89a66526bd5694ad3bef4604b876177e0b44
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..04ec81edf969c
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-api-1.40.0.jar.sha1
@@ -0,0 +1 @@
+6db562f2b74ffaa7253d740e9aa7a3c4f2e392ec
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1
deleted file mode 100644
index 9c3c9f43d153c..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.39.0-alpha.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1a1fd96155e1b58726300bbf8457630713035e51
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.40.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.40.0-alpha.jar.sha1
new file mode 100644
index 0000000000000..bcd7c886b5f6c
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.40.0-alpha.jar.sha1
@@ -0,0 +1 @@
+43115633361430a3c6aaa39fd78363014ac79270
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1
deleted file mode 100644
index 115d4ccb1f34b..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-context-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f0601fb1c06f661afeffbc73a1dbe29797b2f13b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..9716ec518c886
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-context-1.40.0.jar.sha1
@@ -0,0 +1 @@
+bf1db0f288b9baaabdb439ab6179b673b751511e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1
deleted file mode 100644
index a10b92995becd..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-570d71e39e36fe2caad142557bde0c11fcdb3b92
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..c0e79b05aa675
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.40.0.jar.sha1
@@ -0,0 +1 @@
+b883b179c242a1761df2d408fe01ec41b17327a3
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1
deleted file mode 100644
index f43393104296a..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f5b528f8d6f8531836eabba698979516964b24ed
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..1df0ad183c475
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.40.0.jar.sha1
@@ -0,0 +1 @@
+a8c1f9b05ac9fb1259517cf53950ccecaf84ebe1
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1
deleted file mode 100644
index 5adba2ba0f342..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-04fc0e4983253ea58430c3d24b6b3c5c95f84dc9
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..ebeb639a8459c
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.40.0.jar.sha1
@@ -0,0 +1 @@
+8d8b92bcdb0ace48fb5764cc1ad7a0de197d5b8c
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1
deleted file mode 100644
index ea9c293f25025..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a2b8571e36b11c3153d31ec87ec69cc168af8036
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..b630c808d4763
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.40.0.jar.sha1
@@ -0,0 +1 @@
+80fa10130cc7e7626e2581aa7c5871eab7381889
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1
deleted file mode 100644
index dcf23f16ac89f..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1a8947a2e28924ad9374e319150a23837926ca4b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..eda90dc825e6f
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.40.0.jar.sha1
@@ -0,0 +1 @@
+006dcdbf8eb911ad4d11c54fa824f5a97f582850
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1
deleted file mode 100644
index f603af04d8012..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba9afdf3ef1ea51e42999fd68c959e3ceb219399
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..cdd7dc6551b33
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.40.0.jar.sha1
@@ -0,0 +1 @@
+59f260c5412b79a5a40c7d433600248727cd195a
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1
deleted file mode 100644
index f9419f6ccfbee..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fb8168627bf0059445f61081eaa47c4ab787fc2e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..668291498bbae
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.40.0.jar.sha1
@@ -0,0 +1 @@
+7042214012232a5d6a251aca4aa5932014a4946b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1
deleted file mode 100644
index 63269f239eacd..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b6b45155399bc9fa563945f3e3a77416d7165948
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..74f0786e21954
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.40.0.jar.sha1
@@ -0,0 +1 @@
+1c6b884d65f79d40429263ac0ab7ed1422237837
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1
deleted file mode 100644
index f18c8259c1adc..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-522d46926cc06a4c18829da7e4c4340bdf5673c3
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..23ef1bf6e6b2c
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.40.0.jar.sha1
@@ -0,0 +1 @@
+a1c9b33a8660ace82aecb7f1c7ea50093dc87f0a
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1
deleted file mode 100644
index 03b81424f46d5..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.39.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0b72722a5bbea5f46319bf08b2caed5b8f987a92
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.40.0.jar.sha1
new file mode 100644
index 0000000000000..aea753f0df18b
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.40.0.jar.sha1
@@ -0,0 +1 @@
+5145f077bf2821ad243617baf8c1810d29af8566
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1
deleted file mode 100644
index 7cf8e7e8ede28..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.25.0-alpha.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-76b3d4ca0a8f20b27c1590ceece54f0c7fb5857e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1
new file mode 100644
index 0000000000000..7124dcb31da3f
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1
@@ -0,0 +1 @@
+955de1d2de4d3d2bb6ba2498f19c9a06da2f3956
\ No newline at end of file

From a04cf24923e7379d94314ae4f03e2bddfddf1765 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Tue, 9 Jul 2024 17:07:54 -0400
Subject: [PATCH 085/113] Bump jackson from 2.17.1 to 2.17.2 (#14687)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                                  | 1 +
 buildSrc/version.properties                                   | 4 ++--
 client/sniffer/licenses/jackson-core-2.17.1.jar.sha1          | 1 -
 client/sniffer/licenses/jackson-core-2.17.2.jar.sha1          | 1 +
 .../upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1  | 1 -
 .../upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1  | 1 +
 .../upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1     | 1 -
 .../upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1     | 1 +
 libs/core/licenses/jackson-core-2.17.1.jar.sha1               | 1 -
 libs/core/licenses/jackson-core-2.17.2.jar.sha1               | 1 +
 libs/x-content/licenses/jackson-core-2.17.1.jar.sha1          | 1 -
 libs/x-content/licenses/jackson-core-2.17.2.jar.sha1          | 1 +
 .../licenses/jackson-dataformat-cbor-2.17.1.jar.sha1          | 1 -
 .../licenses/jackson-dataformat-cbor-2.17.2.jar.sha1          | 1 +
 .../licenses/jackson-dataformat-smile-2.17.1.jar.sha1         | 1 -
 .../licenses/jackson-dataformat-smile-2.17.2.jar.sha1         | 1 +
 .../licenses/jackson-dataformat-yaml-2.17.1.jar.sha1          | 1 -
 .../licenses/jackson-dataformat-yaml-2.17.2.jar.sha1          | 1 +
 .../ingest-geoip/licenses/jackson-annotations-2.17.1.jar.sha1 | 1 -
 .../ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1 | 1 +
 .../ingest-geoip/licenses/jackson-databind-2.17.1.jar.sha1    | 1 -
 .../ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1    | 1 +
 .../crypto-kms/licenses/jackson-annotations-2.17.1.jar.sha1   | 1 -
 .../crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1   | 1 +
 plugins/crypto-kms/licenses/jackson-databind-2.17.1.jar.sha1  | 1 -
 plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1  | 1 +
 .../licenses/jackson-annotations-2.17.1.jar.sha1              | 1 -
 .../licenses/jackson-annotations-2.17.2.jar.sha1              | 1 +
 .../discovery-ec2/licenses/jackson-databind-2.17.1.jar.sha1   | 1 -
 .../discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1   | 1 +
 .../licenses/jackson-annotations-2.17.1.jar.sha1              | 1 -
 .../licenses/jackson-annotations-2.17.2.jar.sha1              | 1 +
 .../licenses/jackson-databind-2.17.1.jar.sha1                 | 1 -
 .../licenses/jackson-databind-2.17.2.jar.sha1                 | 1 +
 .../licenses/jackson-dataformat-xml-2.17.1.jar.sha1           | 1 -
 .../licenses/jackson-dataformat-xml-2.17.2.jar.sha1           | 1 +
 .../licenses/jackson-datatype-jsr310-2.17.1.jar.sha1          | 1 -
 .../licenses/jackson-datatype-jsr310-2.17.2.jar.sha1          | 1 +
 .../licenses/jackson-module-jaxb-annotations-2.17.1.jar.sha1  | 1 -
 .../licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1  | 1 +
 .../licenses/jackson-annotations-2.17.1.jar.sha1              | 1 -
 .../licenses/jackson-annotations-2.17.2.jar.sha1              | 1 +
 .../repository-s3/licenses/jackson-databind-2.17.1.jar.sha1   | 1 -
 .../repository-s3/licenses/jackson-databind-2.17.2.jar.sha1   | 1 +
 server/licenses/jackson-core-2.17.1.jar.sha1                  | 1 -
 server/licenses/jackson-core-2.17.2.jar.sha1                  | 1 +
 server/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1       | 1 -
 server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1       | 1 +
 server/licenses/jackson-dataformat-smile-2.17.1.jar.sha1      | 1 -
 server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1      | 1 +
 server/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1       | 1 -
 server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1       | 1 +
 52 files changed, 28 insertions(+), 27 deletions(-)
 delete mode 100644 client/sniffer/licenses/jackson-core-2.17.1.jar.sha1
 create mode 100644 client/sniffer/licenses/jackson-core-2.17.2.jar.sha1
 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1
 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1
 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1
 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1
 delete mode 100644 libs/core/licenses/jackson-core-2.17.1.jar.sha1
 create mode 100644 libs/core/licenses/jackson-core-2.17.2.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-core-2.17.1.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-core-2.17.2.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.17.1.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.17.1.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1
 delete mode 100644 plugins/crypto-kms/licenses/jackson-annotations-2.17.1.jar.sha1
 create mode 100644 plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1
 delete mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.17.1.jar.sha1
 create mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.17.1.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.17.1.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.17.1.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/jackson-databind-2.17.1.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.1.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.1.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.17.1.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.17.1.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1
 delete mode 100644 server/licenses/jackson-core-2.17.1.jar.sha1
 create mode 100644 server/licenses/jackson-core-2.17.2.jar.sha1
 delete mode 100644 server/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
 create mode 100644 server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
 delete mode 100644 server/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
 create mode 100644 server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
 delete mode 100644 server/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1
 create mode 100644 server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1807fb6d5e00c..fe26423cde573 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.microsoft.azure:msal4j` from 1.15.1 to 1.16.0 ([#14610](https://github.com/opensearch-project/OpenSearch/pull/14610))
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.13 to 0.9.14 ([#14672](https://github.com/opensearch-project/OpenSearch/pull/14672))
 - Bump `net.minidev:accessors-smart` from 2.5.0 to 2.5.1 ([#14673](https://github.com/opensearch-project/OpenSearch/pull/14673))
+- Bump `jackson` from 2.17.1 to 2.17.2 ([#14687](https://github.com/opensearch-project/OpenSearch/pull/14687))
 
 ### Changed
 - [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index a04fb68f47f55..d62f8c51e616b 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -7,8 +7,8 @@ bundled_jdk = 21.0.3+9
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.17.1
-jackson_databind  = 2.17.1
+jackson           = 2.17.2
+jackson_databind  = 2.17.2
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
diff --git a/client/sniffer/licenses/jackson-core-2.17.1.jar.sha1 b/client/sniffer/licenses/jackson-core-2.17.1.jar.sha1
deleted file mode 100644
index 82dab5981e652..0000000000000
--- a/client/sniffer/licenses/jackson-core-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e52a11644cd59a28ef79f02bddc2cc3bab45edb
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.17.2.jar.sha1 b/client/sniffer/licenses/jackson-core-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..e15f2340980bc
--- /dev/null
+++ b/client/sniffer/licenses/jackson-core-2.17.2.jar.sha1
@@ -0,0 +1 @@
+969a35cb35c86512acbadcdbbbfb044c877db814
\ No newline at end of file
diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 4ceead1b7ae4f..0000000000000
--- a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca7ef6192c9ad05d07bc50da991bf937a84af3a
\ No newline at end of file
diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..411e1d62459fd
--- /dev/null
+++ b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+147b7b9412ffff24339f8aba080b292448e08698
\ No newline at end of file
diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1
deleted file mode 100644
index 7cf1ac1b60301..0000000000000
--- a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0524dcbcccdde7d45a679dfc333e4763feb09079
\ No newline at end of file
diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f2b4dbdc5decb
--- /dev/null
+++ b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1
@@ -0,0 +1 @@
+e6deb029e5901e027c129341fac39e515066b68c
\ No newline at end of file
diff --git a/libs/core/licenses/jackson-core-2.17.1.jar.sha1 b/libs/core/licenses/jackson-core-2.17.1.jar.sha1
deleted file mode 100644
index 82dab5981e652..0000000000000
--- a/libs/core/licenses/jackson-core-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e52a11644cd59a28ef79f02bddc2cc3bab45edb
\ No newline at end of file
diff --git a/libs/core/licenses/jackson-core-2.17.2.jar.sha1 b/libs/core/licenses/jackson-core-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..e15f2340980bc
--- /dev/null
+++ b/libs/core/licenses/jackson-core-2.17.2.jar.sha1
@@ -0,0 +1 @@
+969a35cb35c86512acbadcdbbbfb044c877db814
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-core-2.17.1.jar.sha1 b/libs/x-content/licenses/jackson-core-2.17.1.jar.sha1
deleted file mode 100644
index 82dab5981e652..0000000000000
--- a/libs/x-content/licenses/jackson-core-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e52a11644cd59a28ef79f02bddc2cc3bab45edb
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-core-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-core-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..e15f2340980bc
--- /dev/null
+++ b/libs/x-content/licenses/jackson-core-2.17.2.jar.sha1
@@ -0,0 +1 @@
+969a35cb35c86512acbadcdbbbfb044c877db814
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
deleted file mode 100644
index ff42ed1f92cfe..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba5d8e6ecc62aa0e49c0ce935b8696352dbebc71
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..069e088413ef1
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
@@ -0,0 +1 @@
+57fa7c1b5104bbc4599278d13933a937ee058e68
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.17.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
deleted file mode 100644
index 47d19067cf2a6..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-89683ac4f0a0c2c4f69ea56b90480ed40266dac8
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..28d8c8382aed3
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
@@ -0,0 +1 @@
+20e956b9b6f67138edd39fab7a506ded19638bcb
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1
deleted file mode 100644
index 7946e994c7104..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4c7b8a9ea3f398116a75c146b982b22afebc4ee
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f3e25b7eb253c
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1
@@ -0,0 +1 @@
+78d2c73dbec62044d7cf3b544b2e0d24a1a093b0
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.17.1.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 4ceead1b7ae4f..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca7ef6192c9ad05d07bc50da991bf937a84af3a
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..411e1d62459fd
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+147b7b9412ffff24339f8aba080b292448e08698
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.17.1.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.17.1.jar.sha1
deleted file mode 100644
index 7cf1ac1b60301..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0524dcbcccdde7d45a679dfc333e4763feb09079
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f2b4dbdc5decb
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1
@@ -0,0 +1 @@
+e6deb029e5901e027c129341fac39e515066b68c
\ No newline at end of file
diff --git a/plugins/crypto-kms/licenses/jackson-annotations-2.17.1.jar.sha1 b/plugins/crypto-kms/licenses/jackson-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 4ceead1b7ae4f..0000000000000
--- a/plugins/crypto-kms/licenses/jackson-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca7ef6192c9ad05d07bc50da991bf937a84af3a
\ No newline at end of file
diff --git a/plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..411e1d62459fd
--- /dev/null
+++ b/plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+147b7b9412ffff24339f8aba080b292448e08698
\ No newline at end of file
diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.17.1.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.17.1.jar.sha1
deleted file mode 100644
index 7cf1ac1b60301..0000000000000
--- a/plugins/crypto-kms/licenses/jackson-databind-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0524dcbcccdde7d45a679dfc333e4763feb09079
\ No newline at end of file
diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f2b4dbdc5decb
--- /dev/null
+++ b/plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1
@@ -0,0 +1 @@
+e6deb029e5901e027c129341fac39e515066b68c
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.17.1.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 4ceead1b7ae4f..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca7ef6192c9ad05d07bc50da991bf937a84af3a
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..411e1d62459fd
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+147b7b9412ffff24339f8aba080b292448e08698
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.17.1.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.17.1.jar.sha1
deleted file mode 100644
index 7cf1ac1b60301..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0524dcbcccdde7d45a679dfc333e4763feb09079
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f2b4dbdc5decb
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1
@@ -0,0 +1 @@
+e6deb029e5901e027c129341fac39e515066b68c
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.17.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 4ceead1b7ae4f..0000000000000
--- a/plugins/repository-azure/licenses/jackson-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca7ef6192c9ad05d07bc50da991bf937a84af3a
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..411e1d62459fd
--- /dev/null
+++ b/plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+147b7b9412ffff24339f8aba080b292448e08698
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-databind-2.17.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.17.1.jar.sha1
deleted file mode 100644
index 7cf1ac1b60301..0000000000000
--- a/plugins/repository-azure/licenses/jackson-databind-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0524dcbcccdde7d45a679dfc333e4763feb09079
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f2b4dbdc5decb
--- /dev/null
+++ b/plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1
@@ -0,0 +1 @@
+e6deb029e5901e027c129341fac39e515066b68c
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.1.jar.sha1
deleted file mode 100644
index 3915ab2616beb..0000000000000
--- a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e6a168dba62aa63743b9e2b83f4e0f0dfdc143d3
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f9c31c168926d
--- /dev/null
+++ b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1
@@ -0,0 +1 @@
+ad58f5bd089e743ac6e5999b2d1e3cf8515cea9a
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1
deleted file mode 100644
index db26ebbf738f7..0000000000000
--- a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0969b0c3cb8c75d759e9a6c585c44c9b9f3a4f75
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..a61bf643d69e6
--- /dev/null
+++ b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1
@@ -0,0 +1 @@
+267b85e9ba2892a37be6d80aa9ca1438a0d8c210
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.1.jar.sha1
deleted file mode 100644
index bb8ecfe34d295..0000000000000
--- a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f77e7bf0e64dfcf53bfdcf2764ad7ab92b78a4da
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..d9d7975146c22
--- /dev/null
+++ b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+c2978b818ef2f2b2738b387c143624eab611d917
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.17.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 4ceead1b7ae4f..0000000000000
--- a/plugins/repository-s3/licenses/jackson-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca7ef6192c9ad05d07bc50da991bf937a84af3a
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..411e1d62459fd
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+147b7b9412ffff24339f8aba080b292448e08698
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.17.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.17.1.jar.sha1
deleted file mode 100644
index 7cf1ac1b60301..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0524dcbcccdde7d45a679dfc333e4763feb09079
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f2b4dbdc5decb
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1
@@ -0,0 +1 @@
+e6deb029e5901e027c129341fac39e515066b68c
\ No newline at end of file
diff --git a/server/licenses/jackson-core-2.17.1.jar.sha1 b/server/licenses/jackson-core-2.17.1.jar.sha1
deleted file mode 100644
index 82dab5981e652..0000000000000
--- a/server/licenses/jackson-core-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e52a11644cd59a28ef79f02bddc2cc3bab45edb
\ No newline at end of file
diff --git a/server/licenses/jackson-core-2.17.2.jar.sha1 b/server/licenses/jackson-core-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..e15f2340980bc
--- /dev/null
+++ b/server/licenses/jackson-core-2.17.2.jar.sha1
@@ -0,0 +1 @@
+969a35cb35c86512acbadcdbbbfb044c877db814
\ No newline at end of file
diff --git a/server/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1 b/server/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
deleted file mode 100644
index ff42ed1f92cfe..0000000000000
--- a/server/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba5d8e6ecc62aa0e49c0ce935b8696352dbebc71
\ No newline at end of file
diff --git a/server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 b/server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..069e088413ef1
--- /dev/null
+++ b/server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
@@ -0,0 +1 @@
+57fa7c1b5104bbc4599278d13933a937ee058e68
\ No newline at end of file
diff --git a/server/licenses/jackson-dataformat-smile-2.17.1.jar.sha1 b/server/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
deleted file mode 100644
index 47d19067cf2a6..0000000000000
--- a/server/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-89683ac4f0a0c2c4f69ea56b90480ed40266dac8
\ No newline at end of file
diff --git a/server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 b/server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..28d8c8382aed3
--- /dev/null
+++ b/server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
@@ -0,0 +1 @@
+20e956b9b6f67138edd39fab7a506ded19638bcb
\ No newline at end of file
diff --git a/server/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1 b/server/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1
deleted file mode 100644
index 7946e994c7104..0000000000000
--- a/server/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4c7b8a9ea3f398116a75c146b982b22afebc4ee
\ No newline at end of file
diff --git a/server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 b/server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1
new file mode 100644
index 0000000000000..f3e25b7eb253c
--- /dev/null
+++ b/server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1
@@ -0,0 +1 @@
+78d2c73dbec62044d7cf3b544b2e0d24a1a093b0
\ No newline at end of file

From 5ef41cf483581b86abc145091fea9dd6f636f503 Mon Sep 17 00:00:00 2001
From: Zelin Hao <zelinhao@amazon.com>
Date: Tue, 9 Jul 2024 14:57:05 -0700
Subject: [PATCH 086/113] Add release notes for release 1.3.18 (#14699)

Signed-off-by: Zelin Hao <zelinhao@amazon.com>
---
 release-notes/opensearch.release-notes-1.3.18.md | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 release-notes/opensearch.release-notes-1.3.18.md

diff --git a/release-notes/opensearch.release-notes-1.3.18.md b/release-notes/opensearch.release-notes-1.3.18.md
new file mode 100644
index 0000000000000..75c38dd285a63
--- /dev/null
+++ b/release-notes/opensearch.release-notes-1.3.18.md
@@ -0,0 +1,4 @@
+## 2024-07-09 Version 1.3.18 Release Notes
+
+### Upgrades
+- Bump `netty` from 4.1.110.Final to 4.1.111.Final ([#14356](https://github.com/opensearch-project/OpenSearch/pull/14356))

From c4d960fcca1ae6c3191a5c474cbc867b4431d621 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Tue, 9 Jul 2024 21:20:17 -0400
Subject: [PATCH 087/113] Bump reactor from 3.5.19 to 3.5.20 (#14697)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                                  | 4 ++--
 buildSrc/version.properties                                   | 4 ++--
 .../licenses/reactor-netty-core-1.1.20.jar.sha1               | 1 -
 .../licenses/reactor-netty-core-1.1.21.jar.sha1               | 1 +
 .../licenses/reactor-netty-http-1.1.20.jar.sha1               | 1 -
 .../licenses/reactor-netty-http-1.1.21.jar.sha1               | 1 +
 .../licenses/reactor-netty-core-1.1.20.jar.sha1               | 1 -
 .../licenses/reactor-netty-core-1.1.21.jar.sha1               | 1 +
 .../licenses/reactor-netty-http-1.1.20.jar.sha1               | 1 -
 .../licenses/reactor-netty-http-1.1.21.jar.sha1               | 1 +
 server/licenses/reactor-core-3.5.18.jar.sha1                  | 1 -
 server/licenses/reactor-core-3.5.19.jar.sha1                  | 1 +
 12 files changed, 9 insertions(+), 9 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.21.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.21.jar.sha1
 delete mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.21.jar.sha1
 delete mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.21.jar.sha1
 delete mode 100644 server/licenses/reactor-core-3.5.18.jar.sha1
 create mode 100644 server/licenses/reactor-core-3.5.19.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe26423cde573..62bb73d80f2c1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,8 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
 - Bump `netty` from 4.1.110.Final to 4.1.111.Final ([#14356](https://github.com/opensearch-project/OpenSearch/pull/14356))
 - Bump `org.wiremock:wiremock-standalone` from 3.3.1 to 3.6.0 ([#14361](https://github.com/opensearch-project/OpenSearch/pull/14361))
-- Bump `reactor` from 3.5.17 to 3.5.18 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
-- Bump `reactor-netty` from 1.1.19 to 1.1.20 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
+- Bump `reactor` from 3.5.17 to 3.5.19 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395), [#14697](https://github.com/opensearch-project/OpenSearch/pull/14697))
+- Bump `reactor-netty` from 1.1.19 to 1.1.21 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395), [#14697](https://github.com/opensearch-project/OpenSearch/pull/14697))
 - Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.37.3 to 9.40 ([#14398](https://github.com/opensearch-project/OpenSearch/pull/14398))
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index d62f8c51e616b..855ccc1f87413 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -33,8 +33,8 @@ netty             = 4.1.111.Final
 joda              = 2.12.7
 
 # project reactor
-reactor_netty     = 1.1.20
-reactor           = 3.5.18
+reactor_netty     = 1.1.21
+reactor           = 3.5.19
 
 # client dependencies
 httpclient5       = 5.2.1
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1
deleted file mode 100644
index 2f4d023c88c80..0000000000000
--- a/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1a5ef52a470a82d9313e2e1ad8ba064bdbd38948
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.21.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.21.jar.sha1
new file mode 100644
index 0000000000000..21c16c7430016
--- /dev/null
+++ b/plugins/repository-azure/licenses/reactor-netty-core-1.1.21.jar.sha1
@@ -0,0 +1 @@
+acb98bd08107287c454ce74e7b1ed8e7a018a662
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1
deleted file mode 100644
index 6c031e00e39c1..0000000000000
--- a/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8d4ee98405a5856cf0c9d7c1a70f3f14631e3c46
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.21.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.21.jar.sha1
new file mode 100644
index 0000000000000..648df22873d56
--- /dev/null
+++ b/plugins/repository-azure/licenses/reactor-netty-http-1.1.21.jar.sha1
@@ -0,0 +1 @@
+b83542bb35630ef815b4177e3c670f62e952e695
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1
deleted file mode 100644
index 2f4d023c88c80..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1a5ef52a470a82d9313e2e1ad8ba064bdbd38948
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.21.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.21.jar.sha1
new file mode 100644
index 0000000000000..21c16c7430016
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.21.jar.sha1
@@ -0,0 +1 @@
+acb98bd08107287c454ce74e7b1ed8e7a018a662
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1
deleted file mode 100644
index 6c031e00e39c1..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8d4ee98405a5856cf0c9d7c1a70f3f14631e3c46
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.21.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.21.jar.sha1
new file mode 100644
index 0000000000000..648df22873d56
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.21.jar.sha1
@@ -0,0 +1 @@
+b83542bb35630ef815b4177e3c670f62e952e695
\ No newline at end of file
diff --git a/server/licenses/reactor-core-3.5.18.jar.sha1 b/server/licenses/reactor-core-3.5.18.jar.sha1
deleted file mode 100644
index c503f768beafa..0000000000000
--- a/server/licenses/reactor-core-3.5.18.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3a8157f7d66d71a407eb77ba12bce72a38c5b4da
\ No newline at end of file
diff --git a/server/licenses/reactor-core-3.5.19.jar.sha1 b/server/licenses/reactor-core-3.5.19.jar.sha1
new file mode 100644
index 0000000000000..04b59d2faae04
--- /dev/null
+++ b/server/licenses/reactor-core-3.5.19.jar.sha1
@@ -0,0 +1 @@
+1d49ce1d0df79f28d3927da5f4c46a895b94335f
\ No newline at end of file

From b068355d0d81e031a4022efcb713c93fa4ff0b31 Mon Sep 17 00:00:00 2001
From: Shivansh Arora <hishiv@amazon.com>
Date: Wed, 10 Jul 2024 12:18:49 +0530
Subject: [PATCH 088/113] Add unit tests for read flow of
 RemoteClusterStateService and bug fix for transient settings (#14476)

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
---
 .../PublicationTransportHandler.java          |    1 -
 .../remote/ClusterStateDiffManifest.java      |   28 +-
 .../remote/RemoteClusterStateService.java     |   28 +-
 ...oteClusterStateAttributesManagerTests.java |  165 +--
 .../RemoteClusterStateServiceTests.java       | 1214 ++++++++++++++++-
 .../remote/RemoteClusterStateTestUtils.java   |  227 +++
 .../RemoteGlobalMetadataManagerTests.java     |  125 +-
 .../test/TestClusterStateCustom.java          |   68 +
 8 files changed, 1531 insertions(+), 325 deletions(-)
 create mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateTestUtils.java
 create mode 100644 test/framework/src/main/java/org/opensearch/test/TestClusterStateCustom.java

diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java
index 36eabd51ffda1..62885a12222be 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java
@@ -284,7 +284,6 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest
                 )
             );
             ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(
-                request.getClusterName(),
                 manifest,
                 lastSeen,
                 transportService.getLocalNode().getId()
diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java
index 65ae2675a95da..aca53c92781e4 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java
@@ -25,6 +25,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -152,17 +153,17 @@ public ClusterStateDiffManifest(
         this.settingsMetadataUpdated = settingsMetadataUpdated;
         this.transientSettingsMetadataUpdated = transientSettingsMetadataUpdate;
         this.templatesMetadataUpdated = templatesMetadataUpdated;
-        this.customMetadataUpdated = customMetadataUpdated;
-        this.customMetadataDeleted = customMetadataDeleted;
-        this.indicesUpdated = indicesUpdated;
-        this.indicesDeleted = indicesDeleted;
+        this.customMetadataUpdated = Collections.unmodifiableList(customMetadataUpdated);
+        this.customMetadataDeleted = Collections.unmodifiableList(customMetadataDeleted);
+        this.indicesUpdated = Collections.unmodifiableList(indicesUpdated);
+        this.indicesDeleted = Collections.unmodifiableList(indicesDeleted);
         this.clusterBlocksUpdated = clusterBlocksUpdated;
         this.discoveryNodesUpdated = discoveryNodesUpdated;
-        this.indicesRoutingUpdated = indicesRoutingUpdated;
-        this.indicesRoutingDeleted = indicesRoutingDeleted;
+        this.indicesRoutingUpdated = Collections.unmodifiableList(indicesRoutingUpdated);
+        this.indicesRoutingDeleted = Collections.unmodifiableList(indicesRoutingDeleted);
         this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated;
-        this.clusterStateCustomUpdated = clusterStateCustomUpdated;
-        this.clusterStateCustomDeleted = clusterStateCustomDeleted;
+        this.clusterStateCustomUpdated = Collections.unmodifiableList(clusterStateCustomUpdated);
+        this.clusterStateCustomDeleted = Collections.unmodifiableList(clusterStateCustomDeleted);
     }
 
     public ClusterStateDiffManifest(StreamInput in) throws IOException {
@@ -563,7 +564,16 @@ public static class Builder {
         private List<String> clusterStateCustomUpdated;
         private List<String> clusterStateCustomDeleted;
 
-        public Builder() {}
+        public Builder() {
+            customMetadataUpdated = Collections.emptyList();
+            customMetadataDeleted = Collections.emptyList();
+            indicesUpdated = Collections.emptyList();
+            indicesDeleted = Collections.emptyList();
+            indicesRoutingUpdated = Collections.emptyList();
+            indicesRoutingDeleted = Collections.emptyList();
+            clusterStateCustomUpdated = Collections.emptyList();
+            clusterStateCustomDeleted = Collections.emptyList();
+        }
 
         public Builder fromStateUUID(String fromStateUUID) {
             this.fromStateUUID = fromStateUUID;
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 74abe9cd257b4..3e63f9114ea16 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -976,7 +976,8 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID
         return getClusterStateForManifest(clusterName, clusterMetadataManifest.get(), nodeId, includeEphemeral);
     }
 
-    private ClusterState readClusterStateInParallel(
+    // package private for testing
+    ClusterState readClusterStateInParallel(
         ClusterState previousState,
         ClusterMetadataManifest manifest,
         String clusterUUID,
@@ -1285,7 +1286,7 @@ public ClusterState getClusterStateForManifest(
                 manifest.getCustomMetadataMap(),
                 manifest.getCoordinationMetadata() != null,
                 manifest.getSettingsMetadata() != null,
-                manifest.getTransientSettingsMetadata() != null,
+                includeEphemeral && manifest.getTransientSettingsMetadata() != null,
                 manifest.getTemplatesMetadata() != null,
                 includeEphemeral && manifest.getDiscoveryNodesMetadata() != null,
                 includeEphemeral && manifest.getClusterBlocksMetadata() != null,
@@ -1321,13 +1322,9 @@ public ClusterState getClusterStateForManifest(
 
     }
 
-    public ClusterState getClusterStateUsingDiff(
-        String clusterName,
-        ClusterMetadataManifest manifest,
-        ClusterState previousState,
-        String localNodeId
-    ) throws IOException {
-        assert manifest.getDiffManifest() != null;
+    public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId)
+        throws IOException {
+        assert manifest.getDiffManifest() != null : "Diff manifest null which is required for downloading cluster state";
         ClusterStateDiffManifest diff = manifest.getDiffManifest();
         List<UploadedIndexMetadata> updatedIndices = diff.getIndicesUpdated().stream().map(idx -> {
             Optional<UploadedIndexMetadata> uploadedIndexMetadataOptional = manifest.getIndices()
@@ -1586,6 +1583,19 @@ private boolean isValidClusterUUID(ClusterMetadataManifest manifest) {
         return manifest.isClusterUUIDCommitted();
     }
 
+    // package private setter which are required for injecting mock managers, these setters are not supposed to be used elsewhere
+    void setRemoteIndexMetadataManager(RemoteIndexMetadataManager remoteIndexMetadataManager) {
+        this.remoteIndexMetadataManager = remoteIndexMetadataManager;
+    }
+
+    void setRemoteGlobalMetadataManager(RemoteGlobalMetadataManager remoteGlobalMetadataManager) {
+        this.remoteGlobalMetadataManager = remoteGlobalMetadataManager;
+    }
+
+    void setRemoteClusterStateAttributesManager(RemoteClusterStateAttributesManager remoteClusterStateAttributeManager) {
+        this.remoteClusterStateAttributesManager = remoteClusterStateAttributeManager;
+    }
+
     public void writeMetadataFailed() {
         getStats().stateFailed();
     }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
index fe9ed57fa77b8..3f2edd1a6c5a5 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java
@@ -8,9 +8,7 @@
 
 package org.opensearch.gateway.remote;
 
-import org.opensearch.Version;
 import org.opensearch.action.LatchedActionListener;
-import org.opensearch.cluster.AbstractNamedDiffable;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterState.Custom;
@@ -23,11 +21,8 @@
 import org.opensearch.common.util.TestCapturingListener;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.compress.Compressor;
 import org.opensearch.core.compress.NoneCompressor;
-import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.gateway.remote.model.RemoteClusterBlocks;
 import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms;
 import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes;
@@ -51,6 +46,10 @@
 import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE;
 import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION;
 import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom1;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom2;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom3;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom4;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER;
@@ -338,22 +337,22 @@ public void testGetAsyncMetadataReadAction_Exception() throws IOException, Inter
 
     public void testGetUpdatedCustoms() {
         Map<String, ClusterState.Custom> previousCustoms = Map.of(
-            TestCustom1.TYPE,
-            new TestCustom1("data1"),
-            TestCustom2.TYPE,
-            new TestCustom2("data2"),
-            TestCustom3.TYPE,
-            new TestCustom3("data3")
+            TestClusterStateCustom1.TYPE,
+            new TestClusterStateCustom1("data1"),
+            TestClusterStateCustom2.TYPE,
+            new TestClusterStateCustom2("data2"),
+            TestClusterStateCustom3.TYPE,
+            new TestClusterStateCustom3("data3")
         );
         ClusterState previousState = ClusterState.builder(new ClusterName("test-cluster")).customs(previousCustoms).build();
 
         Map<String, Custom> currentCustoms = Map.of(
-            TestCustom2.TYPE,
-            new TestCustom2("data2"),
-            TestCustom3.TYPE,
-            new TestCustom3("data3-changed"),
-            TestCustom4.TYPE,
-            new TestCustom4("data4")
+            TestClusterStateCustom2.TYPE,
+            new TestClusterStateCustom2("data2"),
+            TestClusterStateCustom3.TYPE,
+            new TestClusterStateCustom3("data3-changed"),
+            TestClusterStateCustom4.TYPE,
+            new TestClusterStateCustom4("data4")
         );
 
         ClusterState currentState = ClusterState.builder(new ClusterName("test-cluster")).customs(currentCustoms).build();
@@ -368,136 +367,14 @@ public void testGetUpdatedCustoms() {
         assertThat(customsDiff.getDeletes(), is(Collections.emptyList()));
 
         Map<String, ClusterState.Custom> expectedCustoms = Map.of(
-            TestCustom3.TYPE,
-            new TestCustom3("data3-changed"),
-            TestCustom4.TYPE,
-            new TestCustom4("data4")
+            TestClusterStateCustom3.TYPE,
+            new TestClusterStateCustom3("data3-changed"),
+            TestClusterStateCustom4.TYPE,
+            new TestClusterStateCustom4("data4")
         );
 
         customsDiff = remoteClusterStateAttributesManager.getUpdatedCustoms(currentState, previousState, true, false);
         assertThat(customsDiff.getUpserts(), is(expectedCustoms));
-        assertThat(customsDiff.getDeletes(), is(List.of(TestCustom1.TYPE)));
-    }
-
-    private static abstract class AbstractTestCustom extends AbstractNamedDiffable<Custom> implements ClusterState.Custom {
-
-        private final String value;
-
-        AbstractTestCustom(String value) {
-            this.value = value;
-        }
-
-        AbstractTestCustom(StreamInput in) throws IOException {
-            this.value = in.readString();
-        }
-
-        @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(value);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder;
-        }
-
-        @Override
-        public boolean isPrivate() {
-            return true;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            AbstractTestCustom that = (AbstractTestCustom) o;
-
-            if (!value.equals(that.value)) return false;
-
-            return true;
-        }
-
-        @Override
-        public int hashCode() {
-            return value.hashCode();
-        }
-    }
-
-    private static class TestCustom1 extends AbstractTestCustom {
-
-        private static final String TYPE = "custom_1";
-
-        TestCustom1(String value) {
-            super(value);
-        }
-
-        TestCustom1(StreamInput in) throws IOException {
-            super(in);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-    }
-
-    private static class TestCustom2 extends AbstractTestCustom {
-
-        private static final String TYPE = "custom_2";
-
-        TestCustom2(String value) {
-            super(value);
-        }
-
-        TestCustom2(StreamInput in) throws IOException {
-            super(in);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-    }
-
-    private static class TestCustom3 extends AbstractTestCustom {
-
-        private static final String TYPE = "custom_3";
-
-        TestCustom3(String value) {
-            super(value);
-        }
-
-        TestCustom3(StreamInput in) throws IOException {
-            super(in);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-    }
-
-    private static class TestCustom4 extends AbstractTestCustom {
-
-        private static final String TYPE = "custom_4";
-
-        TestCustom4(String value) {
-            super(value);
-        }
-
-        TestCustom4(StreamInput in) throws IOException {
-            super(in);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
+        assertThat(customsDiff.getDeletes(), is(List.of(TestClusterStateCustom1.TYPE)));
     }
 }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index d983a4d8c4027..91ddd64cc2ccc 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -9,12 +9,14 @@
 package org.opensearch.gateway.remote;
 
 import org.opensearch.Version;
+import org.opensearch.action.LatchedActionListener;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.RepositoryCleanupInProgress;
-import org.opensearch.cluster.RepositoryCleanupInProgress.Entry;
+import org.opensearch.cluster.block.ClusterBlocks;
 import org.opensearch.cluster.coordination.CoordinationMetadata;
+import org.opensearch.cluster.metadata.DiffableStringMap;
 import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexTemplateMetadata;
@@ -22,10 +24,12 @@
 import org.opensearch.cluster.metadata.TemplatesMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService;
 import org.opensearch.cluster.routing.remote.NoopRemoteRoutingTableService;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobMetadata;
@@ -38,6 +42,7 @@
 import org.opensearch.common.compress.DeflateCompressor;
 import org.opensearch.common.lucene.store.ByteArrayIndexInput;
 import org.opensearch.common.network.NetworkModule;
+import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
@@ -45,13 +50,17 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.compress.Compressor;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute;
 import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest;
 import org.opensearch.gateway.remote.model.RemoteClusterStateManifestInfo;
-import org.opensearch.gateway.remote.model.RemoteIndexMetadata;
+import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata;
+import org.opensearch.gateway.remote.model.RemoteReadResult;
+import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata;
 import org.opensearch.index.remote.RemoteIndexPathUploader;
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.repositories.FilterRepository;
@@ -61,7 +70,6 @@
 import org.opensearch.repositories.blobstore.ChecksumWritableBlobStoreFormat;
 import org.opensearch.repositories.fs.FsRepository;
 import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.test.TestCustomMetadata;
 import org.opensearch.test.VersionUtils;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -75,7 +83,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -92,23 +99,51 @@
 import java.util.stream.Stream;
 
 import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatcher;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static java.util.stream.Collectors.toList;
 import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL;
 import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1;
+import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2;
+import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS;
+import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata1;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata2;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata3;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom1;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom2;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom3;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getFormattedIndexFileName;
+import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteClusterBlocksTests.randomClusterBlocks;
 import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION;
+import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM;
 import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER;
+import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA;
+import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.readFrom;
+import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES;
+import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodesTests.getDiscoveryNodes;
 import static org.opensearch.gateway.remote.model.RemoteGlobalMetadata.GLOBAL_METADATA_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS;
+import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettingsTests.getHashesOfConsistentSettings;
+import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX;
+import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_METADATA_FORMAT;
 import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTINGS_METADATA_FORMAT;
 import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA_FORMAT;
+import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadataTests.getTemplatesMetadata;
+import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
@@ -120,11 +155,19 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyMap;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class RemoteClusterStateServiceTests extends OpenSearchTestCase {
@@ -135,11 +178,22 @@ public class RemoteClusterStateServiceTests extends OpenSearchTestCase {
     private Supplier<RepositoriesService> repositoriesServiceSupplier;
     private RepositoriesService repositoriesService;
     private BlobStoreRepository blobStoreRepository;
+    private Compressor compressor;
     private BlobStore blobStore;
     private Settings settings;
     private boolean publicationEnabled;
+    private NamedWriteableRegistry namedWriteableRegistry;
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
 
+    private static final String NODE_ID = "test-node";
+    private static final String COORDINATION_METADATA_FILENAME = "coordination-metadata-file__1";
+    private static final String PERSISTENT_SETTINGS_FILENAME = "persistent-settings-file__1";
+    private static final String TRANSIENT_SETTINGS_FILENAME = "transient-settings-file__1";
+    private static final String TEMPLATES_METADATA_FILENAME = "templates-metadata-file__1";
+    private static final String DISCOVERY_NODES_FILENAME = "discovery-nodes-file__1";
+    private static final String CLUSTER_BLOCKS_FILENAME = "cluster-blocks-file__1";
+    private static final String HASHES_OF_CONSISTENT_SETTINGS_FILENAME = "consistent-settings-hashes-file__1";
+
     @Before
     public void setup() {
         repositoriesServiceSupplier = mock(Supplier.class);
@@ -164,6 +218,11 @@ public void setup() {
             .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
             .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository")
             .build();
+        List<NamedWriteableRegistry.Entry> writeableEntries = ClusterModule.getNamedWriteables();
+        writeableEntries.add(new NamedWriteableRegistry.Entry(Metadata.Custom.class, CustomMetadata1.TYPE, CustomMetadata1::new));
+        writeableEntries.add(new NamedWriteableRegistry.Entry(Metadata.Custom.class, CustomMetadata2.TYPE, CustomMetadata2::new));
+        writeableEntries.add(new NamedWriteableRegistry.Entry(Metadata.Custom.class, CustomMetadata3.TYPE, CustomMetadata3::new));
+        namedWriteableRegistry = new NamedWriteableRegistry(writeableEntries);
 
         clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         clusterService = mock(ClusterService.class);
@@ -176,6 +235,7 @@ public void setup() {
             ).flatMap(Function.identity()).collect(toList())
         );
 
+        compressor = new DeflateCompressor();
         blobStoreRepository = mock(BlobStoreRepository.class);
         blobStore = mock(BlobStore.class);
         when(blobStoreRepository.blobStore()).thenReturn(blobStore);
@@ -191,7 +251,7 @@ public void setup() {
             () -> 0L,
             threadPool,
             List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)),
-            writableRegistry()
+            namedWriteableRegistry
         );
     }
 
@@ -275,6 +335,7 @@ public void testWriteFullMetadataSuccess() throws IOException {
         assertThat(manifest.getSettingsMetadata(), notNullValue());
         assertThat(manifest.getTemplatesMetadata(), notNullValue());
         assertFalse(manifest.getCustomMetadataMap().isEmpty());
+        assertThat(manifest.getCustomMetadataMap().containsKey(CustomMetadata1.TYPE), is(true));
         assertThat(manifest.getClusterBlocksMetadata(), nullValue());
         assertThat(manifest.getDiscoveryNodesMetadata(), nullValue());
         assertThat(manifest.getTransientSettingsMetadata(), nullValue());
@@ -298,7 +359,12 @@ public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException
             writableRegistry()
         );
         final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager())
-            .customs(Map.of(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(List.of(new Entry("test-repo", 10L)))))
+            .customs(
+                Map.of(
+                    RepositoryCleanupInProgress.TYPE,
+                    new RepositoryCleanupInProgress(List.of(new RepositoryCleanupInProgress.Entry("test-repo", 10L)))
+                )
+            )
             .build();
         mockBlobStoreObjects();
         remoteClusterStateService.start();
@@ -330,6 +396,7 @@ public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException
         assertThat(manifest.getSettingsMetadata(), notNullValue());
         assertThat(manifest.getTemplatesMetadata(), notNullValue());
         assertFalse(manifest.getCustomMetadataMap().isEmpty());
+        assertThat(manifest.getCustomMetadataMap().containsKey(CustomMetadata1.TYPE), is(true));
         assertThat(manifest.getClusterStateCustomMap().size(), is(1));
         assertThat(manifest.getClusterStateCustomMap().containsKey(RepositoryCleanupInProgress.TYPE), is(true));
     }
@@ -388,7 +455,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
             .provideStream(0)
             .getInputStream()
             .readAllBytes();
-        IndexMetadata writtenIndexMetadata = RemoteIndexMetadata.INDEX_METADATA_FORMAT.deserialize(
+        IndexMetadata writtenIndexMetadata = INDEX_METADATA_FORMAT.deserialize(
             capturedWriteContext.get("metadata").getFileName(),
             blobStoreRepository.getNamedXContentRegistry(),
             new BytesArray(writtenBytes)
@@ -445,24 +512,35 @@ public void testTimeoutWhileWritingManifestFile() throws IOException {
 
         ArgumentCaptor<ActionListener<Void>> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
 
-        doAnswer((i) -> { // For Global Metadata
-            actionListenerArgumentCaptor.getValue().onResponse(null);
-            return null;
-        }).doAnswer((i) -> { // For Index Metadata
-            actionListenerArgumentCaptor.getValue().onResponse(null);
-            return null;
-        }).doAnswer((i) -> {
+        doAnswer((i) -> {
             // For Manifest file perform No Op, so latch in code will timeout
             return null;
         }).when(container).asyncBlobUpload(any(WriteContext.class), actionListenerArgumentCaptor.capture());
 
         remoteClusterStateService.start();
-        try {
-            remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10));
-        } catch (Exception e) {
-            assertTrue(e instanceof RemoteStateTransferException);
-            assertTrue(e.getMessage().contains("Timed out waiting for transfer of following metadata to complete"));
-        }
+        RemoteClusterStateService spiedService = spy(remoteClusterStateService);
+        when(
+            spiedService.writeMetadataInParallel(
+                any(),
+                anyList(),
+                anyMap(),
+                anyMap(),
+                anyBoolean(),
+                anyBoolean(),
+                anyBoolean(),
+                anyBoolean(),
+                anyBoolean(),
+                anyBoolean(),
+                anyMap(),
+                anyBoolean(),
+                anyList()
+            )
+        ).thenReturn(new RemoteClusterStateUtils.UploadedMetadataResults());
+        RemoteStateTransferException ex = expectThrows(
+            RemoteStateTransferException.class,
+            () -> spiedService.writeFullMetadata(clusterState, randomAlphaOfLength(10))
+        );
+        assertTrue(ex.getMessage().contains("Timed out waiting for transfer of manifest file to complete"));
     }
 
     public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOException {
@@ -658,6 +736,991 @@ public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws I
         assertThat(manifest.getIndicesRouting().size(), is(1));
     }
 
+    public void testTimeoutWhileWritingMetadata() throws IOException {
+        AsyncMultiStreamBlobContainer container = (AsyncMultiStreamBlobContainer) mockBlobStoreObjects(AsyncMultiStreamBlobContainer.class);
+        doNothing().when(container).asyncBlobUpload(any(), any());
+        int writeTimeout = 2;
+        Settings newSettings = Settings.builder()
+            .put("cluster.remote_store.state.global_metadata.upload_timeout", writeTimeout + "s")
+            .build();
+        clusterSettings.applySettings(newSettings);
+        remoteClusterStateService.start();
+        RemoteStateTransferException exception = assertThrows(
+            RemoteStateTransferException.class,
+            () -> remoteClusterStateService.writeMetadataInParallel(
+                ClusterState.EMPTY_STATE,
+                emptyList(),
+                emptyMap(),
+                emptyMap(),
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                emptyMap(),
+                true,
+                emptyList()
+            )
+        );
+        assertTrue(exception.getMessage().startsWith("Timed out waiting for transfer of following metadata to complete"));
+    }
+
+    public void testGetClusterStateForManifest_IncludeEphemeral() throws IOException {
+        ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().build();
+        mockBlobStoreObjects();
+        remoteClusterStateService.start();
+        RemoteReadResult mockedResult = mock(RemoteReadResult.class);
+        RemoteIndexMetadataManager mockedIndexManager = mock(RemoteIndexMetadataManager.class);
+        RemoteGlobalMetadataManager mockedGlobalMetadataManager = mock(RemoteGlobalMetadataManager.class);
+        RemoteClusterStateAttributesManager mockedClusterStateAttributeManager = mock(RemoteClusterStateAttributesManager.class);
+        remoteClusterStateService.setRemoteIndexMetadataManager(mockedIndexManager);
+        remoteClusterStateService.setRemoteGlobalMetadataManager(mockedGlobalMetadataManager);
+        remoteClusterStateService.setRemoteClusterStateAttributesManager(mockedClusterStateAttributeManager);
+        ArgumentCaptor<LatchedActionListener<RemoteReadResult>> listenerArgumentCaptor = ArgumentCaptor.forClass(
+            LatchedActionListener.class
+        );
+        when(mockedIndexManager.getAsyncIndexMetadataReadAction(any(), anyString(), listenerArgumentCaptor.capture())).thenReturn(
+            () -> listenerArgumentCaptor.getValue().onResponse(mockedResult)
+        );
+        when(mockedGlobalMetadataManager.getAsyncMetadataReadAction(any(), anyString(), listenerArgumentCaptor.capture())).thenReturn(
+            () -> listenerArgumentCaptor.getValue().onResponse(mockedResult)
+        );
+        when(mockedClusterStateAttributeManager.getAsyncMetadataReadAction(anyString(), any(), listenerArgumentCaptor.capture()))
+            .thenReturn(() -> listenerArgumentCaptor.getValue().onResponse(mockedResult));
+        when(mockedResult.getComponent()).thenReturn(COORDINATION_METADATA);
+        RemoteClusterStateService mockService = spy(remoteClusterStateService);
+        mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true);
+        verify(mockService, times(1)).readClusterStateInParallel(
+            any(),
+            eq(manifest),
+            eq(manifest.getClusterUUID()),
+            eq(NODE_ID),
+            eq(manifest.getIndices()),
+            eq(manifest.getCustomMetadataMap()),
+            eq(true),
+            eq(true),
+            eq(true),
+            eq(true),
+            eq(true),
+            eq(true),
+            eq(manifest.getIndicesRouting()),
+            eq(true),
+            eq(manifest.getClusterStateCustomMap()),
+            eq(true)
+        );
+    }
+
+    public void testGetClusterStateForManifest_ExcludeEphemeral() throws IOException {
+        ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().build();
+        mockBlobStoreObjects();
+        remoteClusterStateService.start();
+        RemoteReadResult mockedResult = mock(RemoteReadResult.class);
+        RemoteIndexMetadataManager mockedIndexManager = mock(RemoteIndexMetadataManager.class);
+        RemoteGlobalMetadataManager mockedGlobalMetadataManager = mock(RemoteGlobalMetadataManager.class);
+        RemoteClusterStateAttributesManager mockedClusterStateAttributeManager = mock(RemoteClusterStateAttributesManager.class);
+        ArgumentCaptor<LatchedActionListener<RemoteReadResult>> listenerArgumentCaptor = ArgumentCaptor.forClass(
+            LatchedActionListener.class
+        );
+        when(mockedIndexManager.getAsyncIndexMetadataReadAction(any(), anyString(), listenerArgumentCaptor.capture())).thenReturn(
+            () -> listenerArgumentCaptor.getValue().onResponse(mockedResult)
+        );
+        when(mockedGlobalMetadataManager.getAsyncMetadataReadAction(any(), anyString(), listenerArgumentCaptor.capture())).thenReturn(
+            () -> listenerArgumentCaptor.getValue().onResponse(mockedResult)
+        );
+        when(mockedClusterStateAttributeManager.getAsyncMetadataReadAction(anyString(), any(), listenerArgumentCaptor.capture()))
+            .thenReturn(() -> listenerArgumentCaptor.getValue().onResponse(mockedResult));
+        when(mockedResult.getComponent()).thenReturn(COORDINATION_METADATA);
+        remoteClusterStateService.setRemoteIndexMetadataManager(mockedIndexManager);
+        remoteClusterStateService.setRemoteGlobalMetadataManager(mockedGlobalMetadataManager);
+        remoteClusterStateService.setRemoteClusterStateAttributesManager(mockedClusterStateAttributeManager);
+        RemoteClusterStateService spiedService = spy(remoteClusterStateService);
+        spiedService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, false);
+        verify(spiedService, times(1)).readClusterStateInParallel(
+            any(),
+            eq(manifest),
+            eq(manifest.getClusterUUID()),
+            eq(NODE_ID),
+            eq(manifest.getIndices()),
+            eq(manifest.getCustomMetadataMap()),
+            eq(true),
+            eq(true),
+            eq(false),
+            eq(true),
+            eq(false),
+            eq(false),
+            eq(emptyList()),
+            eq(false),
+            eq(emptyMap()),
+            eq(false)
+        );
+    }
+
+    public void testGetClusterStateFromManifest_CodecV1() throws IOException {
+        ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().codecVersion(CODEC_V1).build();
+        mockBlobStoreObjects();
+        remoteClusterStateService.start();
+        final Index index = new Index("test-index", "index-uuid");
+        final Settings idxSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
+            .build();
+        final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+        RemoteIndexMetadataManager mockedIndexManager = mock(RemoteIndexMetadataManager.class);
+        RemoteGlobalMetadataManager mockedGlobalMetadataManager = mock(RemoteGlobalMetadataManager.class);
+        remoteClusterStateService.setRemoteIndexMetadataManager(mockedIndexManager);
+        remoteClusterStateService.setRemoteGlobalMetadataManager(mockedGlobalMetadataManager);
+        ArgumentCaptor<LatchedActionListener<RemoteReadResult>> listenerArgumentCaptor = ArgumentCaptor.forClass(
+            LatchedActionListener.class
+        );
+        when(mockedIndexManager.getAsyncIndexMetadataReadAction(any(), anyString(), listenerArgumentCaptor.capture())).thenReturn(
+            () -> listenerArgumentCaptor.getValue().onResponse(new RemoteReadResult(indexMetadata, INDEX, INDEX))
+        );
+        when(mockedGlobalMetadataManager.getGlobalMetadata(anyString(), eq(manifest))).thenReturn(Metadata.EMPTY_METADATA);
+        RemoteClusterStateService spiedService = spy(remoteClusterStateService);
+        spiedService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true);
+        verify(spiedService, times(1)).readClusterStateInParallel(
+            any(),
+            eq(manifest),
+            eq(manifest.getClusterUUID()),
+            eq(NODE_ID),
+            eq(manifest.getIndices()),
+            eq(emptyMap()),
+            eq(false),
+            eq(false),
+            eq(false),
+            eq(false),
+            eq(false),
+            eq(false),
+            eq(emptyList()),
+            eq(false),
+            eq(emptyMap()),
+            eq(false)
+        );
+        verify(mockedGlobalMetadataManager, times(1)).getGlobalMetadata(eq(manifest.getClusterUUID()), eq(manifest));
+    }
+
+    public void testGetClusterStateUsingDiffFailWhenDiffManifestAbsent() {
+        ClusterMetadataManifest manifest = ClusterMetadataManifest.builder().build();
+        ClusterState previousState = ClusterState.EMPTY_STATE;
+        AssertionError error = assertThrows(
+            AssertionError.class,
+            () -> remoteClusterStateService.getClusterStateUsingDiff(manifest, previousState, "test-node")
+        );
+        assertEquals("Diff manifest null which is required for downloading cluster state", error.getMessage());
+    }
+
+    public void testGetClusterStateUsingDiff_NoDiff() throws IOException {
+        ClusterStateDiffManifest diffManifest = ClusterStateDiffManifest.builder().build();
+        ClusterState clusterState = generateClusterStateWithAllAttributes().build();
+        ClusterMetadataManifest manifest = ClusterMetadataManifest.builder()
+            .diffManifest(diffManifest)
+            .stateUUID(clusterState.stateUUID())
+            .stateVersion(clusterState.version())
+            .metadataVersion(clusterState.metadata().version())
+            .clusterUUID(clusterState.getMetadata().clusterUUID())
+            .routingTableVersion(clusterState.routingTable().version())
+            .build();
+        ClusterState updatedClusterState = remoteClusterStateService.getClusterStateUsingDiff(manifest, clusterState, "test-node");
+        assertEquals(clusterState.getClusterName(), updatedClusterState.getClusterName());
+        assertEquals(clusterState.metadata().clusterUUID(), updatedClusterState.metadata().clusterUUID());
+        assertEquals(clusterState.metadata().version(), updatedClusterState.metadata().version());
+        assertEquals(clusterState.metadata().coordinationMetadata(), updatedClusterState.metadata().coordinationMetadata());
+        assertEquals(clusterState.metadata().getIndices(), updatedClusterState.metadata().getIndices());
+        assertEquals(clusterState.metadata().templates(), updatedClusterState.metadata().templates());
+        assertEquals(clusterState.metadata().persistentSettings(), updatedClusterState.metadata().persistentSettings());
+        assertEquals(clusterState.metadata().transientSettings(), updatedClusterState.metadata().transientSettings());
+        assertEquals(clusterState.metadata().getCustoms(), updatedClusterState.metadata().getCustoms());
+        assertEquals(clusterState.metadata().hashesOfConsistentSettings(), updatedClusterState.metadata().hashesOfConsistentSettings());
+        assertEquals(clusterState.getCustoms(), updatedClusterState.getCustoms());
+        assertEquals(clusterState.stateUUID(), updatedClusterState.stateUUID());
+        assertEquals(clusterState.version(), updatedClusterState.version());
+        assertEquals(clusterState.getRoutingTable().version(), updatedClusterState.getRoutingTable().version());
+        assertEquals(clusterState.getRoutingTable().getIndicesRouting(), updatedClusterState.getRoutingTable().getIndicesRouting());
+        assertEquals(clusterState.getNodes(), updatedClusterState.getNodes());
+        assertEquals(clusterState.getBlocks(), updatedClusterState.getBlocks());
+    }
+
+    public void testGetClusterStateUsingDiff() throws IOException {
+        ClusterState clusterState = generateClusterStateWithAllAttributes().build();
+        ClusterState.Builder expectedClusterStateBuilder = ClusterState.builder(clusterState);
+        Metadata.Builder mb = Metadata.builder(clusterState.metadata());
+        ClusterStateDiffManifest.Builder diffManifestBuilder = ClusterStateDiffManifest.builder();
+        ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder();
+        BlobContainer blobContainer = mockBlobStoreObjects();
+        if (randomBoolean()) {
+            // updated coordination metadata
+            CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder()
+                .term(clusterState.metadata().coordinationMetadata().term() + 1)
+                .build();
+            mb.coordinationMetadata(coordinationMetadata);
+            diffManifestBuilder.coordinationMetadataUpdated(true);
+            manifestBuilder.coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, COORDINATION_METADATA_FILENAME));
+            when(blobContainer.readBlob(COORDINATION_METADATA_FILENAME)).thenAnswer(i -> {
+                BytesReference bytes = COORDINATION_METADATA_FORMAT.serialize(
+                    coordinationMetadata,
+                    COORDINATION_METADATA_FILENAME,
+                    compressor,
+                    FORMAT_PARAMS
+                );
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            // updated templates
+            TemplatesMetadata templatesMetadata = TemplatesMetadata.builder()
+                .put(
+                    IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3))
+                        .patterns(Arrays.asList("bar-*", "foo-*"))
+                        .settings(Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build())
+                        .build()
+                )
+                .build();
+            mb.templates(templatesMetadata);
+            diffManifestBuilder.templatesMetadataUpdated(true);
+            manifestBuilder.templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, TEMPLATES_METADATA_FILENAME));
+            when(blobContainer.readBlob(TEMPLATES_METADATA_FILENAME)).thenAnswer(i -> {
+                BytesReference bytes = TEMPLATES_METADATA_FORMAT.serialize(
+                    templatesMetadata,
+                    TEMPLATES_METADATA_FILENAME,
+                    compressor,
+                    FORMAT_PARAMS
+                );
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            // updated persistent settings
+            Settings persistentSettings = Settings.builder()
+                .put("random_persistent_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
+                .build();
+            mb.persistentSettings(persistentSettings);
+            diffManifestBuilder.settingsMetadataUpdated(true);
+            manifestBuilder.settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, PERSISTENT_SETTINGS_FILENAME));
+            when(blobContainer.readBlob(PERSISTENT_SETTINGS_FILENAME)).thenAnswer(i -> {
+                BytesReference bytes = RemotePersistentSettingsMetadata.SETTINGS_METADATA_FORMAT.serialize(
+                    persistentSettings,
+                    PERSISTENT_SETTINGS_FILENAME,
+                    compressor,
+                    FORMAT_PARAMS
+                );
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            // updated transient settings
+            Settings transientSettings = Settings.builder()
+                .put("random_transient_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
+                .build();
+            mb.transientSettings(transientSettings);
+            diffManifestBuilder.transientSettingsMetadataUpdate(true);
+            manifestBuilder.transientSettingsMetadata(
+                new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, TRANSIENT_SETTINGS_FILENAME)
+            );
+            when(blobContainer.readBlob(TRANSIENT_SETTINGS_FILENAME)).thenAnswer(i -> {
+                BytesReference bytes = RemoteTransientSettingsMetadata.SETTINGS_METADATA_FORMAT.serialize(
+                    transientSettings,
+                    TRANSIENT_SETTINGS_FILENAME,
+                    compressor,
+                    FORMAT_PARAMS
+                );
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            // updated customs
+            CustomMetadata2 addedCustom = new CustomMetadata2(randomAlphaOfLength(10));
+            mb.putCustom(addedCustom.getWriteableName(), addedCustom);
+            diffManifestBuilder.customMetadataUpdated(Collections.singletonList(addedCustom.getWriteableName()));
+            manifestBuilder.customMetadataMap(
+                Map.of(addedCustom.getWriteableName(), new UploadedMetadataAttribute(addedCustom.getWriteableName(), "custom-md2-file__1"))
+            );
+            when(blobContainer.readBlob("custom-md2-file__1")).thenAnswer(i -> {
+                ChecksumWritableBlobStoreFormat<Metadata.Custom> customMetadataFormat = new ChecksumWritableBlobStoreFormat<>(
+                    "custom",
+                    is -> readFrom(is, namedWriteableRegistry, addedCustom.getWriteableName())
+                );
+                BytesReference bytes = customMetadataFormat.serialize(addedCustom, "custom-md2-file__1", compressor);
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            Set<String> customsToRemove = clusterState.metadata().customs().keySet();
+            customsToRemove.forEach(mb::removeCustom);
+            diffManifestBuilder.customMetadataDeleted(new ArrayList<>(customsToRemove));
+        }
+        if (randomBoolean()) {
+            // updated hashes of consistent settings
+            DiffableStringMap hashesOfConsistentSettings = new DiffableStringMap(Map.of("secure_setting_key", "secure_setting_value"));
+            mb.hashesOfConsistentSettings(hashesOfConsistentSettings);
+            diffManifestBuilder.hashesOfConsistentSettingsUpdated(true);
+            manifestBuilder.hashesOfConsistentSettings(
+                new UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, HASHES_OF_CONSISTENT_SETTINGS_FILENAME)
+            );
+            when(blobContainer.readBlob(HASHES_OF_CONSISTENT_SETTINGS_FILENAME)).thenAnswer(i -> {
+                BytesReference bytes = HASHES_OF_CONSISTENT_SETTINGS_FORMAT.serialize(
+                    hashesOfConsistentSettings,
+                    HASHES_OF_CONSISTENT_SETTINGS_FILENAME,
+                    compressor
+                );
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            // updated index metadata
+            IndexMetadata indexMetadata = new IndexMetadata.Builder("add-test-index").settings(
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .put(IndexMetadata.SETTING_INDEX_UUID, "add-test-index-uuid")
+                    .build()
+            ).numberOfShards(1).numberOfReplicas(0).build();
+            mb.put(indexMetadata, true);
+            diffManifestBuilder.indicesUpdated(Collections.singletonList(indexMetadata.getIndex().getName()));
+            manifestBuilder.indices(
+                List.of(
+                    new UploadedIndexMetadata(indexMetadata.getIndex().getName(), indexMetadata.getIndexUUID(), "add-test-index-file__2")
+                )
+            );
+            when(blobContainer.readBlob("add-test-index-file__2")).thenAnswer(i -> {
+                BytesReference bytes = INDEX_METADATA_FORMAT.serialize(indexMetadata, "add-test-index-file__2", compressor, FORMAT_PARAMS);
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            // remove index metadata
+            Set<String> indicesToDelete = clusterState.metadata().getIndices().keySet();
+            indicesToDelete.forEach(mb::remove);
+            diffManifestBuilder.indicesDeleted(new ArrayList<>(indicesToDelete));
+        }
+        if (randomBoolean()) {
+            // update nodes
+            DiscoveryNode node = new DiscoveryNode("node_id", buildNewFakeTransportAddress(), Version.CURRENT);
+            DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes()).add(node);
+            expectedClusterStateBuilder.nodes(nodesBuilder.build());
+            diffManifestBuilder.discoveryNodesUpdated(true);
+            manifestBuilder.discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, DISCOVERY_NODES_FILENAME));
+            when(blobContainer.readBlob(DISCOVERY_NODES_FILENAME)).thenAnswer(invocationOnMock -> {
+                BytesReference bytes = DISCOVERY_NODES_FORMAT.serialize(nodesBuilder.build(), DISCOVERY_NODES_FILENAME, compressor);
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+        }
+        if (randomBoolean()) {
+            // update blocks
+            ClusterBlocks newClusterBlock = randomClusterBlocks();
+            expectedClusterStateBuilder.blocks(newClusterBlock);
+            diffManifestBuilder.clusterBlocksUpdated(true);
+            manifestBuilder.clusterBlocksMetadata(new UploadedMetadataAttribute(CLUSTER_BLOCKS, CLUSTER_BLOCKS_FILENAME));
+            when(blobContainer.readBlob(CLUSTER_BLOCKS_FILENAME)).thenAnswer(invocationOnMock -> {
+                BytesReference bytes = CLUSTER_BLOCKS_FORMAT.serialize(newClusterBlock, CLUSTER_BLOCKS_FILENAME, compressor);
+                return new ByteArrayInputStream(bytes.streamInput().readAllBytes());
+            });
+
+        }
+        ClusterState expectedClusterState = expectedClusterStateBuilder.metadata(mb).build();
+        ClusterStateDiffManifest diffManifest = diffManifestBuilder.build();
+        manifestBuilder.diffManifest(diffManifest)
+            .stateUUID(clusterState.stateUUID())
+            .stateVersion(clusterState.version())
+            .metadataVersion(clusterState.metadata().version())
+            .clusterUUID(clusterState.getMetadata().clusterUUID())
+            .routingTableVersion(clusterState.getRoutingTable().version());
+
+        remoteClusterStateService.start();
+        ClusterState updatedClusterState = remoteClusterStateService.getClusterStateUsingDiff(
+            manifestBuilder.build(),
+            clusterState,
+            NODE_ID
+        );
+
+        assertEquals(expectedClusterState.getClusterName(), updatedClusterState.getClusterName());
+        assertEquals(expectedClusterState.stateUUID(), updatedClusterState.stateUUID());
+        assertEquals(expectedClusterState.version(), updatedClusterState.version());
+        assertEquals(expectedClusterState.metadata().clusterUUID(), updatedClusterState.metadata().clusterUUID());
+        assertEquals(expectedClusterState.getRoutingTable().version(), updatedClusterState.getRoutingTable().version());
+        assertNotEquals(diffManifest.isClusterBlocksUpdated(), updatedClusterState.getBlocks().equals(clusterState.getBlocks()));
+        assertNotEquals(diffManifest.isDiscoveryNodesUpdated(), updatedClusterState.getNodes().equals(clusterState.getNodes()));
+        assertNotEquals(
+            diffManifest.isCoordinationMetadataUpdated(),
+            updatedClusterState.getMetadata().coordinationMetadata().equals(clusterState.getMetadata().coordinationMetadata())
+        );
+        assertNotEquals(
+            diffManifest.isTemplatesMetadataUpdated(),
+            updatedClusterState.getMetadata().templates().equals(clusterState.getMetadata().getTemplates())
+        );
+        assertNotEquals(
+            diffManifest.isSettingsMetadataUpdated(),
+            updatedClusterState.getMetadata().persistentSettings().equals(clusterState.getMetadata().persistentSettings())
+        );
+        assertNotEquals(
+            diffManifest.isTransientSettingsMetadataUpdated(),
+            updatedClusterState.getMetadata().transientSettings().equals(clusterState.getMetadata().transientSettings())
+        );
+        diffManifest.getIndicesUpdated().forEach(indexName -> {
+            IndexMetadata updatedIndexMetadata = updatedClusterState.metadata().index(indexName);
+            IndexMetadata originalIndexMetadata = clusterState.metadata().index(indexName);
+            assertNotEquals(originalIndexMetadata, updatedIndexMetadata);
+        });
+        diffManifest.getCustomMetadataUpdated().forEach(customMetadataName -> {
+            Metadata.Custom updatedCustomMetadata = updatedClusterState.metadata().custom(customMetadataName);
+            Metadata.Custom originalCustomMetadata = clusterState.metadata().custom(customMetadataName);
+            assertNotEquals(originalCustomMetadata, updatedCustomMetadata);
+        });
+        diffManifest.getClusterStateCustomUpdated().forEach(clusterStateCustomName -> {
+            ClusterState.Custom updateClusterStateCustom = updatedClusterState.customs().get(clusterStateCustomName);
+            ClusterState.Custom originalClusterStateCustom = clusterState.customs().get(clusterStateCustomName);
+            assertNotEquals(originalClusterStateCustom, updateClusterStateCustom);
+        });
+        diffManifest.getIndicesRoutingUpdated().forEach(indexName -> {
+            IndexRoutingTable updatedIndexRoutingTable = updatedClusterState.getRoutingTable().getIndicesRouting().get(indexName);
+            IndexRoutingTable originalIndexingRoutingTable = clusterState.getRoutingTable().getIndicesRouting().get(indexName);
+            assertNotEquals(originalIndexingRoutingTable, updatedIndexRoutingTable);
+        });
+        diffManifest.getIndicesDeleted()
+            .forEach(indexName -> { assertFalse(updatedClusterState.metadata().getIndices().containsKey(indexName)); });
+        diffManifest.getCustomMetadataDeleted().forEach(customMetadataName -> {
+            assertFalse(updatedClusterState.metadata().customs().containsKey(customMetadataName));
+        });
+        diffManifest.getClusterStateCustomDeleted().forEach(clusterStateCustomName -> {
+            assertFalse(updatedClusterState.customs().containsKey(clusterStateCustomName));
+        });
+        diffManifest.getIndicesRoutingDeleted().forEach(indexName -> {
+            assertFalse(updatedClusterState.getRoutingTable().getIndicesRouting().containsKey(indexName));
+        });
+    }
+
+    public void testReadClusterStateInParallel_TimedOut() throws IOException {
+        ClusterState previousClusterState = generateClusterStateWithAllAttributes().build();
+        ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().build();
+        BlobContainer container = mockBlobStoreObjects();
+        int readTimeOut = 2;
+        Settings newSettings = Settings.builder().put("cluster.remote_store.state.read_timeout", readTimeOut + "s").build();
+        clusterSettings.applySettings(newSettings);
+        when(container.readBlob(anyString())).thenAnswer(invocationOnMock -> {
+            Thread.sleep(readTimeOut * 1000 + 100);
+            return null;
+        });
+        remoteClusterStateService.start();
+        RemoteStateTransferException exception = expectThrows(
+            RemoteStateTransferException.class,
+            () -> remoteClusterStateService.readClusterStateInParallel(
+                previousClusterState,
+                manifest,
+                manifest.getClusterUUID(),
+                NODE_ID,
+                emptyList(),
+                emptyMap(),
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                emptyList(),
+                true,
+                emptyMap(),
+                true
+            )
+        );
+        assertEquals("Timed out waiting to read cluster state from remote within timeout " + readTimeOut + "s", exception.getMessage());
+    }
+
+    public void testReadClusterStateInParallel_ExceptionDuringRead() throws IOException {
+        ClusterState previousClusterState = generateClusterStateWithAllAttributes().build();
+        ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().build();
+        BlobContainer container = mockBlobStoreObjects();
+        Exception mockException = new IOException("mock exception");
+        when(container.readBlob(anyString())).thenThrow(mockException);
+        remoteClusterStateService.start();
+        RemoteStateTransferException exception = expectThrows(
+            RemoteStateTransferException.class,
+            () -> remoteClusterStateService.readClusterStateInParallel(
+                previousClusterState,
+                manifest,
+                manifest.getClusterUUID(),
+                NODE_ID,
+                emptyList(),
+                emptyMap(),
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                emptyList(),
+                true,
+                emptyMap(),
+                true
+            )
+        );
+        assertEquals("Exception during reading cluster state from remote", exception.getMessage());
+        assertTrue(exception.getSuppressed().length > 0);
+        assertEquals(mockException, exception.getSuppressed()[0]);
+    }
+
+    public void testReadClusterStateInParallel_UnexpectedResult() throws IOException {
+        ClusterState previousClusterState = generateClusterStateWithAllAttributes().build();
+        // index already present in previous state
+        List<UploadedIndexMetadata> uploadedIndexMetadataList = new ArrayList<>(
+            List.of(new UploadedIndexMetadata("test-index", "test-index-uuid", "test-index-file__2"))
+        );
+        // new index to be added
+        List<UploadedIndexMetadata> newIndicesToRead = List.of(
+            new UploadedIndexMetadata("test-index-1", "test-index-1-uuid", "test-index-1-file__2")
+        );
+        uploadedIndexMetadataList.addAll(newIndicesToRead);
+        // existing custom metadata
+        Map<String, UploadedMetadataAttribute> uploadedCustomMetadataMap = new HashMap<>(
+            Map.of(
+                "custom_md_1",
+                new UploadedMetadataAttribute("custom_md_1", "test-custom1-file__1"),
+                "custom_md_2",
+                new UploadedMetadataAttribute("custom_md_2", "test-custom2-file__1")
+            )
+        );
+        // new custom metadata to be added
+        Map<String, UploadedMetadataAttribute> newCustomMetadataMap = Map.of(
+            "custom_md_3",
+            new UploadedMetadataAttribute("custom_md_3", "test-custom3-file__1")
+        );
+        uploadedCustomMetadataMap.putAll(newCustomMetadataMap);
+        // already existing cluster state customs
+        Map<String, UploadedMetadataAttribute> uploadedClusterStateCustomMap = new HashMap<>(
+            Map.of(
+                "custom_1",
+                new UploadedMetadataAttribute("custom_1", "test-cluster-state-custom1-file__1"),
+                "custom_2",
+                new UploadedMetadataAttribute("custom_2", "test-cluster-state-custom2-file__1")
+            )
+        );
+        // new customs uploaded
+        Map<String, UploadedMetadataAttribute> newClusterStateCustoms = Map.of(
+            "custom_3",
+            new UploadedMetadataAttribute("custom_3", "test-cluster-state-custom3-file__1")
+        );
+        uploadedClusterStateCustomMap.putAll(newClusterStateCustoms);
+        ClusterMetadataManifest manifest = ClusterMetadataManifest.builder()
+            .clusterUUID(previousClusterState.getMetadata().clusterUUID())
+            .indices(uploadedIndexMetadataList)
+            .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, COORDINATION_METADATA_FILENAME))
+            .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, PERSISTENT_SETTINGS_FILENAME))
+            .transientSettingsMetadata(new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, TRANSIENT_SETTINGS_FILENAME))
+            .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, TEMPLATES_METADATA_FILENAME))
+            .hashesOfConsistentSettings(
+                new UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, HASHES_OF_CONSISTENT_SETTINGS_FILENAME)
+            )
+            .customMetadataMap(uploadedCustomMetadataMap)
+            .discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, DISCOVERY_NODES_FILENAME))
+            .clusterBlocksMetadata(new UploadedMetadataAttribute(CLUSTER_BLOCKS, CLUSTER_BLOCKS_FILENAME))
+            .clusterStateCustomMetadataMap(uploadedClusterStateCustomMap)
+            .build();
+
+        RemoteReadResult mockResult = mock(RemoteReadResult.class);
+        RemoteIndexMetadataManager mockIndexMetadataManager = mock(RemoteIndexMetadataManager.class);
+        CheckedRunnable<IOException> mockRunnable = mock(CheckedRunnable.class);
+        ArgumentCaptor<LatchedActionListener<RemoteReadResult>> latchCapture = ArgumentCaptor.forClass(LatchedActionListener.class);
+        when(mockIndexMetadataManager.getAsyncIndexMetadataReadAction(anyString(), anyString(), latchCapture.capture())).thenReturn(
+            mockRunnable
+        );
+        RemoteGlobalMetadataManager mockGlobalMetadataManager = mock(RemoteGlobalMetadataManager.class);
+        when(mockGlobalMetadataManager.getAsyncMetadataReadAction(any(), anyString(), latchCapture.capture())).thenReturn(mockRunnable);
+        RemoteClusterStateAttributesManager mockClusterStateAttributeManager = mock(RemoteClusterStateAttributesManager.class);
+        when(mockClusterStateAttributeManager.getAsyncMetadataReadAction(anyString(), any(), latchCapture.capture())).thenReturn(
+            mockRunnable
+        );
+        doAnswer(invocationOnMock -> {
+            latchCapture.getValue().onResponse(mockResult);
+            return null;
+        }).when(mockRunnable).run();
+        when(mockResult.getComponent()).thenReturn("mock-result");
+        remoteClusterStateService.start();
+        remoteClusterStateService.setRemoteIndexMetadataManager(mockIndexMetadataManager);
+        remoteClusterStateService.setRemoteGlobalMetadataManager(mockGlobalMetadataManager);
+        remoteClusterStateService.setRemoteClusterStateAttributesManager(mockClusterStateAttributeManager);
+        IllegalStateException exception = expectThrows(
+            IllegalStateException.class,
+            () -> remoteClusterStateService.readClusterStateInParallel(
+                previousClusterState,
+                manifest,
+                manifest.getClusterUUID(),
+                NODE_ID,
+                newIndicesToRead,
+                newCustomMetadataMap,
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                emptyList(),
+                true,
+                newClusterStateCustoms,
+                true
+            )
+        );
+        assertEquals("Unknown component: mock-result", exception.getMessage());
+        newIndicesToRead.forEach(
+            uploadedIndexMetadata -> verify(mockIndexMetadataManager, times(1)).getAsyncIndexMetadataReadAction(
+                eq(previousClusterState.getMetadata().clusterUUID()),
+                eq(uploadedIndexMetadata.getUploadedFilename()),
+                any()
+            )
+        );
+        verify(mockGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(COORDINATION_METADATA_FILENAME)),
+            eq(COORDINATION_METADATA),
+            any()
+        );
+        verify(mockGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(PERSISTENT_SETTINGS_FILENAME)),
+            eq(SETTING_METADATA),
+            any()
+        );
+        verify(mockGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(TRANSIENT_SETTINGS_FILENAME)),
+            eq(TRANSIENT_SETTING_METADATA),
+            any()
+        );
+        verify(mockGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(TEMPLATES_METADATA_FILENAME)),
+            eq(TEMPLATES_METADATA),
+            any()
+        );
+        verify(mockGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(HASHES_OF_CONSISTENT_SETTINGS_FILENAME)),
+            eq(HASHES_OF_CONSISTENT_SETTINGS),
+            any()
+        );
+        newCustomMetadataMap.keySet().forEach(uploadedCustomMetadataKey -> {
+            verify(mockGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(newCustomMetadataMap.get(uploadedCustomMetadataKey).getUploadedFilename())),
+                eq(uploadedCustomMetadataKey),
+                any()
+            );
+        });
+        verify(mockClusterStateAttributeManager, times(1)).getAsyncMetadataReadAction(
+            eq(DISCOVERY_NODES),
+            argThat(new BlobNameMatcher(DISCOVERY_NODES_FILENAME)),
+            any()
+        );
+        verify(mockClusterStateAttributeManager, times(1)).getAsyncMetadataReadAction(
+            eq(CLUSTER_BLOCKS),
+            argThat(new BlobNameMatcher(CLUSTER_BLOCKS_FILENAME)),
+            any()
+        );
+        newClusterStateCustoms.keySet().forEach(uploadedClusterStateCustomMetadataKey -> {
+            verify(mockClusterStateAttributeManager, times(1)).getAsyncMetadataReadAction(
+                eq(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, uploadedClusterStateCustomMetadataKey)),
+                argThat(new BlobNameMatcher(newClusterStateCustoms.get(uploadedClusterStateCustomMetadataKey).getUploadedFilename())),
+                any()
+            );
+        });
+    }
+
+    public void testReadClusterStateInParallel_Success() throws IOException {
+        ClusterState previousClusterState = generateClusterStateWithAllAttributes().build();
+        String indexFilename = "test-index-1-file__2";
+        String customMetadataFilename = "test-custom3-file__1";
+        String clusterStateCustomFilename = "test-cluster-state-custom3-file__1";
+        // index already present in previous state
+        List<UploadedIndexMetadata> uploadedIndexMetadataList = new ArrayList<>(
+            List.of(new UploadedIndexMetadata("test-index", "test-index-uuid", "test-index-file__2"))
+        );
+        // new index to be added
+        List<UploadedIndexMetadata> newIndicesToRead = List.of(
+            new UploadedIndexMetadata("test-index-1", "test-index-1-uuid", indexFilename)
+        );
+        uploadedIndexMetadataList.addAll(newIndicesToRead);
+        // existing custom metadata
+        Map<String, UploadedMetadataAttribute> uploadedCustomMetadataMap = new HashMap<>(
+            Map.of(
+                "custom_md_1",
+                new UploadedMetadataAttribute("custom_md_1", "test-custom1-file__1"),
+                "custom_md_2",
+                new UploadedMetadataAttribute("custom_md_2", "test-custom2-file__1")
+            )
+        );
+        // new custom metadata to be added
+        Map<String, UploadedMetadataAttribute> newCustomMetadataMap = Map.of(
+            "custom_md_3",
+            new UploadedMetadataAttribute("custom_md_3", customMetadataFilename)
+        );
+        uploadedCustomMetadataMap.putAll(newCustomMetadataMap);
+        // already existing cluster state customs
+        Map<String, UploadedMetadataAttribute> uploadedClusterStateCustomMap = new HashMap<>(
+            Map.of(
+                "custom_1",
+                new UploadedMetadataAttribute("custom_1", "test-cluster-state-custom1-file__1"),
+                "custom_2",
+                new UploadedMetadataAttribute("custom_2", "test-cluster-state-custom2-file__1")
+            )
+        );
+        // new customs uploaded
+        Map<String, UploadedMetadataAttribute> newClusterStateCustoms = Map.of(
+            "custom_3",
+            new UploadedMetadataAttribute("custom_3", clusterStateCustomFilename)
+        );
+        uploadedClusterStateCustomMap.putAll(newClusterStateCustoms);
+
+        ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().indices(uploadedIndexMetadataList)
+            .customMetadataMap(uploadedCustomMetadataMap)
+            .clusterStateCustomMetadataMap(uploadedClusterStateCustomMap)
+            .build();
+
+        IndexMetadata newIndexMetadata = new IndexMetadata.Builder("test-index-1").state(IndexMetadata.State.OPEN)
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build())
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .build();
+        CustomMetadata3 customMetadata3 = new CustomMetadata3("custom_md_3");
+        CoordinationMetadata updatedCoordinationMetadata = CoordinationMetadata.builder()
+            .term(previousClusterState.metadata().coordinationMetadata().term() + 1)
+            .build();
+        Settings updatedPersistentSettings = Settings.builder()
+            .put("random_persistent_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
+            .build();
+        Settings updatedTransientSettings = Settings.builder()
+            .put("random_transient_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
+            .build();
+        TemplatesMetadata updatedTemplateMetadata = getTemplatesMetadata();
+        DiffableStringMap updatedHashesOfConsistentSettings = getHashesOfConsistentSettings();
+        DiscoveryNodes updatedDiscoveryNodes = getDiscoveryNodes();
+        ClusterBlocks updatedClusterBlocks = randomClusterBlocks();
+        TestClusterStateCustom3 updatedClusterStateCustom3 = new TestClusterStateCustom3("custom_3");
+
+        RemoteIndexMetadataManager mockedIndexManager = mock(RemoteIndexMetadataManager.class);
+        RemoteGlobalMetadataManager mockedGlobalMetadataManager = mock(RemoteGlobalMetadataManager.class);
+        RemoteClusterStateAttributesManager mockedClusterStateAttributeManager = mock(RemoteClusterStateAttributesManager.class);
+
+        when(
+            mockedIndexManager.getAsyncIndexMetadataReadAction(
+                eq(manifest.getClusterUUID()),
+                eq(indexFilename),
+                any(LatchedActionListener.class)
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(newIndexMetadata, INDEX, "test-index-1")
+            );
+        });
+        when(
+            mockedGlobalMetadataManager.getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(customMetadataFilename)),
+                eq("custom_md_3"),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(customMetadata3, CUSTOM_METADATA, "custom_md_3")
+            );
+        });
+        when(
+            mockedGlobalMetadataManager.getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(COORDINATION_METADATA_FILENAME)),
+                eq(COORDINATION_METADATA),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(updatedCoordinationMetadata, COORDINATION_METADATA, COORDINATION_METADATA)
+            );
+        });
+        when(
+            mockedGlobalMetadataManager.getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(PERSISTENT_SETTINGS_FILENAME)),
+                eq(SETTING_METADATA),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(updatedPersistentSettings, SETTING_METADATA, SETTING_METADATA)
+            );
+        });
+        when(
+            mockedGlobalMetadataManager.getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(TRANSIENT_SETTINGS_FILENAME)),
+                eq(TRANSIENT_SETTING_METADATA),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(updatedTransientSettings, TRANSIENT_SETTING_METADATA, TRANSIENT_SETTING_METADATA)
+            );
+        });
+        when(
+            mockedGlobalMetadataManager.getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(TEMPLATES_METADATA_FILENAME)),
+                eq(TEMPLATES_METADATA),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(updatedTemplateMetadata, TEMPLATES_METADATA, TEMPLATES_METADATA)
+            );
+        });
+        when(
+            mockedGlobalMetadataManager.getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(HASHES_OF_CONSISTENT_SETTINGS_FILENAME)),
+                eq(HASHES_OF_CONSISTENT_SETTINGS),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(updatedHashesOfConsistentSettings, HASHES_OF_CONSISTENT_SETTINGS, HASHES_OF_CONSISTENT_SETTINGS)
+            );
+        });
+        when(
+            mockedClusterStateAttributeManager.getAsyncMetadataReadAction(
+                eq(DISCOVERY_NODES),
+                argThat(new BlobNameMatcher(DISCOVERY_NODES_FILENAME)),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(updatedDiscoveryNodes, CLUSTER_STATE_ATTRIBUTE, DISCOVERY_NODES)
+            );
+        });
+        when(
+            mockedClusterStateAttributeManager.getAsyncMetadataReadAction(
+                eq(CLUSTER_BLOCKS),
+                argThat(new BlobNameMatcher(CLUSTER_BLOCKS_FILENAME)),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(updatedClusterBlocks, CLUSTER_STATE_ATTRIBUTE, CLUSTER_BLOCKS)
+            );
+        });
+        when(
+            mockedClusterStateAttributeManager.getAsyncMetadataReadAction(
+                eq(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, updatedClusterStateCustom3.getWriteableName())),
+                argThat(new BlobNameMatcher(clusterStateCustomFilename)),
+                any()
+            )
+        ).thenAnswer(invocationOnMock -> {
+            LatchedActionListener<RemoteReadResult> latchedActionListener = invocationOnMock.getArgument(2, LatchedActionListener.class);
+            return (CheckedRunnable<IOException>) () -> latchedActionListener.onResponse(
+                new RemoteReadResult(
+                    updatedClusterStateCustom3,
+                    CLUSTER_STATE_ATTRIBUTE,
+                    String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, updatedClusterStateCustom3.getWriteableName())
+                )
+            );
+        });
+
+        remoteClusterStateService.start();
+        remoteClusterStateService.setRemoteIndexMetadataManager(mockedIndexManager);
+        remoteClusterStateService.setRemoteGlobalMetadataManager(mockedGlobalMetadataManager);
+        remoteClusterStateService.setRemoteClusterStateAttributesManager(mockedClusterStateAttributeManager);
+
+        ClusterState updatedClusterState = remoteClusterStateService.readClusterStateInParallel(
+            previousClusterState,
+            manifest,
+            manifest.getClusterUUID(),
+            NODE_ID,
+            newIndicesToRead,
+            newCustomMetadataMap,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            emptyList(),
+            true,
+            newClusterStateCustoms,
+            true
+        );
+
+        assertEquals(uploadedIndexMetadataList.size(), updatedClusterState.metadata().indices().size());
+        assertTrue(updatedClusterState.metadata().indices().containsKey("test-index-1"));
+        assertEquals(newIndexMetadata, updatedClusterState.metadata().index(newIndexMetadata.getIndex()));
+        uploadedCustomMetadataMap.keySet().forEach(key -> assertTrue(updatedClusterState.metadata().customs().containsKey(key)));
+        assertEquals(customMetadata3, updatedClusterState.metadata().custom(customMetadata3.getWriteableName()));
+        assertEquals(
+            previousClusterState.metadata().coordinationMetadata().term() + 1,
+            updatedClusterState.metadata().coordinationMetadata().term()
+        );
+        assertEquals(updatedPersistentSettings, updatedClusterState.metadata().persistentSettings());
+        assertEquals(updatedTransientSettings, updatedClusterState.metadata().transientSettings());
+        assertEquals(updatedTemplateMetadata.getTemplates(), updatedClusterState.metadata().templates());
+        assertEquals(updatedHashesOfConsistentSettings, updatedClusterState.metadata().hashesOfConsistentSettings());
+        assertEquals(updatedDiscoveryNodes.getSize(), updatedClusterState.getNodes().getSize());
+        updatedDiscoveryNodes.getNodes().forEach((nodeId, node) -> assertEquals(updatedClusterState.getNodes().get(nodeId), node));
+        assertEquals(updatedDiscoveryNodes.getClusterManagerNodeId(), updatedClusterState.getNodes().getClusterManagerNodeId());
+        assertEquals(updatedClusterBlocks, updatedClusterState.blocks());
+        uploadedClusterStateCustomMap.keySet().forEach(key -> assertTrue(updatedClusterState.customs().containsKey(key)));
+        assertEquals(updatedClusterStateCustom3, updatedClusterState.custom("custom_3"));
+        newIndicesToRead.forEach(
+            uploadedIndexMetadata -> verify(mockedIndexManager, times(1)).getAsyncIndexMetadataReadAction(
+                eq(previousClusterState.getMetadata().clusterUUID()),
+                eq(uploadedIndexMetadata.getUploadedFilename()),
+                any()
+            )
+        );
+        verify(mockedGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(COORDINATION_METADATA_FILENAME)),
+            eq(COORDINATION_METADATA),
+            any()
+        );
+        verify(mockedGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(PERSISTENT_SETTINGS_FILENAME)),
+            eq(SETTING_METADATA),
+            any()
+        );
+        verify(mockedGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(TRANSIENT_SETTINGS_FILENAME)),
+            eq(TRANSIENT_SETTING_METADATA),
+            any()
+        );
+        verify(mockedGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(TEMPLATES_METADATA_FILENAME)),
+            eq(TEMPLATES_METADATA),
+            any()
+        );
+        verify(mockedGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+            argThat(new BlobNameMatcher(HASHES_OF_CONSISTENT_SETTINGS_FILENAME)),
+            eq(HASHES_OF_CONSISTENT_SETTINGS),
+            any()
+        );
+        newCustomMetadataMap.keySet().forEach(uploadedCustomMetadataKey -> {
+            verify(mockedGlobalMetadataManager, times(1)).getAsyncMetadataReadAction(
+                argThat(new BlobNameMatcher(newCustomMetadataMap.get(uploadedCustomMetadataKey).getUploadedFilename())),
+                eq(uploadedCustomMetadataKey),
+                any()
+            );
+        });
+        verify(mockedClusterStateAttributeManager, times(1)).getAsyncMetadataReadAction(
+            eq(DISCOVERY_NODES),
+            argThat(new BlobNameMatcher(DISCOVERY_NODES_FILENAME)),
+            any()
+        );
+        verify(mockedClusterStateAttributeManager, times(1)).getAsyncMetadataReadAction(
+            eq(CLUSTER_BLOCKS),
+            argThat(new BlobNameMatcher(CLUSTER_BLOCKS_FILENAME)),
+            any()
+        );
+        newClusterStateCustoms.keySet().forEach(uploadedClusterStateCustomMetadataKey -> {
+            verify(mockedClusterStateAttributeManager, times(1)).getAsyncMetadataReadAction(
+                eq(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, uploadedClusterStateCustomMetadataKey)),
+                argThat(new BlobNameMatcher(newClusterStateCustoms.get(uploadedClusterStateCustomMetadataKey).getUploadedFilename())),
+                any()
+            );
+        });
+    }
+
     /*
      * Here we will verify the migration of manifest file from codec V0.
      *
@@ -1857,7 +2920,7 @@ private void mockObjectsForGettingPreviousClusterUUID(
         BlobContainer[] mockBlobContainerOrderedArray = new BlobContainer[mockBlobContainerOrderedList.size()];
         mockBlobContainerOrderedList.toArray(mockBlobContainerOrderedArray);
         when(blobStore.blobContainer(ArgumentMatchers.any())).thenReturn(uuidBlobContainer, mockBlobContainerOrderedArray);
-        when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor());
+        when(blobStoreRepository.getCompressor()).thenReturn(compressor);
     }
 
     private ClusterMetadataManifest generateV1ClusterMetadataManifest(
@@ -1986,7 +3049,7 @@ private void mockBlobContainer(
                 }
                 String fileName = uploadedIndexMetadata.getUploadedFilename();
                 when(blobContainer.readBlob(getFormattedIndexFileName(fileName))).thenAnswer((invocationOnMock) -> {
-                    BytesReference bytesIndexMetadata = RemoteIndexMetadata.INDEX_METADATA_FORMAT.serialize(
+                    BytesReference bytesIndexMetadata = INDEX_METADATA_FORMAT.serialize(
                         indexMetadata,
                         fileName,
                         blobStoreRepository.getCompressor(),
@@ -2057,12 +3120,6 @@ private void mockBlobContainerForGlobalMetadata(
                 .stream()
                 .collect(Collectors.toMap(Map.Entry::getKey, entry -> getFileNameFromPath(entry.getValue().getUploadedFilename())));
 
-            // ChecksumBlobStoreFormat<Metadata.Custom> customMetadataFormat = new ChecksumBlobStoreFormat<>(
-            // "custom",
-            // METADATA_NAME_PLAIN_FORMAT,
-            // null
-            // );
-
             ChecksumWritableBlobStoreFormat<Metadata.Custom> customMetadataFormat = new ChecksumWritableBlobStoreFormat<>("custom", null);
             for (Map.Entry<String, String> entry : customFileMap.entrySet()) {
                 String custom = entry.getKey();
@@ -2147,32 +3204,105 @@ static ClusterState.Builder generateClusterStateWithOneIndex() {
             .routingTable(RoutingTable.builder().addAsNew(indexMetadata).version(1L).build());
     }
 
+    static ClusterState.Builder generateClusterStateWithAllAttributes() {
+        final Index index = new Index("test-index", "index-uuid");
+        final Settings idxSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
+            .build();
+        final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
+        final Settings settings = Settings.builder().put("mock-settings", true).build();
+        final Settings transientSettings = Settings.builder().put("mock-transient-settings", true).build();
+        final DiffableStringMap hashesOfConsistentSettings = new DiffableStringMap(emptyMap());
+        final TemplatesMetadata templatesMetadata = TemplatesMetadata.builder()
+            .put(IndexTemplateMetadata.builder("template-1").patterns(List.of("test-index* ")).build())
+            .build();
+        final CustomMetadata1 customMetadata1 = new CustomMetadata1("custom-metadata-1");
+        final CustomMetadata2 customMetadata2 = new CustomMetadata2("custom-metadata-2");
+        final DiscoveryNodes nodes = nodesWithLocalNodeClusterManager();
+        final ClusterBlocks clusterBlocks = randomClusterBlocks();
+        final TestClusterStateCustom1 custom1 = new RemoteClusterStateTestUtils.TestClusterStateCustom1("custom-1");
+        final TestClusterStateCustom2 custom2 = new RemoteClusterStateTestUtils.TestClusterStateCustom2("custom-2");
+        return ClusterState.builder(ClusterName.DEFAULT)
+            .version(1L)
+            .stateUUID("state-uuid")
+            .metadata(
+                Metadata.builder()
+                    .version(randomNonNegativeLong())
+                    .put(indexMetadata, true)
+                    .clusterUUID("cluster-uuid")
+                    .coordinationMetadata(coordinationMetadata)
+                    .persistentSettings(settings)
+                    .transientSettings(transientSettings)
+                    .hashesOfConsistentSettings(hashesOfConsistentSettings)
+                    .templates(templatesMetadata)
+                    .putCustom(customMetadata1.getWriteableName(), customMetadata1)
+                    .putCustom(customMetadata2.getWriteableName(), customMetadata2)
+                    .build()
+            )
+            .routingTable(RoutingTable.builder().addAsNew(indexMetadata).version(1L).build())
+            .nodes(nodes)
+            .blocks(clusterBlocks)
+            .putCustom(custom1.getWriteableName(), custom1)
+            .putCustom(custom2.getWriteableName(), custom2);
+    }
+
+    static ClusterMetadataManifest.Builder generateClusterMetadataManifestWithAllAttributes() {
+        return ClusterMetadataManifest.builder()
+            .codecVersion(CODEC_V2)
+            .clusterUUID("cluster-uuid")
+            .indices(List.of(new UploadedIndexMetadata("test-index", "test-index-uuid", "test-index-file__2")))
+            .customMetadataMap(
+                Map.of(
+                    "custom_md_1",
+                    new UploadedMetadataAttribute("custom_md_1", "test-custom1-file__1"),
+                    "custom_md_2",
+                    new UploadedMetadataAttribute("custom_md_2", "test-custom2-file__1")
+                )
+            )
+            .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, COORDINATION_METADATA_FILENAME))
+            .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, PERSISTENT_SETTINGS_FILENAME))
+            .transientSettingsMetadata(new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, TRANSIENT_SETTINGS_FILENAME))
+            .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, TEMPLATES_METADATA_FILENAME))
+            .hashesOfConsistentSettings(
+                new UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, HASHES_OF_CONSISTENT_SETTINGS_FILENAME)
+            )
+            .discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, DISCOVERY_NODES_FILENAME))
+            .clusterBlocksMetadata(new UploadedMetadataAttribute(CLUSTER_BLOCKS, CLUSTER_BLOCKS_FILENAME))
+            .clusterStateCustomMetadataMap(
+                Map.of(
+                    "custom_1",
+                    new UploadedMetadataAttribute("custom_1", "test-cluster-state-custom1-file__1"),
+                    "custom_2",
+                    new UploadedMetadataAttribute("custom_2", "test-cluster-state-custom2-file__1")
+                )
+            );
+    }
+
     static DiscoveryNodes nodesWithLocalNodeClusterManager() {
         final DiscoveryNode localNode = new DiscoveryNode("cluster-manager-id", buildNewFakeTransportAddress(), Version.CURRENT);
         return DiscoveryNodes.builder().clusterManagerNodeId("cluster-manager-id").localNodeId("cluster-manager-id").add(localNode).build();
     }
 
-    private static class CustomMetadata1 extends TestCustomMetadata {
-        public static final String TYPE = "custom_md_1";
+    private class BlobNameMatcher implements ArgumentMatcher<AbstractRemoteWritableBlobEntity> {
+        private final String expectedBlobName;
 
-        CustomMetadata1(String data) {
-            super(data);
+        BlobNameMatcher(String expectedBlobName) {
+            this.expectedBlobName = expectedBlobName;
         }
 
         @Override
-        public String getWriteableName() {
-            return TYPE;
+        public boolean matches(AbstractRemoteWritableBlobEntity argument) {
+            return argument != null && expectedBlobName.equals(argument.getFullBlobName());
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
-        }
-
-        @Override
-        public EnumSet<Metadata.XContentContext> context() {
-            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        public String toString() {
+            return "BlobNameMatcher[Expected blobName: " + expectedBlobName + "]";
         }
     }
-
 }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateTestUtils.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateTestUtils.java
new file mode 100644
index 0000000000000..b17ffcbaac344
--- /dev/null
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateTestUtils.java
@@ -0,0 +1,227 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.test.TestClusterStateCustom;
+import org.opensearch.test.TestCustomMetadata;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+public class RemoteClusterStateTestUtils {
+    public static class CustomMetadata1 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_1";
+
+        public CustomMetadata1(String data) {
+            super(data);
+        }
+
+        public CustomMetadata1(StreamInput in) throws IOException {
+            super(in.readString());
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    public static class CustomMetadata2 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_2";
+
+        public CustomMetadata2(String data) {
+            super(data);
+        }
+
+        public CustomMetadata2(StreamInput in) throws IOException {
+            super(in.readString());
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    public static class CustomMetadata3 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_3";
+
+        public CustomMetadata3(String data) {
+            super(data);
+        }
+
+        public CustomMetadata3(StreamInput in) throws IOException {
+            super(in.readString());
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    public static class CustomMetadata4 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_4";
+
+        public CustomMetadata4(String data) {
+            super(data);
+        }
+
+        public CustomMetadata4(StreamInput in) throws IOException {
+            super(in.readString());
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
+    public static class CustomMetadata5 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_5";
+
+        public CustomMetadata5(String data) {
+            super(data);
+        }
+
+        public CustomMetadata5(StreamInput in) throws IOException {
+            super(in.readString());
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.API);
+        }
+    }
+
+    public static class TestClusterStateCustom1 extends TestClusterStateCustom {
+
+        public static final String TYPE = "custom_1";
+
+        public TestClusterStateCustom1(String value) {
+            super(value);
+        }
+
+        public TestClusterStateCustom1(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+
+    public static class TestClusterStateCustom2 extends TestClusterStateCustom {
+
+        public static final String TYPE = "custom_2";
+
+        public TestClusterStateCustom2(String value) {
+            super(value);
+        }
+
+        public TestClusterStateCustom2(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+
+    public static class TestClusterStateCustom3 extends TestClusterStateCustom {
+
+        public static final String TYPE = "custom_3";
+
+        public TestClusterStateCustom3(String value) {
+            super(value);
+        }
+
+        public TestClusterStateCustom3(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+
+    public static class TestClusterStateCustom4 extends TestClusterStateCustom {
+
+        public static final String TYPE = "custom_4";
+
+        public TestClusterStateCustom4(String value) {
+            super(value);
+        }
+
+        public TestClusterStateCustom4(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+}
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
index c543f986b3e86..917794ec03c3a 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.gateway.remote;
 
-import org.opensearch.Version;
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
@@ -18,7 +17,6 @@
 import org.opensearch.cluster.metadata.DiffableStringMap;
 import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.Metadata;
-import org.opensearch.cluster.metadata.Metadata.XContentContext;
 import org.opensearch.cluster.metadata.TemplatesMetadata;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.network.NetworkModule;
@@ -43,7 +41,6 @@
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.test.TestCustomMetadata;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
 import org.junit.After;
@@ -51,7 +48,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
@@ -61,6 +57,11 @@
 import static java.util.stream.Collectors.toList;
 import static org.opensearch.cluster.metadata.Metadata.isGlobalStateEquals;
 import static org.opensearch.common.blobstore.stream.write.WritePriority.URGENT;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata1;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata2;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata3;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata4;
+import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata5;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
@@ -699,121 +700,5 @@ public void testGetUpdatedCustoms() {
         );
         assertThat(customsDiff.getUpserts(), is(expectedUpserts));
         assertThat(customsDiff.getDeletes(), is(List.of(CustomMetadata1.TYPE)));
-
-    }
-
-    private static class CustomMetadata1 extends TestCustomMetadata {
-        public static final String TYPE = "custom_md_1";
-
-        CustomMetadata1(String data) {
-            super(data);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-
-        @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
-        }
-
-        @Override
-        public EnumSet<XContentContext> context() {
-            return EnumSet.of(Metadata.XContentContext.GATEWAY);
-        }
-    }
-
-    private static class CustomMetadata2 extends TestCustomMetadata {
-        public static final String TYPE = "custom_md_2";
-
-        CustomMetadata2(String data) {
-            super(data);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-
-        @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
-        }
-
-        @Override
-        public EnumSet<Metadata.XContentContext> context() {
-            return EnumSet.of(Metadata.XContentContext.GATEWAY);
-        }
-    }
-
-    private static class CustomMetadata3 extends TestCustomMetadata {
-        public static final String TYPE = "custom_md_3";
-
-        CustomMetadata3(String data) {
-            super(data);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-
-        @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
-        }
-
-        @Override
-        public EnumSet<Metadata.XContentContext> context() {
-            return EnumSet.of(Metadata.XContentContext.GATEWAY);
-        }
-    }
-
-    private static class CustomMetadata4 extends TestCustomMetadata {
-        public static final String TYPE = "custom_md_4";
-
-        CustomMetadata4(String data) {
-            super(data);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-
-        @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
-        }
-
-        @Override
-        public EnumSet<Metadata.XContentContext> context() {
-            return EnumSet.of(Metadata.XContentContext.GATEWAY);
-        }
-    }
-
-    private static class CustomMetadata5 extends TestCustomMetadata {
-        public static final String TYPE = "custom_md_5";
-
-        CustomMetadata5(String data) {
-            super(data);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-
-        @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
-        }
-
-        @Override
-        public EnumSet<Metadata.XContentContext> context() {
-            return EnumSet.of(XContentContext.API);
-        }
     }
 }
diff --git a/test/framework/src/main/java/org/opensearch/test/TestClusterStateCustom.java b/test/framework/src/main/java/org/opensearch/test/TestClusterStateCustom.java
new file mode 100644
index 0000000000000..ac32b8d227eda
--- /dev/null
+++ b/test/framework/src/main/java/org/opensearch/test/TestClusterStateCustom.java
@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.test;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.AbstractNamedDiffable;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+public abstract class TestClusterStateCustom extends AbstractNamedDiffable<ClusterState.Custom> implements ClusterState.Custom {
+
+    private final String value;
+
+    protected TestClusterStateCustom(String value) {
+        this.value = value;
+    }
+
+    protected TestClusterStateCustom(StreamInput in) throws IOException {
+        this.value = in.readString();
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.CURRENT;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(value);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder;
+    }
+
+    @Override
+    public boolean isPrivate() {
+        return true;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        TestClusterStateCustom that = (TestClusterStateCustom) o;
+
+        if (!value.equals(that.value)) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return value.hashCode();
+    }
+}

From bda839377195fe3f665cb08b2ebf5263690e757c Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Wed, 10 Jul 2024 19:35:09 +0800
Subject: [PATCH 089/113] Update version check for the bug fix of
 match_phrase_prefix_query not working on text field with multiple values and
 index_prefixes (#14703)

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 .../rest-api-spec/test/search/190_index_prefix_search.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml
index 8b031c132f979..6a946fb264560 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml
@@ -138,8 +138,8 @@ setup:
 ---
 "search index prefixes with multiple values":
   - skip:
-      version: " - 2.99.99"
-      reason: "the bug was fixed in 3.0.0"
+      version: " - 2.15.99"
+      reason: "the bug was fixed since 2.16.0"
   - do:
       search:
         rest_total_hits_as_int: true
@@ -154,8 +154,8 @@ setup:
 ---
 "search index prefixes with multiple values and custom position_increment_gap":
   - skip:
-      version: " - 2.99.99"
-      reason: "the bug was fixed in 3.0.0"
+      version: " - 2.15.99"
+      reason: "the bug was fixed since 2.16.0"
   - do:
       search:
         rest_total_hits_as_int: true

From 605543b0abce8dd84983571df5354490d2029113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Vl=C4=8Dek?= <lukas.vlcek@aiven.io>
Date: Wed, 10 Jul 2024 18:22:26 +0200
Subject: [PATCH 090/113] Remove unnecessary cast to int from test (#14696)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Lukáš Vlček <lukas.vlcek@aiven.io>
---
 .../remote/RemoteSegmentTransferTrackerTests.java      | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
index 280598c516c3c..1ec1e9977a9d5 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
@@ -571,13 +571,9 @@ public void testStatsObjectCreationViaStream() throws IOException {
                 assertEquals((int) deserializedStats.uploadBytesStarted, (int) transferTrackerStats.uploadBytesStarted);
                 assertEquals((int) deserializedStats.uploadBytesSucceeded, (int) transferTrackerStats.uploadBytesSucceeded);
                 assertEquals((int) deserializedStats.uploadBytesFailed, (int) transferTrackerStats.uploadBytesFailed);
-                assertEquals((int) deserializedStats.uploadBytesMovingAverage, transferTrackerStats.uploadBytesMovingAverage, 0);
-                assertEquals(
-                    (int) deserializedStats.uploadBytesPerSecMovingAverage,
-                    transferTrackerStats.uploadBytesPerSecMovingAverage,
-                    0
-                );
-                assertEquals((int) deserializedStats.uploadTimeMovingAverage, transferTrackerStats.uploadTimeMovingAverage, 0);
+                assertEquals(deserializedStats.uploadBytesMovingAverage, transferTrackerStats.uploadBytesMovingAverage, 0);
+                assertEquals(deserializedStats.uploadBytesPerSecMovingAverage, transferTrackerStats.uploadBytesPerSecMovingAverage, 0);
+                assertEquals(deserializedStats.uploadTimeMovingAverage, transferTrackerStats.uploadTimeMovingAverage, 0);
                 assertEquals((int) deserializedStats.totalUploadsStarted, (int) transferTrackerStats.totalUploadsStarted);
                 assertEquals((int) deserializedStats.totalUploadsSucceeded, (int) transferTrackerStats.totalUploadsSucceeded);
                 assertEquals((int) deserializedStats.totalUploadsFailed, (int) transferTrackerStats.totalUploadsFailed);

From dfb8449ed8f85e0c1e8601c86d17b833e1641e0f Mon Sep 17 00:00:00 2001
From: kkewwei <kkewwei@163.com>
Date: Thu, 11 Jul 2024 03:45:27 +0800
Subject: [PATCH 091/113] print reason why parent task was cancelled (#14604)

Signed-off-by: kkewwei <kkewwei@163.com>
---
 CHANGELOG.md                                     |  1 +
 .../cluster/node/tasks/CancellableTasksIT.java   |  4 ++--
 .../tasks/TaskCancellationService.java           |  2 +-
 .../java/org/opensearch/tasks/TaskManager.java   | 16 ++++++++++++----
 .../node/tasks/CancellableTasksTests.java        |  2 +-
 5 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 62bb73d80f2c1..813eecbaabfa3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
 - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
+- Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
index bdb36b62ada21..d8a4bed4740bf 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
@@ -327,7 +327,7 @@ public void testFailedToStartChildTaskAfterCancelled() throws Exception {
         mainAction.startSubTask(taskId, subRequest, future);
         TransportException te = expectThrows(TransportException.class, future::actionGet);
         assertThat(te.getCause(), instanceOf(TaskCancelledException.class));
-        assertThat(te.getCause().getMessage(), equalTo("The parent task was cancelled, shouldn't start any child tasks"));
+        assertThat(te.getCause().getMessage(), equalTo("The parent task was cancelled, shouldn't start any child tasks, by user request"));
         allowEntireRequest(rootRequest);
         waitForRootTask(rootTaskFuture);
         ensureAllBansRemoved();
@@ -386,7 +386,7 @@ static void waitForRootTask(ActionFuture<TestResponse> rootTask) {
             assertThat(
                 cause.getMessage(),
                 anyOf(
-                    equalTo("The parent task was cancelled, shouldn't start any child tasks"),
+                    equalTo("The parent task was cancelled, shouldn't start any child tasks, by user request"),
                     containsString("Task cancelled before it started:"),
                     equalTo("Task was cancelled while executing")
                 )
diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java
index 6955a5927ca23..5a4a25ec832bd 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java
@@ -92,7 +92,7 @@ void cancelTaskAndDescendants(CancellableTask task, String reason, boolean waitF
             Collection<DiscoveryNode> childrenNodes = taskManager.startBanOnChildrenNodes(task.getId(), () -> {
                 logger.trace("child tasks of parent [{}] are completed", taskId);
                 groupedListener.onResponse(null);
-            });
+            }, reason);
             taskManager.cancel(task, reason, () -> {
                 logger.trace("task [{}] is cancelled", taskId);
                 groupedListener.onResponse(null);
diff --git a/server/src/main/java/org/opensearch/tasks/TaskManager.java b/server/src/main/java/org/opensearch/tasks/TaskManager.java
index a49968ab85e89..6ad06da9d2fa2 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskManager.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskManager.java
@@ -510,17 +510,22 @@ public Set<TaskId> getBannedTaskIds() {
         return Collections.unmodifiableSet(banedParents.keySet());
     }
 
+    public Collection<DiscoveryNode> startBanOnChildrenNodes(long taskId, Runnable onChildTasksCompleted) {
+        return startBanOnChildrenNodes(taskId, onChildTasksCompleted, "unknown");
+    }
+
     /**
      * Start rejecting new child requests as the parent task was cancelled.
      *
      * @param taskId                the parent task id
      * @param onChildTasksCompleted called when all child tasks are completed or failed
+     * @param reason                the ban reason
      * @return the set of current nodes that have outstanding child tasks
      */
-    public Collection<DiscoveryNode> startBanOnChildrenNodes(long taskId, Runnable onChildTasksCompleted) {
+    public Collection<DiscoveryNode> startBanOnChildrenNodes(long taskId, Runnable onChildTasksCompleted, String reason) {
         final CancellableTaskHolder holder = cancellableTasks.get(taskId);
         if (holder != null) {
-            return holder.startBan(onChildTasksCompleted);
+            return holder.startBan(onChildTasksCompleted, reason);
         } else {
             onChildTasksCompleted.run();
             return Collections.emptySet();
@@ -585,6 +590,7 @@ private static class CancellableTaskHolder {
         private List<Runnable> cancellationListeners = null;
         private Map<DiscoveryNode, Integer> childTasksPerNode = null;
         private boolean banChildren = false;
+        private String banReason;
         private List<Runnable> childTaskCompletedListeners = null;
 
         CancellableTaskHolder(CancellableTask task) {
@@ -662,7 +668,7 @@ public CancellableTask getTask() {
 
         synchronized void registerChildNode(DiscoveryNode node) {
             if (banChildren) {
-                throw new TaskCancelledException("The parent task was cancelled, shouldn't start any child tasks");
+                throw new TaskCancelledException("The parent task was cancelled, shouldn't start any child tasks, " + banReason);
             }
             if (childTasksPerNode == null) {
                 childTasksPerNode = new HashMap<>();
@@ -686,11 +692,13 @@ void unregisterChildNode(DiscoveryNode node) {
             notifyListeners(listeners);
         }
 
-        Set<DiscoveryNode> startBan(Runnable onChildTasksCompleted) {
+        Set<DiscoveryNode> startBan(Runnable onChildTasksCompleted, String reason) {
             final Set<DiscoveryNode> pendingChildNodes;
             final Runnable toRun;
             synchronized (this) {
                 banChildren = true;
+                assert reason != null;
+                banReason = reason;
                 if (childTasksPerNode == null) {
                     pendingChildNodes = Collections.emptySet();
                 } else {
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index 7d706411b6f0d..5b3b08377f19b 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -428,7 +428,7 @@ public void testRegisterAndExecuteChildTaskWhileParentTaskIsBeingCanceled() thro
         );
         assertThat(cancelledException.getMessage(), startsWith("Task cancelled before it started:"));
         CountDownLatch latch = new CountDownLatch(1);
-        taskManager.startBanOnChildrenNodes(parentTaskId.getId(), latch::countDown);
+        taskManager.startBanOnChildrenNodes(parentTaskId.getId(), latch::countDown, cancelledException.getMessage());
         assertTrue("onChildTasksCompleted() is not invoked", latch.await(1, TimeUnit.SECONDS));
     }
 

From 125b77300eb783f502a23ec0329c77e3d7170494 Mon Sep 17 00:00:00 2001
From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com>
Date: Thu, 11 Jul 2024 10:33:23 +0530
Subject: [PATCH 092/113] Use set of shard routing for shard in unassigned
 shard batch check. (#14533)

Signed-off-by: Swetha Guptha <gupthasg@amazon.com>
---
 .../gateway/PrimaryShardBatchAllocator.java   |  8 +++--
 .../gateway/ReplicaShardBatchAllocator.java   | 11 ++++--
 .../PrimaryShardBatchAllocatorTests.java      | 36 +++++++++++++++++--
 3 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java b/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java
index 27f9bedc4e495..c493bf717c97f 100644
--- a/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java
@@ -23,8 +23,10 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * PrimaryShardBatchAllocator is similar to {@link org.opensearch.gateway.PrimaryShardAllocator} only difference is
@@ -82,6 +84,7 @@ public AllocateUnassignedDecision makeAllocationDecision(ShardRouting unassigned
      * @param allocation    the allocation state container object
      */
     public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAllocation allocation) {
+        logger.trace("Starting shard allocation execution for unassigned primary shards: {}", shardRoutings.size());
         HashMap<ShardId, AllocateUnassignedDecision> ineligibleShardAllocationDecisions = new HashMap<>();
         List<ShardRouting> eligibleShards = new ArrayList<>();
         List<ShardRouting> inEligibleShards = new ArrayList<>();
@@ -99,13 +102,13 @@ public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAll
         // only fetch data for eligible shards
         final FetchResult<NodeGatewayStartedShardsBatch> shardsState = fetchData(eligibleShards, inEligibleShards, allocation);
 
+        Set<ShardRouting> batchShardRoutingSet = new HashSet<>(shardRoutings);
         RoutingNodes.UnassignedShards.UnassignedIterator iterator = allocation.routingNodes().unassigned().iterator();
         while (iterator.hasNext()) {
             ShardRouting unassignedShard = iterator.next();
             AllocateUnassignedDecision allocationDecision;
 
-            if (shardRoutings.contains(unassignedShard)) {
-                assert unassignedShard.primary();
+            if (unassignedShard.primary() && batchShardRoutingSet.contains(unassignedShard)) {
                 if (ineligibleShardAllocationDecisions.containsKey(unassignedShard.shardId())) {
                     allocationDecision = ineligibleShardAllocationDecisions.get(unassignedShard.shardId());
                 } else {
@@ -115,6 +118,7 @@ public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAll
                 executeDecision(unassignedShard, allocationDecision, allocation, iterator);
             }
         }
+        logger.trace("Finished shard allocation execution for unassigned primary shards: {}", shardRoutings.size());
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java b/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java
index f2cb3d053440d..7c75f2a5d1a8f 100644
--- a/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java
@@ -28,10 +28,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
-import java.util.stream.Collectors;
 
 /**
  * Allocates replica shards in a batch mode
@@ -117,6 +118,7 @@ public AllocateUnassignedDecision makeAllocationDecision(ShardRouting unassigned
      * @param allocation    the allocation state container object
      */
     public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAllocation allocation) {
+        logger.trace("Starting shard allocation execution for unassigned replica shards: {}", shardRoutings.size());
         List<ShardRouting> eligibleShards = new ArrayList<>();
         List<ShardRouting> ineligibleShards = new ArrayList<>();
         Map<ShardRouting, AllocateUnassignedDecision> ineligibleShardAllocationDecisions = new HashMap<>();
@@ -135,7 +137,11 @@ public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAll
         // only fetch data for eligible shards
         final FetchResult<NodeStoreFilesMetadataBatch> shardsState = fetchData(eligibleShards, ineligibleShards, allocation);
 
-        List<ShardId> shardIdsFromBatch = shardRoutings.stream().map(shardRouting -> shardRouting.shardId()).collect(Collectors.toList());
+        Set<ShardId> shardIdsFromBatch = new HashSet<>();
+        for (ShardRouting shardRouting : shardRoutings) {
+            ShardId shardId = shardRouting.shardId();
+            shardIdsFromBatch.add(shardId);
+        }
         RoutingNodes.UnassignedShards.UnassignedIterator iterator = allocation.routingNodes().unassigned().iterator();
         while (iterator.hasNext()) {
             ShardRouting unassignedShard = iterator.next();
@@ -159,6 +165,7 @@ public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAll
                 executeDecision(unassignedShard, allocateUnassignedDecision, allocation, iterator);
             }
         }
+        logger.trace("Finished shard allocation execution for unassigned replica shards: {}", shardRoutings.size());
     }
 
     private AllocateUnassignedDecision getUnassignedShardAllocationDecision(
diff --git a/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java
index e90850de3fe33..8ad8bcda95f40 100644
--- a/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java
+++ b/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java
@@ -85,7 +85,10 @@ private void allocateAllUnassignedBatch(final RoutingAllocation allocation) {
         final RoutingNodes.UnassignedShards.UnassignedIterator iterator = allocation.routingNodes().unassigned().iterator();
         List<ShardRouting> shardsToBatch = new ArrayList<>();
         while (iterator.hasNext()) {
-            shardsToBatch.add(iterator.next());
+            ShardRouting unassignedShardRouting = iterator.next();
+            if (unassignedShardRouting.primary()) {
+                shardsToBatch.add(unassignedShardRouting);
+            }
         }
         batchAllocator.allocateUnassignedBatch(shardsToBatch, allocation);
     }
@@ -180,6 +183,35 @@ public void testInitializePrimaryShards() {
         assertEquals(2, routingAllocation.routingNodes().getInitialPrimariesIncomingRecoveries(node1.getId()));
     }
 
+    public void testInitializeOnlyPrimaryUnassignedShardsIgnoreReplicaShards() {
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        AllocationDeciders allocationDeciders = randomAllocationDeciders(Settings.builder().build(), clusterSettings, random());
+        setUpShards(1);
+        final RoutingAllocation routingAllocation = routingAllocationWithOnePrimary(allocationDeciders, CLUSTER_RECOVERED, "allocId-0");
+
+        for (ShardId shardId : shardsInBatch) {
+            batchAllocator.addShardData(
+                node1,
+                "allocId-0",
+                shardId,
+                true,
+                new ReplicationCheckpoint(shardId, 20, 101, 1, Codec.getDefault().getName()),
+                null
+            );
+        }
+
+        allocateAllUnassignedBatch(routingAllocation);
+
+        List<ShardRouting> initializingShards = routingAllocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING);
+        assertEquals(1, initializingShards.size());
+        assertTrue(shardsInBatch.contains(initializingShards.get(0).shardId()));
+        assertTrue(initializingShards.get(0).primary());
+        assertEquals(1, routingAllocation.routingNodes().getInitialPrimariesIncomingRecoveries(node1.getId()));
+        List<ShardRouting> unassignedShards = routingAllocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED);
+        assertEquals(1, unassignedShards.size());
+        assertTrue(!unassignedShards.get(0).primary());
+    }
+
     public void testAllocateUnassignedBatchThrottlingAllocationDeciderIsHonoured() {
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         AllocationDeciders allocationDeciders = randomAllocationDeciders(
@@ -258,7 +290,7 @@ private RoutingAllocation routingAllocationWithOnePrimary(
             .routingTable(routingTableBuilder.build())
             .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3))
             .build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, null, null, System.nanoTime());
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, null, System.nanoTime());
     }
 
     private RoutingAllocation routingAllocationWithMultiplePrimaries(

From 2d8c68cbb3eafb0811aafbf0436cd4041261caf9 Mon Sep 17 00:00:00 2001
From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com>
Date: Thu, 11 Jul 2024 10:42:29 +0530
Subject: [PATCH 093/113] Add versioning for UploadedIndexMetadata (#14677)

* Add versioning for UploadedIndexMetadata
* Handle componentPrefix for backward compatibility

Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
---
 .../remote/ClusterMetadataManifest.java       | 65 +++++++++++++++----
 .../remote/ClusterMetadataManifestTests.java  | 23 ++++++-
 .../RemoteClusterStateServiceTests.java       | 14 ++--
 3 files changed, 81 insertions(+), 21 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
index 2786cd432b002..3a66419b1dc20 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
@@ -20,6 +20,7 @@
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.gateway.remote.ClusterMetadataManifest.Builder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -243,7 +244,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
         parser.declareBoolean(ConstructingObjectParser.constructorArg(), COMMITTED_FIELD);
         parser.declareObjectArray(
             ConstructingObjectParser.constructorArg(),
-            (p, c) -> UploadedIndexMetadata.fromXContent(p),
+            (p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
             INDICES_FIELD
         );
         parser.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
@@ -277,7 +278,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
             parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD);
             parser.declareObjectArray(
                 ConstructingObjectParser.constructorArg(),
-                (p, c) -> UploadedIndexMetadata.fromXContent(p),
+                (p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
                 INDICES_ROUTING_FIELD
             );
             parser.declareNamedObject(
@@ -1112,16 +1113,30 @@ private static String componentPrefix(Object[] fields) {
             return (String) fields[3];
         }
 
-        private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER = new ConstructingObjectParser<>(
+        private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V0 = new ConstructingObjectParser<>(
+            "uploaded_index_metadata",
+            fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields))
+        );
+
+        private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V2 = new ConstructingObjectParser<>(
             "uploaded_index_metadata",
             fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields))
         );
 
+        private static final ConstructingObjectParser<UploadedIndexMetadata, Void> CURRENT_PARSER = PARSER_V2;
+
         static {
-            PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
-            PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
-            PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
-            PARSER.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
+            declareParser(PARSER_V0, CODEC_V0);
+            declareParser(PARSER_V2, CODEC_V2);
+        }
+
+        private static void declareParser(ConstructingObjectParser<UploadedIndexMetadata, Void> parser, long codec_version) {
+            parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
+            parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
+            parser.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
+            if (codec_version >= CODEC_V2) {
+                parser.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
+            }
         }
 
         static final String COMPONENT_PREFIX = "index--";
@@ -1130,15 +1145,32 @@ private static String componentPrefix(Object[] fields) {
         private final String indexUUID;
         private final String uploadedFilename;
 
+        private long codecVersion = CODEC_V2;
+
         public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) {
-            this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX);
+            this(indexName, indexUUID, uploadedFileName, CODEC_V2);
+        }
+
+        public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, long codecVersion) {
+            this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX, codecVersion);
         }
 
         public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) {
+            this(indexName, indexUUID, uploadedFileName, componentPrefix, CODEC_V2);
+        }
+
+        public UploadedIndexMetadata(
+            String indexName,
+            String indexUUID,
+            String uploadedFileName,
+            String componentPrefix,
+            long codecVersion
+        ) {
             this.componentPrefix = componentPrefix;
             this.indexName = indexName;
             this.indexUUID = indexUUID;
             this.uploadedFilename = uploadedFileName;
+            this.codecVersion = codecVersion;
         }
 
         public UploadedIndexMetadata(StreamInput in) throws IOException {
@@ -1175,10 +1207,13 @@ public String getComponentPrefix() {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
+            builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
                 .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID())
-                .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath())
-                .field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
+                .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath());
+            if (codecVersion >= CODEC_V2) {
+                builder.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
+            }
+            return builder;
         }
 
         @Override
@@ -1214,9 +1249,13 @@ public String toString() {
             return Strings.toString(MediaTypeRegistry.JSON, this);
         }
 
-        public static UploadedIndexMetadata fromXContent(XContentParser parser) throws IOException {
-            return PARSER.parse(parser, null);
+        public static UploadedIndexMetadata fromXContent(XContentParser parser, long codecVersion) throws IOException {
+            if (codecVersion >= CODEC_V2) {
+                return CURRENT_PARSER.parse(parser, null);
+            }
+            return PARSER_V0.parse(parser, null);
         }
+
     }
 
     /**
diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
index 02471c9cdbbbe..152a6dba6c032 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
@@ -48,7 +48,7 @@
 public class ClusterMetadataManifestTests extends OpenSearchTestCase {
 
     public void testClusterMetadataManifestXContentV0() throws IOException {
-        UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
+        UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0);
         ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
             .clusterTerm(1L)
             .stateVersion(1L)
@@ -74,7 +74,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException {
     }
 
     public void testClusterMetadataManifestXContentV1() throws IOException {
-        UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
+        UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V1);
         ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
             .clusterTerm(1L)
             .stateVersion(1L)
@@ -619,6 +619,24 @@ public void testUploadedIndexMetadataSerializationEqualsHashCode() {
         );
     }
 
+    public void testUploadedIndexMetadataWithoutComponentPrefix() throws IOException {
+        final UploadedIndexMetadata originalUploadedIndexMetadata = new UploadedIndexMetadata(
+            "test-index",
+            "test-index-uuid",
+            "test_file_name",
+            CODEC_V1
+        );
+        final XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        originalUploadedIndexMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+            final UploadedIndexMetadata fromXContentUploadedIndexMetadata = UploadedIndexMetadata.fromXContent(parser, 1L);
+            assertEquals(originalUploadedIndexMetadata, fromXContentUploadedIndexMetadata);
+        }
+    }
+
     private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedIndexMetadata uploadedIndexMetadata) {
         switch (randomInt(2)) {
             case 0:
@@ -642,4 +660,5 @@ private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedInde
         }
         return uploadedIndexMetadata;
     }
+
 }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 91ddd64cc2ccc..6cd9cbbf13848 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -2254,13 +2254,14 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
             .stateVersion(1L)
             .stateUUID("state-uuid")
             .clusterUUID("cluster-uuid")
+            .codecVersion(CODEC_V2)
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
             .build();
 
         BlobContainer blobContainer = mockBlobStoreObjects();
-        mockBlobContainer(blobContainer, expectedManifest, Map.of());
+        mockBlobContainer(blobContainer, expectedManifest, Map.of(), CODEC_V2);
         when(blobContainer.readBlob(uploadedIndexMetadata.getUploadedFilename())).thenThrow(FileNotFoundException.class);
 
         remoteClusterStateService.start();
@@ -2288,11 +2289,11 @@ public void testReadLatestMetadataManifestSuccess() throws IOException {
             .clusterUUID("cluster-uuid")
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
-            .codecVersion(ClusterMetadataManifest.CODEC_V0)
+            .codecVersion(CODEC_V2)
             .previousClusterUUID("prev-cluster-uuid")
             .build();
 
-        mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>());
+        mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>(), CODEC_V2);
         remoteClusterStateService.start();
         final ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
             clusterState.getClusterName().value(),
@@ -2416,10 +2417,10 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
-            .codecVersion(ClusterMetadataManifest.CODEC_V0)
+            .codecVersion(CODEC_V2)
             .build();
 
-        mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
+        mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata), CODEC_V2);
 
         Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestClusterState(
             clusterState.getClusterName().value(),
@@ -2664,6 +2665,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx
             .clusterUUID("cluster-uuid")
             .previousClusterUUID("prev-cluster-uuid")
             .routingTableVersion(1)
+            .codecVersion(CODEC_V2)
             .indicesRouting(List.of(uploadedIndiceRoutingMetadata))
             .build();
 
@@ -3081,7 +3083,7 @@ private void mockBlobContainerForGlobalMetadata(
             FORMAT_PARAMS
         );
         when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
-        if (codecVersion >= ClusterMetadataManifest.CODEC_V2) {
+        if (codecVersion >= CODEC_V2) {
             String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename());
             when(blobContainer.readBlob(COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))).thenAnswer((invocationOnMock) -> {
                 BytesReference bytesReference = COORDINATION_METADATA_FORMAT.serialize(

From 17b799697e1dbfe65671531f722a1433e646b1bc Mon Sep 17 00:00:00 2001
From: Ahmed Sobeh <ahmed.sobeh@aiven.io>
Date: Thu, 11 Jul 2024 17:00:13 +0200
Subject: [PATCH 094/113] Fix: update help output for _cat (#14722)

* fixed help output for _cat

Signed-off-by: ahmedsobeh <ahmed.sobeh@aiven.io>

* updated changelog

Signed-off-by: ahmedsobeh <ahmed.sobeh@aiven.io>

* updated changelog

Signed-off-by: ahmedsobeh <ahmed.sobeh@aiven.io>

---------

Signed-off-by: ahmedsobeh <ahmed.sobeh@aiven.io>
---
 CHANGELOG.md                                                | 1 +
 .../org/opensearch/rest/action/cat/RestNodesAction.java     | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 813eecbaabfa3..cb8e6403aa47e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -65,6 +65,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Handle NPE in GetResult if "found" field is missing ([#14552](https://github.com/opensearch-project/OpenSearch/pull/14552))
 - Refactoring FilterPath.parse by using an iterative approach ([#14200](https://github.com/opensearch-project/OpenSearch/pull/14200))
 - Refactoring Grok.validatePatternBank by using an iterative approach ([#14206](https://github.com/opensearch-project/OpenSearch/pull/14206))
+- Update help output for _cat ([#14722](https://github.com/opensearch-project/OpenSearch/pull/14722))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java
index e11012a23fce7..bffb50cc63401 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java
@@ -171,9 +171,9 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("port", "default:false;alias:po;desc:bound transport port");
         table.addCell("http_address", "default:false;alias:http;desc:bound http address");
 
-        table.addCell("version", "default:false;alias:v;desc:es version");
-        table.addCell("type", "default:false;alias:t;desc:es distribution type");
-        table.addCell("build", "default:false;alias:b;desc:es build hash");
+        table.addCell("version", "default:false;alias:v;desc:os version");
+        table.addCell("type", "default:false;alias:t;desc:os distribution type");
+        table.addCell("build", "default:false;alias:b;desc:os build hash");
         table.addCell("jdk", "default:false;alias:j;desc:jdk version");
         table.addCell("disk.total", "default:false;alias:dt,diskTotal;text-align:right;desc:total disk space");
         table.addCell("disk.used", "default:false;alias:du,diskUsed;text-align:right;desc:used disk space");

From 82bffb1d506366f952c813e349ce41dc037fde46 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Thu, 11 Jul 2024 14:13:22 -0400
Subject: [PATCH 095/113] Fix hdfs-fixture kerb-admin & hadoop-minicluster
 dependencies are not being updated / false positive reports on CVEs (#14729)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 test/fixtures/hdfs-fixture/build.gradle | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index a532bf0c6287b..6ab6d5acb8880 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -33,7 +33,7 @@ apply plugin: 'opensearch.java'
 group = 'hdfs'
 
 versions << [
-  'jetty': '9.4.53.v20231009'
+  'jetty': '9.4.55.v20240627'
 ]
 
 dependencies {
@@ -73,7 +73,12 @@ dependencies {
   api "commons-net:commons-net:3.11.1"
   api "ch.qos.logback:logback-core:1.5.6"
   api "ch.qos.logback:logback-classic:1.2.13"
-  api 'org.apache.kerby:kerb-admin:2.0.3'
+  api "org.jboss.xnio:xnio-nio:3.8.16.Final"
+  api 'org.jline:jline:3.26.2'
+  api ('org.apache.kerby:kerb-admin:2.0.3') {
+    exclude group: "org.jboss.xnio"
+    exclude group: "org.jline"
+  }
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   runtimeOnly("com.squareup.okhttp3:okhttp:4.12.0") {
     exclude group: "com.squareup.okio"

From 9234a4226acf00416108f4f1484a4a49632ecba2 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Thu, 11 Jul 2024 16:55:32 -0400
Subject: [PATCH 096/113] Update to Gradle 8.9 (#14574)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 .../gradle/test/rest/RestResourcesPlugin.java |  56 +++++++++---------
 gradle/wrapper/gradle-wrapper.jar             | Bin 43462 -> 43504 bytes
 gradle/wrapper/gradle-wrapper.properties      |   4 +-
 gradlew                                       |   7 ++-
 gradlew.bat                                   |   2 +
 5 files changed, 38 insertions(+), 31 deletions(-)

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java
index fcadf35593ce6..9396797536052 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java
@@ -81,50 +81,52 @@ public void apply(Project project) {
         // tests
         Configuration testConfig = project.getConfigurations().create("restTestConfig");
         project.getConfigurations().create("restTests");
+
+        if (BuildParams.isInternal()) {
+            // core
+            Dependency restTestdependency = project.getDependencies().project(new HashMap<String, String>() {
+                {
+                    put("path", ":rest-api-spec");
+                    put("configuration", "restTests");
+                }
+            });
+            testConfig.withDependencies(s -> s.add(restTestdependency));
+        } else {
+            Dependency dependency = project.getDependencies().create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
+            testConfig.withDependencies(s -> s.add(dependency));
+        }
+
         Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
             .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.coreConfig = testConfig;
                 task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
-                if (BuildParams.isInternal()) {
-                    // core
-                    Dependency restTestdependency = project.getDependencies().project(new HashMap<String, String>() {
-                        {
-                            put("path", ":rest-api-spec");
-                            put("configuration", "restTests");
-                        }
-                    });
-                    project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
-                } else {
-                    Dependency dependency = project.getDependencies()
-                        .create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
-                    project.getDependencies().add(task.coreConfig.getName(), dependency);
-                }
                 task.dependsOn(task.coreConfig);
             });
 
         // api
         Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity
         project.getConfigurations().create("restSpecs");
+
+        if (BuildParams.isInternal()) {
+            Dependency restSpecDependency = project.getDependencies().project(new HashMap<String, String>() {
+                {
+                    put("path", ":rest-api-spec");
+                    put("configuration", "restSpecs");
+                }
+            });
+            specConfig.withDependencies(s -> s.add(restSpecDependency));
+        } else {
+            Dependency dependency = project.getDependencies().create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
+            specConfig.withDependencies(s -> s.add(dependency));
+        }
+
         Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
                 task.includeCore.set(extension.restApi.getIncludeCore());
                 task.dependsOn(copyRestYamlTestTask);
                 task.coreConfig = specConfig;
                 task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
-                if (BuildParams.isInternal()) {
-                    Dependency restSpecDependency = project.getDependencies().project(new HashMap<String, String>() {
-                        {
-                            put("path", ":rest-api-spec");
-                            put("configuration", "restSpecs");
-                        }
-                    });
-                    project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
-                } else {
-                    Dependency dependency = project.getDependencies()
-                        .create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
-                    project.getDependencies().add(task.coreConfig.getName(), dependency);
-                }
                 task.dependsOn(task.coreConfig);
             });
 
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index d64cd4917707c1f8861d8cb53dd15194d4248596..2c3521197d7c4586c843d1d3e9090525f1898cde 100644
GIT binary patch
delta 34463
zcmY(qRX`kF)3u#IAjsf0xCD212@LM;?(PI<f(@>NyAue(f;$XO2=4Cg1P$=#e%|lo
zKk1`B>Q#GH)wNd-&cI#Hz}3=WfYndTeo)CyX{fOHsQjGa<{e=jamMNwjda<PyWE6-
zP~B#x7leo3nD`qQ)CqpqLwL&Nv*c_uf@6ZzDxH4Kk*=_Q!1=S4w`ejm#xafRiVQ9K
zD|NMG<SxpdU<&W$m3qe4z`XCn(ZbVn07Ju^@)#`N!0VN6<~T~KDoO4tX0Wnd#9^aG
zzw_kBWvKv8AX+m002<gC-eNM-rZnBB9F6nP%6MQjX4aHBE}ky4v9DAk%X3G|ZVInC
z0sH3o9G@FVA`%%F8RyEFms&lE|8}H=IL>tD={CN3>GNchOE9OGPIqr)3v>RcKWR3Z
zF-guIMjE2UF0Wqk1)21791y#}ciBI*bAenY*B<S7Lsj7;64caBml7(9sSP$#!4bz5
zfe$Y|L&2pF$!`g2EFBw-(C>MW_)AeSuM5}vz_~`+1i!Lo?XAEq{TlK5-efNFgHr6o
zD>^vB&%3ZGEWMS>`?tu!@66|uiDvS5`?bF=gIq3rkK(j<_TybyoaDHg8;Y#`;>tXI
z=tXo~e9{U!*hqTe#nZjW4z0mP8A9UUv1}C#R*@yu9G3k;`Me0-BA2&Aw6f`{Ozan2
z8c8Cs#dA-7V)ZwcGKH}jW!Ja&VaUc@mu5a@CObzNot?b{f+~+212lwF;!QKI16FDS
zodx>XN$sk9;t;)maB^s6sr^L32EbMV(uvW%or=|0@U6cU<cnXKOZI@-_2b|FP!$}s
zAe;S8Mg2+heB}qk71p?*lsuXu#LR?_0T)ujf|bz~>kE`_!<=LHLlRGJx@gQI=B(nn
z-GEjDE}*8>3U$n(t^(b^C$q<Dz5^v_5fDC=_*60>STI;}6q&ypp?-2rGpqg7b}pyT
zOARu2x><t9J4jOACSe7Cla*>0HB{&D(d3sp`+}ka+Pca5glh|c=M)Ujn_$ly^X6&u
z%Q4Y*LtB_>i6(YR!?{Os-(^J`(70lZ&Hp1I^?t@~SFL1!m0x6j|NM!-JTDk)%Q^R<
z@e?23FD&9_W{Bgtr&CG&*Oer3Z(Bu2EbV3T9FeQ|-vo5pwzwQ%g&=zFS7b{n6T2ZQ
z*!H(=z<{D9@c`K<z${~)Mo?=i!8kgh3^kh#gZ+tgX&gTG!eX0?^?c|0?IsnpYDup`
zE@9&nF#>mHO&DbUIzpg`+r5207}4D=_P$ONIc5lsFgn)UB-oUE#{r+|uHc^hzv_df
zV`n<Uy?gf){vC8%ni$H6BLQ|&nNh_JVuc@V<M{j`fPiV3mmYOH;{D(UIYmW$_M(id
zfz>8&qry%jXQ33}Bjqcim~BY1?KZ}x453Oh7G@fA(}+m(f$)TY%7n=MeLi{jJ7LMB
zt(mE*vFnep?YpkT_&WPV9*f>uSi#n#@STJmV&SLZnlLsWYI@y+Bs=gzcqche=&<e&
zH!*BfX&4XuI(sK|M6EWNz&xj!)?<o^!;`2$ksx0>cBH2WL)dkR!a95*Ri)JH_4c*-
zl4pPLl^as5_y&6RDE@@7342DNyF&GLJez#eMJjI}#pZN{Y8io{l*D+|f_Y&RQPia@
zNDL;SBERA|B#cjlNC@V<BMj^PW+*!uB6Kia#Q_k#D<Kdw%NW0h5PShX-Ateg=^YLq
zPqOC>U{2csOvB8$HzU$01Q?y)KEfos>W46VMh>P~oQC8k=26-Ku)@C|n^zDP!hO}Y
z_tF}0@*Ds!JMt>?4y|l3?`v#5*oV-=vL7}zehMON^=s1%q+n=^^Z{^mTs7}*->#YL
z)x-~SWE{e?YCarwU$=cS>VzmUh?Q&7?#Xrcce+jeZ|%0!l|H_=D_`77hBfd4Zqk&!
zq-Dnt_?5*$Wsw8zGd@?woEtfYZ2|9L8b>TO6>oMh%`B7iBb)-aCefM~q|S2Cc0t9T
zlu-ZXmM0wd$!gd-dTtik{bqyx32%f;`XUvbUWWJmpHfk8^PQIEsByJm+@+-aj4J#D
z4#Br3pO6z1eIC>X^yKk|<Bgti^oqrbEGF$Jw^;w14UYT$*kTROuXlF!RjFMV72W^U
zoMllbg+6ow&xKMI7=%s{uAdU)(;XWPXi7!aAo)eS$$~Y>PeVwX_4B+IYJyJyc3B`4
zPrM#raacGIzVOexcVB;fcsxS=s<vwC^XKnl?x{xO@E;np^jTyr6HSfEb6qXiZr9dg
z)?Uhot!j3J`wuMIE#*geg>1e&V;Xe$tw&KQ`YaCkHTKe*Al#velxV{3wxx}`7@isG
zp6{+s)CG%HF#JBAQ_jM%zCX5X;J%-*%&jVI?6KpYyzGbq7qf;&hFprh?E5Wyo=bZ)
z8YNycvMNGp1836!-?nihm6jI`^C`EeGryoNZO1AFTQhzFJOA%Q{X(sMYlzABt!&f{
zoDENSuoJQIg5Q#@BUsNJX2h>jkdx4<+ipUymWKFr;w+s>$laIIkfP6nU}r+?J9bZg
zUIxz>RX$kX=C4m(zh-Eg$BsJ4OL&_J38PbHW&7JmR27%efAkqqdvf)Am)VF$+U3WR
z-E#I9H6^)zHLKCs7|Zs<7Bo9VCS3@CDQ;{UTczoEprCKL3ZZW!ffmZFkcWU-V|_M2
zUA9~8tE9<5`59W-UgUmDFp11YlORl3mS3*2#ZHjv{*-1#uMV_oVTy{PY(}AqZv#wF
zJVks)%N6LaHF$$<6p8S8Lqn+5&t}DmLKiC~lE{jPZ39oj{wR&fe*LX-z0m}9ZnZ{U
z>3-5Bh{KKN^n5i!M79Aw5eY=`6fG#aW1_ZG;fw7JM69qk^*(rmO{|Z6rXy?l=K=#_
zE-zd*P|(sskasO(cZ5L~_{Mz&Y@@@Q)5_8l<6vB$<SnmTz>@226O+pDvkFaK8b>%2
zfMtgJ@+cN@w>3)(_uR;s8$sGONbYvoEZ3-)zZk4!`tNzd<0lwt{RAgplo*f@Z)uO`
zzd`ljSqKfHJOLx<Pf6T=jN-Y)-G`fh^*;28D6YUz?Jn$Qe<sb8Bqa)nI_IHAx#i9Y
z$j}`a&gQX1Vaa!Ea{Z|{=AEw)&>ya4_}T`k5Ok1Mpo#MSqf~&ia3uIy{zyuaF}pV6
z)@$ZG5LYh8Gge*LqM_|GiT1*J*uKes=Oku_gMj&;FS`*sfpM+ygN&yOla-^WtIU#$
zuw(_-?DS?6DY7IbON7J)p^IM?N>7x^3)(7wR4PZJu(teex%l>zKAUSNL@~{czc}bR
z)I{Xz<Lxhu3=iQUmF>XqZBU3a;7UQ~PvAx8g-3q-9AEd}1JrlfS8NdPc+!=HJ6Bs(
zCG!0;e0z-22(Uzw>hkEmC&<fdEY_MO4!XLJs6(n80jtDO!nnCLqs#Y_I0IE~UCPp@
zM_JDqY7&S(Y&QK^-o&47T#3E|`NAlIxvHf3kny;RvvXG3GAWOcnn~MYcAb70=&uwJ
zkt37-cyED3`=cXw-}?~Jre&#~t1kA3Jz_#Bn#jD975Ham#lXmn@!Xe|z!$O^A|8Ja
zxE40Kw#qS@4?x}td&6F5(NIU<?GmP1&i-wkqanUS(~JutQ=}J2g$roe>xj?{0p|kc
zM}MMXCF%RLLa#5jG`+}{pDL3M&|%3BlwOi?dq!)KUdv5__zR>u^o|QkYiqr(m3HxF
z6J*DyN#Jpooc$ok=b7{UAVM@<lCUruKn!C)jTs?G3tF=X?02t9ti$}%_-&Apkmnno
z%fbj!k#YwEr={VibqP-VP5rAcB3fluUzRfi(z01WI?#!N!mLc);E?^+GEin57qXIO
zg8kNfR~-pWXV>nwGsr6kozSddwulf5g1{B=0#2)zv!zLXQup^BZ4sv*sEsn)+MA?t
zEL<g?Ek3W6nR6!O+b*MM+O~eGu_sDZwCnE-jqq7Mvt4JiAYMi&B_$Kb75}|i#Oq?4
z^@)_zxi_9dEszN`mdX;tu;@!Yj!NQXjGcs`B*6!}Q`Xp8$`LBVWzdw?k{MZ=kF0i6
z0u$CdmQDt$g$6JM?JyEA2oW1N)I#|wQ>)}3*R?4(J~CpeSJPM!oZ~8<cg4cY=X|VM
z=^|PC37b4k-X|34t*R-GX&L@u*U|J4q?K3(MG@qFTXx0}+s2jwB%hz+*7-k{2(QHj
zQmM#6vaQDzB)m!b_6aC`nE*p#lagJ>;8s_=@6o`IA%{aEA9!GELRvOuncE`s7sH91
zmF=+T!Q6%){?lJn3`5}oW31(^Of|$r%`~gT{eimT7R~*Mg@x+tWM3KE>=Q>nkMG$U
za7r>Yz2LEaA|PsMafvJ(Y>Xzha?=>#B!sYfVob4k5Orb$INFdL@U0(J8Hj&kgWUlO
zPm+R07E+oq^4f4#HvEPANGWLL_!uF{nkHYE&BCH%l1FL_r(Nj@M)*VOD5S42Gk-yT
z^23oAMvpA57H(fkDGMx86Z}rtQhR^L!T2iS!<mO!Z%ty;Vm%l(OCwBkU1^sK>788E
z+^${W1V}J_NwdwdxpXAW8}#6o1(Uu|vhJvubFvQIH1bDl4J4iDJ+181KuDuHwvM?`
z%1@Tnq+7>p{O&p=@QT}4wT;HCb@i)&7int<0#bj8j0sfN3s6|a(l7Bj#7$hxX@~iP
z1HF8RFH}irky&eCN4T94VyK<n?6cO6O!>qGywEGY{Gt0Xl-`|dOU&{Q;Ao;sL>C6N
zXx1y^RZSaL-pG|JN;j9ADjo^XR}gce#seM4QB1?S`L*aB&QlbBIRegMnTkTCks7JU
z<0(b+^Q?HN1&$M1l&I@>HMS;!&bb()a}hhJzsmB?I`poqTrSoO>m_JE5U4=?o;OV6
zBZjt;*%1P>%2{UL=;a4(aI>PRk|mr&F^=v6Fr&xMj8fRCXE5Z2qdre&;$_RNid5!S
zm^XiLK25G6_j4dWkFqjtU7#s;b8h?BYFxV?OE?c~&ME`n`$ix_`mb^AWr+{<eR?ie
zu^n7F<r|0}4&z!1h-oy{4Bz+F69}Q`VpCh(8k$5~F}&!BM0MdLg4qhh@bV;%4Rw1m
zzqF@FV8+w9w|_OPDgrx*NZebP^m@hq4m*)5tuF1+`%o#G^#zx@w5ZsxZaUE7jzw;N
zhD*<Wl`rn3nq7+Ag+JpPe%(D1QRTS`qM+;}+_^&I$1>M9{^^Rl;~KREplwy2q;&xe
zUR0SjHzKVYzuqQ84w$NKVPGVHL_4I)Uw<$uL2-Ml#+5r2X{LLqc*p13<F6YTV#kWf
zqG&w9QQdWL4?_d~CQnBI;?0D)Qg}{>{;w#E*Kwb*1D|v?e;(<>vl@VjnFB^^Y;;b3
z=R@(uRj6D}-h6CCOxAdqn~_SG=bN%^9(Ac?zfRkO5x2VM0+@_qk?MDXvf<z0O)%s7
zMAFU0$m3HcbC#~2Hc64da@3O%^-=j9undudUI{}zbhknzUL#akDQiL_v_tyrinjEO
zI(3QAK0&1x_y)uNn}57R#|)d${jFV)^rH-I`D9H?s`)o)uI9hKceIT0a|7|+J9G4R
z?^G4wKj6MYf*wobKriuVpvC#r;9ThcHk6^a^xn2ODvRhy6<dTeve8{i!Cu=iTXW7{
z{?H_E?=z!{hGvD8B;Y6V{)tUgB~RL}OZbPv*z!^?TUpHIl7qWjaL~c+st5S>=@q_*
z3IM@)er6-OXyE1Z4sU3{8$Y$>8NcnU-nkyWD&2ZaqX1JF_JYL8y}>@V8A5%lX#U3E
zet5PJM`z79q9u5v(OE~{by|Jzlw2<0h`hKpOefhw=fgLTY9M8h+?37k@TWpzAb2Fc
zQMf^aVf!yXlK?@5d-re}!fuAWu0t57ZKSSacwRGJ$0uC}ZgxCTw>cjRk*xCt%w&hh
zoeiIgdz__&u~8s|_TZsGvJ7sjvBW<(C@}Y%#l_ID2&C`0;Eg2Z+pk;IK}4T@W6X5H
z`s?ayU-iF+aNr5--T-^~K~p;}D<LJ~4Vm)-6?_|)BF5gW4JSY7&(G|Av&E;#AyyV$
zSunlmXPqL`WNNYmTNMzV20flD83AxG;E^Ec8R;(;vdTi$02nI6(4u&=rI7r^2x}@}
zb^h(s^Hv9AwRzcBrk1j8D%Js|d7yM)>(*GWOAYDV9JEw!w8ZYzS3;W6*_`#aZw&9J
ziXhBKU3~zd$kKzCAP-=t&cFDeQR*_e*(excIUxKuD@;-twSlP6>wWQU)$|H3Cy+`=
z-#7OW!ZlYzZxkdQpfqVDFU3V2B_-eJS)Fi{fLtRz!K{~7TR~XilNCu=Z;{GIf9KYz
zf3h=Jo+1#_s>z$lc~e)l93h&RqW1VHYN;Yjwg#Qi0yzjN^M4cuL>Ew`_-_wRhi*!f
zLK6vTpgo^Bz?8AsU%#n}^EGigkG3FXen3M;hm#C38P@Zs4{!QZPAU=m7ZV&xKI<yS
zef9C7IM#)bKApw^nm2P0xPH)?)BN&sFvRw*IK$#0AfO}b<r{uW7o8y}SN)F@&yQm%
zn~nNb7d)eJ2BHvG2FG3z@j=payQlcc`;O$D3XPT3JFukw8Xr%@g@bkhB?@#$+AOw(
z(|@nqsNM;#gS0yC1MWhDA!W&4Ru~!5ks5Q~Pml#jZyfXEak(Em<WRqk+Ush$9swq)
zmPM%H4#Ov0YvR0-8rrI^layyN^pm(_yZ9(@AnNKHRrR<^fYJvzs{G9-d<-PIeZxic
zI!fz70I#*sa*eAVT<99Vxgy-<y1w@xbY}slZ7CHAcKSW+wgZc7BMb2FfqhW}?;;BY
zP!O(w$5_aUKLF$CVvI<PnooNWjt084FOTmh7FT(=9D-a~XGZP;^5(x4>_HWNt90Ef
zxClm)ZY?S|n**2cNYy-xBlLAVZ=~+!|7y`(fh+M$#4zl&T^gV8ZaG(RBD!`3?9xcK
zp2+aD(T%QIgrLx5au&TjG1AazI;`8m{K7^!@m>uGCSR;Ut{&?t%3AsF{>0Cm(Kf)2
z?4?|J+!BUg*P~C{?m<lX{_VAm$p$7Sy5n4)MhYq@8YJpQU2G)^4Cs_N!H_NDI5wLC
zuxU!+I__)7xJYF^Pf|zP9)c{~jt|)Y(3ss;D?4|k?Aspv4X!02;`uMjy{H&7#%Q@o
z<nV(d4I5h&#m8g~O!r&@N38SW3XQ`EjZh!<u+JXC85$B2rn?)m6Y00}q3o2FWUH%+
zxkU_~@oBdV%UzT$+qN=8G&dY}bCZ=N#`=O2mzStcgw{uTG7hdoSCO(-*C<hlyRW_D
zI1Ra`6kd?oLFBA13(Fk<4zU<VtAK2o$FG2w@>wPQ#)gDMmro20YVNsVx5oWQMkzQ?
zsQ<ZZLyAH1!oNzFM}TfglR)=rWx*=M9GIjB?ZFULc>%Y>%7_wkJqnSMuZjB9lBM(o
zWut|B7w48cn}4buUBbdPBW_J@H7g=szrKEpb|aE>!4rLm+sO9K%iI75y~2HkUo^iw
zJ3se$8$|W>3}?JU@3h@M^HEFNmvCp|+$-0M?RQ8SMoZ@38%!tz8f8-Ptb@106heiJ
z^Bx!`0=I<O1KwEC{Zqa_WBh?i$W#4Fd46V<qLHAqFT^%K_=ff(6##B-Ub(l(@oz>m
z1!NUhO=9ICM*+||b3a7w*Y#5*Q}K^ar+oMMtekF0JnO>hzHqZKH0&PZ^^M(j;vwf_
z@^|VMBpcw8;4E-9J{<clOdQ-gLs}yNpT|>(u7sHSyZpQbS&N{VQ%ZCh{c1UA5;?R}
z+52*X_tkDQ(s~#-6`z4|Y}3N#a&dgP4S_^tsV=oZ<Y0d)Rr;arA(GJ}DlJV*pmcys
z*Mk;Dg{B7HD(oM1AgV$?pZ(feakO5tAV~SCs_W%F(?;(4m+NEoby{|2mc0XI>r4A1
zaSoPN1czE(UIBrC_r$0HM?RyBGe#lTBL4~JW#A`P^#0wuK)C-2$B6TvM<GJr3h=;Q
z>i@@%K@JAT_IB^T7Zfqc8?<ppV&KJMBtJ-oMFyu3aJ37qp`YqlHwSWDi``;8R3fKH
z;())3jrugRTov;`f2|96Ol0~`tdom%2YtK*BEp7dW){Z1+eUuiY^^+%IXC`kb4D<x
zHzFJB;p^zZ4`bRwAwzCEy&SFu3-$>{wHcSVG_?{(wUG%zhCm=%qP~EqeqKI$9UivF
zv+5IUOs|%@ypo6b+i=xsZ=^G1yeWe)z6IX-EC`F=(|_GCNbHbNp(CZ*lpSu5n`FRA
zhnrc4w+Vh?r>her@Ba_jv0Omp#-H7avZb=j_A~B%V0&FNi#!S8cwn0(Gg-Gi_LMI{
zCg=<XfJ{>g@m{W@u?GQ|yp^yENd;M=W2s-k7Gw2Z(tsD5fTGF{iZ%Ccgjy6O!AB4x
z%&=6jB7^}pyftW2YQpOY1w@%wZy%}-l0qJlOSKZXnN2wo3|hujU+-U~blRF!^;Tan
z0w;Srh0|Q~6*tXf!5-rCD)OYE(%S|^WTpa1KHtpHZ{!;KdcM^#g8Z^+LkbiBHt85m
z;2xv#83lWB(kplfgqv@ZNDcHizwi4-8+WHA$U-HBN<XXr0~J?$Lai#_;trL)VA8+1
zU!;TqE~}9XS7A@|JfNqKkGav_MPU#1&?e(-M~ysHg+5_(nK1v-W}9=?Zer5AMLpDA
zz5CzT^brYv;fMsP%wz!?lZWTP?W}8@vv2IuVI?tan0C9Y`-Wjik0PN~x7}xCz5j)1
zoSKG<CGs%hfuQ;WLC{xnn)TVIkPvvNhsVT7>qsZ`hKcUI3zV3d1ngJP-AMRET*A{>
zb2A>Fk|L|WYV;Eu4>{a6ESi2r3aZL7x}eRc?cf|~bP)6b7%BnsR{Sa>K^0obn?yiJ
zCVvaZ&;d_6WEk${F1SN0{_`(#TuOOH1as&#&xN~+JDzX(D-WU_nLEI}T_VaeLA=bc
zl_UZS$nu#C1yH}YV>N2^9^zye{rDrn(rS99>Fh&jtNY7PP15q%g=RGnxACdCov47=
zwf^<OLlg}xVjt0)p-rbBn7T~vS^5_Drk~dEL?fU{>9zfJaL{y`R#~tvVL#*<`=`Qe
zj_@Me$6sIK=LMFbBrJps7vdaf_HeX?eC+P^{AgSvbEn?n<}NDWiQGQG4^ZOc|GskK
z$Ve2_n8gQ-KZ=s(f`_X!+vM5)4+QmOP()2Fe#IL2toZBf+)8gTVgDSTN1CkP<}!j7
z0SEl>PBg{MnPHkj4wj$mZ?m5x!1ePVEYI(L_sb0OZ*=M%yQb?L{UL(2_*CTVbRxBe
z@{)COwTK1}!*CK0Vi4~AB;HF(MmQf|dsoy(eiQ><S&flYs}X=qI6%JhOL}c}8ReE6
z>WTKcEQlnKOri5xYsqi61Y=I4kzAjn5~{IWrz_l))|<ryzIgBfLDnKw3JSt*BE7Yk
z@<X1~%_3&{c@EWKT@&!_5{hgsv0*uWq^Ag5&!#`hO30t*Ey!ZHlK~#2TL+u093@m<
zmh@mPLF@+cbRf2&AOJqqsB1zg;bVL&+4p#cS%+M6!rPy}l*roS@+s2E+JcG2%rRTv
z2!(JA6$64oDKq`kHYO({m<UUf<`TtSFw}vw1zYukXby+c34(E)w*pi_qs3N7vw>Ls
zvq7xgQs?Xx@`N?f7+3XKLyD~6DRJw*uj*j?yvT3}a;(j_?YOe%hUFcPGWRVBXzpMJ
zM43g6DLFqS9tcTLSg=^&N-y0dXL816v&-nqC0iXdg7kV|PY+js`<tQ)jQ;9->F8dm
z2PuHw&k+8*&9SPQ6f!^5q0&AH(i+z3I7a?8O+S5`g)>}fG|BM&ZnmL;rk)|u{1!aZ
zEZHpAMmK_v$GbrrWNP|^2^s*!0waLW=-h5PZa-4jWYUt(<b#|Bsx3P3#+uA4`mmgX
z9Kz|`zO$@)@d{sIynMxFy0ta)_!*pZfTlj%)9Ee4zRt~F_6}gJArZ1%_&ds}7FnKN
zT=p-*f9G3aBCWvVUn3v-@ZXvQrHP}1zTx3P<Jhsl2HrSYxUcBQibO?|u3Wl@z}_;u
z86>Hr@EA(m3Mc3^uDxwt-me^55FMA9^>hpp26MhqjLg#^Y7OIJ5%ZLdNx&uDgIIqc
zZRZl|n6TyV)0^DDyVtw*jlWkDY&Gw4q;k!UwqSL6&sW$B*5Rc?&)dt29bDB*b6IBY
z6SY6Unsf6AOQdEf=P1inu6(6hVZ0~v-<>;LAlcQ2u?wRWj5VczBT$Op#8IhppP-1t
zfz5H59Aa~yh7EN;BXJsLyjkjqARS5iIhDVPj<=4AJb}m6M@n{xYj3qsR*Q8;h<IbJ
zd)uk&8m`?q_q1NS{$84>VxDyC4vLI;;?^eENOb5QARj#nII5l$MtBCI@5u~(ylFi$
zw6-+$<K_|-Wlyt+M0?(Bv&pcNNg0svOYzThtqKx!i*Dc5NZvH&F*N23|J~!vMN;d~
z^=H()7fOiv1)6`d`Xlx->$XQ}Ca>FWT>q{k)g{Ml(Yv=6aDfe?m<nPFf#(N8Laf1%
zeJ4>|5|kbGtWS}fKWI+})F6`x@||0oJ<?@*&=4(53;s&f1=p}VJ=CsEx0>^(g|+xi
zqlPdy5;`g*i*C=Q(aGeDw!eQg&w>UUj^{o?PrlFI=34qAU2u@BgwrBiaM8zoDTFJ<
zh7nWpv>dr?q;4ZA?}V}|7qWz4W?6#S&m>hs4IwvCBe@-C>+oohsQZ^JC*RfDRm!?y
zS4$7oxcI|##ga*y5hV>J4a%HHl^t$pjY%caL%-FlRb<$A$E!ws?8hf0@(4HdgQ!@>
zds{&g$ocr9W4I84TMa9-(&^_B*&R%^=@?Ntxi|Ejnh;z=!|uVj&3fiTngDPg=0=P2
zB)3#%HetD84ayj??qrxsd<YW&s^_fNRPJu1Lu~ooseFw$wVknfo;k}<f4``0v~+Y_
zKU_m+@|S(3^pR^^<>9nqrBem(8^_u_UY{1@R_vK-0H9N7lBX5K(^O2=0#TtUUGSz{
z%g>qU8#a$DyZ~EMa|8*@`GOhCW3%DN%xuS91T7~iXRr)SG`%=Lfu%U~Z_<B8YLfje
zenkDPv?7UMn^wh-#5zYOX60}ftn+_;r9>`1b=lSi?qpD4$vLh$?HU6t0MydaowUpb
zQr{>_${AMesCEffZo`}K0^~x>RY_ZIG{(r39MP>@=aiM@C;K)jUcfQV8#?SDvq>9D
zI{XeKM%$$XP5`7p3K0T}x;qn)VMo>2t}Ib(6zui;k}<<~KibAb%p)**e>ln<=qyWU
zrRDy|UXFi9y~PdEFIAXejLA{K)6<)Q`?;Q5!KsuEw({!#Rl8*5_F{TP?u|5(Hijv(
ztAA^I5+$A*+*e0V0R~fc{ET-RAS3suZ}TRk3r)xqj~g_hxB`qIK5z(5wxYboz%46G
zq{izIz^5xW1Vq#%lhXaZL&)FJWp0VZNO%2&ADd?+J%K$fM#T_Eke1{dQsx48dUPUY
zLS+DWMJeUSjYL453f@HpRGU6Dv)rw+-c6xB>(=p4U%}<qNSa;?l%-Qbgo6H=D;<Ac
z37*~61%G;Rk9zKyD{WraYx|9VK1aeUEagvgL(r)WImH;+hu8m^8fk3iQO!R?vp|AU
z#xX(31o5DGMHtXZ9zytcGdR#e9xm`5)!T;H26%a+**oa@WGMf=BFxg~`r&GAS7$4y
z^2|*EV<Mt4fFFPU)O>_p>z^I@Ow9`nkUG21?cMIh9}hN?R-d)*6%pr6d@mcb*ixr7
z)>Lo<&2F}~>WT1ybm^9UO{6P9;m+fU^06_$o9gBWL9_}EMZFD=rLJ~&e?fhDnJNBI
zKM=-WR6g7HY5tHf=V~6~QIQ~rakNv<yXlZw2~R->csamU8m28YE=z8+G7K=h%)l6k
zmCpiDInKL6*e#)#Pt;ANmjf`8h-nEt&d}(SBZMI_A{BI#ck-_V7nx)K9_D9K<wX2&
ztb`!H<{xu+%$XOK7#l;ig9boZT>-p@?Zh81#b@{wS?wCcJ%og)8RF*-0z+~)6f#T`
zWqF7_CBcnn=S-1QykC*F0YTsKMVG49BuKQBH%WuDkEy%E?*x&tt%0m>>5^HCOq|ux
zuvFB)JPR-<R9*qK#QvOEy<g2y->W|%$24eEC^AtG3Gp4qdK%pjRijF5Sg3X}uaKEE
z-L5p5aVR!NTM8T`4|2QA@hXiLXRcJveWZ%YeFfV%mO5q#($TJ`*U>hicS+CMj%Ip#
zivoL;dd*araeJK9EA<(tihD50FHWbITBgF9<S1F59NBB@-km2xgxh^jmVKRZHCzb@
zldh9O@Tx{jgxfxH#W4wy6M0z_-aTt2G@3|Yp>E<33A+eMr2;cgI3Gg6<-2o|_g9|>
zv5}i93<J?21Jwu60De+%9&8yD$M-==IBwP;d{BC=F44-LGWWplT_ESz4ovU#BL|G}
zcmR56^)_SYTt?AM`r&<V0_9alA|EwbVwM(BT3!TE$hsCu`Gm;Rg~8}C&9f4B^0>2(
zYf<!;V`YfO6gj*4fZkUAr?-F@aA9;z9|jbBgt=33h^ojL5J7|FUPfqa-W2^EN+IwG
zyucPVo58FAeHt$Nq*Z0Lz|^v35)f<vIMOX%n{Czoseg>TE9?4#nQhP@a|zm#9FST2
z!y+p3B;p>KkUzH!K;GkBW}bWssz)9b>Ulg^)EDca;jDl+q=243BddS$hY^fC6lbpM
z(q_bo4V8~e<Zs<b7cyyni#U}G!!KTEURDqc{bMpk?5mA1@g$Y$Wx|caEJ@!TvSwmW
zqw6$6_V9}M--vO-=^ZBsJr^(lCyvX*KI4W@0G*bBPEKAYRH$kkkjj{cPznS-d|ZUy
zJ?>VeA?0LFD6ZtKcmOH^75#q$E<Zzyhezvuj?3@d%!~X4ws-!zRzAK#UFhVU?-K@7
z2G)}Z&^k%kMrWcDJ1~9s%Y7B4b*Q5pIQJtR#E8_m=dCI`iz(ByzI1@)w#x&yV;Dmb
zBR*c4ZqzdV6%u1E<qjcp>o%a&qvE8Zsqg=$p}u^|>DSWUP5i{6)LAYF4E2DfGZuMJ
zMwxxmkxQf}Q$V3&2w|$`9_SQS^2NVbTHh;atB>=A%!}k-f4*i$X8m}Ni^ppZXk<vI
zv4b?e{$fL=4CN|)0GXH{ZZE#0qrw?r2yJZjI_c{zf+uDL9gwdN)`!v>5_oYF>Gq(&
z0wy{LjJOu}69}~#UFPc;$7ka+=gl(FZCy4xEsk);+he>Nnl>hb5Ud-lj!CNicgd^2
z_Qgr_-&S7*#nLAI7r()P$`x~fy)+y=W~6aNh_humoZr7MWGSWJPLk}$#<b0qDKzby
zfsF}6hSD25(h&SI*1V<_%OA=AVAytldFr(qeYf$b2GB9jc!!oXb+YO|A>w_1n%(@?
z3FnHf1lbxKJbQ9c&i<$(wd{tUTX6DAKs@cXIOBv~!9i{wD@*|kwfX~sjKASrNFGvN
zrFc=!0Bb^OhR2f`%hrp2ibv#KUxl)Np1aixD9{^o=)*U%n%rTHX?FSWL^UGpHpY@7
z<FJKJ-x}kSNUZ~J=(sQmm|=Hgx6NDTIwxW%&M+qtaTFkwf2nmoB#ib#d`3q#i8x4D
z8SO1)?2${?@9$aGnmIq+`mJ>74U}KoIRwxI#>)Pn4($A`nw1%-D}`sGRZD8Z#lF$6
zOeA5)+W2qvA%m^|$WluUU-O+KtMqd;Pd58?qZj})MbxYGO<{z9U&t4D{S2G>e+J9K
ztFZ?}ya>SVOLp9hpW)}G%kTrg*KXXXsLkGdgHb+R-ZXqdkdQC0_)`?6mqo8(EU#d(
zy;u&aVPe6C=YgCRPV!mJ6R6kdY*`e+VGM~`VtC>{k27!9vAZT)x2~AiX5|m1Rq}_=
z;A9LX^nd$l-9&2%4s~p5r6ad-siV`HtxKF}l&xGSYJmP=z!?Mlwmwef$EQq~7;#OE
z)U5eS6dB~~1pkj#9(}T3j!((8Uf%!W49FfUAozijoxInUE7z`~U3Y^}xc3xp){#9D
z<^Tz2xw}@o@fdUZ@hnW#dX6gDOj4R8dV}Dw`u!h@*K)-NrxT8%2`T}EvOImNF_N1S
zy?uo6_ZS>Qga4Xme3j#aX+1qdFFE{NT0Wfusa$^;eL5xGE_66!5_N8!Z~jCAH2=${
z*goHjl|z|kbmIE{cl-PloSTtD+2=CDm~ZHRgXJ8~1(g4W=1c3=2eF#3tah7ho`zm4
z05P&?nyqq$nC?iJ-nK_iBo=u5l#|Ka3H7{UZ&O`~t-=triw=SE7ynzMAE{Mv-{7E_
zViZtA(0^wD{iCCcg@c{54Ro@U5p1QZq_XlEGtdBAQ9@nT?(zLO0#)q55G8_Ug~Xnu
zR-^1~hp|cy&52iogG@o?-^AD8Jb^;@&Ea5jEicDlze6%>?u$-eE};bQ`T6@(bED0J
zKYtdc?%9*<<$2LCBzVx9CA4<AoGdF)iT9-BV}&@WMK}bKMAdQBHB9zIg~33D9XqBR
zD>YV|q-qg*-{yQ;|0=KIgI6~z0DKTtajw2Oms3<u6JfE3d!7x4=<@msnH0k~j9dW|
zZtZ80d$e)`WiYJI%5C4$o_@ZG+NLyWNO7P{Rii0d%$QNbSHy)#lwN8boUCoV3)k>L
zn{C%{P`duw!(F@*P)lFy11|Z&x`E2<=$Ln38>UR~z6~za(3r;45k<v>QK_^QTX%!s
zNzoIFFH8|Y>YVrUL5#mgA-Jh>j7)n<Zx96$&fzU>)5}iVM4%_@^GSwEIBA2g-;43*
z*)i7u*xc8jo2z8&=8t7qo|B-rsGw)b8UXnu`RgE4u!(J8yIJi(5m3~aYsADcfZ!GG
zzqa7p=sg<o&$HnFk3_PIDT7^R*l|Uy*ErkWh5p!iw%95SEqgx*(#2u}$;yHXwBY|t
z$vJDd#N!Akn_xau{*4y#!iM@tZOc&Rc)@cg)>`V_KjiqI*LA-=T;uiNRB;BZZ)~88
z`C%p8%hIev2rxS12@doqsrjgMg3{<L-IS7j<nG-26~xHYI&`%4j=)ZUR_2Nwyo&s*
zJ_0X{=sXdnBCoNzt%@LBIlEx@{+A&hsZt3nyF&Gtc)9fXv28Q&PU0uzxK5QDQ^0tT
z@ll2`vK+vydWK?7I0;u%diXOvNZ*YxV6uZ=wi~x}imLLs*Slf$SL&G~13D2Praj;n
z{D>A&N8A?%Ui5vSHh7!iC^ltF&HqG~;=16=h0{ygy^@HxixUb1XYcR36SB}}o3nxu
z_IpEmGh_CK<+sUh@2zbK9MqO!S5cao=8LSQg0Zv4?ju%ww^mvc0WU$q@!oo#2bv24
z+?c}14L2vlDn%Y0!t*z=$*a!`*|uAVu&NO!z_arim$=btpUPR5XGCG0U3YU`v>yMr
z^zmTdcEa<e${0lJ(<4$dGj^YwOU!gy-6u%trv^j}K^#M&{&D1UGEa2nS)OMS>!APX
zYF>^Q-TP11;{VgtMqC}7>B^2gN-3KYl33gS-p%f!X<_Hr?`rG8{jb9jmuQA9U;BeG
zHj6Pk(UB5c6zwX%SNi*Py*)gk^?+729$bAN-EUd*RKN7{CM4`Q65a1qF*-QWACA&m
zrT)B(M}yih{2r!Tiv5Y&O&=H_OtaHUz96Npo_k0eN|!*s2mLe!Zkuv>^E8Xa43ZwH
zO<I&Nur7C^JWb9`H||S7lnGT!zTpho2vN69_#Qs?kjYy$`Q0!HKi(a7WFX@;&MNY~
zax>I058AZznYGrRJ+`*GmZzMi6yliFmGMge6^j?|PN%ARns!Eg$ufpcLc#1Ns!1@1
zvC7N8M$mRgnixwEtX{ypBS^n`k@t2cCh#_6L6WtQb8E~*Vu+Rr)YsKZRX~hzLG*BE
zaeU#LPo?RLm(Wzltk79Jd1Y$|6<Tz#Ab3VOQL!oqnu8!3Ev#T)CDpRjo9fjjgw)N!
zErQj*AF93?w7h)pV7lh%{nq{Sh43S}4qRhV$tBKz2Q3F#YC0?apCV-cr-&g)Ta?xA
zc-})no|s-UajuvdSqa3zR2cl82_%nvGFwn23tfy0+-dPTJzZ8#!t6MWix9bj!?+yg
zn$~Be?akwbUV?z=Y{qc>aWz1)wf1K1RtqS;qyQMy@H@B805vQ%wfSJB?m&&=^m4i*
z<RSh9mC|UV(AH!)v4!3^(;(w}isKaD1$r>YVH`zTTFbFtNFkAI`Khe4e^CdGZw;O0
zqkQe2|NG_y6D%h(|EZNf&77_!NU%0y={^E=*gKGQ=)LdKPM3zUlM@otH2X07Awv8o
zY8Y7a1^&Yy%b%m{mNQ5sWNMTIq96Wtr>a(hL>Qi&F(ckgKkyvM0IH<_<ah`2Jtc4H
zzJ5Z$n<rF%>}v~Fv-GqDa<PYPnU!!je;%S69V23AFX{jR^q$cSK5eU!uzK5~_gTvx
z=xC#ygr;Z7l0l}tE!ut;$3u!G=k}WM8nnSovw+>pig=3*ZMOx!%cYY)SKzo7ECyem
z9Mj3C)tCYM?C9YIlt1?zTJXNOo&oVxu&uXKJs7i+j8p*Qvu2PAnY}b`KStdpi`trk
ztAO}T8eOC%x)mu+4ps8sYZ=vYJp16SVWEEgQyFKSfWQ@O5id6GfL`|2<}hMXLPszS
zgK><z_7D{#_0cM38i3?q!q=1kU*<f=E!5q~OGR+n-!$_a1@<av6z8(-tT0T>NWOoR
zBRyKeUPevpqKKShD|MZ`R;~#PdNMB3LWjqFKNvH9k+;(`;-pyXM55?qaji#nl~K8m
z_MifoM*W*X9CQiXAOH{cZcP0;Bn10E1)T@62Um>et2ci!J2$5-_HPy(AGif+BJpJ^
ziHWynC_%-NlrFY+(f7HyVvbDIM$5ci_i3?22ZkF>Y8RPBhgx-7k3M2>6m5R24C|~I
z&RPh9xpMGzhN4b<n_$TKj6;x$6eSY#J<6z8-<<eXD{YGtPeh(eK>ii*ryWaN^d(`0
zTOADlU)g`1p+SVMNLztd)c+;XjXox(VHQwqzu>FROvf0`s&|NEv26}(TAe;@=FpZq
zaVs6mp>W0rM3Qg*6x5f_bPJd!6dQGmh?&v0rpBNfS$DW-{4L7#_~-eA@7<2BsZV=X
zo<jFL%Bl|~iN=_%v|5Y<#Gpc-_nu`zPSa3B)eyzNRYkS_S7#Vl4Sx`|M*FHkF=-_)
z(qT@$sP8*6(;+2_2GrUtq?WVOz@%!47omewB^Cj*+)_Y7c8o#IQ%W|Wg+5PNnR*6U
zqsrE$&lE5UYNkcwa9IkWJQ`K6q;9o~nXTNNt4ypA@SdwPa*k(~yPsQD9A`rvY6hIQ
zbB3Wq3J(Ip%Spgn<@)Zq04$gB8>w){3aATmLZOQrs>uzDkXOD=IiX;Ue*B(^4RF%H
zeaZ^*MWn4tBDj(wj114r(`)P96EHq4th-;tWiHhkp2rDlrklX}I@ib-nel0slFoQO
zOeTc<hn5|jpM}{BL^>;Rh7sMIebO`1%u)=GlEj+7HU;c|Nj>2j)J-kpR)s3#+9AiB
zd$hAk6;3pu9(GCR#)#>aCGPYq%r&i02$<m+xZvsp>0L9=7Al<uiMgWi00$%C&<=p;
z&DJ&!X-+7DOQ<NGx053X=x&OTuveGw;P#7I963O-cntp%n;x3upG6$bSZdIscXKZz
zE26w-ESTe#dx<IxxgOyxd~k92x!}?e7=&}7EIUABcs@v7J^4#tE1Z2`vYo|Zfx#?M
zXxRQAUwIpnrd%x;d2(+0wHKQb5LXCF5FM|~=&cF?)N_xB4Bl|JpPd+>IGY<wi6MrB
zwuAQco(edmX)&f$!qgzbDk@I5RR!7e{<pl6;0q^D9rH)D+xQ^qUHFjiVuuZ(dNLoE
zk610d0D=8c-t>dlUO5%eH&M!ZWD&6^NBAj0Y<LF!W8hneCvn#&X!jeuMxFNL9R6gV
ziYL0E$zuPwu-dNyWbfxdM|kfI7dtb9QzzTE{}sJ=aTD#{Kb((+@c&OeMaT<*ofYbx
z;1%phK@uKkWduA*1(PK@!{WoV6$&w4_0RR0*51~cnd9Q6WRU~~+iD70xHZv5yb^q9
zSl{&Z2jJC&)zq%KFf=>9ZDcPg@r@8Y&-}e!aq0S(`}NuQ({;aigCPnq75U9cBH&Y7
ze)W0aD>muAepOKgm7uPg3Dz7G%)nEqTUm_&^^3(>+eEI;$ia`m<pPp!PnmJ$ax-Ni
zriP82FsWaes>>m0QHEkTt^<h|Kl{fGlDwG;CQ*b4lzdo-V$nC8<LA?4QJ%$e&M1jF
zj4MPd95jth&|E!zZ*d9v(}1-nH2c9uw}SWz(o)!u-C;JXs?Old?KmR?ZVapLr)~kt
z-!pxZ2T4qAqb;DH9;hiuAcN#25{6pE|B@nn@LY@;@6X}9ZRu6|3F2X?Musq<a_n+z
zJMV<b3!3d+o(r5xln5wq$mvr`E9}w25J;Qi>=cx^JsBC68#H(3zc~Z$E9I)oSrF$3
zUClHXhMBZ|^1ikm3nL$Z@v|JRhud*IhOvx!6X<(YSX(9LG#yYuZeB{=7-MyPF;?_8
zy2i3iVKG2q!=JHN>~!#Bl{cwa6-yB@b<;8LSj}`f9pw7#x3yTD>C=>1S@H)~(n_K4
z2-yr{2?|1b#lS`qG@+823j;&UE5|2+EdU4nVw5=m>o_gj#K>>(*t=xI7{R)lJhLU{
z4IO6!x@1f$aDVIE@1a0lraN9!(j~_uGlks)!&davUFRNYHflp<|ENwAxsp~4Hun$Q
z$w>@YzXp#VX~)ZP8`_b_sTg(Gt7?oXJW%^Pf0UW%YM+OGjKS}X`yO~{7WH6nX8S6Z
ztl!5AnM2Lo*_}ZLvo%?iV;D2z>#qdpMx*xY2*GGlRzmHCom`VedAoR=(A1nO)Y>;5
zCK-~a;#g5yDgf7_phlkM@)C8s!xOu)N2UnQhif-v5kL$*t=X}L9EyBRq$V(sI{90>
z=ghTPGswRVbTW@dS2H|)QYTY&I$ljbpNPTc_T|FEJkSW7MV!JM4I(ksRqQ8)V5>}v
z2Sf^Z9_v;dKSp_orZm09jb8;C(vzFFJgoYuWRc|Tt_&3k({wPKiD|*m!<X$9n>+<d
zfwbu(US!ljdxEAm@8L<A)Zh;j$1@#3i1Hba7~+=OxccBV?z(KQ^KiX-VG5F8{s<bz
zFTJ%sXJxQmw1gpd<`@dDaxW+0z@mAqX^!9bM3;Rg{vJ$y`jrFPf<taTjgOUiGo`tq
zU8^ynJ_vtmGeduy1>za$(l*!gNRo{xtmqjy1=kGzFkTH=Nc>EL@1Um0BiN1)wBO$i
z6rG={bRcT|%A3s3xh!Bw?=L&_-X+6}L9i~xRj2}-)7fsoq0|;;PS%mcn%_#oV#kAp
zGw^23c8<Fs8D%<c4(Nre;Oy43eq_*f8CVI6Y}O$7=4|A%Bh)_#+dgnF@9B?PZV+FK
z@D@b4V?}Hxb;mfT&3=fp%aQ_W<BpQFv8~7WceKD?ShRTE&*xP)Rp+|r%gOH<b#|`R
zpMk-?U*1{fo}21TS!!|lrVyo(<X*MXXY2ANSehq$FQcbg%rc;PQLx;F?K=D0V>_0~
ze}v9(p};6HM0+qF5^^>BBEI3<e#!JG>d=2<KmW5$FDhY04Ol??L%ZUPasCO+Pj|h!
zL*;#e7cVT$<F2gg5E8<=1=Y;hH6lJKfJK6P>DW&O#|(;wg}?3?uO=w+{*)+^l_-gE
zSw8GV=4_%U4*OU^hibDV38{Qb7P#Y8zh@BM9pEM_o2FuFc2LWrW2jRRB<+IE)G=Vx
zuu?cp2-`hgqlsn|$nx@I%TC!`>bX^G00_oKboOGGXLgyLKXoo$^@L7v;GWqfUFw3<
zekKMWo0LR;TaFY}T<F_}BWybmURg`OY9TP`)1kkA_VfNgEb>t4!O$3MU@pqcw!0w0
zA}SnJ6Lb597|P5W8$OsEHTku2Kw<?~p>9y4V=hx*K%iSn!#LW9W#~OiWf^dXEP$^2
zaok=UyGwy3GRp)bm6Gqr>8-4h@3=2`Eto2|JE6Sufh?%U6;ut1v1d@#EfcQP2chCt
z+mB{Bk5~()7G>wM3KYf7Xh?LGbwg1uWLotmc_}Z_o;XOUDyfU?{9atAT$={v82^w9
z(MW$gINHt4xB3{bdbhRR%T}L?M<kc`={1@bX$@c}pX`50EN?1djSoIxW|Rhd4u^_G
z>cK?!zkLK3(e>zKyei(yq%Nsijm~LV|9mll-XHavFcc$teX7v);H>=oN-+E_Q{c|!
zp<Ol>JV~-9AH}jxf6IF!PxrB9is{_9s@PYth^`pb%DkwghLdAyDREz(csf9)HcVRq
z+2Vn~>{(S&_;bq_qA{v7XbU?yR7;~JrLfo;g$Lkm#ufO1P`QW_`zWW+4+7xzQZnO$
z5&GyJs4-VGb5MEDBc5=zxZh9xEVoY(|2yRv&!T7LAlIs@tw+4n?v1T8M>;hBv}2n)
zcqi+>M*U@uY>4N3eDSAH2Rg@dsl!1py>kO39GMP#qOHipL~*cCac2_vH^6x@xmO|E
zkWeyvl@P$2Iy*mCgVF+b{&|FY*5Ygi82<omgvyGTqv|QTz=sN}TZ6>37i)9YW#Fp&
z?TJTQW+7U)xCE*`Nsx^yaiJ0KSW}}jc-ub)8Z8x(|K7G>`&l<bpFP+cS)jO`N_V-8
zyz*&criMW3Tge5_hcWP_V8=SvD0*^&JQ8c;z^ZkOK$H-b5$|gm$@;EfT;;o$;h7dL
z>{Y&~W=q#^4Gf{}aJ%6kLXsmv6cr=Hi*uB`V26;dr<aSR^xo%$9+~>4C$WrPnHO>g
zg1@A%DvIWPDtXzll39kY6#%j;aN7grYJP9AlJgs3FnC?crv$wC7S4_Z?<_s0j;MmE
z75yQGul2=bY%`l__1X3jxju2$Ws%hNv75ywfAqjgFO7wFsFDOW^)q2%VIF~WhwEW0
z45z^+r+}sJ{q+>X-w(}OiD(!*&cy4X&yM`!L0Fe+_RUfs@=J{AH#K~gArqT=#DcGE
z<spTPWq%oqdho$J0xh~6Q=^v}j=vBe*gL+|ud>!FwY(h&+&811rVCVoOuK)Z<-$EX
zp`TzcUQC256@YWZ*GkE@P_et4D@qpM92fWA6c$MV=^qTu7&g)U?O~-fUR&xFqNiY1
zRd=|zUs_rmFZhKI|H}dcKhy%Okl(#y#QuMi81zsY56Y@757xBQqDNkd+XhLQhp2BB
zBF^aJ<Yyh7pM5zBvii~t`R`{~RErSl{}-#Xjol7q<MNou@wjO9-e0+&FnPZ^6y4Ll
zv-4#qZasZI63$>__D676wLu|yYo6jNJNw^B+Ce;DYK!f$!dNs1*?D^97u^jKS++7S
z5qE%zG#HY-SMUn^_yru=T6v`)CM%K<>_Z>tPe|js`c<|y7<QPU`KuA=GRt?21}m$3
z)egm^_v+KsYdG<vt|ej|@=R$E;WS|)lq&hvDYqCWcNQVq*yoxejj)Sgjj^Yjl?NcX
zDFDlAtM@eY^eKCIx3Bn3tvI+`G-P!;moJ^4fysp(P&5s?@p7p>?qol&)C=>uLWkg5
zmzNcSAG_sL)E9or;i+O}tY^70@h7+=bG1;YDlX{<4zF_?{)K5B&?^tKZ6<$SD%@>F
zY0cl2H7)%zKeDX%Eo7`ky^mzS)s;842cP{_;dzFuyd~Npb4u!bwkkhf8-^C2e3`<?
zuCMcKb49Z{nlgrGOONk{-i&aq@>q8>MuPhgiv0VxHxvrN9_`rJv&GX0fWz-L-Jg^B
zrTsm>)-~j0F1sV=^V?UUi{L2cp%YwpvHwwLaSsCIrGI#({{QfbgDxMqR1Z0TcrO*~
z;`z(A$}o+TN+QHHSvsC2`@?YICZ>s8&hY;SlR#|0PKaZIauCMS*cOpAMn@6@g@rZ+
z+GT--(uT6#mL8^*mMf7BE`(AVj?zLY-2$aI%TjtREu}5AWdGlcWLvfz(%<HnWJcfj
z-uvEteDlov4?jvo%cNeW+poNG>wn72tGczwUOgGD3RXpWs%onuMxs9!*D^698AupW
z9qTDQu4`!>n|)e35b4t+d(+uOx+>VC#nXCiRex_Fq4fu1f`;C`>g;IuS%6KgEa3NK
z<8dsc`?SDP0g~*EC3QU&OZH-QpPowNEUd4rJF9MGAgb@H`mjRGq;?wFRDVQY7mMpm
z3yoB7eQ!#O#`XIBDXqU>Pt~tCe{Q#awQI4YOm?Q3muUO6`nZ<tt>4^zi5|(w<YclZ
zu+KExzcVzp-<c6itggO*X~x)Ub6iWOWm1_uy?SO#%E*8SSZpyY8dd&ki$+?HWK*%s
zK>b9R)oyarG?mI|I@A0U!+**&lW7_bYKF2biJ4BDbi~*$h?kQ`rCC(LG-oO(nPxMU
zfo#Z#n8t)+3Ph87roL-y2!!U4SEW<vR3`WM9hpvJa%7CIt!<Og+JH8nPFHCj`HIon
zWU69v;ml6>NCIM16i~-&+f55;kxC2bL$FE@jH{5p$Z8gxOiP%Y`hTTa_!v{AKQz&-
ztE+dosg?pN)leO5WpNTS>IKdEEn21zMm&?r28Q52{$e2tGL44^Ys=^?m6p=kOy!gJ
zWm*oFGKS@mqj~{|SONA*T2)3XC|J--en+NrnPlNhAmXMqmiXs^*154{EVE{Uc%xqF
zrbcQ~sezg;wQkW;dVezGrdC0qf!0|>JG6xErVZ8_?B(25cZrr-sL&=jKwW>zKyYMY
zdRn1&@Rid0<C+MnR2o`YS>oIhoRl)+X4)b&e?HUVlOtk^(<zwTW;whij!=-BE}&In
zXFHQ`T%bqdF}QRhp(UdsqzoXW^^{Cu7}aml+o{qH+6C)t@qf6nzbm;nY3xsK?iUUZ
zuVHw|MYP8Sn2YhNSD|j$%_1Jif>xl<NWlN1(3;V2Mahbi86q?_SJr9JNDuW2Zu(9r
zg9R<qC9s8->dhvgf^7r+@TXa!2`LC9<CJjGUIJTusY3k-%1g5`J)=+xD=iAagk|Z0
zbVg4EgrODM$A2{4B$Lg=;`O0e8rkmvgzD5YJ`2#7_lN<X(c{bX>AsB@wEO&eU2mN)
z(2^JsyA6qfeOf%LSJx?Y8BU1m=}0P;*H3vVXSjksEcm>#5Xa`}jj5D2fEfH2Xje-M
zUYHgYX}1u_p<<Rn@EZDvN>|fIC+pI5g6KGn%JeZPZ-0!!1})tOab>y=S>3W~x@o{-
z6^;@rhHTgRaoor06T(UUbrK<L%k*y$z*GoaYL0(Ax6MX8ZY6UNxruI8=|=i@Ca=FV
zjODYps&qYlQq-c*B(C!ypHk^s`ZUw>4+@5bO?r=!vckDD+nwK+>2{{|{u4N@g}r(r
z#3beB`G2`XrO(iR6q2H8yS9v;(z-=*`%fk%CVpj%l#pt?g4*)yP|xS-&NBKOeW5_5
zXkVr;A)BGS=+F;j%<x9&4$@s^^kw=A>O|69F0Lne?{U*t=^g?1HKy7R)R*<>%xD>K
zelPqrp$&BF_?^mZ&U<*tWDIuhrw3HJj~--_0)GL8jxYs2@VLev2$;`DG7X6UI9Z)P
zq|z`w46OtLJ1=V3U8B%9@FSsRP+Ze)dQ@;zLq|~>(%J5G-n}dRZ6&kyH|cQ!{Vil(
zBUvQvj*~0_A1JCtaGZW|?6>KdP}!4A%l>(MnVv>A%d;!|qA>*t&-9-JFU4GZhn`jG
z8GrgNsQJ%JSLgNFP`5;(=b+M9GO8cg+ygIz^4i?=eR@IY>IcG?+on?I4+Y47p-DB8
zjrlar)KtoI{#kBcqL&4?ub@Df+zMt*USCD_T8O$J$~oMrC6*TP7j@H5trGV$r0P6I
zV7EZ{MWH`5`DrX*wx&`d;C`jjYoc_PMSqNB290QXlRn_4*F{5hBmEE4DHBC$%EsbR
zQGb7p;)4MAjY@Bd*2F3L?<8typrrUykb$JXr#}c1|BL*QF|18D{ZTYBZ_=M&Ec6IS
ziv{(%>CbeR(9Aog)}hA!xSm1p@K?*ce*-6R%odqGGk?I4@6q3dmHq)4jbw+B?|%#2
zbX;ioJ_tcGO*#d0v?il&mPAi+AKQvsQnPf*?8tX6qfOPsf-ttT+RZX6Dm&RF6beP3
zdotcJDI1Kn7wkq=;Au=BIyoGfXCNVjCKTj+fxU@mxp*d*7aHec0GTUPt`xbN8x%fe
zikv87g)u<XFa`)x7(;`tKRdCl%zvlpDa(FQF6a4-(c(tNXi-s0t1@H6aN>~0cpQaf
zd<7Mi9GR0B@*<TNFs6&?{V6>S&l&9pCl-HEaNX?ZY8MoXaYHGDf}733;(88<{E%)<
z^k)X#To3=_O2$lKPsc9P-MkDAhJ~{x<=xTJw2aRY5SSZIA6Gij5cFzsGk@S)4@C65
zwN^6CwOI9`5c(3?cqRrH_gSq+ox(wtSBZc-Jr5N%^t3N&WB|TT_i4!i3lxwI=*p)Y
zn7fb%HlXhf8OGjhzswj!=Crh~YwQYb+p~UaV@s%YPgiH_);$|Gx3{{v5v?7s<)+cb
zxlT0Bb!OwtE!K>gx6c4v^M9mL0F=It*NfQL0J0O$RCpt746=H1pPNG#AZC|Y`SZt(
zG`yK<fxmbRn_~3f^D$uXR;DE-6l2Brn44`tAjOgDc$;+zFmZqKsnw<WO1^-%tK7vG
zq9rVGYRKsA6^HB)rDuo5<t~+X@@_Ht9zYzz5%-9s*T9%xs!EW^Uw?kI9?&CM(MEAM
zd|xD+#U-^eF-KWv>MBPV_0I|S?}?$t7G<RgM>U%;*_39bCGO*x3+R|<=9WNe!8jH-
zw5ZJS(k@wws?6w1rejjyZ>08aizReJBo%IRb3b3|VuR6Uo&sL?L5j(isqs%CYe@@b
zIID7kF*hyqmy+7D(SPa^xNVm54hVF3{;4I9+mh)F22+_YFP>ux`{F)8l;uRX>1-cH
zXqPnGsFRr|UZwJtjG=1x2^l_tF-mS0@sdC38kMi$kDw8W#zceJowZuV=@agQ_#l5w
znB`g+sb1mhkrXh$X4<U2%FUm!iUg)BMsC71C8xC>y(<-CntwmVwah5#oA_p-U<_5$
zGDc%(b6Z=!QQ%w6YZS&HWovIaN8wMw1B-9N+Vyl=>(yIgy}BrAhpc2}8YL-i*_KY7
ztV+`WKcC?{RKA@t3pu*BtqZJFSd2d)+cc07-Z#4x&7Dnd{yg6)lz@`z%=Sl-`9Z<I
zvq@B}FY%WJ&40l{%|nmhSKk!tZA)f!GeKYiaz;%KM5ZvweU<MKIGA~y{x*{g>~*io
zck_Lshk9JRJs=t><bs*w&DW%*QMrzYtZr})w%D?D^TyWZZ3+)hfHZHb_+blj1dx}-
z`Yr57thGn2H4E-BYweqe1pPoHo=wO0nOC=skqA`Su74$=MZ|5xc;Aabz*JI;TUTsx
zBeECjs&AbV9s2(kKjDH?ew*pEsRM4|QkgMxq)r%2#hz9wB=Xcs?v>1jmKB~>`6+(J
z@(S}J2Q{Q<aLwQD@U#3~m1O=Nj9M3u>{a-ASTnIViecW(FIagWQ%G41y?zS)gpooM
z@<VGKqkqS#V-&Y$lNsd6am~a_n3S50Y&>c<2$7TykMs4LH*UUYfts(!Ncn`?eZl}f
zg)wx@0N0J(X(OJ^=$2()HLn)=Cn~=zx(_9(B@L04%{F_Zn}5!~5Ec5D4ibN6G_AD}
zzxY^T_JF##qM8~B%aZ1OC}X^kQu`JDwaRaZnt!YcRrP7fq>eIihJW1UY{Xhkn>NdX
zKy|<6-wD*;GtE08sLYry<V<QH1Sv(jnToX72i-sNpHzN}-)36<|L{|pF{8{fZx)*4
zMp{2#T-wD4#<BxWH#fupXEv>W<-e)?7k;;B>e$u?v!QhU9jPK6*Y$o8{Tl`N`+QvG
ze}71rVC)fis9TZ<>EJ2JR`80F^2rkB7dihm$1Ta2bR?&wz>e`)w<4)1{3SfS$uKfV
z3R=JT!eY+i7+IIfl3SIgiR|KvBWH*s;OEuF5tq~wLOB^xP<BqZESF|TGvQX!tiq(3
z|Mp-8M+j94YLq<UQrMUO77=nwv&Hv{1%HwlNG60+0mF+VI@+*}+Yy&?lVZ@?0xxr=
z(^TnHX`Y)}t+&&KEIW%omrQ|e2EBBKDpg3BO{Sr4h#<3UH}T5KTE&42rA5Mk7*Z<J
z;?oZt>_Dc7-BbNjpC|dHYJrZCWj-ucmv4;YS~eN!LvwER`NCd`R4Xh5%zP$V^nU>j
zdOkNvbyB_117;mhiTiL_TBcy&Grvl->zO_SlCCX5dFLd`<IhUzJ>q7x-lBj*&ykj^
zR3@z`y0<8XlBHEhlCk7IV=ofWsuF|d)ECS}qnWf?I#-o~5=JFQM8u+7I!^>dg|wEb
zbu4wp#rHGayeYTT>MN+(x3O`nFMpOSERQdpzQv2ui|Z5#Q<D=p76p9*uO;+Tg4e>d
zB(+GbXda|>C<gg{Slr$#(gyLVu*4PBxH}eUBf0=-6H~npz6ZXezWrJ(10XZ|kJf6R
zW@a#7#xRRP5Y#bqIYq%2?-;boe6h4IYkgu-r1Np~n3gcG_CXgI%zf5XB!7hO#H@AV
z%SD8PvA>W55ky@mG13K0wfXAm8yoek3MJG!Hujn$5)Q(6wWb-l4ogu?jj2Q|srw?r
z-TG0$OfmDx%(qcX`Fc`D!WS{3dN*V%SZas3$vFXQy98^y3oT~8Yv>$EX0!uiRae?m
z_}pvK=rBy5Z_#_!8QEmix_@_*w8E8(2{R5kf^056;GzbLOPr2uqFYaG6Fkrv<RIBB
zuZTHP@Q)>($n_51%7~QN<&GT9$WdjE=H}>(a41KM%d2x#e@K3{W|+=-h*mR&2C01e
z2sMP;YjU)9h+1kxOKJ+g*W=&D@=$q4j<ya_SB@^U{)R?qM|jt+j(;KAZ9hVq;}NO}
zhi&DNPJ6lDnWI>F%@HyRtCwOmEmpS|R<c;xFIK|k0{|?){0Ln&Ob18k+HlZ5Lf3}_
zhv|kK-7-YC4bh<y`b;?J2zc#7^x22#i%dbM*ZC0L&2)&5c^%^Y>r9V<bWqOG;Y0i-
zuUwuJV$!S;8V0UF9e)`-{w&rX$<bqn$O|+X%6Y;o@5#|qIr?7EF?jqQ>_2br*NOd^
z4LN#oxd5yL=#MPWN{9Vo^X-Wo{a7IF2hvYWB%eUCkAZq+=NQ<Q2k7Uw(;NW*(gb!6
zQrQT-79OJiJ{kZTAcG2|_9{7g;}BH{TH?LH+sr_ocF-lDTz@9iN4$;^dNb^G?J585
z5dAJkZ{_H1vG){&&e2~Ek*6T2#m4(jd-3Dlpu@3TsZ_jB*ZX(Af5hvkRO}ZENh=jo
z)SzOLRf@=3%)A=}h*t@Mz~J$tu#;BI?7QNW$dr8L0h$Cxj#<1f_u9b}+lN{07@-@%
z`4E?tXLCILFn@b;JbTRA)K+9`56{W*-0*Vc4w@zCd3auq&)7bA`~|NmidVK(Dnc6B
zesSK^=JtbLdpIDDTwERzdgAHZ4|9!Js5?ZRUN~Sw&`}=2dN}Abd01+{M)-vF<81pE
z=oI00_+`uS3NVAH86<s#SA{*idWg?GYQ3%zuMcqZc7G4I4s#pMI!2Wuv>=iLI9?~@
zr+|ky4Rgm7yEDuc2dIe941~qc8V_$7;?7|XLk6+nbrh}e&Tt20EWZ@dRFDoYbwhkn
zj<rL!Cr3CE4%d$Gg76SWhq(7J?`<F91i%BehdB*P4D<dyLwwl?UlDFc<c;u^AnB+X
z;;REg{C`ovhdU1Qb%H*v1;ZZxc#dxfFPHBSjy>J$th974Z0F${3wtVLk_Ty;*J-Pi
zP0IwrAT!Lj<oLFr;^9vZ^Jl|Bmlv;}%kdX;eAfuy4L`CkS91LIN(DUKKXYM@A2i)d
z#?QlWE^#`&wa0R^pe*dC9OfYqf@;k?pn4ra*MA_~1S=1T73q^TL07=5cwK^0KoC02
z-x#GOB0~^^P6ajBVLloTss(B;uOlR+R13rw*Lqzw;|M)oOep{A9Hn`fkB#tC;gA=S
zdhNAP>34GcoSB8g?IKPt%!iLD-$s+f_eZg@9q!2Si?`F#fUqY`!{bM0O7V^G%VB|A
zyMM>SKNg|KKP}+>>?n6|5MlPK3Vto&;nxppD;yk@z4DXPm0z9hxb+U&Fv4$y&G>q=
z799L0$A2&#>CfSgCuu$+9W>s<-&yq3!C{F9N!{d?I|g|+Qd9@*d;GplgY5Fk$LOV+
zoMealKns!!80PWsJ%(}L61B!7l?j1_5P#LRrVv%NBhs{R`;aufHYb&b+mF%A+DGl5
zBemAHtbLFi++KT(wv9*?;awp>ROX~P?e<4#Uf5RKIV{c3NxmUz!LYO#Cxdz*CoRQp
zSvX|#NN06=q_eTU5-T!RmUJ?Ht=XQF8t)f+GnY5nY5>-}WLR1+R5pou?l@Y|F@KEX
zk=jh-yq=Rn9;riE*;S<SC4qvrM$x?L9*?xeN{b$8>lo}PfNKhXO#;FrZCf%VZ9h7W
z<63YWE^s_SlAVQh6B(En9i<9%4AT|2bTQ4Ph2)pI?f2S`$j?bp`>_3(`Fz&?ig-FJ
zoO7KAh@4BDOU>sBXV84Eajr9;>wlbW&OSUt&dug?oAV;`+3oBzpI18%%1wA4blzmb
z-{QPYJmn_2-F$A5JI!a8+-p8Bk*^U?^f5j7uZ}jEz0E3;XbahB2iZwS&l4jj4WRS6
z3O&!w=ymQSl~7LUE99noXd2y1)9E>yK`+ouR%sTOQ@Qjt@<<O;g>;lErGLk1wrw7r
zV)M})+amJXs_9hQa++&vrqgU&Xr8T)=G&5Vy6vOnvt37L*nU7&ws&ZO-9`)TGA**t
zpby#0X|df;etRud+s~#Y_7zlPZ=_oLg%q&wraF6s>g@;VO#2sUseO=^+3%&Z?61(-
z_IKzU`+Kw;Blil&LR#qv&{rzQnG|%i(Q3zLI@gh)2FE^H;~1dx9G|AOj(e%mSwT(C
z71Zp!jar<yQ>*i3S|_ik_3{n0L4Kav<X35<{5EujksbPy$o79wO9u!wto+&^6#xLx
zDU;zw9Fxm!Ie(jbdz(GluHDkTx^9E6ZQ4Cp*}@o%j!C7Iv2K*GVYy9i)4L_PCHH3C
zKoAuX5d;y`ZOBWcqNspfS}0>YWWZ2x3MhyU!66E$h=L+A&-s$9X_w9Q_e;+`-{ZW#
z^Zn2H_I~`}!vGeFRRY^DyKK#pORBr{&?X}ut`1a(x__(dt3y_-*Np0pX~q39D{Rns
z!iXBWZO~+oZu>($Mrf0rjM>$JZar!n_0_!*e@yT7n=HfVT6#jbYZ0wYEXnTgPDZ0N
zVE5?$1-v94G2@1jFyj##-E1Um(naG-8WuGy@rRAg)t9Oe0$RJ3OoWV8X4DXvW+ftx
zk%S(O8h?#_3B9-1NHn&@ZAXtr=PXcAATV*GzFBXK>hVb9*<BlrkfXpOFkOx&9W&Y(
zvf`m+B4Nax#%9{4cSQ{aDs9}XN9RPXkRCnX>`iMM-zvA6RwMH#2^901uxUFh&4fT%
zmP?pjNsiRIMD)<6xZyOeThl_DN_ZJ*?KUIHgnx{vz`WKxj&!7HbM8{w?{Rued(M1v
zKHsK{_q=YI88@Bf0*RW@cIV@=<{eGsG21xrTrWycT7*KBd!eD2zb1R(O@H~k7>Duv
zHPwp=n8;t#1>7~fuM9IaD5w%BpwLtNCe_Sq9eal4oj2DB1#<+(MGR-P&Ig%3t%=!<
zS$|KxI1a~an2Q>L$s;1$9nQJal4dk)Box$YsAKgCiEGni##jr|%So6Y4J@pYBF!;~
zhXwpKhc7&QZ$=e~Sb&ABZ4o)&U~N*dSU`2G^eQh-WCe9tA}~Ae369btLl<C!I4@0`
zGLiyiCAP}Ip6|uUSkAMjkh!MKQoLA^9)CJbU;;V2qRY0TNyk{NJ3U^kOnY~_K;@BB
zLcu5KLh7NAVN*uVr<{z`95sXfpBG2jJSRh&8E7bWE%>B{GjOKB@yEDH!C7Q&df^#X
zi~?{rCuAE|kAjKzt+r#t6s)1h840@A<%i5(O;$Q&tD(opg0)yzgm#=ucf4CSqkqYS
zaTdivk5I~#=1Z9K5M*uV6H??6s9*ynT`vzr2@%Tkr4k+Tr_ib40$fPP7$yLA$cwJ@
zF@`94=op)$x^0t+QAsNY$pi!4e7hp~gO=|yD=^8JTvTiC(HAamYEQ<z*u)-f1l>}t
z+hR~QoKTOz%)IHEg&6iC4vP=3mw&u4wvcSwi$vNBGQE5RoSUs^l+u{A+6s~aMMkXG
z+1g4wD8^Y2<w_nS2m7!^!)8#{7e#4=)sw`MntAKTV!<W&H0!%Gdm2*8ibO(};uU;G
z!{v+vZao|xa~v5E;#>7Oe4f``K{+tm76n(*d6<qtVYDcTDn?n`k{KsEu7ARpit$BU
zO%5GCuizR=3aYPd#umqmuV8sGuEli(j&V8gRm|pr8z_!ZD-tnC(bRl*s8vpqwi|I%
z5e5G1<n>BUA4;pLa26`6RD6?Rq?2K1yMXVAk`&xbks*~{+``Mhg<C1&>4cQEuw+aM
zaI9{}9en8DCh*S9CojIk)qh|k?#iNiCQ}rAmr&iYR<t*_v}8f?dyGqW<NFj*>JiND
ztt+j*c+}Fv&6x&7U~!(Sb1eAz1N@Nf`w?YxGJdhy+seiNNZEYIG1_<^?&pm^P8W?d
ze(p@$nWC`<TizHIL&PKuLfOp(5!FYo71^8O#e*1TDG!miS*@ofyMMclgb`k2=(1it
zOy`uHYl-(JGjNifek5D#G6v@?QSexvgOY{hCmJ5d69R?n)~@m|QSqce?a0C$8AmKd
zPiuG-dl`ogZA+V!ng6MV-S`<@5t0&arOwZb=Qw1$@pDoeidr^}{DPZ--S{QB*lZ=E
z;i|ahRCP1RRMDO2sedBSp`3kzl0HwSW)nlfuPE-e-fgT)5SGoT83RXSWBw9e*6?fm
zW=uKHUizb!^WXB#`JI4hQ1L1`M=sk|JU~Ximc<#lb8Sz;>Pxqpf8d&AIGNJn#Ty)j
z1NbA^Y}pNQ>OfTdiAp+WR>C6390IrFj;YZglitGH8r7(GvVRpWjZd7|r24M{u66B)
zs#VS$?R*!1FT&sO-ssvW<tq3m8RS^4Rv%RhdIO9Ylq(~zK_B2>8s5jh$-O=^9=7^y
z75||~QA6zLW}Lu!YOZh1J$j<uxF~7FLunBCFFxk|2PtY@W;}D|s-TC#l#yK&C~irz
zJ)LlKO7+mYLw^;8gj**rUaSg*9OH7S$E~<Y_R4E3ie%d(GzRSTN62mA)kJ|9M>46m
zNH|;^a$U_RKgla5h>5(igl^ek(~2nL5a_0}ipvA_Xf0k*E-ExJNld0{LZ;<hGrGh%
z-E8jt^9rA+<V{mY6*3YU(;bJMpRS1+vY(!7&yA~szJEO3a#mnjo-|s2#GD^3m^4?5
z*(6)cVFgP@<q73CU28=gPXL~IU8RS{cGKxmk~L4%YNAHeQ5m5Qi2AN%uj3Vd0stCq
zAov*p0NZe?j4ehe>F^DzqAL+IZGJ7<3<z}D#C%puf*qsH=##y!SSZ^IL9KS>i1szf
zxMRkQ(|@;wj9%I7h{c*{;?g%giylU}Dz{iwb(1vGK<-vlnKs!|Mb9}iTt)Rl&NZka
zkkugrMiY(ng3QseY!npaOf1jo3|r35nK+eTYh*`DHa<o;XAO1n1<mxbre>buv@IFy
zG7@V!LWE0&)bvqgQ8=-L-(vt#Z-&xaOj3G@Nq<T&HvleLUxrEa;$BHyE$#OZolzUy
zu)$Zb6BTtkF{OSdD*Zb#%~!Y+GX^p1KJZ@&sxdpguW$$HB<b$!YKJj5*jhV)DJjew
zMqRMBa}f2Cou9%9rA_oNg{6a7Hh{_$PThvZbtyD&Lj&!ppkHM$g;hgn4W2cCbAoMq
z;D70NkFUEQ9VBFZ3VI6Rj@n>w1FfbNQ`!bFEl@z)0)+#Z5e#_hQ|Rd!KrEoRn^aFz
zkzYzz%hher>ixcg6fW`=rr>Nx@enQ!sQqYR{<2^|eUfw?e8;B_<MLoY8^mWHZYp*+
zdCR@!#dlz)Pc8Q@3<kzYql<l-kG{UpaeswZ^dEfHrv+4>`T)Kxkp8${U>g?k*VhCd
zp^yYLvi}<#5TDjrx@{0U$jx*tQn+mhcXsq2e46a@44^-Sd;C6S2=}sK1LQ_OUhgO`
z^4yN+e9Dv9TQ64y1Bw)<aDY=7<mf(-mAEUQ4oj^*)u;J0N8wMeoS(Cj&Vd3ljDOw1
z4ZP#g;4mI13kR{M^r=BSGl*wX*cVV!c;2T5lzy~vz>0i4u)98(^+@R~eUUsG!Ye84
zFa7-?x3cqUXX)$G<2MgYiGWhjq?Q-CE(|sm-68_z>h_O2vME5nX;RodIf)=No(={I
z_<&3QJcPg8kAI}_Vd+OH4z{NsFMmjv3;kunMSh94VNnqD?85uOps%nq=q?kU_JT5@
zwih;eQlhxr)7d^K#-~InWlc&<*#?{A(8f^+C_WmRR{B&Yh3pxhLU9-toLz%rCPi}}
zE!cw^pQlXB3aACUpacU&ZlBUl(Jo4fxpbDVwDn^m{VG||ar9B)9}@K`(SJxmAWro&
z_3yzfUqLoXg`H($!I;FTudPdo6FTJm2@^S|&42H(XbSRW7!)V&=I`{;mWicu@BT7z
zQs!)F9t-K|aFaMsoJ_6z-ICrzjW5#yJRs>~)bugki)ST$8T%!D4F@EBliCNSA5!fl
zN;OuKbR3m0rj=rrq}5`nq<<%iHIl|euXt6QA}$hFNqV)oR?_Rm4oPnoLy|ru_DQ-=
zJTDFa;zjY2<PV6qn7XcU-RK@CR!CYTnq50ww@RKV^S8O~lsr@B`&|Q)XUhDb>p{sg
zWqz0I5y>-U{xR1Rl4r{NQ?6Ge&y@N7t~Vsll=-(^?@FF2^Y6JnkbgW==09{7N}eh4
z?h<ze^O6!zfO$HB=SEZ#@Cuws0NepL)}w%){Dt(+^x#SY;TxdhE&|n$potYICXn@t
z2*!yR#=Dkdf@>`%x-LM8D}+*41ZA#EG0D9K<?ahm;SOQ4y9ZO;o8WU>Qjc2#z59Pq
zO9u!y^MeiK3jhHB6_epc9Fs0q7m}w4lLmSnf6Gb(F%*XXShZTmYQ1gTje=G?4qg`Z
zf*U~;6hT37na-R}qnQiIv@S#+#J6xEf(swOhZ4_JMMMtdob%^9e?s#9@%jc}19Jk8
z4-e<fJGUi+8%jcV#|_$U!wu3nQ(ERKO}sb_iFHGoq$S?$Nb3b13T|vSw(d?_YD*0_
zEsMNZW}YfMsI|Qnwl<kH&7b*pS8En*M|$f8l0+&YGAfB!?UAv37VFKm&9#%ydPIM(
zdeUc9>KFdIEVQN4T|=j2t&EtMI{9_E$cx)DHN2-1mG28IEdMq557#dRO3U?22M($g
zlriC81f!!ELd`)1V?{MBFnGYPgmrGp{4)cn6%<#sg5fMU9E|fi%iTOm9KgiN)zu3o
zSD!J}c*e{V&__#si_#}hO9u$51d|3zY5@QM=aUgu9h0?tNPn1w)HWnB7LQ^GRUjeP
z(zSg-y4St;3UIQ}ZX?^;ZtL2n4`>^*Y>Trk?aBtSQ(D-o$(D8Px^?ZI-PUB?*1fv!
z{YdHme3Fc8%cR@*@zc5A_nq&2=R4<r9-m+R&Zn;t(I(C@T{wI8;I=1PhV;xy!^*aF
zw`7J}+FLSqet*o&>7Hp@$-JF4Fz*;SLw5}<j_PhQGiqc`x`n*k(mkx_T%)~ZY$Tc2
z$C9SVFow3@ogLZ?UT0fn|8OCf!-PAkCpQX<HDX$Z;h5G4W=|Q8i?cAltz&b?wq({g
z)$(MBm`NM73r@zk-^>|ID{W__bHvfJIivHmqmPXlPJd^=<$8K97bHK^(i8eAy)&m<
zBc1z)P8b<4NOeqgIeTQpaF|x5YV1#`#T`tctbN+b*?N{~O)bV<<z=w0G|psl1=l>K
z^y<lE4A<SOTe>>s-s;V!<r$8p=Q@YM*qO64CvoA<-ScjT@Va|U%k*|KJrcWxzPHq{
z*5U!(8-ML_T_Zn~!}hj^n0&Fehr+a&mTDBF2BsbV550rbq|q{J2ve9A)l-0$hbct$
z@^xF7G+HQME8$LE?OL~C!v?02niniPbVo`#)3iI~u<}T`cF+^l>}b2i=5=M-ComP?
zju>8FPIq0VrdV5*EH$|!Ot;e=VudJExcb;2wST}N#u?M~TxGC_!?ccCHCjt|F*PgJ
zf@kJB`|Ml}cmsyrAjO#Kjr^E5p29w+#>$C`Q|54BoDv$fQ9D?3n32P9LPM<W=*)YE
zse?M@JlL!0rj#DX(UtfhZPIA7xNo89?lK)HVLRht+EUVTE40?LfOK;#H9I(V$ZfXK
zc7NI5&9r!?jzPMIDQLR8<sxC)Nv#X%Ub>Izu?LjNqggOH=1@T{9bMn*u8(GI!;M<D
z7tPp~r<+z(x5mL%(#bU3j_B?)V;C6OsAorqO)DEU&gdC0Hy+(M%{Sf=qMjL{T=I92
z`rwLT+sW&&>Gs%MKpd@c!?|2x+D-Rsw10~pU|Rn@A}C1xOlxCribxes0~+n26qDaI
zA2$?e`opx3_KW!rAgbpzU)gFdjAKXh|5w``#F0R|c)Y)Du0_Ihhz^S?k^pk%P>9|p
zIDx)xHH^_~+aA=^$M!<8K~Hy(71nJG(ov0$3Fg{n+QicHk{UcoFg0-esGM}1X@Ad~
zBS?<d9l7c>mZCLw;l4W4a+D8qc)XJS`pUJ5X-f^1ytxwr`@si$lAE?{4G|o;O0l>`
zrr?;~c;{ZEFJ!!3=7=FdGJ?Q^xfNQh4A?i;IJ4}B+A?4olTK(fN++3CRBP97jTJnI
zF!X$o@{%29Dqq5zt&v4zmF$4E8GqYQko@>U1_;EC_6ig|Drn@=DMV9YEUSCaIf$kH
zei3(u#zm9I!Jf(4t`Vm1lltJ&lVHy(eIXE8sy9sUpmz%I_gA#8x^Zv8%w?r2{GdkX
z1SkzRIr>prRK@rqn9j2wG|rUv%t7pQ!2SrmOQRpAcS|Wp-{6gg=|^e5#DDOQVM?H4
z;eM-QeRFr06@ifV(ocvk?_)~N@1c2ien56UjWXid6<wFbYO4An`mpeRM5@Ny3+Rl(
zbU`xgF8Zh#5IvL!n|{2MoS$J@0_^k*xpuDL8B(Fc^sGo&OFzjx`jEid%&0ajY7f^6
zfatIeBxLVwwOXw-I;EOCd4GL8I;x*CqC<vZMLj)_&_*6PZF-{}tyZ6H4Vz9r>W%6i
zevIh)>dk|rIs##^kY67ib8Kw%#-oVFaXG7$ERyA9(NSJUvWiOA5H(!{uOpcWg&-?i
zqPhds%3%tFspHDqqr;A!N0fU`!IdoMs=lv7E*9NYeVfBht~=W5wtrfcc#o#+l8s8!
z(|NMeqjsy@0x{8^j0d00SqRZjp{Kj)&4UHYGxG+z9b-)72I*&J70?+8e?p_@=>-(>
zl6z5vYlP~<2%DU02b!mA{7mS)NS_eLe=<xzM?bHjUHTM)uwXrIBDU|SS$l5gff>CB
zc62^$j+OeC%Nkvg?0*n6EKlkPQ)EUvvfC=;4M&*|I!w}(@V_)eUKLA_t^%`o0PM9L
zV|UKTLn<KtS!oVFL)Q?{mT|@_FU$^-=?e7A^ee*ttH|7QwB8Lh$Ak3i&={ey4+SMm
zFH1;#j$T3N&fB6&@Eme<c}VD`qAe-rH_^|z`mhQuk;Qb^_kR@pQ1Z9v^BTQMzbz-D
zi>k|?M3u!|f2S0?UqZsEIH9*NJS-8lzu;A6-rr-ot=dg9SASoluZUkFH$7X;P=?kY
zX!K?JL-b~<#7wU;b;eS)O;@?h%sPPk{4xEBxb{!sm0AY|>CXVS(<o0c_+LmB_;dQY
z{5&bUko`3UMSlskT$p~5en}AeSLoCF>_RT9YPMpChUjl310o*$Qocj<cT9{r{SUb7
zyw8~+eM{OAdn8QXmU#Ln`e$U@gLrUCREOwaE9Fi3=+LNRpVo&2-v188V4HG54by)L
zRQ`khtGXQSuL|V<0Y7?3JSYt>Gdf>jS%%kn_+Y;Ztbauie*k&Q@=9;erLneIoel2C
zfCMiPTmYnjjxjV!Ar1h1yQ-31h=b@RZt-play?)#cs=ZxOt;5oX)|*e=7k*ASmQ;r
zO4_`=Z&gX-C2$fitvq+iGK1U*^*#IW!Bo{nON%KSxQv@MZsO%Lx21x78z740FSW!f
zJ%f-?XMgR#xdurqd6mWyUX2uh=Si>bnwg#gssR#jDVN{uEi3n(PZ%PFZ|6J25_rBf
z0-u>e4sFe0*Km49ATi7>Kn0f9!uc|rRMR1Dtt6m1LW8^>qFlo}h$@br=Rmpi;mI&>
zOF64Ba2v-pj&TB}f&A09bMg?1id{fn<o+-xxqqV`I(4E};GFW&(;WVov}3%HH;IhR
zkfD^(vt^DjZz(MyHLZxv8}qzPc(%itBkBwf_fC~sDBgh<3XAv5cxxfF3<2U!03Xe&
zz`is!JDHbe;mNmfkH+_LFE*I2^mdL@7(@9D^Dbmctn3bu{98(dlh7lKI0#mGVH(z)
zKYy3Zj)--;jA6Zy^D5&5v$o^>e%>Q?9GLm{i~p^lAn!%ZtF$I~>39XVZxk0bROh^B
zk9cE0AJBLozZIEmy7xG(yHWGztvf<IsL>nr0(2ro1%>zsGMS^EMu+S$r=_;9WwZkg
z)ww}6KOsH_)RkMh?x<ICbu<>@N2R^3(SICQNAzP7(RdB{@@`v*GfeSYLv=cfmTC<f
z(3^*m5~o9A&_)%lVDe@XW#mnNpPfZAT#_;^V_zXZWH^UJ6m3LR2*TSwYLLJG;HyFY
z`lD!=7J&u<gT=H2Ir9WY>%s2_T@_Cso2168v@AU^NzL&qv?6hZBJEdb)g=X=dVg9?
zYf78=0c@!QU6_a$>CPiXT7QAGDM}7Z(0z#_ZA=fmLUj{2z7@Ypo71UDy8GHr-&TLK
zf6a5WCf@Adl<p1`sp~vxi3mP+969Ibi5ssa2I4Q#TbRyM)c;uurU!iOgN?oMDSB|=
zD(y+9I6g^xC+NfA0s{OOtJwzUuz!ukeJ|0G1?Y9wVt#Epor*W6C+T4*d!Awmb^e`+
zt8`2hd<5gi(y5neN#dISS*lO?HcgP9U#UJ~XwDIf)F93nBt8WbF`vY59QLk*jStWQ
zJkES{3dVXbto#gSCt&^8;FoX1oHxK%PZ`RVjar9w@!S{MUm^dc#^EW-?SH#UR=V4t
zn54HiETzwcx&slvSnPCqYuz){n2ZE`&2>e3VglBt4>Z>;xF}}-S~B7<(%B;Y0QR55
z{z-buw>8ilNM3u6I+D$S%?)(p>=eBx-HpvZj{7c*_?K=d()*7<Jf=Q)<Xiv_`BJ;c
zIiKEFq>r74N{MulF2dQ*rGJ8Al=QJ~zb`)MPYedy2kVl9jXxdnmn`&r8ut0w>q?93
zus}1dq%FAFYLsW8ZTQ_XZLh`P2*6(NgS}qGcfGXVWpwsp#Rs}IuKbk*`2}&)I^Vsk
z6S&Q4@oYS?dJ`NwMVBs6!1v<01<?I<a9(_|3|urzWgBlTN5^wl>3>Q(y%%a0i}Y#1
z-F3m;Ieh#Y12UgW?-R)|eX>ZuF-2cc!1>~NS|XSF-6In>zBoZg+ml!6%fk7Uw0LHc
zz8VQk(jOJ+Yu)|^|15ufl$KQd_1eUZZzj`aC%umU6F1&D5XVWcUvDqcUtW@*>xfVd
z@!G2_v`o<z6jii^T2vK{(N;yb(@sTq(;h`rv|rJKbWG7>bR5<t(rH-HQL+^+(0de>
zU*UT{eMHfcUo`jw*u?4r<c@DtY>2s_$`}U{?NjvEm(u&<>B|%mq$Q3weshzrh!=m4
zH~yPq{qO0O>o|+xpE_i3$yVP%gs2l20HBh&_;PzZtwMPqQDk4~L}0tfu;d4uxUM8h
zx$5GP@<Hv#n>d7%rg(9Y8!9@i+9&2l=3<|?le_)g9Z)PQ5ESCo?x4680QstTl-CH_
z5m)j*Epfqj7I|G0-*vpm?U#8&k?((2zg<P{?hZ7>;QYNszIUs?zAIGUr9}em3I$<A
zs5ixZMZW(6J+@q19%>Fhb*w9-ci~gV$1;8(U;p&SDZE^3_CNLX1zM3@E|W%A=rX4;
zwOlLm!AP*(*Bl0rL_(L=6`Hv5>_8;g?VljGOuMhr8|fxKG|7jrCnCW}AbEe8A8O*a
z;rbQWArFQUVyZaIdGyF7WbZ8lvQ6v;yEgG7uqYA&H#G5ad?wWuhnhHBvUGfsN3K^(
zewji7_p=ede8DTP$FEa_M(6|&v8m{z@NJ&XsIgEPpP?ss9mYaeWBd+!UX6vy_yzie
z8Vi;2C+U(J3ze}%uZ)Gt_+?D`yc!FY@z?1aYAjU7Z=eB`u~3ZJ#|<)8RL1SxrN%;K
zoZ+XHo~5{G1p40!tUgK$I7L3rV9Y8@Eg;`_0Z>Z^2tPilXQ&PU0NNXq;YJ*jtBNjv
zYflqF6o%gs=t3z%xd|2&*IQdyR=^LH8WYpRgrrep4Mx6Aw}<V*UdmoD#>fxhSE$jN
z_`x6Gk20R2MM&C)-R$h{nfE#GnVgwFe}DZ3unAM(^yK7C>62cU)*<-~eOtHo^)=lJ
zyq4q2*a>{Y3mU}nkX(`x@nlm*hSem0>o7{ZNZ;O<ZbWN(%QigOG8~nI>Q5dw>RYT0
zOXvK4;<_A&n$p-%65n=wqR{bejviAOu@}cn>s#w3qd~{|=TQiObS+3ii(WV`2`mPo
zZQ7x1xMY3^WvfM@Sq*HPLJh+LQwQ=`ny&P1^Hu$TtXM-zVD=*VoC&`n>n>@37!?>f
zN*sy>#GXLvspC8GGlAj!USU^YC|}skAcN~^Xqe0(jqx#zAj>muU<=IUs~34|v06u2
zahGbSeT-uAG|Vv*B<wTSL7c#R&H9)rl3qE38(0{_JQf9J`Uo`S1hke4xPAu9m`!5|
zx42|^wj6;+musmsWmu!5gnWyC%7tpb#g_%ltB{@|SD-83y8@d7*`1w%h8tHyeJmd+
z%ZJ?fd}Uzfh5vJX5)@T}RqkqqccH*!l{ekX#H&;IR^iy-o@x*n<0q?{%;#c+zcZNN
z(cr&%T;m%^7vKNHRPG0+zd~JE%wV>w$#pf8#qXFtMfw|VuC{UeT)2WpJ6&O+E6jF;
z;~n9>cf~Ip6j<jm#c!}kVfVY(Du($6W;)n}!j_iX$oGOnXJBElU#^X{UW^X`qsn*a
zA5cpN2wnEIZ#$D5jcI?ahqQ_yf+s;y=zX)9CfjZ{VK=P@u@B-~coIDL06vsB5j{8y
z^YQ)mn_2er>-_@&PGFD0%Vu*QJ@Ht`C7Og!xt#L>mqlJGEh<%*ATJUmZc(FfNSB##
zfy_`Y-70r{Iv3jEfR|~Ii!xC44vZ(KNj#>kjsE86E3FB*Oay<UI$}~~5UnyP(KT8}
zZxN4%<6#sexaQA3Fb186Vr3;>D~$|}3Y&(h6^X|1(TcJ}8{Ua3yL1loSfg!2gTekn
ztVO7WNyFQCfwF2ti$UvL8C6{{IPBg01XK~$ThIQx{)~aw>(9F2L#G36*kRDPqA$P*
znq=!@bbQ#RzDpVIfYc*x9=}2N^*2z1E%3epP)i30>M4^xlbnuWe_MAGRTTb?O*?TC
zw6v5$6bS)qZqo=w4J~*9i;eVx4NwO!crrOjhE8U(&P-ZZU9$We^ubqNd73QDTJqqV
z55D;u{1?`JQre~$mu9WZ%=z|x?{A;q|NiAy0GH5U*nIM2xww(4aBEe#)zoy#s-^NN
z%WJl5hX=Oj8cnY%e+ZYt5!@FfY;fPO8p2xj+f6?;UE_`~@~KwcX!4d}D<7hA<#M$$
zMY^)MV_$1K4gr3H8yA&|Ten>yr0v!TT@%u$ScDfRrzVR=Rjj3cjDj)fWv?wQanp7L
zL)Me^LS6EzBMR%1w^~9L%8&g(G;d3f4uLKFIqs5JYKSlle?R1Fyx?%RURbI;6jq>N
zh+(uYf`e8J=hO2&ZQCoTU^AKRV>_^&!W{P-3%oVMaQqOcL1!4cYP)vuF~dMQb1#lK
zj_HWu4TgBXPYuJQYWv&8km~(7Mlh=5I8HE}*mJ#?mxhx%#+9e>eorO0)eg#m6uhb7
zG^KSg`Cbxlf9XizZH9>B@hZcqJ*7VTp6)w1tHLB11}(?)MI0$rLIUS0;Z^atECLmz
zzb6FE#PKdBl;L{}$M%UdWEi4$AS4ew$#8O?ZRr(G4syuHkcGi8a#*gRz@QP|7R93=
zj*A$L;eA}9id+JyWjkK`Mod00;{&DlA!QJFR3&ljf1vI*O1ec{(V=0QA?ELLVls-W
z``ELsu7M`3`vI4MzhVcpJ!9#^KGjq|#b-J`!F7h${dUEFmBLuMbYu>nV^(S3q+UC;
z7s@e_qZG#+N=oo0o$G1>6Y0a{9@&9;EU2+8k|7P6p?HMh|8#X5UnwpxGbHw;%WXHX
zn_~8ne<fP#lF)ExhoOC?U2oe?RjuMZTCZ6tJ(*l@2sf$dNaAE`m*&YDvG!UNU;j%>
zdvw09V+G$(lhoq7L}=qb+OaPSD&;$TuUtG(4;py(h)8|Nord(*d1ZH-Dmw1MqU&RK
ziI)26r-hE(pqnmo4uixe^`qe<N`Hnro&LZOe@TGNY!@+ezJoVIjBR2t_q>a7(_HA_
zR2K<zF>jdJ4$g!)7ve&Q^b1Tf+{(Vd6vI<Y)5k{)=3<DMk!vAtJl*=spzq;Wh%e0P
zNQhfMZ~d7&9mx~AM1m0Im1kSO<(|(!LzJ)!FT1#FcClh+AB>nCd>i725IomG^(Ez(
zD8L!4qlUAX=)EV9!3JfWLB4n1z)!ums&0UuuVLUHP)i30*5f6tnv<ZCa(|(K<w1}q
zl?27Kf`O2bZM(f5T<@B@Ye_WnLHrE9(8R{X7k&UglySCqr=>k?lbhL{|8I78X7|_c
zA3p(L9<~X5y1L3{K8Sf*xL|5gToDT;aYig?m8z^zQ`XdEMJqC#*O|ho!7x~+MzT<5
zg$turF~pS;RSY&GR;6TxR)3Q+&%yG`3&ngIwR*<kOEpEo38PGtb6lcIvdWk)s}j!E
znQq`(IG>qK&t{TERu@0|fDrKKw3=RE&t-)Xh-$i&l5|>BSn5)z)hg3d?<~8msU=ye
z>CHWR!9yT;PU|$KP*qAD<j+~b^g=HIp(q=sF6V1pM<B4@RH@8N9e-x`R*`CrZ{dLn
zg!lht%9(t}?YcMAEE5=RoywKz{NhrsRw3YhFITNh+_SU+T=7!VFW$-(rqxUWWRJi|
zohovXnrx>f(V?zj^n^g~nykv^I)Uz3{78Ty81{n~ZsS&7WH)#Ach3%UyV<js2d{R0
zFe-o_ZuLPAj1lN}N`J-oXZYZ*0D={~FF?TZ9tbdCd7=P82yySB0AU#9-Xj771~YfA
zl`a&!UOT=iAM|m3g2143U2hd&C>D1s=Ahvw9*%Wt<42vTt%|niux3Zww13+oK)-d~
zG>VKHM0ov>KXKaUH(Cc)#9GFVSc4EoUbnRudxi}T8J!VNY=4g*Y7C*Ho7#^wUVt&<
zKN3&ugs1Ur<7<OCJeL<=IKCj>67&ea4^1oBw%@h^+YZ<ko8Pq9dbC0G@TTE+3rA_p
zO3+3V$H%9q-(e(trvZ`hy#|bPZ-RT1p=huHQ=SGyzXJb-AO>+eK^VI5573*KZosq?
zpMj(u5257?^lBu&LF9`ao`sYf9&zx;uK2iv&$;8{4nFUSFF5$3JHFuHORo5YgFkV{
zCmcNEicdQDvO7NM;484|f=_+6!)x%g1CL;L9DE%%T=1xaKZ8v-+-@x1OZ;|0_a9J8
z2MFd71j+6K002-1li@}jlN6Rde_awnSQ^R>8l%uQO&WF!6qOdxN;eu7Q-nHAUeckH
znK(0P3kdECiu+2%6$MdLP?%OK@`LB_gMXCA`(~0RX;Tm9<mAqK?|ygr?tAyWfBwDo
z7k~}8%`kTQbZ_&?z<?AV;%XwWEf7xzLV>uJ&d7>n%9<KZYY9&CI#;-4e{fnHl#FnE
zkjICNY}yEHOG=8tLxD^xoR%_SS=itMTQ?;JHj=L1oX01#ib4~{pcGy&RMeELa<0p2
zie`toZjdjUZ*JPy)ErLm;lL?37=tIOW%W4UBP*OCpy{bdN|zD}kBsP2Cd2i}AxG{W
z<?*bk=`K_<EIcL+OA$p<e^Zg}fnz*wx=>A~GP*{Zrpyh7B^|a-)|8b<&(!>OhWQ08
z$LV}WQ`RD4Od8d3O-;%vhK7#W<7u;XvbxQo0JX@fY(C0RS6^zcd>jo287k@<4tg;k
z3q5e5hLHE@&4ooC)S|`<FXqib6_$A6#vJjr%nJ|Zimw$6-r_i1e^^C9@=@6^4cm2}
z7x&`<4_3oZ94MNUD1_%l01G^1<spW7I+qf=2&QWXbcLa|zM-Q)f7BEe-tNU3tQCMd
z0;IUvBk5c<9ex+)0eMEeXh2XnZe(aK%Gu^fR!y#}l9JEd_AC!%MdIY2h@ibhDlUYn
z$Z=;lO^IP$*&-B2f1Ha+<!0nSZ#%^l!8#9`u%2Oo!AmSM)YO-1i@I&ZCQ*gWlwhJr
zxs}e48;>w7N|jm>3tns$G}U4o!(2g=!}xLHp?+qFvj$ztd<%96=4tCKGG@ADSX{=m
zNZ@ho6rr?EOQ1(G2i@2;GXb&S#U3YtCuVwc*4rJcPm$kZf2+|!X~X6%(QMj{4u)mZ
zOi!(P(dF3hX4ra9l=RKQ$v(kJFS#;ib+z9K^#Gle6LKa>&4oMFJ4C&NBJ7hhPSIjc
zOno$M6iq+l;ExpH9rF68@D3-EgCCf}JJSgVPbI1$?JjPPX!_88InA}KX&=#cFH#s3
zIx<6LeY==wf5DK*jP`hqF%u+|sI)3HfyywfAj=0OMNUX2pLR;T(8c+$g&}Z#q9L>(
zD~t~l&X^VFXp@&w92f8tq+KXMZ&o!an%$#uo^hJh^9-RjEvqE_s%H8{qw(juo4?SC
z{YhO*`|H*ibxm%ZF6r=2QC)bE`d3oZ(~?<!ZyfHJe|{^JKQAxi6%Ss*s|>;a-(mX)
zb!|i%p!VVP>DN6tg*Ry97gUPUJj<}OxaYL1nXE}hxs-O{twImUw<O2r+a8?5JJeOn
z-4ZUmM`R1eXdf5gMV4(VAK*g|-p5Dz$VydQ_=KS<u4rP*ekLxPPjQ}_T%xq9ZhX$L
zwp^<ye}BHd5H7!D*x9A|hb1M;{UfqD=pW+8?JD|>43Eo6nJ4_RTDIQALB8H!3nq37
zcE6>oNG;jZZhXh!vORPsMKfzJ8_*?O7DfGmcrL8A(_NAhSH+JE?u?`xR1|ZThDb;2
zDt`9hC;UQ%94^20-MA*;<$KO0{3b&9y(ENIe@&xj6>X23)Ftc?ax=4pL5FZ06CPOj
zgG%2<Pb(@_2fd`avgrY??-$z<>*<WT=z&*(8gcQ{SB<##ZAni8k}eRDSU~@i0CQml
zDrxrb#$aPG6pW3-H;LL<tZN*LZ(`|9EWe4B79Kp0i{W56HuFcg&;%Z~P~T}GM4uZh
ze{AtJ(_nk3-@;=S+I+id^ki<d*Vjp-o=|^<g{XytzNcw;IJbDz_biR#xlz)>F$-x6
z&si`nj955%8LK)caVl1M8?IPaMPt<WLU;mO-+HX@*|}0L-G;E3gR?W7x9e6H9GT{B
znBvW<o>M85o>MvPUn@(X=!wZq0)at}MK|kJ&K<Rqc0O3RM|#tTjjbDBs~mqf;a<$8
zIF=hnE-|aOzi~Bvr@T|Og86i<xK2%}_`L=1nN}~!S>JggGx6y?Ey21qiw~76MoISk
z+LyUR=2+oJK1IoYOX~R}S1x>iblZ|_oAmqhyU+NpxvjQb;Ht{pO_xn4T+UO<73|gD
zaq0Wtdz^7GoZq-Fu+;61dX%|tud0myO`{vHTlP*oes5OaTBV$=y?3V{mRnFLdQ!Hj
z)lErp+uBchtEPv?ao=?feR1oRVaUdpIVC}+xkgTxPYSGDyR2Zw++VdTe(-~Oh=P%c
zFD5UUvx;?cLREy~<lwo<(ny!wGpI2DPxgQVm&q6BI>~@9BnQ?{+kh7j7^BGZ3r}vC
zuRPgbSbFk<c>*%f8<`nm*%=sYP!wJk1uNV$&qN0K`bt|AMMaWeMf&qirQ!Dt0<A<f
z>FDJ8`4KXRTiO^HPz`BO1{-ofSrz0YR`9K0lLHorGM!h0O0Z3yut19ieErkD1!7DO
zG~nX@7pO{uE-YFOTtaXT=wTxi=Y>zUU+BjIx>jcL#D!u^>AGNjXBL{vAZ}$~KnuVC
z1E3-$;H5MCAlFEP4~z$T=^-$HP(wOqa`hr78Te`EKnLicSpL~^a?K*8$-ft=N<+?q
zW?-0u5gn^0TQByPK^#BKz~G2th_L-+o5j*dCr4Ycg3q*_+`m|qNyu^Xvc-|obKpm+
zGBD_)==PZ0utaRK!4gv$&;gX1%nS@qfG$9_!NzrR<B}K|<iq+*Pz<^20aPG2S=L<<
zoMMpAl2JrasOtk%C_8zX7dYftKu3{G=36Gl^ePl6E;@PsGPcRyz!Tv?$quoM9(X7X
ziXrL|P{jr^h&4703`!`9mgj;LO<JxY16)4>Sv~>`eq9tbPbwj5K&x^fX&o_o$H1U~
zqIOd?L@oQ|Bg^Gwz#}riv?K=%D|r-k8@s@c6Ir1u0~(i50a^-LyMmf7oO;2EvR3Fw
zgF8gPQ1=7g{c3<>(&5P)SNO;vnvv+PKQakyh~7$L8Bq2Q1{!dbhk-!@#SpP+P(|#M
SXRcJ{65?fGI57uQ5&!`B?F@7P

delta 34554
zcmX7vV`H6d(}mmEwr$(CZQE$vU^m*aZQE(=WXEZ2+l}qF_w)XN>&rEBu9;)4xt<3b
zo(HR^Mh47P)@z^^pH!4#b(O8!;$>N+S+v5K5f8RrQ+Qv0_oH#e!pI2>yt4ij>fI9l
zW&-hsVAQg%dpn3NRy$kb_vbM2sr`>bZ48b35m{D=OqX;p8A${^Dp|W&J5mXvUl#_I
zN!~GCBUzj~C%K?<7<Lh)kN@p@FGCC|S+xWQML0moHteKXeb8wJdQH-gjR&7XH4^`G
z18O**eOiwBMhV|yW}+)H1-qhO2aiyh%|2AVjN^2;YOavpiJ0<eag))8h36*<J}UmM
zh5)@`0psU<7DA@>+UZ_q|L)EGG#_*2Zzko-&Kck)Qd2%CpS3{P1co1?$|Sj1?E;PO
z7alI9$X(MDly9AIEZ-vDLhpAKd1x4U#w$OvBtaA{fW9)iD#|AkMrsSaN<NH{Yj<;r
zJR^3Ei65zvOeKyy368On7xFr|udQdRg!uvBg!o!BMrLO4`R~=WSNU)FX3DN?UiRax
zt1Nc*-cP@`?<QFC2v<*!x!PMc9!`H@mKU%%&E@IsfX5sqlj5iC9s3zUdp-4#$Pj{D
z-ud{50Mow^jMOt&t!;$ig-cEDVcZ_LuIYDHKBg^7I=ypK+jgs5kU>z(69;h1iM1#_
z?u?O_aKa>vk=j;AR&*V-p3SY`CI}Uo%eRO(Dr-Te<99WQhi>y&l%UiS%W2m(d#woD
zW?alFl75!1NiUzVqgqY98fSQNjhX3uZ&orB08Y*DFD;sjIddWoJF;S_@{Lx#SQk+9
zvSQ-620z0D7cy8-u_7u?PqYt?R0m2k%PWj%V(L|MCO(@3%l&pzEy7ijNv(VXU9byn
z@6=4zL|qk*7!@QWd9imT9i%y}1#6+%w=s%WmsHbw@{UVc^?nL*GsnACaLnTbr9A>B
zK)H-$tB`>jt9LSw<e?{;w%G1n2P1|J0yx{P*t9jy^kFBQA#W^mMH}$k)?g@xYa+r6
zlJ|^>aY+4!F1q(YO!E7@?SX3X-Ug4r($QrmJnM8m#;#LN`kE>?<{vbCZbhKOrMpux
zTU=<lXsfLv18QvxbhhA4+*COFL{lUUKwTI^?+b!W)%bL{5ICeYgxi==?r7ml!xm!7
zhdowRK-_IXtccBJOk{G;-<SoBvpq6sZLl$N`1P40zF|?WL@Z|Q9X1BGIgTLA`g8zK
z6>02hy${;n&ikcP8PqufhT9nJU>s;dyl;&~|Cs+o{9pCu{cRF+0{iyuH~6=tIZXVd
zR~pJBC3Hf-g%Y|bhTuGyd~3-sm}kaX5=T?p$V?48h4{h2;_u{<w17YEUL6d7r?`IW
zC*$~_<n<G0=2K)oe-lc+anc79OB&v^xcsx>b}8s~Jar{39PnL7DsXpxcX#3zx@f9K
zkkrw9s2*>)&=fLY{=xeIYVICff2Id5cc*~l7ztSsU@xuXYdV1(lLGZ5)?mXyIDf1-
zA7j3P{C5s?$Y-kg60&XML*y93zrir8CNq*EMx)Kw)XA(N({9t-<qoKDSToyx=0O<F
zcZOoLjIgey#`z<{D=F=|jxIUj7G)oJ<z}l*I|pAYR$!%#pP+(2spDO4`pdL|xx}1{
zoCp>XAdX;rjxk`OF%4-0x?ne@LlBQMJe5+$Ir{Oj`@#qe+_-z!g5qQ2<A}CkiPnG%
z#<9e?dl5B^C&2WW>SxKQy1ex_x^Huj%u+S@Ef<hwN)i9H)8f1HlVw`IOD9pOk3F=2
zUFdA$C*zaFwUQBueYyoochf_SCZNJv(o>EPP-70KeL@7@PBfadCUBt%`huTknOCj{
z;v?wZ2&wsL@-iBa(iFd)7duJTY8z-q5^HR-R9d*ex2m^A-~uCvz9B-1C$2xXL#>ow
z!O<5&jhbM&@m=l_aW3F>vjJyy27gY}!9PSU3kITbrbs#Gm<n97W+KiQNM{Zc)RZRJ
zO9VM|sK^tMcU~i!8>0gD?~Tub8ZFFK$X?pdv-%EeopaGB#$rDQHELW!8bVt`%?&>0
zrZUQ0!yP(uzVK?jWJ8^n915hO$v1SLV_&$-2y(iDIg}GDFRo!JzQF#gJoWu^UW0#?
z*OC-SPMEY!LYcIZO95!sv{#-t!3Z!CfomqgzFJld>~CTFKGcr^sUai5s-y^vI5K={
z)cmQthQuKS07e8nLfaIYQ5f}PJQqcmokx?%yzFH*`%k}RyXCt1C<ET#?;-nOj&O_w
z0RjAX8jLRfx+{_%VvA`D$(8(CLoDsiJS|qdlA=Gf(}R42xBwFy^Y@K2Y2B5F73lIC
z_Y!h7$sAGI9OU(?0{R3(+-fA%SuZuc8k~w~-j=n9^J*)UAglq-zi6`BA?FFqiPk`=
zFg88a>hfv5KAeMWbq^2MNft;@`hMyhWg50(!jdAn;Jyx4Yt)^^DVCSu?xRu^<p^cd
z`AfT9K@W351Yp;%_)?5A+{-Sd(?fRZxF>$*&&=O6#JVShU_N3?D)|$5pyP8A!f)`|
z>t0k&S66T*es5(_cs>0F=twYJUrQMqYa2HQvy)d+XW&rai?m;8nW9tL9Ivp9qi2-`
zOQM<}D*g`<R@=@6xNeyfv+%3f>28wJ54H~1U!+)vQh)(cpuf^&8uteU$G{9BUhOL|
zBX{5E1**;hlc0ZAi(r@)IK{Y*ro_UL8Ztf8n{Xnwn=s=qH;<JNq#i!T6Ol{k1N4b^
z$<%S3ch|aFn`rrM7xp7Akdf|09MH!w#X^*^qWGG{=N4~T_X*vr=<9Sr1u0vsR}sts
z)nFJ(24Ixno*7<(J<m!__?DA*(CbmQp|*(D0(fX(I-?~pJ$dd#K$~K_UvNXKlCI0J
zAi?sKWY{_$3e#h0k$v$pwmR{i3$N!{^!LlUzPj3g^GFVYwXlrd2LlB<R)>fxkK+uL
zY)0pvf6-iHfX+{F8&6LzG;&d%^5g`_&GEEx0GU=cJM*}RecV-AqHSK@{TMi<FO;*^
zVIJ;K`_Wr?eL=2LPxeiG(NZ1XC=-->r1jaFf&R{@?|ieOUnmb?lQxCN!GnAqcii9$
z{a!Y{Vfz)xD!m2VfPH=`bk5m6dG{LfgtA4ITT?Sckn<92rt@pG+sk>3UhTQx9ywF3
z=<e;Zhr+VTogSTRTa`0dZhDEYFlz}iYJ-Ca|IInsl^}*}ws*9OfO%7=435_R)~{jX
z|C))|=(?2={M$D}=%iUpf}~t>$|U(bN<fa5trxoGzi!H8bL1;zwAZzO0dvgcXe@7*
zY^o`jnIo&H5j7~yXg%$-pqf(9s|<SUx!UU?X^UEuJZ$s34K@~hg}#3V7Z`DSr`YnM
zvIU1teWhe~+xESB+c<W5In8tK0k4NOX6yQKGIQC&FBWX~Q<%G1#g|49BQj)83&2%|
zUs<JZ#bIxoeZa-tcILuIpp|3uGrjDI|D_t;j#v#bOj@j6furiEu;aSGahA)d$u{2o
zqNVUdWs5~-2cc><=6-B4+UbYWxfQUOe8cmEDY3QL$;mOw&X2;q9x9qNz3J97)3^jb
zdlzkDYLKm^5?3IV>t3fdWwNpq3qY;hsj<HM2+6N#zwmbwcBhBpw2~<e#bkw}$`!Kn
z1)=dUBB{1-93?8Rgm^nY*tF;B7wJ85X%C|9a*1^ara~8e6Ph?%g(rj#aA~z51%f(0
zrd%qQA0cZ0Z2^@t)yso?$#(rr$H2>=pk9;P!wVmjP|6Dw^ez7_&DH9X33$T=Q{>Nl
zv*a*QMM1-2XQ)O=3n@X+RO~S`N13QM81^ZzljPJIFBh%x<~No?@z_&LAl)ap!AflS
zb{yFXU(Uw(dw%NR_l7%eN2VVX;^Ln{I1G+yPQr1AY+0MapBnJ3k1>Zdrw^3aUig*!
z?xQe8C0LW;EDY(qe_P!Z#Q^jP3u$Z3hQpy^w7?jI;~XTz0ju$DQNc4LUyX}+S5zh>
zGkB%~XU+L?3pw&j!i|x6C+RyP+_XYNm9<?|3Dg4wCcC?VQr<bzeKrSpw8b>`rtH<m
z^q>pqxvoCdV_MXg847oHhYJqO+{t!xxdbsw4Ugn($Cwkm<z&(cEdbmmHs@&T9(Qe{
zY0mG0W~lWDbhWX{<STjlgSTP0kYxL@HoRpRDzEHe(+0b)#tmEv>^+36&goy$vkaFs
zrH6F29eMPXyoBha7X^b+N*a!>VZ<&G<WRC+Em14oermb3Ag6>f3eeE+Bgz7PB-6X7
z_%2M~{sTwC^iQVjH9#fVa3IO6E4b*S%M;#WhHa^L+=DP%arD_`eW5G0<9Tk=Ci?P@
z6tJXhej{ZWF=idj32x7dp{zmQY;;D2*11&-(~wifGXLmD6C-XR=K3c>S^_+x!3OuB
z%D&!EOk;V4Sq6eQcE{UEDsPMtED*;qgcJU^UwLwjE-Ww54d73fQ`<mR8A?eX*vOu(
z;s!AeRF7I0k*#FrPh}Ask{q+pF}CGNj<O)=(?Ka}%Tcfb=@JBbsSO=CT&lP}la&!T
zB*Gtuxukvb<;Db%3gAA$>9Sv%^H>juEKmxN+*aD=0Q+ZFH1_J(*$~9&JyUJ6!>(Nj
zi3Z6zWC%Yz0ZjX>thi~rH+lqv<9nkI3?Ghn7@!u3Ef){G(0Pvwnxc&(YeC=Kg2-7z
zr>a^@b_QClXs?Obplq@Lq-l5>W);Y^JbCYk^n8G`8PzCH^rnY5Zk-AN6|7Pn=oF(H
zxE#8LkI;;}K7I^UK55Z)c=zn7OX_XVgFlEGSO}~H^y|wd7piw*b1$kA!0*X*DQ~O`
z*vFvc5Jy7(fFMRq>XA8Tq`E>EF35{?(_;yAdbO8rrmrlb&LceV%;U3haVV}Koh9C|
zTZnR0a(*yN^Hp9u*h+eAdn)d}vPCo3k?GCz1w>OOeme(Mbo*A7)*nEmmUt?eN_vA;
z=~2}K_}BtDXJM-y5fn^v>QQo+%*FdZQFNz^j&rYhmZHgDA-TH47#Wjn_@iH4?6R{J
z%+C8LYIy>{3~A@|y4kN8YZZp72F8F@dOZWp>N0-DyVb4UQd_t^`P)zsCoygL_>>x|
z2Hyu7;n(4G&?wCB4YVUIVg0K!CALjRsb}&4aLS|}0t`C}orYqhFe7N~h9XQ_bIW*f
zGlDCIE`&wwyFX1U>}g#P0xRRn2q9%FPRfm{-M7;}6cS(V6;kn@6!$y06lO>8AE_!O
z{|W{HEA<ltk<^3&v(zsFX*bwn-7C9ar58A%@T#a2aBXBI3o8y|0%iXyV|l}sg<m!U
zT-k`jQb^8oQ(X_pliK#72^1&??md*<37!Bk(#NKKogWc3``(E}^iGHbqW0g7K=ab9
zUpQLYqhr5^vY&U1qg(xAv~ioY%|o`fES|sX{~Wc{ICx(f+xCyUQrPRZo%C)iUsb?a
zew2`T0vB@=rlln7nHw`MOepMp{ML^Ja1HQ#K5&NkV!sK<&i;~{TAs9+9gJ+;Lz$da
zS};~j46utcwL=M>bI0eD$z9tQvWth7y>qpTKQ0$EDsJkQxAaV2+gE28Al8W%t`Pbh
zP<xNDbl}BXrCZDI-ZIp5KNryWfd5bbbb`{6SD?OqGei3JO+_9Y0^=JD49vHrX%anv
zx});E3Pu1^ut&6#2_NeA4BT3`JGz8npG~rjS&&I~|7dIHW39&8ddY=MbOQ07^K~#4
zAG+`}_`B-xd@Z|;49eO;uCH?5F>l#%_S@a^6Y;lH6BfUfZNRKwS#x_keQ`;Rjg@qj
zZRwQ<VVZO-GvahB#0&T?91+}W8%P`g+CV*WTDcQ*ECFq*EHu?RdUx6LGyt0&7`Ke<
z(79}oUUG8J6UjGrhsg*-n+Rzoc4V@;e92&Fv0kzs^dCx1BLpw}k)lbrZ+!!4b9rh_
z{{2T!5K`M=B=D3khI9L_t!YX2loweH^^dUIKs*I^r|TIqcqitK3=NKDIU!(-Fg-cJ
za`ZI<4s`pPup!z94p%kUdn`lUG3}x)93kQpWvT(t+fqvi%M1$&iX<2wVFk(pY2$hw
z;e()Ntun1;jRBx1Z_weKQ&VAEOVyzxlq3KpHRdO0*!<udc458*N}Tk<yvh(lq)+w`
zFEEJtD{)&$;xqmiHT}}u4L}!;Y0mFCG2c*If`gd*;VV3Vr8aURePSQ2wwxyp-Zg;}
z2y5*0{Ns5&4yo$wW`M$Wj^j~)lpJ%p0cE61bW)~~n2lIIinNG2#<)ly#Dd$&{nJPv
zg4t&sQ>XZd-rWngbYC}r6X)VCJ-=D54A+81%(L*8?+&r7(wOxDSNn!t(U}!;5|sjq
zc5yF5$V!;%C#T+T3*AD+A({T)#p$H_<$nDd#M)KOLbd*KoW~9E19BBd-UwBX1<0h9
z8lNI&7Z_r4bx;`%5&;ky+y7PD9F^;Qk{`J@z!jJKyJ|s@lY^y!r9p^75D)_TJ6S*T
zLA7AA*m}Y|5~)-`cyB+lUE9CS_`iB;MM&0f<l?>X**f;$n($fQ1_Zo=u>|n~r$<Jr
zR^#jf&wU3t7icxNwdO00+#eaU-n3>HvkOUK(gv_L&@DE0b4#ya{HN)8bNQMl9hCva
zi~j0v&plR<c*E(e0zzsewSSCWoYNe#zBz~4gPJgFZQI3UCOBaQQRa$tLad>sp?_zR
zA}uI4n;^_Ko5`N-HCw_1BMLd#OAmmIY#ol4M^UjLL-UAat+xA+zxrFqKc@V5Zqan_
z+L<vzaOW^{oV^!V7EZrt5zgiq1K|N3u8LNf1;g77T|EW9D_#STe{m<8-*ti95##sc
zyGd7O;e|{=b{s4r=uzKM#MLq2_E;Cl(9-pAG6@1SP&CT#ClJkT<QXn*!E+nh?X8yH
zcMM9Cu5gnFv(px!)SJCU0$OR|_#*aP@{anWc1hQYPQ<vIYfp}%E>oVX-Ub2mT7Dk_
z<+_3?XWBEM84@J_F}FDe-hl@}x@v-s1AR{_YD!_fMgagH6s9uyi6pW3gdhauG>+H?
zi<5^{dp*5-9v`|m*ceT&`Hqv77oBQ+Da!=?dDO&9jo;<Pz>=JkzrQKx^o$RqAgzL{
zjK@n)JW~lzxB>(o(21ibI}i|r3e;17zT<i-A5WFI)l0=DfsHm5h*a$|nW5w2Z&g&C
zfIgxfWxL<b7<;2;0#?vaiBe<amMP~Hu(BM=D~XPJ7}ZO*rIjib5G!Hs^`n3(-k2!C
z!UiS)1Z$XWmt$B+Er?b&6mkk#-D8jLY4v(Z+xBe4-ze{3Z8c}rWsNw!vUg{(e_E!O
zGy`+{2}jJyQpV^h@Z#T~xRhsp27NBzs^L@HyvOdwuq!yc4;~zS%a~%ld&pD6w$+^X
z4e>jdEl5c`Cn-KAlR7EPp84M@!8~CywES-`mxKJ@Dsf6B18_!XMIq$Q3rTDeIgJ3X
zB1)voa#V{iY^ju>*Cdg&UCbx?d3UMArPRHZauE}c@<I9zuOvh8{QaF~!8NmT14ZcB
zXDiDmH{)Nqe}et`?!f-+>Fdk;z85OcA&Th>ZN%}=VU%3b9={Q(@M4QaeuGE(BbZ{U
z?WPD<in~-M2H?eVIA^jvfi8U5&YUg%&UM*~ACg=ZhWs!uCP(%s`pq9sl~Rn<t^myp
zyBzUD^1Z%w8~>G+sjJSz1OYFpdImKYHUa@ELn%n&PR9&I7B$<-c3e|{tPH*u@hs<V
z<H3(ufuU6=m@PPl*Z5lGM+(I8E&;iIFkSFjg3169JAkH{U;A7oj2f|AG+4C#qrq+Q
zSEMgDyQI)s?9FfgI&>)Ci>Z@5$M?lP(#d#QIz}~()P7mt`<2PT4oHH}R&#dIx4<lE
zZM@x|FQ}SFXpD?iesWRPGe4;I$q-cmdj*5H_;}S&ml}cX6{lN;Hc9cQTAqNu`75Pu
zm)lpl05-+8mc%^XjQ*+==x*CFMVtN~s|N?yyj?nb@+0b65tp1aoBu<}P|sqT#oxbu
z^9TFiuY0AD1GFwI&Vgx(N@^#!xc>uq943D8gVbaa2&Fygr<d;h^~M>Sk3*whGr~Jn
zR4QnS@83UZ_BUGw<CYZIe2@PKT3-tK{$nVwgnd*Bz&L7Z=B%q|uGZL4XMBC&BC8z`
z)4RakhXH*19!wvlHiX`r;-qnceEtmXoaf9Zp<!_s;6!03b&^hmte9c568Cp_f0FC4
z@(8kb-pvrKkck`L9ixZbpEUWNQ0z@i@*Zn<GA0aPaURQm-ybWQ)C=W4nKD;in<YK)
z$64%N3;kIt{W*W&vFr90Wr=?+S%KEmK=Dy9&+KBnRXYEkzmZq_)xy)7^gtx11&*2l
zFT&6!U^|U`oJ*VvM^5Eh;*~96yk0B!PnHB-7=?Z<!C6~0J+Dn$C3)1UuE9Wd>;?@T
zo5jA#potERcBv+dd8V$xTh)COur`TQ^^Yb&cdBcesjHlA3O8SBeKrVj!-D3+_p6%P
zP@e{|^-G-C(}g+=bAuAy8)wcS{$XB?I=|r=&=TvbqeyXiuG43RR>R72Ry7d6RS;n^
zO5J-QIc@)sz_l6%Lg5zA8cgNK^GK_b-Z+M{RLYk5=O|6c%!1u6YMm3jJg{TfS*L%2
zA<*7$@wgJ(M*gyTzz8+7{iRP_e~(CCbGB}FN-#`&1ntct@`5gB-u6oUp3#QDxyF8v
zOjxr}pS{5RpK1l7+l(bC)0>M;%7L?@<gTpQHS2vT6B!6J&3W7)M@B+Riqg<&33yLj
zDz(l!s5Pz!g$so{@5)~9fIqA36)y?X;K=l3yijdZuqQpxQ-60#B4i){*o9>6t}S&a
zx0gP8^sXi(g2_g8+8-1~hKO;9Nn%_S%9djd*;nCLadHpVx(S0tixw2{Q}vOPCWvZg
zjYc6LQ~nIZ*b0m_uN~l{&2df2*ZmBU8dv`#o+^5p>D5l%9@(Y-g%`|$%nQ|SSRm0c
zLZV)45DS8d#v(z6gj&6|ay@MP23leodS8-GWIMH8_YCScX#Xr)mbuvXqSHo*)cY9g
z#Ea+NvHIA)@`L+)T|f$Etx;-vrE3;Gk^O@IN@1{lpg&XzU5Eh3!w;6l=Q$k|%7nj^
z|HGu}c59-Ilzu^w<93il$cRf@C(4Cr2S!!E&7#)GgUH@py?O;Vl&joXrep=2A|3Vn
zH+e$Ctmdy3B^fh%12D$nQk^j|v=>_3JAdKP<DgTp;^DIt;)F-EZ+0`lzs+uJ77mXF
zKO=n36r1cJ6ZeI&O)*O1QP@8JX7{q1%3ybU`ux1R!~W&-hs|$w8=(7Htli#B52kNC
zQ^VL@u3up6CP>t2YVusbNW&CL?M*?`K1mK*!&-9Ecp~>V1w{EK(429OT>DJAV21fG
z=XP=%m+0vV4LdIi#(~XpaUY$~fQ=xA#5?V%xGRr_|5WWV=uoG_Z&{fae)`2~u{6-p
zG>E>8j({w7njU-5Lai|2HhDPn<YS^8ehQhE5I<iRyh-9mT)mWhU$mn4$2>tQ(X@yB
z9l?NGoKB5N98fWrkdN3g8ox7Vic|gfTF~jIfXkm|9Yuu-p>v3d{5&hC+ZD%mh|_=*
zD5v*u(SuLxzX~owH!mJQi%Z=ALvdjyt9U6baVY<88B>{HApAJ~>`buHVGQd%KUu(d
z5#{NEKk6Vy08_8*E(?hqZe2L?P2$>!0~26N(rVzB9KbF&JQOIaU{SumX!TsYzR%wB
z<5EgJXDJ=1L_SNCNZcBWBNeN+Y`)B%R(wEA?}Wi@mp(jcw9&^1EMSM58?68gwnXF`
zzT0_7>)e<HsEdDR$NlH-Oz4z)4YK4*R4;r~#@Bf-qNf*W7N2rb$7_0t8fk5+&(AR?
zC<bDf1Ag^T@+r>p%6hid-*DZ42eU)tFcFz7@bo=<<wtaeOw^nO{PSb9Z|&*5Akk|K
zT2kaOkz6nI=><d29ZzfLPsac!Z^b$VBXIcmh|xmM@r6)Mz5F3JrGc)HCM;QKXdFn2
zN)F<RDq^r=d;jb2Z_(C<ZvTXP1meFPC<#c8lVpTKloUq~l@!Z@3NZ9Vm&6DNgWy6`
zT}ws|4y{dC<8Z`V4@yo&7N=2Lrz)W}q0uSpoVQq?bXwW`hPgX1VxJc#-*ykw$5f~i
zC|ut1-c`N^gj;icG!hG34qg7q@S4iL%Vod%yqRXM`*vS6e0@J#RM8K8!&}P;Cm)AA
z)coSDZ19_26LgSk3ec~JtbuRjAHdkaY|=S$lB0)roUPH^c%Bj9Hk?JTOI5=(QpBAL
z3sAe_Ulf+sv37C%@|)xs+e!7Qj4K_W!%anrHBA})sm;TbBl+0B6}Z)(%yc<S4D7=h
zfb2Z7*=sa~gIe^IJ52<6s26k2%UEB?h3dcZ_+%c$(i1?60)R)50crnWc*@*MJ7j?m
z+#_q#5p|jLUHhtmm6?Sn8@r$&TZ(yj%-c?;b-Jx|^5Dlk#go9U4yb7#!fULIqQBds
z&2I{W54>~CrLXpNDM}tv*-B(ZF`(9^RiM9W4xC%@ZHv=>w(&~$Wta%)Z;d!{J;e@z
zX1Gkw^XrHOfYHR#<ZxGz&lw$T6G0Nxnpuw>hAU=G`v43E$Iq}*gwqm@-mPac0HOZ0
zVtfu7>CQYS_F@n6n#CGcC5R%4{+P4m7uVlg3axX}B(_kf((>W?EhIO&rQ{iUO$16X
zv{Abj3ZApUrc<F~?Lk!w&f2Xtq>ar7Ck}B1%RvnR%uocMlKsRxV9Qqe^Y_5<iO7aU
zq3dedVPYS67#l$e6weB_b04g?APY5;b;`8Mge$lbj+y6C?EmJGf9^Ws=8R+H6IIt7
zKe|lohcPUhO~ppHQMA6xg5WT0HL*b(&d+ITYX&Vxg_jkc$c~ekjBwTWLMPMT{$RcV
z!!#NM$Q>C$xQW@9QdCcF%W#!zj;!xWc+0#VQ*}u&rJ7)zc+{vpw+nV?{tdd&Xs`NV
zKUp|dV98WbWl*_MoyzM0xv8tTNJChwifP!9WM^GD|Mkc75$F;j$K%Y8K@7?uJjq-w
zz*|>EH5jH&oTKlIzueAN2926Uo1OryC|Cmkyo<gXO1RvR0{t_&+f8HnilyXjJ;i5c
zxvz56DxWcDNyd~<XL{%_niD)o#)a~_BvNhsPHlrS(q>QZABt#FtHz)QmQvSX35o`f
z<^*5XXxexj+Q-a#2h4(?_*|!5Pjph@?Na8Z>K%AAjNr3T!7RN;7c)1SqAJfHY|xAV
z1f;p%lSdE8I}E4~tRH(l*rK?OZ>mB4C{3e%E-bUng2ymerg8?M$rXC!D?3O}_mka?
zm*Y~JMu+_F7O4T;#nFv)?Ru6<Jo9R8v(kHoGs?{2=?i%Iyi}0!42){}rxVvL!pJk>
z92r|old*4ZB$*6M<gaMZQRWjo$?vcL>40B;V&2w->#>4DEu0;#vHSgXdEzm{+VS48
z7U1tVn<u)EUy|Q1O0Ir9Rk;P;jjZ*6(#9^$Kg9d7dC=gOZg$lTVjzn?JP?E=U!^HZ
zZX5_RsBK=U`f|HJ?<gCGcIs3j%>#AnQ3z#gP26$!dmS5&JsXsrR>~rWA}%qd{92+j
zu+wYAqrJYOA%WC9nZ>BKH&;9vMSW_59z5LtzS4Q@o5vcrWjg+<k0<>28#&$*8SMYP
z!l5=|p@x6YnmNq>23sQ(^du5K)TB&K8t{P`@T4J5cEFL@qwtsCmn~p>>*b=37y!kB
zn6x{#KjM{S9O_otGQub*K)iIjtE2NfiV~zD2x{4r)IUD(Y8%r`n;#)ujIrl8Sa+L{
z<QB-&R!kciC=u%_Pu&BK7EGV1yUp_s@Qqpd^*Cf6xOgo*V|4$hGSSKA2WpaB33(Y*
zN`4+3BzbZ%lDtqn<HlZ9zS#|E)QFj}w@z>>ixGoZJ1K@;wTUbRRFgnltN_U*^EOJS
zRo4Y+S`cP}e-zNtdl^S5#%oN#HLjmq$W^(Y6=5tM#RBK-M14RO7X(8Gliy3+&9fO;
zXn{60%0sWh1_g1Z2r0MuGwSGUE;l4TI*M!$5dm&v9pO7@KlW@j_Qboe<WL{+D=0|S
z04rq}Cn#r|Qh`OLI1_QL5U}R{geifFF9P8T)vM{2=Wygi8p}p~9owhqk+?19n*#yM
zj*#{xPp0@cJDT3i?GOww`QWntOHm1f!c}va;b?tOb2%YIGP6RX6g6=_2i^T=N30Lx
zGGxIOD~)lgV;r;otK)h-PN}|RW44>Dd1k9!7S)jIwBza-V#1)(7ht|sjY}a19sO!T
z2VEW7nB0!zP=Sx17-6S$r=A)MZikCjlQHE)%_Ka|OY4+jgGOw=I3CM`3ui^=o0p7u
z?xujpg#dRVZCg|{%!^DvoR*~;QBH8ia6%4pOh<#t+e_u!8gjuk<TWg3Zfj1VT(rT(
z+;cFX>_Aic=|*H24Yq~Wup1dTRQs0nlZOy+30f16;f7EYh*^*i9hTZ`h`015%{i|4
z?$7qC3&kt#(jI#<76B<Zc8_jzF}-jz?c}RJj?}d~o|A$peyx&2tAof6rbF8`JJ<LA
zkx}wt?p2W^anecKr7&{Wp)yE)mY79tNY^xv{LL@QzaTPd04d4;6lK5S1YUv7FN_j~
ztrXYDtR<2@iHMgRaE>iz=bl=k<lj_Eu32rlOy9O^>=&qyaH><ZvCyu=u73u5Amj3Z
zlBLffeJyS(F;wLAij_z7LBu;aGHquG4Ecd?iDly_Tmu`)-}iF7q9@!(siy|rCL-T)
zMJ=;>foM#zA7}N`Ji~<RlOT@^-5)rOA8|yZid=(^iWhCiV9#ueyabSRKG4M!ILEeK
z@}E_xM=5xQ^8{3sm^736?!4juH{#woOSkhU{pHmMs7%U%L{L7L?9}ZSTh126kqS##
z7l>)-f-t&tR4^do)-5t?Hz_Q+X~S2bZx{t+MEjwy3kGfbv(ij^@;<DrdI3T`-v~_9
zcRbkI9fdmuaS>=?H_^FIIu*HP_7mpV)NS{MY-Rr7&rvWo@Wd~{Lt!8|66rq`GdGu%
z@<(<7bYcZKCt%_RmTpAjx=TNvdh+ZiLkMN+hT;=tC?%vQQGc7WrCPIYZwYTW`;x|N
zrlEz1yf95FiloUU^(onr3A3>+96;;6aL?($@!JwiQ2hO|^<dM|7%UlhRzx6gJV9|=
z=NpG%ffNZg)Ud#-8(4E`FwI`_0dIbwuD$x#G#SyFPIa|i#MJAmCx}gJ+(iX-<$2`F
zq$b)!gO0}32UorvmcjGcs0oMXeFUsIgX?OLA?6&a$PuFr{46GnR!g|y(}h#A`6l)i
zpT!HijVb^7e>i)b4pCJ7-y&a~B#J`#FO!3uBp{5G<W@uu@K85&o0q~6#LtppE&cVY
z3Bv{xQ-;i}LN-60B2*1suMd=Fi%Y|7@52axZ|b=Wiwk^5eg{9X4}(q%4D5N5_Gm)`
zg~VyFCwfkIKW(@@ZGAlTra6CO$RA_b*yz#){B82N7AYpQ9)s<e%0{=46^?9=ZEQ-l
zAQWxGXxNdHjc*vyWIT!33B;oq*FC${iQp|GUP9ZdljM~?wy^$O5X<tj{P6elLK%T5
zC#svA*D5wmh9fQFpm0}|k!**l^Nlj<UP85>LQfhOAOMUV7$0|d$=_y&jl>va$3u-H
z_+H*|UXBPLe%N2Uk<CC2<5os`NfNoGP!Ib0m9`%)&9#>wu1*)kt!$Y>(IH3`YbEt;
znb1uB*{UgwG{pQnh>h@vyCE!6B~!k}NxEai#iY{$!_w54s5!6jG9%pr=S~3Km^EEA
z)sCnnau+ZY)(}IK#(3jGGADw8V7#v~<&y5cF=5_Ypkrs3&<giP_=TY>7{}%(4KM7)
zuSHVqo~g#1kzNwXc39%hL8atpa1Wd#V^uL<!G-fTwDKo?kirzAqPxES@VyF*HS5iQ
z6u^SQDo@?kL(sZYU;lwg?P>=W^&E)fvGivt)B!M)?)Y#Ze&zU6O_I?1wj)*M;b*dE
zqlcwgX#eVuZj2GKgBu@QB(#LHMd`qk<08i$hG1@g1;zD*#(9PHjVWl*5!;ER{Q#A9
zyQ%fu<$U?dOW=&_#~{nrq{RRyD8upRi}c-m!n)DZw9P>WGs>o1vefI}ujt_`O@l#Z
z%xnOt4&e}LlM1-0*dd?|EvrAO-$fX8i{aTP^2wsmSDd!Xc9DxJB=x1}6|yM~QQPbl
z0xrJcQNtWHgt*MdGmt<!{59BQ@6R8^J{W0F$k->j%x6SWYd?uGnrx4{m{6A9bYx`m
z$*UAs@9?3s;@Jl19%$!3TxPlCkawEk12FADYJClt0N@O@Pxxhj+Kk(1jK~<na;r+|
zm{2yx7Tja~_uQ<c+6H|8mu<-YUr>laR0*KGAc7%C4nI^v2NShTc4#?!p{0@p<m@p~
zP+-C1vUzlE>0T#HSIRndH;#Ts0YECtlSR}~{Uck+keoJq6iH)(Zc~C!fBe2~4(Wd>
zR<rpD32Yz=|MVv!jF0SQWXAJmWFDV0ka3bIedmZ8+smZ53Ho~&j~&2Ke3DNyKLA+A
zbJ10q_<tT*Npv&sy}xR;{lmk%mN;2WFCXUz0F+vcRuJB#5+NyF41aEUQ~yy+_{tj7
zrTp6j;lWAkEZhiI7QwVMiDXtF6(d<{RD?!c&lT5iMqpaD7bA4G&{!+@%C<d+c70Wi
zU;*do=_MRGv5LZbeY<RwC*Ablvqx17uLc~M%ux><4I1zMeW$<0xww(@09!l?;oDiq
zk8qjS9Lxv$<5m#j(?4VLDgLz;8b$B%XO|9i7^1M;V{aGC#JT)c+L=BgCfO5k>CTlI
zOlf~DzcopV29Dajzt*OcYvaUH{UJPaD$;spv%>{y8goE+bDD$~HQbON>W*~JD`;`-
zZEcCP<ta~{4Akw!C0ePB!7KfHnqyMC?0a{DvplTx4je+~_=&7<z9BH2!>SdlCvANe
z=?|+e{6AW$f(H;BND>uy1MvQ`pri>SafK5bK!YAE>0URAW9RS8#LWUHBOc&BNQ9T+
zJpg~Eky!u!9WBk)!$Z?!^3M~o_VPERYnk1NmzVYaGH;1h+;st==-;jzF~2LTn+x*k
zvywHZg7~=<axHmC^DNM(rrxXXZ+6q8kDUy#Wo11Iv58)OcYVE^Pyh_LpKFMxqfz^m
zX-?%QT4qvLZS`UJtdA|zc7nI&w@Ai(hyYF^v>aiJe=OhS@U>1fYGvT1+jsAaiaM;)
zay2xsMKhO+FIeK?|K{G4SJOEt*eX?!>K8jpsZWW8c!X|JR#v(1+Ey5NM^TB1n|_40
z@Db2gH}PNT+3YEyqXP8U@)`E|Xat<{K5K;eK7O0yV72m|b!o43!e-!P>iW>7-9HN7
zmmc7)JX0^lPz<DsX{e!8YOmOSO*e%?uh^R5cfwUMA1cm{WQM4>F#>$#D~nU^3f!~Q
zQWly&oZEb1847&czU;dg?=dS>z3lJkADL1innNtE(f?~OxM`%A_PBp?Lj;zDDomdw
zoC=eKBnzA5DamDVIk!-AoSMv~QchAOt&5fk#G=s!$FD}9rL0yDjwDkw<9>|UUuyVm
z&o7y|6Ut5WI0!G$M?NiMUy%;s3ugPKJU_+B!Z$eMFm}A**6Z8jHg)_qVmzG-uG7bj
zfb6twRQ2wVgd)WY00}ux=jqy@YH4ldI*;T^2iAk+@0u`r_Fu(hmc3}!u-Pb>BDIf{
zCN<?+PO@76t^M~>DDv_Ko`U@<h{fuj6s>})TZvuE=#74~E4SUh)<>8kxZ=7`E?#|c
zdDKEoHxbEq;VVpkk^b&~>-y`uO<UK(6@qDtV}WxIHiDW6Fs=Oj9`v(O*Al1E76K`*
zJpPUt>~mX=X0bmP!=F1G1YiluyeEg!D*8Fq-h=Ny<ihcl5CnIQP~m5~*+=tV5EK_@
z;F@I)hdX~?{m>E-2S;^F6j=QMtU<k@K}zH2fHq|>zN4oPedvc*q(BCpbg~*As!D@U
z3(sz|;Pe1hn08P<o)tz^6{_0K@t@xxURQdhHo}v;@M76*{4a;hniVAv>_cDQ(klZ6
z;P`q(5_V?*kJYBBrA1^yDgJD|)X1FV_<LnK&#C2?lx^wxJ`?B?Y+;SDXi8|Z0Waj`
zkfD4-F3*H!jF054^gZ3S5xE`McvCD2myEl5(og_>*~sO>?8Sy~I9WdK5K8bc7aeNC
zDb{Fe>y3N^{mrD<dm^3E&Vl+mCGGvrwso}I%<(WEzOn>1+GyH{F?@9}YQ2Om3t`nt
zQ(}MS8M?6Vk>B=*j*yibz6QCdR=ALgTUcKx61){O@1WkPp-v$$4}e#KgK<N~t~;2S
zxPkz4<?YE$L@Ji%7x}Ycray*%EZnc8%G%R=;^20vexFLyVE?wgrEWYm%wFmDv$A6S
z;FnvyM^ZAT9vH!Czs1MO8Y+9fm2`UD@{gu@{-iUlqi;M)NyA|}D)n;tgni%*-M87Z
zrxY`r-uq#uU7xxF<)32NCL<X)rw|)2EdGJz6tDbc&K9%qKZBP_+AxmxpX40;Pja4Q
z#Q)3Yi(-62#WR+SN<#jF3{|OO`hEJ467}KOwM+62ek}8w1%{iKORJ1B|0HMs_q0vD
zgkrRE=D$>`HG~2@#A?`BF8em`ah6+8hH-DNA2>@02WWk9(fzhL_iz|~H~qEViQ(*{
zV;3tjb<%&r!whm6<KTa;Z&Svx(I3Dl{C=o5o%icCS*ze|9kyEo81pbg3aHBh$%tqi
zO^d2^A6vdf5s7))JlFz`+M4&D?K_s>B`XtWmmrMWi=#ZO&`{h9`->HVxQ)^_oOS{W
z!BzVRjdx5@pCXl#87ovlp<^QU;s<*d$)+|vI;Ai(!8Tjll^mi6!o~CpnlgZAK>6=V
zm38^kT`D$_$v@UYeFyVhnsMZI1m`E&8<{V07>bBEI1=fg3cji*N?7pBzuamD`X|^^
zm!)<lrp3t)f;D+J`&arUA{h~5GAW}`YKXM7cbi&qt^^%;RV+@eSZs(3gfwQ7Il^wR
zgw2TwPg8s@AkmZuFhiEZGYj~&&!a1uTKY~SC|)bqctGlhtwgW3iBK?3h-i(2rwJ-P
z?*!Vj6Ko>2v?s|6T&H-_^y`KM&$!0!9tai9x&)5<(&sY6B`3D{$$KMAX3@&`SW;X0
zB-}obt^I;|#o_bR>eOv?P>=UC6CGTXIM+lSu?Uy+R9~O;q|c2+FafBP;E)B5M9H<q
z36O%f4We9cD!RaEFO2QAjn8OEs)0DsupfZKYPw!RZw5F4nwyN;FyuIngDkb`=$?&p
zF2TFkR56Z+kSzToYD%o`U(%^~PP4^++$qOD#N>JgRIpF|GvRi*E+JTBI~T?T*X}r)
zefUd*(+3n_YHZZS(g8)+7=pNV9QR^>Qs8t+iEpbJS!9;wio&9rn=19C0G#<yg$YZL
zjccm%L?k$q_x@t@P<lcl{xx(@SVx3sFpyo5sk5J`Wm}+FtML=sq=Dtxrr6KAaO>Ax
zM-tWHp_YlJ<b?O=20XtJZ>vXWsUqJUr^`OYFA4wkgL`cSOV;w4?tp>GT1jq}-qPoN
zp&G}*;+#+Zh&vqDOp>gRL#^O7;s2yWqs+U4_+R4`{l9rEt-ud(kZ*JZm#0M{4K(OH
zb<7kgkgbakPE=G&!#cNkvSgpU{KLkc6)dNU$}BQelv+t+gemD5;)F-0(%cjYUFcm{
zxaUt??ycI({X5Gkk@K<a<*;?zZ`*hKrsv%M>IR$WCqy4!wkeO_j)?O7=lFL@zJDfz
zrJJRDePaPzCAB)hPOL%05T5D*hq|L5-GG&s5sB97pCT23toUrTxRB{!lejfX_xg(y
z;VQ+X91I;EUOB;=mTkswkW0~F$<oE;TsR91qN?zYft5oMd;^F{?x@|gO<W5Y(KiX>
zS%M}ATlKkIg??F?I|%gd<Xbr0$vQ#Yv3PQHLovgQujPQ$nUr)0Z1GhgO9@*A8ojQY
zwLpIXJ6Rf28rhQ@9`5ko-@kq-`8I^6s{@I++iX26ONTi7T`Wm~*~68Lek!um)dW3m
zNS!FQvGZPuTH&*5?PD%RWsc@+YUnYBa*<q&vu&J}_@HjP#B<xIz-IPr;ziEeNX%U7
z{2nUhfmJ|g>YBhU(h$LqkhE!Xx$7kPS{2U4wLujF_4O+d8^e<sEgeoOW+LM*7!>j{
zgSo(;vA)|(KT8R_n_aQ$YqDQaI9Stqi7u=+l~~*u^3-WsfA$=w=VX6H%gf!6X|O#X
z*U6Wg#naq%yrf&|`*$O!?cS<s6ao87Ru3$BSA77uB8lfJ%#WNb3^|8yl{I@3XkrI(
zJgt1sZOB!FTT<0jS&V`X5%1ab%LR!W%{s@SArDPyD{m<hnQzx-(jw@%8EE|ul^M=+
zesZFnS+it?KOjMwnUHL)A{*=WZ=sf$Lf-W@s2go-(8O3#by~pGFY*dX2h%ohQnzfs
zwIjfgmcvtjlv+XaW*8-pvY!WwfkeAt`S3gx^{M%O*8CF6VJcqpTMV)x8Sdsu>94GD
zk}Gx%{UU!kx|HFb+{f(RA2h+t#A!32`fxL}QlXUM{QF3m&{=7+hz@aXMq*FirZk?W
zoQ~ZCOx>S?o>3`+tC&N0x4R`%m)%O$b@BkW;6zE+aBzeYi47~78w$d~uypaV*p$kQ
zJf34Q+pp~vg6)yeTT&qWbnR2|SifwK2gA7fzy#W(DyM^bdCjnee42Ws>5mM9W6_`j
zC(|n5Fa&=MT$$@?p~)!IlLezYa}=Uw21^Fz-I#?_AOk(7Ttxm;#>RDD_9EloqhvrS
z&7fpbd$q_e21Al+bcz|o{(^p}AG>jX0B}ZZRfzk$WLbNLC{y|lZ|&a(=bOE6Mxum{
zM=Nd+-I2A-N&2giWM2oAH`O&QecJn6%uYl0GWlpx&2*)BIfl3h&2E(>#ODt4oG}Dq
z__73?sw2-TOWq@d&gmYKd<iKo&=r8}Zr?JqOn1s50dI0cNjF}-c{?&Hz5cB3HhBX)
z${^V&RtR{CsrRLf!?KxfOlNemEhWK%QpeqxD-?X>h`a}-_6YQ5```}bEBEmWLj))O
z?*eUM4tw0Cwrr+4Ml^9JkKW9e4|_^oal0*sS-u_Xovjo8RJ18x_m7v!j$eR@-{2(Y
z?&K4ZR8^T{MGHL#C(+ZAs6&k}r07Xqo1WzaMLo9V;I<9a6jx2wH2qeU?kv25MJxoj
zJKzX`Un|;_e&KY%R2jU~<5lm-`$EjIJLDP~11_5?&W#t3I{~+0Ze++pOh2B4c1Mde
zSgj$ODQQm7gk&w{wwfE1_@V(g!C=2Hd%Gwj{{-_K4S|nZu+vk}@k(?&13iccsLkQo
z<h;ZAnp{#9Ey_QcGt8aIJ@N)XMhg3=bc~HZs<28<6Vcv6=C3+=2(xal&%ow9r;}2d
zw|9bJBQ1)^12O!e@-UD2#$5V0+%zZ41tq8UxaP75u>_t8#Ah$HVB-MRyzpab*OHOp
zl`$tEcUcF9_=3*qh8KTaW$znGztA7Obzb`QW5IQN+8XC=l%+$FVgZ|*XCU?G4w)}!
zmEY+2!(!%R5;h`>W(ACqB|7`GTSp4{d)eEC8O)Mhsr$dQG}WVBk$aN1->sTSV7E)K
zBqr;^#^bZJJX4E_{9gdPo8e?Ry>ZrE&qM)zF5<O!k#@8yI;8@YdgAaHwRYo~PGeSl
zDTThdwE~4P^B}RF(%vYzE9v&J87|VRsaIA)vPz0V+n?~Rvo2gCvUCr`{FGAV$EOE5
zd6pw_Dv!AP{mhLin0T_el%J7Du$u_Ba*4jz_;My<R#<>z20DP0`)IIm_!vm&s2mzl
z2;EPI{HgFH-Mp&fIL^6f74>19^>o^AOj`uyL0+Nb##Slvi9K4LQSs>f+$j?cn9<mj
zieNNixK}$@qK0mprFZhLE<IH!gf50ZH0vp@`@wp=8egh_Ki4E(AEGq@WB!b7>Z__C
zAkyZ9C;#uRi3cDYoTA>AT<|*pt{K70oZKG*S1F$r?KE=$4~W3!u53yUvh~(kMrClS
zXC?Dmgv4iS`>~wBPJJFL_C8x2tEg*PCDX2=rHQ@z+Zs)Kkr;FYG`GnbUXqdipzvHE
z1aZ>G6|e`}Q#)Kru0)(SZnUCN#d<Gt44{&d27e++`2xrdaX0E|gVQ)W*}}Ox*>N2H
zd1}r&xGs<zfr+x^()^{Y1UJ;@R0%&9u+-$3A%g|foWzAw*Xx6f<iTqDhPaYlD-f$r
zP5+@uvX}wg;FkC+oc_!X=-H#zQWteppW=;SBPa8+dJ8;gvf{^-k()KxbwcKuI&8_d
zke(`-RKUK6cQ<(gAqjK}ih<cg+h(dOn3q(C<GUhn*Qy8H;`i5tGFNSiHZL^iu^kg#
zHz+UYQAlWZo(+G9+*gFHO+o?0WeER=x9#8D!B51%I19%|084P(tMC%(Gyg|k`d~=j
z1{ZDwN9VgbuE5!s`whsOBzPoq^K%~Tt1IeS2GH;YZ%PKW5uI9skV{dC0gL7A^46Q$
z&z;u!G<Tb1L0VzlNxyRo(7zVSnVx85$a08H;RTXg=h~E4A@dDeUxQ$&&8GQK^+P@m
zijtmTokm3Qi3S9@y0D+tU4q`pcFY&-qhN$2V^sReZlV$MhTOQ^-;G5V#V8fVG$m9S
z0mFAL4Q7QY`(-+5F_)y?7Ygm-#hET&({7hnC@aivI%RzUIiL=R$8~}w#`^6%#4M37
zp5TrNfdOoa0A)AXhOhSeiLbPJejFy)wgI(1!pDWs3Ar<nrJDH4mtXz?&u0dGh1n~k
z*v<c+6GwM~>aAeEed9#?|0HzMGA7pl2=aehy_zsRV8RKV6+^I8woDd%4J8v9hs$x{
zl*V61wSumovRVWtetd1eJ%i^#z`_~~^B;aeuD`6LgHL66F0b^G5@om^&_3REtGmhz
z%j^9{U`BH7-~P_>c_yu9sE+kk)|2`C)-ygYhR?g~gH`OK@JFAGg0O)ng-JzSZMjw<
z2f&vA7@qAhrVyoz64A!JaTVa>jb5=I0c<PFnWwv`<3{7CrJh8Oo-vquiY~b_AC3Hr
ztnQPPhJHDMi{*c?GS?I?uf>bRuTv;gM<a&j)iA&{?jB-W%9UX^AEu}sZ0Ki<g;a+D
z`yLgTENm=NL#_&OE84?60w36OPG8#l)-JaIn$Y+*&ASDRGxCRu64+E;_dN?7+}vQn
zo7^7ZQy{(ggn~3*ZHLde$7_xJ3c8hkF1@T3G3ER(XBeWiNV__sSFjj(BP1!CA5Uij
zf#twm-!Rp<s+~20Ez`ReuBI7LFUnq>F@4bX3DVV#!VWZEo>PWHeMQtU!!7ptMzb{H
ze`E4ZG!rr4A8>j2AK(A<!A`>0Vh6mNY0|*1BbLhs4?>jmi6fRaQwed-Z?0d=eT@Hg
zLS(%af5#q%h@txY2KaYmJBu>}ZESUv-G02~cJ-(ADz6u8rLVECbAR7+KV~a!DI83H
zd!Z(<r&PayxII^4gN5aNhUy5@hr)bsq97X4US^3#=|g9mK!&-LQ}73Z@xN=ztiyWZ
zm2+h7SAr_M-uvK%yY%p|IUQ@6z<nr*aYO>Ekz%vjA-|%4-YpgfymMzxm_RjZg%ruo
zT4^x)f*%Ufvg_n`&55cK;~QChP6~Fy_Z67HA`UtdW)@$Xk-2+|opk6A@y0~3Qb;V%
z%+B@ArKl|<U*EFDzaIRPp3Z=;9bNVOwbz;xij3dVU~8#|?SWYWSfh(t#&CbW2=6I8
z1hX{0V+Nkq$T~nCt1_8FVM9dH>Q^DJW&xuBZD#~SurH7XXf*uE0@|ccNd&MA%Ts*1
zg7TU!xY}~*AOY+tAnFR(Fu)e@^9V!Rm65$;G$-?6e%7w7p9WT098%-R?u#J+zLot@
z4H7R>G8;q~_^uxC_Z=-548YRA`r`CsPDL!^$v0Yy<^KSoKwiJaCt&dlW?p^7Y_<9c
z3n#c<y~)c7^bffE9+=2-yiOta15ir`2n&9G&yycmS$~yT349ynm49zE(u~H+VJl7|
zLW1HXM2;<+gpfla5MN20$WE{mf^uNS(%2S}G)6}f(-yY0w1qAUU3!JGJ?LS&rG?T&
zHn<Q<AwWwn+Cu5w?M+X5vUHd9|IJ9YWy>MWFUe@W@4ffE`}pQduRZ)I5v`G8On2RI
zL)V5k)PMBq(Zfb6Ruig;_SMwaM9t)2JfUafW-6F8V+PjKM#9iD1~v!uOfWiNL=R_j
z$xKbCPfuiw`kKN1U{W6p#s!Vo+Suw#*7O24y`hNTmrEqDkQvZ}tMO{2`r|3XNXJwC
zSUqB-GdK(D8yYTd*bs~vM{3@r5;JMtW-c8ywtvPG2Gepg-QU=s)?*2y@n~8f95m96
z+pO1p_FIP@Pbnlb&AnDXqBkb=RDa{H-fN9$Rv{OYoWwrU{J??m#C~^HFtMrjN~Spz
zt1SsVlTk=x^7b3q-DxumB4DxAv}x1?YHb=BBbrOcvqOzjVK#ZlL$frhpxI1I&JL^4
zTz{rnIH(26vL$9Zf7%ffyC7agUX3bg9@D~^pcI<VQ~|PKGin(ViYXFJ8QI3YU11Pf
zNIs1g(0NRr$vX<2&*aRSd%N1Fp|t~T30<JkV)B=wwJTJ^<i?rZ@R20Mm{&xdtZff1
zrxhA4qYGh`J{VV}K#A3CIE7X*xs2;_@P9F;>Ogp^SvS@0_fS0rHL9Zq*vjT4ZZ-;<
zjl1>i0E~DMlLHLFe*&dK6lIzW57ySu#Tu=qwM<ph_Z3<Pma?Ypk>h#+h*$yk2HIFb
z>nT*!OJPT$OPLhmOCaK*%WUy42dzuvsd)CXDdLTLrH7iRS)<L=Vyktxo;Ffh4}Y}@
z0-I^8O|(-_8XnqiufblfgALc}>E$Zzgab4TrcDG#Hg058>HuG9V=$qMph{<;l?`Ri
zEyGDUBkrQzLi1NJtvoj(mN?yl$vw8i+u{fXdFV>oD0cQS`6mT>G!chOCzE<A7Dmbd
zLdHld6oF9#HoXHH?WcpVzMe>!M}POG4yVkcsa=D@;o&t554oCp+<>_TZ~ZFu!frP4
zU=Fl`17;Hbhh*q72kj_XUp7O8XXeU24I1gAe!<PaS!J-H6}lF-(9`|7Bvi0xXb}EX
z8J?1YKr%r|HytLh#n-7ch@iYK7dNshrLoeg5=>Z;8OmghWKbAdr6WwUEq^k(Y&_8z
zj%SeljzOqyBkQ*T{RNL0@|%7B?116lab<@;U^MhM_=By<A+ftTIdnu%<c!(|m9EDL
zF(VUA#|!$YD(4Juq&EpZZz#sGb$U}etq&mv{|(yJGlCbU6(!m&26DKRWe~iH-lEZ)
z>8;asX*oe`l13GJ8<aOeVSl@x8GyJ7y#ra@wrZw%$YvFe5-x@8&<Anct@KXeB>z5*
z5VjTi4+vl>1TM8OFqzvHGm)^9If&dr@6zaY`cEcbpgfH2v+vgE7J84UMd4{&7eL;p
z(c9_$OzU1R7?w91eP-GY=k8o@VPB!Un6?GZ;<PsQrl}Cqlw08EZhwtFK>t-tik9u#
zvqoC)70K;GOln-bWzDpZYO;db3+qtNN9djk`Y?U8NTp<7p^qb*p}pudj%BUzM(7UH
zy%qEc`XuT^%33b1Ck5~E(5L7=0rzR9`q$N${pil>S#W+o{57c$^%{6jXLl7mylgTC
zJD;ToHF|(P$0P-VDu1113cl`fO??oskdG7^5dm<nI)6x`QF<6!2?h^bz4=g`N}tC|
zQqQ8eW<2z$;HHjFp)_Q2y_tf0R|ne5X6rA|;{y7N$h>B%MB4r5SOQ*GRGZ)={o>ds
z>9kPUQ%r0Ab$o@MK{hL}EBvA<4GAv_oC7bVTzr|H)#yv~6@O3*T%M^d=yP+!DwVzl
zmBv#s<Jhh~co~$}?xAncb3#|&6u#U(lRCdGK%PgFYNnbBX`HFKoag+j+<u2%6x@Cn
z1;XN1Fl+JpePplYs0*T;!<|~=84-Um5B<<CaX&(;E?@)?351^%`$LPLrXprrIxPb3
zryBiCl+KE%nSaWn5zXv0hCK8O(NNt^zhqi4Rg~sjJP{N1_tz*s2!P*c^pa>zT%!L@
zp@s&_ia!GxNcwyFgCOxo<d3XrZ+hrYqLF!t{tRQGM?hLEs@Fq*rF(^D{$^<%Y8*@$
zS@aIC*yjCTaFW1mksYw|2MqZS`afZ%|A&lbs?#j=1%DnoBQ*Q~grK<=ouWC~QZWxp
zqNjL(9i=U)RXP=R7CM*lrfO<InAq9mVTDzVUF;SLg`zJ#7w?#s4JO(P_Mj^8ET#+3
z4M*WQNC@F61-VdQ@8C)|&*iFO<6H`$@H~yHMXu(}DleEJCyH2M3=nmU0fJP<(BK%%
zO>HX+X@7dgvR{(Rc?n~*xScUt%qyo=g)w5da7a@kfkHC5f{IFx%*o4ng~rPm)5Yw;
zw2^`5jQ4|6i@zwi9u9D=8;Zrap%z2I!`5JN3kOAh$h0K~vqK(kg#U3hW2TTZ@#_r_
zuYrSM;o@m|cf2&M;Y$Pr=7tL<l}<DS{d&R*_<y!zQKMk34NQw{R{bc1{$wtZjl;$L
zcI(lga!BKiBH%74K*dZW<4>7cfFCjZdTPi91>|OQ<ueY<dyxmWqwfuy=Hc9+!Wcvr
zwzh3;+8f@}v#qPCHQd&-x4W&YXZx;>HV-$Uwc{<^Jl;4rh{n0WYMi;%o-qsd8G>t`
zQ-2D8(zo(95gXe<cIufb?|_X#)<_-kaM%XK>{3}cf6_?9yO@>*O2@DnMi0IM0|s|7
zttz7!JH98}Y&!xefmFwP>`Q>D`_oUYE!S7_mAp^my?hl~!ZN3Z&HjFI$bM0J_S;+@
z)c61&5|i&S#33B9M<l(?jQORe42i<!M}ONfBbpO!6!*gSMY360Rx49;lzpb5ag<U2
zTyX!sIBIN3R+@5Div^QQ>vme=0gk(Yj(KKL8KhQ>V+m7_DV!+plI5r>jJ{+xCiSCc
z`tY83(lA9*;dT!X@^x-D8ExhQ@OlJNOt(y3U<O&sS}4mojgySIX-%M<)V7T_m4Cr)
zx0QEHpu_XeD}&sg%49K^D*T4hIi|QT?gK<wPWD>P_9ldOS+k8hnRVig8sESest%o%
z;j<W{>}Clsg_Ca5_>KG)G$OIMXfS(ocFQ<>%6$;u%x@EBc{_~MsPZjH3YcHB?RH<~
z;dk0a0@D>EH({DmGJ2n}HyvkMGJnIh%sA;g_+3K57^-Gv&8F^__Vz-f!0)!MQ5b`i
zqoef_mEQ*sEWHiuFftjv-)N2Z8=|Bgx097+l$5w-TRn5KDo+Fae1PxP_%6mQq=HuS
zP*%8{9H>3e?BNgbhlQLUK_uk{V@U3p*8>NdMN#@Fe@vi#yja%I#t$?$$AA0VQ(42x
z0mDFwS%-M|lb{3O|He|F-NJ`0?$h{Q{<P?i9v8pv7fll4eC}k^H0v2ur3XdEHW_n@
zvb$^7j<%LPDxaJJY28Hehz&Uk$m`>SHul5z+L*m&!#!fJJqj;3jztr>O#Fy-E!z~0
zLOmUN3K~L8HkR|Nwiywi&40)E3vRgB<4otz96rleEBpjg`mCW*>Nn*WDNrlBS2nlV
zdOxl4ll+uzZtGeG6`^DdE!@@cGyElu6#g>Yp&=1HtTN^eSMqQSqq&E_W@quQ!v*8$
z+|%d|%rshx=j?UN8s|+=?8>FG$a<4ngKuN*X)$w&m{snhX#>vXAAhv&&-}3>HGiL(
z_9x8fVZXSs^sD>=(;RT!)SEFAxvXK^@<r>SkiV<(^P-nfQ+mo2Io4{LcX;>*{6kT1
zf8-?bXHN4L2l2NaD^3zncNc1-nY1lw-EQ*FFcGJZs{9L$e=aJlCR8<`r&0!z{?fpt
ztJbK!nz3wF0D<a$&3`Wm)ZZ{IJ}aogzk^$5(?elMDU!!bq{Tky{(=9f@$dQN(n<aG
zJyT(fAC=b4Kuf~R7*~i(x5UuIl?~3bHpBpCHJyLuziIpz{(H&Pux9{<Lg~X~;>;ur
zV^Cy@9RmCxjK=X*#$+N#;gcRdLx}GuB`W$sS&0-$g7}56F@GLO#-t)SB+Mj^M7&p(
z6cp|<Vm6y*7J;u1iZ3MAERB(KwIaDSNfQ~woj_g?N5HddOJgjhCxvpdq97cYa@j3;
zq*-tzX?Af^EPOk#!Xb<*12sx>#ig#l@GT+ik-Xx2!!l_e8s;ehRK%E%3_0F#P1+Hc
zYSW_5-U2TRC4ZkLEs)OhP@Dbhd?Cw$($5_;U|V4>EzzV(=>k+4<b4rVw(X`~IoYc?
zaH&)S14_#vNJf0RK^i}nEn)Y(z|7}KE2Ikr`bzjPEY?tHY!iAe6&*lxZTIQIUaC|p
zECkF#?pg6V4R#?LNp(`a&;#ZdZ<+gSMQT(ruvuRfI)9ApfC)*rl%afJ8s*7nrHwvu
zv)O1<z@clTi#4i{)|T8mltL+z_=XZs9Wf6h<!mf7D9(IATryfo8>Eezv|b9qyP_f%
zJ<_EjASxvcKW!7qG9kWy8P-j=tyX_g&Hf!tUH*8gxIDQ$`d6;VtZYyv@r?#q71eqQ
zuVwU8hJV-Mv?Dc1&FBmy<lBluKPztERjEyU>ML`_H0h2++J;ImVNPoF!}q{<%zspm
zX8~m8`|*10*R2fZ&ze^H4}rQEqeM{`zr#4%AJ6!6_9qfm>cr6#TEf6N09|0P_S;v9
z5PmmirL$iSA{@-4#T<wr>OxVGx|!+=_0&Hxs(;xvNvL&VY_&!l9JH6|vKHhzEX6SO
zrIYcL;g1S;8$`*n#4IE;{|-Iv?@OCWf7FZ_y^yVFseR%m<}9p51Z(??En=Zh=pMqj
ze{7=8N(YOdYb_d`rseakM&DL5mx|f<X42KgD-#Q55;8_KS$u#+)fJa8ow;PMk=`SC
zpns+0B*iMIh&fX6kELEB_n+~rf=KmNIrB6pPt|!^l&7VUaGq9H)#j=3xbkA%C|&Xp
zZDI-!Q`NpP+8pVq%hQ$-YON2isw=*4!@nI>;i}F&b&b&8JY8k~4Uf_O$iai1BXmeU
zNxJh9s*6M%Rncy_%IMBhysGXbnZ?!Xuz#8ntNV&8IjkHNE0L-p09L)>B;7blH;>WV
zBO!T=Zixg>&~16TbA;YILdVDG1Cfw3=#xk2gAdWim_ja}>mfoTdz?@EoZ|Oqm>vV^
zkdmhp$NA$vr7ADPq{=ZG1+G9H8$R<a)*s2!W5>w{GzH3e!l(4)>FGRuHRK#VbAKQ9
zzi#a}i2b>n^YpEC0<apOFW8{fPWc+^*AsdAo<KSZq#syFet{%Ufcg~I1laBe>Bo1`
zLID4d1?(E8iZS|GWQ2ZxDhM<{hEz!HQ}gtz<1|mu62FVQ%?%c4hui|nZ9%=o=NzM#
zB0hId)o(}WcX@g_Pk#}6PebTD{eS&9d5ePDY`pf24==BVoX&M>wd#YqUc2YDlRjs)
zDqkZctyV2jL#jnqEg@?&^J)knJ~ada!)H#xPI@V`uZmNmGxAjcXcicGX7PKSPX<#g
zkFwS|Mz@3W5w57p<$3lA_U3v1gte)?#MWM3nCC^2b?V(zDd>55ah{j%8-G6YoX--)
zr#PxrA&nwmQ!ur){W+f;35p|ERz-!Lc=o;%TqhP9j#IY}4!Akwtcqei5^`BQtd?&Q
zK4HJCl|M=ggxlfGk>~Yb22nFi#u#smczM$ZUwX>^d71e6Ah+!Ea@#1k<E+zEEkX+-
zIf3JIL4FE)dFLp1&agYjT7PkZiU`eMM7-8{dG`o+k8%Vuhy|asWQ7kzIw51#b%>^-
zbokLQ!dK^6Kkj&9jH8iA{TMHcjBsp(`%m!UjxkOGJXn8%GqA)cAMF|8>&N(wkq$)O
z7~cSr&bkqPb8v*;3iwFp34Vv5Pg}sSmv7DUZIN}#-NLbF`&`ww&VPmNynK6cPlHU#
zFwOG09My_tnP3EDM)}S>zc-|M`Te8(!AQsrU*dc6{E0EX7fvLv!|SK2RWS6Kxy$qX
zfaO~XUOx-Z5=Ya^J+_a96k$B|1fKvE=+#OBn$H<>55q^WVx<M?J#`^>(5L#`f>KZr
zI>8T((-L7Jh(V!(nt%HQe?Ah@iqzabXIO}+6^X5^_qppP5js^$sPNM@PV)qRag3jg
zgnbaxC)Y!tPv`krD+Nb7M37unh#gD59TthNj$>mx(wXOP+(oN{!k9D*k8fG|#6QN*
zM+9ztkC(qA;*P&p#QXj!?&J_+?8o!?CrK~=^k#j%lS7J6d4G!b7FOpw-+ec2ALE}#
ztl;`(JvjJPo_}k3(VrrnPtg*DIcU6szm@d#&7=I<Fljq|O6M@oM}F#ranC%kbC}&e
z<urY#E-x{B2U?I^z_5;g<uTeEwneS;N~#b@2vJ*a2~kr&CTWq55vig+FU^T`oTj<;
zkIEk=t=_SwV}F>O+);m;_KZoDk%M7CROO}W4*3yU9C6flk4lU3(&7=xKPoN9$pNpl
zDlau)w;~dDc%_TFz0zu|UxF0{E33L0Z=3ezrOQ4m^kyyZbkqTC%c@bSRj6zl^W1r=
zsACw%D{Zxm^V7W4?v-{5E4xcnzA9MM);O9^>+wn*c7IOvO1mat#{t|k0PGYHUg?Te
zBhsEzlQ^yi$5$3Po+8Or#dQlAm<LYLJhTh5Rkw7A7SVDFNImlX@=4`cNtZ|EFIvxM
z<QK%#Rq0yj$`QF%x*DwKTGx8(xyjXHJzHE`T?U@*uCVpI!gauU_P7k|d4uaF<#If4
za@}UZ-+$q{Q#_T+l^xczL)mRTyOfyq>{o6SPc$)6{MSG`t;S{}Nwk|Bw4Y=$(D1~`
zMMG$NZbZZLE;Ks#kVdGb^hxs2eKd>ir`hy1nnTagT-K<PtEfu)G<l_mX`Xb7s-?$i
zzGE>haQJDVV+HvfwRE0i9W8RS(D{ztwAe8~OMe_Gy1?;P@;lx^OC8^&8pq#gne3qD
zvO+85Idq|1MJwe11>}0FmDkcLc|Fz1O;j&mMM3!xHONtFly9bs<d4y6d6?G7_tVAl
z3$#}LGhHITLhIz$XuWeOZE&uljm{v2oLgy=GfbB{ucgh-3|;0tLYF(=LtC8pQIo5Z
znm1i*sl|0CwYuI%ZLa5OtI|r_lv`=L@)c@VUZfq$%g`0(b?8eX$3Fp3O9u$6?>Zp=
z6aWB?DU;C^9FxIqIe*i8dz(GluG`YRvTlQ}ZQ8wBMi`H+11Xd;){T;FQf`ym_HIdT
zxw%<4ULqnQiUNY#fhed{bPCKaEfg4_ZZJSmR31)Vg5U#DR8+vtbG{^9+GV)@e(AaA
z`@Zu&-#O>ofAE2a0W1-#1$JC<#oFbUR(9&)Ek-<28LSL<Re#o2hOJny8Py%riU(p=
z#GrNCh#Gprpv@jV5eWAf;Vp?|EKwQk(xVBZwz9W75Yu}DCi93ai!5$f(n)9S<yrc!
zWHd?(i5@+$m{23$ZN?45Hsjs2y4A1~B#YLU*DbAE9*7uQD=$|e1hh6QX@`w7%%~wS
z)3UqkyKOxZHGk^+Y`wSFu<M)S2}h4c=Py=JATW8ezE!V}>hbRSb2~R1VMjrsz%03%
zbj)ad*oudfwr#|n`X(aNJEMjIl?b=$(fLs;tVcJPy=iF^TO^rj)iZvQKrx?*m<eaK
zX$zEPG284+vZvPD(NUtf;nc5R+e}0ye2k=ancYcSdVf@4VM`uE+S9Np_bAa9Iz47$
z{vt9yA5q47KDWY*o6ac$cGXzD^I=)_K8ocr>$vcIFG^5a1P{u+&```@)4cGezkFUy
zz(oF<;l(6O=C4@-?kc7$!yF9?`~n5!dh*|ts)a4%V@TF{bB$0iUtmJF;jGa)km+bm
z&Jt!V^?%|x9Is&kssyGTX4&R&&aFzC(THIysMb)!;uT`os>h7+8l;aCvjFOtSv`50
zeGrcb1gefacqDB`6tP&0B`j?z8DD2@QPCivI#&9W7bmcQ8Y~x>mp6iAq)68VSs~6#
zGeH?ij<W3S(y<%7I2ot#?y01%nP4tntV}Ph5`UOP#;)v4Ska_otkInwFIK0mJ6)jS
z=qaJ#V{{?)aM(ytMlPb9?5_IA4Eldod<60Q3~b^;L%wUqSsKp7*#gHOBawo0NZa(m
zGp!go;9O3~R;(FQCfc?q93!S+EqTtc>0XzQs=bD^bVyf2kC6uJu)YXwIG^r#mu^Or
zwtsOB`9bfdlqt=ZFc%=i(l$_~$iq;<CQ7<qUWigfD9@Kl6yyuhq2c4Wh{7150_>0#
zo#`-!DS0T2O;J6OAQ5AdRxXkX2D<omb?0|lzRRq@BxQ1|T3e64q@l4b8S`QbTeKN;
zOSLYGi?ZQ_dj#g>P1kIRVJqUWIC#Beg@3V)cqhED(^in`<%f%NlNF6p8k5w7f}}u^
z5$kofw-5#SIBTIi$!la_AGT@O3d;JTD6Oz~;#g9(aO3z|a49Zhd6#FSA-SxyZC$cg
z@Cgl9avgB%k;u4kWQq{qs;lrRK6f?cz*t=rTto3N9fRCxQ4&oZqiu6$o%FaCpMNdJ
zXK)=EbmYE*&r?!Re{D6kIbM7LrxfFQe36P{TrS**dAx8F`7vsBcN-*VM!q}LA~#9e
z&A6qA9RFpqdNrpHrIkODEfszhU*$5=!DVNMfbXcB6x>FhA(39(&d0xouan2q2`PJF
z$+#3?U)_N_Iq2V{;+>mMUVNLo!GC7lm96TTOi}P1s_KrlvaPAPIa?IJ%XR5)e2+Xz
zGlJQ*eYMpWk6L=9DKmfwG~~HD$5KDPj~}pp_fR$`<CmDdE!_N%md&Gv5#fE<#e_dh
zH;r`t+@01lMQe2ld1F)z6Ji@wWw%m^s62YF$leq$9>555d62BlN?n!g>VGn9BeK@e
zWxskjn>ZPbvg?oJ34&}Ak7;-mKjI28x|^oS?Egf=9_*#$rK%KZp_$B!$Jv-YctXGv
zj#>#?d6L`o9y~=!(qtv05r5or{9Szg{gkaeekuo)O+Te{%#%aekSTbEJd)76jP*8E
znb}q23dMMD`~uHv_&I(#u7A;Huj5BH+Fx@{KPMpSRJ=gOk;w@w9wa4yldS-fa$S#Y
z^`(cv-*UGwoJ>*o;$`;2OL&EJwi0!5nhjLEM$MLEZd+uSLuKc<V;TWSrPuL>M&0B0
z+1`_`9Gr3_`Yi$1`nJ(NlCwvYf5e}P@CW>PY}b-}75s%1a;z4skALboP3MOd%H@$)
zp}*p98s5RXWL}>ck63*P75^Yl(WvU^W}M3Cj9lBAdUU(ZxHxIV!|Ch&9{$Dj|0b_>
zn(<7`RlF}S{V)|diid^KY3oBysUCU}s5nR!<%EU?8okLdZe)7gikqabyimd=2NL1t
zQo8Xd1Ca1&_^+V(-hV?~-*&ic=bD-kev((HqKHpwbVrWZR)m*bpqtJaT)1g^YW9kW
zVv;5%h{=@i*-O(L?@eZUcjnHCQfdRFdCm?^nmJ==&ITzlMU*qospO!lyhqYDP1i)3
z@QrCxq*zRM92Pl46Eo$sydbe4u8P^z3A*I2z=}Mnxbdj>W`8VWQqM2u5^qt-0+x@-
zHM%2Yup$;vdCt6@(o5rK<@74?I$l(1;yAI8ngq=^G*u;g9j~aNB0{UR0@a6$NWyUZ
z#x^6Ibodtf=~~6i1iu9nTvX`7iaHicj2)xZ=#!JISR{uBv6!aS!_wC#<Bl0KPTv+b
zdgUl1mTF=dH-ENS*$QMt{JE{j-$mR=LL()a-%OA`etN8#5kGYUxmTx(<yjfhVYE?C
z@53A!!7=3|XJbXq+}C_=&IUcvW*??#&icoQ<eWK;BHTppW*XJGB)2AGorb-RJw&z5
zLn9q|7M{r^{MrHtdNYt-S`yMbOG0{=(M#V&=gTmK-hb1p2&*RgLV~laLz_1F3U*^^
z&2CJu-HnRCZp`#~c4PJs=H3VXD?ugZrRhA|NG5T2UsgSW1)-MeK}=6!;RtF%)g42q
zmFS`rmX$WrzZ6a!!iqsuq;Oh48fsE#>PH>XOr>8%D1|eI(Gogm5a)$j_o8sX^+C-p
zv=ft!DSzlGMB1xEp-ps}PE2nd#LQp;kp(@2m>mih)~3+YK8RRQaW|@kjYR>;T`gDp
zq16U_1u0zY^Q7SHK=Cjx3918VX8ej!P~Ate4!!MDM{s2*s14zh4>uOO8@=V;^5Q!&
z$ETKimxO{7q|(Jc%|~CKZok?q1`fUA(}Jo`y?-B{6G(sDAkdGc{PiV)N5~~Xjr9Kt
zJH)4Tl=ctdRx&f~ixj>wjBm9M9D0KED;&f?3OfTnWf=FeVuNJH0A6e_FDkqPdwt42
zJX$MHg@TG?r?7)l7-H|<pXK;aP#uv{eX36jWI_>0pInr4lHx!P8Nr^=CZ>3lv>U>Y
zhkvjyh5bP_g{OULP#Hig`>Dvs3wvrqSwobL(w~tb!}wJS&zHV9YE5=u?I=AU4SjWV
zO9YjIMzy@iby29X=ytKFT-|Z-qHN^pH&Zg(nG=7i2(%pv7I0ike>aRbcj4_6{$Bde
z6#mms5yO+xQcs}t1F}Z6j^Mwc!iVrqD1YShbcEcchuR9tglO|L7N$f&d0|J}kWf;h
zm{KJrO8T*djc*+hWg#CeOdApvWc`SkN&7=$7P)ReIeIUue1&CVPEaj)2udhe<n}3T
z<l3T?m`8y-&^mz0^s8VN^XX%1U|3YA1W~_Am?~jlFXs9RvcB<jl!^LrSx{SDlYbIR
z+T~@55IC<jaPNp%7UIkw5)FI!I~~U|ae}0MzD%5$2_)@f5xoL#5f_M)XbQhu!ksk3
zG91MH5<Z4~67HvgjNyxTS;E)whJ<h79SPqTf+6)-O~MIcnuHZ%u7s81L<vt8YbA_{
zUhy1Z7mI5pyiQy%;SJ(e32zg_5`R7@9+&WO@uWBah^NIf5<ehbW9Yidb)D;dajJyx
ziIuJ!T{ladA>+5W`X$bg@!MQ?OPnF&J6-okoFU`8T)QRCknthc6B1|0_*1TDCC-rX
z7hEq%oFU_{xL%hyL&o29y(@8sj30EnCC-p=s)kKe88@Q>JiDAt)wLaNY+XbFz1BVS
zL@dNLRAFy|io2*{eh7_dip6SpMK>mh7$&+JFv)c`CcD<5#I*sXt_xA-axlexD$3nw
zVXAu#rn%Q+y88n7+?%8vx2)ps{{c`-2M9FbluW}5006p^;dxnq+e!m55QhI)wOUte
zJ>7V>3ZA+y^#Dc18$lElK|$~`-JNcu*#pV8UWh)3Z{dXqUibh$lsH=z5gEwL{Q2fj
zNZvnQ-vDf2PT=w3;k<e0wq$TaN$BplLEEjnK^kXD%Uq?27pEbyZb*}~#G4&yy`V$E
zjqS$P-Dyj0sqSVikr&I%Q-w#k6*jgvTFjc}&wRV9H4C;Qy#}|GL@FUNDv4O_k+FOh
z>&^Ae^^@j$M1ODMq|d0-FZ_2|XiKHLhEB;^88I<+^6PSu7q?|<cuyrN-xpX|{%i0b
zUbz$~Ew5ZUa4=Cp8C3@(7#+nBY7QzGE2?RM!2^CJG`K1A&kUeeP+ZjthO^*sFwW~N
zck?K50M*s2s~1eJJ!7i=jF~5(kCyrtrB6^x2MCiS?FIm90RRB!lR*$2lgxWae=!NE
zZAfA)9>ox<KtisiYx{z9uY2zm;AElQM)vBqu5G#&x`r&<;%rO1vVrWB7WQhgWnH^&
zUAt$uHCeZHudaJPQu;riBwMy+(rx_s>D=%8&Ue1^o%27B&#!&!lh=u83+I?Fo;!DF
z$CE8Xdghd2Wm~#iGQ%zHEg3sMe`e-%&$O*%-p(4BcZ{5&y9O3VbvKzAH8Q8%Lf&oZ
z9@cZN(cUsPlFaL4NmFEG@6K-Cwq*#s&W_6d;X*El33pUaZpP5CMoh~v9Mc-X>}kVs
zaTexxbZqU|k<1#WTb>FLGiif%!O0j8m^p)Kwe5^_jyQTYXLO!%^szC+f9dSETu;yC
zg5+mfeo{ZJcjk0!r1QYgNh9M0sg9{GXOD~+4%3=cjr}RLxRWWAwa-{NThB7BtHrpx
zybRXW#@S4+;F_nEUOkzN;kx^DOIN3K*4n&h!3_{scdu!g-Y%v`W4F-omO9m1Jg9r4
zJ+5oyhjQ57_Arw#*7k6if0oj6je^v`l>A?58l)zTR!~Ej!nCBG0<<zrA*z?J+rp&L
zV)<GPm+5NPhE1C`F^$x`QrT%cQ_5kc-6@BaN6LMJo*-jq+buS4DAPWHqWEvb@cMGP
z>oPUP+Nxx!$(>=ko$io(N14La#|EhdE-=oTuIDNfJrbr3)T+^Xf4YmQS+N#8GuPQ?
z=W@UlaOwsr##C?Q$Gq_r_Axb9PE?#ShXdo3(5Q{t!J5O29EKAbVr|D}-#bhl)G6n|
zUQIJndK^br;)AqBqpjkw#iqO4bfARojE8AkNz3ifTF(Nu&9T(n0N5$F*+KWn{%)qF
zvvmy8y-Y#V-6IzXf732%T}=1U{Y;NPs7xNsg2^$53UcY_##VP@G;14f)Uv&3#<o1&
zw4%B-4z`j`rr~x(_naNWz{o~DJ34Gy*{E?=&v?1<=;mp@@s1Gn%nH|%zk}2VR}9-u
zUWZNZiOmPZu^Km=&lRRU^ngaY=|QHA-ywp6)X%g&_LhiLe*rY0(Oyb1`Can~LxHA0
zO#A78nC~FsctP`(t>(fwb~OKgwcQ~c3ABsH``hMQBut0th^QhVpEHL-^bWxZ^lhtQ
zj9%OJpr$^y4~h+Xy5kwnhRs1brqOZ1T-$7$SbAPkgC{Aa296(-lTI-0eQN~C@wy{d
zoyJnM#xC4fe`i{W5@8OHR}x-dx&AP1tAUcYb|PRu_)t%B%eL(yf&<qm7T3rW9Mc=`
z+=ujqjJMkyF(gPv5O^@RLW`9F`{o*Fmi<s$225kpNsUJ76w|6|?fR{;f(H?Xo)1!9
zvcqiUOL((2vS_T59k4cQ8;Xt}U&sJ~*s)%LqDKXde_T3^Xex|lbq_iRvDDu$;?B~z
zXz~QuGdbHe!t`BI5nOr->{+ER1R_iIhUs1OZsGmziq=&(?k$+PtW<^X)#$tcrD2An
z-|`GqF}@F`^X!L=v!y-r5IY^PKR`dI(f892Nx4RE;Ejgqhv|UC@Q+|hpkm>EYh!)$
zcb64`e~|amkBKhtLuFgoLksNufb4t*WyG^9x~_=TRQ1Q{L&E!EsT%Jrp!*5aMai(c
z=_6u5^hq9U`q5HyewJw&u+uZ-+PQ*fNKFpYb0T3q{Ur0~!vbqFqgt(~JzOgQqQg3n
zkiE0jYPHhnhHCQU_3`Mae%go*8HN@0^gKcve|hAL>5X=@T79-PY&!X<H>!L1F`^r*
zHxG{L2!z2xeq(gZv9Zw`k0Kh!<*ZV&NS2dDM|mB|3i$~-m@b0Xk<5fbkd-Y_-GOT5
zFonU?apmpNVaLuR$~~vxN|tj~Z`UCgi|($z%@HTp9c^`6txCK{Q+CNlrRnKBS?NQ&
ze^qXQm}pPNgHPrygy^Txx6OF-P{H!dyn$}V7!$cc`k6TebXLNj(C7tv5rw?uUKHUP
zq525ICa2ng=II(g8#*u1$Heg;57W=l&ueIxK7k-CSWlRU?K^7Lo|!x_s~5qJ&PU9#
zQvY&AqpOk~f`;Wu9bt;hYDe~1g}mV?fAc|yNtzP=muJbVVhPeUU=~gOKHD+&<cvI|
z0^*;R=HOFw9RY0_XT9>m+#s2*K)+1CBJ974%so%*Jy3HzNWTt^5gPkZP{Q<zgv4j)
z)!9v{i9F(1dahHzrW)GAjxLV{y;QU%#r!7vIaeQ6p(V1I?)r{`A4>ifeO9B_f9SX6
zWOPw=`BSK}xa;qfV)qM3I29-K7KVo<q7D?is?7U)Ot)3LY4+;xYxE`YOJv5g)eL25
z{eebbq(4M&rb^88DjpKCRBh8$E;G9h;68s$e<H5^Q+TD;fkFB+!0imm6Ab<rQU(5;
zzA8UY$u4AnO+isYEf=O=q+b$*fBqHvv_89#%c`2K7@{Hi8_9r(2fvi>5d9q!qfY+=
z?z-RuCP?3qcElbD(>Eoa{)zq>+4c|~l@iq<`qxT%Q$9L8>ey%WA%XY5LowK<Tu{UG
zp8}Qtp#Q2ahX<+x`G3HV9ukj6!^?~=6j+wwbsjm?uL5fc#2-Z7vb@qee{*@PZBA!{
z`!OKFiv$+{X`o{ajZ%mM0N?JaWEbKfx`|u7)Ph_O*b82dI40BWu{qkz9)@`_hc(uC
z$()k5VDDR1(s&u1#LFwsowCdzH$r{SK5{5kb<xsd3J0#>W{sP8<?L;#Am0XvBKb=#
zu}aS%B-A<kXVxHTHLuY)f5K~#0`NTP)m<~wQ$RI9;w|O!nclK;--U!BQuubhLo9*k
zTO;t9Ddf=B9B~b&X9N<%+yYc!c|Dvjvq?1_;?_z6nkO{K8zIUyyhEts=y?v7OCO#L
z3$~PV3In%sT;mugU>e9jV>_n~qo~*g<VEfebCNsip;ISn1<olSe?86Nk4Zbmn|X`K
z*a{g+89iI(2=lhm5>nHu*n%<7JA~&RICDgu;o;t?QVYd9(L!PI-dS%ggq9&d+y&sH
zSryoqrsgK|(kwjrHtx~*e(uEv)0N)NaSCH7zhT~uOo^2}0g`{qiEt8ngb@e9DlbgK
zl0S*ucdNf$Y}joKf9r*uR~a9ivmNL6^Ioyz0MpL@hoB(uL(QwSCV1(11-EY$7d2Gp
zymzm7;{YGjct5`#nQXfEIHS8!bLQ3^As*D|O?nYJ5u$=Zd=#0?QBR}8c9_#r+t)MN
zfrjebpqif$9|!8nEnRoiE4exv3-M#p-qvW2t0VexiD<lre|&kjEyNk7_^qKjokdHv
zh7IV=!$^tKAu(v9idHZ=lSCPL)8A)jkqnomOv~6`$PXFLVG%`}Q5AylwvQU*??3oz
zkiY&Y8m1+nLBL=sEkn*ciP1*z`$5;l)5j;N@fxj24@^>X{3@+VT%}0+Ra$dd!Ka?q
z(z?<neh^#ge?S)};zhdiXpuHvByEb40~2(AqDY(51Furu#LaKSuhG_YYLf20zRS1W
z-|S!aJgtj2`?sXK14Zh(Kw2V#kDW))`+4H#>xqH*%k(y;3l#N#nu6&8U;AKVZ+wa#
z8n{M#(<zQm(!L4$5V(K<|D|fS!Fg<Bao@{ybP;-;fAyGOpH8RZjp<2x1j=5ZSVEnD
zXW|+i7X=^1`0;e=6<U_K?thNz6TU4I<mXqbj~JSF#1b`#GX{xIz(mZa@EeD{>tN%9
zv<QzgADxD=-U}-~i`fZS|8@A~n=t21@YPd>vSp*zVO>1;x%OAdf4OmZigNp}k(KWD
zCno8ge+|p&Q=#ra#4i>*liptUEHx%00bg@nk)E7@wdn)Rb&D>E*}syE_=|L|NZ*6~
z=dpj1p7w1IGzXH`pQnywb6{%&-8?r%?@4!K^N-@bizEK!n~L=QqY#g&4<0=qfJ45}
zE^;oU_ZR6WE<mM3x=5Gc?B`QKF-rP)x8D~ZfAl8?1MP!#$-%~-j`&L!dLa$_{Ok>-
z#SK`XnO4&_+-xn%v(PsDZkx8(Qg8%dulK=Tui?91+V3(td$HmJ-5yu|N`m}?xM_p$
zzO@P5X03QOo>;pDj-8^*7b)O->HH$-{suTNy;KG+nx(Rhx0j>i`D=7Fo!$pEi$(gR
zf8g$h;O;y=evJW{&!qQ@WSBl#q~DmL&ne)1{sJwNOa1QAiJPCFpkwXHYxG6o{8Cyx
zGf7{L1SaW^iu9Fke}jLHzdl0CD*k$X;^x<r>9UjF!2gMx?;eQbq&IG~7wIoA%g+r&
zsD^m$RTf&I=qidT+Cr_0#%Q~u_s}jyOZU)TMN@P@(L;1x(c^Ri)+N$uSkY0k6)n(v
z6qR4$dp~_x(UM;@_ygF)>LTQhuT^Y_xuD7z2NUg6^w*cu`{U^=6cMB)PBeaflh243
ze@`_2n_~U%>6IHei{PI+WN*n<-$I0_6BhxXlDYUwdpxZ|c_2|_U+F|(yU4KQ2b;LA
zBucsJ($Vrk?I)Tzgp;OtX^|T$I;`0*=0@gXpSY8|{oEZ;EUOR{;??e;xD^2TvUrr&
z3EB}?@;@zc!FLvULlfV1qR8!6cvF$@e^$R;MegnnG{oTieMP=+yT86GRNtjV0__R~
zVMM4m#eGG7;37S~Qd=2n4nKXoE2MYfQ^&^&elTDE%ttA_Qfu}<{meyLm0T&4Mpx(x
zr!cirEApX8u-(@j29QKTm(~@UxcS^bB-rhrAh%4ruhE<7CO$mLM{Xn{!AKx^e}x}z
z;&<Ts71$vUh7w|`ILvwM=w)Q@Z3D7R>;}6w@uRRP5&}0g@d<n;)fI=DI9sxGe3D0Q
zR7-w^LIU@55x;!FD{sVaP=rS4BoDEv=wtA0p3bVVP#K@4CwT+LLf2yaE_y+Yh06Fv
z`j{FEmGQ^vvKkAOu|%(oh06F9O!}-E3zhL#>1%2%RK{<j0hF;&jM>KxGFDW^?cAlt
zLS>xcXOy0$xM&3W-wv!kMvFK_KF(mwDoZUQ-?sr!O9u!`Lm;-F4gdhY8<Wu$9Fxn6
zE`R$@6G0TlzmdyADVA~*EGVwGwy>;4O&V%U42cOzgT@++{5Rb_Y!~)Y_JT1+9)zb*
zqnP-I58y)?&(I<yv|%SZyL0CIoo^;*=Fi_>zX7bl6gWOQdQ<(RH>I^tfvvCW)~>#y
zTcO`}J(;*+VECa;9FNE&852*oWNcV1vVZpD)Q|P`UFpTNqPHExmu<sUIqWBP<>^|J
zwNdqq-%UM_193|l6&_OHxB*e*1`bCLDT>*P<JHy2t5w6ML;dpzgg~YdrjbQ2U5^Cj
z!^kn6NZKBmA0pXqQ)Jdd$Dv5WpuohQJd~y<1IK*T+#@T|Ff1^A9(}<?V9TaQ8h=Iv
zh7MAf5&^LzF#OC7+~m2yU~#9kEih0lZD*0i^$axRvdCa;6hp}Kj2AirGsXIO-bSMK
zQB&N7H8eiP3Hcgkm<dz%ndu;IXuf8B!W2ormc<Rsjbaw_0#iK%8VUkhTLruzi$zRl
zuz*{<WO=$BfkzkDveA-JgAP&(EPtw%>b*8!6ELqrE-i8iy7Ij%u-2E|-0W*uxf<$W
z`9N7d<iZy>`evT{Ki4BcStVHJs&4Qp6v);2&~2rDlcKi@M}=#uL12{Myecx^iy{8c
zVw`(}N3*!b4ak(=|HMS$2PVHlJ$X!Fx~nO4HM#P4Odcci4L6rhaQjTSgiAYJVW}(3
zcZ6dd;k|d|FB}wD<$jpIV3ES^cd=y*as#G1*to(L7Ee&T3=W)vrT%_}6Rcdu_!2Ox
zd<XskP)i30>YK3HJOTg!9+QD19g|)V50gKZ2$Phk9FvcY6@RORP(={Ilb|T{zS&HZ
zZ8w{+o7RKa2k|XD2_Ad^A4;5v9-M{w_<sA%Ec^NO{s91E*kkar+1y7@JQi|^CgD>q
z=X}6rk(Ww~N);x^iv)>V)F>R%WhPu8Gn7lW${nB1g?2dLWg6t73{<@%IZZ~BaZFho
z{msu;S`%=Y2!BRo(WJ^CT4hqAYqXBuA|4G-hEb5<nm_x%7<3+rm6dp{G%`3UY#OFk
zBpSm<E6NgWcqgjt(mBjyjkXO92Kx)K7NHg<4R7NG%Bl@52CZ{tRCU4Nuy^wlP(@g8
zJ^SzaGX`F-KkI@Ioes2Nhe6|tz5pX=lC6u;@nM_#E*`l9QjfHG{080E-#`tv!21SJ
zO9u$2Uk=(N0RRAMlR*$2liiL#e@k=IRuukj<3z4(N=OI+w-jS&>X+gsK<hft2Q)M;
zdC(Y0MIZE9zLrH~$s@@LOr}i#23d5=mX$o<nX>4vi|+ax`Y)QE>yX5GbXw0?()rHg
zp2v6Y?|;Ai6~Hta44Y4$EEhLYRc@>br<N+EjCv}cGTdg%vUy;+j@EQ*e}ZsN*n<0l
zlnw4{Mnf2@e!J<XN_B4gBA;rRT9dalOU9^Ou1#H;u9B@+-0`osZJPkU!L=z8g=1Qd
z5T50jgsutCr?UtzPF|WUYBjNz`Y8&=pjO<rXNa4YEf}({XBJG4*KAQ(_js!%yh7!%
zABg5{&m|$Sgg?g}QhN#_e+<J9`5G_S+%XGF)rZ0e6bvzpnj$#p#c&}#-?O&df;?;{
zvn{p*%P!1uzp=nuGYlsVfeKWHp{tf-1)nesq%-&8h~t=w7~W!t_s3K)Os1B<T@0Dt
zk7ER*Do)@e!^yqSdwD51rD9x08t;!(c3SPA5_p0)6`Y~84kq7Af8uQw3B1D)v0Ywc
zIK5}o=ijq^7w6<LNrpkgc6|{?N|uno1xk2TJg$g99@5_vf@b1)4-?YyeTHNER0=W-
zQ7aH04_uGo)IQ(xYn%e)kdGk?jbY@lUO|CDA-*k&M?oA@7>(frOjAV;0o1acsC^@*
zn3r)y+I>hF1TIxce;hk#yN!}<5g)5iP-2MryPTMe;_5#3Y?~{f39Ej<y})0w0@|Lj
z^x>Fts-NL=6`$fd!<&A)>c385EL}b_hc7TIt#4AVZQ2VNn8;C%V-97h_=;pxPGBN^
zxZEQ<FpmW}mPK-;K{FJOaQvU4ZsTjo<z<G%o_4wI1Xfebe?BrqOI{KR?(8tMue9rJ
z%dW{)yhrOb>v^u1TyF>`Dd|Y+WNVk^$vUz2S`^>>OG|rnzOP~h-%^w0;yXlW?LXSF
zFAFN=d;B0nJdh6>c=m{s`j9&f&t2!$-EFF>xC?`>kKH9&>Z_j?I&y<<y2yD95})Lc
zhL0%y8BCJ?f4~rFfX!?dF@3&+;~~a2F_?Sa#kj67ba1+hgua;9^lS%bL!1wBF+}<W
zCK+yJU*HnMQ<UiABL;mj#LURG5I3G~{iV_Oa4f`^dMOg()-PLs=FUX&gf5XHL}lgK
z)^EAz^Un|^?Bc5~?&@8v=-CG&dI#Ty_%Q^}bx?anD)}XF7VD_v9431231qNAS!a;%
z$s72&_m<TS(DMxp`~y%+2MBtsPVk(Qy^?Z&TOO={C{HR0ifIJ{AtBq|_Ihx=Hh0&O
zXyk+V8GNCMjfpS(0DdUrZ0}A>d)Ov7vpfIa?C#9&uirm@0zd|~2z<J{%w<1_1zJ33
znk24>#gaHD7ORz-qEb_-YRO7fVmPlel~IFXuuP3)vCN9+M!jN)D<!HLY)-7n$qKE1
zB^91ST9ndDnF4C{tizY;s)`dvnI_XXWSXpKjOmId<7|WJ2Cjwk+4+U}Y*Jzk@uLq2
z0e`+)*Ng1AqA~)}sxGHwok}W8UFft{V|wbHWw)wYNs;S%YtLTr5D1>p22H6{lT-VJ
zGgdUc&`&^+6vNb&LY?af1om1gjhU%`gWT>aQtk0gJTQUq-oH$Flkd1w_lBBf0;BCy
z`7+HcE$8bM0^avZ&C0|*OB=uyFRJ?aTcyIPb&~+uB{0^Ysv=R7ZMP*l&{d2c6X;)4
zG{sye&>M>%3NQkre(=Ig+{%mG#`fOM=|O%cclvVw)s7Fw1@Oa-0qBDX0)tL}srdd3
zAKVr|u!4652w2`d0fsD36d(v8?%fw448z=eKw!vV=Ju7+g<@B0$2aAJ0j^IF7?!W<
ztpbe1;%>zpHr&Lcv2JbrusgL?(as#!?0ARvZ(9Tyw9dPLBI6nnUO(iIo%Z>S_JI|#
zma!w&AcT?E9qq-QVS__Pcf=Ea+vSIvK<Mzg_9KlKV8Zy1#1lK_X*^r_+8~YR^5P4}
z7sOkF9^v<)Wd+*~+BRU@A=_f}TNYc7Hi#bHH2iMjXaTblw9&-T(aFho7z^KOLL_{r
z36tEL;@)&98f?OhrwP%oz<(i#LEKIdh93L_^e1MUFzdwUAZf=#X!!zWeTi=n`C^CX
zA?1cg9Q>gxKI!0TcYM;pGp_iegD<(`iw?f*icdNCBX@kt!LzRTw1Yo($EO{91y)_~
zna_534W4x25$ukGuftOpJnG=jV8ac!8;kc6zdg|V2T)4~2x;QgE$@>LmS2BOn-Id%
zP<GH51==)e*e#n11+>zQ28t;HPLr2p=wv3&Oj;JfT|seQL0nM~MJ-CF6-0jU9DeYR
z@_64&(j;x_;hdb@dGFotF5i9czW2|+H~#{#7PlELoIc&#dNMd5B?h^g3~ml4Qo(RA
zp=EQjBAK$LMzUIx)4a|VE*XEE7Bi9&No06p(8y<v;O3O17(5)zr6XA>7m<aH<oe{A
z4QmHTS8pxiQ+Y+92_qv#*3givX<6l5m(?`Q4sqQeUo_v^vaY2ylH^0dQywq|Z%oVU
z3BFraI73j=)6ujpB^4eW)}>sI>(K*_+;xm6@}{P{;bNG3R2q_^ill$0qum2XdBSv~
zj!flrjWkV}8w?9NY@NI*E76{b`7I2yOInW8*^Z{HMa7sj>JplolG6-L9n;6tX6xj2
zn?nKGDyy>jD8s78N_*AgXzF9AX>98AVK(M^;YK|n@6nqZ^So$4y$?Rjnt@s@@WF!_
z;%ku)Ud$9<OBj6Rakzi6jDqCHWWzLU*Zn@+j|aS1jsS6>Xi~Bio)1CH@sgE?7-s2Q
zO70|>uI<+qhK9zbjuQPbQ&f114=b=z09Fwo&CMQ3=c?)OJGTfZGU7uMLc(z~Lu*;i
zHb=5*a$S{_V&=AIc_1$mC;vnQ?IluiBSJ+^IKxRw46Caap*(-$LQE<*qx*Z?DW)h^
zd(nb5408-#VUeM}u~J*qZ5`H&Dr}$xlV!>~=nQ%A2*bQ|r4_N@!zMvf12!|v6f`-E
zA159fr-nFf(3Q+@#Wuk_ZM}KMRF@3%tC$uEJdW)mlpT{2=#k8f2Ro-GAQpVs?IiHT
zRBz6DyJPh!@>_pyHI|XqZrB*hXFcd(STxD>#HtTnj{R<iP4`4e|1_NJ6`b^vD`Q$$
z`<zhs(@8lY=VH`cxKq6Y)JrJBemUn9E$+nBPt!)(^fM0rP#NAK@5lt+;Y4!q0~2^>
zI_co4MD?WI#m!+&AKWKrxt2HWBiimm8X2J@Gq@Vt#l(MB42sNXkJlShK|+a2t3nf~
z9K#Z_+$Sk=QZo6ZQ{saz&VK_8f$J9yVJq^&_z>ZYX>pD=c{zsT0)B$DOC{*dt0qOW
z>sW&4oM!brL%2=LE6ISWnE}yg0)_4tD7E51O4qW1RV$2DEgqb%=<rTa-FT7uS4z{f
zl39Ik9PEFXd@EExFE8U2FJ8i{459zuVf!+=HY^QL`*8a7>t39~8?^CDDrIS&Wms6=
zbK2Eh-Xx=3%DVAZsfQF>l4J92FV5i|>Z;Xl2{+y&vIS$bk4x|}%eIvd@Szv)<D+6^
zl`3w0!cdz~G_hqr6PL}WI8RM3Sy@#NK4(~2tyO<iKVM%6mtQh$@6rN8l9K0vVcE<C
z2KjNjiUA|X6LL!C$$+d@tv7K{toPS~iS2^jZ|N#hEB3Sp-!Zst58Wxz%-X94bQxDf
zQGXzwORMd4m!!p2@gsvLC+P+i#hj)g($TtxpS}1Azfd8Es&9P{Mg+Wi&$*1>L<qlE
zgs^`@)96`6+hdfvgdIX*W|}bQ5RPHo3k!cx>D%aOMWyPXmsD3iJHYjQVmo3Dol!SE
z@M=&mE`Iu|7uUWm=}AD+4I&bA=>HbL+*kq^&HmjSY7T`%@iF*sp&=gc8pHfiEF8t+
zQ7pCa;CWn%gd*{&Kf;B_@vw!)P77iBTx)+}qra5~Tf#>yJZ7QIzl%ms7DjvgoiyqR
zAE~hrv(V>%nuZ4pi--Ns(kM|Fr7Rq^khSof1=GT?g_BpXtn(I5#a*}Ij(62GJN`%C
z<=Drl3ZC?LG0U$s-Dq50A)NbSTPi=_%})kwxho&E==wkE(LH}@{{)3qO|C%#YF=3$
zdiA?ni$9)wR*=E-zD>6#=i#B!N#gG&-1E6KkNw7xOU%m~-nh!XQ{HJ<!rc8=T(_oF
zeBXrsO2vWOl$DC2B6ckgYmtAy;$wWzPA9|v=4YoIme9MasB(Dzm-njLU%%MipH|W<
z{)U}vVr56^?I$dY`hbUa?qcJ*rMpCT(Z}{_zxLlZj_X^!P?^c3FMfS!#jMPH&f~%R
zxV~8>=8J4JS5MC7j80GfF1F!!W{h{y?1Y6gJv#Es?z-Mhy6*8qFYB=KY5fJ$eA5<B
zUX?jnaGp<_N3Lb-;Wt-+Bf#~bI%b`~tp}42%vr_it^2-4X>$JDWZC&|wm9Vh`;wc1
z=hdk(0FO+816Kit$%z66lMChx$ilBF2VOs5jG{_Fm|^llWu?h^^R#6V_b)Rr*r2Go
zCJIq?W1a~s_?F7ag7Zb0%OoM9-t$dmLAMF|0NpViXalO=LkbX8`{$d;BCcg)V6a88
zp-~y6${p-l#0_8!3>GM=&ZvP@X-r<TKua2Q6BIAdMrWYUQT=p(^2Y`KvY<PRmIJ$t
z=YVRAP)rHahPbGHp_VN2eKYnbs?u~Js$MKKmqlFNfF8>J1|U_6z{_d)L2hS-94p_r
zNR&C&lwq=fmEz=Gi{xeDN1+4Vql040S4)s8GqAtmXGCMf(rRml$p-dPz{AsxWx*#7
z1I<|s^p_oqSz`7Kll2`vz-A#%!)0L5M^WYL$S|3)N@Q}Svnp66{FqRnt&S)votz;m
zA;;+IfmI{UMr2?xK~eqK4W?QPtP=SQA4L?Exn2;JaX#W;mGFaPfWAVFN$n7b${>49
zkV+ZQVI((!sx|@ru8U%(NZ90nWgaq!b@vPmS||zvBY+B|C!b%YB@17*Bg(*_graC;
zF33Ka$q#Y`z%B!>QGqN`0osXb-`Pr#N^_7ZX~ZBQ1A_vJd9x>9Ty7%^AMXK%usn+V
z#4d>c>{h7C!iPA3cA@5E<z9%YDJyhj!96NqiuVKF6KjBC){;JmsxN(NV7G%Cq`+_r
x0IEcF`|?Q)lOIk3yPXAGr1Y;8WBxT2Bq%-EZ<_ezTPu0ESa^Xa_s;>9CIF-wP*MN@

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index f5bdef81deb70..c7f182843385d 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
+distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
diff --git a/gradlew b/gradlew
index 1aa94a4269074..f5feea6d6b116 100755
--- a/gradlew
+++ b/gradlew
@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -84,7 +86,8 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
diff --git a/gradlew.bat b/gradlew.bat
index 7101f8e4676fc..9b42019c7915b 100644
--- a/gradlew.bat
+++ b/gradlew.bat
@@ -13,6 +13,8 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################

From 084852519ea884b9ada0acbb798facb006ab8ee0 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Thu, 11 Jul 2024 17:43:33 -0400
Subject: [PATCH 097/113] Fix hdfs-fixture hadoop-minicluster dependencies are
 not being updated / false positive reports on CVEs (#14732)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 test/fixtures/hdfs-fixture/build.gradle | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 6ab6d5acb8880..a3c2932be64c4 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -52,6 +52,8 @@ dependencies {
     exclude module: "logback-classic"
     exclude module: "avro"
     exclude group: 'org.apache.kerby'
+    exclude group: 'com.nimbusds'
+    exclude module: "commons-configuration2"
   }
   api "org.codehaus.jettison:jettison:${versions.jettison}"
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
@@ -75,6 +77,8 @@ dependencies {
   api "ch.qos.logback:logback-classic:1.2.13"
   api "org.jboss.xnio:xnio-nio:3.8.16.Final"
   api 'org.jline:jline:3.26.2'
+  api 'org.apache.commons:commons-configuration2:2.11.0'
+  api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api ('org.apache.kerby:kerb-admin:2.0.3') {
     exclude group: "org.jboss.xnio"
     exclude group: "org.jline"

From 6b8b3efe01a62c221f308a2e3b019d75a7f5ad8a Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Fri, 12 Jul 2024 06:05:06 +0800
Subject: [PATCH 098/113] Add `strict_allow_templates` dynamic mapping option
 (#14555)

* The dynamic mapping parameter supports strict_allow_templates

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Modify change log

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Modify skip version in yml test file

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Refactor some code

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Keep the old methods

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* change public to private

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Optimize some code

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Do not override toString method for Dynamic

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Optimize some code and modify the changelog

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

---------

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../test/index/110_strict_allow_templates.yml | 155 ++++++++
 .../indices.put_mapping/all_path_options.yml  |  31 ++
 .../index/mapper/DocumentParser.java          | 175 ++++++---
 .../opensearch/index/mapper/ObjectMapper.java |   5 +-
 .../mapper/StrictDynamicMappingException.java |   4 +-
 .../index/mapper/CopyToMapperTests.java       |  40 +++
 .../index/mapper/DocumentParserTests.java     | 334 ++++++++++++++++++
 8 files changed, 688 insertions(+), 57 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cb8e6403aa47e..fbe2b7f50d446 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Workload Management] Add QueryGroup schema ([13669](https://github.com/opensearch-project/OpenSearch/pull/13669))
 - Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
+- Add `strict_allow_templates` dynamic mapping option ([#14555](https://github.com/opensearch-project/OpenSearch/pull/14555))
 - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
 - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml
new file mode 100644
index 0000000000000..b3899e295eb61
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml
@@ -0,0 +1,155 @@
+---
+"Index documents with setting dynamic parameter to strict_allow_templates in the mapping of the index":
+  - skip:
+      version: " - 2.99.99"
+      reason: "introduced in 3.0.0"
+
+  - do:
+      indices.create:
+        index:    test_1
+        body:
+          mappings:
+            dynamic: strict_allow_templates
+            dynamic_templates: [
+              {
+                strings: {
+                  "match": "stringField*",
+                  "match_mapping_type": "string",
+                  "mapping": {
+                    "type": "keyword"
+                  }
+                }
+              },
+              {
+                object: {
+                  "match": "objectField*",
+                  "match_mapping_type": "object",
+                  "mapping": {
+                    "type": "object",
+                    "properties": {
+                      "bar1": {
+                        "type": "keyword"
+                      },
+                      "bar2": {
+                        "type": "text"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                boolean: {
+                  "match": "booleanField*",
+                  "match_mapping_type": "boolean",
+                  "mapping": {
+                    "type": "boolean"
+                  }
+                }
+              },
+              {
+                double: {
+                  "match": "doubleField*",
+                  "match_mapping_type": "double",
+                  "mapping": {
+                    "type": "double"
+                  }
+                }
+              },
+              {
+                long: {
+                  "match": "longField*",
+                  "match_mapping_type": "long",
+                  "mapping": {
+                    "type": "long"
+                  }
+                }
+              },
+              {
+                array: {
+                  "match": "arrayField*",
+                  "mapping": {
+                    "type": "keyword"
+                  }
+                }
+              },
+              {
+                date: {
+                  "match": "dateField*",
+                  "match_mapping_type": "date",
+                  "mapping": {
+                    "type": "date"
+                  }
+                }
+              }
+            ]
+            properties:
+              test1:
+                type: text
+
+  - do:
+      catch: /mapping set to strict_allow_templates, dynamic introduction of \[test2\] within \[\_doc\] is not allowed/
+      index:
+        index:   test_1
+        id:      1
+        body:    {
+          stringField: bar,
+          objectField: {
+            bar1: "bar1",
+            bar2: "bar2"
+          },
+          test1: test1,
+          test2: test2
+        }
+
+  - do:
+      index:
+        index:   test_1
+        id:      1
+        body:    {
+          stringField: bar,
+          objectField: {
+            bar1: "bar1",
+            bar2: "bar2"
+          },
+          booleanField: true,
+          doubleField: 1.0,
+          longField: 100,
+          arrayField: ["1","2"],
+          dateField: "2024-06-25T05:11:51.243Z",
+          test1: test1
+        }
+
+  - do:
+      get:
+        index: test_1
+        id: 1
+  - match: { _source: {
+      stringField: bar,
+      objectField: {
+        bar1: "bar1",
+        bar2: "bar2"
+      },
+      booleanField: true,
+      doubleField: 1.0,
+      longField: 100,
+      arrayField: [ "1","2" ],
+      dateField: "2024-06-25T05:11:51.243Z",
+      test1: test1
+    }
+  }
+
+  - do:
+      indices.get_mapping: {
+        index: test_1
+      }
+
+  - match: {test_1.mappings.dynamic:     strict_allow_templates}
+  - match: {test_1.mappings.properties.stringField.type:     keyword}
+  - match: {test_1.mappings.properties.objectField.properties.bar1.type:     keyword}
+  - match: {test_1.mappings.properties.objectField.properties.bar2.type:     text}
+  - match: {test_1.mappings.properties.booleanField.type:     boolean}
+  - match: {test_1.mappings.properties.doubleField.type:     double}
+  - match: {test_1.mappings.properties.longField.type:     long}
+  - match: {test_1.mappings.properties.arrayField.type:     keyword}
+  - match: {test_1.mappings.properties.dateField.type:     date}
+  - match: {test_1.mappings.properties.test1.type:     text}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
index ca7a21df20ea4..f579891478b19 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
@@ -159,3 +159,34 @@ setup:
       indices.get_mapping: {}
 
   - match: {test_index1.mappings.properties.text.type:     text}
+
+---
+"post a mapping with setting dynamic to strict_allow_templates":
+  - skip:
+      version: " - 2.99.99"
+      reason: "introduced in 3.0.0"
+  - do:
+      indices.put_mapping:
+        index: test_index1
+        body:
+          dynamic: strict_allow_templates
+          dynamic_templates: [
+            {
+              strings: {
+                "match": "foo*",
+                "match_mapping_type": "string",
+                "mapping": {
+                  "type": "keyword"
+                }
+              }
+            }
+          ]
+          properties:
+            test1:
+              type: text
+
+  - do:
+      indices.get_mapping: {}
+
+  - match: {test_index1.mappings.dynamic:     strict_allow_templates}
+  - match: {test_index1.mappings.properties.test1.type:     text}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
index f276d6ee2e579..c6815ebe8d91a 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
@@ -54,6 +54,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 
 import static org.opensearch.index.mapper.FieldMapper.IGNORE_MALFORMED_SETTING;
 
@@ -545,22 +546,32 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, mapper);
             ObjectMapper parentMapper = parentMapperTuple.v2();
             ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
-            if (dynamic == ObjectMapper.Dynamic.STRICT) {
-                throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName);
-            } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.OBJECT);
-                if (builder == null) {
-                    builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
-                }
-                Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
-                objectMapper = builder.build(builderContext);
-                context.addDynamicMapper(objectMapper);
-                context.path().add(currentFieldName);
-                parseObjectOrField(context, objectMapper);
-                context.path().remove();
-            } else {
-                // not dynamic, read everything up to end object
-                context.parser().skipChildren();
+            switch (dynamic) {
+                case STRICT:
+                    throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName);
+                case TRUE:
+                case STRICT_ALLOW_TEMPLATES:
+                    Mapper.Builder builder = findTemplateBuilder(
+                        context,
+                        currentFieldName,
+                        XContentFieldType.OBJECT,
+                        dynamic,
+                        mapper.fullPath()
+                    );
+
+                    if (builder == null) {
+                        builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
+                    }
+                    Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
+                    objectMapper = builder.build(builderContext);
+                    context.addDynamicMapper(objectMapper);
+                    context.path().add(currentFieldName);
+                    parseObjectOrField(context, objectMapper);
+                    context.path().remove();
+                    break;
+                case FALSE:
+                    // not dynamic, read everything up to end object
+                    context.parser().skipChildren();
             }
             for (int i = 0; i < parentMapperTuple.v1(); i++) {
                 context.path().remove();
@@ -591,31 +602,44 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
                 Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
                 parentMapper = parentMapperTuple.v2();
                 ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
-                if (dynamic == ObjectMapper.Dynamic.STRICT) {
-                    throw new StrictDynamicMappingException(parentMapper.fullPath(), arrayFieldName);
-                } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, XContentFieldType.OBJECT);
-                    if (builder == null) {
-                        parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
-                    } else {
-                        Mapper.BuilderContext builderContext = new Mapper.BuilderContext(
-                            context.indexSettings().getSettings(),
-                            context.path()
+                switch (dynamic) {
+                    case STRICT:
+                        throw new StrictDynamicMappingException(
+                            dynamic.name().toLowerCase(Locale.ROOT),
+                            parentMapper.fullPath(),
+                            arrayFieldName
                         );
-                        mapper = builder.build(builderContext);
-                        assert mapper != null;
-                        if (parsesArrayValue(mapper)) {
-                            context.addDynamicMapper(mapper);
-                            context.path().add(arrayFieldName);
-                            parseObjectOrField(context, mapper);
-                            context.path().remove();
-                        } else {
+                    case TRUE:
+                    case STRICT_ALLOW_TEMPLATES:
+                        Mapper.Builder builder = findTemplateBuilder(
+                            context,
+                            arrayFieldName,
+                            XContentFieldType.OBJECT,
+                            dynamic,
+                            parentMapper.fullPath()
+                        );
+                        if (builder == null) {
                             parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                        } else {
+                            Mapper.BuilderContext builderContext = new Mapper.BuilderContext(
+                                context.indexSettings().getSettings(),
+                                context.path()
+                            );
+                            mapper = builder.build(builderContext);
+                            assert mapper != null;
+                            if (parsesArrayValue(mapper)) {
+                                context.addDynamicMapper(mapper);
+                                context.path().add(arrayFieldName);
+                                parseObjectOrField(context, mapper);
+                                context.path().remove();
+                            } else {
+                                parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                            }
                         }
-                    }
-                } else {
-                    // TODO: shouldn't this skip, not parse?
-                    parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                        break;
+                    case FALSE:
+                        // TODO: shouldn't this skip, not parse?
+                        parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
                 }
                 for (int i = 0; i < parentMapperTuple.v1(); i++) {
                     context.path().remove();
@@ -692,11 +716,12 @@ private static void parseNullValue(ParseContext context, ObjectMapper parentMapp
         throws IOException {
         // we can only handle null values if we have mappings for them
         Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths);
+        ObjectMapper.Dynamic dynamic = parentMapper.dynamic();
         if (mapper != null) {
             // TODO: passing null to an object seems bogus?
             parseObjectOrField(context, mapper);
-        } else if (parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT) {
-            throw new StrictDynamicMappingException(parentMapper.fullPath(), lastFieldName);
+        } else if (dynamic == ObjectMapper.Dynamic.STRICT || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) {
+            throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), lastFieldName);
         }
     }
 
@@ -711,7 +736,9 @@ private static Mapper.Builder<?> newFloatBuilder(String name, Settings settings)
     private static Mapper.Builder<?> createBuilderFromDynamicValue(
         final ParseContext context,
         XContentParser.Token token,
-        String currentFieldName
+        String currentFieldName,
+        ObjectMapper.Dynamic dynamic,
+        String fullPath
     ) throws IOException {
         if (token == XContentParser.Token.VALUE_STRING) {
             String text = context.parser().text();
@@ -733,13 +760,13 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
             }
 
             if (parseableAsLong && context.root().numericDetection()) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
+                Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
                 if (builder == null) {
                     builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
                 }
                 return builder;
             } else if (parseableAsDouble && context.root().numericDetection()) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
+                Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
                 if (builder == null) {
                     builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings());
                 }
@@ -755,7 +782,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
                         // failure to parse this, continue
                         continue;
                     }
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, dateTimeFormatter);
+                    Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath);
                     if (builder == null) {
                         boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
                         builder = new DateFieldMapper.Builder(
@@ -771,7 +798,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
                 }
             }
 
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
+            Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
             if (builder == null) {
                 builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField(
                     new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
@@ -783,7 +810,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
             if (numberType == XContentParser.NumberType.INT
                 || numberType == XContentParser.NumberType.LONG
                 || numberType == XContentParser.NumberType.BIG_INTEGER) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
+                Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
                 if (builder == null) {
                     builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
                 }
@@ -791,7 +818,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
             } else if (numberType == XContentParser.NumberType.FLOAT
                 || numberType == XContentParser.NumberType.DOUBLE
                 || numberType == XContentParser.NumberType.BIG_DECIMAL) {
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
+                    Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
                     if (builder == null) {
                         // no templates are defined, we use float by default instead of double
                         // since this is much more space-efficient and should be enough most of
@@ -801,19 +828,19 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
                     return builder;
                 }
         } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN);
+            Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath);
             if (builder == null) {
                 builder = new BooleanFieldMapper.Builder(currentFieldName);
             }
             return builder;
         } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY);
+            Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath);
             if (builder == null) {
                 builder = new BinaryFieldMapper.Builder(currentFieldName);
             }
             return builder;
         } else {
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
+            Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
             if (builder != null) {
                 return builder;
             }
@@ -832,13 +859,13 @@ private static void parseDynamicValue(
     ) throws IOException {
         ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
         if (dynamic == ObjectMapper.Dynamic.STRICT) {
-            throw new StrictDynamicMappingException(parentMapper.fullPath(), currentFieldName);
+            throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), currentFieldName);
         }
         if (dynamic == ObjectMapper.Dynamic.FALSE) {
             return;
         }
         final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
-        final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName);
+        final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
         Mapper mapper = builder.build(builderContext);
         context.addDynamicMapper(mapper);
 
@@ -926,9 +953,16 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
 
                 switch (dynamic) {
                     case STRICT:
-                        throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
+                        throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]);
+                    case STRICT_ALLOW_TEMPLATES:
                     case TRUE:
-                        Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT);
+                        Mapper.Builder builder = findTemplateBuilder(
+                            context,
+                            paths[i],
+                            XContentFieldType.OBJECT,
+                            dynamic,
+                            parent.fullPath()
+                        );
                         if (builder == null) {
                             builder = new ObjectMapper.Builder(paths[i]).enabled(true);
                         }
@@ -1010,4 +1044,37 @@ private static Mapper getMapper(final ParseContext context, ObjectMapper objectM
         }
         return objectMapper.getMapper(subfields[subfields.length - 1]);
     }
+
+    // Throws exception if no dynamic templates found but `dynamic` is set to strict_allow_templates
+    @SuppressWarnings("rawtypes")
+    private static Mapper.Builder findTemplateBuilder(
+        ParseContext context,
+        String name,
+        XContentFieldType matchType,
+        ObjectMapper.Dynamic dynamic,
+        String fieldFullPath
+    ) {
+        Mapper.Builder builder = context.root().findTemplateBuilder(context, name, matchType);
+        if (builder == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) {
+            throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), fieldFullPath, name);
+        }
+
+        return builder;
+    }
+
+    // Throws exception if no dynamic templates found but `dynamic` is set to strict_allow_templates
+    @SuppressWarnings("rawtypes")
+    private static Mapper.Builder findTemplateBuilder(
+        ParseContext context,
+        String name,
+        DateFormatter dateFormat,
+        ObjectMapper.Dynamic dynamic,
+        String fieldFullPath
+    ) {
+        Mapper.Builder builder = context.root().findTemplateBuilder(context, name, dateFormat);
+        if (builder == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) {
+            throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), fieldFullPath, name);
+        }
+        return builder;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
index be3adfe8b2c4e..533e6ca73d737 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
@@ -92,7 +92,8 @@ public static class Defaults {
     public enum Dynamic {
         TRUE,
         FALSE,
-        STRICT
+        STRICT,
+        STRICT_ALLOW_TEMPLATES
     }
 
     /**
@@ -297,6 +298,8 @@ protected static boolean parseObjectOrDocumentTypeProperties(
                 String value = fieldNode.toString();
                 if (value.equalsIgnoreCase("strict")) {
                     builder.dynamic(Dynamic.STRICT);
+                } else if (value.equalsIgnoreCase("strict_allow_templates")) {
+                    builder.dynamic(Dynamic.STRICT_ALLOW_TEMPLATES);
                 } else {
                     boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic");
                     builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE);
diff --git a/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java b/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java
index 9127641128dad..0524c672011c5 100644
--- a/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java
+++ b/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java
@@ -43,8 +43,8 @@
  */
 public class StrictDynamicMappingException extends MapperParsingException {
 
-    public StrictDynamicMappingException(String path, String fieldName) {
-        super("mapping set to strict, dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed");
+    public StrictDynamicMappingException(String dynamic, String path, String fieldName) {
+        super("mapping set to " + dynamic + ", dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed");
     }
 
     public StrictDynamicMappingException(StreamInput in) throws IOException {
diff --git a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java
index b274cf28429e8..7a8c4ffe35021 100644
--- a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java
@@ -247,6 +247,46 @@ public void testCopyToStrictDynamicInnerObjectParsing() throws Exception {
         assertThat(e.getMessage(), startsWith("mapping set to strict, dynamic introduction of [very] within [_doc] is not allowed"));
     }
 
+    public void testCopyToStrictAllowTemplatesDynamicInnerObjectParsing() throws Exception {
+        DocumentMapper docMapper = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "test");
+                        b.startObject("mapping").field("type", "object").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+            b.startObject("properties");
+            {
+                b.startObject("copy_test");
+                {
+                    b.field("type", "text");
+                    b.field("copy_to", "very.inner.field");
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+
+        MapperParsingException e = expectThrows(
+            MapperParsingException.class,
+            () -> docMapper.parse(source(b -> b.field("copy_test", "foo")))
+        );
+
+        assertThat(
+            e.getMessage(),
+            startsWith("mapping set to strict_allow_templates, dynamic introduction of [very] within [_doc] is not allowed")
+        );
+    }
+
     public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception {
 
         DocumentMapper docMapper = createDocumentMapper(mapping(b -> {
diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java
index ecab9da8c6b6c..15e2b6649b0be 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java
@@ -878,6 +878,340 @@ public void testDynamicStrictDottedFieldNameLong() throws Exception {
         assertEquals("mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed", exception.getMessage());
     }
 
+    public void testDynamicStrictAllowTemplatesDottedFieldNameLong() throws Exception {
+        DocumentMapper documentMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates")));
+        StrictDynamicMappingException exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> documentMapper.parse(source(b -> b.field("foo.bar.baz", 0)))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        DocumentMapper documentMapperWithDynamicTemplates = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("path_match", "foo.bar.baz");
+                        b.startObject("mapping").field("type", "long").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }));
+        exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> documentMapperWithDynamicTemplates.parse(source(b -> b.field("foo.bar.baz", 0)))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "foo");
+                        b.field("match_mapping_type", "object");
+                        b.startObject("mapping").field("type", "object").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            {
+                b.startObject();
+                {
+                    b.startObject("test1");
+                    {
+                        b.field("match", "bar");
+                        b.field("match_mapping_type", "object");
+                        b.startObject("mapping").field("type", "object").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            {
+                b.startObject();
+                {
+                    b.startObject("test2");
+                    {
+                        b.field("path_match", "foo.bar.baz");
+                        b.startObject("mapping").field("type", "long").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }));
+
+        ParsedDocument doc = mapper.parse(source(b -> b.field("foo.bar.baz", 0)));
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+    }
+
+    public void testDynamicAllowTemplatesStrictLongArray() throws Exception {
+        DocumentMapper documentMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates")));
+        StrictDynamicMappingException exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> documentMapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray()))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        DocumentMapper documentMapperWithDynamicTemplates = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "test");
+                        b.startObject("mapping").field("type", "long").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }));
+        exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> documentMapperWithDynamicTemplates.parse(source(b -> b.startArray("foo").value(0).value(1).endArray()))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "foo");
+                        b.startObject("mapping").field("type", "long").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }));
+
+        ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray()));
+        assertEquals(4, doc.rootDoc().getFields("foo").length);
+    }
+
+    public void testDynamicStrictAllowTemplatesDottedFieldNameObject() throws Exception {
+        DocumentMapper documentMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates")));
+        StrictDynamicMappingException exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> documentMapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        DocumentMapper documentMapperWithDynamicTemplates = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "test");
+                        b.startObject("mapping").field("type", "long").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }));
+        exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> documentMapperWithDynamicTemplates.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "foo");
+                        b.field("match_mapping_type", "object");
+                        b.startObject("mapping").field("type", "object").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            {
+                b.startObject();
+                {
+                    b.startObject("test1");
+                    {
+                        b.field("match", "bar");
+                        b.field("match_mapping_type", "object");
+                        b.startObject("mapping").field("type", "object").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            {
+                b.startObject();
+                {
+                    b.startObject("test2");
+                    {
+                        b.field("match", "baz");
+                        b.field("match_mapping_type", "object");
+                        b.startObject("mapping").field("type", "object").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            {
+                b.startObject();
+                {
+                    b.startObject("test3");
+                    {
+                        b.field("path_match", "foo.bar.baz.a");
+                        b.startObject("mapping").field("type", "long").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }));
+
+        ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()));
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+    }
+
+    public void testDynamicStrictAllowTemplatesObject() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "test");
+                        b.field("match_mapping_type", "object");
+                        b.startObject("mapping").field("type", "object").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            {
+                b.startObject();
+                {
+                    b.startObject("test1");
+                    {
+                        b.field("match", "test1");
+                        b.startObject("mapping").field("type", "keyword").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }
+
+        ));
+        StrictDynamicMappingException exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> mapper.parse(source(b -> b.startObject("foo").field("bar", "baz").endObject()))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        ParsedDocument doc = mapper.parse(source(b -> b.startObject("test").field("test1", "baz").endObject()));
+        assertEquals(2, doc.rootDoc().getFields("test.test1").length);
+    }
+
+    public void testDynamicStrictAllowTemplatesValue() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "strict_allow_templates");
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "test*");
+                        b.field("match_mapping_type", "string");
+                        b.startObject("mapping").field("type", "keyword").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }
+
+        ));
+        StrictDynamicMappingException exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> mapper.parse(source(b -> b.field("bar", "baz")))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [bar] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+
+        ParsedDocument doc = mapper.parse(source(b -> b.field("test1", "baz")));
+        assertEquals(2, doc.rootDoc().getFields("test1").length);
+    }
+
+    public void testDynamicStrictAllowTemplatesNull() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates")));
+        StrictDynamicMappingException exception = expectThrows(
+            StrictDynamicMappingException.class,
+            () -> mapper.parse(source(b -> b.nullField("bar")))
+        );
+        assertEquals(
+            "mapping set to strict_allow_templates, dynamic introduction of [bar] within [_doc] is not allowed",
+            exception.getMessage()
+        );
+    }
+
     public void testDynamicDottedFieldNameObject() throws Exception {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
         ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()));

From afa479b2c5ce9a22220bf2f4de49ae4ca69c3bc7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:13:25 -0400
Subject: [PATCH 099/113] Bump net.minidev:json-smart from 2.5.0 to 2.5.1 in
 /plugins/repository-azure (#14748)

* Bump net.minidev:json-smart in /plugins/repository-azure

Bumps [net.minidev:json-smart](https://github.com/netplex/json-smart-v2) from 2.5.0 to 2.5.1.
- [Release notes](https://github.com/netplex/json-smart-v2/releases)
- [Commits](https://github.com/netplex/json-smart-v2/compare/2.5.0...2.5.1)

---
updated-dependencies:
- dependency-name: net.minidev:json-smart
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                                | 1 +
 plugins/repository-azure/build.gradle                       | 2 +-
 plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1 | 1 -
 plugins/repository-azure/licenses/json-smart-2.5.1.jar.sha1 | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/json-smart-2.5.1.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fbe2b7f50d446..6c260f8be9ca3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.13 to 0.9.14 ([#14672](https://github.com/opensearch-project/OpenSearch/pull/14672))
 - Bump `net.minidev:accessors-smart` from 2.5.0 to 2.5.1 ([#14673](https://github.com/opensearch-project/OpenSearch/pull/14673))
 - Bump `jackson` from 2.17.1 to 2.17.2 ([#14687](https://github.com/opensearch-project/OpenSearch/pull/14687))
+- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#14748](https://github.com/opensearch-project/OpenSearch/pull/14748))
 
 ### Changed
 - [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187))
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 0f822a02e05d8..980940e35b0b0 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -68,7 +68,7 @@ dependencies {
   api 'com.nimbusds:lang-tag:1.7'
   // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,
   // selected the higher version which is 2.5.0
-  api 'net.minidev:json-smart:2.5.0'
+  api 'net.minidev:json-smart:2.5.1'
   api 'net.minidev:accessors-smart:2.5.1'
   api "org.ow2.asm:asm:${versions.asm}"
   // End of transitive dependencies for azure-identity
diff --git a/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
deleted file mode 100644
index 3ec055efa1255..0000000000000
--- a/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-57a64f421b472849c40e77d2e7cce3a141b41e99
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/json-smart-2.5.1.jar.sha1 b/plugins/repository-azure/licenses/json-smart-2.5.1.jar.sha1
new file mode 100644
index 0000000000000..fe23968afce1e
--- /dev/null
+++ b/plugins/repository-azure/licenses/json-smart-2.5.1.jar.sha1
@@ -0,0 +1 @@
+4c11d2808d009132dfbbf947ebf37de6bf266c8e
\ No newline at end of file

From 26ac836751fd91afe7c140c08c5fd0bce3b216bb Mon Sep 17 00:00:00 2001
From: Chenyang Ji <cyji@amazon.com>
Date: Mon, 15 Jul 2024 12:56:33 -0700
Subject: [PATCH 100/113] remove query insights plugin from core (#14743)

Signed-off-by: Chenyang Ji <cyji@amazon.com>
---
 plugins/query-insights/build.gradle           |  18 -
 .../QueryInsightsPluginTransportIT.java       | 274 -------------
 .../plugin/insights/TopQueriesRestIT.java     | 107 -----
 .../plugin/insights/QueryInsightsPlugin.java  | 125 ------
 .../insights/core/exporter/DebugExporter.java |  61 ---
 .../core/exporter/LocalIndexExporter.java     | 113 ------
 .../core/exporter/QueryInsightsExporter.java  |  26 --
 .../QueryInsightsExporterFactory.java         | 143 -------
 .../insights/core/exporter/SinkType.java      |  66 ----
 .../insights/core/exporter/package-info.java  |  12 -
 .../core/listener/QueryInsightsListener.java  | 202 ----------
 .../insights/core/listener/package-info.java  |  12 -
 .../core/service/QueryInsightsService.java    | 283 -------------
 .../core/service/TopQueriesService.java       | 372 ------------------
 .../insights/core/service/package-info.java   |  12 -
 .../plugin/insights/package-info.java         |  12 -
 .../insights/rules/action/package-info.java   |  12 -
 .../rules/action/top_queries/TopQueries.java  |  77 ----
 .../action/top_queries/TopQueriesAction.java  |  32 --
 .../action/top_queries/TopQueriesRequest.java |  62 ---
 .../top_queries/TopQueriesResponse.java       | 143 -------
 .../action/top_queries/package-info.java      |  12 -
 .../insights/rules/model/Attribute.java       |  82 ----
 .../insights/rules/model/MetricType.java      | 119 ------
 .../rules/model/SearchQueryRecord.java        | 183 ---------
 .../insights/rules/model/package-info.java    |  12 -
 .../rules/resthandler/package-info.java       |  12 -
 .../top_queries/RestTopQueriesAction.java     |  99 -----
 .../resthandler/top_queries/package-info.java |  12 -
 .../rules/transport/package-info.java         |  12 -
 .../TransportTopQueriesAction.java            | 148 -------
 .../transport/top_queries/package-info.java   |  12 -
 .../settings/QueryInsightsSettings.java       | 304 --------------
 .../insights/settings/package-info.java       |  12 -
 .../insights/QueryInsightsPluginTests.java    | 113 ------
 .../insights/QueryInsightsTestUtils.java      | 205 ----------
 .../core/exporter/DebugExporterTests.java     |  37 --
 .../exporter/LocalIndexExporterTests.java     |  99 -----
 .../QueryInsightsExporterFactoryTests.java    |  89 -----
 .../listener/QueryInsightsListenerTests.java  | 217 ----------
 .../service/QueryInsightsServiceTests.java    |  65 ---
 .../core/service/TopQueriesServiceTests.java  | 112 ------
 .../top_queries/TopQueriesRequestTests.java   |  43 --
 .../top_queries/TopQueriesResponseTests.java  |  71 ----
 .../action/top_queries/TopQueriesTests.java   |  35 --
 .../rules/model/SearchQueryRecordTests.java   |  71 ----
 .../RestTopQueriesActionTests.java            |  70 ----
 .../TransportTopQueriesActionTests.java       |  85 ----
 48 files changed, 4495 deletions(-)
 delete mode 100644 plugins/query-insights/build.gradle
 delete mode 100644 plugins/query-insights/src/internalClusterTest/java/org/opensearch/plugin/insights/QueryInsightsPluginTransportIT.java
 delete mode 100644 plugins/query-insights/src/javaRestTest/java/org/opensearch/plugin/insights/TopQueriesRestIT.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/DebugExporter.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporter.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporter.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactory.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/SinkType.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueries.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesAction.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequest.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponse.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesAction.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesAction.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/package-info.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java
 delete mode 100644 plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/package-info.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/DebugExporterTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporterTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactoryTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequestTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecordTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesActionTests.java
 delete mode 100644 plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java

diff --git a/plugins/query-insights/build.gradle b/plugins/query-insights/build.gradle
deleted file mode 100644
index eabbd395bd3bd..0000000000000
--- a/plugins/query-insights/build.gradle
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-opensearchplugin {
-  description 'OpenSearch Query Insights Plugin.'
-  classname 'org.opensearch.plugin.insights.QueryInsightsPlugin'
-}
-
-dependencies {
-}
diff --git a/plugins/query-insights/src/internalClusterTest/java/org/opensearch/plugin/insights/QueryInsightsPluginTransportIT.java b/plugins/query-insights/src/internalClusterTest/java/org/opensearch/plugin/insights/QueryInsightsPluginTransportIT.java
deleted file mode 100644
index 04e715444f50a..0000000000000
--- a/plugins/query-insights/src/internalClusterTest/java/org/opensearch/plugin/insights/QueryInsightsPluginTransportIT.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights;
-
-import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.opensearch.action.admin.cluster.node.info.NodeInfo;
-import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
-import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.opensearch.action.admin.cluster.node.info.PluginsAndModules;
-import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
-import org.opensearch.action.index.IndexResponse;
-import org.opensearch.action.search.SearchResponse;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesAction;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesRequest;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesResponse;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugins.Plugin;
-import org.opensearch.plugins.PluginInfo;
-import org.opensearch.test.OpenSearchIntegTestCase;
-import org.junit.Assert;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE;
-import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
-
-/**
- * Transport Action tests for Query Insights Plugin
- */
-
-@OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST)
-public class QueryInsightsPluginTransportIT extends OpenSearchIntegTestCase {
-
-    private final int TOTAL_NUMBER_OF_NODES = 2;
-    private final int TOTAL_SEARCH_REQUESTS = 5;
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(QueryInsightsPlugin.class);
-    }
-
-    /**
-     * Test Query Insights Plugin is installed
-     */
-    public void testQueryInsightPluginInstalled() {
-        NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
-        nodesInfoRequest.addMetric(NodesInfoRequest.Metric.PLUGINS.metricName());
-        NodesInfoResponse nodesInfoResponse = OpenSearchIntegTestCase.client().admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
-        List<PluginInfo> pluginInfos = nodesInfoResponse.getNodes()
-            .stream()
-            .flatMap(
-                (Function<NodeInfo, Stream<PluginInfo>>) nodeInfo -> nodeInfo.getInfo(PluginsAndModules.class).getPluginInfos().stream()
-            )
-            .collect(Collectors.toList());
-        Assert.assertTrue(
-            pluginInfos.stream().anyMatch(pluginInfo -> pluginInfo.getName().equals("org.opensearch.plugin.insights.QueryInsightsPlugin"))
-        );
-    }
-
-    /**
-     * Test get top queries when feature disabled
-     */
-    public void testGetTopQueriesWhenFeatureDisabled() {
-        TopQueriesRequest request = new TopQueriesRequest(MetricType.LATENCY);
-        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
-        Assert.assertNotEquals(0, response.failures().size());
-        Assert.assertEquals(
-            "Cannot get top n queries for [latency] when it is not enabled.",
-            response.failures().get(0).getCause().getCause().getMessage()
-        );
-    }
-
-    /**
-     * Test update top query record when feature enabled
-     */
-    public void testUpdateRecordWhenFeatureDisabledThenEnabled() throws ExecutionException, InterruptedException {
-        Settings commonSettings = Settings.builder().put(TOP_N_LATENCY_QUERIES_ENABLED.getKey(), "false").build();
-
-        logger.info("--> starting nodes for query insight testing");
-        List<String> nodes = internalCluster().startNodes(TOTAL_NUMBER_OF_NODES, Settings.builder().put(commonSettings).build());
-
-        logger.info("--> waiting for nodes to form a cluster");
-        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
-        assertFalse(health.isTimedOut());
-
-        assertAcked(
-            prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 2))
-        );
-        ensureStableCluster(2);
-        logger.info("--> creating indices for query insight testing");
-        for (int i = 0; i < 5; i++) {
-            IndexResponse response = client().prepareIndex("test_" + i).setId("" + i).setSource("field_" + i, "value_" + i).get();
-            assertEquals("CREATED", response.status().toString());
-        }
-        // making search requests to get top queries
-        for (int i = 0; i < TOTAL_SEARCH_REQUESTS; i++) {
-            SearchResponse searchResponse = internalCluster().client(randomFrom(nodes))
-                .prepareSearch()
-                .setQuery(QueryBuilders.matchAllQuery())
-                .get();
-            assertEquals(searchResponse.getFailedShards(), 0);
-        }
-
-        TopQueriesRequest request = new TopQueriesRequest(MetricType.LATENCY);
-        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
-        Assert.assertNotEquals(0, response.failures().size());
-        Assert.assertEquals(
-            "Cannot get top n queries for [latency] when it is not enabled.",
-            response.failures().get(0).getCause().getCause().getMessage()
-        );
-
-        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().persistentSettings(
-            Settings.builder().put(TOP_N_LATENCY_QUERIES_ENABLED.getKey(), "true").build()
-        );
-        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get());
-        TopQueriesRequest request2 = new TopQueriesRequest(MetricType.LATENCY);
-        TopQueriesResponse response2 = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request2).actionGet();
-        Assert.assertEquals(0, response2.failures().size());
-        Assert.assertEquals(TOTAL_NUMBER_OF_NODES, response2.getNodes().size());
-        for (int i = 0; i < TOTAL_NUMBER_OF_NODES; i++) {
-            Assert.assertEquals(0, response2.getNodes().get(i).getTopQueriesRecord().size());
-        }
-
-        internalCluster().stopAllNodes();
-    }
-
-    /**
-     * Test get top queries when feature enabled
-     */
-    public void testGetTopQueriesWhenFeatureEnabled() throws InterruptedException {
-        Settings commonSettings = Settings.builder()
-            .put(TOP_N_LATENCY_QUERIES_ENABLED.getKey(), "true")
-            .put(TOP_N_LATENCY_QUERIES_SIZE.getKey(), "100")
-            .put(TOP_N_LATENCY_QUERIES_WINDOW_SIZE.getKey(), "600s")
-            .build();
-
-        logger.info("--> starting nodes for query insight testing");
-        List<String> nodes = internalCluster().startNodes(TOTAL_NUMBER_OF_NODES, Settings.builder().put(commonSettings).build());
-
-        logger.info("--> waiting for nodes to form a cluster");
-        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
-        assertFalse(health.isTimedOut());
-
-        assertAcked(
-            prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 2))
-        );
-        ensureStableCluster(2);
-        logger.info("--> creating indices for query insight testing");
-        for (int i = 0; i < 5; i++) {
-            IndexResponse response = client().prepareIndex("test_" + i).setId("" + i).setSource("field_" + i, "value_" + i).get();
-            assertEquals("CREATED", response.status().toString());
-        }
-        // making search requests to get top queries
-        for (int i = 0; i < TOTAL_SEARCH_REQUESTS; i++) {
-            SearchResponse searchResponse = internalCluster().client(randomFrom(nodes))
-                .prepareSearch()
-                .setQuery(QueryBuilders.matchAllQuery())
-                .get();
-            assertEquals(searchResponse.getFailedShards(), 0);
-        }
-        // Sleep to wait for queue drained to top queries store
-        Thread.sleep(6000);
-        TopQueriesRequest request = new TopQueriesRequest(MetricType.LATENCY);
-        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
-        Assert.assertEquals(0, response.failures().size());
-        Assert.assertEquals(TOTAL_NUMBER_OF_NODES, response.getNodes().size());
-        Assert.assertEquals(TOTAL_SEARCH_REQUESTS, response.getNodes().stream().mapToInt(o -> o.getTopQueriesRecord().size()).sum());
-
-        internalCluster().stopAllNodes();
-    }
-
-    /**
-     * Test get top queries with small top n size
-     */
-    public void testGetTopQueriesWithSmallTopN() throws InterruptedException {
-        Settings commonSettings = Settings.builder()
-            .put(TOP_N_LATENCY_QUERIES_ENABLED.getKey(), "true")
-            .put(TOP_N_LATENCY_QUERIES_SIZE.getKey(), "1")
-            .put(TOP_N_LATENCY_QUERIES_WINDOW_SIZE.getKey(), "600s")
-            .build();
-
-        logger.info("--> starting nodes for query insight testing");
-        List<String> nodes = internalCluster().startNodes(TOTAL_NUMBER_OF_NODES, Settings.builder().put(commonSettings).build());
-
-        logger.info("--> waiting for nodes to form a cluster");
-        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
-        assertFalse(health.isTimedOut());
-
-        assertAcked(
-            prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 2))
-        );
-        ensureStableCluster(2);
-        logger.info("--> creating indices for query insight testing");
-        for (int i = 0; i < 5; i++) {
-            IndexResponse response = client().prepareIndex("test_" + i).setId("" + i).setSource("field_" + i, "value_" + i).get();
-            assertEquals("CREATED", response.status().toString());
-        }
-        // making search requests to get top queries
-        for (int i = 0; i < TOTAL_SEARCH_REQUESTS; i++) {
-            SearchResponse searchResponse = internalCluster().client(randomFrom(nodes))
-                .prepareSearch()
-                .setQuery(QueryBuilders.matchAllQuery())
-                .get();
-            assertEquals(searchResponse.getFailedShards(), 0);
-        }
-        Thread.sleep(6000);
-        TopQueriesRequest request = new TopQueriesRequest(MetricType.LATENCY);
-        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
-        Assert.assertEquals(0, response.failures().size());
-        Assert.assertEquals(TOTAL_NUMBER_OF_NODES, response.getNodes().size());
-        Assert.assertEquals(2, response.getNodes().stream().mapToInt(o -> o.getTopQueriesRecord().size()).sum());
-
-        internalCluster().stopAllNodes();
-    }
-
-    /**
-     * Test get top queries with small window size
-     */
-    public void testGetTopQueriesWithSmallWindowSize() throws InterruptedException {
-        Settings commonSettings = Settings.builder()
-            .put(TOP_N_LATENCY_QUERIES_ENABLED.getKey(), "true")
-            .put(TOP_N_LATENCY_QUERIES_SIZE.getKey(), "100")
-            .put(TOP_N_LATENCY_QUERIES_WINDOW_SIZE.getKey(), "1m")
-            .build();
-
-        logger.info("--> starting nodes for query insight testing");
-        List<String> nodes = internalCluster().startNodes(TOTAL_NUMBER_OF_NODES, Settings.builder().put(commonSettings).build());
-
-        logger.info("--> waiting for nodes to form a cluster");
-        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
-        assertFalse(health.isTimedOut());
-
-        assertAcked(
-            prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 2))
-        );
-        ensureStableCluster(2);
-        logger.info("--> creating indices for query insight testing");
-        for (int i = 0; i < 5; i++) {
-            IndexResponse response = client().prepareIndex("test_" + i).setId("" + i).setSource("field_" + i, "value_" + i).get();
-            assertEquals("CREATED", response.status().toString());
-        }
-        // making search requests to get top queries
-        for (int i = 0; i < TOTAL_SEARCH_REQUESTS; i++) {
-            SearchResponse searchResponse = internalCluster().client(randomFrom(nodes))
-                .prepareSearch()
-                .setQuery(QueryBuilders.matchAllQuery())
-                .get();
-            assertEquals(searchResponse.getFailedShards(), 0);
-        }
-
-        TopQueriesRequest request = new TopQueriesRequest(MetricType.LATENCY);
-        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
-        Assert.assertEquals(0, response.failures().size());
-        Assert.assertEquals(TOTAL_NUMBER_OF_NODES, response.getNodes().size());
-        Thread.sleep(6000);
-        internalCluster().stopAllNodes();
-    }
-}
diff --git a/plugins/query-insights/src/javaRestTest/java/org/opensearch/plugin/insights/TopQueriesRestIT.java b/plugins/query-insights/src/javaRestTest/java/org/opensearch/plugin/insights/TopQueriesRestIT.java
deleted file mode 100644
index 57dea6ad8d5ff..0000000000000
--- a/plugins/query-insights/src/javaRestTest/java/org/opensearch/plugin/insights/TopQueriesRestIT.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights;
-
-import org.opensearch.client.Request;
-import org.opensearch.client.Response;
-import org.opensearch.common.xcontent.LoggingDeprecationHandler;
-import org.opensearch.common.xcontent.json.JsonXContent;
-import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.test.rest.OpenSearchRestTestCase;
-import org.junit.Assert;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Rest Action tests for Query Insights
- */
-public class TopQueriesRestIT extends OpenSearchRestTestCase {
-
-    /**
-     * test Query Insights is installed
-     * @throws IOException IOException
-     */
-    @SuppressWarnings("unchecked")
-    public void testQueryInsightsPluginInstalled() throws IOException {
-        Request request = new Request("GET", "/_cat/plugins?s=component&h=name,component,version,description&format=json");
-        Response response = client().performRequest(request);
-        List<Object> pluginsList = JsonXContent.jsonXContent.createParser(
-            NamedXContentRegistry.EMPTY,
-            LoggingDeprecationHandler.INSTANCE,
-            response.getEntity().getContent()
-        ).list();
-        Assert.assertTrue(
-            pluginsList.stream().map(o -> (Map<String, Object>) o).anyMatch(plugin -> plugin.get("component").equals("query-insights"))
-        );
-    }
-
-    /**
-     * test enabling top queries
-     * @throws IOException IOException
-     */
-    public void testTopQueriesResponses() throws IOException {
-        // Enable Top N Queries feature
-        Request request = new Request("PUT", "/_cluster/settings");
-        request.setJsonEntity(defaultTopQueriesSettings());
-        Response response = client().performRequest(request);
-
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-
-        // Create documents for search
-        request = new Request("POST", "/my-index-0/_doc");
-        request.setJsonEntity(createDocumentsBody());
-        response = client().performRequest(request);
-
-        Assert.assertEquals(201, response.getStatusLine().getStatusCode());
-
-        // Do Search
-        request = new Request("GET", "/my-index-0/_search?size=20&pretty");
-        request.setJsonEntity(searchBody());
-        response = client().performRequest(request);
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-        response = client().performRequest(request);
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-
-        // Get Top Queries
-        request = new Request("GET", "/_insights/top_queries?pretty");
-        response = client().performRequest(request);
-
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-        String top_requests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
-        Assert.assertTrue(top_requests.contains("top_queries"));
-        Assert.assertEquals(2, top_requests.split("searchType", -1).length - 1);
-    }
-
-    private String defaultTopQueriesSettings() {
-        return "{\n"
-            + "    \"persistent\" : {\n"
-            + "        \"search.top_n_queries.latency.enabled\" : \"true\",\n"
-            + "        \"search.top_n_queries.latency.window_size\" : \"600s\",\n"
-            + "        \"search.top_n_queries.latency.top_n_size\" : 5\n"
-            + "    }\n"
-            + "}";
-    }
-
-    private String createDocumentsBody() {
-        return "{\n"
-            + "  \"@timestamp\": \"2099-11-15T13:12:00\",\n"
-            + "  \"message\": \"this is document 1\",\n"
-            + "  \"user\": {\n"
-            + "    \"id\": \"cyji\"\n"
-            + "  }\n"
-            + "}";
-    }
-
-    private String searchBody() {
-        return "{}";
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java
deleted file mode 100644
index bba676436c39a..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights;
-
-import org.opensearch.action.ActionRequest;
-import org.opensearch.client.Client;
-import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
-import org.opensearch.cluster.node.DiscoveryNodes;
-import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.IndexScopedSettings;
-import org.opensearch.common.settings.Setting;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.settings.SettingsFilter;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.concurrent.OpenSearchExecutors;
-import org.opensearch.core.action.ActionResponse;
-import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
-import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.env.Environment;
-import org.opensearch.env.NodeEnvironment;
-import org.opensearch.plugin.insights.core.listener.QueryInsightsListener;
-import org.opensearch.plugin.insights.core.service.QueryInsightsService;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesAction;
-import org.opensearch.plugin.insights.rules.resthandler.top_queries.RestTopQueriesAction;
-import org.opensearch.plugin.insights.rules.transport.top_queries.TransportTopQueriesAction;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.plugins.ActionPlugin;
-import org.opensearch.plugins.Plugin;
-import org.opensearch.repositories.RepositoriesService;
-import org.opensearch.rest.RestController;
-import org.opensearch.rest.RestHandler;
-import org.opensearch.script.ScriptService;
-import org.opensearch.threadpool.ExecutorBuilder;
-import org.opensearch.threadpool.ScalingExecutorBuilder;
-import org.opensearch.threadpool.ThreadPool;
-import org.opensearch.watcher.ResourceWatcherService;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.function.Supplier;
-
-/**
- * Plugin class for Query Insights.
- */
-public class QueryInsightsPlugin extends Plugin implements ActionPlugin {
-    /**
-     * Default constructor
-     */
-    public QueryInsightsPlugin() {}
-
-    @Override
-    public Collection<Object> createComponents(
-        final Client client,
-        final ClusterService clusterService,
-        final ThreadPool threadPool,
-        final ResourceWatcherService resourceWatcherService,
-        final ScriptService scriptService,
-        final NamedXContentRegistry xContentRegistry,
-        final Environment environment,
-        final NodeEnvironment nodeEnvironment,
-        final NamedWriteableRegistry namedWriteableRegistry,
-        final IndexNameExpressionResolver indexNameExpressionResolver,
-        final Supplier<RepositoriesService> repositoriesServiceSupplier
-    ) {
-        // create top n queries service
-        final QueryInsightsService queryInsightsService = new QueryInsightsService(clusterService.getClusterSettings(), threadPool, client);
-        return List.of(queryInsightsService, new QueryInsightsListener(clusterService, queryInsightsService));
-    }
-
-    @Override
-    public List<ExecutorBuilder<?>> getExecutorBuilders(final Settings settings) {
-        return List.of(
-            new ScalingExecutorBuilder(
-                QueryInsightsSettings.QUERY_INSIGHTS_EXECUTOR,
-                1,
-                Math.min((OpenSearchExecutors.allocatedProcessors(settings) + 1) / 2, QueryInsightsSettings.MAX_THREAD_COUNT),
-                TimeValue.timeValueMinutes(5)
-            )
-        );
-    }
-
-    @Override
-    public List<RestHandler> getRestHandlers(
-        final Settings settings,
-        final RestController restController,
-        final ClusterSettings clusterSettings,
-        final IndexScopedSettings indexScopedSettings,
-        final SettingsFilter settingsFilter,
-        final IndexNameExpressionResolver indexNameExpressionResolver,
-        final Supplier<DiscoveryNodes> nodesInCluster
-    ) {
-        return List.of(new RestTopQueriesAction());
-    }
-
-    @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return List.of(new ActionPlugin.ActionHandler<>(TopQueriesAction.INSTANCE, TransportTopQueriesAction.class));
-    }
-
-    @Override
-    public List<Setting<?>> getSettings() {
-        return List.of(
-            // Settings for top N queries
-            QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED,
-            QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE,
-            QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE,
-            QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS,
-            QueryInsightsSettings.TOP_N_CPU_QUERIES_ENABLED,
-            QueryInsightsSettings.TOP_N_CPU_QUERIES_SIZE,
-            QueryInsightsSettings.TOP_N_CPU_QUERIES_WINDOW_SIZE,
-            QueryInsightsSettings.TOP_N_CPU_EXPORTER_SETTINGS,
-            QueryInsightsSettings.TOP_N_MEMORY_QUERIES_ENABLED,
-            QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE,
-            QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE,
-            QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS
-        );
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/DebugExporter.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/DebugExporter.java
deleted file mode 100644
index 116bd26e1f9bc..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/DebugExporter.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-
-import java.util.List;
-
-/**
- * Debug exporter for development purpose
- */
-public final class DebugExporter implements QueryInsightsExporter {
-    /**
-     * Logger of the debug exporter
-     */
-    private final Logger logger = LogManager.getLogger();
-
-    /**
-     * Constructor of DebugExporter
-     */
-    private DebugExporter() {}
-
-    private static class InstanceHolder {
-        private static final DebugExporter INSTANCE = new DebugExporter();
-    }
-
-    /**
-     Get the singleton instance of DebugExporter
-     *
-     @return DebugExporter instance
-     */
-    public static DebugExporter getInstance() {
-        return InstanceHolder.INSTANCE;
-    }
-
-    /**
-     * Write the list of SearchQueryRecord to debug log
-     *
-     * @param records list of {@link SearchQueryRecord}
-     */
-    @Override
-    public void export(final List<SearchQueryRecord> records) {
-        logger.debug("QUERY_INSIGHTS_RECORDS: " + records.toString());
-    }
-
-    /**
-     * Close the debugger exporter sink
-     */
-    @Override
-    public void close() {
-        logger.debug("Closing the DebugExporter..");
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporter.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporter.java
deleted file mode 100644
index c19fe3655098b..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporter.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.opensearch.action.bulk.BulkRequestBuilder;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.client.Client;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.action.ActionListener;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.util.List;
-
-/**
- * Local index exporter for exporting query insights data to local OpenSearch indices.
- */
-public final class LocalIndexExporter implements QueryInsightsExporter {
-    /**
-     * Logger of the local index exporter
-     */
-    private final Logger logger = LogManager.getLogger();
-    private final Client client;
-    private DateTimeFormatter indexPattern;
-
-    /**
-     * Constructor of LocalIndexExporter
-     *
-     * @param client OS client
-     * @param indexPattern the pattern of index to export to
-     */
-    public LocalIndexExporter(final Client client, final DateTimeFormatter indexPattern) {
-        this.indexPattern = indexPattern;
-        this.client = client;
-    }
-
-    /**
-     * Getter of indexPattern
-     *
-     * @return indexPattern
-     */
-    public DateTimeFormatter getIndexPattern() {
-        return indexPattern;
-    }
-
-    /**
-     * Setter of indexPattern
-     *
-     * @param indexPattern index pattern
-     * @return the current LocalIndexExporter
-     */
-    public LocalIndexExporter setIndexPattern(DateTimeFormatter indexPattern) {
-        this.indexPattern = indexPattern;
-        return this;
-    }
-
-    /**
-     * Export a list of SearchQueryRecord to a local index
-     *
-     * @param records list of {@link SearchQueryRecord}
-     */
-    @Override
-    public void export(final List<SearchQueryRecord> records) {
-        if (records == null || records.size() == 0) {
-            return;
-        }
-        try {
-            final String index = getDateTimeFromFormat();
-            final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk().setTimeout(TimeValue.timeValueMinutes(1));
-            for (SearchQueryRecord record : records) {
-                bulkRequestBuilder.add(
-                    new IndexRequest(index).source(record.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
-                );
-            }
-            bulkRequestBuilder.execute(new ActionListener<BulkResponse>() {
-                @Override
-                public void onResponse(BulkResponse bulkItemResponses) {}
-
-                @Override
-                public void onFailure(Exception e) {
-                    logger.error("Failed to execute bulk operation for query insights data: ", e);
-                }
-            });
-        } catch (final Exception e) {
-            logger.error("Unable to index query insights data: ", e);
-        }
-    }
-
-    /**
-     * Close the exporter sink
-     */
-    @Override
-    public void close() {
-        logger.debug("Closing the LocalIndexExporter..");
-    }
-
-    private String getDateTimeFromFormat() {
-        return indexPattern.print(DateTime.now(DateTimeZone.UTC));
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporter.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporter.java
deleted file mode 100644
index 42e5354eb1640..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporter.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-
-import java.io.Closeable;
-import java.util.List;
-
-/**
- * Base interface for Query Insights exporters
- */
-public interface QueryInsightsExporter extends Closeable {
-    /**
-     * Export a list of SearchQueryRecord to the exporter sink
-     *
-     * @param records list of {@link SearchQueryRecord}
-     */
-    void export(final List<SearchQueryRecord> records);
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactory.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactory.java
deleted file mode 100644
index 016911761a3d0..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactory.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.opensearch.client.Client;
-import org.opensearch.common.settings.Settings;
-import org.joda.time.format.DateTimeFormat;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Set;
-
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_QUERIES_INDEX_PATTERN;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_QUERIES_EXPORTER_TYPE;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORTER_TYPE;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORT_INDEX;
-
-/**
- * Factory class for validating and creating exporters based on provided settings
- */
-public class QueryInsightsExporterFactory {
-    /**
-     * Logger of the query insights exporter factory
-     */
-    private final Logger logger = LogManager.getLogger();
-    final private Client client;
-    final private Set<QueryInsightsExporter> exporters;
-
-    /**
-     * Constructor of QueryInsightsExporterFactory
-     *
-     * @param client OS client
-     */
-    public QueryInsightsExporterFactory(final Client client) {
-        this.client = client;
-        this.exporters = new HashSet<>();
-    }
-
-    /**
-     * Validate exporter sink config
-     *
-     * @param settings exporter sink config {@link Settings}
-     * @throws IllegalArgumentException if provided exporter sink config settings are invalid
-     */
-    public void validateExporterConfig(final Settings settings) throws IllegalArgumentException {
-        // Disable exporter if the EXPORTER_TYPE setting is null
-        if (settings.get(EXPORTER_TYPE) == null) {
-            return;
-        }
-        SinkType type;
-        try {
-            type = SinkType.parse(settings.get(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE));
-        } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException(
-                String.format(
-                    Locale.ROOT,
-                    "Invalid exporter type [%s], type should be one of %s",
-                    settings.get(EXPORTER_TYPE),
-                    SinkType.allSinkTypes()
-                )
-            );
-        }
-        switch (type) {
-            case LOCAL_INDEX:
-                final String indexPattern = settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN);
-                if (indexPattern.length() == 0) {
-                    throw new IllegalArgumentException("Empty index pattern configured for the exporter");
-                }
-                try {
-                    DateTimeFormat.forPattern(indexPattern);
-                } catch (Exception e) {
-                    throw new IllegalArgumentException(
-                        String.format(Locale.ROOT, "Invalid index pattern [%s] configured for the exporter", indexPattern)
-                    );
-                }
-        }
-    }
-
-    /**
-     * Create an exporter based on provided parameters
-     *
-     * @param type The type of exporter to create
-     * @param indexPattern the index pattern if creating a index exporter
-     * @return QueryInsightsExporter the created exporter sink
-     */
-    public QueryInsightsExporter createExporter(SinkType type, String indexPattern) {
-        if (SinkType.LOCAL_INDEX.equals(type)) {
-            QueryInsightsExporter exporter = new LocalIndexExporter(client, DateTimeFormat.forPattern(indexPattern));
-            this.exporters.add(exporter);
-            return exporter;
-        }
-        return DebugExporter.getInstance();
-    }
-
-    /**
-     * Update an exporter based on provided parameters
-     *
-     * @param exporter The exporter to update
-     * @param indexPattern the index pattern if creating a index exporter
-     * @return QueryInsightsExporter the updated exporter sink
-     */
-    public QueryInsightsExporter updateExporter(QueryInsightsExporter exporter, String indexPattern) {
-        if (exporter.getClass() == LocalIndexExporter.class) {
-            ((LocalIndexExporter) exporter).setIndexPattern(DateTimeFormat.forPattern(indexPattern));
-        }
-        return exporter;
-    }
-
-    /**
-     * Close an exporter
-     *
-     * @param exporter the exporter to close
-     */
-    public void closeExporter(QueryInsightsExporter exporter) throws IOException {
-        if (exporter != null) {
-            exporter.close();
-            this.exporters.remove(exporter);
-        }
-    }
-
-    /**
-     * Close all exporters
-     *
-     */
-    public void closeAllExporters() {
-        for (QueryInsightsExporter exporter : exporters) {
-            try {
-                closeExporter(exporter);
-            } catch (IOException e) {
-                logger.error("Fail to close query insights exporter, error: ", e);
-            }
-        }
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/SinkType.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/SinkType.java
deleted file mode 100644
index c90c9c76b6706..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/SinkType.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import java.util.Arrays;
-import java.util.Locale;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * Type of supported sinks
- */
-public enum SinkType {
-    /** debug exporter */
-    DEBUG("debug"),
-    /** local index exporter */
-    LOCAL_INDEX("local_index");
-
-    private final String type;
-
-    SinkType(String type) {
-        this.type = type;
-    }
-
-    @Override
-    public String toString() {
-        return type;
-    }
-
-    /**
-     * Parse SinkType from String
-     * @param type the String representation of the SinkType
-     * @return SinkType
-     */
-    public static SinkType parse(final String type) {
-        return valueOf(type.toUpperCase(Locale.ROOT));
-    }
-
-    /**
-     * Get all valid SinkTypes
-     *
-     * @return A set contains all valid SinkTypes
-     */
-    public static Set<SinkType> allSinkTypes() {
-        return Arrays.stream(values()).collect(Collectors.toSet());
-    }
-
-    /**
-     * Get Sink type from exporter
-     *
-     * @param exporter the {@link QueryInsightsExporter}
-     * @return SinkType associated with this exporter
-     */
-    public static SinkType getSinkTypeFromExporter(QueryInsightsExporter exporter) {
-        if (exporter.getClass().equals(LocalIndexExporter.class)) {
-            return SinkType.LOCAL_INDEX;
-        }
-        return SinkType.DEBUG;
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/package-info.java
deleted file mode 100644
index 7164411194f85..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Query Insights exporter
- */
-package org.opensearch.plugin.insights.core.exporter;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java
deleted file mode 100644
index a1f810ad5987c..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.listener;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.opensearch.action.search.SearchPhaseContext;
-import org.opensearch.action.search.SearchRequest;
-import org.opensearch.action.search.SearchRequestContext;
-import org.opensearch.action.search.SearchRequestOperationsListener;
-import org.opensearch.action.search.SearchTask;
-import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.inject.Inject;
-import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.plugin.insights.core.service.QueryInsightsService;
-import org.opensearch.plugin.insights.rules.model.Attribute;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.tasks.Task;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNEnabledSetting;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNSizeSetting;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNWindowSizeSetting;
-
-/**
- * The listener for query insights services.
- * It forwards query-related data to the appropriate query insights stores,
- * either for each request or for each phase.
- *
- * @opensearch.internal
- */
-public final class QueryInsightsListener extends SearchRequestOperationsListener {
-    private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
-
-    private static final Logger log = LogManager.getLogger(QueryInsightsListener.class);
-
-    private final QueryInsightsService queryInsightsService;
-    private final ClusterService clusterService;
-
-    /**
-     * Constructor for QueryInsightsListener
-     *
-     * @param clusterService The Node's cluster service.
-     * @param queryInsightsService The topQueriesByLatencyService associated with this listener
-     */
-    @Inject
-    public QueryInsightsListener(final ClusterService clusterService, final QueryInsightsService queryInsightsService) {
-        this.clusterService = clusterService;
-        this.queryInsightsService = queryInsightsService;
-        // Setting endpoints set up for top n queries, including enabling top n queries, window size and top n size
-        // Expected metricTypes are Latency, CPU and Memory.
-        for (MetricType type : MetricType.allMetricTypes()) {
-            clusterService.getClusterSettings()
-                .addSettingsUpdateConsumer(getTopNEnabledSetting(type), v -> this.setEnableTopQueries(type, v));
-            clusterService.getClusterSettings()
-                .addSettingsUpdateConsumer(
-                    getTopNSizeSetting(type),
-                    v -> this.queryInsightsService.setTopNSize(type, v),
-                    v -> this.queryInsightsService.validateTopNSize(type, v)
-                );
-            clusterService.getClusterSettings()
-                .addSettingsUpdateConsumer(
-                    getTopNWindowSizeSetting(type),
-                    v -> this.queryInsightsService.setWindowSize(type, v),
-                    v -> this.queryInsightsService.validateWindowSize(type, v)
-                );
-
-            this.setEnableTopQueries(type, clusterService.getClusterSettings().get(getTopNEnabledSetting(type)));
-            this.queryInsightsService.validateTopNSize(type, clusterService.getClusterSettings().get(getTopNSizeSetting(type)));
-            this.queryInsightsService.setTopNSize(type, clusterService.getClusterSettings().get(getTopNSizeSetting(type)));
-            this.queryInsightsService.validateWindowSize(type, clusterService.getClusterSettings().get(getTopNWindowSizeSetting(type)));
-            this.queryInsightsService.setWindowSize(type, clusterService.getClusterSettings().get(getTopNWindowSizeSetting(type)));
-        }
-    }
-
-    /**
-     * Enable or disable top queries insights collection for {@link MetricType}
-     * This function will enable or disable the corresponding listeners
-     * and query insights services.
-     *
-     * @param metricType {@link MetricType}
-     * @param enabled boolean
-     */
-    public void setEnableTopQueries(final MetricType metricType, final boolean enabled) {
-        boolean isAllMetricsDisabled = !queryInsightsService.isEnabled();
-        this.queryInsightsService.enableCollection(metricType, enabled);
-        if (!enabled) {
-            // disable QueryInsightsListener only if all metrics collections are disabled now.
-            if (!queryInsightsService.isEnabled()) {
-                super.setEnabled(false);
-                this.queryInsightsService.stop();
-            }
-        } else {
-            super.setEnabled(true);
-            // restart QueryInsightsListener only if none of metrics collections is enabled before.
-            if (isAllMetricsDisabled) {
-                this.queryInsightsService.stop();
-                this.queryInsightsService.start();
-            }
-        }
-
-    }
-
-    @Override
-    public boolean isEnabled() {
-        return super.isEnabled();
-    }
-
-    @Override
-    public void onPhaseStart(SearchPhaseContext context) {}
-
-    @Override
-    public void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) {}
-
-    @Override
-    public void onPhaseFailure(SearchPhaseContext context, Throwable cause) {}
-
-    @Override
-    public void onRequestStart(SearchRequestContext searchRequestContext) {}
-
-    @Override
-    public void onRequestEnd(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
-        constructSearchQueryRecord(context, searchRequestContext);
-    }
-
-    @Override
-    public void onRequestFailure(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
-        constructSearchQueryRecord(context, searchRequestContext);
-    }
-
-    private void constructSearchQueryRecord(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
-        SearchTask searchTask = context.getTask();
-        List<TaskResourceInfo> tasksResourceUsages = searchRequestContext.getPhaseResourceUsage();
-        tasksResourceUsages.add(
-            new TaskResourceInfo(
-                searchTask.getAction(),
-                searchTask.getId(),
-                searchTask.getParentTaskId().getId(),
-                clusterService.localNode().getId(),
-                searchTask.getTotalResourceStats()
-            )
-        );
-
-        final SearchRequest request = context.getRequest();
-        try {
-            Map<MetricType, Number> measurements = new HashMap<>();
-            if (queryInsightsService.isCollectionEnabled(MetricType.LATENCY)) {
-                measurements.put(
-                    MetricType.LATENCY,
-                    TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos())
-                );
-            }
-            if (queryInsightsService.isCollectionEnabled(MetricType.CPU)) {
-                measurements.put(
-                    MetricType.CPU,
-                    tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getCpuTimeInNanos()).mapToLong(Long::longValue).sum()
-                );
-            }
-            if (queryInsightsService.isCollectionEnabled(MetricType.MEMORY)) {
-                measurements.put(
-                    MetricType.MEMORY,
-                    tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getMemoryInBytes()).mapToLong(Long::longValue).sum()
-                );
-            }
-            Map<Attribute, Object> attributes = new HashMap<>();
-            attributes.put(Attribute.SEARCH_TYPE, request.searchType().toString().toLowerCase(Locale.ROOT));
-            attributes.put(Attribute.SOURCE, request.source().toString(FORMAT_PARAMS));
-            attributes.put(Attribute.TOTAL_SHARDS, context.getNumShards());
-            attributes.put(Attribute.INDICES, request.indices());
-            attributes.put(Attribute.PHASE_LATENCY_MAP, searchRequestContext.phaseTookMap());
-            attributes.put(Attribute.TASK_RESOURCE_USAGES, tasksResourceUsages);
-
-            Map<String, Object> labels = new HashMap<>();
-            // Retrieve user provided label if exists
-            String userProvidedLabel = context.getTask().getHeader(Task.X_OPAQUE_ID);
-            if (userProvidedLabel != null) {
-                labels.put(Task.X_OPAQUE_ID, userProvidedLabel);
-            }
-            attributes.put(Attribute.LABELS, labels);
-            // construct SearchQueryRecord from attributes and measurements
-            SearchQueryRecord record = new SearchQueryRecord(request.getOrCreateAbsoluteStartMillis(), measurements, attributes);
-            queryInsightsService.addRecord(record);
-        } catch (Exception e) {
-            log.error(String.format(Locale.ROOT, "fail to ingest query insight data, error: %s", e));
-        }
-    }
-
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/package-info.java
deleted file mode 100644
index 3cb9cacf7fd1c..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Listeners for Query Insights
- */
-package org.opensearch.plugin.insights.core.listener;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java
deleted file mode 100644
index c63430a1a726c..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.service;
-
-import org.opensearch.client.Client;
-import org.opensearch.common.inject.Inject;
-import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.threadpool.Scheduler;
-import org.opensearch.threadpool.ThreadPool;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getExporterSettings;
-
-/**
- * Service responsible for gathering, analyzing, storing and exporting
- * information related to search queries
- *
- * @opensearch.internal
- */
-public class QueryInsightsService extends AbstractLifecycleComponent {
-    /**
-     * The internal OpenSearch thread pool that execute async processing and exporting tasks
-     */
-    private final ThreadPool threadPool;
-
-    /**
-     * Services to capture top n queries for different metric types
-     */
-    private final Map<MetricType, TopQueriesService> topQueriesServices;
-
-    /**
-     * Flags for enabling insight data collection for different metric types
-     */
-    private final Map<MetricType, Boolean> enableCollect;
-
-    /**
-     * The internal thread-safe queue to ingest the search query data and subsequently forward to processors
-     */
-    private final LinkedBlockingQueue<SearchQueryRecord> queryRecordsQueue;
-
-    /**
-     * Holds a reference to delayed operation {@link Scheduler.Cancellable} so it can be cancelled when
-     * the service closed concurrently.
-     */
-    protected volatile Scheduler.Cancellable scheduledFuture;
-
-    /**
-     * Query Insights exporter factory
-     */
-    final QueryInsightsExporterFactory queryInsightsExporterFactory;
-
-    /**
-     * Constructor of the QueryInsightsService
-     *
-     * @param clusterSettings OpenSearch cluster level settings
-     * @param threadPool The OpenSearch thread pool to run async tasks
-     * @param client OS client
-     */
-    @Inject
-    public QueryInsightsService(final ClusterSettings clusterSettings, final ThreadPool threadPool, final Client client) {
-        enableCollect = new HashMap<>();
-        queryRecordsQueue = new LinkedBlockingQueue<>(QueryInsightsSettings.QUERY_RECORD_QUEUE_CAPACITY);
-        this.threadPool = threadPool;
-        this.queryInsightsExporterFactory = new QueryInsightsExporterFactory(client);
-        // initialize top n queries services and configurations consumers
-        topQueriesServices = new HashMap<>();
-        for (MetricType metricType : MetricType.allMetricTypes()) {
-            enableCollect.put(metricType, false);
-            topQueriesServices.put(metricType, new TopQueriesService(metricType, threadPool, queryInsightsExporterFactory));
-        }
-        for (MetricType type : MetricType.allMetricTypes()) {
-            clusterSettings.addSettingsUpdateConsumer(
-                getExporterSettings(type),
-                (settings -> setExporter(type, settings)),
-                (settings -> validateExporterConfig(type, settings))
-            );
-        }
-    }
-
-    /**
-     * Ingest the query data into in-memory stores
-     *
-     * @param record the record to ingest
-     */
-    public boolean addRecord(final SearchQueryRecord record) {
-        boolean shouldAdd = false;
-        for (Map.Entry<MetricType, TopQueriesService> entry : topQueriesServices.entrySet()) {
-            if (!enableCollect.get(entry.getKey())) {
-                continue;
-            }
-            List<SearchQueryRecord> currentSnapshot = entry.getValue().getTopQueriesCurrentSnapshot();
-            // skip add to top N queries store if the incoming record is smaller than the Nth record
-            if (currentSnapshot.size() < entry.getValue().getTopNSize()
-                || SearchQueryRecord.compare(record, currentSnapshot.get(0), entry.getKey()) > 0) {
-                shouldAdd = true;
-                break;
-            }
-        }
-        if (shouldAdd) {
-            return queryRecordsQueue.offer(record);
-        }
-        return false;
-    }
-
-    /**
-     * Drain the queryRecordsQueue into internal stores and services
-     */
-    public void drainRecords() {
-        final List<SearchQueryRecord> records = new ArrayList<>();
-        queryRecordsQueue.drainTo(records);
-        records.sort(Comparator.comparingLong(SearchQueryRecord::getTimestamp));
-        for (MetricType metricType : MetricType.allMetricTypes()) {
-            if (enableCollect.get(metricType)) {
-                // ingest the records into topQueriesService
-                topQueriesServices.get(metricType).consumeRecords(records);
-            }
-        }
-    }
-
-    /**
-     * Get the top queries service based on metricType
-     * @param metricType {@link MetricType}
-     * @return {@link TopQueriesService}
-     */
-    public TopQueriesService getTopQueriesService(final MetricType metricType) {
-        return topQueriesServices.get(metricType);
-    }
-
-    /**
-     * Set flag to enable or disable Query Insights data collection
-     *
-     * @param metricType {@link MetricType}
-     * @param enable Flag to enable or disable Query Insights data collection
-     */
-    public void enableCollection(final MetricType metricType, final boolean enable) {
-        this.enableCollect.put(metricType, enable);
-        this.topQueriesServices.get(metricType).setEnabled(enable);
-    }
-
-    /**
-     * Get if the Query Insights data collection is enabled for a MetricType
-     *
-     * @param metricType {@link MetricType}
-     * @return if the Query Insights data collection is enabled
-     */
-    public boolean isCollectionEnabled(final MetricType metricType) {
-        return this.enableCollect.get(metricType);
-    }
-
-    /**
-     * Check if query insights service is enabled
-     *
-     * @return if query insights service is enabled
-     */
-    public boolean isEnabled() {
-        for (MetricType t : MetricType.allMetricTypes()) {
-            if (isCollectionEnabled(t)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Validate the window size config for a metricType
-     *
-     * @param type {@link MetricType}
-     * @param windowSize {@link TimeValue}
-     */
-    public void validateWindowSize(final MetricType type, final TimeValue windowSize) {
-        if (topQueriesServices.containsKey(type)) {
-            topQueriesServices.get(type).validateWindowSize(windowSize);
-        }
-    }
-
-    /**
-     * Set window size for a metricType
-     *
-     * @param type {@link MetricType}
-     * @param windowSize {@link TimeValue}
-     */
-    public void setWindowSize(final MetricType type, final TimeValue windowSize) {
-        if (topQueriesServices.containsKey(type)) {
-            topQueriesServices.get(type).setWindowSize(windowSize);
-        }
-    }
-
-    /**
-     * Validate the top n size config for a metricType
-     *
-     * @param type {@link MetricType}
-     * @param topNSize top n size
-     */
-    public void validateTopNSize(final MetricType type, final int topNSize) {
-        if (topQueriesServices.containsKey(type)) {
-            topQueriesServices.get(type).validateTopNSize(topNSize);
-        }
-    }
-
-    /**
-     * Set the top n size config for a metricType
-     *
-     * @param type {@link MetricType}
-     * @param topNSize top n size
-     */
-    public void setTopNSize(final MetricType type, final int topNSize) {
-        if (topQueriesServices.containsKey(type)) {
-            topQueriesServices.get(type).setTopNSize(topNSize);
-        }
-    }
-
-    /**
-     * Set the exporter config for a metricType
-     *
-     * @param type {@link MetricType}
-     * @param settings exporter settings
-     */
-    public void setExporter(final MetricType type, final Settings settings) {
-        if (topQueriesServices.containsKey(type)) {
-            topQueriesServices.get(type).setExporter(settings);
-        }
-    }
-
-    /**
-     * Validate the exporter config for a metricType
-     *
-     * @param type {@link MetricType}
-     * @param settings exporter settings
-     */
-    public void validateExporterConfig(final MetricType type, final Settings settings) {
-        if (topQueriesServices.containsKey(type)) {
-            topQueriesServices.get(type).validateExporterConfig(settings);
-        }
-    }
-
-    @Override
-    protected void doStart() {
-        if (isEnabled()) {
-            scheduledFuture = threadPool.scheduleWithFixedDelay(
-                this::drainRecords,
-                QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL,
-                QueryInsightsSettings.QUERY_INSIGHTS_EXECUTOR
-            );
-        }
-    }
-
-    @Override
-    protected void doStop() {
-        if (scheduledFuture != null) {
-            scheduledFuture.cancel();
-        }
-    }
-
-    @Override
-    protected void doClose() throws IOException {
-        // close all top n queries service
-        for (TopQueriesService topQueriesService : topQueriesServices.values()) {
-            topQueriesService.close();
-        }
-        // close any unclosed resources
-        queryInsightsExporterFactory.closeAllExporters();
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java
deleted file mode 100644
index bbe8b8fc40dac..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.service;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporter;
-import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory;
-import org.opensearch.plugin.insights.core.exporter.SinkType;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.threadpool.ThreadPool;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Locale;
-import java.util.PriorityQueue;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_QUERIES_INDEX_PATTERN;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_QUERIES_EXPORTER_TYPE;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORTER_TYPE;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORT_INDEX;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.QUERY_INSIGHTS_EXECUTOR;
-
-/**
- * Service responsible for gathering and storing top N queries
- * with high latency or resource usage
- *
- * @opensearch.internal
- */
-public class TopQueriesService {
-    /**
-     * Logger of the local index exporter
-     */
-    private final Logger logger = LogManager.getLogger();
-    private boolean enabled;
-    /**
-     * The metric type to measure top n queries
-     */
-    private final MetricType metricType;
-    private int topNSize;
-    /**
-     * The window size to keep the top n queries
-     */
-    private TimeValue windowSize;
-    /**
-     * The current window start timestamp
-     */
-    private long windowStart;
-    /**
-     * The internal thread-safe store that holds the top n queries insight data
-     */
-    private final PriorityQueue<SearchQueryRecord> topQueriesStore;
-
-    /**
-     * The AtomicReference of a snapshot of the current window top queries for getters to consume
-     */
-    private final AtomicReference<List<SearchQueryRecord>> topQueriesCurrentSnapshot;
-
-    /**
-     * The AtomicReference of a snapshot of the last window top queries for getters to consume
-     */
-    private final AtomicReference<List<SearchQueryRecord>> topQueriesHistorySnapshot;
-
-    /**
-     * Factory for validating and creating exporters
-     */
-    private final QueryInsightsExporterFactory queryInsightsExporterFactory;
-
-    /**
-     * The internal OpenSearch thread pool that execute async processing and exporting tasks
-     */
-    private final ThreadPool threadPool;
-
-    /**
-     * Exporter for exporting top queries data
-     */
-    private QueryInsightsExporter exporter;
-
-    TopQueriesService(
-        final MetricType metricType,
-        final ThreadPool threadPool,
-        final QueryInsightsExporterFactory queryInsightsExporterFactory
-    ) {
-        this.enabled = false;
-        this.metricType = metricType;
-        this.threadPool = threadPool;
-        this.queryInsightsExporterFactory = queryInsightsExporterFactory;
-        this.topNSize = QueryInsightsSettings.DEFAULT_TOP_N_SIZE;
-        this.windowSize = QueryInsightsSettings.DEFAULT_WINDOW_SIZE;
-        this.windowStart = -1L;
-        this.exporter = null;
-        topQueriesStore = new PriorityQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType));
-        topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>());
-        topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>());
-    }
-
-    /**
-     * Set the top N size for TopQueriesService service.
-     *
-     * @param topNSize the top N size to set
-     */
-    public void setTopNSize(final int topNSize) {
-        this.topNSize = topNSize;
-    }
-
-    /**
-     * Get the current configured top n size
-     *
-     * @return top n size
-     */
-    public int getTopNSize() {
-        return topNSize;
-    }
-
-    /**
-     * Validate the top N size based on the internal constrains
-     *
-     * @param size the wanted top N size
-     */
-    public void validateTopNSize(final int size) {
-        if (size < 1 || size > QueryInsightsSettings.MAX_N_SIZE) {
-            throw new IllegalArgumentException(
-                "Top N size setting for ["
-                    + metricType
-                    + "]"
-                    + " should be between 1 and "
-                    + QueryInsightsSettings.MAX_N_SIZE
-                    + ", was ("
-                    + size
-                    + ")"
-            );
-        }
-    }
-
-    /**
-     * Set enable flag for the service
-     * @param enabled boolean
-     */
-    public void setEnabled(final boolean enabled) {
-        this.enabled = enabled;
-    }
-
-    /**
-     * Set the window size for top N queries service
-     *
-     * @param windowSize window size to set
-     */
-    public void setWindowSize(final TimeValue windowSize) {
-        this.windowSize = windowSize;
-        // reset the window start time since the window size has changed
-        this.windowStart = -1L;
-    }
-
-    /**
-     * Validate if the window size is valid, based on internal constrains.
-     *
-     * @param windowSize the window size to validate
-     */
-    public void validateWindowSize(final TimeValue windowSize) {
-        if (windowSize.compareTo(QueryInsightsSettings.MAX_WINDOW_SIZE) > 0
-            || windowSize.compareTo(QueryInsightsSettings.MIN_WINDOW_SIZE) < 0) {
-            throw new IllegalArgumentException(
-                "Window size setting for ["
-                    + metricType
-                    + "]"
-                    + " should be between ["
-                    + QueryInsightsSettings.MIN_WINDOW_SIZE
-                    + ","
-                    + QueryInsightsSettings.MAX_WINDOW_SIZE
-                    + "]"
-                    + "was ("
-                    + windowSize
-                    + ")"
-            );
-        }
-        if (!(QueryInsightsSettings.VALID_WINDOW_SIZES_IN_MINUTES.contains(windowSize) || windowSize.getMinutes() % 60 == 0)) {
-            throw new IllegalArgumentException(
-                "Window size setting for ["
-                    + metricType
-                    + "]"
-                    + " should be multiple of 1 hour, or one of "
-                    + QueryInsightsSettings.VALID_WINDOW_SIZES_IN_MINUTES
-                    + ", was ("
-                    + windowSize
-                    + ")"
-            );
-        }
-    }
-
-    /**
-     * Set up the top queries exporter based on provided settings
-     *
-     * @param settings exporter config {@link Settings}
-     */
-    public void setExporter(final Settings settings) {
-        if (settings.get(EXPORTER_TYPE) != null) {
-            SinkType expectedType = SinkType.parse(settings.get(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE));
-            if (exporter != null && expectedType == SinkType.getSinkTypeFromExporter(exporter)) {
-                queryInsightsExporterFactory.updateExporter(exporter, settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN));
-            } else {
-                try {
-                    queryInsightsExporterFactory.closeExporter(this.exporter);
-                } catch (IOException e) {
-                    logger.error("Fail to close the current exporter when updating exporter, error: ", e);
-                }
-                this.exporter = queryInsightsExporterFactory.createExporter(
-                    SinkType.parse(settings.get(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE)),
-                    settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN)
-                );
-            }
-        } else {
-            // Disable exporter if exporter type is set to null
-            try {
-                queryInsightsExporterFactory.closeExporter(this.exporter);
-                this.exporter = null;
-            } catch (IOException e) {
-                logger.error("Fail to close the current exporter when disabling exporter, error: ", e);
-            }
-        }
-    }
-
-    /**
-     * Validate provided settings for top queries exporter
-     *
-     * @param settings settings exporter config {@link Settings}
-     */
-    public void validateExporterConfig(Settings settings) {
-        queryInsightsExporterFactory.validateExporterConfig(settings);
-    }
-
-    /**
-     * Get all top queries records that are in the current top n queries store
-     * Optionally include top N records from the last window.
-     *
-     * By default, return the records in sorted order.
-     *
-     * @param includeLastWindow if the top N queries from the last window should be included
-     * @return List of the records that are in the query insight store
-     * @throws IllegalArgumentException if query insight is disabled in the cluster
-     */
-    public List<SearchQueryRecord> getTopQueriesRecords(final boolean includeLastWindow) throws IllegalArgumentException {
-        if (!enabled) {
-            throw new IllegalArgumentException(
-                String.format(Locale.ROOT, "Cannot get top n queries for [%s] when it is not enabled.", metricType.toString())
-            );
-        }
-        // read from window snapshots
-        final List<SearchQueryRecord> queries = new ArrayList<>(topQueriesCurrentSnapshot.get());
-        if (includeLastWindow) {
-            queries.addAll(topQueriesHistorySnapshot.get());
-        }
-        return Stream.of(queries)
-            .flatMap(Collection::stream)
-            .sorted((a, b) -> SearchQueryRecord.compare(a, b, metricType) * -1)
-            .collect(Collectors.toList());
-    }
-
-    /**
-     * Consume records to top queries stores
-     *
-     * @param records a list of {@link SearchQueryRecord}
-     */
-    void consumeRecords(final List<SearchQueryRecord> records) {
-        final long currentWindowStart = calculateWindowStart(System.currentTimeMillis());
-        List<SearchQueryRecord> recordsInLastWindow = new ArrayList<>();
-        List<SearchQueryRecord> recordsInThisWindow = new ArrayList<>();
-        for (SearchQueryRecord record : records) {
-            // skip the records that does not have the corresponding measurement
-            if (!record.getMeasurements().containsKey(metricType)) {
-                continue;
-            }
-            if (record.getTimestamp() < currentWindowStart) {
-                recordsInLastWindow.add(record);
-            } else {
-                recordsInThisWindow.add(record);
-            }
-        }
-        // add records in last window, if there are any, to the top n store
-        addToTopNStore(recordsInLastWindow);
-        // rotate window and reset window start if necessary
-        rotateWindowIfNecessary(currentWindowStart);
-        // add records in current window, if there are any, to the top n store
-        addToTopNStore(recordsInThisWindow);
-        // update the current window snapshot for getters to consume
-        final List<SearchQueryRecord> newSnapShot = new ArrayList<>(topQueriesStore);
-        newSnapShot.sort((a, b) -> SearchQueryRecord.compare(a, b, metricType));
-        topQueriesCurrentSnapshot.set(newSnapShot);
-    }
-
-    private void addToTopNStore(final List<SearchQueryRecord> records) {
-        topQueriesStore.addAll(records);
-        // remove top elements for fix sizing priority queue
-        while (topQueriesStore.size() > topNSize) {
-            topQueriesStore.poll();
-        }
-    }
-
-    /**
-     * Reset the current window and rotate the data to history snapshot for top n queries,
-     * This function would be invoked zero time or only once in each consumeRecords call
-     *
-     * @param newWindowStart the new windowStart to set to
-     */
-    private void rotateWindowIfNecessary(final long newWindowStart) {
-        // reset window if the current window is outdated
-        if (windowStart < newWindowStart) {
-            final List<SearchQueryRecord> history = new ArrayList<>();
-            // rotate the current window to history store only if the data belongs to the last window
-            if (windowStart == newWindowStart - windowSize.getMillis()) {
-                history.addAll(topQueriesStore);
-            }
-            topQueriesHistorySnapshot.set(history);
-            topQueriesStore.clear();
-            topQueriesCurrentSnapshot.set(new ArrayList<>());
-            windowStart = newWindowStart;
-            // export to the configured sink
-            if (exporter != null) {
-                threadPool.executor(QUERY_INSIGHTS_EXECUTOR).execute(() -> exporter.export(history));
-            }
-        }
-    }
-
-    /**
-     * Calculate the window start for the given timestamp
-     *
-     * @param timestamp the given timestamp to calculate window start
-     */
-    private long calculateWindowStart(final long timestamp) {
-        final LocalDateTime currentTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.of("UTC"));
-        LocalDateTime windowStartTime = currentTime.truncatedTo(ChronoUnit.HOURS);
-        while (!windowStartTime.plusMinutes(windowSize.getMinutes()).isAfter(currentTime)) {
-            windowStartTime = windowStartTime.plusMinutes(windowSize.getMinutes());
-        }
-        return windowStartTime.toInstant(ZoneOffset.UTC).getEpochSecond() * 1000;
-    }
-
-    /**
-     * Get the current top queries snapshot from the AtomicReference.
-     *
-     * @return a list of {@link SearchQueryRecord}
-     */
-    public List<SearchQueryRecord> getTopQueriesCurrentSnapshot() {
-        return topQueriesCurrentSnapshot.get();
-    }
-
-    /**
-     * Close the top n queries service
-     */
-    public void close() throws IOException {
-        queryInsightsExporterFactory.closeExporter(this.exporter);
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/package-info.java
deleted file mode 100644
index 5068f28234f6d..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Service Classes for Query Insights
- */
-package org.opensearch.plugin.insights.core.service;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/package-info.java
deleted file mode 100644
index 04d1f9bfff7e1..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Base Package of Query Insights
- */
-package org.opensearch.plugin.insights;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/package-info.java
deleted file mode 100644
index 9b6b5856f7d27..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Transport Actions, Requests and Responses for Query Insights
- */
-package org.opensearch.plugin.insights.rules.action;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueries.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueries.java
deleted file mode 100644
index 26cff82aae52e..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueries.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.action.top_queries;
-
-import org.opensearch.action.support.nodes.BaseNodeResponse;
-import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.xcontent.ToXContentObject;
-import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Holds all top queries records by resource usage or latency on a node
- * Mainly used in the top N queries node response workflow.
- *
- * @opensearch.internal
- */
-public class TopQueries extends BaseNodeResponse implements ToXContentObject {
-    /** The store to keep the top queries records */
-    private final List<SearchQueryRecord> topQueriesRecords;
-
-    /**
-     * Create the TopQueries Object from StreamInput
-     * @param in A {@link StreamInput} object.
-     * @throws IOException IOException
-     */
-    public TopQueries(final StreamInput in) throws IOException {
-        super(in);
-        topQueriesRecords = in.readList(SearchQueryRecord::new);
-    }
-
-    /**
-     * Create the TopQueries Object
-     * @param node A node that is part of the cluster.
-     * @param searchQueryRecords A list of SearchQueryRecord associated in this TopQueries.
-     */
-    public TopQueries(final DiscoveryNode node, final List<SearchQueryRecord> searchQueryRecords) {
-        super(node);
-        topQueriesRecords = searchQueryRecords;
-    }
-
-    @Override
-    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
-        if (topQueriesRecords != null) {
-            for (SearchQueryRecord record : topQueriesRecords) {
-                record.toXContent(builder, params);
-            }
-        }
-        return builder;
-    }
-
-    @Override
-    public void writeTo(final StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeList(topQueriesRecords);
-
-    }
-
-    /**
-     * Get all top queries records
-     *
-     * @return the top queries records in this node response
-     */
-    public List<SearchQueryRecord> getTopQueriesRecord() {
-        return topQueriesRecords;
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesAction.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesAction.java
deleted file mode 100644
index b8ed69fa5692b..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesAction.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.action.top_queries;
-
-import org.opensearch.action.ActionType;
-
-/**
- * Transport action for cluster/node level top queries information.
- *
- * @opensearch.internal
- */
-public class TopQueriesAction extends ActionType<TopQueriesResponse> {
-
-    /**
-     * The TopQueriesAction Instance.
-     */
-    public static final TopQueriesAction INSTANCE = new TopQueriesAction();
-    /**
-     * The name of this Action
-     */
-    public static final String NAME = "cluster:admin/opensearch/insights/top_queries";
-
-    private TopQueriesAction() {
-        super(NAME, TopQueriesResponse::new);
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequest.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequest.java
deleted file mode 100644
index 3bdff2c403161..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.action.top_queries;
-
-import org.opensearch.action.support.nodes.BaseNodesRequest;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-
-import java.io.IOException;
-
-/**
- * A request to get cluster/node level top queries information.
- *
- * @opensearch.internal
- */
-public class TopQueriesRequest extends BaseNodesRequest<TopQueriesRequest> {
-
-    final MetricType metricType;
-
-    /**
-     * Constructor for TopQueriesRequest
-     *
-     * @param in A {@link StreamInput} object.
-     * @throws IOException if the stream cannot be deserialized.
-     */
-    public TopQueriesRequest(final StreamInput in) throws IOException {
-        super(in);
-        this.metricType = MetricType.readFromStream(in);
-    }
-
-    /**
-     * Get top queries from nodes based on the nodes ids specified.
-     * If none are passed, cluster level top queries will be returned.
-     *
-     * @param metricType {@link MetricType}
-     * @param nodesIds the nodeIds specified in the request
-     */
-    public TopQueriesRequest(final MetricType metricType, final String... nodesIds) {
-        super(nodesIds);
-        this.metricType = metricType;
-    }
-
-    /**
-     * Get the type of requested metrics
-     */
-    public MetricType getMetricType() {
-        return metricType;
-    }
-
-    @Override
-    public void writeTo(final StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeString(metricType.toString());
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponse.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponse.java
deleted file mode 100644
index 2e66bb7f77baf..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponse.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.action.top_queries;
-
-import org.opensearch.action.FailedNodeException;
-import org.opensearch.action.support.nodes.BaseNodesResponse;
-import org.opensearch.cluster.ClusterName;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.xcontent.ToXContentFragment;
-import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.plugin.insights.rules.model.Attribute;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Transport response for cluster/node level top queries information.
- *
- * @opensearch.internal
- */
-public class TopQueriesResponse extends BaseNodesResponse<TopQueries> implements ToXContentFragment {
-
-    private static final String CLUSTER_LEVEL_RESULTS_KEY = "top_queries";
-    private final MetricType metricType;
-    private final int top_n_size;
-
-    /**
-     * Constructor for TopQueriesResponse.
-     *
-     * @param in A {@link StreamInput} object.
-     * @throws IOException if the stream cannot be deserialized.
-     */
-    public TopQueriesResponse(final StreamInput in) throws IOException {
-        super(in);
-        top_n_size = in.readInt();
-        metricType = in.readEnum(MetricType.class);
-    }
-
-    /**
-     * Constructor for TopQueriesResponse
-     *
-     * @param clusterName The current cluster name
-     * @param nodes A list that contains top queries results from all nodes
-     * @param failures A list that contains FailedNodeException
-     * @param top_n_size The top N size to return to the user
-     * @param metricType the {@link MetricType} to be returned in this response
-     */
-    public TopQueriesResponse(
-        final ClusterName clusterName,
-        final List<TopQueries> nodes,
-        final List<FailedNodeException> failures,
-        final int top_n_size,
-        final MetricType metricType
-    ) {
-        super(clusterName, nodes, failures);
-        this.top_n_size = top_n_size;
-        this.metricType = metricType;
-    }
-
-    @Override
-    protected List<TopQueries> readNodesFrom(final StreamInput in) throws IOException {
-        return in.readList(TopQueries::new);
-    }
-
-    @Override
-    protected void writeNodesTo(final StreamOutput out, final List<TopQueries> nodes) throws IOException {
-        out.writeList(nodes);
-        out.writeLong(top_n_size);
-        out.writeEnum(metricType);
-    }
-
-    @Override
-    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
-        final List<TopQueries> results = getNodes();
-        postProcess(results);
-        builder.startObject();
-        toClusterLevelResult(builder, params, results);
-        return builder.endObject();
-    }
-
-    @Override
-    public String toString() {
-        try {
-            final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            this.toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.toString();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
-    }
-
-    /**
-     * Post process the top queries results to add customized attributes
-     *
-     * @param results the top queries results
-     */
-    private void postProcess(final List<TopQueries> results) {
-        for (TopQueries topQueries : results) {
-            final String nodeId = topQueries.getNode().getId();
-            for (SearchQueryRecord record : topQueries.getTopQueriesRecord()) {
-                record.addAttribute(Attribute.NODE_ID, nodeId);
-            }
-        }
-    }
-
-    /**
-     * Merge top n queries results from nodes into cluster level results in XContent format.
-     *
-     * @param builder XContent builder
-     * @param params serialization parameters
-     * @param results top queries results from all nodes
-     * @throws IOException if an error occurs
-     */
-    private void toClusterLevelResult(final XContentBuilder builder, final Params params, final List<TopQueries> results)
-        throws IOException {
-        final List<SearchQueryRecord> all_records = results.stream()
-            .map(TopQueries::getTopQueriesRecord)
-            .flatMap(Collection::stream)
-            .sorted((a, b) -> SearchQueryRecord.compare(a, b, metricType) * -1)
-            .limit(top_n_size)
-            .collect(Collectors.toList());
-        builder.startArray(CLUSTER_LEVEL_RESULTS_KEY);
-        for (SearchQueryRecord record : all_records) {
-            record.toXContent(builder, params);
-        }
-        builder.endArray();
-    }
-
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/package-info.java
deleted file mode 100644
index 3cc7900e5ce7d..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Transport Actions, Requests and Responses for Top N Queries
- */
-package org.opensearch.plugin.insights.rules.action.top_queries;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java
deleted file mode 100644
index dcdb085fdc6fa..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.model;
-
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-import java.util.Locale;
-
-/**
- * Valid attributes for a search query record
- *
- * @opensearch.internal
- */
-public enum Attribute {
-    /**
-     * The search query type
-     */
-    SEARCH_TYPE,
-    /**
-     * The search query source
-     */
-    SOURCE,
-    /**
-     * Total shards queried
-     */
-    TOTAL_SHARDS,
-    /**
-     * The indices involved
-     */
-    INDICES,
-    /**
-     * The per phase level latency map for a search query
-     */
-    PHASE_LATENCY_MAP,
-    /**
-     * The node id for this request
-     */
-    NODE_ID,
-    /**
-     * Tasks level resource usages in this request
-     */
-    TASK_RESOURCE_USAGES,
-    /**
-     * Custom search request labels
-     */
-    LABELS;
-
-    /**
-     * Read an Attribute from a StreamInput
-     *
-     * @param in the StreamInput to read from
-     * @return Attribute
-     * @throws IOException IOException
-     */
-    static Attribute readFromStream(final StreamInput in) throws IOException {
-        return Attribute.valueOf(in.readString().toUpperCase(Locale.ROOT));
-    }
-
-    /**
-     * Write Attribute to a StreamOutput
-     *
-     * @param out the StreamOutput to write
-     * @param attribute the Attribute to write
-     * @throws IOException IOException
-     */
-    static void writeTo(final StreamOutput out, final Attribute attribute) throws IOException {
-        out.writeString(attribute.toString());
-    }
-
-    @Override
-    public String toString() {
-        return this.name().toLowerCase(Locale.ROOT);
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java
deleted file mode 100644
index 4694c757f4ef2..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.model;
-
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.Locale;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * Valid metric types for a search query record
- *
- * @opensearch.internal
- */
-public enum MetricType implements Comparator<Number> {
-    /**
-     * Latency metric type
-     */
-    LATENCY,
-    /**
-     * CPU usage metric type
-     */
-    CPU,
-    /**
-     * JVM heap usage metric type
-     */
-    MEMORY;
-
-    /**
-     * Read a MetricType from a StreamInput
-     *
-     * @param in the StreamInput to read from
-     * @return MetricType
-     * @throws IOException IOException
-     */
-    public static MetricType readFromStream(final StreamInput in) throws IOException {
-        return fromString(in.readString());
-    }
-
-    /**
-     * Create MetricType from String
-     *
-     * @param metricType the String representation of MetricType
-     * @return MetricType
-     */
-    public static MetricType fromString(final String metricType) {
-        return MetricType.valueOf(metricType.toUpperCase(Locale.ROOT));
-    }
-
-    /**
-     * Write MetricType to a StreamOutput
-     *
-     * @param out the StreamOutput to write
-     * @param metricType the MetricType to write
-     * @throws IOException IOException
-     */
-    static void writeTo(final StreamOutput out, final MetricType metricType) throws IOException {
-        out.writeString(metricType.toString());
-    }
-
-    @Override
-    public String toString() {
-        return this.name().toLowerCase(Locale.ROOT);
-    }
-
-    /**
-     * Get all valid metrics
-     *
-     * @return A set of String that contains all valid metrics
-     */
-    public static Set<MetricType> allMetricTypes() {
-        return Arrays.stream(values()).collect(Collectors.toSet());
-    }
-
-    /**
-     * Compare two numbers based on the metric type
-     *
-     * @param a the first Number to be compared.
-     * @param b the second Number to be compared.
-     * @return a negative integer, zero, or a positive integer as the first argument is less than, equal to, or greater than the second
-     */
-    public int compare(final Number a, final Number b) {
-        switch (this) {
-            case LATENCY:
-            case CPU:
-            case MEMORY:
-                return Long.compare(a.longValue(), b.longValue());
-        }
-        return -1;
-    }
-
-    /**
-     * Parse a value with the correct type based on MetricType
-     *
-     * @param o the generic object to parse
-     * @return {@link Number}
-     */
-    Number parseValue(final Object o) {
-        switch (this) {
-            case LATENCY:
-            case CPU:
-            case MEMORY:
-                return (Long) o;
-            default:
-                return (Number) o;
-        }
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java
deleted file mode 100644
index fec00a680ae58..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.model;
-
-import org.opensearch.core.common.Strings;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.core.xcontent.MediaTypeRegistry;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.core.xcontent.ToXContentObject;
-import org.opensearch.core.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * SearchQueryRecord represents a minimal atomic record stored in the Query Insight Framework,
- * which contains extensive information related to a search query.
- *
- * @opensearch.internal
- */
-public class SearchQueryRecord implements ToXContentObject, Writeable {
-    private final long timestamp;
-    private final Map<MetricType, Number> measurements;
-    private final Map<Attribute, Object> attributes;
-
-    /**
-     * Constructor of SearchQueryRecord
-     *
-     * @param in the StreamInput to read the SearchQueryRecord from
-     * @throws IOException IOException
-     * @throws ClassCastException ClassCastException
-     */
-    public SearchQueryRecord(final StreamInput in) throws IOException, ClassCastException {
-        this.timestamp = in.readLong();
-        measurements = new HashMap<>();
-        in.readMap(MetricType::readFromStream, StreamInput::readGenericValue)
-            .forEach(((metricType, o) -> measurements.put(metricType, metricType.parseValue(o))));
-        this.attributes = in.readMap(Attribute::readFromStream, StreamInput::readGenericValue);
-    }
-
-    /**
-     * Constructor of SearchQueryRecord
-     *
-     * @param timestamp The timestamp of the query.
-     * @param measurements A list of Measurement associated with this query
-     * @param attributes A list of Attributes associated with this query
-     */
-    public SearchQueryRecord(final long timestamp, Map<MetricType, Number> measurements, final Map<Attribute, Object> attributes) {
-        if (measurements == null) {
-            throw new IllegalArgumentException("Measurements cannot be null");
-        }
-        this.measurements = measurements;
-        this.attributes = attributes;
-        this.timestamp = timestamp;
-    }
-
-    /**
-     * Returns the observation time of the metric.
-     *
-     * @return the observation time in milliseconds
-     */
-    public long getTimestamp() {
-        return timestamp;
-    }
-
-    /**
-     * Returns the measurement associated with the specified name.
-     *
-     * @param name the name of the measurement
-     * @return the measurement object, or null if not found
-     */
-    public Number getMeasurement(final MetricType name) {
-        return measurements.get(name);
-    }
-
-    /**
-     * Returns a map of all the measurements associated with the metric.
-     *
-     * @return a map of measurement names to measurement objects
-     */
-    public Map<MetricType, Number> getMeasurements() {
-        return measurements;
-    }
-
-    /**
-     * Returns a map of the attributes associated with the metric.
-     *
-     * @return a map of attribute keys to attribute values
-     */
-    public Map<Attribute, Object> getAttributes() {
-        return attributes;
-    }
-
-    /**
-     * Add an attribute to this record
-     *
-     * @param attribute attribute to add
-     * @param value the value associated with the attribute
-     */
-    public void addAttribute(final Attribute attribute, final Object value) {
-        attributes.put(attribute, value);
-    }
-
-    @Override
-    public XContentBuilder toXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
-        builder.startObject();
-        builder.field("timestamp", timestamp);
-        for (Map.Entry<Attribute, Object> entry : attributes.entrySet()) {
-            builder.field(entry.getKey().toString(), entry.getValue());
-        }
-        for (Map.Entry<MetricType, Number> entry : measurements.entrySet()) {
-            builder.field(entry.getKey().toString(), entry.getValue());
-        }
-        return builder.endObject();
-    }
-
-    /**
-     * Write a SearchQueryRecord to a StreamOutput
-     *
-     * @param out the StreamOutput to write
-     * @throws IOException IOException
-     */
-    @Override
-    public void writeTo(final StreamOutput out) throws IOException {
-        out.writeLong(timestamp);
-        out.writeMap(measurements, (stream, metricType) -> MetricType.writeTo(out, metricType), StreamOutput::writeGenericValue);
-        out.writeMap(attributes, (stream, attribute) -> Attribute.writeTo(out, attribute), StreamOutput::writeGenericValue);
-    }
-
-    /**
-     * Compare two SearchQueryRecord, based on the given MetricType
-     *
-     * @param a the first SearchQueryRecord to compare
-     * @param b the second SearchQueryRecord to compare
-     * @param metricType the MetricType to compare on
-     * @return 0 if the first SearchQueryRecord is numerically equal to the second SearchQueryRecord;
-     *        -1 if the first SearchQueryRecord is numerically less than the second SearchQueryRecord;
-     *         1 if the first SearchQueryRecord is numerically greater than the second SearchQueryRecord.
-     */
-    public static int compare(final SearchQueryRecord a, final SearchQueryRecord b, final MetricType metricType) {
-        return metricType.compare(a.getMeasurement(metricType), b.getMeasurement(metricType));
-    }
-
-    /**
-     * Check if a SearchQueryRecord is deep equal to another record
-     *
-     * @param o the other SearchQueryRecord record
-     * @return true if two records are deep equal, false otherwise.
-     */
-    @Override
-    public boolean equals(final Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof SearchQueryRecord)) {
-            return false;
-        }
-        final SearchQueryRecord other = (SearchQueryRecord) o;
-        return timestamp == other.getTimestamp()
-            && measurements.equals(other.getMeasurements())
-            && attributes.size() == other.getAttributes().size();
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(timestamp, measurements, attributes);
-    }
-
-    @Override
-    public String toString() {
-        return Strings.toString(MediaTypeRegistry.JSON, this);
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/package-info.java
deleted file mode 100644
index c59ec1550f54b..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Data Models for Query Insight Records
- */
-package org.opensearch.plugin.insights.rules.model;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/package-info.java
deleted file mode 100644
index 3787f05f65552..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Rest Handlers for Query Insights
- */
-package org.opensearch.plugin.insights.rules.resthandler;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesAction.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesAction.java
deleted file mode 100644
index 6aa511c626ab1..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesAction.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.resthandler.top_queries;
-
-import org.opensearch.client.node.NodeClient;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.core.common.Strings;
-import org.opensearch.core.rest.RestStatus;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesAction;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesRequest;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesResponse;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.rest.BaseRestHandler;
-import org.opensearch.rest.BytesRestResponse;
-import org.opensearch.rest.RestChannel;
-import org.opensearch.rest.RestRequest;
-import org.opensearch.rest.RestResponse;
-import org.opensearch.rest.action.RestResponseListener;
-
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_QUERIES_BASE_URI;
-import static org.opensearch.rest.RestRequest.Method.GET;
-
-/**
- * Rest action to get Top N queries by certain metric type
- *
- * @opensearch.api
- */
-public class RestTopQueriesAction extends BaseRestHandler {
-    /** The metric types that are allowed in top N queries */
-    static final Set<String> ALLOWED_METRICS = MetricType.allMetricTypes().stream().map(MetricType::toString).collect(Collectors.toSet());
-
-    /**
-     * Constructor for RestTopQueriesAction
-     */
-    public RestTopQueriesAction() {}
-
-    @Override
-    public List<Route> routes() {
-        return List.of(
-            new Route(GET, TOP_QUERIES_BASE_URI),
-            new Route(GET, String.format(Locale.ROOT, "%s/{nodeId}", TOP_QUERIES_BASE_URI))
-        );
-    }
-
-    @Override
-    public String getName() {
-        return "query_insights_top_queries_action";
-    }
-
-    @Override
-    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) {
-        final TopQueriesRequest topQueriesRequest = prepareRequest(request);
-        topQueriesRequest.timeout(request.param("timeout"));
-
-        return channel -> client.execute(TopQueriesAction.INSTANCE, topQueriesRequest, topQueriesResponse(channel));
-    }
-
-    static TopQueriesRequest prepareRequest(final RestRequest request) {
-        final String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
-        final String metricType = request.param("type", MetricType.LATENCY.toString());
-        if (!ALLOWED_METRICS.contains(metricType)) {
-            throw new IllegalArgumentException(
-                String.format(Locale.ROOT, "request [%s] contains invalid metric type [%s]", request.path(), metricType)
-            );
-        }
-        return new TopQueriesRequest(MetricType.fromString(metricType), nodesIds);
-    }
-
-    @Override
-    protected Set<String> responseParams() {
-        return Settings.FORMAT_PARAMS;
-    }
-
-    @Override
-    public boolean canTripCircuitBreaker() {
-        return false;
-    }
-
-    private RestResponseListener<TopQueriesResponse> topQueriesResponse(final RestChannel channel) {
-        return new RestResponseListener<>(channel) {
-            @Override
-            public RestResponse buildResponse(final TopQueriesResponse response) throws Exception {
-                return new BytesRestResponse(RestStatus.OK, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS));
-            }
-        };
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/package-info.java
deleted file mode 100644
index 087cf7d765f8c..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Rest Handlers for Top N Queries
- */
-package org.opensearch.plugin.insights.rules.resthandler.top_queries;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/package-info.java
deleted file mode 100644
index f3a1c70b9af57..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Transport Actions for Query Insights.
- */
-package org.opensearch.plugin.insights.rules.transport;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesAction.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesAction.java
deleted file mode 100644
index 7949b70a16db6..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesAction.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.transport.top_queries;
-
-import org.opensearch.action.FailedNodeException;
-import org.opensearch.action.support.ActionFilters;
-import org.opensearch.action.support.nodes.TransportNodesAction;
-import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.inject.Inject;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.plugin.insights.core.service.QueryInsightsService;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueries;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesAction;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesRequest;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesResponse;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.threadpool.ThreadPool;
-import org.opensearch.transport.TransportRequest;
-import org.opensearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Transport action for cluster/node level top queries information.
- *
- * @opensearch.internal
- */
-public class TransportTopQueriesAction extends TransportNodesAction<
-    TopQueriesRequest,
-    TopQueriesResponse,
-    TransportTopQueriesAction.NodeRequest,
-    TopQueries> {
-
-    private final QueryInsightsService queryInsightsService;
-
-    /**
-     * Create the TransportTopQueriesAction Object
-
-     * @param threadPool The OpenSearch thread pool to run async tasks
-     * @param clusterService The clusterService of this node
-     * @param transportService The TransportService of this node
-     * @param queryInsightsService The topQueriesByLatencyService associated with this Transport Action
-     * @param actionFilters the action filters
-     */
-    @Inject
-    public TransportTopQueriesAction(
-        final ThreadPool threadPool,
-        final ClusterService clusterService,
-        final TransportService transportService,
-        final QueryInsightsService queryInsightsService,
-        final ActionFilters actionFilters
-    ) {
-        super(
-            TopQueriesAction.NAME,
-            threadPool,
-            clusterService,
-            transportService,
-            actionFilters,
-            TopQueriesRequest::new,
-            NodeRequest::new,
-            ThreadPool.Names.GENERIC,
-            TopQueries.class
-        );
-        this.queryInsightsService = queryInsightsService;
-    }
-
-    @Override
-    protected TopQueriesResponse newResponse(
-        final TopQueriesRequest topQueriesRequest,
-        final List<TopQueries> responses,
-        final List<FailedNodeException> failures
-    ) {
-        int size;
-        switch (topQueriesRequest.getMetricType()) {
-            case CPU:
-                size = clusterService.getClusterSettings().get(QueryInsightsSettings.TOP_N_CPU_QUERIES_SIZE);
-                break;
-            case MEMORY:
-                size = clusterService.getClusterSettings().get(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE);
-                break;
-            default:
-                size = clusterService.getClusterSettings().get(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE);
-        }
-        return new TopQueriesResponse(clusterService.getClusterName(), responses, failures, size, topQueriesRequest.getMetricType());
-    }
-
-    @Override
-    protected NodeRequest newNodeRequest(final TopQueriesRequest request) {
-        return new NodeRequest(request);
-    }
-
-    @Override
-    protected TopQueries newNodeResponse(final StreamInput in) throws IOException {
-        return new TopQueries(in);
-    }
-
-    @Override
-    protected TopQueries nodeOperation(final NodeRequest nodeRequest) {
-        final TopQueriesRequest request = nodeRequest.request;
-        return new TopQueries(
-            clusterService.localNode(),
-            queryInsightsService.getTopQueriesService(request.getMetricType()).getTopQueriesRecords(true)
-        );
-    }
-
-    /**
-     * Inner Node Top Queries Request
-     *
-     * @opensearch.internal
-     */
-    public static class NodeRequest extends TransportRequest {
-
-        final TopQueriesRequest request;
-
-        /**
-         * Create the NodeResponse object from StreamInput
-         *
-         * @param in the StreamInput to read the object
-         * @throws IOException IOException
-         */
-        public NodeRequest(StreamInput in) throws IOException {
-            super(in);
-            request = new TopQueriesRequest(in);
-        }
-
-        /**
-         * Create the NodeResponse object from a TopQueriesRequest
-         * @param request the TopQueriesRequest object
-         */
-        public NodeRequest(final TopQueriesRequest request) {
-            this.request = request;
-        }
-
-        @Override
-        public void writeTo(final StreamOutput out) throws IOException {
-            super.writeTo(out);
-            request.writeTo(out);
-        }
-    }
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/package-info.java
deleted file mode 100644
index 54da0980deff8..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/transport/top_queries/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Transport Actions for Top N Queries.
- */
-package org.opensearch.plugin.insights.rules.transport.top_queries;
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java
deleted file mode 100644
index 25309b5721792..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.settings;
-
-import org.opensearch.common.settings.Setting;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.plugin.insights.core.exporter.SinkType;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Settings for Query Insights Plugin
- *
- * @opensearch.api
- * @opensearch.experimental
- */
-public class QueryInsightsSettings {
-    /**
-     * Executors settings
-     */
-    public static final String QUERY_INSIGHTS_EXECUTOR = "query_insights_executor";
-    /**
-     * Max number of thread
-     */
-    public static final int MAX_THREAD_COUNT = 5;
-    /**
-     * Max number of requests for the consumer to collect at one time
-     */
-    public static final int QUERY_RECORD_QUEUE_CAPACITY = 1000;
-    /**
-     * Time interval for record queue consumer to run
-     */
-    public static final TimeValue QUERY_RECORD_QUEUE_DRAIN_INTERVAL = new TimeValue(5, TimeUnit.SECONDS);
-    /**
-     * Default Values and Settings
-     */
-    public static final TimeValue MAX_WINDOW_SIZE = new TimeValue(1, TimeUnit.DAYS);
-    /**
-     * Minimal window size
-     */
-    public static final TimeValue MIN_WINDOW_SIZE = new TimeValue(1, TimeUnit.MINUTES);
-    /**
-     * Valid window sizes
-     */
-    public static final Set<TimeValue> VALID_WINDOW_SIZES_IN_MINUTES = new HashSet<>(
-        Arrays.asList(
-            new TimeValue(1, TimeUnit.MINUTES),
-            new TimeValue(5, TimeUnit.MINUTES),
-            new TimeValue(10, TimeUnit.MINUTES),
-            new TimeValue(30, TimeUnit.MINUTES)
-        )
-    );
-
-    /** Default N size for top N queries */
-    public static final int MAX_N_SIZE = 100;
-    /** Default window size in seconds to keep the top N queries with latency data in query insight store */
-    public static final TimeValue DEFAULT_WINDOW_SIZE = new TimeValue(60, TimeUnit.SECONDS);
-    /** Default top N size to keep the data in query insight store */
-    public static final int DEFAULT_TOP_N_SIZE = 3;
-    /**
-     * Query Insights base uri
-     */
-    public static final String PLUGINS_BASE_URI = "/_insights";
-
-    /**
-     * Settings for Top Queries
-     *
-     */
-    public static final String TOP_QUERIES_BASE_URI = PLUGINS_BASE_URI + "/top_queries";
-    /** Default prefix for top N queries feature */
-    public static final String TOP_N_QUERIES_SETTING_PREFIX = "search.insights.top_queries";
-    /** Default prefix for top N queries by latency feature */
-    public static final String TOP_N_LATENCY_QUERIES_PREFIX = TOP_N_QUERIES_SETTING_PREFIX + ".latency";
-    /** Default prefix for top N queries by cpu feature */
-    public static final String TOP_N_CPU_QUERIES_PREFIX = TOP_N_QUERIES_SETTING_PREFIX + ".cpu";
-    /** Default prefix for top N queries by memory feature */
-    public static final String TOP_N_MEMORY_QUERIES_PREFIX = TOP_N_QUERIES_SETTING_PREFIX + ".memory";
-    /**
-     * Boolean setting for enabling top queries by latency.
-     */
-    public static final Setting<Boolean> TOP_N_LATENCY_QUERIES_ENABLED = Setting.boolSetting(
-        TOP_N_LATENCY_QUERIES_PREFIX + ".enabled",
-        false,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Int setting to define the top n size for top queries by latency.
-     */
-    public static final Setting<Integer> TOP_N_LATENCY_QUERIES_SIZE = Setting.intSetting(
-        TOP_N_LATENCY_QUERIES_PREFIX + ".top_n_size",
-        DEFAULT_TOP_N_SIZE,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Time setting to define the window size in seconds for top queries by latency.
-     */
-    public static final Setting<TimeValue> TOP_N_LATENCY_QUERIES_WINDOW_SIZE = Setting.positiveTimeSetting(
-        TOP_N_LATENCY_QUERIES_PREFIX + ".window_size",
-        DEFAULT_WINDOW_SIZE,
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic
-    );
-
-    /**
-     * Boolean setting for enabling top queries by cpu.
-     */
-    public static final Setting<Boolean> TOP_N_CPU_QUERIES_ENABLED = Setting.boolSetting(
-        TOP_N_CPU_QUERIES_PREFIX + ".enabled",
-        false,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Int setting to define the top n size for top queries by cpu.
-     */
-    public static final Setting<Integer> TOP_N_CPU_QUERIES_SIZE = Setting.intSetting(
-        TOP_N_CPU_QUERIES_PREFIX + ".top_n_size",
-        DEFAULT_TOP_N_SIZE,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Time setting to define the window size in seconds for top queries by cpu.
-     */
-    public static final Setting<TimeValue> TOP_N_CPU_QUERIES_WINDOW_SIZE = Setting.positiveTimeSetting(
-        TOP_N_CPU_QUERIES_PREFIX + ".window_size",
-        DEFAULT_WINDOW_SIZE,
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic
-    );
-
-    /**
-     * Boolean setting for enabling top queries by memory.
-     */
-    public static final Setting<Boolean> TOP_N_MEMORY_QUERIES_ENABLED = Setting.boolSetting(
-        TOP_N_MEMORY_QUERIES_PREFIX + ".enabled",
-        false,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Int setting to define the top n size for top queries by memory.
-     */
-    public static final Setting<Integer> TOP_N_MEMORY_QUERIES_SIZE = Setting.intSetting(
-        TOP_N_MEMORY_QUERIES_PREFIX + ".top_n_size",
-        DEFAULT_TOP_N_SIZE,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Time setting to define the window size in seconds for top queries by memory.
-     */
-    public static final Setting<TimeValue> TOP_N_MEMORY_QUERIES_WINDOW_SIZE = Setting.positiveTimeSetting(
-        TOP_N_MEMORY_QUERIES_PREFIX + ".window_size",
-        DEFAULT_WINDOW_SIZE,
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic
-    );
-
-    /**
-     * Config key for exporter type
-     */
-    public static final String EXPORTER_TYPE = "type";
-    /**
-     * Config key for export index
-     */
-    public static final String EXPORT_INDEX = "config.index";
-
-    /**
-     * Settings and defaults for top queries exporters
-     */
-    private static final String TOP_N_LATENCY_QUERIES_EXPORTER_PREFIX = TOP_N_LATENCY_QUERIES_PREFIX + ".exporter.";
-    /**
-     * Prefix for top n queries by cpu exporters
-     */
-    private static final String TOP_N_CPU_QUERIES_EXPORTER_PREFIX = TOP_N_CPU_QUERIES_PREFIX + ".exporter.";
-    /**
-     * Prefix for top n queries by memory exporters
-     */
-    private static final String TOP_N_MEMORY_QUERIES_EXPORTER_PREFIX = TOP_N_MEMORY_QUERIES_PREFIX + ".exporter.";
-    /**
-     * Default index pattern of top n queries
-     */
-    public static final String DEFAULT_TOP_N_QUERIES_INDEX_PATTERN = "'top_queries-'YYYY.MM.dd";
-    /**
-     * Default exporter type of top queries
-     */
-    public static final String DEFAULT_TOP_QUERIES_EXPORTER_TYPE = SinkType.LOCAL_INDEX.toString();
-
-    /**
-     * Settings for the exporter of top latency queries
-     */
-    public static final Setting<Settings> TOP_N_LATENCY_EXPORTER_SETTINGS = Setting.groupSetting(
-        TOP_N_LATENCY_QUERIES_EXPORTER_PREFIX,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Settings for the exporter of top cpu queries
-     */
-    public static final Setting<Settings> TOP_N_CPU_EXPORTER_SETTINGS = Setting.groupSetting(
-        TOP_N_CPU_QUERIES_EXPORTER_PREFIX,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Settings for the exporter of top cpu queries
-     */
-    public static final Setting<Settings> TOP_N_MEMORY_EXPORTER_SETTINGS = Setting.groupSetting(
-        TOP_N_MEMORY_QUERIES_EXPORTER_PREFIX,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Get the enabled setting based on type
-     * @param type MetricType
-     * @return enabled setting
-     */
-    public static Setting<Boolean> getTopNEnabledSetting(MetricType type) {
-        switch (type) {
-            case CPU:
-                return TOP_N_CPU_QUERIES_ENABLED;
-            case MEMORY:
-                return TOP_N_MEMORY_QUERIES_ENABLED;
-            default:
-                return TOP_N_LATENCY_QUERIES_ENABLED;
-        }
-    }
-
-    /**
-     * Get the top n size setting based on type
-     * @param type MetricType
-     * @return top n size setting
-     */
-    public static Setting<Integer> getTopNSizeSetting(MetricType type) {
-        switch (type) {
-            case CPU:
-                return TOP_N_CPU_QUERIES_SIZE;
-            case MEMORY:
-                return TOP_N_MEMORY_QUERIES_SIZE;
-            default:
-                return TOP_N_LATENCY_QUERIES_SIZE;
-        }
-    }
-
-    /**
-     * Get the window size setting based on type
-     * @param type MetricType
-     * @return top n queries window size setting
-     */
-    public static Setting<TimeValue> getTopNWindowSizeSetting(MetricType type) {
-        switch (type) {
-            case CPU:
-                return TOP_N_CPU_QUERIES_WINDOW_SIZE;
-            case MEMORY:
-                return TOP_N_MEMORY_QUERIES_WINDOW_SIZE;
-            default:
-                return TOP_N_LATENCY_QUERIES_WINDOW_SIZE;
-        }
-    }
-
-    /**
-     * Get the exporter settings based on type
-     * @param type MetricType
-     * @return exporter setting
-     */
-    public static Setting<Settings> getExporterSettings(MetricType type) {
-        switch (type) {
-            case CPU:
-                return TOP_N_CPU_EXPORTER_SETTINGS;
-            case MEMORY:
-                return TOP_N_MEMORY_EXPORTER_SETTINGS;
-            default:
-                return TOP_N_LATENCY_EXPORTER_SETTINGS;
-        }
-    }
-
-    /**
-     * Default constructor
-     */
-    public QueryInsightsSettings() {}
-}
diff --git a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/package-info.java b/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/package-info.java
deleted file mode 100644
index f3152bbf966cb..0000000000000
--- a/plugins/query-insights/src/main/java/org/opensearch/plugin/insights/settings/package-info.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/**
- * Settings for Query Insights Plugin
- */
-package org.opensearch.plugin.insights.settings;
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java
deleted file mode 100644
index 2efe9085a39ee..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights;
-
-import org.opensearch.action.ActionRequest;
-import org.opensearch.client.Client;
-import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.core.action.ActionResponse;
-import org.opensearch.plugin.insights.core.listener.QueryInsightsListener;
-import org.opensearch.plugin.insights.core.service.QueryInsightsService;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesAction;
-import org.opensearch.plugin.insights.rules.resthandler.top_queries.RestTopQueriesAction;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.plugins.ActionPlugin;
-import org.opensearch.rest.RestHandler;
-import org.opensearch.test.ClusterServiceUtils;
-import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.threadpool.ExecutorBuilder;
-import org.opensearch.threadpool.ScalingExecutorBuilder;
-import org.opensearch.threadpool.ThreadPool;
-import org.junit.Before;
-
-import java.util.Arrays;
-import java.util.List;
-
-import static org.mockito.Mockito.mock;
-
-public class QueryInsightsPluginTests extends OpenSearchTestCase {
-
-    private QueryInsightsPlugin queryInsightsPlugin;
-
-    private final Client client = mock(Client.class);
-    private ClusterService clusterService;
-    private final ThreadPool threadPool = mock(ThreadPool.class);
-
-    @Before
-    public void setup() {
-        queryInsightsPlugin = new QueryInsightsPlugin();
-        Settings.Builder settingsBuilder = Settings.builder();
-        Settings settings = settingsBuilder.build();
-        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        QueryInsightsTestUtils.registerAllQueryInsightsSettings(clusterSettings);
-        clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, threadPool);
-    }
-
-    public void testGetSettings() {
-        assertEquals(
-            Arrays.asList(
-                QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED,
-                QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE,
-                QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE,
-                QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS,
-                QueryInsightsSettings.TOP_N_CPU_QUERIES_ENABLED,
-                QueryInsightsSettings.TOP_N_CPU_QUERIES_SIZE,
-                QueryInsightsSettings.TOP_N_CPU_QUERIES_WINDOW_SIZE,
-                QueryInsightsSettings.TOP_N_CPU_EXPORTER_SETTINGS,
-                QueryInsightsSettings.TOP_N_MEMORY_QUERIES_ENABLED,
-                QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE,
-                QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE,
-                QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS
-            ),
-            queryInsightsPlugin.getSettings()
-        );
-    }
-
-    public void testCreateComponent() {
-        List<Object> components = (List<Object>) queryInsightsPlugin.createComponents(
-            client,
-            clusterService,
-            threadPool,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null
-        );
-        assertEquals(2, components.size());
-        assertTrue(components.get(0) instanceof QueryInsightsService);
-        assertTrue(components.get(1) instanceof QueryInsightsListener);
-    }
-
-    public void testGetExecutorBuilders() {
-        Settings.Builder settingsBuilder = Settings.builder();
-        Settings settings = settingsBuilder.build();
-        List<ExecutorBuilder<?>> executorBuilders = queryInsightsPlugin.getExecutorBuilders(settings);
-        assertEquals(1, executorBuilders.size());
-        assertTrue(executorBuilders.get(0) instanceof ScalingExecutorBuilder);
-    }
-
-    public void testGetRestHandlers() {
-        List<RestHandler> components = queryInsightsPlugin.getRestHandlers(Settings.EMPTY, null, null, null, null, null, null);
-        assertEquals(1, components.size());
-        assertTrue(components.get(0) instanceof RestTopQueriesAction);
-    }
-
-    public void testGetActions() {
-        List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> components = queryInsightsPlugin.getActions();
-        assertEquals(1, components.size());
-        assertTrue(components.get(0).getAction() instanceof TopQueriesAction);
-    }
-
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java
deleted file mode 100644
index 7fa4e9841c20e..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights;
-
-import org.opensearch.action.search.SearchType;
-import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.util.Maps;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueries;
-import org.opensearch.plugin.insights.rules.model.Attribute;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.test.VersionUtils;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.opensearch.test.OpenSearchTestCase.buildNewFakeTransportAddress;
-import static org.opensearch.test.OpenSearchTestCase.random;
-import static org.opensearch.test.OpenSearchTestCase.randomAlphaOfLengthBetween;
-import static org.opensearch.test.OpenSearchTestCase.randomArray;
-import static org.opensearch.test.OpenSearchTestCase.randomIntBetween;
-import static org.opensearch.test.OpenSearchTestCase.randomLong;
-import static org.opensearch.test.OpenSearchTestCase.randomLongBetween;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-final public class QueryInsightsTestUtils {
-
-    public QueryInsightsTestUtils() {}
-
-    public static List<SearchQueryRecord> generateQueryInsightRecords(int count) {
-        return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0);
-    }
-
-    /**
-     * Creates a List of random Query Insight Records for testing purpose
-     */
-    public static List<SearchQueryRecord> generateQueryInsightRecords(int lower, int upper, long startTimeStamp, long interval) {
-        List<SearchQueryRecord> records = new ArrayList<>();
-        int countOfRecords = randomIntBetween(lower, upper);
-        long timestamp = startTimeStamp;
-        for (int i = 0; i < countOfRecords; ++i) {
-            Map<MetricType, Number> measurements = Map.of(
-                MetricType.LATENCY,
-                randomLongBetween(1000, 10000),
-                MetricType.CPU,
-                randomLongBetween(1000, 10000),
-                MetricType.MEMORY,
-                randomLongBetween(1000, 10000)
-            );
-
-            Map<String, Long> phaseLatencyMap = new HashMap<>();
-            int countOfPhases = randomIntBetween(2, 5);
-            for (int j = 0; j < countOfPhases; ++j) {
-                phaseLatencyMap.put(randomAlphaOfLengthBetween(5, 10), randomLong());
-            }
-            Map<Attribute, Object> attributes = new HashMap<>();
-            attributes.put(Attribute.SEARCH_TYPE, SearchType.QUERY_THEN_FETCH.toString().toLowerCase(Locale.ROOT));
-            attributes.put(Attribute.SOURCE, "{\"size\":20}");
-            attributes.put(Attribute.TOTAL_SHARDS, randomIntBetween(1, 100));
-            attributes.put(Attribute.INDICES, randomArray(1, 3, Object[]::new, () -> randomAlphaOfLengthBetween(5, 10)));
-            attributes.put(Attribute.PHASE_LATENCY_MAP, phaseLatencyMap);
-
-            records.add(new SearchQueryRecord(timestamp, measurements, attributes));
-            timestamp += interval;
-        }
-        return records;
-    }
-
-    public static TopQueries createRandomTopQueries() {
-        DiscoveryNode node = new DiscoveryNode(
-            "node_for_top_queries_test",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            emptySet(),
-            VersionUtils.randomVersion(random())
-        );
-        List<SearchQueryRecord> records = generateQueryInsightRecords(10);
-
-        return new TopQueries(node, records);
-    }
-
-    public static TopQueries createFixedTopQueries() {
-        DiscoveryNode node = new DiscoveryNode(
-            "node_for_top_queries_test",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            emptySet(),
-            VersionUtils.randomVersion(random())
-        );
-        List<SearchQueryRecord> records = new ArrayList<>();
-        records.add(createFixedSearchQueryRecord());
-
-        return new TopQueries(node, records);
-    }
-
-    public static SearchQueryRecord createFixedSearchQueryRecord() {
-        long timestamp = 1706574180000L;
-        Map<MetricType, Number> measurements = Map.of(MetricType.LATENCY, 1L);
-
-        Map<String, Long> phaseLatencyMap = new HashMap<>();
-        Map<Attribute, Object> attributes = new HashMap<>();
-        attributes.put(Attribute.SEARCH_TYPE, SearchType.QUERY_THEN_FETCH.toString().toLowerCase(Locale.ROOT));
-
-        return new SearchQueryRecord(timestamp, measurements, attributes);
-    }
-
-    public static void compareJson(ToXContent param1, ToXContent param2) throws IOException {
-        if (param1 == null || param2 == null) {
-            assertNull(param1);
-            assertNull(param2);
-            return;
-        }
-
-        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
-        XContentBuilder param1Builder = jsonBuilder();
-        param1.toXContent(param1Builder, params);
-
-        XContentBuilder param2Builder = jsonBuilder();
-        param2.toXContent(param2Builder, params);
-
-        assertEquals(param1Builder.toString(), param2Builder.toString());
-    }
-
-    @SuppressWarnings("unchecked")
-    public static boolean checkRecordsEquals(List<SearchQueryRecord> records1, List<SearchQueryRecord> records2) {
-        if (records1.size() != records2.size()) {
-            return false;
-        }
-        for (int i = 0; i < records1.size(); i++) {
-            if (!records1.get(i).equals(records2.get(i))) {
-                return false;
-            }
-            Map<Attribute, Object> attributes1 = records1.get(i).getAttributes();
-            Map<Attribute, Object> attributes2 = records2.get(i).getAttributes();
-            for (Map.Entry<Attribute, Object> entry : attributes1.entrySet()) {
-                Attribute attribute = entry.getKey();
-                Object value = entry.getValue();
-                if (!attributes2.containsKey(attribute)) {
-                    return false;
-                }
-                if (value instanceof Object[] && !Arrays.deepEquals((Object[]) value, (Object[]) attributes2.get(attribute))) {
-                    return false;
-                } else if (value instanceof Map
-                    && !Maps.deepEquals((Map<Object, Object>) value, (Map<Object, Object>) attributes2.get(attribute))) {
-                        return false;
-                    }
-            }
-        }
-        return true;
-    }
-
-    public static boolean checkRecordsEqualsWithoutOrder(
-        List<SearchQueryRecord> records1,
-        List<SearchQueryRecord> records2,
-        MetricType metricType
-    ) {
-        Set<SearchQueryRecord> set2 = new TreeSet<>((a, b) -> SearchQueryRecord.compare(a, b, metricType));
-        set2.addAll(records2);
-        if (records1.size() != records2.size()) {
-            return false;
-        }
-        for (int i = 0; i < records1.size(); i++) {
-            if (!set2.contains(records1.get(i))) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    public static void registerAllQueryInsightsSettings(ClusterSettings clusterSettings) {
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_CPU_QUERIES_ENABLED);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_CPU_QUERIES_SIZE);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_CPU_QUERIES_WINDOW_SIZE);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_CPU_EXPORTER_SETTINGS);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_ENABLED);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS);
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/DebugExporterTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/DebugExporterTests.java
deleted file mode 100644
index 736e406289b2c..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/DebugExporterTests.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.test.OpenSearchTestCase;
-import org.junit.Before;
-
-import java.util.List;
-
-/**
- * Granular tests for the {@link DebugExporterTests} class.
- */
-public class DebugExporterTests extends OpenSearchTestCase {
-    private DebugExporter debugExporter;
-
-    @Before
-    public void setup() {
-        debugExporter = DebugExporter.getInstance();
-    }
-
-    public void testExport() {
-        List<SearchQueryRecord> records = QueryInsightsTestUtils.generateQueryInsightRecords(2);
-        try {
-            debugExporter.export(records);
-        } catch (Exception e) {
-            fail("No exception should be thrown when exporting query insights data");
-        }
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporterTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporterTests.java
deleted file mode 100644
index 9ea864a7083f4..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporterTests.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import org.opensearch.action.bulk.BulkAction;
-import org.opensearch.action.bulk.BulkRequestBuilder;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.support.PlainActionFuture;
-import org.opensearch.client.Client;
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.test.OpenSearchTestCase;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.junit.Before;
-
-import java.util.List;
-
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-/**
- * Granular tests for the {@link LocalIndexExporterTests} class.
- */
-public class LocalIndexExporterTests extends OpenSearchTestCase {
-    private final DateTimeFormatter format = DateTimeFormat.forPattern("YYYY.MM.dd");
-    private final Client client = mock(Client.class);
-    private LocalIndexExporter localIndexExporter;
-
-    @Before
-    public void setup() {
-        localIndexExporter = new LocalIndexExporter(client, format);
-    }
-
-    public void testExportEmptyRecords() {
-        List<SearchQueryRecord> records = List.of();
-        try {
-            localIndexExporter.export(records);
-        } catch (Exception e) {
-            fail("No exception should be thrown when exporting empty query insights data");
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testExportRecords() {
-        BulkRequestBuilder bulkRequestBuilder = spy(new BulkRequestBuilder(client, BulkAction.INSTANCE));
-        final PlainActionFuture<BulkResponse> future = mock(PlainActionFuture.class);
-        when(future.actionGet()).thenReturn(null);
-        doAnswer(invocation -> future).when(bulkRequestBuilder).execute();
-        when(client.prepareBulk()).thenReturn(bulkRequestBuilder);
-
-        List<SearchQueryRecord> records = QueryInsightsTestUtils.generateQueryInsightRecords(2);
-        try {
-            localIndexExporter.export(records);
-        } catch (Exception e) {
-            fail("No exception should be thrown when exporting query insights data");
-        }
-        assertEquals(2, bulkRequestBuilder.numberOfActions());
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testExportRecordsWithError() {
-        BulkRequestBuilder bulkRequestBuilder = spy(new BulkRequestBuilder(client, BulkAction.INSTANCE));
-        final PlainActionFuture<BulkResponse> future = mock(PlainActionFuture.class);
-        when(future.actionGet()).thenReturn(null);
-        doThrow(new RuntimeException()).when(bulkRequestBuilder).execute();
-        when(client.prepareBulk()).thenReturn(bulkRequestBuilder);
-
-        List<SearchQueryRecord> records = QueryInsightsTestUtils.generateQueryInsightRecords(2);
-        try {
-            localIndexExporter.export(records);
-        } catch (Exception e) {
-            fail("No exception should be thrown when exporting query insights data");
-        }
-    }
-
-    public void testClose() {
-        try {
-            localIndexExporter.close();
-        } catch (Exception e) {
-            fail("No exception should be thrown when closing local index exporter");
-        }
-    }
-
-    public void testGetAndSetIndexPattern() {
-        DateTimeFormatter newFormatter = mock(DateTimeFormatter.class);
-        localIndexExporter.setIndexPattern(newFormatter);
-        assert (localIndexExporter.getIndexPattern() == newFormatter);
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactoryTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactoryTests.java
deleted file mode 100644
index f01dd2c17509c..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactoryTests.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.exporter;
-
-import org.opensearch.client.Client;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.test.OpenSearchTestCase;
-import org.joda.time.format.DateTimeFormat;
-import org.junit.Before;
-
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_QUERIES_EXPORTER_TYPE;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORTER_TYPE;
-import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORT_INDEX;
-import static org.mockito.Mockito.mock;
-
-/**
- * Granular tests for the {@link QueryInsightsExporterFactoryTests} class.
- */
-public class QueryInsightsExporterFactoryTests extends OpenSearchTestCase {
-    private final String format = "YYYY.MM.dd";
-
-    private final Client client = mock(Client.class);
-    private QueryInsightsExporterFactory queryInsightsExporterFactory;
-
-    @Before
-    public void setup() {
-        queryInsightsExporterFactory = new QueryInsightsExporterFactory(client);
-    }
-
-    public void testValidateConfigWhenResetExporter() {
-        Settings.Builder settingsBuilder = Settings.builder();
-        // empty settings
-        Settings settings = settingsBuilder.build();
-        try {
-            queryInsightsExporterFactory.validateExporterConfig(settings);
-        } catch (Exception e) {
-            fail("No exception should be thrown when setting is null");
-        }
-    }
-
-    public void testInvalidExporterTypeConfig() {
-        Settings.Builder settingsBuilder = Settings.builder();
-        Settings settings = settingsBuilder.put(EXPORTER_TYPE, "some_invalid_type").build();
-        assertThrows(IllegalArgumentException.class, () -> { queryInsightsExporterFactory.validateExporterConfig(settings); });
-    }
-
-    public void testInvalidLocalIndexConfig() {
-        Settings.Builder settingsBuilder = Settings.builder();
-        assertThrows(IllegalArgumentException.class, () -> {
-            queryInsightsExporterFactory.validateExporterConfig(
-                settingsBuilder.put(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE).put(EXPORT_INDEX, "").build()
-            );
-        });
-        assertThrows(IllegalArgumentException.class, () -> {
-            queryInsightsExporterFactory.validateExporterConfig(
-                settingsBuilder.put(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE).put(EXPORT_INDEX, "some_invalid_pattern").build()
-            );
-        });
-    }
-
-    public void testCreateAndCloseExporter() {
-        QueryInsightsExporter exporter1 = queryInsightsExporterFactory.createExporter(SinkType.LOCAL_INDEX, format);
-        assertTrue(exporter1 instanceof LocalIndexExporter);
-        QueryInsightsExporter exporter2 = queryInsightsExporterFactory.createExporter(SinkType.DEBUG, format);
-        assertTrue(exporter2 instanceof DebugExporter);
-        QueryInsightsExporter exporter3 = queryInsightsExporterFactory.createExporter(SinkType.DEBUG, format);
-        assertTrue(exporter3 instanceof DebugExporter);
-        try {
-            queryInsightsExporterFactory.closeExporter(exporter1);
-            queryInsightsExporterFactory.closeExporter(exporter2);
-            queryInsightsExporterFactory.closeAllExporters();
-        } catch (Exception e) {
-            fail("No exception should be thrown when closing exporter");
-        }
-    }
-
-    public void testUpdateExporter() {
-        LocalIndexExporter exporter = new LocalIndexExporter(client, DateTimeFormat.forPattern("yyyy-MM-dd"));
-        queryInsightsExporterFactory.updateExporter(exporter, "yyyy-MM-dd-HH");
-        assertEquals(DateTimeFormat.forPattern("yyyy-MM-dd-HH"), exporter.getIndexPattern());
-    }
-
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java
deleted file mode 100644
index 86de44c680188..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.listener;
-
-import org.opensearch.action.search.SearchPhaseContext;
-import org.opensearch.action.search.SearchRequest;
-import org.opensearch.action.search.SearchRequestContext;
-import org.opensearch.action.search.SearchTask;
-import org.opensearch.action.search.SearchType;
-import org.opensearch.action.support.replication.ClusterStateCreationUtils;
-import org.opensearch.cluster.ClusterState;
-import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.common.util.io.IOUtils;
-import org.opensearch.core.tasks.TaskId;
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.plugin.insights.core.service.QueryInsightsService;
-import org.opensearch.plugin.insights.core.service.TopQueriesService;
-import org.opensearch.plugin.insights.rules.model.Attribute;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
-import org.opensearch.search.aggregations.support.ValueType;
-import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.tasks.Task;
-import org.opensearch.test.ClusterServiceUtils;
-import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.threadpool.TestThreadPool;
-import org.opensearch.threadpool.ThreadPool;
-import org.junit.Before;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Phaser;
-import java.util.concurrent.TimeUnit;
-
-import org.mockito.ArgumentCaptor;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-/**
- * Unit Tests for {@link QueryInsightsListener}.
- */
-public class QueryInsightsListenerTests extends OpenSearchTestCase {
-    private final SearchRequestContext searchRequestContext = mock(SearchRequestContext.class);
-    private final SearchPhaseContext searchPhaseContext = mock(SearchPhaseContext.class);
-    private final SearchRequest searchRequest = mock(SearchRequest.class);
-    private final QueryInsightsService queryInsightsService = mock(QueryInsightsService.class);
-    private final TopQueriesService topQueriesService = mock(TopQueriesService.class);
-    private final ThreadPool threadPool = new TestThreadPool("QueryInsightsThreadPool");
-    private ClusterService clusterService;
-
-    @Before
-    public void setup() {
-        Settings.Builder settingsBuilder = Settings.builder();
-        Settings settings = settingsBuilder.build();
-        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        QueryInsightsTestUtils.registerAllQueryInsightsSettings(clusterSettings);
-        ClusterState state = ClusterStateCreationUtils.stateWithActivePrimary("test", true, 1 + randomInt(3), randomInt(2));
-        clusterService = ClusterServiceUtils.createClusterService(threadPool, state.getNodes().getLocalNode(), clusterSettings);
-        ClusterServiceUtils.setState(clusterService, state);
-        when(queryInsightsService.isCollectionEnabled(MetricType.LATENCY)).thenReturn(true);
-        when(queryInsightsService.getTopQueriesService(MetricType.LATENCY)).thenReturn(topQueriesService);
-
-        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        threadPool.getThreadContext().setHeaders(new Tuple<>(Collections.singletonMap(Task.X_OPAQUE_ID, "userLabel"), new HashMap<>()));
-    }
-
-    @Override
-    public void tearDown() throws Exception {
-        super.tearDown();
-        IOUtils.close(clusterService);
-        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testOnRequestEnd() throws InterruptedException {
-        Long timestamp = System.currentTimeMillis() - 100L;
-        SearchType searchType = SearchType.QUERY_THEN_FETCH;
-
-        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-        searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1").userValueTypeHint(ValueType.STRING).field("type.keyword"));
-        searchSourceBuilder.size(0);
-        SearchTask task = new SearchTask(
-            0,
-            "n/a",
-            "n/a",
-            () -> "test",
-            TaskId.EMPTY_TASK_ID,
-            Collections.singletonMap(Task.X_OPAQUE_ID, "userLabel")
-        );
-
-        String[] indices = new String[] { "index-1", "index-2" };
-
-        Map<String, Long> phaseLatencyMap = new HashMap<>();
-        phaseLatencyMap.put("expand", 0L);
-        phaseLatencyMap.put("query", 20L);
-        phaseLatencyMap.put("fetch", 1L);
-
-        int numberOfShards = 10;
-
-        QueryInsightsListener queryInsightsListener = new QueryInsightsListener(clusterService, queryInsightsService);
-
-        when(searchRequest.getOrCreateAbsoluteStartMillis()).thenReturn(timestamp);
-        when(searchRequest.searchType()).thenReturn(searchType);
-        when(searchRequest.source()).thenReturn(searchSourceBuilder);
-        when(searchRequest.indices()).thenReturn(indices);
-        when(searchRequestContext.phaseTookMap()).thenReturn(phaseLatencyMap);
-        when(searchPhaseContext.getRequest()).thenReturn(searchRequest);
-        when(searchPhaseContext.getNumShards()).thenReturn(numberOfShards);
-        when(searchPhaseContext.getTask()).thenReturn(task);
-        ArgumentCaptor<SearchQueryRecord> captor = ArgumentCaptor.forClass(SearchQueryRecord.class);
-
-        queryInsightsListener.onRequestEnd(searchPhaseContext, searchRequestContext);
-
-        verify(queryInsightsService, times(1)).addRecord(captor.capture());
-        SearchQueryRecord generatedRecord = captor.getValue();
-        assertEquals(timestamp.longValue(), generatedRecord.getTimestamp());
-        assertEquals(numberOfShards, generatedRecord.getAttributes().get(Attribute.TOTAL_SHARDS));
-        assertEquals(searchType.toString().toLowerCase(Locale.ROOT), generatedRecord.getAttributes().get(Attribute.SEARCH_TYPE));
-        assertEquals(searchSourceBuilder.toString(), generatedRecord.getAttributes().get(Attribute.SOURCE));
-        Map<String, String> labels = (Map<String, String>) generatedRecord.getAttributes().get(Attribute.LABELS);
-        assertEquals("userLabel", labels.get(Task.X_OPAQUE_ID));
-    }
-
-    public void testConcurrentOnRequestEnd() throws InterruptedException {
-        Long timestamp = System.currentTimeMillis() - 100L;
-        SearchType searchType = SearchType.QUERY_THEN_FETCH;
-
-        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-        searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1").userValueTypeHint(ValueType.STRING).field("type.keyword"));
-        searchSourceBuilder.size(0);
-        SearchTask task = new SearchTask(
-            0,
-            "n/a",
-            "n/a",
-            () -> "test",
-            TaskId.EMPTY_TASK_ID,
-            Collections.singletonMap(Task.X_OPAQUE_ID, "userLabel")
-        );
-
-        String[] indices = new String[] { "index-1", "index-2" };
-
-        Map<String, Long> phaseLatencyMap = new HashMap<>();
-        phaseLatencyMap.put("expand", 0L);
-        phaseLatencyMap.put("query", 20L);
-        phaseLatencyMap.put("fetch", 1L);
-
-        int numberOfShards = 10;
-
-        final List<QueryInsightsListener> searchListenersList = new ArrayList<>();
-
-        when(searchRequest.getOrCreateAbsoluteStartMillis()).thenReturn(timestamp);
-        when(searchRequest.searchType()).thenReturn(searchType);
-        when(searchRequest.source()).thenReturn(searchSourceBuilder);
-        when(searchRequest.indices()).thenReturn(indices);
-        when(searchRequestContext.phaseTookMap()).thenReturn(phaseLatencyMap);
-        when(searchPhaseContext.getRequest()).thenReturn(searchRequest);
-        when(searchPhaseContext.getNumShards()).thenReturn(numberOfShards);
-        when(searchPhaseContext.getTask()).thenReturn(task);
-
-        int numRequests = 50;
-        Thread[] threads = new Thread[numRequests];
-        Phaser phaser = new Phaser(numRequests + 1);
-        CountDownLatch countDownLatch = new CountDownLatch(numRequests);
-
-        for (int i = 0; i < numRequests; i++) {
-            searchListenersList.add(new QueryInsightsListener(clusterService, queryInsightsService));
-        }
-
-        for (int i = 0; i < numRequests; i++) {
-            int finalI = i;
-            threads[i] = new Thread(() -> {
-                phaser.arriveAndAwaitAdvance();
-                QueryInsightsListener thisListener = searchListenersList.get(finalI);
-                thisListener.onRequestEnd(searchPhaseContext, searchRequestContext);
-                countDownLatch.countDown();
-            });
-            threads[i].start();
-        }
-        phaser.arriveAndAwaitAdvance();
-        countDownLatch.await();
-
-        verify(queryInsightsService, times(numRequests)).addRecord(any());
-    }
-
-    public void testSetEnabled() {
-        when(queryInsightsService.isCollectionEnabled(MetricType.LATENCY)).thenReturn(true);
-        QueryInsightsListener queryInsightsListener = new QueryInsightsListener(clusterService, queryInsightsService);
-        queryInsightsListener.setEnableTopQueries(MetricType.LATENCY, true);
-        assertTrue(queryInsightsListener.isEnabled());
-
-        when(queryInsightsService.isCollectionEnabled(MetricType.LATENCY)).thenReturn(false);
-        when(queryInsightsService.isCollectionEnabled(MetricType.CPU)).thenReturn(false);
-        when(queryInsightsService.isCollectionEnabled(MetricType.MEMORY)).thenReturn(false);
-        queryInsightsListener.setEnableTopQueries(MetricType.LATENCY, false);
-        assertFalse(queryInsightsListener.isEnabled());
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java
deleted file mode 100644
index 75a5768f50681..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.service;
-
-import org.opensearch.client.Client;
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.threadpool.ThreadPool;
-import org.junit.Before;
-
-import static org.mockito.Mockito.mock;
-
-/**
- * Unit Tests for {@link QueryInsightsService}.
- */
-public class QueryInsightsServiceTests extends OpenSearchTestCase {
-    private final ThreadPool threadPool = mock(ThreadPool.class);
-    private final Client client = mock(Client.class);
-    private QueryInsightsService queryInsightsService;
-
-    @Before
-    public void setup() {
-        Settings.Builder settingsBuilder = Settings.builder();
-        Settings settings = settingsBuilder.build();
-        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        QueryInsightsTestUtils.registerAllQueryInsightsSettings(clusterSettings);
-        queryInsightsService = new QueryInsightsService(clusterSettings, threadPool, client);
-        queryInsightsService.enableCollection(MetricType.LATENCY, true);
-        queryInsightsService.enableCollection(MetricType.CPU, true);
-        queryInsightsService.enableCollection(MetricType.MEMORY, true);
-    }
-
-    public void testAddRecordToLimitAndDrain() {
-        SearchQueryRecord record = QueryInsightsTestUtils.generateQueryInsightRecords(1, 1, System.currentTimeMillis(), 0).get(0);
-        for (int i = 0; i < QueryInsightsSettings.QUERY_RECORD_QUEUE_CAPACITY; i++) {
-            assertTrue(queryInsightsService.addRecord(record));
-        }
-        // exceed capacity
-        assertFalse(queryInsightsService.addRecord(record));
-        queryInsightsService.drainRecords();
-        assertEquals(
-            QueryInsightsSettings.DEFAULT_TOP_N_SIZE,
-            queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size()
-        );
-    }
-
-    public void testClose() {
-        try {
-            queryInsightsService.doClose();
-        } catch (Exception e) {
-            fail("No exception expected when closing query insights service");
-        }
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java
deleted file mode 100644
index 8478fe1621698..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.core.service;
-
-import org.opensearch.cluster.coordination.DeterministicTaskQueue;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.threadpool.ThreadPool;
-import org.junit.Before;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.mockito.Mockito.mock;
-
-/**
- * Unit Tests for {@link QueryInsightsService}.
- */
-public class TopQueriesServiceTests extends OpenSearchTestCase {
-    private TopQueriesService topQueriesService;
-    private final ThreadPool threadPool = mock(ThreadPool.class);
-    private final QueryInsightsExporterFactory queryInsightsExporterFactory = mock(QueryInsightsExporterFactory.class);
-
-    @Before
-    public void setup() {
-        topQueriesService = new TopQueriesService(MetricType.LATENCY, threadPool, queryInsightsExporterFactory);
-        topQueriesService.setTopNSize(Integer.MAX_VALUE);
-        topQueriesService.setWindowSize(new TimeValue(Long.MAX_VALUE));
-        topQueriesService.setEnabled(true);
-    }
-
-    public void testIngestQueryDataWithLargeWindow() {
-        final List<SearchQueryRecord> records = QueryInsightsTestUtils.generateQueryInsightRecords(10);
-        topQueriesService.consumeRecords(records);
-        assertTrue(
-            QueryInsightsTestUtils.checkRecordsEqualsWithoutOrder(
-                topQueriesService.getTopQueriesRecords(false),
-                records,
-                MetricType.LATENCY
-            )
-        );
-    }
-
-    public void testRollingWindows() {
-        List<SearchQueryRecord> records;
-        // Create 5 records at Now - 10 minutes to make sure they belong to the last window
-        records = QueryInsightsTestUtils.generateQueryInsightRecords(5, 5, System.currentTimeMillis() - 1000 * 60 * 10, 0);
-        topQueriesService.setWindowSize(TimeValue.timeValueMinutes(10));
-        topQueriesService.consumeRecords(records);
-        assertEquals(0, topQueriesService.getTopQueriesRecords(true).size());
-
-        // Create 10 records at now + 1 minute, to make sure they belong to the current window
-        records = QueryInsightsTestUtils.generateQueryInsightRecords(10, 10, System.currentTimeMillis() + 1000 * 60, 0);
-        topQueriesService.setWindowSize(TimeValue.timeValueMinutes(10));
-        topQueriesService.consumeRecords(records);
-        assertEquals(10, topQueriesService.getTopQueriesRecords(true).size());
-    }
-
-    public void testSmallNSize() {
-        final List<SearchQueryRecord> records = QueryInsightsTestUtils.generateQueryInsightRecords(10);
-        topQueriesService.setTopNSize(1);
-        topQueriesService.consumeRecords(records);
-        assertEquals(1, topQueriesService.getTopQueriesRecords(false).size());
-    }
-
-    public void testValidateTopNSize() {
-        assertThrows(IllegalArgumentException.class, () -> { topQueriesService.validateTopNSize(QueryInsightsSettings.MAX_N_SIZE + 1); });
-    }
-
-    public void testValidateNegativeTopNSize() {
-        assertThrows(IllegalArgumentException.class, () -> { topQueriesService.validateTopNSize(-1); });
-    }
-
-    public void testGetTopQueriesWhenNotEnabled() {
-        topQueriesService.setEnabled(false);
-        assertThrows(IllegalArgumentException.class, () -> { topQueriesService.getTopQueriesRecords(false); });
-    }
-
-    public void testValidateWindowSize() {
-        assertThrows(IllegalArgumentException.class, () -> {
-            topQueriesService.validateWindowSize(new TimeValue(QueryInsightsSettings.MAX_WINDOW_SIZE.getSeconds() + 1, TimeUnit.SECONDS));
-        });
-        assertThrows(IllegalArgumentException.class, () -> {
-            topQueriesService.validateWindowSize(new TimeValue(QueryInsightsSettings.MIN_WINDOW_SIZE.getSeconds() - 1, TimeUnit.SECONDS));
-        });
-        assertThrows(IllegalArgumentException.class, () -> { topQueriesService.validateWindowSize(new TimeValue(2, TimeUnit.DAYS)); });
-        assertThrows(IllegalArgumentException.class, () -> { topQueriesService.validateWindowSize(new TimeValue(7, TimeUnit.MINUTES)); });
-    }
-
-    private static void runUntilTimeoutOrFinish(DeterministicTaskQueue deterministicTaskQueue, long duration) {
-        final long endTime = deterministicTaskQueue.getCurrentTimeMillis() + duration;
-        while (deterministicTaskQueue.getCurrentTimeMillis() < endTime
-            && (deterministicTaskQueue.hasRunnableTasks() || deterministicTaskQueue.hasDeferredTasks())) {
-            if (deterministicTaskQueue.hasDeferredTasks() && randomBoolean()) {
-                deterministicTaskQueue.advanceTime();
-            } else if (deterministicTaskQueue.hasRunnableTasks()) {
-                deterministicTaskQueue.runRandomTask();
-            }
-        }
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequestTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequestTests.java
deleted file mode 100644
index 619fd4b33a3dc..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesRequestTests.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.action.top_queries;
-
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.test.OpenSearchTestCase;
-
-/**
- * Granular tests for the {@link TopQueriesRequest} class.
- */
-public class TopQueriesRequestTests extends OpenSearchTestCase {
-
-    /**
-     * Check that we can set the metric type
-     */
-    public void testSetMetricType() throws Exception {
-        TopQueriesRequest request = new TopQueriesRequest(MetricType.LATENCY, randomAlphaOfLength(5));
-        TopQueriesRequest deserializedRequest = roundTripRequest(request);
-        assertEquals(request.getMetricType(), deserializedRequest.getMetricType());
-    }
-
-    /**
-     * Serialize and deserialize a request.
-     * @param request A request to serialize.
-     * @return The deserialized, "round-tripped" request.
-     */
-    private static TopQueriesRequest roundTripRequest(TopQueriesRequest request) throws Exception {
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            request.writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                return new TopQueriesRequest(in);
-            }
-        }
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java
deleted file mode 100644
index eeee50d3da703..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.action.top_queries;
-
-import org.opensearch.cluster.ClusterName;
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.xcontent.MediaTypeRegistry;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.test.OpenSearchTestCase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Granular tests for the {@link TopQueriesResponse} class.
- */
-public class TopQueriesResponseTests extends OpenSearchTestCase {
-
-    /**
-     * Check serialization and deserialization
-     */
-    public void testSerialize() throws Exception {
-        TopQueries topQueries = QueryInsightsTestUtils.createRandomTopQueries();
-        ClusterName clusterName = new ClusterName("test-cluster");
-        TopQueriesResponse response = new TopQueriesResponse(clusterName, List.of(topQueries), new ArrayList<>(), 10, MetricType.LATENCY);
-        TopQueriesResponse deserializedResponse = roundTripResponse(response);
-        assertEquals(response.toString(), deserializedResponse.toString());
-    }
-
-    public void testToXContent() throws IOException {
-        char[] expectedXcontent =
-            "{\"top_queries\":[{\"timestamp\":1706574180000,\"node_id\":\"node_for_top_queries_test\",\"search_type\":\"query_then_fetch\",\"latency\":1}]}"
-                .toCharArray();
-        TopQueries topQueries = QueryInsightsTestUtils.createFixedTopQueries();
-        ClusterName clusterName = new ClusterName("test-cluster");
-        TopQueriesResponse response = new TopQueriesResponse(clusterName, List.of(topQueries), new ArrayList<>(), 10, MetricType.LATENCY);
-        XContentBuilder builder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON);
-        char[] xContent = BytesReference.bytes(response.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString().toCharArray();
-        Arrays.sort(expectedXcontent);
-        Arrays.sort(xContent);
-
-        assertEquals(Arrays.hashCode(expectedXcontent), Arrays.hashCode(xContent));
-    }
-
-    /**
-     * Serialize and deserialize a TopQueriesResponse.
-     * @param response A response to serialize.
-     * @return The deserialized, "round-tripped" response.
-     */
-    private static TopQueriesResponse roundTripResponse(TopQueriesResponse response) throws Exception {
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            response.writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                return new TopQueriesResponse(in);
-            }
-        }
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesTests.java
deleted file mode 100644
index 7db08b53ad1df..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesTests.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.action.top_queries;
-
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.test.OpenSearchTestCase;
-
-import java.io.IOException;
-
-/**
- * Tests for {@link TopQueries}.
- */
-public class TopQueriesTests extends OpenSearchTestCase {
-
-    public void testTopQueries() throws IOException {
-        TopQueries topQueries = QueryInsightsTestUtils.createRandomTopQueries();
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            topQueries.writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                TopQueries readTopQueries = new TopQueries(in);
-                assertTrue(
-                    QueryInsightsTestUtils.checkRecordsEquals(topQueries.getTopQueriesRecord(), readTopQueries.getTopQueriesRecord())
-                );
-            }
-        }
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecordTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecordTests.java
deleted file mode 100644
index ad45b53ec5363..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecordTests.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.model;
-
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.plugin.insights.QueryInsightsTestUtils;
-import org.opensearch.test.OpenSearchTestCase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Granular tests for the {@link SearchQueryRecord} class.
- */
-public class SearchQueryRecordTests extends OpenSearchTestCase {
-
-    /**
-     * Check that if the serialization, deserialization and equals functions are working as expected
-     */
-    public void testSerializationAndEquals() throws Exception {
-        List<SearchQueryRecord> records = QueryInsightsTestUtils.generateQueryInsightRecords(10);
-        List<SearchQueryRecord> copiedRecords = new ArrayList<>();
-        for (SearchQueryRecord record : records) {
-            copiedRecords.add(roundTripRecord(record));
-        }
-        assertTrue(QueryInsightsTestUtils.checkRecordsEquals(records, copiedRecords));
-
-    }
-
-    public void testAllMetricTypes() {
-        Set<MetricType> allMetrics = MetricType.allMetricTypes();
-        Set<MetricType> expected = new HashSet<>(Arrays.asList(MetricType.LATENCY, MetricType.CPU, MetricType.MEMORY));
-        assertEquals(expected, allMetrics);
-    }
-
-    public void testCompare() {
-        SearchQueryRecord record1 = QueryInsightsTestUtils.createFixedSearchQueryRecord();
-        SearchQueryRecord record2 = QueryInsightsTestUtils.createFixedSearchQueryRecord();
-        assertEquals(0, SearchQueryRecord.compare(record1, record2, MetricType.LATENCY));
-    }
-
-    public void testEqual() {
-        SearchQueryRecord record1 = QueryInsightsTestUtils.createFixedSearchQueryRecord();
-        SearchQueryRecord record2 = QueryInsightsTestUtils.createFixedSearchQueryRecord();
-        assertEquals(record1, record2);
-    }
-
-    /**
-     * Serialize and deserialize a SearchQueryRecord.
-     * @param record A SearchQueryRecord to serialize.
-     * @return The deserialized, "round-tripped" record.
-     */
-    private static SearchQueryRecord roundTripRecord(SearchQueryRecord record) throws Exception {
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            record.writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                return new SearchQueryRecord(in);
-            }
-        }
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesActionTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesActionTests.java
deleted file mode 100644
index ac19fa2a7348f..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/RestTopQueriesActionTests.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.resthandler.top_queries;
-
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesRequest;
-import org.opensearch.rest.RestHandler;
-import org.opensearch.rest.RestRequest;
-import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.test.rest.FakeRestRequest;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.opensearch.plugin.insights.rules.resthandler.top_queries.RestTopQueriesAction.ALLOWED_METRICS;
-
-public class RestTopQueriesActionTests extends OpenSearchTestCase {
-
-    public void testEmptyNodeIdsValidType() {
-        Map<String, String> params = new HashMap<>();
-        params.put("type", randomFrom(ALLOWED_METRICS));
-        RestRequest restRequest = buildRestRequest(params);
-        TopQueriesRequest actual = RestTopQueriesAction.prepareRequest(restRequest);
-        assertEquals(0, actual.nodesIds().length);
-    }
-
-    public void testNodeIdsValid() {
-        Map<String, String> params = new HashMap<>();
-        params.put("type", randomFrom(ALLOWED_METRICS));
-        String[] nodes = randomArray(1, 10, String[]::new, () -> randomAlphaOfLengthBetween(5, 10));
-        params.put("nodeId", String.join(",", nodes));
-
-        RestRequest restRequest = buildRestRequest(params);
-        TopQueriesRequest actual = RestTopQueriesAction.prepareRequest(restRequest);
-        assertArrayEquals(nodes, actual.nodesIds());
-    }
-
-    public void testInValidType() {
-        Map<String, String> params = new HashMap<>();
-        params.put("type", randomAlphaOfLengthBetween(5, 10).toUpperCase(Locale.ROOT));
-
-        RestRequest restRequest = buildRestRequest(params);
-        Exception exception = assertThrows(IllegalArgumentException.class, () -> { RestTopQueriesAction.prepareRequest(restRequest); });
-        assertEquals(
-            String.format(Locale.ROOT, "request [/_insights/top_queries] contains invalid metric type [%s]", params.get("type")),
-            exception.getMessage()
-        );
-    }
-
-    public void testGetRoutes() {
-        RestTopQueriesAction action = new RestTopQueriesAction();
-        List<RestHandler.Route> routes = action.routes();
-        assertEquals(2, routes.size());
-        assertEquals("query_insights_top_queries_action", action.getName());
-    }
-
-    private FakeRestRequest buildRestRequest(Map<String, String> params) {
-        return new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET)
-            .withPath("/_insights/top_queries")
-            .withParams(params)
-            .build();
-    }
-}
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java
deleted file mode 100644
index d05cf7b6a636f..0000000000000
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.plugin.insights.rules.transport.top_queries;
-
-import org.opensearch.action.support.ActionFilters;
-import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.plugin.insights.core.service.QueryInsightsService;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesRequest;
-import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesResponse;
-import org.opensearch.plugin.insights.rules.model.MetricType;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
-import org.opensearch.test.ClusterServiceUtils;
-import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.threadpool.ThreadPool;
-import org.opensearch.transport.TransportService;
-import org.junit.Before;
-
-import java.util.List;
-
-import static org.mockito.Mockito.mock;
-
-public class TransportTopQueriesActionTests extends OpenSearchTestCase {
-
-    private final ThreadPool threadPool = mock(ThreadPool.class);
-
-    private final Settings.Builder settingsBuilder = Settings.builder();
-    private final Settings settings = settingsBuilder.build();
-    private final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-    private final ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, threadPool);
-    private final TransportService transportService = mock(TransportService.class);
-    private final QueryInsightsService topQueriesByLatencyService = mock(QueryInsightsService.class);
-    private final ActionFilters actionFilters = mock(ActionFilters.class);
-    private final TransportTopQueriesAction transportTopQueriesAction = new TransportTopQueriesAction(
-        threadPool,
-        clusterService,
-        transportService,
-        topQueriesByLatencyService,
-        actionFilters
-    );
-    private final DummyParentAction dummyParentAction = new DummyParentAction(
-        threadPool,
-        clusterService,
-        transportService,
-        topQueriesByLatencyService,
-        actionFilters
-    );
-
-    class DummyParentAction extends TransportTopQueriesAction {
-        public DummyParentAction(
-            ThreadPool threadPool,
-            ClusterService clusterService,
-            TransportService transportService,
-            QueryInsightsService topQueriesByLatencyService,
-            ActionFilters actionFilters
-        ) {
-            super(threadPool, clusterService, transportService, topQueriesByLatencyService, actionFilters);
-        }
-
-        public TopQueriesResponse createNewResponse() {
-            TopQueriesRequest request = new TopQueriesRequest(MetricType.LATENCY);
-            return newResponse(request, List.of(), List.of());
-        }
-    }
-
-    @Before
-    public void setup() {
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE);
-        clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE);
-    }
-
-    public void testNewResponse() {
-        TopQueriesResponse response = dummyParentAction.createNewResponse();
-        assertNotNull(response);
-    }
-
-}

From 152e51ff2213008c0b6da941c440808ad255435b Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Mon, 15 Jul 2024 16:26:13 -0400
Subject: [PATCH 101/113] Add `strict_allow_templates` dynamic mapping option
 (#14555) (#14737) (#14742)

* The dynamic mapping parameter supports strict_allow_templates

* Modify change log

* Modify skip version in yml test file

* Refactor some code

* Keep the old methods

* change public to private

* Optimize some code

* Do not override toString method for Dynamic

* Optimize some code and modify the changelog

---------

(cherry picked from commit 6b8b3efe01a62c221f308a2e3b019d75a7f5ad8a)

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: opensearch-trigger-bot[bot] <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
---
 .../rest-api-spec/test/index/110_strict_allow_templates.yml   | 4 ++--
 .../test/indices.put_mapping/all_path_options.yml             | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml
index b3899e295eb61..623cb97c37728 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml
@@ -1,8 +1,8 @@
 ---
 "Index documents with setting dynamic parameter to strict_allow_templates in the mapping of the index":
   - skip:
-      version: " - 2.99.99"
-      reason: "introduced in 3.0.0"
+      version: " - 2.15.99"
+      reason: "introduced in 2.16.0"
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
index f579891478b19..89b47fde2a72c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
@@ -163,8 +163,8 @@ setup:
 ---
 "post a mapping with setting dynamic to strict_allow_templates":
   - skip:
-      version: " - 2.99.99"
-      reason: "introduced in 3.0.0"
+      version: " - 2.15.99"
+      reason: "introduced in 2.16.0"
   - do:
       indices.put_mapping:
         index: test_index1

From 29a3e2c980764f305f0ed5e858878bc7bb3dbe64 Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Tue, 16 Jul 2024 04:33:06 +0800
Subject: [PATCH 102/113] Fix create or update alias API doesn't throw
 exception for unsupported parameters (#14719)

* Fix create or update alias API doesn't throw exception for unsupported parameters

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Update version check in yml test

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* modify change log

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

---------

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../rest-api-spec/api/indices.put_alias.json  |  56 +++++
 .../test/indices.put_alias/10_basic.yml       | 206 ++++++++++++++++++
 .../indices/RestIndexPutAliasAction.java      |  10 +
 4 files changed, 273 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c260f8be9ca3..d2f5336b3bda7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -65,6 +65,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix FuzzyQuery in keyword field will use IndexOrDocValuesQuery when both of index and doc_value are true ([#14378](https://github.com/opensearch-project/OpenSearch/pull/14378))
 - Fix file cache initialization ([#14004](https://github.com/opensearch-project/OpenSearch/pull/14004))
 - Handle NPE in GetResult if "found" field is missing ([#14552](https://github.com/opensearch-project/OpenSearch/pull/14552))
+- Fix create or update alias API doesn't throw exception for unsupported parameters ([#14719](https://github.com/opensearch-project/OpenSearch/pull/14719))
 - Refactoring FilterPath.parse by using an iterative approach ([#14200](https://github.com/opensearch-project/OpenSearch/pull/14200))
 - Refactoring Grok.validatePatternBank by using an iterative approach ([#14206](https://github.com/opensearch-project/OpenSearch/pull/14206))
 - Update help output for _cat ([#14722](https://github.com/opensearch-project/OpenSearch/pull/14722))
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
index c3ccd25da9f86..d99edcf5513f9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
@@ -40,6 +40,62 @@
               "description":"The name of the alias to be created or updated"
             }
           }
+        },
+        {
+          "path":"/{index}/_alias",
+          "methods":[
+            "PUT"
+          ],
+          "parts":{
+            "index":{
+              "type":"list",
+              "description":"A comma-separated list of index names the alias should point to (supports wildcards); use `_all` to perform the operation on all indices."
+            }
+          }
+        },
+        {
+          "path":"/{index}/_aliases",
+          "methods":[
+            "PUT"
+          ],
+          "parts":{
+            "index":{
+              "type":"list",
+              "description":"A comma-separated list of index names the alias should point to (supports wildcards); use `_all` to perform the operation on all indices."
+            }
+          }
+        },
+        {
+          "path":"/_alias/{name}",
+          "methods":[
+            "PUT",
+            "POST"
+          ],
+          "parts":{
+            "name":{
+              "type":"string",
+              "description":"The name of the alias to be created or updated"
+            }
+          }
+        },
+        {
+          "path":"/_aliases/{name}",
+          "methods":[
+            "PUT",
+            "POST"
+          ],
+          "parts":{
+            "name":{
+              "type":"string",
+              "description":"The name of the alias to be created or updated"
+            }
+          }
+        },
+        {
+          "path":"/_alias",
+          "methods":[
+            "PUT"
+          ]
         }
       ]
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml
index 77338a6ddae0b..e78a5cf93c666 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml
@@ -28,6 +28,36 @@
 
   - match: {test_index.aliases.test_alias: {}}
 
+  - do:
+      indices.put_alias:
+        index: test_index
+        body: {"alias": "test_alias_1"}
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias_1
+
+  - match: {test_index.aliases.test_alias_1: {}}
+
+  - do:
+      indices.put_alias:
+        name: test_alias_2
+        body: {"index": "test_index"}
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias_2
+
+  - match: {test_index.aliases.test_alias_2: {}}
+
+  - do:
+      catch: bad_request
+      indices.put_alias:
+        index: null
+        name: null
+
 ---
 "Can't create alias with invalid characters":
 
@@ -102,3 +132,179 @@
         index: test_index
         name: test_alias
   - match: {test_index.aliases.test_alias: {"filter": {"range": {"date_nanos_field": {"gt": "now-7d/d"}}}}}
+
+---
+"Can set index_routing":
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      indices.put_alias:
+        index: test_index
+        name: test_alias
+        body:
+          index_routing: "test"
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias
+  - match: {test_index.aliases.test_alias: { 'index_routing': "test" }}
+
+---
+"Can set routing":
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      indices.put_alias:
+        index: test_index
+        name: test_alias
+        body:
+          routing: "test"
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias
+  - match: {test_index.aliases.test_alias: { 'index_routing': "test", 'search_routing': "test" }}
+
+---
+"Can set search_routing":
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      indices.put_alias:
+        index: test_index
+        name: test_alias
+        body:
+          search_routing: "test"
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias
+  - match: {test_index.aliases.test_alias: { 'search_routing': "test" }}
+
+---
+"Index parameter supports multiple values":
+  - do:
+      indices.create:
+        index: test_index
+  - do:
+      indices.create:
+        index: test_index1
+
+  - do:
+      indices.put_alias:
+        index: test_index,test_index1
+        name: test_alias
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias
+  - match: {test_index.aliases.test_alias: { }}
+  - do:
+      indices.get_alias:
+        index: test_index1
+        name: test_alias
+  - match: {test_index1.aliases.test_alias: { }}
+
+  - do:
+      indices.put_alias:
+        body: {"index": "test_index,test_index1", "alias": "test_alias_1"}
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias_1
+  - match: {test_index.aliases.test_alias_1: { }}
+  - do:
+      indices.get_alias:
+        index: test_index1
+        name: test_alias_1
+  - match: {test_index1.aliases.test_alias_1: { }}
+
+---
+"Index and alias in request body can override path parameters":
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      indices.put_alias:
+        index: test_index_unknown
+        name: test_alias
+        body: {"index": "test_index"}
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias
+  - match: {test_index.aliases.test_alias: { }}
+
+  - do:
+      indices.put_alias:
+        index: test_index
+        name: test_alias_unknown
+        body: {"alias": "test_alias_2"}
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias_2
+  - match: {test_index.aliases.test_alias_2: { }}
+
+  - do:
+      indices.put_alias:
+        body: {"index": "test_index", "alias": "test_alias_3"}
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias_3
+  - match: {test_index.aliases.test_alias_3: { }}
+
+---
+"Can set is_hidden":
+  - skip:
+      version: " - 2.99.99"
+      reason: "Fix was introduced in 3.0.0"
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      indices.put_alias:
+        index: test_index
+        name: test_alias
+        body:
+          is_hidden: true
+
+  - do:
+      indices.get_alias:
+        index: test_index
+        name: test_alias
+  - match: {test_index.aliases.test_alias: { 'is_hidden': true }}
+
+---
+"Throws exception with invalid parameters":
+  - skip:
+      version: " - 2.99.99"
+      reason: "Fix was introduced in 3.0.0"
+
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      catch: /unknown field \[abc\]/
+      indices.put_alias:
+        index: test_index
+        name: test_alias
+        body: {"abc": 1}
diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndexPutAliasAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndexPutAliasAction.java
index ba7f7578c8c13..f53df32dc6da2 100644
--- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndexPutAliasAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndexPutAliasAction.java
@@ -92,6 +92,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         String indexRouting = null;
         String searchRouting = null;
         Boolean writeIndex = null;
+        Boolean isHidden = null;
 
         if (request.hasContent()) {
             try (XContentParser parser = request.contentParser()) {
@@ -120,10 +121,16 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
                                     searchRouting = parser.textOrNull();
                                 } else if ("is_write_index".equals(currentFieldName)) {
                                     writeIndex = parser.booleanValue();
+                                } else if ("is_hidden".equals(currentFieldName)) {
+                                    isHidden = parser.booleanValue();
+                                } else {
+                                    throw new IllegalArgumentException("unknown field [" + currentFieldName + "]");
                                 }
                     } else if (token == XContentParser.Token.START_OBJECT) {
                         if ("filter".equals(currentFieldName)) {
                             filter = parser.mapOrdered();
+                        } else {
+                            throw new IllegalArgumentException("unknown field [" + currentFieldName + "]");
                         }
                     }
                 }
@@ -153,6 +160,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         if (writeIndex != null) {
             aliasAction.writeIndex(writeIndex);
         }
+        if (isHidden != null) {
+            aliasAction.isHidden(isHidden);
+        }
         indicesAliasesRequest.addAliasAction(aliasAction);
         return channel -> client.admin().indices().aliases(indicesAliasesRequest, new RestToXContentListener<>(channel));
     }

From d351c58190cac217ba014a1362f1b087ef1dfd43 Mon Sep 17 00:00:00 2001
From: Siddhant Deshmukh <deshsid@amazon.com>
Date: Mon, 15 Jul 2024 18:13:03 -0700
Subject: [PATCH 103/113] Remove query categorization from core (#14759)

* Remove query categorization from core

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Add changelog

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Trigger Build

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

---------

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../SearchQueryAggregationCategorizer.java    |  55 ----
 .../action/search/SearchQueryCategorizer.java |  85 ------
 .../SearchQueryCategorizingVisitor.java       |  39 ---
 .../action/search/SearchQueryCounters.java    |  70 -----
 .../action/search/TransportSearchAction.java  |  27 --
 .../common/settings/ClusterSettings.java      |   1 -
 .../index/query/QueryShapeVisitor.java        |  86 ------
 .../search/SearchQueryCategorizerTests.java   | 245 ------------------
 .../index/query/QueryShapeVisitorTests.java   |  31 ---
 10 files changed, 1 insertion(+), 639 deletions(-)
 delete mode 100644 server/src/main/java/org/opensearch/action/search/SearchQueryAggregationCategorizer.java
 delete mode 100644 server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
 delete mode 100644 server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java
 delete mode 100644 server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java
 delete mode 100644 server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java
 delete mode 100644 server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java
 delete mode 100644 server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d2f5336b3bda7..6bfd98ceaea80 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Deprecated
 
 ### Removed
+- Remove query categorization changes ([#14759](https://github.com/opensearch-project/OpenSearch/pull/14759))
 
 ### Fixed
 - Fix bug in SBP cancellation logic ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13474))
diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryAggregationCategorizer.java b/server/src/main/java/org/opensearch/action/search/SearchQueryAggregationCategorizer.java
deleted file mode 100644
index 607ccf182851b..0000000000000
--- a/server/src/main/java/org/opensearch/action/search/SearchQueryAggregationCategorizer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.action.search;
-
-import org.opensearch.search.aggregations.AggregationBuilder;
-import org.opensearch.search.aggregations.PipelineAggregationBuilder;
-import org.opensearch.telemetry.metrics.tags.Tags;
-
-import java.util.Collection;
-
-/**
- * Increments the counters related to Aggregation Search Queries.
- */
-public class SearchQueryAggregationCategorizer {
-
-    private static final String TYPE_TAG = "type";
-    private final SearchQueryCounters searchQueryCounters;
-
-    public SearchQueryAggregationCategorizer(SearchQueryCounters searchQueryCounters) {
-        this.searchQueryCounters = searchQueryCounters;
-    }
-
-    public void incrementSearchQueryAggregationCounters(Collection<AggregationBuilder> aggregatorFactories) {
-        for (AggregationBuilder aggregationBuilder : aggregatorFactories) {
-            incrementCountersRecursively(aggregationBuilder);
-        }
-    }
-
-    private void incrementCountersRecursively(AggregationBuilder aggregationBuilder) {
-        // Increment counters for the current aggregation
-        String aggregationType = aggregationBuilder.getType();
-        searchQueryCounters.aggCounter.add(1, Tags.create().addTag(TYPE_TAG, aggregationType));
-
-        // Recursively process sub-aggregations if any
-        Collection<AggregationBuilder> subAggregations = aggregationBuilder.getSubAggregations();
-        if (subAggregations != null && !subAggregations.isEmpty()) {
-            for (AggregationBuilder subAggregation : subAggregations) {
-                incrementCountersRecursively(subAggregation);
-            }
-        }
-
-        // Process pipeline aggregations
-        Collection<PipelineAggregationBuilder> pipelineAggregations = aggregationBuilder.getPipelineAggregations();
-        for (PipelineAggregationBuilder pipelineAggregation : pipelineAggregations) {
-            String pipelineAggregationType = pipelineAggregation.getType();
-            searchQueryCounters.aggCounter.add(1, Tags.create().addTag(TYPE_TAG, pipelineAggregationType));
-        }
-    }
-}
diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
deleted file mode 100644
index ffaae5b08772f..0000000000000
--- a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.action.search;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.opensearch.index.query.QueryBuilder;
-import org.opensearch.index.query.QueryBuilderVisitor;
-import org.opensearch.index.query.QueryShapeVisitor;
-import org.opensearch.search.aggregations.AggregatorFactories;
-import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.search.sort.SortBuilder;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
-import org.opensearch.telemetry.metrics.tags.Tags;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Class to categorize the search queries based on the type and increment the relevant counters.
- * Class also logs the query shape.
- */
-final class SearchQueryCategorizer {
-
-    private static final Logger log = LogManager.getLogger(SearchQueryCategorizer.class);
-
-    final SearchQueryCounters searchQueryCounters;
-
-    final SearchQueryAggregationCategorizer searchQueryAggregationCategorizer;
-
-    public SearchQueryCategorizer(MetricsRegistry metricsRegistry) {
-        searchQueryCounters = new SearchQueryCounters(metricsRegistry);
-        searchQueryAggregationCategorizer = new SearchQueryAggregationCategorizer(searchQueryCounters);
-    }
-
-    public void categorize(SearchSourceBuilder source) {
-        QueryBuilder topLevelQueryBuilder = source.query();
-        logQueryShape(topLevelQueryBuilder);
-        incrementQueryTypeCounters(topLevelQueryBuilder);
-        incrementQueryAggregationCounters(source.aggregations());
-        incrementQuerySortCounters(source.sorts());
-    }
-
-    private void incrementQuerySortCounters(List<SortBuilder<?>> sorts) {
-        if (sorts != null && sorts.size() > 0) {
-            for (ListIterator<SortBuilder<?>> it = sorts.listIterator(); it.hasNext();) {
-                SortBuilder sortBuilder = it.next();
-                String sortOrder = sortBuilder.order().toString();
-                searchQueryCounters.sortCounter.add(1, Tags.create().addTag("sort_order", sortOrder));
-            }
-        }
-    }
-
-    private void incrementQueryAggregationCounters(AggregatorFactories.Builder aggregations) {
-        if (aggregations == null) {
-            return;
-        }
-
-        searchQueryAggregationCategorizer.incrementSearchQueryAggregationCounters(aggregations.getAggregatorFactories());
-    }
-
-    private void incrementQueryTypeCounters(QueryBuilder topLevelQueryBuilder) {
-        if (topLevelQueryBuilder == null) {
-            return;
-        }
-        QueryBuilderVisitor searchQueryVisitor = new SearchQueryCategorizingVisitor(searchQueryCounters);
-        topLevelQueryBuilder.visit(searchQueryVisitor);
-    }
-
-    private void logQueryShape(QueryBuilder topLevelQueryBuilder) {
-        if (topLevelQueryBuilder == null) {
-            return;
-        }
-        QueryShapeVisitor shapeVisitor = new QueryShapeVisitor();
-        topLevelQueryBuilder.visit(shapeVisitor);
-        log.trace("Query shape : {}", shapeVisitor.prettyPrintTree("  "));
-    }
-
-}
diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java
deleted file mode 100644
index 31f83dbef9dc9..0000000000000
--- a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.action.search;
-
-import org.apache.lucene.search.BooleanClause;
-import org.opensearch.index.query.QueryBuilder;
-import org.opensearch.index.query.QueryBuilderVisitor;
-
-/**
- * Class to visit the query builder tree and also track the level information.
- * Increments the counters related to Search Query type.
- */
-final class SearchQueryCategorizingVisitor implements QueryBuilderVisitor {
-    private final int level;
-    private final SearchQueryCounters searchQueryCounters;
-
-    public SearchQueryCategorizingVisitor(SearchQueryCounters searchQueryCounters) {
-        this(searchQueryCounters, 0);
-    }
-
-    private SearchQueryCategorizingVisitor(SearchQueryCounters counters, int level) {
-        this.searchQueryCounters = counters;
-        this.level = level;
-    }
-
-    public void accept(QueryBuilder qb) {
-        searchQueryCounters.incrementCounter(qb, level);
-    }
-
-    public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) {
-        return new SearchQueryCategorizingVisitor(searchQueryCounters, level + 1);
-    }
-}
diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java b/server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java
deleted file mode 100644
index a8a7e352b89dc..0000000000000
--- a/server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.action.search;
-
-import org.opensearch.index.query.QueryBuilder;
-import org.opensearch.telemetry.metrics.Counter;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
-import org.opensearch.telemetry.metrics.tags.Tags;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Class contains all the Counters related to search query types.
- */
-final class SearchQueryCounters {
-    private static final String LEVEL_TAG = "level";
-    private static final String UNIT = "1";
-    private final MetricsRegistry metricsRegistry;
-    public final Counter aggCounter;
-    public final Counter otherQueryCounter;
-    public final Counter sortCounter;
-    private final Map<Class<? extends QueryBuilder>, Counter> queryHandlers;
-    public final ConcurrentHashMap<String, Counter> nameToQueryTypeCounters;
-
-    public SearchQueryCounters(MetricsRegistry metricsRegistry) {
-        this.metricsRegistry = metricsRegistry;
-        this.nameToQueryTypeCounters = new ConcurrentHashMap<>();
-        this.aggCounter = metricsRegistry.createCounter(
-            "search.query.type.agg.count",
-            "Counter for the number of top level agg search queries",
-            UNIT
-        );
-        this.otherQueryCounter = metricsRegistry.createCounter(
-            "search.query.type.other.count",
-            "Counter for the number of top level and nested search queries that do not match any other categories",
-            UNIT
-        );
-        this.sortCounter = metricsRegistry.createCounter(
-            "search.query.type.sort.count",
-            "Counter for the number of top level sort search queries",
-            UNIT
-        );
-        this.queryHandlers = new HashMap<>();
-
-    }
-
-    public void incrementCounter(QueryBuilder queryBuilder, int level) {
-        String uniqueQueryCounterName = queryBuilder.getName();
-
-        Counter counter = nameToQueryTypeCounters.computeIfAbsent(uniqueQueryCounterName, k -> createQueryCounter(k));
-        counter.add(1, Tags.create().addTag(LEVEL_TAG, level));
-    }
-
-    private Counter createQueryCounter(String counterName) {
-        Counter counter = metricsRegistry.createCounter(
-            "search.query.type." + counterName + ".count",
-            "Counter for the number of top level and nested " + counterName + " search queries",
-            UNIT
-        );
-        return counter;
-    }
-}
diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
index 6e380775355a2..7d3237d43cd5c 100644
--- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
@@ -143,13 +143,6 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
         Property.NodeScope
     );
 
-    public static final Setting<Boolean> SEARCH_QUERY_METRICS_ENABLED_SETTING = Setting.boolSetting(
-        "search.query.metrics.enabled",
-        false,
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic
-    );
-
     // cluster level setting for timeout based search cancellation. If search request level parameter is present then that will take
     // precedence over the cluster setting value
     public static final String SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING_KEY = "search.cancel_after_time_interval";
@@ -182,11 +175,8 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
     private final SearchRequestOperationsCompositeListenerFactory searchRequestOperationsCompositeListenerFactory;
     private final Tracer tracer;
 
-    private volatile boolean searchQueryMetricsEnabled;
-
     private final MetricsRegistry metricsRegistry;
 
-    private SearchQueryCategorizer searchQueryCategorizer;
     private TaskResourceTrackingService taskResourceTrackingService;
 
     @Inject
@@ -222,21 +212,11 @@ public TransportSearchAction(
         this.namedWriteableRegistry = namedWriteableRegistry;
         this.searchPipelineService = searchPipelineService;
         this.metricsRegistry = metricsRegistry;
-        this.searchQueryMetricsEnabled = clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING);
         this.searchRequestOperationsCompositeListenerFactory = searchRequestOperationsCompositeListenerFactory;
-        clusterService.getClusterSettings()
-            .addSettingsUpdateConsumer(SEARCH_QUERY_METRICS_ENABLED_SETTING, this::setSearchQueryMetricsEnabled);
         this.tracer = tracer;
         this.taskResourceTrackingService = taskResourceTrackingService;
     }
 
-    private void setSearchQueryMetricsEnabled(boolean searchQueryMetricsEnabled) {
-        this.searchQueryMetricsEnabled = searchQueryMetricsEnabled;
-        if ((this.searchQueryMetricsEnabled == true) && this.searchQueryCategorizer == null) {
-            this.searchQueryCategorizer = new SearchQueryCategorizer(metricsRegistry);
-        }
-    }
-
     private Map<String, AliasFilter> buildPerIndexAliasFilter(
         SearchRequest request,
         ClusterState clusterState,
@@ -473,13 +453,6 @@ private void executeRequest(
             }
 
             ActionListener<SearchRequest> requestTransformListener = ActionListener.wrap(sr -> {
-                if (searchQueryMetricsEnabled) {
-                    try {
-                        searchQueryCategorizer.categorize(sr.source());
-                    } catch (Exception e) {
-                        logger.error("Error while trying to categorize the query.", e);
-                    }
-                }
 
                 ActionListener<SearchSourceBuilder> rewriteListener = buildRewriteListener(
                     sr,
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 5dcf23ae52294..0648fad619dc7 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -404,7 +404,6 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SearchService.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS,
                 TransportSearchAction.SHARD_COUNT_LIMIT_SETTING,
                 TransportSearchAction.SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING,
-                TransportSearchAction.SEARCH_QUERY_METRICS_ENABLED_SETTING,
                 TransportSearchAction.SEARCH_PHASE_TOOK_ENABLED,
                 SearchRequestStats.SEARCH_REQUEST_STATS_ENABLED,
                 RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE,
diff --git a/server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java b/server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java
deleted file mode 100644
index 3ba13bc7a2da4..0000000000000
--- a/server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.index.query;
-
-import org.apache.lucene.search.BooleanClause;
-import org.opensearch.common.SetOnce;
-
-import java.util.ArrayList;
-import java.util.EnumMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * Class to traverse the QueryBuilder tree and capture the query shape
- */
-public final class QueryShapeVisitor implements QueryBuilderVisitor {
-    private final SetOnce<String> queryType = new SetOnce<>();
-    private final Map<BooleanClause.Occur, List<QueryShapeVisitor>> childVisitors = new EnumMap<>(BooleanClause.Occur.class);
-
-    @Override
-    public void accept(QueryBuilder qb) {
-        queryType.set(qb.getName());
-    }
-
-    @Override
-    public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) {
-        // Should get called once per Occur value
-        if (childVisitors.containsKey(occur)) {
-            throw new IllegalStateException("child visitor already called for " + occur);
-        }
-        final List<QueryShapeVisitor> childVisitorList = new ArrayList<>();
-        QueryBuilderVisitor childVisitorWrapper = new QueryBuilderVisitor() {
-            QueryShapeVisitor currentChild;
-
-            @Override
-            public void accept(QueryBuilder qb) {
-                currentChild = new QueryShapeVisitor();
-                childVisitorList.add(currentChild);
-                currentChild.accept(qb);
-            }
-
-            @Override
-            public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) {
-                return currentChild.getChildVisitor(occur);
-            }
-        };
-        childVisitors.put(occur, childVisitorList);
-        return childVisitorWrapper;
-    }
-
-    String toJson() {
-        StringBuilder outputBuilder = new StringBuilder("{\"type\":\"").append(queryType.get()).append("\"");
-        for (Map.Entry<BooleanClause.Occur, List<QueryShapeVisitor>> entry : childVisitors.entrySet()) {
-            outputBuilder.append(",\"").append(entry.getKey().name().toLowerCase(Locale.ROOT)).append("\"[");
-            boolean first = true;
-            for (QueryShapeVisitor child : entry.getValue()) {
-                if (!first) {
-                    outputBuilder.append(",");
-                }
-                outputBuilder.append(child.toJson());
-                first = false;
-            }
-            outputBuilder.append("]");
-        }
-        outputBuilder.append("}");
-        return outputBuilder.toString();
-    }
-
-    public String prettyPrintTree(String indent) {
-        StringBuilder outputBuilder = new StringBuilder(indent).append(queryType.get()).append("\n");
-        for (Map.Entry<BooleanClause.Occur, List<QueryShapeVisitor>> entry : childVisitors.entrySet()) {
-            outputBuilder.append(indent).append("  ").append(entry.getKey().name().toLowerCase(Locale.ROOT)).append(":\n");
-            for (QueryShapeVisitor child : entry.getValue()) {
-                outputBuilder.append(child.prettyPrintTree(indent + "    "));
-            }
-        }
-        return outputBuilder.toString();
-    }
-}
diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java
deleted file mode 100644
index 4878a463729f9..0000000000000
--- a/server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.action.search;
-
-import org.opensearch.index.query.BoolQueryBuilder;
-import org.opensearch.index.query.BoostingQueryBuilder;
-import org.opensearch.index.query.MatchNoneQueryBuilder;
-import org.opensearch.index.query.MatchQueryBuilder;
-import org.opensearch.index.query.MultiMatchQueryBuilder;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.index.query.QueryStringQueryBuilder;
-import org.opensearch.index.query.RangeQueryBuilder;
-import org.opensearch.index.query.RegexpQueryBuilder;
-import org.opensearch.index.query.TermQueryBuilder;
-import org.opensearch.index.query.WildcardQueryBuilder;
-import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder;
-import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder;
-import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig;
-import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.search.sort.ScoreSortBuilder;
-import org.opensearch.search.sort.SortOrder;
-import org.opensearch.telemetry.metrics.Counter;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
-import org.opensearch.telemetry.metrics.tags.Tags;
-import org.opensearch.test.OpenSearchTestCase;
-import org.junit.Before;
-
-import java.util.Arrays;
-
-import org.mockito.ArgumentCaptor;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public final class SearchQueryCategorizerTests extends OpenSearchTestCase {
-
-    private static final String MULTI_TERMS_AGGREGATION = "multi_terms";
-
-    private MetricsRegistry metricsRegistry;
-
-    private SearchQueryCategorizer searchQueryCategorizer;
-
-    @Before
-    public void setup() {
-        metricsRegistry = mock(MetricsRegistry.class);
-        when(metricsRegistry.createCounter(any(String.class), any(String.class), any(String.class))).thenAnswer(
-            invocation -> mock(Counter.class)
-        );
-        searchQueryCategorizer = new SearchQueryCategorizer(metricsRegistry);
-    }
-
-    public void testAggregationsQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.aggregation(
-            new MultiTermsAggregationBuilder("agg1").terms(
-                Arrays.asList(
-                    new MultiTermsValuesSourceConfig.Builder().setFieldName("username").build(),
-                    new MultiTermsValuesSourceConfig.Builder().setFieldName("rating").build()
-                )
-            )
-        );
-        sourceBuilder.size(0);
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.aggCounter).add(eq(1.0d), any(Tags.class));
-
-        // capture the arguments passed to the aggCounter.add method
-        ArgumentCaptor<Double> valueCaptor = ArgumentCaptor.forClass(Double.class);
-        ArgumentCaptor<Tags> tagsCaptor = ArgumentCaptor.forClass(Tags.class);
-
-        // Verify that aggCounter.add was called with the expected arguments
-        verify(searchQueryCategorizer.searchQueryCounters.aggCounter).add(valueCaptor.capture(), tagsCaptor.capture());
-
-        double actualValue = valueCaptor.getValue();
-        String actualTag = (String) tagsCaptor.getValue().getTagsMap().get("type");
-
-        assertEquals(1.0d, actualValue, 0.0001);
-        assertEquals(MULTI_TERMS_AGGREGATION, actualTag);
-    }
-
-    public void testBoolQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        sourceBuilder.query(new BoolQueryBuilder().must(new MatchQueryBuilder("searchText", "fox")));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("bool")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("match")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testFunctionScoreQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        sourceBuilder.query(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro")));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("function_score")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testMatchQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        sourceBuilder.query(QueryBuilders.matchQuery("tags", "php"));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("match")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testMatchPhraseQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        sourceBuilder.query(QueryBuilders.matchPhraseQuery("tags", "php"));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("match_phrase")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testMultiMatchQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        sourceBuilder.query(new MultiMatchQueryBuilder("foo bar", "myField"));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("multi_match")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testOtherQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        BoostingQueryBuilder queryBuilder = new BoostingQueryBuilder(
-            new TermQueryBuilder("unmapped_field", "foo"),
-            new MatchNoneQueryBuilder()
-        );
-        sourceBuilder.query(queryBuilder);
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("boosting")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("match_none")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("term")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testQueryStringQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder("foo:*");
-        sourceBuilder.query(queryBuilder);
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("query_string")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testRangeQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        RangeQueryBuilder rangeQuery = new RangeQueryBuilder("date");
-        rangeQuery.gte("1970-01-01");
-        rangeQuery.lt("1982-01-01");
-        sourceBuilder.query(rangeQuery);
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("range")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testRegexQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.query(new RegexpQueryBuilder("field", "text"));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("regexp")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testSortQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.query(QueryBuilders.matchQuery("tags", "ruby"));
-        sourceBuilder.sort("creationDate", SortOrder.DESC);
-        sourceBuilder.sort(new ScoreSortBuilder());
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("match")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.sortCounter, times(2)).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testTermQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        sourceBuilder.query(QueryBuilders.termQuery("field", "value2"));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("term")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testWildcardQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-        sourceBuilder.query(new WildcardQueryBuilder("field", "text"));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("wildcard")).add(eq(1.0d), any(Tags.class));
-    }
-
-    public void testComplexQuery() {
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.size(50);
-
-        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("field", "value2");
-        MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("tags", "php");
-        RegexpQueryBuilder regexpQueryBuilder = new RegexpQueryBuilder("field", "text");
-        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().must(termQueryBuilder)
-            .filter(matchQueryBuilder)
-            .should(regexpQueryBuilder);
-        sourceBuilder.query(boolQueryBuilder);
-        sourceBuilder.aggregation(new RangeAggregationBuilder("agg1").field("num"));
-
-        searchQueryCategorizer.categorize(sourceBuilder);
-
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("term")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("match")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("regexp")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.nameToQueryTypeCounters.get("bool")).add(eq(1.0d), any(Tags.class));
-        verify(searchQueryCategorizer.searchQueryCounters.aggCounter).add(eq(1.0d), any(Tags.class));
-    }
-}
diff --git a/server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java b/server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java
deleted file mode 100644
index 18b814aec61c2..0000000000000
--- a/server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.index.query;
-
-import org.opensearch.test.OpenSearchTestCase;
-
-import static org.junit.Assert.assertEquals;
-
-public final class QueryShapeVisitorTests extends OpenSearchTestCase {
-    public void testQueryShapeVisitor() {
-        QueryBuilder builder = new BoolQueryBuilder().must(new TermQueryBuilder("foo", "bar"))
-            .filter(new ConstantScoreQueryBuilder(new RangeQueryBuilder("timestamp").from("12345677").to("2345678")))
-            .should(
-                new BoolQueryBuilder().must(new MatchQueryBuilder("text", "this is some text"))
-                    .mustNot(new RegexpQueryBuilder("color", "red.*"))
-            )
-            .must(new TermsQueryBuilder("genre", "action", "drama", "romance"));
-        QueryShapeVisitor shapeVisitor = new QueryShapeVisitor();
-        builder.visit(shapeVisitor);
-        assertEquals(
-            "{\"type\":\"bool\",\"must\"[{\"type\":\"term\"},{\"type\":\"terms\"}],\"filter\"[{\"type\":\"constant_score\",\"filter\"[{\"type\":\"range\"}]}],\"should\"[{\"type\":\"bool\",\"must\"[{\"type\":\"match\"}],\"must_not\"[{\"type\":\"regexp\"}]}]}",
-            shapeVisitor.toJson()
-        );
-    }
-}

From d33d24e9ff48bd20c12636349ec8c0eb67b38eb2 Mon Sep 17 00:00:00 2001
From: Kaushal Kumar <ravi.kaushal97@gmail.com>
Date: Mon, 15 Jul 2024 22:05:57 -0700
Subject: [PATCH 104/113] Add changes to propagate queryGroupId across child
 requests and nodes (#14614)

* add query group header propagator

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* apply spotless check

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add new propagator in ThreadContext

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* spotlessApply

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* address comments

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Bump com.microsoft.azure:msal4j from 1.15.1 to 1.16.0 in /plugins/repository-azure (#14610)

* Bump com.microsoft.azure:msal4j in /plugins/repository-azure

Bumps [com.microsoft.azure:msal4j](https://github.com/AzureAD/microsoft-authentication-library-for-java) from 1.15.1 to 1.16.0.
- [Release notes](https://github.com/AzureAD/microsoft-authentication-library-for-java/releases)
- [Changelog](https://github.com/AzureAD/microsoft-authentication-library-for-java/blob/dev/changelog.txt)
- [Commits](https://github.com/AzureAD/microsoft-authentication-library-for-java/compare/v1.15.1...v1.16.0)

---
updated-dependencies:
- dependency-name: com.microsoft.azure:msal4j
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* [Bugfix] Fix ICacheKeySerializerTests flakiness (#14564)

* Fix testInvalidInput flakiness

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

* Addressed andrross's comment

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

* rerun security check

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

---------

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
Co-authored-by: Peter Alfonsi <petealft@amazon.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Correct typo in method name (#14621)

Signed-off-by: vatsal <vatsal.v.anand@gmail.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Refactoring FilterPath.parse by using an iterative approach instead of recursion. (#14200)

* Refactor FilterPath parse function (#12067)
Signed-off-by: Robin Friedmann <robinfriedmann.rf@gmail.com>

* Implement unit tests for FilterPathTests (#12067)
Signed-off-by: Robin Friedmann <robinfriedmann.rf@gmail.com>

* Write warn log if Filter is empty; Add comments (#12067)
Signed-off-by: Robin Friedmann <robinfriedmann.rf@gmail.com>

* Add changelog

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Remove unnecessary log statement

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Remove unused logger

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Spotless apply

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

* Remove incorrect changelog

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

---------

Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>
Co-authored-by: Robin Friedmann <robinfriedmann.rf@gmail.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Removing String format in RemoteStoreMigrationAllocationDecider to optimise performance(#14612)

Signed-off-by: RS146BIJAY <rishavsagar4b1@gmail.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Clear templates before Adding; Use NamedWriteableAwareStreamInput for RemoteCustomMetadata; Correct the check for deciding upload of HashesOfConsistentSettings (#14513)

* Clear templates before Adding; Use NamedWriteableAwareStreamInput for RemoteCustomMetadata
* Correct the check for deciding upload of hashes of consistent settings

Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add changelog

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* add PR link changelog

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Improve reroute performance by optimising List.removeAll in LocalShardsBalancer to filter remote search shard from relocation decision (#14613)

Signed-off-by: RS146BIJAY <rishavsagar4b1@gmail.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Fix assertion failure while deleting remote backed index (#14601)

Signed-off-by: Sachin Kale <kalsac@amazon.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Allow system index warning in OpenSearchRestTestCase.refreshAllIndices (#14635)

* Allow system index warning

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add to CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Address code review comments

Signed-off-by: Craig Perkins <cwperx@amazon.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Star tree codec changes (#14514)

---------
Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Bump com.github.spullara.mustache.java:compiler from 0.9.13 to 0.9.14 in /modules/lang-mustache (#14672)

* Bump com.github.spullara.mustache.java:compiler

Bumps [com.github.spullara.mustache.java:compiler](https://github.com/spullara/mustache.java) from 0.9.13 to 0.9.14.
- [Commits](https://github.com/spullara/mustache.java/compare/mustache.java-0.9.13...mustache.java-0.9.14)

---
updated-dependencies:
- dependency-name: com.github.spullara.mustache.java:compiler
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* Bump net.minidev:accessors-smart from 2.5.0 to 2.5.1 in /plugins/repository-azure (#14673)

* Bump net.minidev:accessors-smart in /plugins/repository-azure

Bumps [net.minidev:accessors-smart](https://github.com/netplex/json-smart-v2) from 2.5.0 to 2.5.1.
- [Release notes](https://github.com/netplex/json-smart-v2/releases)
- [Commits](https://github.com/netplex/json-smart-v2/compare/2.5.0...2.5.1)

---
updated-dependencies:
- dependency-name: net.minidev:accessors-smart
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

* move query group thread context propagator out of ThreadContext

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>

---------

Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Peter Alfonsi <petealft@amazon.com>
Signed-off-by: vatsal <vatsal.v.anand@gmail.com>
Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>
Signed-off-by: RS146BIJAY <rishavsagar4b1@gmail.com>
Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
Signed-off-by: Sachin Kale <kalsac@amazon.com>
Signed-off-by: Craig Perkins <cwperx@amazon.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
Co-authored-by: Peter Alfonsi <peter.alfonsi@gmail.com>
Co-authored-by: Peter Alfonsi <petealft@amazon.com>
Co-authored-by: Vatsal <36672090+imvtsl@users.noreply.github.com>
Co-authored-by: Siddhant Deshmukh <deshsid@amazon.com>
Co-authored-by: Robin Friedmann <robinfriedmann.rf@gmail.com>
Co-authored-by: rishavz_sagar <rishavsagar4b1@gmail.com>
Co-authored-by: Sooraj Sinha <81695996+soosinha@users.noreply.github.com>
Co-authored-by: Sachin Kale <sachinpkale@gmail.com>
Co-authored-by: Craig Perkins <cwperx@amazon.com>
Co-authored-by: Bharathwaj G <bharath78910@gmail.com>
---
 CHANGELOG.md                                  |  1 +
 ...ueryGroupThreadContextStatePropagator.java | 53 +++++++++++++++++++
 .../java/org/opensearch/wlm/package-info.java | 13 +++++
 ...roupThreadContextStatePropagatorTests.java | 30 +++++++++++
 4 files changed, 97 insertions(+)
 create mode 100644 server/src/main/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagator.java
 create mode 100644 server/src/main/java/org/opensearch/wlm/package-info.java
 create mode 100644 server/src/test/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagatorTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6bfd98ceaea80..6f666dbf3b8d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
 - Add `strict_allow_templates` dynamic mapping option ([#14555](https://github.com/opensearch-project/OpenSearch/pull/14555))
 - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
+- [Workload Management] add queryGroupId header propagator across requests and nodes ([#14614](https://github.com/opensearch-project/OpenSearch/pull/14614))
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
 - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
 
diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagator.java b/server/src/main/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagator.java
new file mode 100644
index 0000000000000..06d223907082e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagator.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.wlm;
+
+import org.opensearch.common.util.concurrent.ThreadContextStatePropagator;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class is used to propagate QueryGroup related headers to request and nodes
+ */
+public class QueryGroupThreadContextStatePropagator implements ThreadContextStatePropagator {
+    // TODO: move this constant to QueryGroupService class once the QueryGroup monitoring framework PR is ready
+    public static List<String> PROPAGATED_HEADERS = List.of("queryGroupId");
+
+    /**
+     * @param source current context transient headers
+     * @return the map of header and their values to be propagated across request threadContexts
+     */
+    @Override
+    @SuppressWarnings("removal")
+    public Map<String, Object> transients(Map<String, Object> source) {
+        final Map<String, Object> transientHeaders = new HashMap<>();
+
+        for (String headerName : PROPAGATED_HEADERS) {
+            transientHeaders.compute(headerName, (k, v) -> source.get(headerName));
+        }
+        return transientHeaders;
+    }
+
+    /**
+     * @param source current context headers
+     * @return map of header and their values to be propagated across nodes
+     */
+    @Override
+    @SuppressWarnings("removal")
+    public Map<String, String> headers(Map<String, Object> source) {
+        final Map<String, String> propagatedHeaders = new HashMap<>();
+
+        for (String headerName : PROPAGATED_HEADERS) {
+            propagatedHeaders.compute(headerName, (k, v) -> (String) source.get(headerName));
+        }
+        return propagatedHeaders;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/wlm/package-info.java b/server/src/main/java/org/opensearch/wlm/package-info.java
new file mode 100644
index 0000000000000..fa4731d95cc34
--- /dev/null
+++ b/server/src/main/java/org/opensearch/wlm/package-info.java
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * This package contains workload management constructs
+ */
+
+package org.opensearch.wlm;
diff --git a/server/src/test/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagatorTests.java b/server/src/test/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagatorTests.java
new file mode 100644
index 0000000000000..ad5d7f569a56e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/wlm/QueryGroupThreadContextStatePropagatorTests.java
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.wlm;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.Map;
+
+public class QueryGroupThreadContextStatePropagatorTests extends OpenSearchTestCase {
+
+    public void testTransients() {
+        QueryGroupThreadContextStatePropagator sut = new QueryGroupThreadContextStatePropagator();
+        Map<String, Object> source = Map.of("queryGroupId", "adgarja0r235te");
+        Map<String, Object> transients = sut.transients(source);
+        assertEquals("adgarja0r235te", transients.get("queryGroupId"));
+    }
+
+    public void testHeaders() {
+        QueryGroupThreadContextStatePropagator sut = new QueryGroupThreadContextStatePropagator();
+        Map<String, Object> source = Map.of("queryGroupId", "adgarja0r235te");
+        Map<String, String> headers = sut.headers(source);
+        assertEquals("adgarja0r235te", headers.get("queryGroupId"));
+    }
+}

From ba9bdacc05928c82a4244b06c5c0e8283406dc7d Mon Sep 17 00:00:00 2001
From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com>
Date: Tue, 16 Jul 2024 13:15:05 +0530
Subject: [PATCH 105/113] Add consumers to remote store based index settings
 (#14764)

Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com>
---
 .../MigrationBaseTestCase.java                | 16 +++++++++++
 .../RemoteStoreMigrationTestCase.java         |  9 +++++++
 .../org/opensearch/index/IndexSettings.java   | 27 ++++++++++++++++---
 .../blobstore/BlobStoreRepository.java        |  2 +-
 4 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
index 5be9b25512704..2bea36ed80c9f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
@@ -13,6 +13,8 @@
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
+import org.opensearch.action.admin.indices.get.GetIndexRequest;
+import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.bulk.BulkRequest;
 import org.opensearch.action.bulk.BulkResponse;
 import org.opensearch.action.delete.DeleteResponse;
@@ -21,12 +23,17 @@
 import org.opensearch.client.Requests;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.routing.RoutingNode;
 import org.opensearch.common.Priority;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.index.Index;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.indices.IndicesService;
 import org.opensearch.repositories.fs.ReloadableFsRepository;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Before;
@@ -261,4 +268,13 @@ public ClusterHealthStatus waitForRelocation(TimeValue t) {
         }
         return actionGet.getStatus();
     }
+
+    protected IndexShard getIndexShard(String dataNode, String indexName) throws ExecutionException, InterruptedException {
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, dataNode);
+        GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
+        String uuid = getIndexResponse.getSettings().get(indexName).get(IndexMetadata.SETTING_INDEX_UUID);
+        IndexService indexService = indicesService.indexService(new Index(indexName, uuid));
+        return indexService.getShard(0);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java
index e0e25db4ca722..4d37b2a1feb88 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java
@@ -17,6 +17,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.shard.IndexShard;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.snapshots.SnapshotInfo;
 import org.opensearch.test.OpenSearchIntegTestCase;
@@ -216,4 +217,12 @@ public void testEndToEndRemoteMigration() throws Exception {
             asyncIndexingService.getIndexedDocs()
         );
     }
+
+    public void testRemoteSettingPropagatedToIndexShardAfterMigration() throws Exception {
+        testEndToEndRemoteMigration();
+        IndexShard indexShard = getIndexShard(primaryNodeName("test"), "test");
+        assertTrue(indexShard.indexSettings().isRemoteStoreEnabled());
+        assertEquals(MigrationBaseTestCase.REPOSITORY_NAME, indexShard.indexSettings().getRemoteStoreRepository());
+        assertEquals(MigrationBaseTestCase.REPOSITORY_2_NAME, indexShard.indexSettings().getRemoteStoreTranslogRepository());
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 96458ecc49ddc..a833d66fab5d9 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -732,11 +732,11 @@ public static IndexMergePolicy fromString(String text) {
     private final Settings nodeSettings;
     private final int numberOfShards;
     private final ReplicationType replicationType;
-    private final boolean isRemoteStoreEnabled;
+    private volatile boolean isRemoteStoreEnabled;
     private final boolean isStoreLocalityPartial;
     private volatile TimeValue remoteTranslogUploadBufferInterval;
-    private final String remoteStoreTranslogRepository;
-    private final String remoteStoreRepository;
+    private volatile String remoteStoreTranslogRepository;
+    private volatile String remoteStoreRepository;
     private int remoteTranslogKeepExtraGen;
     private Version extendedCompatibilitySnapshotVersion;
 
@@ -1132,6 +1132,15 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
             this::setDocIdFuzzySetFalsePositiveProbability
         );
         scopedSettings.addSettingsUpdateConsumer(ALLOW_DERIVED_FIELDS, this::setAllowDerivedField);
+        scopedSettings.addSettingsUpdateConsumer(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, this::setRemoteStoreEnabled);
+        scopedSettings.addSettingsUpdateConsumer(
+            IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING,
+            this::setRemoteStoreRepository
+        );
+        scopedSettings.addSettingsUpdateConsumer(
+            IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING,
+            this::setRemoteStoreTranslogRepository
+        );
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1950,4 +1959,16 @@ public RemoteStorePathStrategy getRemoteStorePathStrategy() {
     public boolean isTranslogMetadataEnabled() {
         return isTranslogMetadataEnabled;
     }
+
+    public void setRemoteStoreEnabled(boolean isRemoteStoreEnabled) {
+        this.isRemoteStoreEnabled = isRemoteStoreEnabled;
+    }
+
+    public void setRemoteStoreRepository(String remoteStoreRepository) {
+        this.remoteStoreRepository = remoteStoreRepository;
+    }
+
+    public void setRemoteStoreTranslogRepository(String remoteStoreTranslogRepository) {
+        this.remoteStoreTranslogRepository = remoteStoreTranslogRepository;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index 53c44f743c781..02290b6a5e566 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -2678,7 +2678,7 @@ public void snapshotRemoteStoreIndexShard(
         final ShardId shardId = store.shardId();
         try {
             final String generation = snapshotStatus.generation();
-            logger.info("[{}] [{}] snapshot to [{}] [{}] ...", shardId, snapshotId, metadata.name(), generation);
+            logger.info("[{}] [{}] shallow copy snapshot to [{}] [{}] ...", shardId, snapshotId, metadata.name(), generation);
             final BlobContainer shardContainer = shardContainer(indexId, shardId);
 
             long indexTotalFileSize = 0;

From 6ad57bf393bda06da1c463d7c276d4a61d3cb299 Mon Sep 17 00:00:00 2001
From: Craig Perkins <craig5008@gmail.com>
Date: Tue, 16 Jul 2024 08:16:11 -0400
Subject: [PATCH 106/113] Add matchesPluginSystemIndexPattern to
 SystemIndexRegistry (#14750)

* Add matchesPluginSystemIndexPattern to SystemIndexRegistry

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add to CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Use single data structure to keep track of system indices

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Address code review comments

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add test for getAllDescriptors

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java

Co-authored-by: Andriy Redko <drreta@gmail.com>
Signed-off-by: Craig Perkins <craig5008@gmail.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
Signed-off-by: Craig Perkins <craig5008@gmail.com>
Co-authored-by: Andriy Redko <drreta@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 .../indices/SystemIndexRegistry.java          |  36 ++++---
 .../org/opensearch/indices/SystemIndices.java |   5 +-
 .../main/java/org/opensearch/node/Node.java   |   5 +-
 .../indices/SystemIndicesTests.java           | 100 ++++++++++++++----
 5 files changed, 112 insertions(+), 35 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f666dbf3b8d5..7050e7d23b24d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Workload Management] add queryGroupId header propagator across requests and nodes ([#14614](https://github.com/opensearch-project/OpenSearch/pull/14614))
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
 - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
+- Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https://github.com/opensearch-project/OpenSearch/pull/14750))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
index d9608e220d924..ab2cbd4ef1a73 100644
--- a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
+++ b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
@@ -15,13 +15,13 @@
 import org.opensearch.common.regex.Regex;
 import org.opensearch.tasks.TaskResultsService;
 
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonList;
@@ -45,25 +45,35 @@ public class SystemIndexRegistry {
     );
 
     private volatile static String[] SYSTEM_INDEX_PATTERNS = new String[0];
-    volatile static Collection<SystemIndexDescriptor> SYSTEM_INDEX_DESCRIPTORS = Collections.emptyList();
+    private volatile static Map<String, Collection<SystemIndexDescriptor>> SYSTEM_INDEX_DESCRIPTORS_MAP = Collections.emptyMap();
 
     static void register(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) {
         final Map<String, Collection<SystemIndexDescriptor>> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
         checkForOverlappingPatterns(descriptorsMap);
-        List<SystemIndexDescriptor> descriptors = pluginAndModulesDescriptors.values()
-            .stream()
-            .flatMap(Collection::stream)
-            .collect(Collectors.toList());
-        descriptors.add(TASK_INDEX_DESCRIPTOR);
 
-        SYSTEM_INDEX_DESCRIPTORS = descriptors.stream().collect(Collectors.toUnmodifiableList());
-        SYSTEM_INDEX_PATTERNS = descriptors.stream().map(SystemIndexDescriptor::getIndexPattern).toArray(String[]::new);
+        SYSTEM_INDEX_DESCRIPTORS_MAP = descriptorsMap;
+        SYSTEM_INDEX_PATTERNS = getAllDescriptors().stream().map(SystemIndexDescriptor::getIndexPattern).toArray(String[]::new);
     }
 
-    public static List<String> matchesSystemIndexPattern(String... indexExpressions) {
-        return Arrays.stream(indexExpressions)
-            .filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern))
-            .collect(Collectors.toList());
+    public static Set<String> matchesSystemIndexPattern(Set<String> indexExpressions) {
+        return indexExpressions.stream().filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern)).collect(Collectors.toSet());
+    }
+
+    public static Set<String> matchesPluginSystemIndexPattern(String pluginClassName, Set<String> indexExpressions) {
+        if (!SYSTEM_INDEX_DESCRIPTORS_MAP.containsKey(pluginClassName)) {
+            return Collections.emptySet();
+        }
+        String[] pluginSystemIndexPatterns = SYSTEM_INDEX_DESCRIPTORS_MAP.get(pluginClassName)
+            .stream()
+            .map(SystemIndexDescriptor::getIndexPattern)
+            .toArray(String[]::new);
+        return indexExpressions.stream()
+            .filter(pattern -> Regex.simpleMatch(pluginSystemIndexPatterns, pattern))
+            .collect(Collectors.toSet());
+    }
+
+    static List<SystemIndexDescriptor> getAllDescriptors() {
+        return SYSTEM_INDEX_DESCRIPTORS_MAP.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/indices/SystemIndices.java b/server/src/main/java/org/opensearch/indices/SystemIndices.java
index bbf58fe91512f..6e9e5e7707877 100644
--- a/server/src/main/java/org/opensearch/indices/SystemIndices.java
+++ b/server/src/main/java/org/opensearch/indices/SystemIndices.java
@@ -63,7 +63,7 @@ public class SystemIndices {
 
     public SystemIndices(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) {
         SystemIndexRegistry.register(pluginAndModulesDescriptors);
-        this.runAutomaton = buildCharacterRunAutomaton(SystemIndexRegistry.SYSTEM_INDEX_DESCRIPTORS);
+        this.runAutomaton = buildCharacterRunAutomaton(SystemIndexRegistry.getAllDescriptors());
     }
 
     /**
@@ -91,7 +91,8 @@ public boolean isSystemIndex(String indexName) {
      * @throws IllegalStateException if multiple descriptors match the name
      */
     public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) {
-        final List<SystemIndexDescriptor> matchingDescriptors = SystemIndexRegistry.SYSTEM_INDEX_DESCRIPTORS.stream()
+        final List<SystemIndexDescriptor> matchingDescriptors = SystemIndexRegistry.getAllDescriptors()
+            .stream()
             .filter(descriptor -> descriptor.matchesIndexPattern(name))
             .collect(Collectors.toList());
 
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 96a716af7f1a1..281f697d9bb79 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -699,7 +699,10 @@ protected Node(
                 pluginsService.filterPlugins(SystemIndexPlugin.class)
                     .stream()
                     .collect(
-                        Collectors.toMap(plugin -> plugin.getClass().getSimpleName(), plugin -> plugin.getSystemIndexDescriptors(settings))
+                        Collectors.toMap(
+                            plugin -> plugin.getClass().getCanonicalName(),
+                            plugin -> plugin.getSystemIndexDescriptors(settings)
+                        )
                     )
             );
             final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap);
diff --git a/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java b/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java
index 8ac457c32d53a..ca9370645dec3 100644
--- a/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java
+++ b/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java
@@ -44,6 +44,8 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
@@ -155,32 +157,61 @@ public void testSystemIndexMatching() {
         );
 
         assertThat(
-            SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index2"),
-            equalTo(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2))
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1", ".system-index2")),
+            equalTo(Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2))
         );
-        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index1"), equalTo(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1)));
-        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index2"), equalTo(List.of(SystemIndexPlugin2.SYSTEM_INDEX_2)));
-        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern1"), equalTo(List.of(".system-index-pattern1")));
         assertThat(
-            SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern-sub*"),
-            equalTo(List.of(".system-index-pattern-sub*"))
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1")),
+            equalTo(Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1))
         );
         assertThat(
-            SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern1", ".system-index-pattern2"),
-            equalTo(List.of(".system-index-pattern1", ".system-index-pattern2"))
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index2")),
+            equalTo(Set.of(SystemIndexPlugin2.SYSTEM_INDEX_2))
         );
         assertThat(
-            SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index-pattern1"),
-            equalTo(List.of(".system-index1", ".system-index-pattern1"))
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index-pattern1")),
+            equalTo(Set.of(".system-index-pattern1"))
         );
         assertThat(
-            SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index-pattern1", ".not-system"),
-            equalTo(List.of(".system-index1", ".system-index-pattern1"))
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index-pattern-sub*")),
+            equalTo(Set.of(".system-index-pattern-sub*"))
+        );
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index-pattern1", ".system-index-pattern2")),
+            equalTo(Set.of(".system-index-pattern1", ".system-index-pattern2"))
+        );
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1", ".system-index-pattern1")),
+            equalTo(Set.of(".system-index1", ".system-index-pattern1"))
+        );
+        assertThat(
+            SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1", ".system-index-pattern1", ".not-system")),
+            equalTo(Set.of(".system-index1", ".system-index-pattern1"))
+        );
+        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".not-system")), equalTo(Collections.emptySet()));
+    }
+
+    public void testRegisteredSystemIndexGetAllDescriptors() {
+        SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
+        SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
+        SystemIndices pluginSystemIndices = new SystemIndices(
+            Map.of(
+                SystemIndexPlugin1.class.getCanonicalName(),
+                plugin1.getSystemIndexDescriptors(Settings.EMPTY),
+                SystemIndexPlugin2.class.getCanonicalName(),
+                plugin2.getSystemIndexDescriptors(Settings.EMPTY)
+            )
+        );
+        assertEquals(
+            SystemIndexRegistry.getAllDescriptors()
+                .stream()
+                .map(SystemIndexDescriptor::getIndexPattern)
+                .collect(Collectors.toUnmodifiableList()),
+            List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2, TASK_INDEX + "*")
         );
-        assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".not-system"), equalTo(Collections.emptyList()));
     }
 
-    public void testRegisteredSystemIndexExpansion() {
+    public void testRegisteredSystemIndexMatching() {
         SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
         SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
         SystemIndices pluginSystemIndices = new SystemIndices(
@@ -191,12 +222,43 @@ public void testRegisteredSystemIndexExpansion() {
                 plugin2.getSystemIndexDescriptors(Settings.EMPTY)
             )
         );
-        List<String> systemIndices = SystemIndexRegistry.matchesSystemIndexPattern(
-            SystemIndexPlugin1.SYSTEM_INDEX_1,
-            SystemIndexPlugin2.SYSTEM_INDEX_2
+        Set<String> systemIndices = SystemIndexRegistry.matchesSystemIndexPattern(
+            Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2)
         );
         assertEquals(2, systemIndices.size());
-        assertTrue(systemIndices.containsAll(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2)));
+        assertTrue(systemIndices.containsAll(Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2)));
+    }
+
+    public void testRegisteredSystemIndexMatchingForPlugin() {
+        SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
+        SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
+        SystemIndices pluginSystemIndices = new SystemIndices(
+            Map.of(
+                SystemIndexPlugin1.class.getCanonicalName(),
+                plugin1.getSystemIndexDescriptors(Settings.EMPTY),
+                SystemIndexPlugin2.class.getCanonicalName(),
+                plugin2.getSystemIndexDescriptors(Settings.EMPTY)
+            )
+        );
+        Set<String> systemIndicesForPlugin1 = SystemIndexRegistry.matchesPluginSystemIndexPattern(
+            SystemIndexPlugin1.class.getCanonicalName(),
+            Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2, "other-index")
+        );
+        assertEquals(1, systemIndicesForPlugin1.size());
+        assertTrue(systemIndicesForPlugin1.contains(SystemIndexPlugin1.SYSTEM_INDEX_1));
+
+        Set<String> systemIndicesForPlugin2 = SystemIndexRegistry.matchesPluginSystemIndexPattern(
+            SystemIndexPlugin2.class.getCanonicalName(),
+            Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2, "other-index")
+        );
+        assertEquals(1, systemIndicesForPlugin2.size());
+        assertTrue(systemIndicesForPlugin2.contains(SystemIndexPlugin2.SYSTEM_INDEX_2));
+
+        Set<String> noMatchingSystemIndices = SystemIndexRegistry.matchesPluginSystemIndexPattern(
+            SystemIndexPlugin2.class.getCanonicalName(),
+            Set.of("other-index")
+        );
+        assertEquals(0, noMatchingSystemIndices.size());
     }
 
     static final class SystemIndexPlugin1 extends Plugin implements SystemIndexPlugin {

From 54af34ec80ab9eb3c4831c9e0c4b1bcd5983c3a3 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <81609427+mgodwan@users.noreply.github.com>
Date: Tue, 16 Jul 2024 22:17:27 +0530
Subject: [PATCH 107/113] SPI for loading ABC templates (#14659)

* SPI for loading ABC templates

Signed-off-by: mgodwan <mgodwan@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../ClusterStateSystemTemplateLoader.java     | 108 +++++++++++
 .../applicationtemplates/SystemTemplate.java  |  43 ++++
 .../SystemTemplateLoader.java                 |  26 +++
 .../SystemTemplateMetadata.java               |  68 +++++++
 .../SystemTemplateRepository.java             |  37 ++++
 .../SystemTemplatesPlugin.java                |  31 +++
 .../SystemTemplatesService.java               | 183 ++++++++++++++++++
 .../TemplateRepositoryMetadata.java           |  34 ++++
 .../applicationtemplates/package-info.java    |  10 +
 .../common/settings/ClusterSettings.java      |   5 +-
 .../common/settings/FeatureFlagSettings.java  |   3 +-
 .../opensearch/common/util/FeatureFlags.java  |  14 +-
 .../main/java/org/opensearch/node/Node.java   |  13 +-
 ...ClusterStateSystemTemplateLoaderTests.java | 148 ++++++++++++++
 .../SystemTemplatesServiceTests.java          |  90 +++++++++
 .../TestSystemTemplatesRepositoryPlugin.java  |  72 +++++++
 .../test/OpenSearchIntegTestCase.java         |   3 +
 .../test/OpenSearchSingleNodeTestCase.java    |   1 +
 19 files changed, 886 insertions(+), 4 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoader.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplate.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateLoader.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateRepository.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesPlugin.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesService.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/TemplateRepositoryMetadata.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/applicationtemplates/package-info.java
 create mode 100644 server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java
 create mode 100644 server/src/test/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesServiceTests.java
 create mode 100644 test/framework/src/main/java/org/opensearch/cluster/service/applicationtemplates/TestSystemTemplatesRepositoryPlugin.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7050e7d23b24d..f885261f404ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
 - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
 - Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https://github.com/opensearch-project/OpenSearch/pull/14750))
+- Add Plugin interface for loading application based configuration templates (([#14659](https://github.com/opensearch-project/OpenSearch/issues/14659)))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoader.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoader.java
new file mode 100644
index 0000000000000..332960ef49064
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoader.java
@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchCorruptionException;
+import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction;
+import org.opensearch.client.Client;
+import org.opensearch.client.OriginSettingClient;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.ComponentTemplate;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+/**
+ * Class responsible for loading the component templates provided by a repository into the cluster state.
+ */
+@ExperimentalApi
+public class ClusterStateSystemTemplateLoader implements SystemTemplateLoader {
+
+    private final Client client;
+
+    private final Supplier<ClusterState> clusterStateSupplier;
+
+    private static final Logger logger = LogManager.getLogger(SystemTemplateLoader.class);
+
+    public static final String TEMPLATE_LOADER_IDENTIFIER = "system_template_loader";
+    public static final String TEMPLATE_TYPE_KEY = "_type";
+
+    public ClusterStateSystemTemplateLoader(Client client, Supplier<ClusterState> clusterStateSupplier) {
+        this.client = new OriginSettingClient(client, TEMPLATE_LOADER_IDENTIFIER);
+        this.clusterStateSupplier = clusterStateSupplier;
+    }
+
+    @Override
+    public boolean loadTemplate(SystemTemplate template) throws IOException {
+        final ComponentTemplate existingTemplate = clusterStateSupplier.get()
+            .metadata()
+            .componentTemplates()
+            .get(template.templateMetadata().fullyQualifiedName());
+
+        if (existingTemplate != null
+            && !SystemTemplateMetadata.COMPONENT_TEMPLATE_TYPE.equals(
+                Objects.toString(existingTemplate.metadata().get(TEMPLATE_TYPE_KEY))
+            )) {
+            throw new OpenSearchCorruptionException(
+                "Attempting to create " + template.templateMetadata().name() + " which has already been created through some other source."
+            );
+        }
+
+        if (existingTemplate != null && existingTemplate.version() >= template.templateMetadata().version()) {
+            logger.debug(
+                "Skipping putting template {} as its existing version [{}] is >= fetched version [{}]",
+                template.templateMetadata().fullyQualifiedName(),
+                existingTemplate.version(),
+                template.templateMetadata().version()
+            );
+            return false;
+        }
+
+        ComponentTemplate newTemplate = null;
+        try (
+            XContentParser contentParser = JsonXContent.jsonXContent.createParser(
+                NamedXContentRegistry.EMPTY,
+                DeprecationHandler.IGNORE_DEPRECATIONS,
+                template.templateContent().utf8ToString()
+            )
+        ) {
+            newTemplate = ComponentTemplate.parse(contentParser);
+        }
+
+        if (!Objects.equals(newTemplate.version(), template.templateMetadata().version())) {
+            throw new OpenSearchCorruptionException(
+                "Template version mismatch for "
+                    + template.templateMetadata().name()
+                    + ". Version in metadata: "
+                    + template.templateMetadata().version()
+                    + " , Version in content: "
+                    + newTemplate.version()
+            );
+        }
+
+        final PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(
+            template.templateMetadata().fullyQualifiedName()
+        ).componentTemplate(newTemplate);
+
+        return client.admin()
+            .indices()
+            .execute(PutComponentTemplateAction.INSTANCE, request)
+            .actionGet(TimeValue.timeValueMillis(30000))
+            .isAcknowledged();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplate.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplate.java
new file mode 100644
index 0000000000000..e11ded7ef5546
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplate.java
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.common.bytes.BytesReference;
+
+/**
+ * Encapsulates the information and content about a system template available within a repository.
+ */
+@ExperimentalApi
+public class SystemTemplate {
+
+    private final BytesReference templateContent;
+
+    private final SystemTemplateMetadata templateMetadata;
+
+    private final TemplateRepositoryMetadata repositoryMetadata;
+
+    public SystemTemplate(BytesReference templateContent, SystemTemplateMetadata templateInfo, TemplateRepositoryMetadata repositoryInfo) {
+        this.templateContent = templateContent;
+        this.templateMetadata = templateInfo;
+        this.repositoryMetadata = repositoryInfo;
+    }
+
+    public BytesReference templateContent() {
+        return templateContent;
+    }
+
+    public SystemTemplateMetadata templateMetadata() {
+        return templateMetadata;
+    }
+
+    public TemplateRepositoryMetadata repositoryMetadata() {
+        return repositoryMetadata;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateLoader.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateLoader.java
new file mode 100644
index 0000000000000..077580aed5a64
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateLoader.java
@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Interface to load template into the OpenSearch runtime.
+ */
+@ExperimentalApi
+public interface SystemTemplateLoader {
+
+    /**
+     * @param template Templated to be loaded
+     * @throws IOException If an exceptional situation is encountered while parsing/loading the template
+     */
+    boolean loadTemplate(SystemTemplate template) throws IOException;
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java
new file mode 100644
index 0000000000000..9bbe27ac0e281
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java
@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Metadata information about a template available in a template repository.
+ */
+@ExperimentalApi
+public class SystemTemplateMetadata {
+
+    private final long version;
+    private final String type;
+    private final String name;
+
+    private static final String DELIMITER = "@";
+
+    public static final String COMPONENT_TEMPLATE_TYPE = "@abc_template";
+
+    public SystemTemplateMetadata(long version, String type, String name) {
+        this.version = version;
+        this.type = type;
+        this.name = name;
+    }
+
+    public String type() {
+        return type;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public long version() {
+        return version;
+    }
+
+    /**
+     * Gets the metadata using fully qualified name for the template
+     * @param fullyQualifiedName (e.g. @abc_template@logs@1)
+     * @return Metadata object based on name
+     */
+    public static SystemTemplateMetadata fromComponentTemplate(String fullyQualifiedName) {
+        assert fullyQualifiedName.length() > 1 : "System template name must have at least one component";
+        assert fullyQualifiedName.substring(1, fullyQualifiedName.indexOf(DELIMITER, 1)).equals(COMPONENT_TEMPLATE_TYPE);
+
+        return new SystemTemplateMetadata(
+            Long.parseLong(fullyQualifiedName.substring(fullyQualifiedName.lastIndexOf(DELIMITER))),
+            COMPONENT_TEMPLATE_TYPE,
+            fullyQualifiedName.substring(0, fullyQualifiedName.lastIndexOf(DELIMITER))
+        );
+    }
+
+    public static SystemTemplateMetadata fromComponentTemplateInfo(String name, long version) {
+        return new SystemTemplateMetadata(version, COMPONENT_TEMPLATE_TYPE, name);
+    }
+
+    public final String fullyQualifiedName() {
+        return type + DELIMITER + name + DELIMITER + version;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateRepository.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateRepository.java
new file mode 100644
index 0000000000000..9cf302b8874f2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateRepository.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Repository interface around the templates provided by a store (e.g. code repo, remote file store, etc)
+ */
+@ExperimentalApi
+public interface SystemTemplateRepository extends AutoCloseable {
+
+    /**
+     * @return Metadata about the repository
+     */
+    TemplateRepositoryMetadata metadata();
+
+    /**
+     * @return Metadata for all available templates
+     */
+    Iterable<SystemTemplateMetadata> listTemplates() throws IOException;
+
+    /**
+     *
+     * @param template metadata about template to be fetched
+     * @return The actual template content
+     */
+    SystemTemplate getTemplate(SystemTemplateMetadata template) throws IOException;
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesPlugin.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesPlugin.java
new file mode 100644
index 0000000000000..54871e6db7010
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesPlugin.java
@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Plugin interface to expose the template maintaining logic.
+ */
+@ExperimentalApi
+public interface SystemTemplatesPlugin {
+
+    /**
+     * @return repository implementation from which templates are to be fetched.
+     */
+    SystemTemplateRepository loadRepository() throws IOException;
+
+    /**
+     * @param templateInfo Metadata about the template to load
+     * @return Implementation of TemplateLoader which determines how to make the template available at runtime.
+     */
+    SystemTemplateLoader loaderFor(SystemTemplateMetadata templateInfo);
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesService.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesService.java
new file mode 100644
index 0000000000000..ccb9272fa57b1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesService.java
@@ -0,0 +1,183 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.cluster.LocalNodeClusterManagerListener;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Service class to orchestrate execution around available templates' management.
+ */
+@ExperimentalApi
+public class SystemTemplatesService implements LocalNodeClusterManagerListener {
+
+    public static final Setting<Boolean> SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED = Setting.boolSetting(
+        "cluster.application_templates.enabled",
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    private final List<SystemTemplatesPlugin> systemTemplatesPluginList;
+    private final ThreadPool threadPool;
+
+    private final AtomicBoolean loaded = new AtomicBoolean(false);
+
+    private volatile boolean enabledTemplates;
+
+    private volatile Stats latestStats;
+
+    private static final Logger logger = LogManager.getLogger(SystemTemplatesService.class);
+
+    public SystemTemplatesService(
+        List<SystemTemplatesPlugin> systemTemplatesPluginList,
+        ThreadPool threadPool,
+        ClusterSettings clusterSettings,
+        Settings settings
+    ) {
+        this.systemTemplatesPluginList = systemTemplatesPluginList;
+        this.threadPool = threadPool;
+        if (settings.getAsBoolean(SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED.getKey(), false)) {
+            setEnabledTemplates(settings.getAsBoolean(SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED.getKey(), false));
+        }
+        clusterSettings.addSettingsUpdateConsumer(SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED, this::setEnabledTemplates);
+    }
+
+    @Override
+    public void onClusterManager() {
+        threadPool.generic().execute(() -> refreshTemplates(false));
+    }
+
+    @Override
+    public void offClusterManager() {
+        // do nothing
+    }
+
+    public void verifyRepositories() {
+        refreshTemplates(true);
+    }
+
+    public Stats stats() {
+        return latestStats;
+    }
+
+    void refreshTemplates(boolean verification) {
+        int templatesLoaded = 0;
+        int failedLoadingTemplates = 0;
+        int failedLoadingRepositories = 0;
+        List<Exception> exceptions = new ArrayList<>();
+
+        if (loaded.compareAndSet(false, true) && enabledTemplates) {
+            for (SystemTemplatesPlugin plugin : systemTemplatesPluginList) {
+                try (SystemTemplateRepository repository = plugin.loadRepository()) {
+
+                    final TemplateRepositoryMetadata repositoryMetadata = repository.metadata();
+                    logger.debug(
+                        "Loading templates from repository: {} at version {}",
+                        repositoryMetadata.id(),
+                        repositoryMetadata.version()
+                    );
+
+                    for (SystemTemplateMetadata templateMetadata : repository.listTemplates()) {
+                        try {
+                            final SystemTemplate template = repository.getTemplate(templateMetadata);
+
+                            // Load plugin if not in verification phase.
+                            if (!verification && plugin.loaderFor(templateMetadata).loadTemplate(template)) {
+                                templatesLoaded++;
+                            }
+
+                        } catch (Exception ex) {
+                            exceptions.add(ex);
+                            logger.error(
+                                new ParameterizedMessage(
+                                    "Failed loading template  {} from repository: {}",
+                                    templateMetadata.fullyQualifiedName(),
+                                    repositoryMetadata.id()
+                                ),
+                                ex
+                            );
+                            failedLoadingTemplates++;
+                        }
+                    }
+                } catch (Exception ex) {
+                    exceptions.add(ex);
+                    failedLoadingRepositories++;
+                    logger.error(new ParameterizedMessage("Failed loading repository from plugin: {}", plugin.getClass().getName()), ex);
+                }
+            }
+
+            logger.debug(
+                "Stats: Total Loaded Templates: [{}], Failed Loading Templates: [{}], Failed Loading Repositories: [{}]",
+                templatesLoaded,
+                failedLoadingTemplates,
+                failedLoadingRepositories
+            );
+
+            // End exceptionally if invoked in verification context
+            if (verification && (failedLoadingRepositories > 0 || failedLoadingTemplates > 0)) {
+                latestStats = new Stats(templatesLoaded, failedLoadingTemplates, failedLoadingRepositories);
+                throw new IllegalStateException("Some of the repositories could not be loaded or are corrupted: " + exceptions);
+            }
+        }
+
+        latestStats = new Stats(templatesLoaded, failedLoadingTemplates, failedLoadingRepositories);
+    }
+
+    private void setEnabledTemplates(boolean enabled) {
+        if (!FeatureFlags.isEnabled(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING)) {
+            throw new IllegalArgumentException(
+                "Application Based Configuration Templates is under an experimental feature and can be activated only by enabling "
+                    + FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING.getKey()
+                    + " feature flag."
+            );
+        }
+        enabledTemplates = enabled;
+    }
+
+    /**
+     * Class to record stats for templates loaded through the listener in a single iteration.
+     */
+    @ExperimentalApi
+    public static class Stats {
+        private final long templatesLoaded;
+        private final long failedLoadingTemplates;
+        private final long failedLoadingRepositories;
+
+        public Stats(long templatesLoaded, long failedLoadingTemplates, long failedLoadingRepositories) {
+            this.templatesLoaded = templatesLoaded;
+            this.failedLoadingTemplates = failedLoadingTemplates;
+            this.failedLoadingRepositories = failedLoadingRepositories;
+        }
+
+        public long getTemplatesLoaded() {
+            return templatesLoaded;
+        }
+
+        public long getFailedLoadingTemplates() {
+            return failedLoadingTemplates;
+        }
+
+        public long getFailedLoadingRepositories() {
+            return failedLoadingRepositories;
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/TemplateRepositoryMetadata.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/TemplateRepositoryMetadata.java
new file mode 100644
index 0000000000000..7ab4553aade0e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/TemplateRepositoryMetadata.java
@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * The information to uniquely identify a template repository.
+ */
+@ExperimentalApi
+public class TemplateRepositoryMetadata {
+
+    private final String id;
+    private final long version;
+
+    public TemplateRepositoryMetadata(String id, long version) {
+        this.id = id;
+        this.version = version;
+    }
+
+    public String id() {
+        return id;
+    }
+
+    public long version() {
+        return version;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/package-info.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/package-info.java
new file mode 100644
index 0000000000000..3fef2aab07d43
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/applicationtemplates/package-info.java
@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Core classes responsible for handling all application based configuration templates related operations. */
+package org.opensearch.cluster.applicationtemplates;
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 0648fad619dc7..b4826e1a59428 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -49,6 +49,7 @@
 import org.opensearch.cluster.NodeConnectionsService;
 import org.opensearch.cluster.action.index.MappingUpdatedAction;
 import org.opensearch.cluster.action.shard.ShardStateAction;
+import org.opensearch.cluster.applicationtemplates.SystemTemplatesService;
 import org.opensearch.cluster.coordination.ClusterBootstrapService;
 import org.opensearch.cluster.coordination.ClusterFormationFailureHelper;
 import org.opensearch.cluster.coordination.Coordinator;
@@ -757,7 +758,9 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING,
 
                 // Composite index settings
-                CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING
+                CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING,
+
+                SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
index b6166f5d3cce1..d893d8d92be3b 100644
--- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
@@ -38,6 +38,7 @@ protected FeatureFlagSettings(
         FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         FeatureFlags.PLUGGABLE_CACHE_SETTING,
         FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING,
-        FeatureFlags.STAR_TREE_INDEX_SETTING
+        FeatureFlags.STAR_TREE_INDEX_SETTING,
+        FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING
     );
 }
diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
index ceb2559a0e16c..9d57e6939e3ae 100644
--- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
+++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -107,6 +107,16 @@ public class FeatureFlags {
     public static final String STAR_TREE_INDEX = "opensearch.experimental.feature.composite_index.star_tree.enabled";
     public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope);
 
+    /**
+     * Gates the functionality of application based configuration templates.
+     */
+    public static final String APPLICATION_BASED_CONFIGURATION_TEMPLATES = "opensearch.experimental.feature.application_templates.enabled";
+    public static final Setting<Boolean> APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING = Setting.boolSetting(
+        APPLICATION_BASED_CONFIGURATION_TEMPLATES,
+        false,
+        Property.NodeScope
+    );
+
     private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
         REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         EXTENSIONS_SETTING,
@@ -116,8 +126,10 @@ public class FeatureFlags {
         TIERED_REMOTE_INDEX_SETTING,
         PLUGGABLE_CACHE_SETTING,
         REMOTE_PUBLICATION_EXPERIMENTAL_SETTING,
-        STAR_TREE_INDEX_SETTING
+        STAR_TREE_INDEX_SETTING,
+        APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING
     );
+
     /**
      * Should store the settings from opensearch.yml.
      */
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 281f697d9bb79..d91b2a45a48c6 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -68,6 +68,8 @@
 import org.opensearch.cluster.InternalClusterInfoService;
 import org.opensearch.cluster.NodeConnectionsService;
 import org.opensearch.cluster.action.index.MappingUpdatedAction;
+import org.opensearch.cluster.applicationtemplates.SystemTemplatesPlugin;
+import org.opensearch.cluster.applicationtemplates.SystemTemplatesService;
 import org.opensearch.cluster.coordination.PersistedStateRegistry;
 import org.opensearch.cluster.metadata.AliasValidator;
 import org.opensearch.cluster.metadata.IndexTemplateMetadata;
@@ -669,11 +671,20 @@ protected Node(
             resourcesToClose.add(clusterService);
             final Set<Setting<?>> consistentSettings = settingsModule.getConsistentSettings();
             if (consistentSettings.isEmpty() == false) {
-                clusterService.addLocalNodeMasterListener(
+                clusterService.addLocalNodeClusterManagerListener(
                     new ConsistentSettingsService(settings, clusterService, consistentSettings).newHashPublisher()
                 );
             }
 
+            SystemTemplatesService systemTemplatesService = new SystemTemplatesService(
+                pluginsService.filterPlugins(SystemTemplatesPlugin.class),
+                threadPool,
+                clusterService.getClusterSettings(),
+                settings
+            );
+            systemTemplatesService.verifyRepositories();
+            clusterService.addLocalNodeClusterManagerListener(systemTemplatesService);
+
             final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client);
             final UsageService usageService = new UsageService();
 
diff --git a/server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java b/server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java
new file mode 100644
index 0000000000000..63caccc87e67a
--- /dev/null
+++ b/server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java
@@ -0,0 +1,148 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.OpenSearchCorruptionException;
+import org.opensearch.cluster.metadata.ComponentTemplate;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+
+import java.io.IOException;
+import java.util.UUID;
+
+public class ClusterStateSystemTemplateLoaderTests extends OpenSearchSingleNodeTestCase {
+
+    public static final String SAMPLE_TEMPLATE = "{\n"
+        + "  \"template\": {\n"
+        + "    \"settings\": {\n"
+        + "       \"index\": {\n"
+        + "           \"codec\": \"best_compression\",\n"
+        + "           \"merge.policy\": \"log_byte_size\",\n"
+        + "           \"refresh_interval\": \"60s\"\n"
+        + "       }\n"
+        + "    }\n"
+        + "  },\n"
+        + "  \"_meta\": {\n"
+        + "    \"_type\": \"@abc_template\",\n"
+        + "    \"_version\": 1\n"
+        + "  },\n"
+        + "  \"version\": 1\n"
+        + "}";
+
+    public static final String SAMPLE_TEMPLATE_V2 = "{\n"
+        + "  \"template\": {\n"
+        + "    \"settings\": {\n"
+        + "       \"index\": {\n"
+        + "           \"codec\": \"best_compression\",\n"
+        + "           \"merge.policy\": \"log_byte_size\",\n"
+        + "           \"refresh_interval\": \"60s\"\n"
+        + "       }\n"
+        + "    }\n"
+        + "  },\n"
+        + "  \"_meta\": {\n"
+        + "    \"_type\": \"@abc_template\",\n"
+        + "    \"_version\": 2\n"
+        + "  },\n"
+        + "  \"version\": 2\n"
+        + "}";
+
+    public void testLoadTemplate() throws IOException {
+        ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
+            node().client(),
+            () -> node().injector().getInstance(ClusterService.class).state()
+        );
+
+        TemplateRepositoryMetadata repositoryMetadata = new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L);
+        SystemTemplateMetadata metadata = SystemTemplateMetadata.fromComponentTemplateInfo("dummy", 1L);
+
+        // Load for the first time
+        assertTrue(
+            loader.loadTemplate(
+                new SystemTemplate(
+                    new BytesArray(SAMPLE_TEMPLATE),
+                    metadata,
+                    new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
+                )
+            )
+        );
+        assertTrue(
+            node().injector()
+                .getInstance(ClusterService.class)
+                .state()
+                .metadata()
+                .componentTemplates()
+                .containsKey(metadata.fullyQualifiedName())
+        );
+        XContentParser parser = JsonXContent.jsonXContent.createParser(
+            NamedXContentRegistry.EMPTY,
+            DeprecationHandler.IGNORE_DEPRECATIONS,
+            SAMPLE_TEMPLATE
+        );
+        assertEquals(
+            node().injector().getInstance(ClusterService.class).state().metadata().componentTemplates().get(metadata.fullyQualifiedName()),
+            ComponentTemplate.parse(parser)
+        );
+
+        // Retry and ensure loading does not happen again with same version
+        assertFalse(
+            loader.loadTemplate(
+                new SystemTemplate(
+                    new BytesArray(SAMPLE_TEMPLATE),
+                    metadata,
+                    new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
+                )
+            )
+        );
+
+        // Retry with new template version
+        SystemTemplateMetadata newVersionMetadata = SystemTemplateMetadata.fromComponentTemplateInfo("dummy", 2L);
+        assertTrue(loader.loadTemplate(new SystemTemplate(new BytesArray(SAMPLE_TEMPLATE_V2), newVersionMetadata, repositoryMetadata)));
+        parser = JsonXContent.jsonXContent.createParser(
+            NamedXContentRegistry.EMPTY,
+            DeprecationHandler.IGNORE_DEPRECATIONS,
+            SAMPLE_TEMPLATE_V2
+        );
+        assertEquals(
+            node().injector()
+                .getInstance(ClusterService.class)
+                .state()
+                .metadata()
+                .componentTemplates()
+                .get(newVersionMetadata.fullyQualifiedName()),
+            ComponentTemplate.parse(parser)
+        );
+    }
+
+    public void testLoadTemplateVersionMismatch() throws IOException {
+        ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
+            node().client(),
+            () -> node().injector().getInstance(ClusterService.class).state()
+        );
+
+        TemplateRepositoryMetadata repositoryMetadata = new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L);
+        SystemTemplateMetadata metadata = SystemTemplateMetadata.fromComponentTemplateInfo("dummy", 2L);
+
+        // Load for the first time
+        assertThrows(
+            OpenSearchCorruptionException.class,
+            () -> loader.loadTemplate(
+                new SystemTemplate(
+                    new BytesArray(SAMPLE_TEMPLATE),
+                    metadata,
+                    new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
+                )
+            )
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesServiceTests.java b/server/src/test/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesServiceTests.java
new file mode 100644
index 0000000000000..4addf3802b40d
--- /dev/null
+++ b/server/src/test/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesServiceTests.java
@@ -0,0 +1,90 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.cluster.service.applicationtemplates.TestSystemTemplatesRepositoryPlugin;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.mockito.Mockito;
+
+import static org.opensearch.common.settings.ClusterSettings.BUILT_IN_CLUSTER_SETTINGS;
+import static org.mockito.Mockito.when;
+
+public class SystemTemplatesServiceTests extends OpenSearchTestCase {
+
+    private SystemTemplatesService systemTemplatesService;
+
+    public void testSystemTemplatesLoaded() throws IOException {
+        setupService(true);
+
+        systemTemplatesService.onClusterManager();
+        SystemTemplatesService.Stats stats = systemTemplatesService.stats();
+        assertNotNull(stats);
+        assertEquals(stats.getTemplatesLoaded(), 1L);
+        assertEquals(stats.getFailedLoadingTemplates(), 0L);
+        assertEquals(stats.getFailedLoadingRepositories(), 1L);
+    }
+
+    public void testSystemTemplatesVerify() throws IOException {
+        setupService(false);
+
+        systemTemplatesService.verifyRepositories();
+
+        SystemTemplatesService.Stats stats = systemTemplatesService.stats();
+        assertNotNull(stats);
+        assertEquals(stats.getTemplatesLoaded(), 0L);
+        assertEquals(stats.getFailedLoadingTemplates(), 0L);
+        assertEquals(stats.getFailedLoadingRepositories(), 0L);
+    }
+
+    public void testSystemTemplatesVerifyWithFailingRepository() throws IOException {
+        setupService(true);
+
+        assertThrows(IllegalStateException.class, () -> systemTemplatesService.verifyRepositories());
+
+        SystemTemplatesService.Stats stats = systemTemplatesService.stats();
+        assertNotNull(stats);
+        assertEquals(stats.getTemplatesLoaded(), 0L);
+        assertEquals(stats.getFailedLoadingTemplates(), 0L);
+        assertEquals(stats.getFailedLoadingRepositories(), 1L);
+    }
+
+    void setupService(boolean errorFromMockPlugin) throws IOException {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, true).build());
+
+        ThreadPool mockPool = Mockito.mock(ThreadPool.class);
+        when(mockPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService());
+
+        List<SystemTemplatesPlugin> plugins = new ArrayList<>();
+        plugins.add(new TestSystemTemplatesRepositoryPlugin());
+
+        if (errorFromMockPlugin) {
+            SystemTemplatesPlugin mockPlugin = Mockito.mock(SystemTemplatesPlugin.class);
+            when(mockPlugin.loadRepository()).thenThrow(new IOException());
+            plugins.add(mockPlugin);
+        }
+
+        ClusterSettings mockSettings = new ClusterSettings(Settings.EMPTY, BUILT_IN_CLUSTER_SETTINGS);
+        systemTemplatesService = new SystemTemplatesService(
+            plugins,
+            mockPool,
+            mockSettings,
+            Settings.builder().put(SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED.getKey(), true).build()
+        );
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/cluster/service/applicationtemplates/TestSystemTemplatesRepositoryPlugin.java b/test/framework/src/main/java/org/opensearch/cluster/service/applicationtemplates/TestSystemTemplatesRepositoryPlugin.java
new file mode 100644
index 0000000000000..c5245c7109d8f
--- /dev/null
+++ b/test/framework/src/main/java/org/opensearch/cluster/service/applicationtemplates/TestSystemTemplatesRepositoryPlugin.java
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.service.applicationtemplates;
+
+import org.opensearch.cluster.applicationtemplates.SystemTemplate;
+import org.opensearch.cluster.applicationtemplates.SystemTemplateLoader;
+import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
+import org.opensearch.cluster.applicationtemplates.SystemTemplateRepository;
+import org.opensearch.cluster.applicationtemplates.SystemTemplatesPlugin;
+import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.plugins.Plugin;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+public class TestSystemTemplatesRepositoryPlugin extends Plugin implements SystemTemplatesPlugin {
+
+    private final SystemTemplateMetadata templateMetadata = SystemTemplateMetadata.fromComponentTemplateInfo("dummy", 1);
+
+    private final TemplateRepositoryMetadata repoMetadata = new TemplateRepositoryMetadata("test", 1);
+
+    private final SystemTemplate systemTemplate = new SystemTemplate(
+        BytesReference.fromByteBuffer(ByteBuffer.wrap("content".getBytes(StandardCharsets.UTF_8))),
+        templateMetadata,
+        repoMetadata
+    );
+
+    @Override
+    public SystemTemplateRepository loadRepository() throws IOException {
+        return new SystemTemplateRepository() {
+            @Override
+            public TemplateRepositoryMetadata metadata() {
+                return repoMetadata;
+            }
+
+            @Override
+            public List<SystemTemplateMetadata> listTemplates() throws IOException {
+                return List.of(templateMetadata);
+            }
+
+            @Override
+            public SystemTemplate getTemplate(SystemTemplateMetadata template) throws IOException {
+                return systemTemplate;
+            }
+
+            @Override
+            public void close() throws Exception {}
+        };
+    }
+
+    @Override
+    public SystemTemplateLoader loaderFor(SystemTemplateMetadata templateMetadata) {
+        return new SystemTemplateLoader() { // Asserting Loader
+            @Override
+            public boolean loadTemplate(SystemTemplate template) throws IOException {
+                assert template.templateMetadata() == TestSystemTemplatesRepositoryPlugin.this.templateMetadata;
+                assert template.repositoryMetadata() == repoMetadata;
+                assert template.templateContent() == systemTemplate.templateContent();
+                return true;
+            }
+        };
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index ca5ddf21710af..7a50502e418e2 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -90,6 +90,7 @@
 import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.cluster.service.applicationtemplates.TestSystemTemplatesRepositoryPlugin;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.Priority;
 import org.opensearch.common.collect.Tuple;
@@ -682,6 +683,7 @@ protected Settings featureFlagSettings() {
         }
         // Enabling Telemetry setting by default
         featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true);
+        featureSettings.put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING.getKey(), true);
 
         return featureSettings.build();
     }
@@ -2168,6 +2170,7 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
         if (addMockTelemetryPlugin()) {
             mocks.add(MockTelemetryPlugin.class);
         }
+        mocks.add(TestSystemTemplatesRepositoryPlugin.class);
         return Collections.unmodifiableList(mocks);
     }
 
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java
index 45ea63e862df6..1dfad60c04155 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java
@@ -438,6 +438,7 @@ protected Settings featureFlagSettings() {
             featureSettings.put(builtInFlag.getKey(), builtInFlag.getDefaultRaw(Settings.EMPTY));
         }
         featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true);
+        featureSettings.put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING.getKey(), true);
         return featureSettings.build();
     }
 

From 8ae728c5610900888449af2eae2ef8b23d664538 Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Wed, 17 Jul 2024 04:34:09 +0800
Subject: [PATCH 108/113] Fix bulk upsert ignores the default_pipeline and
 final_pipeline when the auto-created index matches the index template
 (#12891)

* Fix bulk upsert ignores the default_pipeline and final_pipeline when auto-created index matches with the index template

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Modify changelog & comment

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Use new approach

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Fix test failure

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

---------

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../rest-api-spec/test/ingest/70_bulk.yml     | 38 +++++++++++++++++++
 .../action/update/UpdateRequest.java          |  4 +-
 .../action/update/UpdateRequestTests.java     |  4 +-
 .../opensearch/ingest/IngestServiceTests.java | 14 +++++++
 5 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f885261f404ae..b863b9d13e789 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactoring FilterPath.parse by using an iterative approach ([#14200](https://github.com/opensearch-project/OpenSearch/pull/14200))
 - Refactoring Grok.validatePatternBank by using an iterative approach ([#14206](https://github.com/opensearch-project/OpenSearch/pull/14206))
 - Update help output for _cat ([#14722](https://github.com/opensearch-project/OpenSearch/pull/14722))
+- Fix bulk upsert ignores the default_pipeline and final_pipeline when auto-created index matches the index template ([#12891](https://github.com/opensearch-project/OpenSearch/pull/12891))
 
 ### Security
 
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
index edb7b77eb8d28..8830503940f4d 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
@@ -41,6 +41,10 @@ teardown:
       ingest.delete_pipeline:
         id: "pipeline2"
         ignore: 404
+  - do:
+      indices.delete_index_template:
+        name: test_index_template_for_bulk
+        ignore: 404
 
 ---
 "Test bulk request without default pipeline":
@@ -168,6 +172,40 @@ teardown:
         id: test_id3
   - match: { _source: {"f1": "v2", "f2": 47, "field1": "value1"}}
 
+# related issue: https://github.com/opensearch-project/OpenSearch/issues/12888
+---
+"Test bulk upsert honors default_pipeline and final_pipeline when the auto-created index matches with the index template":
+  - skip:
+      version: " - 2.99.99"
+      reason: "fixed in 3.0.0"
+  - do:
+      indices.put_index_template:
+        name: test_for_bulk_upsert_index_template
+        body:
+          index_patterns: test_bulk_upsert_*
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+              default_pipeline: pipeline1
+              final_pipeline: pipeline2
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"update": {"_index": "test_bulk_upsert_index", "_id": "test_id3"}}'
+          - '{"upsert": {"f1": "v2", "f2": 47}, "doc": {"x": 1}}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: created }
+
+  - do:
+      get:
+        index: test_bulk_upsert_index
+        id: test_id3
+  - match: { _source: {"f1": "v2", "f2": 47, "field1": "value1", "field2": "value2"}}
+
 ---
 "Test bulk API with batch enabled happy case":
   - skip:
diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
index 9654bd1c114ba..6cb5e049e0f1e 100644
--- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
+++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
@@ -717,7 +717,7 @@ public IndexRequest doc() {
 
     private IndexRequest safeDoc() {
         if (doc == null) {
-            doc = new IndexRequest();
+            doc = new IndexRequest(index);
         }
         return doc;
     }
@@ -803,7 +803,7 @@ public IndexRequest upsertRequest() {
 
     private IndexRequest safeUpsertRequest() {
         if (upsertRequest == null) {
-            upsertRequest = new IndexRequest();
+            upsertRequest = new IndexRequest(index);
         }
         return upsertRequest;
     }
diff --git a/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java
index b70fda0d86240..e85dfa8cca556 100644
--- a/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java
@@ -247,6 +247,7 @@ public void testFromXContent() throws Exception {
         assertThat(params, notNullValue());
         assertThat(params.size(), equalTo(1));
         assertThat(params.get("param1").toString(), equalTo("value1"));
+        assertThat(request.upsertRequest().index(), equalTo("test"));
         Map<String, Object> upsertDoc = XContentHelper.convertToMap(
             request.upsertRequest().source(),
             true,
@@ -304,6 +305,7 @@ public void testFromXContent() throws Exception {
             )
         );
         Map<String, Object> doc = request.doc().sourceAsMap();
+        assertThat(request.doc().index(), equalTo("test"));
         assertThat(doc.get("field1").toString(), equalTo("value1"));
         assertThat(((Map<String, Object>) doc.get("compound")).get("field2").toString(), equalTo("value2"));
     }
@@ -662,7 +664,7 @@ public void testToString() throws IOException {
             request.toString(),
             equalTo(
                 "update {[test][1], doc_as_upsert[false], "
-                    + "doc[index {[null][null], source[{\"body\":\"bar\"}]}], scripted_upsert[false], detect_noop[true]}"
+                    + "doc[index {[test][null], source[{\"body\":\"bar\"}]}], scripted_upsert[false], detect_noop[true]}"
             )
         );
     }
diff --git a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
index 684297c11c140..e61fbb6e1dbff 100644
--- a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
@@ -1605,6 +1605,13 @@ public void testResolveRequiredOrDefaultPipelineDefaultPipeline() {
         assertThat(result, is(true));
         assertThat(indexRequest.isPipelineResolved(), is(true));
         assertThat(indexRequest.getPipeline(), equalTo("default-pipeline"));
+
+        // index name matches with ITMD for bulk upsert
+        UpdateRequest updateRequest = new UpdateRequest("idx", "id1").upsert(emptyMap()).script(mockScript("1"));
+        result = IngestService.resolvePipelines(updateRequest, TransportBulkAction.getIndexWriteRequest(updateRequest), metadata);
+        assertThat(result, is(true));
+        assertThat(updateRequest.upsertRequest().isPipelineResolved(), is(true));
+        assertThat(updateRequest.upsertRequest().getPipeline(), equalTo("default-pipeline"));
     }
 
     public void testResolveFinalPipeline() {
@@ -1642,6 +1649,13 @@ public void testResolveFinalPipeline() {
         assertThat(indexRequest.isPipelineResolved(), is(true));
         assertThat(indexRequest.getPipeline(), equalTo("_none"));
         assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
+
+        // index name matches with ITMD for bulk upsert:
+        UpdateRequest updateRequest = new UpdateRequest("idx", "id1").upsert(emptyMap()).script(mockScript("1"));
+        result = IngestService.resolvePipelines(updateRequest, TransportBulkAction.getIndexWriteRequest(updateRequest), metadata);
+        assertThat(result, is(true));
+        assertThat(updateRequest.upsertRequest().isPipelineResolved(), is(true));
+        assertThat(updateRequest.upsertRequest().getFinalPipeline(), equalTo("final-pipeline"));
     }
 
     public void testResolveRequestOrDefaultPipelineAndFinalPipeline() {

From 0e31c02585059db726ebaa71b57dd8dbad7f6007 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <81609427+mgodwan@users.noreply.github.com>
Date: Wed, 17 Jul 2024 13:57:20 +0530
Subject: [PATCH 109/113] Fix flaky test due to node being used across all
 tests (#14787)

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../ClusterStateSystemTemplateLoaderTests.java               | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java b/server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java
index 63caccc87e67a..c7cfab6d38e04 100644
--- a/server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java
+++ b/server/src/test/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoaderTests.java
@@ -145,4 +145,9 @@ public void testLoadTemplateVersionMismatch() throws IOException {
             )
         );
     }
+
+    @Override
+    protected boolean resetNodeAfterTest() {
+        return true;
+    }
 }

From a2cef8fcdeeaed4ca11f6c6d69a9c9cb5cd4d2c5 Mon Sep 17 00:00:00 2001
From: Sarthak Aggarwal <sarthagg@amazon.com>
Date: Wed, 17 Jul 2024 15:26:49 +0530
Subject: [PATCH 110/113] Star Tree Implementation [OnHeap] (#14512)

---------
Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
---
 .../composite/Composite99DocValuesWriter.java |   5 +-
 .../datacube/startree/StarTreeDocument.java   |  34 +
 .../aggregators/CountValueAggregator.java     |  66 ++
 .../aggregators/MetricAggregatorInfo.java     | 130 ++++
 .../aggregators/SumValueAggregator.java       |  97 +++
 .../startree/aggregators/ValueAggregator.java |  64 ++
 .../aggregators/ValueAggregatorFactory.java   |  56 ++
 .../numerictype/StarTreeNumericType.java      |  66 ++
 .../StarTreeNumericTypeConverters.java        |  58 ++
 .../aggregators/numerictype/package-info.java |  14 +
 .../startree/aggregators/package-info.java    |  14 +
 .../startree/builder/BaseStarTreeBuilder.java | 668 +++++++++++++++++
 .../builder/OnHeapStarTreeBuilder.java        | 213 ++++++
 .../startree/builder/StarTreeBuilder.java     |  29 +
 .../StarTreeDocValuesIteratorAdapter.java     |  82 ++
 .../startree/builder/StarTreesBuilder.java    | 114 +++
 .../startree/builder/package-info.java        |  14 +
 .../datacube/startree/package-info.java       |   2 +
 .../utils/SequentialDocValuesIterator.java    | 137 ++++
 .../datacube/startree/utils/TreeNode.java     |  65 ++
 .../datacube/startree/utils/package-info.java |  14 +
 .../StarTreeDocValuesFormatTests.java         |  36 -
 .../CountValueAggregatorTests.java            |  53 ++
 .../MetricAggregatorInfoTests.java            | 123 +++
 .../aggregators/SumValueAggregatorTests.java  |  72 ++
 .../ValueAggregatorFactoryTests.java          |  27 +
 .../builder/BaseStarTreeBuilderTests.java     | 216 ++++++
 .../builder/OnHeapStarTreeBuilderTests.java   | 706 ++++++++++++++++++
 ...StarTreeDocValuesIteratorAdapterTests.java | 139 ++++
 .../StarTreeValuesIteratorFactoryTests.java   | 131 ++++
 .../builder/StarTreesBuilderTests.java        | 132 ++++
 .../SequentialDocValuesIteratorTests.java     |  46 ++
 32 files changed, 3586 insertions(+), 37 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java
 create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java

diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java
index 75bbf78dbdad2..3753b20a8bea3 100644
--- a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java
+++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.index.MergeState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder;
 import org.opensearch.index.mapper.CompositeMappedFieldType;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.StarTreeMapper;
@@ -98,7 +99,9 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer,
         if (compositeFieldSet.isEmpty()) {
             for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) {
                 if (mappedType instanceof StarTreeMapper.StarTreeFieldType) {
-                    // TODO : Call StarTree builder
+                    try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, state, mapperService)) {
+                        starTreesBuilder.build();
+                    }
                 }
             }
         }
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java
new file mode 100644
index 0000000000000..0ce2b3a5cdac5
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java
@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.Arrays;
+
+/**
+ * Star tree document
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeDocument {
+    public final Long[] dimensions;
+    public final Object[] metrics;
+
+    public StarTreeDocument(Long[] dimensions, Object[] metrics) {
+        this.dimensions = dimensions;
+        this.metrics = metrics;
+    }
+
+    @Override
+    public String toString() {
+        return Arrays.toString(dimensions) + " | " + Arrays.toString(metrics);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java
new file mode 100644
index 0000000000000..d72f4a292dc0a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+
+/**
+ * Count value aggregator for star tree
+ *
+ * @opensearch.experimental
+ */
+public class CountValueAggregator implements ValueAggregator<Long> {
+    public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.LONG;
+    public static final long DEFAULT_INITIAL_VALUE = 1L;
+
+    @Override
+    public MetricStat getAggregationType() {
+        return MetricStat.COUNT;
+    }
+
+    @Override
+    public StarTreeNumericType getAggregatedValueType() {
+        return VALUE_AGGREGATOR_TYPE;
+    }
+
+    @Override
+    public Long getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) {
+        return DEFAULT_INITIAL_VALUE;
+    }
+
+    @Override
+    public Long mergeAggregatedValueAndSegmentValue(Long value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) {
+        return value + 1;
+    }
+
+    @Override
+    public Long mergeAggregatedValues(Long value, Long aggregatedValue) {
+        return value + aggregatedValue;
+    }
+
+    @Override
+    public Long getInitialAggregatedValue(Long value) {
+        return value;
+    }
+
+    @Override
+    public int getMaxAggregatedValueByteSize() {
+        return Long.BYTES;
+    }
+
+    @Override
+    public Long toLongValue(Long value) {
+        return value;
+    }
+
+    @Override
+    public Long toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) {
+        return value;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java
new file mode 100644
index 0000000000000..46f1b1ac11063
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java
@@ -0,0 +1,130 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+
+import java.util.Comparator;
+import java.util.Objects;
+
+/**
+ * Builds aggregation function and doc values field pair to support various aggregations
+ *
+ * @opensearch.experimental
+ */
+public class MetricAggregatorInfo implements Comparable<MetricAggregatorInfo> {
+
+    public static final String DELIMITER = "_";
+    private final String metric;
+    private final String starFieldName;
+    private final MetricStat metricStat;
+    private final String field;
+    private final ValueAggregator valueAggregators;
+    private final StarTreeNumericType starTreeNumericType;
+    private final SequentialDocValuesIterator metricStatReader;
+
+    /**
+     * Constructor for MetricAggregatorInfo
+     */
+    public MetricAggregatorInfo(
+        MetricStat metricStat,
+        String field,
+        String starFieldName,
+        IndexNumericFieldData.NumericType numericType,
+        SequentialDocValuesIterator metricStatReader
+    ) {
+        this.metricStat = metricStat;
+        this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat);
+        this.starTreeNumericType = StarTreeNumericType.fromNumericType(numericType);
+        this.metricStatReader = metricStatReader;
+        this.field = field;
+        this.starFieldName = starFieldName;
+        this.metric = toFieldName();
+    }
+
+    /**
+     * @return metric type
+     */
+    public MetricStat getMetricStat() {
+        return metricStat;
+    }
+
+    /**
+     * @return field Name
+     */
+    public String getField() {
+        return field;
+    }
+
+    /**
+     * @return the metric stat name
+     */
+    public String getMetric() {
+        return metric;
+    }
+
+    /**
+     * @return aggregator for the field value
+     */
+    public ValueAggregator getValueAggregators() {
+        return valueAggregators;
+    }
+
+    /**
+     * @return star tree aggregated value type
+     */
+    public StarTreeNumericType getAggregatedValueType() {
+        return starTreeNumericType;
+    }
+
+    /**
+     * @return metric value reader iterator
+     */
+    public SequentialDocValuesIterator getMetricStatReader() {
+        return metricStatReader;
+    }
+
+    /**
+     * @return field name with metric type and field
+     */
+    public String toFieldName() {
+        return starFieldName + DELIMITER + field + DELIMITER + metricStat.getTypeName();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(toFieldName());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof MetricAggregatorInfo) {
+            MetricAggregatorInfo anotherPair = (MetricAggregatorInfo) obj;
+            return metricStat == anotherPair.metricStat && field.equals(anotherPair.field);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return toFieldName();
+    }
+
+    @Override
+    public int compareTo(MetricAggregatorInfo other) {
+        return Comparator.comparing((MetricAggregatorInfo o) -> o.field)
+            .thenComparing((MetricAggregatorInfo o) -> o.metricStat)
+            .compare(this, other);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java
new file mode 100644
index 0000000000000..543b0f7f42374
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java
@@ -0,0 +1,97 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.apache.lucene.util.NumericUtils;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+import org.opensearch.search.aggregations.metrics.CompensatedSum;
+
+/**
+ * Sum value aggregator for star tree
+ *
+ * @opensearch.experimental
+ */
+public class SumValueAggregator implements ValueAggregator<Double> {
+
+    public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE;
+    private double sum = 0;
+    private double compensation = 0;
+    private CompensatedSum kahanSummation = new CompensatedSum(0, 0);
+
+    @Override
+    public MetricStat getAggregationType() {
+        return MetricStat.SUM;
+    }
+
+    @Override
+    public StarTreeNumericType getAggregatedValueType() {
+        return VALUE_AGGREGATOR_TYPE;
+    }
+
+    @Override
+    public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) {
+        kahanSummation.reset(0, 0);
+        kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue));
+        compensation = kahanSummation.delta();
+        sum = kahanSummation.value();
+        return kahanSummation.value();
+    }
+
+    @Override
+    public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) {
+        assert kahanSummation.value() == value;
+        kahanSummation.reset(sum, compensation);
+        kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue));
+        compensation = kahanSummation.delta();
+        sum = kahanSummation.value();
+        return kahanSummation.value();
+    }
+
+    @Override
+    public Double mergeAggregatedValues(Double value, Double aggregatedValue) {
+        assert kahanSummation.value() == aggregatedValue;
+        kahanSummation.reset(sum, compensation);
+        kahanSummation.add(value);
+        compensation = kahanSummation.delta();
+        sum = kahanSummation.value();
+        return kahanSummation.value();
+    }
+
+    @Override
+    public Double getInitialAggregatedValue(Double value) {
+        kahanSummation.reset(0, 0);
+        kahanSummation.add(value);
+        compensation = kahanSummation.delta();
+        sum = kahanSummation.value();
+        return kahanSummation.value();
+    }
+
+    @Override
+    public int getMaxAggregatedValueByteSize() {
+        return Double.BYTES;
+    }
+
+    @Override
+    public Long toLongValue(Double value) {
+        try {
+            return NumericUtils.doubleToSortableLong(value);
+        } catch (Exception e) {
+            throw new IllegalStateException("Cannot convert " + value + " to sortable long", e);
+        }
+    }
+
+    @Override
+    public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) {
+        try {
+            return type.getDoubleValue(value);
+        } catch (Exception e) {
+            throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java
new file mode 100644
index 0000000000000..3dd1f85845c17
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java
@@ -0,0 +1,64 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+
+/**
+ * A value aggregator that pre-aggregates on the input values for a specific type of aggregation.
+ *
+ * @opensearch.experimental
+ */
+public interface ValueAggregator<A> {
+
+    /**
+     * Returns the type of the aggregation.
+     */
+    MetricStat getAggregationType();
+
+    /**
+     * Returns the data type of the aggregated value.
+     */
+    StarTreeNumericType getAggregatedValueType();
+
+    /**
+     * Returns the initial aggregated value.
+     */
+    A getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType);
+
+    /**
+     * Applies a segment doc value to the current aggregated value.
+     */
+    A mergeAggregatedValueAndSegmentValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType);
+
+    /**
+     * Applies an aggregated value to the current aggregated value.
+     */
+    A mergeAggregatedValues(A value, A aggregatedValue);
+
+    /**
+     * Clones an aggregated value.
+     */
+    A getInitialAggregatedValue(A value);
+
+    /**
+     * Returns the maximum size in bytes of the aggregated values seen so far.
+     */
+    int getMaxAggregatedValueByteSize();
+
+    /**
+     * Converts an aggregated value into a Long type.
+     */
+    Long toLongValue(A value);
+
+    /**
+     * Converts an aggregated value from a Long type.
+     */
+    A toStarTreeNumericTypeValue(Long rawValue, StarTreeNumericType type);
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java
new file mode 100644
index 0000000000000..4ee0b0b5b13f8
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java
@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+
+/**
+ * Value aggregator factory for a given aggregation type
+ *
+ * @opensearch.experimental
+ */
+public class ValueAggregatorFactory {
+    private ValueAggregatorFactory() {}
+
+    /**
+     * Returns a new instance of value aggregator for the given aggregation type.
+     *
+     * @param aggregationType Aggregation type
+     * @return Value aggregator
+     */
+    public static ValueAggregator getValueAggregator(MetricStat aggregationType) {
+        switch (aggregationType) {
+            // other metric types (count, min, max, avg) will be supported in the future
+            case SUM:
+                return new SumValueAggregator();
+            case COUNT:
+                return new CountValueAggregator();
+            default:
+                throw new IllegalStateException("Unsupported aggregation type: " + aggregationType);
+        }
+    }
+
+    /**
+     * Returns the data type of the aggregated value for the given aggregation type.
+     *
+     * @param aggregationType Aggregation type
+     * @return Data type of the aggregated value
+     */
+    public static StarTreeNumericType getAggregatedValueType(MetricStat aggregationType) {
+        switch (aggregationType) {
+            // other metric types (count, min, max, avg) will be supported in the future
+            case SUM:
+                return SumValueAggregator.VALUE_AGGREGATOR_TYPE;
+            case COUNT:
+                return CountValueAggregator.VALUE_AGGREGATOR_TYPE;
+            default:
+                throw new IllegalStateException("Unsupported aggregation type: " + aggregationType);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java
new file mode 100644
index 0000000000000..57fe573a6a93c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype;
+
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+
+import java.util.function.Function;
+
+/**
+ * Enum to map Star Tree Numeric Types to Lucene's Numeric Type
+ *
+ * @opensearch.experimental
+ */
+public enum StarTreeNumericType {
+
+    // TODO: Handle scaled floats
+    HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble),
+    FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble),
+    LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble),
+    DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble),
+    INT(IndexNumericFieldData.NumericType.INT, StarTreeNumericTypeConverters::intToDouble),
+    SHORT(IndexNumericFieldData.NumericType.SHORT, StarTreeNumericTypeConverters::shortToDouble),
+    BYTE(IndexNumericFieldData.NumericType.BYTE, StarTreeNumericTypeConverters::bytesToDouble),
+    UNSIGNED_LONG(IndexNumericFieldData.NumericType.UNSIGNED_LONG, StarTreeNumericTypeConverters::unsignedlongToDouble);
+
+    final IndexNumericFieldData.NumericType numericType;
+    final Function<Long, Double> converter;
+
+    StarTreeNumericType(IndexNumericFieldData.NumericType numericType, Function<Long, Double> converter) {
+        this.numericType = numericType;
+        this.converter = converter;
+    }
+
+    public double getDoubleValue(long rawValue) {
+        return this.converter.apply(rawValue);
+    }
+
+    public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericType numericType) {
+        switch (numericType) {
+            case HALF_FLOAT:
+                return StarTreeNumericType.HALF_FLOAT;
+            case FLOAT:
+                return StarTreeNumericType.FLOAT;
+            case LONG:
+                return StarTreeNumericType.LONG;
+            case DOUBLE:
+                return StarTreeNumericType.DOUBLE;
+            case INT:
+                return StarTreeNumericType.INT;
+            case SHORT:
+                return StarTreeNumericType.SHORT;
+            case UNSIGNED_LONG:
+                return StarTreeNumericType.UNSIGNED_LONG;
+            case BYTE:
+                return StarTreeNumericType.BYTE;
+            default:
+                throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]");
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java
new file mode 100644
index 0000000000000..eb7647c4f9851
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype;
+
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
+import org.apache.lucene.util.NumericUtils;
+import org.opensearch.common.Numbers;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Numeric converters used during aggregations of metric values
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeNumericTypeConverters {
+
+    public static double halfFloatPointToDouble(Long value) {
+        return HalfFloatPoint.sortableShortToHalfFloat((short) value.longValue());
+    }
+
+    public static double floatPointToDouble(Long value) {
+        return NumericUtils.sortableIntToFloat((int) value.longValue());
+    }
+
+    public static double longToDouble(Long value) {
+        return (double) value;
+    }
+
+    public static double intToDouble(Long value) {
+        return (double) value;
+    }
+
+    public static double shortToDouble(Long value) {
+        return (double) value;
+    }
+
+    public static Double sortableLongtoDouble(Long value) {
+        return NumericUtils.sortableLongToDouble(value);
+    }
+
+    public static double unsignedlongToDouble(Long value) {
+        return Numbers.unsignedLongToDouble(value);
+    }
+
+    public static double bytesToDouble(Long value) {
+        byte[] bytes = new byte[8];
+        NumericUtils.longToSortableBytes(value, bytes, 0);
+        return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, 0));
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java
new file mode 100644
index 0000000000000..fe5c2a7ceb254
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Numeric Types for Composite Index Star Tree
+ *
+ *  @opensearch.experimental
+ */
+package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype;
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java
new file mode 100644
index 0000000000000..bddd6a46fbbe8
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Aggregators for Composite Index Star Tree
+ *
+ * @opensearch.experimental
+ */
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java
new file mode 100644
index 0000000000000..0a363bfad8fe1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java
@@ -0,0 +1,668 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
+import org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode;
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+import org.opensearch.index.mapper.Mapper;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.NumberFieldMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode.ALL;
+
+/**
+ * Builder for star tree. Defines the algorithm to construct star-tree
+ * See {@link StarTreesBuilder} for information around the construction of star-trees based on star-tree fields
+ *
+ * @opensearch.experimental
+ */
+public abstract class BaseStarTreeBuilder implements StarTreeBuilder {
+
+    private static final Logger logger = LogManager.getLogger(BaseStarTreeBuilder.class);
+
+    /**
+     * Default value for star node
+     */
+    public static final int STAR_IN_DOC_VALUES_INDEX = -1;
+
+    protected final Set<Integer> skipStarNodeCreationForDimensions;
+
+    protected final List<MetricAggregatorInfo> metricAggregatorInfos;
+    protected final int numMetrics;
+    protected final int numDimensions;
+    protected int numStarTreeDocs;
+    protected int totalSegmentDocs;
+    protected int numStarTreeNodes;
+    protected final int maxLeafDocuments;
+
+    protected final TreeNode rootNode = getNewNode();
+
+    protected SequentialDocValuesIterator[] dimensionReaders;
+
+    // We do not close these producers as they are empty doc value producers (where close() is unsupported)
+    protected Map<String, DocValuesProducer> fieldProducerMap;
+
+    private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorAdapter;
+    private final StarTreeField starTreeField;
+
+    /**
+     * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters.
+     *
+     * @param starTreeField    holds the configuration for the star tree
+     * @param fieldProducerMap helps return the doc values iterator for each type based on field name
+     * @param state            stores the segment write state
+     * @param mapperService    helps to find the original type of the field
+     */
+    protected BaseStarTreeBuilder(
+        StarTreeField starTreeField,
+        Map<String, DocValuesProducer> fieldProducerMap,
+        SegmentWriteState state,
+        MapperService mapperService
+    ) throws IOException {
+
+        logger.debug("Building in base star tree builder");
+
+        this.starTreeField = starTreeField;
+        StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig();
+        this.fieldProducerMap = fieldProducerMap;
+        this.starTreeDocValuesIteratorAdapter = new StarTreeDocValuesIteratorAdapter();
+
+        List<Dimension> dimensionsSplitOrder = starTreeField.getDimensionsOrder();
+        this.numDimensions = dimensionsSplitOrder.size();
+
+        this.skipStarNodeCreationForDimensions = new HashSet<>();
+        this.totalSegmentDocs = state.segmentInfo.maxDoc();
+        this.dimensionReaders = new SequentialDocValuesIterator[numDimensions];
+        Set<String> skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims();
+
+        for (int i = 0; i < numDimensions; i++) {
+            String dimension = dimensionsSplitOrder.get(i).getField();
+            if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) {
+                this.skipStarNodeCreationForDimensions.add(i);
+            }
+            FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension);
+            DocValuesType dimensionDocValuesType = dimensionFieldInfos.getDocValuesType();
+            dimensionReaders[i] = starTreeDocValuesIteratorAdapter.getDocValuesIterator(
+                dimensionDocValuesType,
+                dimensionFieldInfos,
+                fieldProducerMap.get(dimensionFieldInfos.name)
+            );
+        }
+
+        this.metricAggregatorInfos = generateMetricAggregatorInfos(mapperService, state);
+        this.numMetrics = metricAggregatorInfos.size();
+        this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs();
+    }
+
+    /**
+     * Generates the configuration required to perform aggregation for all the metrics on a field
+     *
+     * @return list of MetricAggregatorInfo
+     */
+    public List<MetricAggregatorInfo> generateMetricAggregatorInfos(MapperService mapperService, SegmentWriteState state)
+        throws IOException {
+        List<MetricAggregatorInfo> metricAggregatorInfos = new ArrayList<>();
+        for (Metric metric : this.starTreeField.getMetrics()) {
+            for (MetricStat metricStat : metric.getMetrics()) {
+                IndexNumericFieldData.NumericType numericType;
+                SequentialDocValuesIterator metricStatReader;
+                Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField());
+                if (fieldMapper instanceof NumberFieldMapper) {
+                    numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType();
+                } else {
+                    logger.error("unsupported mapper type");
+                    throw new IllegalStateException("unsupported mapper type");
+                }
+
+                FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField());
+                DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType();
+                if (metricStat != MetricStat.COUNT) {
+                    metricStatReader = starTreeDocValuesIteratorAdapter.getDocValuesIterator(
+                        metricDocValuesType,
+                        metricFieldInfos,
+                        fieldProducerMap.get(metricFieldInfos.name)
+                    );
+                } else {
+                    metricStatReader = new SequentialDocValuesIterator();
+                }
+
+                MetricAggregatorInfo metricAggregatorInfo = new MetricAggregatorInfo(
+                    metricStat,
+                    metric.getField(),
+                    starTreeField.getName(),
+                    numericType,
+                    metricStatReader
+                );
+                metricAggregatorInfos.add(metricAggregatorInfo);
+            }
+        }
+        return metricAggregatorInfos;
+    }
+
+    /**
+     * Adds a document to the star-tree.
+     *
+     * @param starTreeDocument star tree document to be added
+     * @throws IOException if an I/O error occurs while adding the document
+     */
+    public abstract void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException;
+
+    /**
+     * Returns the document of the given document id in the star-tree.
+     *
+     * @param docId document id
+     * @return star tree document
+     * @throws IOException if an I/O error occurs while fetching the star-tree document
+     */
+    public abstract StarTreeDocument getStarTreeDocument(int docId) throws IOException;
+
+    /**
+     * Retrieves the list of star-tree documents in the star-tree.
+     *
+     * @return Star tree documents
+     */
+    public abstract List<StarTreeDocument> getStarTreeDocuments();
+
+    /**
+     * Returns the value of the dimension for the given dimension id and document in the star-tree.
+     *
+     * @param docId       document id
+     * @param dimensionId dimension id
+     * @return dimension value
+     */
+    public abstract Long getDimensionValue(int docId, int dimensionId) throws IOException;
+
+    /**
+     * Sorts and aggregates the star-tree document in the segment, and returns a star-tree document iterator for all the
+     * aggregated star-tree document.
+     *
+     * @return Iterator for the aggregated star-tree document
+     */
+    public abstract Iterator<StarTreeDocument> sortAndAggregateStarTreeDocuments() throws IOException;
+
+    /**
+     * Generates aggregated star-tree documents for star-node.
+     *
+     * @param startDocId  start document id (inclusive) in the star-tree
+     * @param endDocId    end document id (exclusive) in the star-tree
+     * @param dimensionId dimension id of the star-node
+     * @return Iterator for the aggregated star-tree documents
+     */
+    public abstract Iterator<StarTreeDocument> generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId)
+        throws IOException;
+
+    /**
+     * Returns the star-tree document from the segment
+     *
+     * @throws IOException when we are unable to build a star tree document from the segment
+     */
+    protected StarTreeDocument getSegmentStarTreeDocument(int currentDocId) throws IOException {
+        Long[] dimensions = getStarTreeDimensionsFromSegment(currentDocId);
+        Object[] metrics = getStarTreeMetricsFromSegment(currentDocId);
+        return new StarTreeDocument(dimensions, metrics);
+    }
+
+    /**
+     * Returns the dimension values for the next document from the segment
+     *
+     * @return dimension values for each of the star-tree dimension
+     * @throws IOException when we are unable to iterate to the next doc for the given dimension readers
+     */
+    private Long[] getStarTreeDimensionsFromSegment(int currentDocId) throws IOException {
+        Long[] dimensions = new Long[numDimensions];
+        for (int i = 0; i < numDimensions; i++) {
+            try {
+                dimensions[i] = getValuesFromSegment(dimensionReaders[i], currentDocId);
+            } catch (Exception e) {
+                logger.error("unable to read the dimension values from the segment", e);
+                throw new IllegalStateException("unable to read the dimension values from the segment", e);
+            }
+
+        }
+        return dimensions;
+    }
+
+    /**
+     * Returns the next value from the iterator of respective field
+     *
+     * @param iterator     respective field iterator
+     * @param currentDocId current document id
+     * @return the next value for the field
+     * @throws IOException when we are unable to iterate to the next doc for the given iterator
+     */
+    private Long getValuesFromSegment(SequentialDocValuesIterator iterator, int currentDocId) throws IOException {
+        try {
+            starTreeDocValuesIteratorAdapter.nextDoc(iterator, currentDocId);
+        } catch (IOException e) {
+            logger.error("unable to iterate to next doc", e);
+            throw new RuntimeException("unable to iterate to next doc", e);
+        }
+        return starTreeDocValuesIteratorAdapter.getNextValue(iterator, currentDocId);
+    }
+
+    /**
+     * Returns the metric values for the next document from the segment
+     *
+     * @return metric values for each of the star-tree metric
+     * @throws IOException when we are unable to iterate to the next doc for the given metric readers
+     */
+    private Object[] getStarTreeMetricsFromSegment(int currentDocId) throws IOException {
+        Object[] metrics = new Object[numMetrics];
+        for (int i = 0; i < numMetrics; i++) {
+            SequentialDocValuesIterator metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader();
+            if (metricStatReader != null) {
+                try {
+                    metrics[i] = getValuesFromSegment(metricStatReader, currentDocId);
+                } catch (Exception e) {
+                    logger.error("unable to read the metric values from the segment", e);
+                    throw new IllegalStateException("unable to read the metric values from the segment", e);
+                }
+            } else {
+                throw new IllegalStateException("metric readers are empty");
+            }
+        }
+        return metrics;
+    }
+
+    /**
+     * Merges a star-tree document from the segment into an aggregated star-tree document.
+     * A new aggregated star-tree document is created if the aggregated segment document is null.
+     *
+     * @param aggregatedSegmentDocument aggregated star-tree document
+     * @param segmentDocument           segment star-tree document
+     * @return merged star-tree document
+     */
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    protected StarTreeDocument reduceSegmentStarTreeDocuments(
+        StarTreeDocument aggregatedSegmentDocument,
+        StarTreeDocument segmentDocument
+    ) {
+        if (aggregatedSegmentDocument == null) {
+            Long[] dimensions = Arrays.copyOf(segmentDocument.dimensions, numDimensions);
+            Object[] metrics = new Object[numMetrics];
+            for (int i = 0; i < numMetrics; i++) {
+                try {
+                    ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators();
+                    StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType();
+                    metrics[i] = metricValueAggregator.getInitialAggregatedValueForSegmentDocValue(
+                        getLong(segmentDocument.metrics[i]),
+                        starTreeNumericType
+                    );
+                } catch (Exception e) {
+                    logger.error("Cannot parse initial segment doc value", e);
+                    throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]");
+                }
+            }
+            return new StarTreeDocument(dimensions, metrics);
+        } else {
+            for (int i = 0; i < numMetrics; i++) {
+                try {
+                    ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators();
+                    StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType();
+                    aggregatedSegmentDocument.metrics[i] = metricValueAggregator.mergeAggregatedValueAndSegmentValue(
+                        aggregatedSegmentDocument.metrics[i],
+                        getLong(segmentDocument.metrics[i]),
+                        starTreeNumericType
+                    );
+                } catch (Exception e) {
+                    logger.error("Cannot apply segment doc value for aggregation", e);
+                    throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]");
+                }
+            }
+            return aggregatedSegmentDocument;
+        }
+    }
+
+    /**
+     * Safely converts the metric value of object type to long.
+     *
+     * @param metric value of the metric
+     * @return converted metric value to long
+     */
+    private static long getLong(Object metric) {
+
+        Long metricValue = null;
+        try {
+            if (metric instanceof Long) {
+                metricValue = (long) metric;
+            } else if (metric != null) {
+                metricValue = Long.valueOf(String.valueOf(metric));
+            }
+        } catch (Exception e) {
+            throw new IllegalStateException("unable to cast segment metric", e);
+        }
+
+        if (metricValue == null) {
+            throw new IllegalStateException("unable to cast segment metric");
+        }
+        return metricValue;
+    }
+
+    /**
+     * Merges a star-tree document into an aggregated star-tree document.
+     * A new aggregated star-tree document is created if the aggregated document is null.
+     *
+     * @param aggregatedDocument aggregated star-tree document
+     * @param starTreeDocument   segment star-tree document
+     * @return merged star-tree document
+     */
+    @SuppressWarnings("unchecked")
+    public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocument, StarTreeDocument starTreeDocument) {
+        // aggregate the documents
+        if (aggregatedDocument == null) {
+            Long[] dimensions = Arrays.copyOf(starTreeDocument.dimensions, numDimensions);
+            Object[] metrics = new Object[numMetrics];
+            for (int i = 0; i < numMetrics; i++) {
+                try {
+                    metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().getInitialAggregatedValue(starTreeDocument.metrics[i]);
+                } catch (Exception e) {
+                    logger.error("Cannot get value for aggregation", e);
+                    throw new IllegalStateException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]");
+                }
+            }
+            return new StarTreeDocument(dimensions, metrics);
+        } else {
+            for (int i = 0; i < numMetrics; i++) {
+                try {
+                    aggregatedDocument.metrics[i] = metricAggregatorInfos.get(i)
+                        .getValueAggregators()
+                        .mergeAggregatedValues(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]);
+                } catch (Exception e) {
+                    logger.error("Cannot apply value to aggregated document for aggregation", e);
+                    throw new IllegalStateException(
+                        "Cannot apply value to aggregated document for aggregation [" + starTreeDocument.metrics[i] + "]"
+                    );
+                }
+            }
+            return aggregatedDocument;
+        }
+    }
+
+    /**
+     * Builds the star tree using total segment documents
+     *
+     * @throws IOException when we are unable to build star-tree
+     */
+    public void build() throws IOException {
+        long startTime = System.currentTimeMillis();
+        logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName());
+
+        if (totalSegmentDocs == 0) {
+            logger.debug("No documents found in the segment");
+            return;
+        }
+
+        Iterator<StarTreeDocument> starTreeDocumentIterator = sortAndAggregateStarTreeDocuments();
+        logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime));
+        build(starTreeDocumentIterator);
+        logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime));
+    }
+
+    /**
+     * Builds the star tree using Star-Tree Document
+     *
+     * @param starTreeDocumentIterator contains the sorted and aggregated documents
+     * @throws IOException when we are unable to build star-tree
+     */
+    void build(Iterator<StarTreeDocument> starTreeDocumentIterator) throws IOException {
+        int numSegmentStarTreeDocument = totalSegmentDocs;
+
+        while (starTreeDocumentIterator.hasNext()) {
+            appendToStarTree(starTreeDocumentIterator.next());
+        }
+        int numStarTreeDocument = numStarTreeDocs;
+        logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument);
+
+        if (numStarTreeDocs == 0) {
+            // TODO: Uncomment when segment codec and file formats is ready
+            // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes);
+            return;
+        }
+
+        constructStarTree(rootNode, 0, numStarTreeDocs);
+        int numStarTreeDocumentUnderStarNode = numStarTreeDocs - numStarTreeDocument;
+        logger.debug(
+            "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node",
+            numStarTreeNodes,
+            numStarTreeDocumentUnderStarNode
+        );
+
+        createAggregatedDocs(rootNode);
+        int numAggregatedStarTreeDocument = numStarTreeDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode;
+        logger.debug("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument);
+
+        // TODO: When StarTree Codec is ready
+        // Create doc values indices in disk
+        // Serialize and save in disk
+        // Write star tree metadata for off heap implementation
+
+    }
+
+    /**
+     * Adds a document to star-tree
+     *
+     * @param starTreeDocument star-tree document
+     * @throws IOException throws an exception if we are unable to add the doc
+     */
+    private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException {
+        appendStarTreeDocument(starTreeDocument);
+        numStarTreeDocs++;
+    }
+
+    /**
+     * Returns a new star-tree node
+     *
+     * @return return new star-tree node
+     */
+    private TreeNode getNewNode() {
+        numStarTreeNodes++;
+        return new TreeNode();
+    }
+
+    /**
+     * Implements the algorithm to construct a star-tree
+     *
+     * @param node       star-tree node
+     * @param startDocId start document id
+     * @param endDocId   end document id
+     * @throws IOException throws an exception if we are unable to construct the tree
+     */
+    private void constructStarTree(TreeNode node, int startDocId, int endDocId) throws IOException {
+
+        int childDimensionId = node.dimensionId + 1;
+        if (childDimensionId == numDimensions) {
+            return;
+        }
+
+        // Construct all non-star children nodes
+        node.childDimensionId = childDimensionId;
+        Map<Long, TreeNode> children = constructNonStarNodes(startDocId, endDocId, childDimensionId);
+        node.children = children;
+
+        // Construct star-node if required
+        if (!skipStarNodeCreationForDimensions.contains(childDimensionId) && children.size() > 1) {
+            children.put((long) ALL, constructStarNode(startDocId, endDocId, childDimensionId));
+        }
+
+        // Further split on child nodes if required
+        for (TreeNode child : children.values()) {
+            if (child.endDocId - child.startDocId > maxLeafDocuments) {
+                constructStarTree(child, child.startDocId, child.endDocId);
+            }
+        }
+    }
+
+    /**
+     * Constructs non star tree nodes
+     *
+     * @param startDocId  start document id (inclusive)
+     * @param endDocId    end document id (exclusive)
+     * @param dimensionId id of the dimension in the star tree
+     * @return root node with non-star nodes constructed
+     * @throws IOException throws an exception if we are unable to construct non-star nodes
+     */
+    private Map<Long, TreeNode> constructNonStarNodes(int startDocId, int endDocId, int dimensionId) throws IOException {
+        Map<Long, TreeNode> nodes = new HashMap<>();
+        int nodeStartDocId = startDocId;
+        Long nodeDimensionValue = getDimensionValue(startDocId, dimensionId);
+        for (int i = startDocId + 1; i < endDocId; i++) {
+            Long dimensionValue = getDimensionValue(i, dimensionId);
+            if (!dimensionValue.equals(nodeDimensionValue)) {
+                TreeNode child = getNewNode();
+                child.dimensionId = dimensionId;
+                child.dimensionValue = nodeDimensionValue;
+                child.startDocId = nodeStartDocId;
+                child.endDocId = i;
+                nodes.put(nodeDimensionValue, child);
+
+                nodeStartDocId = i;
+                nodeDimensionValue = dimensionValue;
+            }
+        }
+        TreeNode lastNode = getNewNode();
+        lastNode.dimensionId = dimensionId;
+        lastNode.dimensionValue = nodeDimensionValue;
+        lastNode.startDocId = nodeStartDocId;
+        lastNode.endDocId = endDocId;
+        nodes.put(nodeDimensionValue, lastNode);
+        return nodes;
+    }
+
+    /**
+     * Constructs star tree nodes
+     *
+     * @param startDocId  start document id (inclusive)
+     * @param endDocId    end document id (exclusive)
+     * @param dimensionId id of the dimension in the star tree
+     * @return root node with star nodes constructed
+     * @throws IOException throws an exception if we are unable to construct non-star nodes
+     */
+    private TreeNode constructStarNode(int startDocId, int endDocId, int dimensionId) throws IOException {
+        TreeNode starNode = getNewNode();
+        starNode.dimensionId = dimensionId;
+        starNode.dimensionValue = ALL;
+        starNode.isStarNode = true;
+        starNode.startDocId = numStarTreeDocs;
+        Iterator<StarTreeDocument> starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId);
+        while (starTreeDocumentIterator.hasNext()) {
+            appendToStarTree(starTreeDocumentIterator.next());
+        }
+        starNode.endDocId = numStarTreeDocs;
+        return starNode;
+    }
+
+    /**
+     * Returns aggregated star-tree document
+     *
+     * @param node star-tree node
+     * @return aggregated star-tree documents
+     * @throws IOException throws an exception upon failing to create new aggregated docs based on star tree
+     */
+    private StarTreeDocument createAggregatedDocs(TreeNode node) throws IOException {
+        StarTreeDocument aggregatedStarTreeDocument = null;
+        if (node.children == null) {
+
+            // For leaf node
+            if (node.startDocId == node.endDocId - 1) {
+                // If it has only one document, use it as the aggregated document
+                aggregatedStarTreeDocument = getStarTreeDocument(node.startDocId);
+                node.aggregatedDocId = node.startDocId;
+            } else {
+                // If it has multiple documents, aggregate all of them
+                for (int i = node.startDocId; i < node.endDocId; i++) {
+                    aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i));
+                }
+                if (null == aggregatedStarTreeDocument) {
+                    throw new IllegalStateException("aggregated star-tree document is null after reducing the documents");
+                }
+                for (int i = node.dimensionId + 1; i < numDimensions; i++) {
+                    aggregatedStarTreeDocument.dimensions[i] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX);
+                }
+                node.aggregatedDocId = numStarTreeDocs;
+                appendToStarTree(aggregatedStarTreeDocument);
+            }
+        } else {
+            // For non-leaf node
+            if (node.children.containsKey((long) ALL)) {
+                // If it has star child, use the star child aggregated document directly
+                for (TreeNode child : node.children.values()) {
+                    if (child.isStarNode) {
+                        aggregatedStarTreeDocument = createAggregatedDocs(child);
+                        node.aggregatedDocId = child.aggregatedDocId;
+                    } else {
+                        createAggregatedDocs(child);
+                    }
+                }
+            } else {
+                // If no star child exists, aggregate all aggregated documents from non-star children
+                if (node.children.values().size() == 1) {
+                    for (TreeNode child : node.children.values()) {
+                        aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child));
+                        node.aggregatedDocId = child.aggregatedDocId;
+                    }
+                } else {
+                    for (TreeNode child : node.children.values()) {
+                        aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child));
+                    }
+                    if (null == aggregatedStarTreeDocument) {
+                        throw new IllegalStateException("aggregated star-tree document is null after reducing the documents");
+                    }
+                    for (int i = node.dimensionId + 1; i < numDimensions; i++) {
+                        aggregatedStarTreeDocument.dimensions[i] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX);
+                    }
+                    node.aggregatedDocId = numStarTreeDocs;
+                    appendToStarTree(aggregatedStarTreeDocument);
+                }
+            }
+        }
+        return aggregatedStarTreeDocument;
+    }
+
+    /**
+     * Handles the dimension of date time field type
+     *
+     * @param fieldName name of the field
+     * @param val       value of the field
+     * @return returns the converted dimension of the field to a particular granularity
+     */
+    private long handleDateDimension(final String fieldName, final long val) {
+        // TODO: handle timestamp granularity
+        return val;
+    }
+
+    public void close() throws IOException {
+
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java
new file mode 100644
index 0000000000000..caeb24838da62
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java
@@ -0,0 +1,213 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.SegmentWriteState;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.mapper.MapperService;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * On heap single tree builder
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder {
+
+    private final List<StarTreeDocument> starTreeDocuments = new ArrayList<>();
+
+    /**
+     * Constructor for OnHeapStarTreeBuilder
+     *
+     * @param starTreeField     star-tree field
+     * @param fieldProducerMap  helps with document values producer for a particular field
+     * @param segmentWriteState segment write state
+     * @param mapperService     helps with the numeric type of field
+     * @throws IOException throws an exception we are unable to construct an onheap star-tree
+     */
+    public OnHeapStarTreeBuilder(
+        StarTreeField starTreeField,
+        Map<String, DocValuesProducer> fieldProducerMap,
+        SegmentWriteState segmentWriteState,
+        MapperService mapperService
+    ) throws IOException {
+        super(starTreeField, fieldProducerMap, segmentWriteState, mapperService);
+    }
+
+    @Override
+    public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {
+        starTreeDocuments.add(starTreeDocument);
+    }
+
+    @Override
+    public StarTreeDocument getStarTreeDocument(int docId) throws IOException {
+        return starTreeDocuments.get(docId);
+    }
+
+    @Override
+    public List<StarTreeDocument> getStarTreeDocuments() {
+        return starTreeDocuments;
+    }
+
+    @Override
+    public Long getDimensionValue(int docId, int dimensionId) throws IOException {
+        return starTreeDocuments.get(docId).dimensions[dimensionId];
+    }
+
+    @Override
+    public Iterator<StarTreeDocument> sortAndAggregateStarTreeDocuments() throws IOException {
+        int numDocs = totalSegmentDocs;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs];
+        for (int currentDocId = 0; currentDocId < numDocs; currentDocId++) {
+            starTreeDocuments[currentDocId] = getSegmentStarTreeDocument(currentDocId);
+        }
+
+        return sortAndAggregateStarTreeDocuments(starTreeDocuments);
+    }
+
+    /**
+     * Sort, aggregates and merges the star-tree documents
+     *
+     * @param starTreeDocuments star-tree documents
+     * @return iterator for star-tree documents
+     */
+    Iterator<StarTreeDocument> sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments) {
+
+        // sort all the documents
+        sortStarTreeDocumentsFromDimensionId(starTreeDocuments, 0);
+
+        // merge the documents
+        return mergeStarTreeDocuments(starTreeDocuments);
+    }
+
+    /**
+     * Merges the star-tree documents
+     *
+     * @param starTreeDocuments star-tree documents
+     * @return iterator to aggregate star-tree documents
+     */
+    private Iterator<StarTreeDocument> mergeStarTreeDocuments(StarTreeDocument[] starTreeDocuments) {
+        return new Iterator<>() {
+            boolean hasNext = true;
+            StarTreeDocument currentStarTreeDocument = starTreeDocuments[0];
+            // starting from 1 since we have already fetched the 0th document
+            int docId = 1;
+
+            @Override
+            public boolean hasNext() {
+                return hasNext;
+            }
+
+            @Override
+            public StarTreeDocument next() {
+                // aggregate as we move on to the next doc
+                StarTreeDocument next = reduceSegmentStarTreeDocuments(null, currentStarTreeDocument);
+                while (docId < starTreeDocuments.length) {
+                    StarTreeDocument starTreeDocument = starTreeDocuments[docId];
+                    docId++;
+                    if (Arrays.equals(starTreeDocument.dimensions, next.dimensions) == false) {
+                        currentStarTreeDocument = starTreeDocument;
+                        return next;
+                    } else {
+                        next = reduceSegmentStarTreeDocuments(next, starTreeDocument);
+                    }
+                }
+                hasNext = false;
+                return next;
+            }
+        };
+    }
+
+    /**
+     * Generates a star-tree for a given star-node
+     *
+     * @param startDocId  Start document id in the star-tree
+     * @param endDocId    End document id (exclusive) in the star-tree
+     * @param dimensionId Dimension id of the star-node
+     * @return iterator for star-tree documents of star-node
+     * @throws IOException throws when unable to generate star-tree for star-node
+     */
+    @Override
+    public Iterator<StarTreeDocument> generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId)
+        throws IOException {
+        int numDocs = endDocId - startDocId;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs];
+        for (int i = 0; i < numDocs; i++) {
+            starTreeDocuments[i] = getStarTreeDocument(startDocId + i);
+        }
+
+        // sort star tree documents from given dimension id (as previous dimension ids have already been processed)
+        sortStarTreeDocumentsFromDimensionId(starTreeDocuments, dimensionId + 1);
+
+        return new Iterator<StarTreeDocument>() {
+            boolean hasNext = true;
+            StarTreeDocument currentStarTreeDocument = starTreeDocuments[0];
+            int docId = 1;
+
+            private boolean hasSameDimensions(StarTreeDocument starTreeDocument1, StarTreeDocument starTreeDocument2) {
+                for (int i = dimensionId + 1; i < numDimensions; i++) {
+                    if (!Objects.equals(starTreeDocument1.dimensions[i], starTreeDocument2.dimensions[i])) {
+                        return false;
+                    }
+                }
+                return true;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return hasNext;
+            }
+
+            @Override
+            public StarTreeDocument next() {
+                StarTreeDocument next = reduceStarTreeDocuments(null, currentStarTreeDocument);
+                next.dimensions[dimensionId] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX);
+                while (docId < numDocs) {
+                    StarTreeDocument starTreeDocument = starTreeDocuments[docId];
+                    docId++;
+                    if (!hasSameDimensions(starTreeDocument, currentStarTreeDocument)) {
+                        currentStarTreeDocument = starTreeDocument;
+                        return next;
+                    } else {
+                        next = reduceStarTreeDocuments(next, starTreeDocument);
+                    }
+                }
+                hasNext = false;
+                return next;
+            }
+        };
+    }
+
+    /**
+     * Sorts the star-tree documents from the given dimension id
+     *
+     * @param starTreeDocuments star-tree documents
+     * @param dimensionId id of the dimension
+     */
+    private void sortStarTreeDocumentsFromDimensionId(StarTreeDocument[] starTreeDocuments, int dimensionId) {
+        Arrays.sort(starTreeDocuments, (o1, o2) -> {
+            for (int i = dimensionId; i < numDimensions; i++) {
+                if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) {
+                    return Long.compare(o1.dimensions[i], o2.dimensions[i]);
+                }
+            }
+            return 0;
+        });
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java
new file mode 100644
index 0000000000000..20af1b3bc7935
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java
@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * A star-tree builder that builds a single star-tree.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface StarTreeBuilder extends Closeable {
+
+    /**
+     * Builds the star tree based on star-tree field
+     * @throws IOException when we are unable to build star-tree
+     */
+    void build() throws IOException;
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java
new file mode 100644
index 0000000000000..cb0350bb110b0
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java
@@ -0,0 +1,82 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
+
+import java.io.IOException;
+
+/**
+ * A factory class to return respective doc values iterator based on the doc volues type.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreeDocValuesIteratorAdapter {
+
+    /**
+     * Creates an iterator for the given doc values type and field using the doc values producer
+     */
+    public SequentialDocValuesIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer)
+        throws IOException {
+        switch (type) {
+            case SORTED_NUMERIC:
+                return new SequentialDocValuesIterator(producer.getSortedNumeric(field));
+            default:
+                throw new IllegalArgumentException("Unsupported DocValuesType: " + type);
+        }
+    }
+
+    /**
+     * Returns the next value for the given iterator
+     */
+    public Long getNextValue(SequentialDocValuesIterator sequentialDocValuesIterator, int currentDocId) throws IOException {
+        if (sequentialDocValuesIterator.getDocIdSetIterator() instanceof SortedNumericDocValues) {
+            SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) sequentialDocValuesIterator.getDocIdSetIterator();
+            if (sequentialDocValuesIterator.getDocId() < 0 || sequentialDocValuesIterator.getDocId() == DocIdSetIterator.NO_MORE_DOCS) {
+                throw new IllegalStateException("invalid doc id to fetch the next value");
+            }
+
+            if (sequentialDocValuesIterator.getDocValue() == null) {
+                sequentialDocValuesIterator.setDocValue(sortedNumericDocValues.nextValue());
+                return sequentialDocValuesIterator.getDocValue();
+            }
+
+            if (sequentialDocValuesIterator.getDocId() == currentDocId) {
+                Long nextValue = sequentialDocValuesIterator.getDocValue();
+                sequentialDocValuesIterator.setDocValue(null);
+                return nextValue;
+            } else {
+                return null;
+            }
+        } else {
+            throw new IllegalStateException("Unsupported Iterator: " + sequentialDocValuesIterator.getDocIdSetIterator().toString());
+        }
+    }
+
+    /**
+     * Moves to the next doc in the iterator
+     * Returns the doc id for the next document from the given iterator
+     */
+    public int nextDoc(SequentialDocValuesIterator iterator, int currentDocId) throws IOException {
+        if (iterator.getDocValue() != null) {
+            return iterator.getDocId();
+        }
+        iterator.setDocId(iterator.getDocIdSetIterator().nextDoc());
+        iterator.setDocValue(this.getNextValue(iterator, currentDocId));
+        return iterator.getDocId();
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java
new file mode 100644
index 0000000000000..eaf9ae1dcdaa1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java
@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.SegmentWriteState;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.mapper.CompositeMappedFieldType;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.StarTreeMapper;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Builder to construct star-trees based on multiple star-tree fields.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class StarTreesBuilder implements Closeable {
+
+    private static final Logger logger = LogManager.getLogger(StarTreesBuilder.class);
+
+    private final List<StarTreeField> starTreeFields;
+    private final SegmentWriteState state;
+    private final Map<String, DocValuesProducer> fieldProducerMap;
+    private final MapperService mapperService;
+
+    public StarTreesBuilder(
+        Map<String, DocValuesProducer> fieldProducerMap,
+        SegmentWriteState segmentWriteState,
+        MapperService mapperService
+    ) {
+        List<StarTreeField> starTreeFields = new ArrayList<>();
+        for (CompositeMappedFieldType compositeMappedFieldType : mapperService.getCompositeFieldTypes()) {
+            if (compositeMappedFieldType instanceof StarTreeMapper.StarTreeFieldType) {
+                StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) compositeMappedFieldType;
+                starTreeFields.add(
+                    new StarTreeField(
+                        starTreeFieldType.name(),
+                        starTreeFieldType.getDimensions(),
+                        starTreeFieldType.getMetrics(),
+                        starTreeFieldType.getStarTreeConfig()
+                    )
+                );
+            }
+        }
+
+        this.starTreeFields = starTreeFields;
+        this.fieldProducerMap = fieldProducerMap;
+        this.state = segmentWriteState;
+        this.mapperService = mapperService;
+    }
+
+    /**
+     * Builds the star-trees.
+     */
+    public void build() throws IOException {
+        if (starTreeFields.isEmpty()) {
+            logger.debug("no star-tree fields found, returning from star-tree builder");
+            return;
+        }
+        long startTime = System.currentTimeMillis();
+        int numStarTrees = starTreeFields.size();
+        logger.debug("Starting building {} star-trees with star-tree fields", numStarTrees);
+
+        // Build all star-trees
+        for (StarTreeField starTreeField : starTreeFields) {
+            try (StarTreeBuilder starTreeBuilder = getStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService)) {
+                starTreeBuilder.build();
+            }
+        }
+        logger.debug("Took {} ms to building {} star-trees with star-tree fields", System.currentTimeMillis() - startTime, numStarTrees);
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    StarTreeBuilder getStarTreeBuilder(
+        StarTreeField starTreeField,
+        Map<String, DocValuesProducer> fieldProducerMap,
+        SegmentWriteState state,
+        MapperService mapperService
+    ) throws IOException {
+        switch (starTreeField.getStarTreeConfig().getBuildMode()) {
+            case ON_HEAP:
+                return new OnHeapStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService);
+            default:
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "No star tree implementation is available for [%s] build mode",
+                        starTreeField.getStarTreeConfig().getBuildMode()
+                    )
+                );
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java
new file mode 100644
index 0000000000000..9c97b076371a3
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Builders for Composite Index Star Tree
+ *
+ * @opensearch.experimental
+ */
+package org.opensearch.index.compositeindex.datacube.startree.builder;
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java
index 4f4e670478e2f..6d6cb420f4a9e 100644
--- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java
@@ -7,5 +7,7 @@
  */
 /**
  * Core classes for handling star tree index.
+ *
+ * @opensearch.experimental
  */
 package org.opensearch.index.compositeindex.datacube.startree;
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java
new file mode 100644
index 0000000000000..cf5f3e94c1ca6
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java
@@ -0,0 +1,137 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.utils;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Coordinates the reading of documents across multiple DocIdSetIterators.
+ * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class SequentialDocValuesIterator {
+
+    /**
+     * The doc id set iterator associated for each field.
+     */
+    private final DocIdSetIterator docIdSetIterator;
+
+    /**
+     * The value associated with the latest document.
+     */
+    private Long docValue;
+
+    /**
+     * The id of the latest document.
+     */
+    private int docId;
+
+    /**
+     * Constructs a new SequentialDocValuesIterator instance with the given DocIdSetIterator.
+     *
+     * @param docIdSetIterator the DocIdSetIterator to be associated with this instance
+     */
+    public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) {
+        this.docIdSetIterator = docIdSetIterator;
+    }
+
+    /**
+     * Constructs a new SequentialDocValuesIterator instance with the given SortedNumericDocValues.
+     *
+     */
+    public SequentialDocValuesIterator() {
+        this.docIdSetIterator = new SortedNumericDocValues() {
+            @Override
+            public long nextValue() throws IOException {
+                return 0;
+            }
+
+            @Override
+            public int docValueCount() {
+                return 0;
+            }
+
+            @Override
+            public boolean advanceExact(int i) throws IOException {
+                return false;
+            }
+
+            @Override
+            public int docID() {
+                return 0;
+            }
+
+            @Override
+            public int nextDoc() throws IOException {
+                return 0;
+            }
+
+            @Override
+            public int advance(int i) throws IOException {
+                return 0;
+            }
+
+            @Override
+            public long cost() {
+                return 0;
+            }
+        };
+    }
+
+    /**
+     * Returns the value associated with the latest document.
+     *
+     * @return the value associated with the latest document
+     */
+    public Long getDocValue() {
+        return docValue;
+    }
+
+    /**
+     * Sets the value associated with the latest document.
+     *
+     * @param docValue the value to be associated with the latest document
+     */
+    public void setDocValue(Long docValue) {
+        this.docValue = docValue;
+    }
+
+    /**
+     * Returns the id of the latest document.
+     *
+     * @return the id of the latest document
+     */
+    public int getDocId() {
+        return docId;
+    }
+
+    /**
+     * Sets the id of the latest document.
+     *
+     * @param docId the ID of the latest document
+     */
+    public void setDocId(int docId) {
+        this.docId = docId;
+    }
+
+    /**
+     * Returns the DocIdSetIterator associated with this instance.
+     *
+     * @return the DocIdSetIterator associated with this instance
+     */
+    public DocIdSetIterator getDocIdSetIterator() {
+        return docIdSetIterator;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java
new file mode 100644
index 0000000000000..5cf737c61ab2d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.utils;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.Map;
+
+/**
+ * /**
+ * Represents a node in a tree data structure, specifically designed for a star-tree implementation.
+ * A star-tree node will represent both star and non-star nodes.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class TreeNode {
+
+    public static final int ALL = -1;
+
+    /**
+     * The dimension id for the dimension (field) associated with this star-tree node.
+     */
+    public int dimensionId = ALL;
+
+    /**
+     * The starting document id (inclusive) associated with this star-tree node.
+     */
+    public int startDocId = ALL;
+
+    /**
+     * The ending document id (exclusive) associated with this star-tree node.
+     */
+    public int endDocId = ALL;
+
+    /**
+     * The aggregated document id associated with this star-tree node.
+     */
+    public int aggregatedDocId = ALL;
+
+    /**
+     * The child dimension identifier associated with this star-tree node.
+     */
+    public int childDimensionId = ALL;
+
+    /**
+     * The value of the dimension associated with this star-tree node.
+     */
+    public long dimensionValue = ALL;
+
+    /**
+     * A flag indicating whether this node is a star node (a node that represents an aggregation of all dimensions).
+     */
+    public boolean isStarNode = false;
+
+    /**
+     * A map containing the child nodes of this star-tree node, keyed by their dimension id.
+     */
+    public Map<Long, TreeNode> children;
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java
new file mode 100644
index 0000000000000..c7e8b04d42178
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Utility to support Composite Index Star Tree
+ *
+ * @opensearch.experimental
+ */
+package org.opensearch.index.compositeindex.datacube.startree.utils;
diff --git a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java
index 6c6d26656e4de..31df9a49bebfb 100644
--- a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java
+++ b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java
@@ -12,12 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.lucene99.Lucene99Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.SortedNumericDocValuesField;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
-import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.common.Rounding;
 import org.opensearch.index.codec.composite.Composite99Codec;
@@ -31,7 +26,6 @@
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.StarTreeMapper;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -77,34 +71,4 @@ private static StarTreeField getStarTreeField(List<Rounding.DateTimeUnit> d1Cale
 
         return new StarTreeField("starTree", dims, metrics, config);
     }
-
-    public void testStarTreeDocValues() throws IOException {
-        Directory directory = newDirectory();
-        IndexWriterConfig conf = newIndexWriterConfig(null);
-        conf.setMergePolicy(newLogMergePolicy());
-        RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
-        Document doc = new Document();
-        doc.add(new SortedNumericDocValuesField("sndv", 1));
-        doc.add(new SortedNumericDocValuesField("dv", 1));
-        doc.add(new SortedNumericDocValuesField("field", 1));
-        iw.addDocument(doc);
-        doc.add(new SortedNumericDocValuesField("sndv", 1));
-        doc.add(new SortedNumericDocValuesField("dv", 1));
-        doc.add(new SortedNumericDocValuesField("field", 1));
-        iw.addDocument(doc);
-        iw.forceMerge(1);
-        doc.add(new SortedNumericDocValuesField("sndv", 2));
-        doc.add(new SortedNumericDocValuesField("dv", 2));
-        doc.add(new SortedNumericDocValuesField("field", 2));
-        iw.addDocument(doc);
-        doc.add(new SortedNumericDocValuesField("sndv", 2));
-        doc.add(new SortedNumericDocValuesField("dv", 2));
-        doc.add(new SortedNumericDocValuesField("field", 2));
-        iw.addDocument(doc);
-        iw.forceMerge(1);
-        iw.close();
-
-        // TODO : validate star tree structures that got created
-        directory.close();
-    }
 }
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java
new file mode 100644
index 0000000000000..e30e203406a6c
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class CountValueAggregatorTests extends OpenSearchTestCase {
+    private final CountValueAggregator aggregator = new CountValueAggregator();
+
+    public void testGetAggregationType() {
+        assertEquals(MetricStat.COUNT.getTypeName(), aggregator.getAggregationType().getTypeName());
+    }
+
+    public void testGetAggregatedValueType() {
+        assertEquals(CountValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType());
+    }
+
+    public void testGetInitialAggregatedValueForSegmentDocValue() {
+        assertEquals(1L, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong(), StarTreeNumericType.LONG), 0.0);
+    }
+
+    public void testMergeAggregatedValueAndSegmentValue() {
+        assertEquals(3L, aggregator.mergeAggregatedValueAndSegmentValue(2L, 3L, StarTreeNumericType.LONG), 0.0);
+    }
+
+    public void testMergeAggregatedValues() {
+        assertEquals(5L, aggregator.mergeAggregatedValues(2L, 3L), 0.0);
+    }
+
+    public void testGetInitialAggregatedValue() {
+        assertEquals(3L, aggregator.getInitialAggregatedValue(3L), 0.0);
+    }
+
+    public void testGetMaxAggregatedValueByteSize() {
+        assertEquals(Long.BYTES, aggregator.getMaxAggregatedValueByteSize());
+    }
+
+    public void testToLongValue() {
+        assertEquals(3L, aggregator.toLongValue(3L), 0.0);
+    }
+
+    public void testToStarTreeNumericTypeValue() {
+        assertEquals(3L, aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.LONG), 0.0);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java
new file mode 100644
index 0000000000000..d08f637a3f0a9
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java
@@ -0,0 +1,123 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class MetricAggregatorInfoTests extends OpenSearchTestCase {
+
+    public void testConstructor() {
+        MetricAggregatorInfo pair = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column1",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        assertEquals(MetricStat.SUM, pair.getMetricStat());
+        assertEquals("column1", pair.getField());
+    }
+
+    public void testCountStarConstructor() {
+        MetricAggregatorInfo pair = new MetricAggregatorInfo(
+            MetricStat.COUNT,
+            "anything",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        assertEquals(MetricStat.COUNT, pair.getMetricStat());
+        assertEquals("anything", pair.getField());
+    }
+
+    public void testToFieldName() {
+        MetricAggregatorInfo pair = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column2",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        assertEquals("star_tree_field_column2_sum", pair.toFieldName());
+    }
+
+    public void testEquals() {
+        MetricAggregatorInfo pair1 = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column1",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        MetricAggregatorInfo pair2 = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column1",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        assertEquals(pair1, pair2);
+        assertNotEquals(
+            pair1,
+            new MetricAggregatorInfo(MetricStat.COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null)
+        );
+        assertNotEquals(
+            pair1,
+            new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null)
+        );
+    }
+
+    public void testHashCode() {
+        MetricAggregatorInfo pair1 = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column1",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        MetricAggregatorInfo pair2 = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column1",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        assertEquals(pair1.hashCode(), pair2.hashCode());
+    }
+
+    public void testCompareTo() {
+        MetricAggregatorInfo pair1 = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column1",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        MetricAggregatorInfo pair2 = new MetricAggregatorInfo(
+            MetricStat.SUM,
+            "column2",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        MetricAggregatorInfo pair3 = new MetricAggregatorInfo(
+            MetricStat.COUNT,
+            "column1",
+            "star_tree_field",
+            IndexNumericFieldData.NumericType.DOUBLE,
+            null
+        );
+        assertTrue(pair1.compareTo(pair2) < 0);
+        assertTrue(pair2.compareTo(pair1) > 0);
+        assertTrue(pair1.compareTo(pair3) > 0);
+        assertTrue(pair3.compareTo(pair1) < 0);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java
new file mode 100644
index 0000000000000..3fb627e7cd434
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.apache.lucene.util.NumericUtils;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
+
+public class SumValueAggregatorTests extends OpenSearchTestCase {
+
+    private SumValueAggregator aggregator;
+
+    @Before
+    public void setup() {
+        aggregator = new SumValueAggregator();
+    }
+
+    public void testGetAggregationType() {
+        assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName());
+    }
+
+    public void testGetAggregatedValueType() {
+        assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType());
+    }
+
+    public void testGetInitialAggregatedValueForSegmentDocValue() {
+        assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(1L, StarTreeNumericType.LONG), 0.0);
+        assertThrows(
+            NullPointerException.class,
+            () -> aggregator.getInitialAggregatedValueForSegmentDocValue(null, StarTreeNumericType.DOUBLE)
+        );
+    }
+
+    public void testMergeAggregatedValueAndSegmentValue() {
+        aggregator.getInitialAggregatedValue(2.0);
+        assertEquals(5.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0);
+    }
+
+    public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() {
+        aggregator.getInitialAggregatedValue(2.0);
+        assertThrows(NullPointerException.class, () -> aggregator.mergeAggregatedValueAndSegmentValue(2.0, null, StarTreeNumericType.LONG));
+    }
+
+    public void testMergeAggregatedValues() {
+        aggregator.getInitialAggregatedValue(3.0);
+        assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0);
+    }
+
+    public void testGetInitialAggregatedValue() {
+        assertEquals(3.14, aggregator.getInitialAggregatedValue(3.14), 0.0);
+    }
+
+    public void testGetMaxAggregatedValueByteSize() {
+        assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize());
+    }
+
+    public void testToLongValue() {
+        assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.toLongValue(3.14), 0.0);
+    }
+
+    public void testToStarTreeNumericTypeValue() {
+        assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java
new file mode 100644
index 0000000000000..ce61ab839cc61
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class ValueAggregatorFactoryTests extends OpenSearchTestCase {
+
+    public void testGetValueAggregatorForSumType() {
+        ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.SUM);
+        assertNotNull(aggregator);
+        assertEquals(SumValueAggregator.class, aggregator.getClass());
+    }
+
+    public void testGetAggregatedValueTypeForSumType() {
+        StarTreeNumericType starTreeNumericType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM);
+        assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, starTreeNumericType);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java
new file mode 100644
index 0000000000000..b78130e72aba1
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java
@@ -0,0 +1,216 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.Version;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.NumericDimension;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo;
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.DocumentMapper;
+import org.opensearch.index.mapper.Mapper;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.MappingLookup;
+import org.opensearch.index.mapper.NumberFieldMapper;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class BaseStarTreeBuilderTests extends OpenSearchTestCase {
+
+    private static BaseStarTreeBuilder builder;
+    private static MapperService mapperService;
+    private static List<Dimension> dimensionsOrder;
+    private static List<String> fields = List.of(
+        "field1",
+        "field2",
+        "field3",
+        "field4",
+        "field5",
+        "field6",
+        "field7",
+        "field8",
+        "field9",
+        "field10"
+    );
+    private static List<Metric> metrics;
+    private static Directory directory;
+    private static FieldInfo[] fieldsInfo;
+    private static SegmentWriteState state;
+    private static StarTreeField starTreeField;
+
+    @BeforeClass
+    public static void setup() throws IOException {
+
+        dimensionsOrder = List.of(
+            new NumericDimension("field1"),
+            new NumericDimension("field3"),
+            new NumericDimension("field5"),
+            new NumericDimension("field8")
+        );
+        metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM)));
+
+        starTreeField = new StarTreeField(
+            "test",
+            dimensionsOrder,
+            metrics,
+            new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP)
+        );
+
+        DocValuesProducer docValuesProducer = mock(DocValuesProducer.class);
+        directory = newFSDirectory(createTempDir());
+        SegmentInfo segmentInfo = new SegmentInfo(
+            directory,
+            Version.LATEST,
+            Version.LUCENE_9_11_0,
+            "test_segment",
+            5,
+            false,
+            false,
+            new Lucene99Codec(),
+            new HashMap<>(),
+            UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8),
+            new HashMap<>(),
+            null
+        );
+
+        fieldsInfo = new FieldInfo[fields.size()];
+        Map<String, DocValuesProducer> fieldProducerMap = new HashMap<>();
+        for (int i = 0; i < fieldsInfo.length; i++) {
+            fieldsInfo[i] = new FieldInfo(
+                fields.get(i),
+                i,
+                false,
+                false,
+                true,
+                IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
+                DocValuesType.SORTED_NUMERIC,
+                -1,
+                Collections.emptyMap(),
+                0,
+                0,
+                0,
+                0,
+                VectorEncoding.FLOAT32,
+                VectorSimilarityFunction.EUCLIDEAN,
+                false,
+                false
+            );
+            fieldProducerMap.put(fields.get(i), docValuesProducer);
+        }
+        FieldInfos fieldInfos = new FieldInfos(fieldsInfo);
+        state = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random()));
+
+        mapperService = mock(MapperService.class);
+        DocumentMapper documentMapper = mock(DocumentMapper.class);
+        when(mapperService.documentMapper()).thenReturn(documentMapper);
+        Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build();
+        NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        MappingLookup fieldMappers = new MappingLookup(
+            Set.of(numberFieldMapper1, numberFieldMapper2),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            0,
+            null
+        );
+        when(documentMapper.mappers()).thenReturn(fieldMappers);
+
+        builder = new BaseStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService) {
+            @Override
+            public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {}
+
+            @Override
+            public StarTreeDocument getStarTreeDocument(int docId) throws IOException {
+                return null;
+            }
+
+            @Override
+            public List<StarTreeDocument> getStarTreeDocuments() {
+                return List.of();
+            }
+
+            @Override
+            public Long getDimensionValue(int docId, int dimensionId) throws IOException {
+                return 0L;
+            }
+
+            @Override
+            public Iterator<StarTreeDocument> sortAndAggregateStarTreeDocuments() throws IOException {
+                return null;
+            }
+
+            @Override
+            public Iterator<StarTreeDocument> generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId)
+                throws IOException {
+                return null;
+            }
+        };
+    }
+
+    public void test_generateMetricAggregatorInfos() throws IOException {
+        List<MetricAggregatorInfo> metricAggregatorInfos = builder.generateMetricAggregatorInfos(mapperService, state);
+        List<MetricAggregatorInfo> expectedMetricAggregatorInfos = List.of(
+            new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null),
+            new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null)
+        );
+        assertEquals(metricAggregatorInfos, expectedMetricAggregatorInfos);
+    }
+
+    public void test_reduceStarTreeDocuments() {
+        StarTreeDocument starTreeDocument1 = new StarTreeDocument(new Long[] { 1L, 3L, 5L, 8L }, new Double[] { 4.0, 8.0 });
+        StarTreeDocument starTreeDocument2 = new StarTreeDocument(new Long[] { 1L, 3L, 5L, 8L }, new Double[] { 10.0, 6.0 });
+
+        StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new Long[] { 1L, 3L, 5L, 8L }, new Double[] { 14.0, 14.0 });
+        StarTreeDocument mergedStarTreeDocument = builder.reduceStarTreeDocuments(null, starTreeDocument1);
+        StarTreeDocument resultStarTreeDocument = builder.reduceStarTreeDocuments(mergedStarTreeDocument, starTreeDocument2);
+
+        assertEquals(resultStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]);
+        assertEquals(resultStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        directory.close();
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java
new file mode 100644
index 0000000000000..4e107e78d27be
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java
@@ -0,0 +1,706 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.Version;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.NumericDimension;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.DocumentMapper;
+import org.opensearch.index.mapper.Mapper;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.MappingLookup;
+import org.opensearch.index.mapper.NumberFieldMapper;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase {
+
+    private OnHeapStarTreeBuilder builder;
+    private MapperService mapperService;
+    private List<Dimension> dimensionsOrder;
+    private List<String> fields = List.of();
+    private List<Metric> metrics;
+    private Directory directory;
+    private FieldInfo[] fieldsInfo;
+    private StarTreeField compositeField;
+    private Map<String, DocValuesProducer> fieldProducerMap;
+    private SegmentWriteState writeState;
+
+    @Before
+    public void setup() throws IOException {
+        fields = List.of("field1", "field2", "field3", "field4", "field5", "field6", "field7", "field8", "field9", "field10");
+
+        dimensionsOrder = List.of(
+            new NumericDimension("field1"),
+            new NumericDimension("field3"),
+            new NumericDimension("field5"),
+            new NumericDimension("field8")
+        );
+        metrics = List.of(
+            new Metric("field2", List.of(MetricStat.SUM)),
+            new Metric("field4", List.of(MetricStat.SUM)),
+            new Metric("field6", List.of(MetricStat.COUNT))
+        );
+
+        DocValuesProducer docValuesProducer = mock(DocValuesProducer.class);
+
+        compositeField = new StarTreeField(
+            "test",
+            dimensionsOrder,
+            metrics,
+            new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP)
+        );
+        directory = newFSDirectory(createTempDir());
+        SegmentInfo segmentInfo = new SegmentInfo(
+            directory,
+            Version.LATEST,
+            Version.LUCENE_9_11_0,
+            "test_segment",
+            5,
+            false,
+            false,
+            new Lucene99Codec(),
+            new HashMap<>(),
+            UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8),
+            new HashMap<>(),
+            null
+        );
+
+        fieldsInfo = new FieldInfo[fields.size()];
+        fieldProducerMap = new HashMap<>();
+        for (int i = 0; i < fieldsInfo.length; i++) {
+            fieldsInfo[i] = new FieldInfo(
+                fields.get(i),
+                i,
+                false,
+                false,
+                true,
+                IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
+                DocValuesType.SORTED_NUMERIC,
+                -1,
+                Collections.emptyMap(),
+                0,
+                0,
+                0,
+                0,
+                VectorEncoding.FLOAT32,
+                VectorSimilarityFunction.EUCLIDEAN,
+                false,
+                false
+            );
+            fieldProducerMap.put(fields.get(i), docValuesProducer);
+        }
+        FieldInfos fieldInfos = new FieldInfos(fieldsInfo);
+        writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random()));
+
+        mapperService = mock(MapperService.class);
+        DocumentMapper documentMapper = mock(DocumentMapper.class);
+        when(mapperService.documentMapper()).thenReturn(documentMapper);
+        Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build();
+        NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.DOUBLE, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        MappingLookup fieldMappers = new MappingLookup(
+            Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            0,
+            null
+        );
+        when(documentMapper.mappers()).thenReturn(fieldMappers);
+        builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService);
+    }
+
+    public void test_sortAndAggregateStarTreeDocuments() throws IOException {
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble() });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() });
+
+        List<StarTreeDocument> inorderStarTreeDocuments = List.of(
+            new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }),
+            new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L })
+        );
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator();
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]);
+            long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]);
+            long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]);
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        int numOfAggregatedDocuments = 0;
+        while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) {
+            StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next();
+            StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next();
+
+            assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]);
+            assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]);
+            assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]);
+            assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]);
+            assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]);
+            assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]);
+            assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]);
+
+            numOfAggregatedDocuments++;
+        }
+
+        assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments);
+
+    }
+
+    public void test_sortAndAggregateStarTreeDocuments_nullMetric() throws IOException {
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble() });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, null, randomDouble() });
+        StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 });
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            Long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]);
+            Long metric2 = starTreeDocuments[i].metrics[1] != null
+                ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1])
+                : null;
+            Long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]);
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+
+        StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next();
+        assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]);
+        assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]);
+        assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]);
+        assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]);
+        assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]);
+        assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]);
+
+        assertThrows(
+            "Null metric should have resulted in IllegalStateException",
+            IllegalStateException.class,
+            segmentStarTreeDocumentIterator::next
+        );
+
+    }
+
+    public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException {
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 10.0, 6.0, randomDouble() });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 14.0, 12.0, randomDouble() });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 11.0, 16.0, randomDouble() });
+
+        List<StarTreeDocument> inorderStarTreeDocuments = List.of(
+            new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }),
+            new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Object[] { 35.0, 34.0, 3L })
+        );
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator();
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]);
+            long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]);
+            long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]);
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        int numOfAggregatedDocuments = 0;
+        while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) {
+            StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next();
+            StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next();
+
+            assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]);
+            assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]);
+            assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]);
+            assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]);
+            assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]);
+            assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]);
+            assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]);
+
+            numOfAggregatedDocuments++;
+        }
+
+        assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments);
+
+    }
+
+    public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() throws IOException {
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { Double.MAX_VALUE, 10.0, randomDouble() });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, Double.MIN_VALUE, randomDouble() });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() });
+
+        List<StarTreeDocument> inorderStarTreeDocuments = List.of(
+            new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { Double.MAX_VALUE + 9, 14.0, 2L }),
+            new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, Double.MIN_VALUE + 22, 3L })
+        );
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator();
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]);
+            long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]);
+            long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]);
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        int numOfAggregatedDocuments = 0;
+        while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) {
+            StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next();
+            StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next();
+
+            assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]);
+            assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]);
+            assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]);
+            assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]);
+            assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]);
+            assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]);
+            assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]);
+
+            numOfAggregatedDocuments++;
+        }
+
+        assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments);
+
+    }
+
+    public void test_build_halfFloatMetrics() throws IOException {
+
+        mapperService = mock(MapperService.class);
+        DocumentMapper documentMapper = mock(DocumentMapper.class);
+        when(mapperService.documentMapper()).thenReturn(documentMapper);
+        Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build();
+        NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.HALF_FLOAT, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.HALF_FLOAT, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        MappingLookup fieldMappers = new MappingLookup(
+            Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            0,
+            null
+        );
+        when(documentMapper.mappers()).thenReturn(fieldMappers);
+        builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService);
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(
+            new Long[] { 2L, 4L, 3L, 4L },
+            new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10), new HalfFloatPoint("field6", 10) }
+        );
+        starTreeDocuments[1] = new StarTreeDocument(
+            new Long[] { 3L, 4L, 2L, 1L },
+            new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6), new HalfFloatPoint("field6", 10) }
+        );
+        starTreeDocuments[2] = new StarTreeDocument(
+            new Long[] { 3L, 4L, 2L, 1L },
+            new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12), new HalfFloatPoint("field6", 10) }
+        );
+        starTreeDocuments[3] = new StarTreeDocument(
+            new Long[] { 2L, 4L, 3L, 4L },
+            new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4), new HalfFloatPoint("field6", 10) }
+        );
+        starTreeDocuments[4] = new StarTreeDocument(
+            new Long[] { 3L, 4L, 2L, 1L },
+            new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16), new HalfFloatPoint("field6", 10) }
+        );
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = HalfFloatPoint.halfFloatToSortableShort(
+                ((HalfFloatPoint) starTreeDocuments[i].metrics[0]).numericValue().floatValue()
+            );
+            long metric2 = HalfFloatPoint.halfFloatToSortableShort(
+                ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue()
+            );
+            long metric3 = HalfFloatPoint.halfFloatToSortableShort(
+                ((HalfFloatPoint) starTreeDocuments[i].metrics[2]).numericValue().floatValue()
+            );
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        builder.build(segmentStarTreeDocumentIterator);
+
+        List<StarTreeDocument> resultStarTreeDocuments = builder.getStarTreeDocuments();
+        assertEquals(7, resultStarTreeDocuments.size());
+
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator();
+        assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator);
+    }
+
+    public void test_build_floatMetrics() throws IOException {
+
+        mapperService = mock(MapperService.class);
+        DocumentMapper documentMapper = mock(DocumentMapper.class);
+        when(mapperService.documentMapper()).thenReturn(documentMapper);
+        Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build();
+        NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.FLOAT, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.FLOAT, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        MappingLookup fieldMappers = new MappingLookup(
+            Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            0,
+            null
+        );
+        when(documentMapper.mappers()).thenReturn(fieldMappers);
+        builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService);
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Float[] { 12.0F, 10.0F, randomFloat() });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Float[] { 10.0F, 6.0F, randomFloat() });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Float[] { 14.0F, 12.0F, randomFloat() });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Float[] { 9.0F, 4.0F, randomFloat() });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Float[] { 11.0F, 16.0F, randomFloat() });
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]);
+            long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]);
+            long metric3 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[2]);
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        builder.build(segmentStarTreeDocumentIterator);
+
+        List<StarTreeDocument> resultStarTreeDocuments = builder.getStarTreeDocuments();
+        assertEquals(7, resultStarTreeDocuments.size());
+
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator();
+        assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator);
+    }
+
+    public void test_build_longMetrics() throws IOException {
+
+        mapperService = mock(MapperService.class);
+        DocumentMapper documentMapper = mock(DocumentMapper.class);
+        when(mapperService.documentMapper()).thenReturn(documentMapper);
+        Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build();
+        NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.LONG, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.LONG, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        MappingLookup fieldMappers = new MappingLookup(
+            Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            0,
+            null
+        );
+        when(documentMapper.mappers()).thenReturn(fieldMappers);
+        builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService);
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 12L, 10L, randomLong() });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 10L, 6L, randomLong() });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 14L, 12L, randomLong() });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 9L, 4L, randomLong() });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 11L, 16L, randomLong() });
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = (Long) starTreeDocuments[i].metrics[0];
+            long metric2 = (Long) starTreeDocuments[i].metrics[1];
+            long metric3 = (Long) starTreeDocuments[i].metrics[2];
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        builder.build(segmentStarTreeDocumentIterator);
+
+        List<StarTreeDocument> resultStarTreeDocuments = builder.getStarTreeDocuments();
+        assertEquals(7, resultStarTreeDocuments.size());
+
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator();
+        assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator);
+    }
+
+    private static Iterator<StarTreeDocument> getExpectedStarTreeDocumentIterator() {
+        List<StarTreeDocument> expectedStarTreeDocuments = List.of(
+            new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }),
+            new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L }),
+            new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L }),
+            new StarTreeDocument(new Long[] { -1L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }),
+            new StarTreeDocument(new Long[] { -1L, 4L, -1L, 1L }, new Object[] { 35.0, 34.0, 3L }),
+            new StarTreeDocument(new Long[] { -1L, 4L, -1L, 4L }, new Object[] { 21.0, 14.0, 2L }),
+            new StarTreeDocument(new Long[] { -1L, 4L, -1L, -1L }, new Object[] { 56.0, 48.0, 5L })
+        );
+        return expectedStarTreeDocuments.iterator();
+    }
+
+    public void test_build() throws IOException {
+
+        int noOfStarTreeDocuments = 5;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble() });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() });
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]);
+            long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]);
+            long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]);
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        builder.build(segmentStarTreeDocumentIterator);
+
+        List<StarTreeDocument> resultStarTreeDocuments = builder.getStarTreeDocuments();
+        assertEquals(7, resultStarTreeDocuments.size());
+
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator();
+        assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator);
+    }
+
+    private void assertStarTreeDocuments(
+        List<StarTreeDocument> resultStarTreeDocuments,
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator
+    ) {
+        Iterator<StarTreeDocument> resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator();
+        while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) {
+            StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next();
+            StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next();
+
+            assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]);
+            assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]);
+            assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]);
+            assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]);
+            assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]);
+            assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]);
+            assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]);
+        }
+    }
+
+    public void test_build_starTreeDataset() throws IOException {
+
+        fields = List.of("fieldC", "fieldB", "fieldL", "fieldI");
+
+        dimensionsOrder = List.of(new NumericDimension("fieldC"), new NumericDimension("fieldB"), new NumericDimension("fieldL"));
+        metrics = List.of(new Metric("fieldI", List.of(MetricStat.SUM)));
+
+        DocValuesProducer docValuesProducer = mock(DocValuesProducer.class);
+
+        compositeField = new StarTreeField(
+            "test",
+            dimensionsOrder,
+            metrics,
+            new StarTreeFieldConfiguration(1, Set.of(), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP)
+        );
+        SegmentInfo segmentInfo = new SegmentInfo(
+            directory,
+            Version.LATEST,
+            Version.LUCENE_9_11_0,
+            "test_segment",
+            7,
+            false,
+            false,
+            new Lucene99Codec(),
+            new HashMap<>(),
+            UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8),
+            new HashMap<>(),
+            null
+        );
+
+        fieldsInfo = new FieldInfo[fields.size()];
+        fieldProducerMap = new HashMap<>();
+        for (int i = 0; i < fieldsInfo.length; i++) {
+            fieldsInfo[i] = new FieldInfo(
+                fields.get(i),
+                i,
+                false,
+                false,
+                true,
+                IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
+                DocValuesType.SORTED_NUMERIC,
+                -1,
+                Collections.emptyMap(),
+                0,
+                0,
+                0,
+                0,
+                VectorEncoding.FLOAT32,
+                VectorSimilarityFunction.EUCLIDEAN,
+                false,
+                false
+            );
+            fieldProducerMap.put(fields.get(i), docValuesProducer);
+        }
+        FieldInfos fieldInfos = new FieldInfos(fieldsInfo);
+        writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random()));
+
+        mapperService = mock(MapperService.class);
+        DocumentMapper documentMapper = mock(DocumentMapper.class);
+        when(mapperService.documentMapper()).thenReturn(documentMapper);
+        Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build();
+        NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("fieldI", NumberFieldMapper.NumberType.DOUBLE, false, true)
+            .build(new Mapper.BuilderContext(settings, new ContentPath()));
+        MappingLookup fieldMappers = new MappingLookup(
+            Set.of(numberFieldMapper1),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            0,
+            null
+        );
+        when(documentMapper.mappers()).thenReturn(fieldMappers);
+        builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService);
+
+        int noOfStarTreeDocuments = 7;
+        StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        starTreeDocuments[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Double[] { 400.0 });
+        starTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Double[] { 200.0 });
+        starTreeDocuments[2] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Double[] { 300.0 });
+        starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Double[] { 100.0 });
+        starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Double[] { 600.0 });
+        starTreeDocuments[5] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Double[] { 200.0 });
+        starTreeDocuments[6] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Double[] { 400.0 });
+
+        StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments];
+        for (int i = 0; i < noOfStarTreeDocuments; i++) {
+            long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]);
+            segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1 });
+        }
+
+        Iterator<StarTreeDocument> segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments);
+        builder.build(segmentStarTreeDocumentIterator);
+
+        List<StarTreeDocument> resultStarTreeDocuments = builder.getStarTreeDocuments();
+        List<StarTreeDocument> expectedStarTreeDocuments = List.of(
+            new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0 }),
+            new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0 }),
+            new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0 }),
+            new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Object[] { 300.0 }),
+            new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Object[] { 600.0 }),
+            new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Object[] { 400.0 }),
+            new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Object[] { 200.0 }),
+            new StarTreeDocument(new Long[] { -1L, 11L, 21L }, new Object[] { 1000.0 }),
+            new StarTreeDocument(new Long[] { -1L, 12L, 21L }, new Object[] { 400.0 }),
+            new StarTreeDocument(new Long[] { -1L, 12L, 22L }, new Object[] { 200.0 }),
+            new StarTreeDocument(new Long[] { -1L, 12L, 23L }, new Object[] { 200.0 }),
+            new StarTreeDocument(new Long[] { -1L, 13L, 21L }, new Object[] { 100.0 }),
+            new StarTreeDocument(new Long[] { -1L, 13L, 23L }, new Object[] { 300.0 }),
+            new StarTreeDocument(new Long[] { -1L, -1L, 21L }, new Object[] { 1500.0 }),
+            new StarTreeDocument(new Long[] { -1L, -1L, 22L }, new Object[] { 200.0 }),
+            new StarTreeDocument(new Long[] { -1L, -1L, 23L }, new Object[] { 500.0 }),
+            new StarTreeDocument(new Long[] { -1L, -1L, -1L }, new Object[] { 2200.0 }),
+            new StarTreeDocument(new Long[] { -1L, 12L, -1L }, new Object[] { 800.0 }),
+            new StarTreeDocument(new Long[] { -1L, 13L, -1L }, new Object[] { 400.0 }),
+            new StarTreeDocument(new Long[] { 1L, -1L, 21L }, new Object[] { 400.0 }),
+            new StarTreeDocument(new Long[] { 1L, -1L, 22L }, new Object[] { 200.0 }),
+            new StarTreeDocument(new Long[] { 1L, -1L, -1L }, new Object[] { 600.0 }),
+            new StarTreeDocument(new Long[] { 2L, 13L, -1L }, new Object[] { 400.0 }),
+            new StarTreeDocument(new Long[] { 3L, -1L, 21L }, new Object[] { 1000.0 }),
+            new StarTreeDocument(new Long[] { 3L, -1L, 23L }, new Object[] { 200.0 }),
+            new StarTreeDocument(new Long[] { 3L, -1L, -1L }, new Object[] { 1200.0 }),
+            new StarTreeDocument(new Long[] { 3L, 12L, -1L }, new Object[] { 600.0 })
+        );
+
+        Iterator<StarTreeDocument> expectedStarTreeDocumentIterator = expectedStarTreeDocuments.iterator();
+        Iterator<StarTreeDocument> resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator();
+        while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) {
+            StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next();
+            StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next();
+
+            assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]);
+            assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]);
+            assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]);
+            assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]);
+        }
+
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        directory.close();
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java
new file mode 100644
index 0000000000000..9c2621401faa4
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java
@@ -0,0 +1,139 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class StarTreeDocValuesIteratorAdapterTests extends OpenSearchTestCase {
+
+    private StarTreeDocValuesIteratorAdapter adapter;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        adapter = new StarTreeDocValuesIteratorAdapter();
+    }
+
+    public void testGetDocValuesIterator() throws IOException {
+        DocValuesProducer mockProducer = mock(DocValuesProducer.class);
+        SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class);
+
+        when(mockProducer.getSortedNumeric(any())).thenReturn(mockSortedNumericDocValues);
+
+        SequentialDocValuesIterator iterator = adapter.getDocValuesIterator(DocValuesType.SORTED_NUMERIC, any(), mockProducer);
+
+        assertNotNull(iterator);
+        assertEquals(mockSortedNumericDocValues, iterator.getDocIdSetIterator());
+    }
+
+    public void testGetDocValuesIteratorWithUnsupportedType() {
+        DocValuesProducer mockProducer = mock(DocValuesProducer.class);
+        FieldInfo fieldInfo = new FieldInfo(
+            "random_field",
+            0,
+            false,
+            false,
+            true,
+            IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
+            DocValuesType.SORTED_NUMERIC,
+            -1,
+            Collections.emptyMap(),
+            0,
+            0,
+            0,
+            0,
+            VectorEncoding.FLOAT32,
+            VectorSimilarityFunction.EUCLIDEAN,
+            false,
+            false
+        );
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
+            adapter.getDocValuesIterator(DocValuesType.BINARY, fieldInfo, mockProducer);
+        });
+
+        assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage());
+    }
+
+    public void testGetNextValue() throws IOException {
+        SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class);
+        SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues);
+        iterator.setDocId(1);
+        when(mockSortedNumericDocValues.nextValue()).thenReturn(42L);
+
+        Long nextValue = adapter.getNextValue(iterator, 1);
+
+        assertEquals(Long.valueOf(42L), nextValue);
+        assertEquals(Long.valueOf(42L), iterator.getDocValue());
+    }
+
+    public void testGetNextValueWithInvalidDocId() {
+        SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class);
+        SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues);
+        iterator.setDocId(DocIdSetIterator.NO_MORE_DOCS);
+
+        IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { adapter.getNextValue(iterator, 1); });
+
+        assertEquals("invalid doc id to fetch the next value", exception.getMessage());
+    }
+
+    public void testGetNextValueWithUnsupportedIterator() {
+        DocIdSetIterator mockIterator = mock(DocIdSetIterator.class);
+        SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockIterator);
+
+        IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { adapter.getNextValue(iterator, 1); });
+
+        assertEquals("Unsupported Iterator: " + mockIterator.toString(), exception.getMessage());
+    }
+
+    public void testNextDoc() throws IOException {
+        SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class);
+        SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues);
+        when(mockSortedNumericDocValues.nextDoc()).thenReturn(2, 3, DocIdSetIterator.NO_MORE_DOCS);
+        when(mockSortedNumericDocValues.nextValue()).thenReturn(42L, 32L);
+
+        int nextDocId = adapter.nextDoc(iterator, 1);
+        assertEquals(2, nextDocId);
+        assertEquals(Long.valueOf(42L), adapter.getNextValue(iterator, nextDocId));
+
+        nextDocId = adapter.nextDoc(iterator, 2);
+        assertEquals(3, nextDocId);
+        when(mockSortedNumericDocValues.nextValue()).thenReturn(42L, 32L);
+
+    }
+
+    public void testNextDoc_noMoreDocs() throws IOException {
+        SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class);
+        SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues);
+        when(mockSortedNumericDocValues.nextDoc()).thenReturn(2, DocIdSetIterator.NO_MORE_DOCS);
+        when(mockSortedNumericDocValues.nextValue()).thenReturn(42L, 32L);
+
+        int nextDocId = adapter.nextDoc(iterator, 1);
+        assertEquals(2, nextDocId);
+        assertEquals(Long.valueOf(42L), adapter.getNextValue(iterator, nextDocId));
+
+        assertThrows(IllegalStateException.class, () -> adapter.nextDoc(iterator, 2));
+
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java
new file mode 100644
index 0000000000000..1aba67533d52e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java
@@ -0,0 +1,131 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import org.mockito.Mockito;
+
+import static org.mockito.Mockito.when;
+
+public class StarTreeValuesIteratorFactoryTests extends OpenSearchTestCase {
+
+    private static StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorAdapter;
+    private static FieldInfo mockFieldInfo;
+
+    @BeforeClass
+    public static void setup() {
+        starTreeDocValuesIteratorAdapter = new StarTreeDocValuesIteratorAdapter();
+        mockFieldInfo = new FieldInfo(
+            "field",
+            1,
+            false,
+            false,
+            true,
+            IndexOptions.NONE,
+            DocValuesType.NONE,
+            -1,
+            Collections.emptyMap(),
+            0,
+            0,
+            0,
+            0,
+            VectorEncoding.FLOAT32,
+            VectorSimilarityFunction.EUCLIDEAN,
+            false,
+            false
+        );
+    }
+
+    public void testCreateIterator_SortedNumeric() throws IOException {
+        DocValuesProducer producer = Mockito.mock(DocValuesProducer.class);
+        SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class);
+        when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator);
+        SequentialDocValuesIterator result = starTreeDocValuesIteratorAdapter.getDocValuesIterator(
+            DocValuesType.SORTED_NUMERIC,
+            mockFieldInfo,
+            producer
+        );
+        assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass());
+    }
+
+    public void testCreateIterator_UnsupportedType() {
+        DocValuesProducer producer = Mockito.mock(DocValuesProducer.class);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
+            starTreeDocValuesIteratorAdapter.getDocValuesIterator(DocValuesType.BINARY, mockFieldInfo, producer);
+        });
+        assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage());
+    }
+
+    public void testGetNextValue_SortedNumeric() throws IOException {
+        SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class);
+        when(iterator.nextDoc()).thenReturn(0);
+        when(iterator.nextValue()).thenReturn(123L);
+        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator);
+        sequentialDocValuesIterator.getDocIdSetIterator().nextDoc();
+        long result = starTreeDocValuesIteratorAdapter.getNextValue(sequentialDocValuesIterator, 0);
+        assertEquals(123L, result);
+    }
+
+    public void testGetNextValue_UnsupportedIterator() {
+        DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class);
+        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator);
+
+        IllegalStateException exception = expectThrows(IllegalStateException.class, () -> {
+            starTreeDocValuesIteratorAdapter.getNextValue(sequentialDocValuesIterator, 0);
+        });
+        assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage());
+    }
+
+    public void testNextDoc() throws IOException {
+        SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class);
+        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator);
+        when(iterator.nextDoc()).thenReturn(5);
+
+        int result = starTreeDocValuesIteratorAdapter.nextDoc(sequentialDocValuesIterator, 5);
+        assertEquals(5, result);
+    }
+
+    public void test_multipleCoordinatedDocumentReader() throws IOException {
+        SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class);
+        SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class);
+
+        SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(iterator1);
+        SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(iterator2);
+
+        when(iterator1.nextDoc()).thenReturn(0);
+        when(iterator2.nextDoc()).thenReturn(1);
+
+        when(iterator1.nextValue()).thenReturn(9L);
+        when(iterator2.nextValue()).thenReturn(9L);
+
+        starTreeDocValuesIteratorAdapter.nextDoc(sequentialDocValuesIterator1, 0);
+        starTreeDocValuesIteratorAdapter.nextDoc(sequentialDocValuesIterator2, 0);
+        assertEquals(0, sequentialDocValuesIterator1.getDocId());
+        assertEquals(9L, (long) sequentialDocValuesIterator1.getDocValue());
+        assertNotEquals(0, sequentialDocValuesIterator2.getDocId());
+        assertEquals(1, sequentialDocValuesIterator2.getDocId());
+        assertEquals(9L, (long) sequentialDocValuesIterator2.getDocValue());
+
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java
new file mode 100644
index 0000000000000..518c6729c2e1a
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java
@@ -0,0 +1,132 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.builder;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.Version;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.StarTreeMapper;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+public class StarTreesBuilderTests extends OpenSearchTestCase {
+
+    private MapperService mapperService;
+    private SegmentWriteState segmentWriteState;
+    private DocValuesProducer docValuesProducer;
+    private StarTreeMapper.StarTreeFieldType starTreeFieldType;
+    private StarTreeField starTreeField;
+    private Map<String, DocValuesProducer> fieldProducerMap;
+    private Directory directory;
+
+    public void setUp() throws Exception {
+        super.setUp();
+        mapperService = mock(MapperService.class);
+        directory = newFSDirectory(createTempDir());
+        SegmentInfo segmentInfo = new SegmentInfo(
+            directory,
+            Version.LATEST,
+            Version.LUCENE_9_11_0,
+            "test_segment",
+            5,
+            false,
+            false,
+            new Lucene99Codec(),
+            new HashMap<>(),
+            UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8),
+            new HashMap<>(),
+            null
+        );
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[0]);
+        segmentWriteState = new SegmentWriteState(
+            InfoStream.getDefault(),
+            segmentInfo.dir,
+            segmentInfo,
+            fieldInfos,
+            null,
+            newIOContext(random())
+        );
+        docValuesProducer = mock(DocValuesProducer.class);
+        StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(
+            1,
+            new HashSet<>(),
+            StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP
+        );
+        starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration);
+        starTreeFieldType = new StarTreeMapper.StarTreeFieldType("star_tree", starTreeField);
+        fieldProducerMap = new HashMap<>();
+        fieldProducerMap.put("field1", docValuesProducer);
+    }
+
+    public void test_buildWithNoStarTreeFields() throws IOException {
+        when(mapperService.getCompositeFieldTypes()).thenReturn(new HashSet<>());
+
+        StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService);
+        starTreesBuilder.build();
+
+        verifyNoInteractions(docValuesProducer);
+    }
+
+    public void test_getStarTreeBuilder() throws IOException {
+        when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType));
+        StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService);
+        StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService);
+        assertTrue(starTreeBuilder instanceof OnHeapStarTreeBuilder);
+    }
+
+    public void test_getStarTreeBuilder_illegalArgument() {
+        when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType));
+        StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(1, new HashSet<>(), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP);
+        StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration);
+        StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService);
+        assertThrows(IllegalArgumentException.class, () -> starTreesBuilder.getStarTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService));
+    }
+
+    public void test_closeWithNoStarTreeFields() throws IOException {
+        StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(
+            1,
+            new HashSet<>(),
+            StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP
+        );
+        StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration);
+        starTreeFieldType = new StarTreeMapper.StarTreeFieldType("star_tree", starTreeField);
+        when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType));
+        StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService);
+        starTreesBuilder.close();
+
+        verifyNoInteractions(docValuesProducer);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        directory.close();
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java
new file mode 100644
index 0000000000000..76b612e3677f7
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.utils;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.opensearch.index.fielddata.AbstractNumericDocValues;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+
+public class SequentialDocValuesIteratorTests extends OpenSearchTestCase {
+
+    public void test_sequentialDocValuesIterator() {
+        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(new AbstractNumericDocValues() {
+            @Override
+            public long longValue() throws IOException {
+                return 0;
+            }
+
+            @Override
+            public boolean advanceExact(int i) throws IOException {
+                return false;
+            }
+
+            @Override
+            public int docID() {
+                return 0;
+            }
+        });
+
+        assertTrue(sequentialDocValuesIterator.getDocIdSetIterator() instanceof AbstractNumericDocValues);
+        assertEquals(sequentialDocValuesIterator.getDocId(), 0);
+    }
+
+    public void test_sequentialDocValuesIterator_default() {
+        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator();
+        assertTrue(sequentialDocValuesIterator.getDocIdSetIterator() instanceof SortedNumericDocValues);
+    }
+
+}

From fca520fe5af3ee496520ab59d7a1da8e0a5703a7 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Wed, 17 Jul 2024 15:34:07 -0400
Subject: [PATCH 111/113] Add Gao Binlong as maintainer (#14796)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 .github/CODEOWNERS | 2 +-
 MAINTAINERS.md     | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 8ceecb3abb4a2..1aefeee710f47 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gaobinlong @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index 3298ceb15463c..f77c69ddeff2a 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -14,6 +14,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Charlotte Henkle         | [CEHENKLE](https://github.com/CEHENKLE)                 | Amazon      |
 | Dan Widdis               | [dbwiddis](https://github.com/dbwiddis)                 | Amazon      |
 | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock)                     | Amazon      |
+| Gao Binlong              | [gaobinlong](https://github.com/gaobinlong)             | Amazon      |
 | Gaurav Bafna             | [gbbafna](https://github.com/gbbafna)                   | Amazon      |
 | Jay Deng                 | [jed326](https://github.com/jed326)                     | Amazon      |
 | Kunal Kotwani            | [kotwanikunal](https://github.com/kotwanikunal)         | Amazon      |

From b3b743d69cc4fc10dbd43f90df9731e51b63991b Mon Sep 17 00:00:00 2001
From: Sagar <99425694+sgup432@users.noreply.github.com>
Date: Wed, 17 Jul 2024 19:02:05 -0700
Subject: [PATCH 112/113] Clear ehcache disk cache files during initialization
 (#14738)

* Clear ehcache disk cache files during initialization

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Adding UT to fix line coverage

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Addressing comment

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Adding more Uts for better line coverage

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Throwing exception in case we fail to clear cache files during startup

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Adding more UTs

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Adding a UT for more coverage

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Fixing gradle build

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Update ehcache disk cache close() logic

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

---------

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>
---
 .../cache/store/disk/EhcacheDiskCache.java    |  50 ++-
 .../store/disk/EhCacheDiskCacheTests.java     | 293 ++++++++++++++++++
 2 files changed, 328 insertions(+), 15 deletions(-)

diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
index b4c62fbf85cb8..4a95b04de3952 100644
--- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
+++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
@@ -60,7 +60,6 @@
 import java.util.function.ToLongBiFunction;
 
 import org.ehcache.Cache;
-import org.ehcache.CachePersistenceException;
 import org.ehcache.PersistentCacheManager;
 import org.ehcache.config.builders.CacheConfigurationBuilder;
 import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
@@ -104,8 +103,6 @@ public class EhcacheDiskCache<K, V> implements ICache<K, V> {
     // Unique id associated with this cache.
     private final static String UNIQUE_ID = UUID.randomUUID().toString();
     private final static String THREAD_POOL_ALIAS_PREFIX = "ehcachePool";
-    private final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB
-
     // A Cache manager can create many caches.
     private final PersistentCacheManager cacheManager;
 
@@ -127,13 +124,18 @@ public class EhcacheDiskCache<K, V> implements ICache<K, V> {
     private final Serializer<K, byte[]> keySerializer;
     private final Serializer<V, byte[]> valueSerializer;
 
+    final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB
+    final static String CACHE_DATA_CLEANUP_DURING_INITIALIZATION_EXCEPTION = "Failed to delete ehcache disk cache under "
+        + "path: %s during initialization. Please clean this up manually and restart the process";
+
     /**
      * Used in computeIfAbsent to synchronize loading of a given key. This is needed as ehcache doesn't provide a
      * computeIfAbsent method.
      */
     Map<ICacheKey<K>, CompletableFuture<Tuple<ICacheKey<K>, V>>> completableFutureMap = new ConcurrentHashMap<>();
 
-    private EhcacheDiskCache(Builder<K, V> builder) {
+    @SuppressForbidden(reason = "Ehcache uses File.io")
+    EhcacheDiskCache(Builder<K, V> builder) {
         this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null");
         this.valueType = Objects.requireNonNull(builder.valueType, "Value type shouldn't be null");
         this.expireAfterAccess = Objects.requireNonNull(builder.getExpireAfterAcess(), "ExpireAfterAccess value shouldn't " + "be null");
@@ -151,6 +153,18 @@ private EhcacheDiskCache(Builder<K, V> builder) {
         if (this.storagePath == null || this.storagePath.isBlank()) {
             throw new IllegalArgumentException("Storage path shouldn't be null or empty");
         }
+        // Delete all the previous disk cache related files/data. We don't persist data between process restart for
+        // now which is why need to do this. Clean up in case there was a non graceful restart and we had older disk
+        // cache data still lying around.
+        Path ehcacheDirectory = Paths.get(this.storagePath);
+        if (Files.exists(ehcacheDirectory)) {
+            try {
+                logger.info("Found older disk cache data lying around during initialization under path: {}", this.storagePath);
+                IOUtils.rm(ehcacheDirectory);
+            } catch (IOException e) {
+                throw new OpenSearchException(String.format(CACHE_DATA_CLEANUP_DURING_INITIALIZATION_EXCEPTION, this.storagePath), e);
+            }
+        }
         if (builder.threadPoolAlias == null || builder.threadPoolAlias.isBlank()) {
             this.threadPoolAlias = THREAD_POOL_ALIAS_PREFIX + "DiskWrite#" + UNIQUE_ID;
         } else {
@@ -175,6 +189,11 @@ private EhcacheDiskCache(Builder<K, V> builder) {
         }
     }
 
+    // Package private for testing
+    PersistentCacheManager getCacheManager() {
+        return this.cacheManager;
+    }
+
     @SuppressWarnings({ "rawtypes" })
     private Cache<ICacheKey, ByteArrayWrapper> buildCache(Duration expireAfterAccess, Builder<K, V> builder) {
         // Creating the cache requires permissions specified in plugin-security.policy
@@ -255,7 +274,7 @@ Map<ICacheKey<K>, CompletableFuture<Tuple<ICacheKey<K>, V>>> getCompletableFutur
     }
 
     @SuppressForbidden(reason = "Ehcache uses File.io")
-    private PersistentCacheManager buildCacheManager() {
+    PersistentCacheManager buildCacheManager() {
         // In case we use multiple ehCaches, we can define this cache manager at a global level.
         // Creating the cache manager also requires permissions specified in plugin-security.policy
         return AccessController.doPrivileged((PrivilegedAction<PersistentCacheManager>) () -> {
@@ -444,20 +463,21 @@ public void refresh() {
     @Override
     @SuppressForbidden(reason = "Ehcache uses File.io")
     public void close() {
-        cacheManager.removeCache(this.diskCacheAlias);
-        cacheManager.close();
         try {
-            cacheManager.destroyCache(this.diskCacheAlias);
-            // Delete all the disk cache related files/data
-            Path ehcacheDirectory = Paths.get(this.storagePath);
-            if (Files.exists(ehcacheDirectory)) {
+            cacheManager.close();
+        } catch (Exception e) {
+            logger.error(() -> new ParameterizedMessage("Exception occurred while trying to close ehcache manager"), e);
+        }
+        // Delete all the disk cache related files/data in case it is present
+        Path ehcacheDirectory = Paths.get(this.storagePath);
+        if (Files.exists(ehcacheDirectory)) {
+            try {
                 IOUtils.rm(ehcacheDirectory);
+            } catch (IOException e) {
+                logger.error(() -> new ParameterizedMessage("Failed to delete ehcache disk cache data under path: {}", this.storagePath));
             }
-        } catch (CachePersistenceException e) {
-            throw new OpenSearchException("Exception occurred while destroying ehcache and associated data", e);
-        } catch (IOException e) {
-            logger.error(() -> new ParameterizedMessage("Failed to delete ehcache disk cache data under path: {}", this.storagePath));
         }
+
     }
 
     /**
diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
index 29551befd3e9f..2bc24227bb513 100644
--- a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
+++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
@@ -25,6 +25,7 @@
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.bytes.CompositeBytesReference;
@@ -34,6 +35,8 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -47,10 +50,17 @@
 import java.util.concurrent.Phaser;
 import java.util.function.ToLongBiFunction;
 
+import org.ehcache.PersistentCacheManager;
+
 import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_LISTENER_MODE_SYNC_KEY;
 import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_MAX_SIZE_IN_BYTES_KEY;
 import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_STORAGE_PATH_KEY;
+import static org.opensearch.cache.store.disk.EhcacheDiskCache.MINIMUM_MAX_SIZE_IN_BYTES;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
 
 @ThreadLeakFilters(filters = { EhcacheThreadLeakFilter.class })
 public class EhCacheDiskCacheTests extends OpenSearchSingleNodeTestCase {
@@ -882,6 +892,289 @@ public void testStatsTrackingDisabled() throws Exception {
         }
     }
 
+    public void testDiskCacheFilesAreClearedUpDuringCloseAndInitialization() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            String path = env.nodePaths()[0].path.toString() + "/request_cache";
+            // Create a dummy file to simulate a scenario where the data is already in the disk cache storage path
+            // beforehand.
+            Files.createDirectory(Path.of(path));
+            Path dummyFilePath = Files.createFile(Path.of(path + "/testing.txt"));
+            assertTrue(Files.exists(dummyFilePath));
+            ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                .setStoragePath(path)
+                .setIsEventListenerModeSync(true)
+                .setKeyType(String.class)
+                .setValueType(String.class)
+                .setKeySerializer(new StringSerializer())
+                .setDiskCacheAlias("test1")
+                .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setThreadPoolAlias("")
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setRemovalListener(removalListener)
+                .setWeigher(weigher)
+                .setStatsTrackingEnabled(false)
+                .build();
+            int randomKeys = randomIntBetween(10, 100);
+            for (int i = 0; i < randomKeys; i++) {
+                ICacheKey<String> iCacheKey = getICacheKey(UUID.randomUUID().toString());
+                ehcacheTest.put(iCacheKey, UUID.randomUUID().toString());
+                assertEquals(0, ehcacheTest.count()); // Expect count of 0 if NoopCacheStatsHolder is used
+                assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), ehcacheTest.stats().getTotalStats());
+            }
+            // Verify that older data was wiped out after initialization
+            assertFalse(Files.exists(dummyFilePath));
+
+            // Verify that there is data present under desired path by explicitly verifying the folder name by prefix
+            // (used from disk cache alias)
+            assertTrue(Files.exists(Path.of(path)));
+            boolean folderExists = Files.walk(Path.of(path))
+                .filter(Files::isDirectory)
+                .anyMatch(path1 -> path1.getFileName().toString().startsWith("test1"));
+            assertTrue(folderExists);
+            ehcacheTest.close();
+            assertFalse(Files.exists(Path.of(path))); // Verify everything is cleared up now after close()
+        }
+    }
+
+    public void testDiskCacheCloseCalledTwiceAndVerifyDiskDataIsCleanedUp() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            String path = env.nodePaths()[0].path.toString() + "/request_cache";
+            ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias(null)
+                .setStoragePath(path)
+                .setIsEventListenerModeSync(true)
+                .setKeyType(String.class)
+                .setValueType(String.class)
+                .setKeySerializer(new StringSerializer())
+                .setDiskCacheAlias("test1")
+                .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setRemovalListener(removalListener)
+                .setWeigher(weigher)
+                .setStatsTrackingEnabled(false)
+                .build();
+            int randomKeys = randomIntBetween(10, 100);
+            for (int i = 0; i < randomKeys; i++) {
+                ICacheKey<String> iCacheKey = getICacheKey(UUID.randomUUID().toString());
+                ehcacheTest.put(iCacheKey, UUID.randomUUID().toString());
+                assertEquals(0, ehcacheTest.count()); // Expect count storagePath 0 if NoopCacheStatsHolder is used
+                assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), ehcacheTest.stats().getTotalStats());
+            }
+            ehcacheTest.close();
+            assertFalse(Files.exists(Path.of(path))); // Verify everything is cleared up now after close()
+            // Call it again. This will throw an exception.
+            ehcacheTest.close();
+        }
+    }
+
+    public void testDiskCacheCloseAfterCleaningUpFilesManually() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            String path = env.nodePaths()[0].path.toString() + "/request_cache";
+            ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias(null)
+                .setStoragePath(path)
+                .setIsEventListenerModeSync(true)
+                .setKeyType(String.class)
+                .setValueType(String.class)
+                .setKeySerializer(new StringSerializer())
+                .setDiskCacheAlias("test1")
+                .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setRemovalListener(removalListener)
+                .setWeigher(weigher)
+                .setStatsTrackingEnabled(false)
+                .build();
+            int randomKeys = randomIntBetween(10, 100);
+            for (int i = 0; i < randomKeys; i++) {
+                ICacheKey<String> iCacheKey = getICacheKey(UUID.randomUUID().toString());
+                ehcacheTest.put(iCacheKey, UUID.randomUUID().toString());
+                assertEquals(0, ehcacheTest.count()); // Expect count storagePath 0 if NoopCacheStatsHolder is used
+                assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), ehcacheTest.stats().getTotalStats());
+            }
+            IOUtils.rm(Path.of(path));
+            ehcacheTest.close();
+        }
+    }
+
+    public void testEhcacheDiskCacheWithoutStoragePathDefined() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            assertThrows(
+                IllegalArgumentException.class,
+                () -> new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                    .setIsEventListenerModeSync(true)
+                    .setKeyType(String.class)
+                    .setValueType(String.class)
+                    .setKeySerializer(new StringSerializer())
+                    .setDiskCacheAlias("test1")
+                    .setValueSerializer(new StringSerializer())
+                    .setDimensionNames(List.of(dimensionName))
+                    .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .setSettings(settings)
+                    .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                    .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                    .setRemovalListener(removalListener)
+                    .setWeigher(weigher)
+                    .setStatsTrackingEnabled(false)
+                    .build()
+            );
+        }
+    }
+
+    public void testEhcacheDiskCacheWithoutStoragePathNull() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            assertThrows(
+                IllegalArgumentException.class,
+                () -> new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                    .setStoragePath(null)
+                    .setIsEventListenerModeSync(true)
+                    .setKeyType(String.class)
+                    .setValueType(String.class)
+                    .setKeySerializer(new StringSerializer())
+                    .setDiskCacheAlias("test1")
+                    .setValueSerializer(new StringSerializer())
+                    .setDimensionNames(List.of(dimensionName))
+                    .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .setSettings(settings)
+                    .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                    .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                    .setRemovalListener(removalListener)
+                    .setWeigher(weigher)
+                    .setStatsTrackingEnabled(false)
+                    .build()
+            );
+        }
+    }
+
+    public void testEhcacheWithStorageSizeLowerThanMinimumExpected() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            assertThrows(
+                IllegalArgumentException.class,
+                () -> new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                    .setIsEventListenerModeSync(true)
+                    .setKeyType(String.class)
+                    .setValueType(String.class)
+                    .setKeySerializer(new StringSerializer())
+                    .setDiskCacheAlias("test1")
+                    .setValueSerializer(new StringSerializer())
+                    .setDimensionNames(List.of(dimensionName))
+                    .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .setSettings(settings)
+                    .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                    .setMaximumWeightInBytes(MINIMUM_MAX_SIZE_IN_BYTES)
+                    .setRemovalListener(removalListener)
+                    .setWeigher(weigher)
+                    .setStatsTrackingEnabled(false)
+                    .build()
+            );
+        }
+    }
+
+    public void testEhcacheWithStorageSizeZero() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            assertThrows(
+                IllegalArgumentException.class,
+                () -> new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                    .setIsEventListenerModeSync(true)
+                    .setKeyType(String.class)
+                    .setValueType(String.class)
+                    .setKeySerializer(new StringSerializer())
+                    .setDiskCacheAlias("test1")
+                    .setValueSerializer(new StringSerializer())
+                    .setDimensionNames(List.of(dimensionName))
+                    .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .setSettings(settings)
+                    .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                    .setMaximumWeightInBytes(0)
+                    .setRemovalListener(removalListener)
+                    .setWeigher(weigher)
+                    .setStatsTrackingEnabled(false)
+                    .build()
+            );
+        }
+    }
+
+    public void testEhcacheCloseWithDestroyCacheMethodThrowingException() throws Exception {
+        EhcacheDiskCache<String, String> ehcacheDiskCache = new MockEhcahceDiskCache(createDummyBuilder(null));
+        PersistentCacheManager cacheManager = ehcacheDiskCache.getCacheManager();
+        doNothing().when(cacheManager).removeCache(anyString());
+        doNothing().when(cacheManager).close();
+        doThrow(new RuntimeException("test")).when(cacheManager).destroyCache(anyString());
+        ehcacheDiskCache.close();
+    }
+
+    static class MockEhcahceDiskCache extends EhcacheDiskCache<String, String> {
+
+        public MockEhcahceDiskCache(Builder<String, String> builder) {
+            super(builder);
+        }
+
+        @Override
+        PersistentCacheManager buildCacheManager() {
+            PersistentCacheManager cacheManager = mock(PersistentCacheManager.class);
+            return cacheManager;
+        }
+    }
+
+    private EhcacheDiskCache.Builder<String, String> createDummyBuilder(String storagePath) throws IOException {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            if (storagePath == null || storagePath.isBlank()) {
+                storagePath = env.nodePaths()[0].path.toString() + "/request_cache";
+            }
+            return (EhcacheDiskCache.Builder<String, String>) new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias(
+                "ehcacheTest"
+            )
+                .setIsEventListenerModeSync(true)
+                .setStoragePath(storagePath)
+                .setKeyType(String.class)
+                .setValueType(String.class)
+                .setKeySerializer(new StringSerializer())
+                .setDiskCacheAlias("test1")
+                .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setRemovalListener(removalListener)
+                .setWeigher(weigher)
+                .setStatsTrackingEnabled(false);
+        }
+    }
+
     private List<String> getRandomDimensions(List<String> dimensionNames) {
         Random rand = Randomness.get();
         int bound = 3;

From 4abcf395897fda619a186afe53fbb5cfaa34b841 Mon Sep 17 00:00:00 2001
From: Arpit-Bandejiya <abandeji@amazon.com>
Date: Thu, 18 Jul 2024 14:33:13 +0530
Subject: [PATCH 113/113] Refactor remote-routing-table service inline with
 remote state interfaces (#14668)

---------
Signed-off-by: Arpit Bandejiya <abandeji@amazon.com>
Signed-off-by: Arpit-Bandejiya <abandeji@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../InternalRemoteRoutingTableService.java    | 232 +++----------
 .../remote/NoopRemoteRoutingTableService.java |  14 +-
 .../remote/RemoteRoutingTableService.java     |  14 +-
 .../RemoteRoutingTableServiceFactory.java     |   5 +-
 .../AbstractRemoteWritableBlobEntity.java     |   4 +
 .../common/settings/ClusterSettings.java      |   6 +-
 .../remote/RemoteClusterStateService.java     |  45 ++-
 .../model/RemoteClusterStateBlobStore.java    |  22 +-
 .../model/RemoteRoutingTableBlobStore.java    | 108 ++++++
 .../routingtable/IndexRoutingTableHeader.java |  81 -----
 .../routingtable/RemoteIndexRoutingTable.java | 136 ++++----
 ...RemoteRoutingTableServiceFactoryTests.java |   6 +-
 .../RemoteRoutingTableServiceTests.java       | 303 ++++++-----------
 .../remote/ClusterMetadataManifestTests.java  |   4 +-
 .../RemoteClusterStateServiceTests.java       |   9 +-
 .../RemoteRoutingTableBlobStoreTests.java     | 133 ++++++++
 .../IndexRoutingTableHeaderTests.java         |  32 --
 .../RemoteIndexRoutingTableTests.java         | 307 +++++++++++++++---
 19 files changed, 799 insertions(+), 663 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java
 delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java
 create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStoreTests.java
 delete mode 100644 server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b863b9d13e789..0417cc14ee86f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
 - Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https://github.com/opensearch-project/OpenSearch/pull/14750))
 - Add Plugin interface for loading application based configuration templates (([#14659](https://github.com/opensearch-project/OpenSearch/issues/14659)))
+- Refactor remote-routing-table service inline with remote state interfaces([#14668](https://github.com/opensearch-project/OpenSearch/pull/14668))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java
index cc1b0713393f3..f3f245ee9f8f0 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java
@@ -11,35 +11,24 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.store.IndexInput;
 import org.opensearch.action.LatchedActionListener;
-import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.DiffableUtils;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.common.CheckedRunnable;
-import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
-import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobPath;
-import org.opensearch.common.blobstore.stream.write.WritePriority;
-import org.opensearch.common.blobstore.transfer.RemoteTransferContainer;
-import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream;
-import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
-import org.opensearch.common.lucene.store.ByteArrayIndexInput;
+import org.opensearch.common.remote.RemoteWritableEntityStore;
 import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.index.Index;
+import org.opensearch.core.compress.Compressor;
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
 import org.opensearch.gateway.remote.RemoteStateTransferException;
+import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore;
 import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable;
-import org.opensearch.index.remote.RemoteStoreEnums;
-import org.opensearch.index.remote.RemoteStorePathStrategy;
-import org.opensearch.index.remote.RemoteStoreUtils;
+import org.opensearch.index.translog.transfer.BlobStoreTransferService;
 import org.opensearch.node.Node;
 import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
 import org.opensearch.repositories.RepositoriesService;
@@ -52,12 +41,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.ExecutorService;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
-import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled;
 
 /**
@@ -67,64 +54,29 @@
  */
 public class InternalRemoteRoutingTableService extends AbstractLifecycleComponent implements RemoteRoutingTableService {
 
-    /**
-     * This setting is used to set the remote routing table store blob store path type strategy.
-     */
-    public static final Setting<RemoteStoreEnums.PathType> REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING = new Setting<>(
-        "cluster.remote_store.routing_table.path_type",
-        RemoteStoreEnums.PathType.HASHED_PREFIX.toString(),
-        RemoteStoreEnums.PathType::parseString,
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic
-    );
-
-    /**
-     * This setting is used to set the remote routing table store blob store path hash algorithm strategy.
-     * This setting will come to effect if the {@link #REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING}
-     * is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
-     */
-    public static final Setting<RemoteStoreEnums.PathHashAlgorithm> REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING = new Setting<>(
-        "cluster.remote_store.routing_table.path_hash_algo",
-        RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64.toString(),
-        RemoteStoreEnums.PathHashAlgorithm::parseString,
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic
-    );
-
-    public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing";
-    public static final String INDEX_ROUTING_FILE_PREFIX = "index_routing";
-    public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--";
-
     private static final Logger logger = LogManager.getLogger(InternalRemoteRoutingTableService.class);
     private final Settings settings;
     private final Supplier<RepositoriesService> repositoriesService;
+    private Compressor compressor;
+    private RemoteWritableEntityStore<IndexRoutingTable, RemoteIndexRoutingTable> remoteIndexRoutingTableStore;
+    private final ClusterSettings clusterSettings;
     private BlobStoreRepository blobStoreRepository;
-    private RemoteStoreEnums.PathType pathType;
-    private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo;
-    private ThreadPool threadPool;
+    private final ThreadPool threadPool;
+    private final String clusterName;
 
     public InternalRemoteRoutingTableService(
         Supplier<RepositoriesService> repositoriesService,
         Settings settings,
         ClusterSettings clusterSettings,
-        ThreadPool threadpool
+        ThreadPool threadpool,
+        String clusterName
     ) {
         assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled";
         this.repositoriesService = repositoriesService;
         this.settings = settings;
-        this.pathType = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING);
-        this.pathHashAlgo = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING);
-        clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, this::setPathTypeSetting);
-        clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting);
         this.threadPool = threadpool;
-    }
-
-    private void setPathTypeSetting(RemoteStoreEnums.PathType pathType) {
-        this.pathType = pathType;
-    }
-
-    private void setPathHashAlgoSetting(RemoteStoreEnums.PathHashAlgorithm pathHashAlgo) {
-        this.pathHashAlgo = pathHashAlgo;
+        this.clusterName = clusterName;
+        this.clusterSettings = clusterSettings;
     }
 
     public List<IndexRoutingTable> getIndicesRouting(RoutingTable routingTable) {
@@ -151,43 +103,32 @@ public DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRouting
 
     /**
      * Create async action for writing one {@code IndexRoutingTable} to remote store
-     * @param clusterState current cluster state
+     * @param term current term
+     * @param version current version
+     * @param clusterUUID current cluster UUID
      * @param indexRouting indexRoutingTable to write to remote store
      * @param latchedActionListener listener for handling async action response
-     * @param clusterBasePath base path for remote file
      * @return returns runnable async action
      */
-    public CheckedRunnable<IOException> getIndexRoutingAsyncAction(
-        ClusterState clusterState,
+    @Override
+    public CheckedRunnable<IOException> getAsyncIndexRoutingWriteAction(
+        String clusterUUID,
+        long term,
+        long version,
         IndexRoutingTable indexRouting,
-        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener,
-        BlobPath clusterBasePath
+        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener
     ) {
 
-        BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN);
-        BlobPath path = pathType.path(
-            RemoteStorePathStrategy.PathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(),
-            pathHashAlgo
-        );
-        final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path);
-
-        final String fileName = getIndexRoutingFileName(clusterState.term(), clusterState.version());
+        RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable(indexRouting, clusterUUID, compressor, term, version);
 
         ActionListener<Void> completionListener = ActionListener.wrap(
-            resp -> latchedActionListener.onResponse(
-                new ClusterMetadataManifest.UploadedIndexMetadata(
-                    indexRouting.getIndex().getName(),
-                    indexRouting.getIndex().getUUID(),
-                    path.buildAsString() + fileName,
-                    INDEX_ROUTING_METADATA_PREFIX
-                )
-            ),
+            resp -> latchedActionListener.onResponse(remoteIndexRoutingTable.getUploadedMetadata()),
             ex -> latchedActionListener.onFailure(
                 new RemoteStateTransferException("Exception in writing index to remote store: " + indexRouting.getIndex().toString(), ex)
             )
         );
 
-        return () -> uploadIndex(indexRouting, fileName, blobContainer, completionListener);
+        return () -> remoteIndexRoutingTableStore.writeAsync(remoteIndexRoutingTable, completionListener);
     }
 
     /**
@@ -214,111 +155,21 @@ public List<ClusterMetadataManifest.UploadedIndexMetadata> getAllUploadedIndices
         return new ArrayList<>(allUploadedIndicesRouting.values());
     }
 
-    private void uploadIndex(
-        IndexRoutingTable indexRouting,
-        String fileName,
-        BlobContainer blobContainer,
-        ActionListener<Void> completionListener
-    ) {
-        RemoteIndexRoutingTable indexRoutingInput = new RemoteIndexRoutingTable(indexRouting);
-        BytesReference bytesInput = null;
-        try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
-            indexRoutingInput.writeTo(streamOutput);
-            bytesInput = streamOutput.bytes();
-        } catch (IOException e) {
-            logger.error("Failed to serialize IndexRoutingTable for [{}]: [{}]", indexRouting, e);
-            completionListener.onFailure(e);
-            return;
-        }
-
-        if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) {
-            try {
-                blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true);
-                completionListener.onResponse(null);
-            } catch (IOException e) {
-                logger.error("Failed to write IndexRoutingTable to remote store for indexRouting [{}]: [{}]", indexRouting, e);
-                completionListener.onFailure(e);
-            }
-            return;
-        }
-
-        try (IndexInput input = new ByteArrayIndexInput("indexrouting", BytesReference.toBytes(bytesInput))) {
-            try (
-                RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer(
-                    fileName,
-                    fileName,
-                    input.length(),
-                    true,
-                    WritePriority.URGENT,
-                    (size, position) -> new OffsetRangeIndexInputStream(input, size, position),
-                    null,
-                    false
-                )
-            ) {
-                ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(
-                    remoteTransferContainer.createWriteContext(),
-                    completionListener
-                );
-            } catch (IOException e) {
-                logger.error("Failed to write IndexRoutingTable to remote store for indexRouting [{}]: [{}]", indexRouting, e);
-                completionListener.onFailure(e);
-            }
-        } catch (IOException e) {
-            logger.error(
-                "Failed to create transfer object for IndexRoutingTable for remote store upload for indexRouting [{}]: [{}]",
-                indexRouting,
-                e
-            );
-            completionListener.onFailure(e);
-        }
-    }
-
     @Override
     public CheckedRunnable<IOException> getAsyncIndexRoutingReadAction(
+        String clusterUUID,
         String uploadedFilename,
-        Index index,
         LatchedActionListener<IndexRoutingTable> latchedActionListener
     ) {
-        int idx = uploadedFilename.lastIndexOf("/");
-        String blobFileName = uploadedFilename.substring(idx + 1);
-        BlobContainer blobContainer = blobStoreRepository.blobStore()
-            .blobContainer(BlobPath.cleanPath().add(uploadedFilename.substring(0, idx)));
 
-        return () -> readAsync(
-            blobContainer,
-            blobFileName,
-            index,
-            threadPool.executor(ThreadPool.Names.REMOTE_STATE_READ),
-            ActionListener.wrap(
-                response -> latchedActionListener.onResponse(response.getIndexRoutingTable()),
-                latchedActionListener::onFailure
-            )
+        ActionListener<IndexRoutingTable> actionListener = ActionListener.wrap(
+            latchedActionListener::onResponse,
+            latchedActionListener::onFailure
         );
-    }
 
-    private void readAsync(
-        BlobContainer blobContainer,
-        String name,
-        Index index,
-        ExecutorService executorService,
-        ActionListener<RemoteIndexRoutingTable> listener
-    ) {
-        executorService.execute(() -> {
-            try {
-                listener.onResponse(read(blobContainer, name, index));
-            } catch (Exception e) {
-                listener.onFailure(e);
-            }
-        });
-    }
+        RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable(uploadedFilename, clusterUUID, compressor);
 
-    private RemoteIndexRoutingTable read(BlobContainer blobContainer, String path, Index index) {
-        try {
-            return new RemoteIndexRoutingTable(blobContainer.readBlob(path), index);
-        } catch (IOException | AssertionError e) {
-            logger.error(() -> new ParameterizedMessage("RoutingTable read failed for path {}", path), e);
-            throw new RemoteStateTransferException("Failed to read RemoteRoutingTable from Manifest with error ", e);
-        }
+        return () -> remoteIndexRoutingTableStore.readAsync(remoteIndexRoutingTable, actionListener);
     }
 
     @Override
@@ -335,16 +186,6 @@ public List<ClusterMetadataManifest.UploadedIndexMetadata> getUpdatedIndexRoutin
         }).collect(Collectors.toList());
     }
 
-    private String getIndexRoutingFileName(long term, long version) {
-        return String.join(
-            DELIMITER,
-            INDEX_ROUTING_FILE_PREFIX,
-            RemoteStoreUtils.invertLong(term),
-            RemoteStoreUtils.invertLong(version),
-            RemoteStoreUtils.invertLong(System.currentTimeMillis())
-        );
-    }
-
     @Override
     protected void doClose() throws IOException {
         if (blobStoreRepository != null) {
@@ -362,6 +203,16 @@ protected void doStart() {
         final Repository repository = repositoriesService.get().repository(remoteStoreRepo);
         assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository";
         blobStoreRepository = (BlobStoreRepository) repository;
+        compressor = blobStoreRepository.getCompressor();
+
+        this.remoteIndexRoutingTableStore = new RemoteRoutingTableBlobStore<>(
+            new BlobStoreTransferService(blobStoreRepository.blobStore(), threadPool),
+            blobStoreRepository,
+            clusterName,
+            threadPool,
+            ThreadPool.Names.REMOTE_STATE_READ,
+            clusterSettings
+        );
     }
 
     @Override
@@ -377,5 +228,4 @@ public void deleteStaleIndexRoutingPaths(List<String> stalePaths) throws IOExcep
             throw e;
         }
     }
-
 }
diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java
index 6236d107d0220..4636e492df28f 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java
@@ -9,14 +9,11 @@
 package org.opensearch.cluster.routing.remote;
 
 import org.opensearch.action.LatchedActionListener;
-import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.DiffableUtils;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.common.CheckedRunnable;
-import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
-import org.opensearch.core.index.Index;
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
 
 import java.io.IOException;
@@ -42,11 +39,12 @@ public DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRouting
     }
 
     @Override
-    public CheckedRunnable<IOException> getIndexRoutingAsyncAction(
-        ClusterState clusterState,
+    public CheckedRunnable<IOException> getAsyncIndexRoutingWriteAction(
+        String clusterUUID,
+        long term,
+        long version,
         IndexRoutingTable indexRouting,
-        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener,
-        BlobPath clusterBasePath
+        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener
     ) {
         // noop
         return () -> {};
@@ -64,8 +62,8 @@ public List<ClusterMetadataManifest.UploadedIndexMetadata> getAllUploadedIndices
 
     @Override
     public CheckedRunnable<IOException> getAsyncIndexRoutingReadAction(
+        String clusterUUID,
         String uploadedFilename,
-        Index index,
         LatchedActionListener<IndexRoutingTable> latchedActionListener
     ) {
         // noop
diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java
index d455dfb58eabc..d319123bc2cee 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java
@@ -9,16 +9,13 @@
 package org.opensearch.cluster.routing.remote;
 
 import org.opensearch.action.LatchedActionListener;
-import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.DiffableUtils;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.common.CheckedRunnable;
-import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.lifecycle.LifecycleComponent;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.index.Index;
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
 
 import java.io.IOException;
@@ -47,8 +44,8 @@ public IndexRoutingTable read(StreamInput in, String key) throws IOException {
     List<IndexRoutingTable> getIndicesRouting(RoutingTable routingTable);
 
     CheckedRunnable<IOException> getAsyncIndexRoutingReadAction(
+        String clusterUUID,
         String uploadedFilename,
-        Index index,
         LatchedActionListener<IndexRoutingTable> latchedActionListener
     );
 
@@ -62,11 +59,12 @@ DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>>
         RoutingTable after
     );
 
-    CheckedRunnable<IOException> getIndexRoutingAsyncAction(
-        ClusterState clusterState,
+    CheckedRunnable<IOException> getAsyncIndexRoutingWriteAction(
+        String clusterUUID,
+        long term,
+        long version,
         IndexRoutingTable indexRouting,
-        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener,
-        BlobPath clusterBasePath
+        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener
     );
 
     List<ClusterMetadataManifest.UploadedIndexMetadata> getAllUploadedIndicesRouting(
diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java
index 82837191a30b7..56dfa03215a64 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java
@@ -34,10 +34,11 @@ public static RemoteRoutingTableService getService(
         Supplier<RepositoriesService> repositoriesService,
         Settings settings,
         ClusterSettings clusterSettings,
-        ThreadPool threadPool
+        ThreadPool threadPool,
+        String clusterName
     ) {
         if (isRemoteRoutingTableEnabled(settings)) {
-            return new InternalRemoteRoutingTableService(repositoriesService, settings, clusterSettings, threadPool);
+            return new InternalRemoteRoutingTableService(repositoriesService, settings, clusterSettings, threadPool, clusterName);
         }
         return new NoopRemoteRoutingTableService();
     }
diff --git a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java
index 23fc9d3ad77cb..237c077cb673c 100644
--- a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java
+++ b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java
@@ -40,6 +40,10 @@ public AbstractRemoteWritableBlobEntity(
         this.namedXContentRegistry = namedXContentRegistry;
     }
 
+    public AbstractRemoteWritableBlobEntity(final String clusterUUID, final Compressor compressor) {
+        this(clusterUUID, compressor, null);
+    }
+
     public abstract BlobPathParameters getBlobPathParameters();
 
     public abstract String getType();
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index b4826e1a59428..49801fd3834b8 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -78,7 +78,6 @@
 import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
-import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService;
 import org.opensearch.cluster.service.ClusterApplierService;
 import org.opensearch.cluster.service.ClusterManagerService;
 import org.opensearch.cluster.service.ClusterManagerTaskThrottler;
@@ -108,6 +107,7 @@
 import org.opensearch.gateway.ShardsBatchGatewayAllocator;
 import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager;
 import org.opensearch.gateway.remote.RemoteClusterStateService;
+import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore;
 import org.opensearch.http.HttpTransportSettings;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexSettings;
@@ -730,8 +730,8 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING,
                 IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
                 IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING,
-                InternalRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING,
-                InternalRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING,
+                RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING,
+                RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING,
 
                 // Admission Control Settings
                 AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 3e63f9114ea16..7e7a93e1d42ec 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -26,7 +26,6 @@
 import org.opensearch.cluster.node.DiscoveryNodes.Builder;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
-import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService;
 import org.opensearch.cluster.routing.remote.RemoteRoutingTableService;
 import org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceFactory;
 import org.opensearch.cluster.service.ClusterService;
@@ -43,7 +42,6 @@
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
-import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute;
@@ -98,7 +96,6 @@
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer;
-import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getClusterMetadataBasePath;
 import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM;
 import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER;
@@ -107,6 +104,7 @@
 import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
 
 /**
@@ -146,7 +144,7 @@ public class RemoteClusterStateService implements Closeable {
     private final List<IndexMetadataUploadListener> indexMetadataUploadListeners;
     private BlobStoreRepository blobStoreRepository;
     private BlobStoreTransferService blobStoreTransferService;
-    private final RemoteRoutingTableService remoteRoutingTableService;
+    private RemoteRoutingTableService remoteRoutingTableService;
     private volatile TimeValue slowWriteLoggingThreshold;
 
     private final RemotePersistenceStats remoteStateStats;
@@ -197,16 +195,17 @@ public RemoteClusterStateService(
         this.remoteStateStats = new RemotePersistenceStats();
         this.namedWriteableRegistry = namedWriteableRegistry;
         this.indexMetadataUploadListeners = indexMetadataUploadListeners;
+        this.isPublicationEnabled = FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL)
+            && RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings)
+            && RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings);
         this.remoteRoutingTableService = RemoteRoutingTableServiceFactory.getService(
             repositoriesService,
             settings,
             clusterSettings,
-            threadPool
+            threadpool,
+            ClusterName.CLUSTER_NAME_SETTING.get(settings).value()
         );
-        this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService, remoteRoutingTableService);
-        this.isPublicationEnabled = FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL)
-            && RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings)
-            && RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings);
+        remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService, remoteRoutingTableService);
     }
 
     /**
@@ -663,16 +662,13 @@ UploadedMetadataResults writeMetadataInParallel(
         });
         indicesRoutingToUpload.forEach(indexRoutingTable -> {
             uploadTasks.put(
-                InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(),
-                remoteRoutingTableService.getIndexRoutingAsyncAction(
-                    clusterState,
+                INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(),
+                remoteRoutingTableService.getAsyncIndexRoutingWriteAction(
+                    clusterState.metadata().clusterUUID(),
+                    clusterState.term(),
+                    clusterState.version(),
                     indexRoutingTable,
-                    listener,
-                    getClusterMetadataBasePath(
-                        blobStoreRepository,
-                        clusterState.getClusterName().value(),
-                        clusterState.metadata().clusterUUID()
-                    )
+                    listener
                 )
             );
         });
@@ -723,7 +719,7 @@ UploadedMetadataResults writeMetadataInParallel(
         UploadedMetadataResults response = new UploadedMetadataResults();
         results.forEach((name, uploadedMetadata) -> {
             if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class)
-                && uploadedMetadata.getComponent().contains(InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) {
+                && uploadedMetadata.getComponent().contains(INDEX_ROUTING_METADATA_PREFIX)) {
                 response.uploadedIndicesRoutingMetadata.add((UploadedIndexMetadata) uploadedMetadata);
             } else if (name.startsWith(CUSTOM_METADATA)) {
                 // component name for custom metadata will look like custom--<metadata-attribute>
@@ -897,9 +893,8 @@ public void start() {
         final Repository repository = repositoriesService.get().repository(remoteStoreRepo);
         assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository";
         blobStoreRepository = (BlobStoreRepository) repository;
-        this.remoteRoutingTableService.start();
-        blobStoreTransferService = new BlobStoreTransferService(getBlobStore(), threadpool);
         String clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings).value();
+        blobStoreTransferService = new BlobStoreTransferService(getBlobStore(), threadpool);
 
         remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(
             clusterSettings,
@@ -931,6 +926,8 @@ public void start() {
             namedWriteableRegistry,
             threadpool
         );
+
+        remoteRoutingTableService.start();
         remoteClusterStateCleanupManager.start();
     }
 
@@ -1022,10 +1019,10 @@ ClusterState readClusterStateInParallel(
 
         LatchedActionListener<IndexRoutingTable> routingTableLatchedActionListener = new LatchedActionListener<>(
             ActionListener.wrap(response -> {
-                logger.debug("Successfully read cluster state component from remote");
+                logger.debug(() -> new ParameterizedMessage("Successfully read index-routing for index {}", response.getIndex().getName()));
                 readIndexRoutingTableResults.add(response);
             }, ex -> {
-                logger.error("Failed to read cluster state from remote", ex);
+                logger.error(() -> new ParameterizedMessage("Failed to read index-routing from remote"), ex);
                 exceptionList.add(ex);
             }),
             latch
@@ -1034,8 +1031,8 @@ ClusterState readClusterStateInParallel(
         for (UploadedIndexMetadata indexRouting : indicesRoutingToRead) {
             asyncMetadataReadActions.add(
                 remoteRoutingTableService.getAsyncIndexRoutingReadAction(
+                    clusterUUID,
                     indexRouting.getUploadedFilename(),
-                    new Index(indexRouting.getIndexName(), indexRouting.getIndexUUID()),
                     routingTableLatchedActionListener
                 )
             );
diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java
index 1dd23443f1252..cd8b8aa41ad65 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java
@@ -23,6 +23,8 @@
 import java.io.InputStream;
 import java.util.concurrent.ExecutorService;
 
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN;
+
 /**
  * Abstract class for a blob type storage
  *
@@ -88,18 +90,26 @@ public void readAsync(final U entity, final ActionListener<T> listener) {
         });
     }
 
-    private BlobPath getBlobPathForUpload(final AbstractRemoteWritableBlobEntity<T> obj) {
-        BlobPath blobPath = blobStoreRepository.basePath()
-            .add(RemoteClusterStateUtils.encodeString(clusterName))
-            .add("cluster-state")
-            .add(obj.clusterUUID());
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    public BlobPath getBlobPathPrefix(String clusterUUID) {
+        return blobStoreRepository.basePath()
+            .add(RemoteClusterStateUtils.encodeString(getClusterName()))
+            .add(CLUSTER_STATE_PATH_TOKEN)
+            .add(clusterUUID);
+    }
+
+    public BlobPath getBlobPathForUpload(final AbstractRemoteWritableBlobEntity<T> obj) {
+        BlobPath blobPath = getBlobPathPrefix(obj.clusterUUID());
         for (String token : obj.getBlobPathParameters().getPathTokens()) {
             blobPath = blobPath.add(token);
         }
         return blobPath;
     }
 
-    private BlobPath getBlobPathForDownload(final AbstractRemoteWritableBlobEntity<T> obj) {
+    public BlobPath getBlobPathForDownload(final AbstractRemoteWritableBlobEntity<T> obj) {
         String[] pathTokens = obj.getBlobPathTokens();
         BlobPath blobPath = new BlobPath();
         if (pathTokens == null || pathTokens.length < 1) {
diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java
new file mode 100644
index 0000000000000..7c4a5bf2236a1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java
@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote.model;
+
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity;
+import org.opensearch.common.remote.RemoteWriteableEntity;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable;
+import org.opensearch.index.remote.RemoteStoreEnums;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
+import org.opensearch.index.translog.transfer.BlobStoreTransferService;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.threadpool.ThreadPool;
+
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE;
+
+/**
+ * Extends the RemoteClusterStateBlobStore to support {@link RemoteIndexRoutingTable}
+ *
+ * @param <IndexRoutingTable> which can be uploaded to / downloaded from blob store
+ * @param <U> The concrete class implementing {@link RemoteWriteableEntity} which is used as a wrapper for IndexRoutingTable entity.
+ */
+public class RemoteRoutingTableBlobStore<IndexRoutingTable, U extends AbstractRemoteWritableBlobEntity<IndexRoutingTable>> extends
+    RemoteClusterStateBlobStore<IndexRoutingTable, U> {
+
+    /**
+     * This setting is used to set the remote routing table store blob store path type strategy.
+     */
+    public static final Setting<RemoteStoreEnums.PathType> REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING = new Setting<>(
+        "cluster.remote_store.routing_table.path_type",
+        RemoteStoreEnums.PathType.HASHED_PREFIX.toString(),
+        RemoteStoreEnums.PathType::parseString,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * This setting is used to set the remote routing table store blob store path hash algorithm strategy.
+     * This setting will come to effect if the {@link #REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING}
+     * is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
+     */
+    public static final Setting<RemoteStoreEnums.PathHashAlgorithm> REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING = new Setting<>(
+        "cluster.remote_store.routing_table.path_hash_algo",
+        RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64.toString(),
+        RemoteStoreEnums.PathHashAlgorithm::parseString,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    private RemoteStoreEnums.PathType pathType;
+    private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo;
+
+    public RemoteRoutingTableBlobStore(
+        BlobStoreTransferService blobStoreTransferService,
+        BlobStoreRepository blobStoreRepository,
+        String clusterName,
+        ThreadPool threadPool,
+        String executor,
+        ClusterSettings clusterSettings
+    ) {
+        super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool, executor);
+        this.pathType = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING);
+        this.pathHashAlgo = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING);
+        clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, this::setPathTypeSetting);
+        clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting);
+    }
+
+    @Override
+    public BlobPath getBlobPathForUpload(final AbstractRemoteWritableBlobEntity<IndexRoutingTable> obj) {
+        assert obj.getBlobPathParameters().getPathTokens().size() == 1 : "Unexpected tokens in RemoteRoutingTableObject";
+        BlobPath indexRoutingPath = getBlobPathPrefix(obj.clusterUUID()).add(INDEX_ROUTING_TABLE);
+
+        BlobPath path = pathType.path(
+            RemoteStorePathStrategy.PathInput.builder()
+                .basePath(indexRoutingPath)
+                .indexUUID(String.join("", obj.getBlobPathParameters().getPathTokens()))
+                .build(),
+            pathHashAlgo
+        );
+        return path;
+    }
+
+    private void setPathTypeSetting(RemoteStoreEnums.PathType pathType) {
+        this.pathType = pathType;
+    }
+
+    private void setPathHashAlgoSetting(RemoteStoreEnums.PathHashAlgorithm pathHashAlgo) {
+        this.pathHashAlgo = pathHashAlgo;
+    }
+
+    // For testing only
+    protected RemoteStoreEnums.PathType getPathTypeSetting() {
+        return pathType;
+    }
+
+    // For testing only
+    protected RemoteStoreEnums.PathHashAlgorithm getPathHashAlgoSetting() {
+        return pathHashAlgo;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java
deleted file mode 100644
index 5baea6adba0c7..0000000000000
--- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.gateway.remote.routingtable;
-
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFormatTooNewException;
-import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.common.io.stream.Writeable;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- * The stored header information for the individual index routing table
- */
-public class IndexRoutingTableHeader implements Writeable {
-
-    public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec";
-    public static final int INITIAL_VERSION = 1;
-    public static final int CURRENT_VERSION = INITIAL_VERSION;
-    private final String indexName;
-
-    public IndexRoutingTableHeader(String indexName) {
-        this.indexName = indexName;
-    }
-
-    /**
-     * Reads the contents on the stream into the corresponding {@link IndexRoutingTableHeader}
-     *
-     * @param in streamInput
-     * @throws IOException exception thrown on failing to read from stream.
-     */
-    public IndexRoutingTableHeader(StreamInput in) throws IOException {
-        try {
-            readHeaderVersion(in);
-            indexName = in.readString();
-        } catch (EOFException e) {
-            throw new IOException("index routing header truncated", e);
-        }
-    }
-
-    private void readHeaderVersion(final StreamInput in) throws IOException {
-        try {
-            CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION);
-        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
-            throw new IOException("index routing table header corrupted", e);
-        }
-    }
-
-    /**
-     * Write the IndexRoutingTable to given stream.
-     *
-     * @param out stream to write
-     * @throws IOException exception thrown on failing to write to stream.
-     */
-    public void writeTo(StreamOutput out) throws IOException {
-        try {
-            CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION);
-            out.writeString(indexName);
-            out.flush();
-        } catch (IOException e) {
-            throw new IOException("Failed to write IndexRoutingTable header", e);
-        }
-    }
-
-    public String getIndexName() {
-        return indexName;
-    }
-
-}
diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java
index 17c55190da07f..40b5bafde2b13 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java
@@ -9,92 +9,106 @@
 package org.opensearch.gateway.remote.routingtable;
 
 import org.opensearch.cluster.routing.IndexRoutingTable;
-import org.opensearch.cluster.routing.IndexShardRoutingTable;
-import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput;
-import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput;
-import org.opensearch.core.common.io.stream.InputStreamStreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.common.io.Streams;
+import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity;
+import org.opensearch.common.remote.BlobPathParameters;
+import org.opensearch.core.compress.Compressor;
 import org.opensearch.core.index.Index;
+import org.opensearch.gateway.remote.ClusterMetadataManifest;
+import org.opensearch.index.remote.RemoteStoreUtils;
+import org.opensearch.repositories.blobstore.ChecksumWritableBlobStoreFormat;
 
-import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.List;
+
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
 
 /**
  * Remote store object for IndexRoutingTable
  */
-public class RemoteIndexRoutingTable implements Writeable {
+public class RemoteIndexRoutingTable extends AbstractRemoteWritableBlobEntity<IndexRoutingTable> {
 
-    private final IndexRoutingTable indexRoutingTable;
+    public static final String INDEX_ROUTING_TABLE = "index-routing";
+    public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--";
+    public static final String INDEX_ROUTING_FILE = "index_routing";
+    private IndexRoutingTable indexRoutingTable;
+    private final Index index;
+    private long term;
+    private long version;
+    private BlobPathParameters blobPathParameters;
+    public static final ChecksumWritableBlobStoreFormat<IndexRoutingTable> INDEX_ROUTING_TABLE_FORMAT =
+        new ChecksumWritableBlobStoreFormat<>("index-routing-table", IndexRoutingTable::readFrom);
 
-    public RemoteIndexRoutingTable(IndexRoutingTable indexRoutingTable) {
+    public RemoteIndexRoutingTable(
+        IndexRoutingTable indexRoutingTable,
+        String clusterUUID,
+        Compressor compressor,
+        long term,
+        long version
+    ) {
+        super(clusterUUID, compressor);
+        this.index = indexRoutingTable.getIndex();
         this.indexRoutingTable = indexRoutingTable;
+        this.term = term;
+        this.version = version;
     }
 
     /**
      * Reads data from inputStream and creates RemoteIndexRoutingTable object with the {@link IndexRoutingTable}
-     * @param inputStream input stream with index routing data
-     * @param index index for the current routing data
-     * @throws IOException exception thrown on failing to read from stream.
+     * @param blobName name of the blob, which contains the index routing data
+     * @param clusterUUID UUID of the cluster
+     * @param compressor Compressor object
      */
-    public RemoteIndexRoutingTable(InputStream inputStream, Index index) throws IOException {
-        try {
-            try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) {
-                // Read the Table Header first and confirm the index
-                IndexRoutingTableHeader indexRoutingTableHeader = new IndexRoutingTableHeader(in);
-                assert indexRoutingTableHeader.getIndexName().equals(index.getName());
+    public RemoteIndexRoutingTable(String blobName, String clusterUUID, Compressor compressor) {
+        super(clusterUUID, compressor);
+        this.index = null;
+        this.term = -1;
+        this.version = -1;
+        this.blobName = blobName;
+    }
 
-                int numberOfShardRouting = in.readVInt();
-                IndexRoutingTable.Builder indicesRoutingTable = IndexRoutingTable.builder(index);
-                for (int idx = 0; idx < numberOfShardRouting; idx++) {
-                    IndexShardRoutingTable indexShardRoutingTable = IndexShardRoutingTable.Builder.readFrom(in);
-                    indicesRoutingTable.addIndexShard(indexShardRoutingTable);
-                }
-                verifyCheckSum(in);
-                indexRoutingTable = indicesRoutingTable.build();
-            }
-        } catch (EOFException e) {
-            throw new IOException("Indices Routing table is corrupted", e);
+    @Override
+    public BlobPathParameters getBlobPathParameters() {
+        if (blobPathParameters == null) {
+            blobPathParameters = new BlobPathParameters(List.of(indexRoutingTable.getIndex().getUUID()), INDEX_ROUTING_FILE);
         }
+        return blobPathParameters;
     }
 
-    public IndexRoutingTable getIndexRoutingTable() {
-        return indexRoutingTable;
+    @Override
+    public String getType() {
+        return INDEX_ROUTING_TABLE;
     }
 
-    /**
-     * Writes {@link IndexRoutingTable} to the given stream
-     * @param streamOutput output stream to write
-     * @throws IOException exception thrown on failing to write to stream.
-     */
     @Override
-    public void writeTo(StreamOutput streamOutput) throws IOException {
-        try {
-            BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(streamOutput);
-            IndexRoutingTableHeader indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName());
-            indexRoutingTableHeader.writeTo(out);
-            out.writeVInt(indexRoutingTable.shards().size());
-            for (IndexShardRoutingTable next : indexRoutingTable) {
-                IndexShardRoutingTable.Builder.writeTo(next, out);
-            }
-            out.writeLong(out.getChecksum());
-            out.flush();
-        } catch (IOException e) {
-            throw new IOException("Failed to write IndexRoutingTable to stream", e);
+    public String generateBlobFileName() {
+        if (blobFileName == null) {
+            blobFileName = String.join(
+                DELIMITER,
+                getBlobPathParameters().getFilePrefix(),
+                RemoteStoreUtils.invertLong(term),
+                RemoteStoreUtils.invertLong(version),
+                RemoteStoreUtils.invertLong(System.currentTimeMillis())
+            );
         }
+        return blobFileName;
     }
 
-    private void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException {
-        long expectedChecksum = in.getChecksum();
-        long readChecksum = in.readLong();
-        if (readChecksum != expectedChecksum) {
-            throw new IOException(
-                "checksum verification failed - expected: 0x"
-                    + Long.toHexString(expectedChecksum)
-                    + ", got: 0x"
-                    + Long.toHexString(readChecksum)
-            );
-        }
+    @Override
+    public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() {
+        assert blobName != null;
+        assert index != null;
+        return new ClusterMetadataManifest.UploadedIndexMetadata(index.getName(), index.getUUID(), blobName, INDEX_ROUTING_METADATA_PREFIX);
+    }
+
+    @Override
+    public InputStream serialize() throws IOException {
+        return INDEX_ROUTING_TABLE_FORMAT.serialize(indexRoutingTable, generateBlobFileName(), getCompressor()).streamInput();
+    }
+
+    @Override
+    public IndexRoutingTable deserialize(InputStream in) throws IOException {
+        return INDEX_ROUTING_TABLE_FORMAT.deserialize(blobName, Streams.readFully(in));
     }
 }
diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java
index 39294ee8da41e..86f4b9502d6ab 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java
@@ -40,7 +40,8 @@ public void testGetServiceWhenRemoteRoutingDisabled() {
             repositoriesService,
             settings,
             new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            "test-cluster"
         );
         assertTrue(service instanceof NoopRemoteRoutingTableService);
     }
@@ -56,7 +57,8 @@ public void testGetServiceWhenRemoteRoutingEnabled() {
             repositoriesService,
             settings,
             new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            "test-cluster"
         );
         assertTrue(service instanceof InternalRemoteRoutingTableService);
     }
diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
index 839ebe1ff8301..564c7f7aed304 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
@@ -19,27 +19,25 @@
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.CheckedRunnable;
-import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.BlobStore;
-import org.opensearch.common.blobstore.stream.write.WriteContext;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.common.compress.DeflateCompressor;
-import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.common.util.TestCapturingListener;
 import org.opensearch.core.action.ActionListener;
-import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.compress.Compressor;
+import org.opensearch.core.compress.NoneCompressor;
 import org.opensearch.core.index.Index;
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
-import org.opensearch.gateway.remote.RemoteStateTransferException;
-import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable;
+import org.opensearch.gateway.remote.RemoteClusterStateUtils;
 import org.opensearch.index.remote.RemoteStoreEnums;
 import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteStoreUtils;
+import org.opensearch.index.translog.transfer.BlobStoreTransferService;
 import org.opensearch.repositories.FilterRepository;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.RepositoryMissingException;
@@ -51,33 +49,37 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.function.Supplier;
 
-import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
-import static org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX;
-import static org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN;
 import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL;
 import static org.opensearch.gateway.remote.ClusterMetadataManifestTests.randomUploadedIndexMetadataList;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_FILE;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE_FORMAT;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
-import static org.mockito.ArgumentMatchers.anyLong;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.ArgumentMatchers.startsWith;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -92,6 +94,8 @@ public class RemoteRoutingTableServiceTests extends OpenSearchTestCase {
     private BlobPath basePath;
     private ClusterSettings clusterSettings;
     private ClusterService clusterService;
+    private Compressor compressor;
+    private BlobStoreTransferService blobStoreTransferService;
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
 
     @Before
@@ -105,6 +109,7 @@ public void setup() {
             .build();
         clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         clusterService = mock(ClusterService.class);
+        blobStoreTransferService = mock(BlobStoreTransferService.class);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         blobStoreRepository = mock(BlobStoreRepository.class);
         when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor());
@@ -112,18 +117,20 @@ public void setup() {
         blobContainer = mock(BlobContainer.class);
         when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository);
         when(blobStoreRepository.blobStore()).thenReturn(blobStore);
+        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
         Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build();
         FeatureFlags.initializeFeatureFlags(nodeSettings);
-
+        compressor = new NoneCompressor();
         basePath = BlobPath.cleanPath().add("base-path");
-
+        when(blobStoreRepository.basePath()).thenReturn(basePath);
         remoteRoutingTableService = new InternalRemoteRoutingTableService(
             repositoriesServiceSupplier,
             settings,
             new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            "test-cluster"
         );
-
+        remoteRoutingTableService.doStart();
     }
 
     @After
@@ -141,7 +148,8 @@ public void testFailInitializationWhenRemoteRoutingDisabled() {
                 repositoriesServiceSupplier,
                 settings,
                 new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                threadPool
+                threadPool,
+                "test-cluster"
             )
         );
     }
@@ -347,136 +355,13 @@ public void testGetIndicesRoutingMapDiffIndexDeleted() {
         assertEquals(indexName, diff.getDeletes().get(0));
     }
 
-    public void testGetIndexRoutingAsyncAction() throws IOException {
-        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        ClusterState clusterState = createClusterState(indexName);
-        BlobPath expectedPath = getPath();
-
-        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> listener = mock(LatchedActionListener.class);
-        when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer);
-
-        remoteRoutingTableService.start();
-        CheckedRunnable<IOException> runnable = remoteRoutingTableService.getIndexRoutingAsyncAction(
-            clusterState,
-            clusterState.routingTable().getIndicesRouting().get(indexName),
-            listener,
-            basePath
-        );
-        assertNotNull(runnable);
-        runnable.run();
-
-        String expectedFilePrefix = String.join(
-            DELIMITER,
-            INDEX_ROUTING_FILE_PREFIX,
-            RemoteStoreUtils.invertLong(clusterState.term()),
-            RemoteStoreUtils.invertLong(clusterState.version())
-        );
-        verify(blobContainer, times(1)).writeBlob(startsWith(expectedFilePrefix), any(StreamInput.class), anyLong(), eq(true));
-        verify(listener, times(1)).onResponse(any(ClusterMetadataManifest.UploadedMetadata.class));
-    }
-
-    public void testGetIndexRoutingAsyncActionFailureInBlobRepo() throws IOException {
-        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        ClusterState clusterState = createClusterState(indexName);
-        BlobPath expectedPath = getPath();
-
-        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> listener = mock(LatchedActionListener.class);
-        when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer);
-        doThrow(new IOException("testing failure")).when(blobContainer).writeBlob(anyString(), any(StreamInput.class), anyLong(), eq(true));
-
-        remoteRoutingTableService.start();
-        CheckedRunnable<IOException> runnable = remoteRoutingTableService.getIndexRoutingAsyncAction(
-            clusterState,
-            clusterState.routingTable().getIndicesRouting().get(indexName),
-            listener,
-            basePath
-        );
-        assertNotNull(runnable);
-        runnable.run();
-        String expectedFilePrefix = String.join(
-            DELIMITER,
-            INDEX_ROUTING_FILE_PREFIX,
-            RemoteStoreUtils.invertLong(clusterState.term()),
-            RemoteStoreUtils.invertLong(clusterState.version())
-        );
-        verify(blobContainer, times(1)).writeBlob(startsWith(expectedFilePrefix), any(StreamInput.class), anyLong(), eq(true));
-        verify(listener, times(1)).onFailure(any(RemoteStateTransferException.class));
-    }
-
-    public void testGetIndexRoutingAsyncActionAsyncRepo() throws IOException {
-        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        ClusterState clusterState = createClusterState(indexName);
-        BlobPath expectedPath = getPath();
-
-        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> listener = mock(LatchedActionListener.class);
-        blobContainer = mock(AsyncMultiStreamBlobContainer.class);
-        when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer);
-        ArgumentCaptor<ActionListener<Void>> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
-        ArgumentCaptor<WriteContext> writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class);
-        ConcurrentHashMap<String, WriteContext> capturedWriteContext = new ConcurrentHashMap<>();
-
-        doAnswer((i) -> {
-            actionListenerArgumentCaptor.getValue().onResponse(null);
-            WriteContext writeContext = writeContextArgumentCaptor.getValue();
-            capturedWriteContext.put(writeContext.getFileName().split(DELIMITER)[0], writeContextArgumentCaptor.getValue());
-            return null;
-        }).when((AsyncMultiStreamBlobContainer) blobContainer)
-            .asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture());
-
-        remoteRoutingTableService.start();
-        CheckedRunnable<IOException> runnable = remoteRoutingTableService.getIndexRoutingAsyncAction(
-            clusterState,
-            clusterState.routingTable().getIndicesRouting().get(indexName),
-            listener,
-            basePath
-        );
-        assertNotNull(runnable);
-        runnable.run();
-
-        String expectedFilePrefix = String.join(
-            DELIMITER,
-            INDEX_ROUTING_FILE_PREFIX,
-            RemoteStoreUtils.invertLong(clusterState.term()),
-            RemoteStoreUtils.invertLong(clusterState.version())
-        );
-        assertEquals(1, actionListenerArgumentCaptor.getAllValues().size());
-        assertEquals(1, writeContextArgumentCaptor.getAllValues().size());
-        assertNotNull(capturedWriteContext.get("index_routing"));
-        assertEquals(capturedWriteContext.get("index_routing").getWritePriority(), WritePriority.URGENT);
-        assertTrue(capturedWriteContext.get("index_routing").getFileName().startsWith(expectedFilePrefix));
-    }
-
-    public void testGetIndexRoutingAsyncActionAsyncRepoFailureInRepo() throws IOException {
-        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        ClusterState clusterState = createClusterState(indexName);
-        BlobPath expectedPath = getPath();
-
-        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> listener = mock(LatchedActionListener.class);
-        blobContainer = mock(AsyncMultiStreamBlobContainer.class);
-        when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer);
-
-        doThrow(new IOException("Testing failure")).when((AsyncMultiStreamBlobContainer) blobContainer)
-            .asyncBlobUpload(any(WriteContext.class), any(ActionListener.class));
-
-        remoteRoutingTableService.start();
-        CheckedRunnable<IOException> runnable = remoteRoutingTableService.getIndexRoutingAsyncAction(
-            clusterState,
-            clusterState.routingTable().getIndicesRouting().get(indexName),
-            listener,
-            basePath
-        );
-        assertNotNull(runnable);
-        runnable.run();
-        verify(listener, times(1)).onFailure(any(RemoteStateTransferException.class));
-    }
-
     public void testGetAllUploadedIndicesRouting() {
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().build();
         final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata = new ClusterMetadataManifest.UploadedIndexMetadata(
             "test-index",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
 
         List<ClusterMetadataManifest.UploadedIndexMetadata> allIndiceRoutingMetadata = remoteRoutingTableService
@@ -491,7 +376,7 @@ public void testGetAllUploadedIndicesRoutingExistingIndexInManifest() {
             "test-index",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
             .indicesRouting(List.of(uploadedIndexMetadata))
@@ -509,7 +394,7 @@ public void testGetAllUploadedIndicesRoutingNewIndexFromManifest() {
             "test-index",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
             .indicesRouting(List.of(uploadedIndexMetadata))
@@ -518,7 +403,7 @@ public void testGetAllUploadedIndicesRoutingNewIndexFromManifest() {
             "test-index2",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
 
         List<ClusterMetadataManifest.UploadedIndexMetadata> allIndiceRoutingMetadata = remoteRoutingTableService
@@ -534,13 +419,13 @@ public void testGetAllUploadedIndicesRoutingIndexDeleted() {
             "test-index",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata(
             "test-index2",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
             .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2))
@@ -558,13 +443,13 @@ public void testGetAllUploadedIndicesRoutingNoChange() {
             "test-index",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata(
             "test-index2",
             "index-uuid",
             "index-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
             .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2))
@@ -640,69 +525,83 @@ public void testIndicesRoutingDiffWhenIndexDeletedAndAdded() {
         );
     }
 
-    public void testGetAsyncIndexMetadataReadAction() throws Exception {
+    public void testGetAsyncIndexRoutingReadAction() throws Exception {
         String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
         ClusterState clusterState = createClusterState(indexName);
         String uploadedFileName = String.format(Locale.ROOT, "index-routing/" + indexName);
-        Index index = new Index(indexName, "uuid-01");
-
-        LatchedActionListener<IndexRoutingTable> listener = mock(LatchedActionListener.class);
-        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
-        BytesStreamOutput streamOutput = new BytesStreamOutput();
-        RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable(
-            clusterState.routingTable().getIndicesRouting().get(indexName)
+        when(blobContainer.readBlob(indexName)).thenReturn(
+            INDEX_ROUTING_TABLE_FORMAT.serialize(
+                clusterState.getRoutingTable().getIndicesRouting().get(indexName),
+                uploadedFileName,
+                compressor
+            ).streamInput()
         );
-        remoteIndexRoutingTable.writeTo(streamOutput);
-        when(blobContainer.readBlob(indexName)).thenReturn(streamOutput.bytes().streamInput());
-        remoteRoutingTableService.start();
+        TestCapturingListener<IndexRoutingTable> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
 
-        CheckedRunnable<IOException> runnable = remoteRoutingTableService.getAsyncIndexRoutingReadAction(uploadedFileName, index, listener);
-        assertNotNull(runnable);
-        runnable.run();
+        remoteRoutingTableService.getAsyncIndexRoutingReadAction(
+            "cluster-uuid",
+            uploadedFileName,
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
 
-        assertBusy(() -> verify(blobContainer, times(1)).readBlob(any()));
-        assertBusy(() -> verify(listener, times(1)).onResponse(any(IndexRoutingTable.class)));
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        IndexRoutingTable indexRoutingTable = listener.getResult();
+        assertEquals(clusterState.getRoutingTable().getIndicesRouting().get(indexName), indexRoutingTable);
     }
 
-    public void testGetAsyncIndexMetadataReadActionFailureForIncorrectIndex() throws Exception {
+    public void testGetAsyncIndexRoutingWriteAction() throws Exception {
         String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
         ClusterState clusterState = createClusterState(indexName);
-        String uploadedFileName = String.format(Locale.ROOT, "index-routing/" + indexName);
-        Index index = new Index("incorrect-index", "uuid-01");
-
-        LatchedActionListener<IndexRoutingTable> listener = mock(LatchedActionListener.class);
-        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
-        BytesStreamOutput streamOutput = new BytesStreamOutput();
-        RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable(
-            clusterState.routingTable().getIndicesRouting().get(indexName)
+        Iterable<String> remotePath = HASHED_PREFIX.path(
+            RemoteStorePathStrategy.PathInput.builder()
+                .basePath(
+                    new BlobPath().add("base-path")
+                        .add(RemoteClusterStateUtils.encodeString(ClusterName.DEFAULT.toString()))
+                        .add(CLUSTER_STATE_PATH_TOKEN)
+                        .add(clusterState.metadata().clusterUUID())
+                        .add(INDEX_ROUTING_TABLE)
+                )
+                .indexUUID(clusterState.getRoutingTable().indicesRouting().get(indexName).getIndex().getUUID())
+                .build(),
+            FNV_1A_BASE64
         );
-        remoteIndexRoutingTable.writeTo(streamOutput);
-        when(blobContainer.readBlob(anyString())).thenReturn(streamOutput.bytes().streamInput());
-        remoteRoutingTableService.doStart();
-
-        CheckedRunnable<IOException> runnable = remoteRoutingTableService.getAsyncIndexRoutingReadAction(uploadedFileName, index, listener);
-        assertNotNull(runnable);
-        runnable.run();
-
-        assertBusy(() -> verify(blobContainer, times(1)).readBlob(any()));
-        assertBusy(() -> verify(listener, times(1)).onFailure(any(Exception.class)));
-    }
-
-    public void testGetAsyncIndexMetadataReadActionFailureInBlobRepo() throws Exception {
-        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        String uploadedFileName = String.format(Locale.ROOT, "index-routing/" + indexName);
-        Index index = new Index(indexName, "uuid-01");
-
-        LatchedActionListener<IndexRoutingTable> listener = mock(LatchedActionListener.class);
-        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
-        doThrow(new IOException("testing failure")).when(blobContainer).readBlob(indexName);
-        remoteRoutingTableService.doStart();
-
-        CheckedRunnable<IOException> runnable = remoteRoutingTableService.getAsyncIndexRoutingReadAction(uploadedFileName, index, listener);
-        assertNotNull(runnable);
-        runnable.run();
 
-        assertBusy(() -> verify(listener, times(1)).onFailure(any(RemoteStateTransferException.class)));
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.getArgument(4, ActionListener.class).onResponse(null);
+            return null;
+        }).when(blobStoreTransferService)
+            .uploadBlob(any(InputStream.class), eq(remotePath), anyString(), eq(WritePriority.URGENT), any(ActionListener.class));
+
+        TestCapturingListener<ClusterMetadataManifest.UploadedMetadata> listener = new TestCapturingListener<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        remoteRoutingTableService.getAsyncIndexRoutingWriteAction(
+            clusterState.metadata().clusterUUID(),
+            clusterState.term(),
+            clusterState.version(),
+            clusterState.getRoutingTable().indicesRouting().get(indexName),
+            new LatchedActionListener<>(listener, latch)
+        ).run();
+        latch.await();
+        assertNull(listener.getFailure());
+        assertNotNull(listener.getResult());
+        ClusterMetadataManifest.UploadedMetadata uploadedMetadata = listener.getResult();
+
+        assertEquals(INDEX_ROUTING_METADATA_PREFIX + indexName, uploadedMetadata.getComponent());
+        String uploadedFileName = uploadedMetadata.getUploadedFilename();
+        String[] pathTokens = uploadedFileName.split(PATH_DELIMITER);
+        assertEquals(8, pathTokens.length);
+        assertEquals(pathTokens[1], "base-path");
+        String[] fileNameTokens = pathTokens[7].split(DELIMITER);
+
+        assertEquals(4, fileNameTokens.length);
+        assertEquals(fileNameTokens[0], INDEX_ROUTING_FILE);
+        assertEquals(fileNameTokens[1], RemoteStoreUtils.invertLong(1L));
+        assertEquals(fileNameTokens[2], RemoteStoreUtils.invertLong(2L));
+        assertThat(RemoteStoreUtils.invertLong(fileNameTokens[3]), lessThanOrEqualTo(System.currentTimeMillis()));
     }
 
     public void testGetUpdatedIndexRoutingTableMetadataWhenNoChange() {
@@ -758,7 +657,7 @@ private ClusterState createClusterState(String indexName) {
     }
 
     private BlobPath getPath() {
-        BlobPath indexRoutingPath = basePath.add(INDEX_ROUTING_PATH_TOKEN);
+        BlobPath indexRoutingPath = basePath.add(INDEX_ROUTING_TABLE);
         return RemoteStoreEnums.PathType.HASHED_PREFIX.path(
             RemoteStorePathStrategy.PathInput.builder().basePath(indexRoutingPath).indexUUID("uuid").build(),
             RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64
diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
index 152a6dba6c032..256161af1a3e2 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
@@ -13,7 +13,6 @@
 import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
-import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -44,6 +43,7 @@
 import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX;
 
 public class ClusterMetadataManifestTests extends OpenSearchTestCase {
 
@@ -545,7 +545,7 @@ public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOExc
             "test-index",
             "test-uuid",
             "routing-path",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
             .clusterTerm(1L)
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 6cd9cbbf13848..ebd3488d06007 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -117,6 +117,7 @@
 import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom1;
 import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom2;
 import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.TestClusterStateCustom3;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getFormattedIndexFileName;
@@ -126,7 +127,6 @@
 import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM;
 import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA_FORMAT;
-import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER;
 import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.readFrom;
 import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES;
@@ -144,6 +144,7 @@
 import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA_FORMAT;
 import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadataTests.getTemplatesMetadata;
 import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
@@ -2603,7 +2604,7 @@ public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException {
             "test-index",
             "index-uuid",
             "routing-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
             .indices(List.of(uploadedIndexMetadata))
@@ -2654,7 +2655,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx
             "test-index",
             "index-uuid",
             "routing-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
 
         final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
@@ -2710,7 +2711,7 @@ public void testWriteIncrementalMetadataSuccessWithRoutingTable() throws IOExcep
             "test-index",
             "index-uuid",
             "routing-filename",
-            InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX
+            INDEX_ROUTING_METADATA_PREFIX
         );
         final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
             .indices(List.of(uploadedIndexMetadata))
diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStoreTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStoreTests.java
new file mode 100644
index 0000000000000..ea0f3264cfe4f
--- /dev/null
+++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStoreTests.java
@@ -0,0 +1,133 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote.model;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.routing.IndexRoutingTable;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.compress.DeflateCompressor;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.index.Index;
+import org.opensearch.gateway.remote.RemoteClusterStateUtils;
+import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable;
+import org.opensearch.index.remote.RemoteStoreEnums;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
+import org.opensearch.index.translog.transfer.BlobStoreTransferService;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_PREFIX;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class RemoteRoutingTableBlobStoreTests extends OpenSearchTestCase {
+
+    private RemoteRoutingTableBlobStore<IndexRoutingTable, RemoteIndexRoutingTable> remoteIndexRoutingTableStore;
+    ClusterSettings clusterSettings;
+    ThreadPool threadPool;
+
+    @Before
+    public void setup() {
+        clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        BlobStoreTransferService blobStoreTransferService = mock(BlobStoreTransferService.class);
+        BlobStoreRepository blobStoreRepository = mock(BlobStoreRepository.class);
+        BlobPath blobPath = new BlobPath().add("base-path");
+        when(blobStoreRepository.basePath()).thenReturn(blobPath);
+
+        threadPool = new TestThreadPool(getClass().getName());
+        this.remoteIndexRoutingTableStore = new RemoteRoutingTableBlobStore<>(
+            blobStoreTransferService,
+            blobStoreRepository,
+            "test-cluster",
+            threadPool,
+            ThreadPool.Names.REMOTE_STATE_READ,
+            clusterSettings
+        );
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+
+    }
+
+    public void testRemoteRoutingTablePathTypeSetting() {
+        // Assert the default is HASHED_PREFIX
+        assertEquals(HASHED_PREFIX.toString(), remoteIndexRoutingTableStore.getPathTypeSetting().toString());
+
+        Settings newSettings = Settings.builder()
+            .put("cluster.remote_store.routing_table.path_type", RemoteStoreEnums.PathType.FIXED.toString())
+            .build();
+        clusterSettings.applySettings(newSettings);
+        assertEquals(RemoteStoreEnums.PathType.FIXED.toString(), remoteIndexRoutingTableStore.getPathTypeSetting().toString());
+    }
+
+    public void testRemoteRoutingTableHashAlgoSetting() {
+        // Assert the default is FNV_1A_BASE64
+        assertEquals(FNV_1A_BASE64.toString(), remoteIndexRoutingTableStore.getPathHashAlgoSetting().toString());
+
+        Settings newSettings = Settings.builder()
+            .put("cluster.remote_store.routing_table.path_hash_algo", RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString())
+            .build();
+        clusterSettings.applySettings(newSettings);
+        assertEquals(
+            RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString(),
+            remoteIndexRoutingTableStore.getPathHashAlgoSetting().toString()
+        );
+    }
+
+    public void testGetBlobPathForUpload() {
+
+        Index index = new Index("test-idx", "index-uuid");
+        Settings idxSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
+            .build();
+
+        IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+
+        IndexRoutingTable indexRoutingTable = new IndexRoutingTable.Builder(index).initializeAsNew(indexMetadata).build();
+
+        RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+            indexRoutingTable,
+            "cluster-uuid",
+            new DeflateCompressor(),
+            2L,
+            3L
+        );
+        BlobPath blobPath = remoteIndexRoutingTableStore.getBlobPathForUpload(remoteObjectForUpload);
+        BlobPath expectedPath = HASHED_PREFIX.path(
+            RemoteStorePathStrategy.PathInput.builder()
+                .basePath(
+                    new BlobPath().add("base-path")
+                        .add(RemoteClusterStateUtils.encodeString("test-cluster"))
+                        .add(CLUSTER_STATE_PATH_TOKEN)
+                        .add("cluster-uuid")
+                        .add(INDEX_ROUTING_TABLE)
+                )
+                .indexUUID(index.getUUID())
+                .build(),
+            FNV_1A_BASE64
+        );
+        assertEquals(expectedPath, blobPath);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java
deleted file mode 100644
index a3f0ac36a40f1..0000000000000
--- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.gateway.remote.routingtable;
-
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.common.io.stream.BytesStreamInput;
-import org.opensearch.test.OpenSearchTestCase;
-
-import java.io.IOException;
-
-public class IndexRoutingTableHeaderTests extends OpenSearchTestCase {
-
-    public void testIndexRoutingTableHeader() throws IOException {
-        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
-        IndexRoutingTableHeader header = new IndexRoutingTableHeader(indexName);
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            header.writeTo(out);
-
-            BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes);
-            IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in);
-            assertEquals(indexName, headerRead.getIndexName());
-
-        }
-    }
-
-}
diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java
index 72066d8afb45b..29d4ffa978851 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java
@@ -13,16 +13,136 @@
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
-import org.opensearch.cluster.routing.ShardRoutingState;
-import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.compress.DeflateCompressor;
+import org.opensearch.common.remote.BlobPathParameters;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.compress.Compressor;
+import org.opensearch.core.compress.NoneCompressor;
+import org.opensearch.gateway.remote.ClusterMetadataManifest;
+import org.opensearch.gateway.remote.RemoteClusterStateUtils;
+import org.opensearch.index.remote.RemoteStoreUtils;
+import org.opensearch.index.translog.transfer.BlobStoreTransferService;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
 
 import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.io.InputStream;
+import java.util.List;
+
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_FILE;
+import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class RemoteIndexRoutingTableTests extends OpenSearchTestCase {
 
-    public void testRoutingTableInput() {
+    private static final String TEST_BLOB_NAME = "/test-path/test-blob-name";
+    private static final String TEST_BLOB_PATH = "test-path";
+    private static final String TEST_BLOB_FILE_NAME = "test-blob-name";
+    private static final String INDEX_ROUTING_TABLE_TYPE = "test-index-routing-table";
+    private static final long STATE_VERSION = 3L;
+    private static final long STATE_TERM = 2L;
+    private String clusterUUID;
+    private BlobStoreTransferService blobStoreTransferService;
+    private BlobStoreRepository blobStoreRepository;
+    private String clusterName;
+    private ClusterSettings clusterSettings;
+    private Compressor compressor;
+    private NamedWriteableRegistry namedWriteableRegistry;
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Before
+    public void setup() {
+        clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        this.clusterUUID = "test-cluster-uuid";
+        this.blobStoreTransferService = mock(BlobStoreTransferService.class);
+        this.blobStoreRepository = mock(BlobStoreRepository.class);
+        BlobPath blobPath = new BlobPath().add("/path");
+        when(blobStoreRepository.basePath()).thenReturn(blobPath);
+        when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor());
+        compressor = new NoneCompressor();
+        namedWriteableRegistry = writableRegistry();
+        this.clusterName = "test-cluster-name";
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+    }
+
+    public void testClusterUUID() {
+        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
+        int numberOfShards = randomIntBetween(1, 10);
+        int numberOfReplicas = randomIntBetween(1, 10);
+        Metadata metadata = Metadata.builder()
+            .put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+            )
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
+
+        initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> {
+            RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+                indexRoutingTable,
+                clusterUUID,
+                compressor,
+                STATE_TERM,
+                STATE_VERSION
+            );
+            assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID);
+
+            RemoteIndexRoutingTable remoteObjectForDownload = new RemoteIndexRoutingTable(TEST_BLOB_NAME, clusterUUID, compressor);
+            assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID);
+        });
+    }
+
+    public void testFullBlobName() {
+        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
+        int numberOfShards = randomIntBetween(1, 10);
+        int numberOfReplicas = randomIntBetween(1, 10);
+        Metadata metadata = Metadata.builder()
+            .put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+            )
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
+
+        initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> {
+            RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+                indexRoutingTable,
+                clusterUUID,
+                compressor,
+                STATE_TERM,
+                STATE_VERSION
+            );
+            assertThat(remoteObjectForUpload.getFullBlobName(), nullValue());
+
+            RemoteIndexRoutingTable remoteObjectForDownload = new RemoteIndexRoutingTable(TEST_BLOB_NAME, clusterUUID, compressor);
+            assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME));
+        });
+    }
+
+    public void testBlobFileName() {
         String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
         int numberOfShards = randomIntBetween(1, 10);
         int numberOfReplicas = randomIntBetween(1, 10);
@@ -37,51 +157,164 @@ public void testRoutingTableInput() {
 
         RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
 
-        initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> {
-            RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables);
-            try (BytesStreamOutput streamOutput = new BytesStreamOutput();) {
-                indexRouting.writeTo(streamOutput);
-                RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable(
-                    streamOutput.bytes().streamInput(),
-                    metadata.index(indexName).getIndex()
-                );
-                IndexRoutingTable indexRoutingTable = remoteIndexRoutingTable.getIndexRoutingTable();
-                assertEquals(numberOfShards, indexRoutingTable.getShards().size());
-                assertEquals(metadata.index(indexName).getIndex(), indexRoutingTable.getIndex());
-                assertEquals(
-                    numberOfShards * (1 + numberOfReplicas),
-                    indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size()
-                );
+        initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> {
+            RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+                indexRoutingTable,
+                clusterUUID,
+                compressor,
+                STATE_TERM,
+                STATE_VERSION
+            );
+            assertThat(remoteObjectForUpload.getBlobFileName(), nullValue());
+
+            RemoteIndexRoutingTable remoteObjectForDownload = new RemoteIndexRoutingTable(TEST_BLOB_NAME, clusterUUID, compressor);
+            assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME));
+        });
+    }
+
+    public void testBlobPathTokens() {
+        String uploadedFile = "user/local/opensearch/routingTable";
+        RemoteIndexRoutingTable remoteObjectForDownload = new RemoteIndexRoutingTable(uploadedFile, clusterUUID, compressor);
+        assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[] { "user", "local", "opensearch", "routingTable" }));
+    }
+
+    public void testBlobPathParameters() {
+        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
+        int numberOfShards = randomIntBetween(1, 10);
+        int numberOfReplicas = randomIntBetween(1, 10);
+        Metadata metadata = Metadata.builder()
+            .put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+            )
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
+
+        initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> {
+            RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+                indexRoutingTable,
+                clusterUUID,
+                compressor,
+                STATE_TERM,
+                STATE_VERSION
+            );
+            assertThat(remoteObjectForUpload.getBlobFileName(), nullValue());
+
+            BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters();
+            assertThat(params.getPathTokens(), is(List.of(indexRoutingTable.getIndex().getUUID())));
+            String expectedPrefix = INDEX_ROUTING_FILE;
+            assertThat(params.getFilePrefix(), is(expectedPrefix));
+        });
+    }
+
+    public void testGenerateBlobFileName() {
+        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
+        int numberOfShards = randomIntBetween(1, 10);
+        int numberOfReplicas = randomIntBetween(1, 10);
+        Metadata metadata = Metadata.builder()
+            .put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+            )
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
+
+        initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> {
+            RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+                indexRoutingTable,
+                clusterUUID,
+                compressor,
+                STATE_TERM,
+                STATE_VERSION
+            );
+
+            String blobFileName = remoteObjectForUpload.generateBlobFileName();
+            String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER);
+            assertEquals(nameTokens[0], INDEX_ROUTING_FILE);
+            assertEquals(nameTokens[1], RemoteStoreUtils.invertLong(STATE_TERM));
+            assertEquals(nameTokens[2], RemoteStoreUtils.invertLong(STATE_VERSION));
+            assertThat(RemoteStoreUtils.invertLong(nameTokens[3]), lessThanOrEqualTo(System.currentTimeMillis()));
+        });
+    }
+
+    public void testGetUploadedMetadata() throws IOException {
+        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
+        int numberOfShards = randomIntBetween(1, 10);
+        int numberOfReplicas = randomIntBetween(1, 10);
+        Metadata metadata = Metadata.builder()
+            .put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+            )
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
+
+        initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> {
+            RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+                indexRoutingTable,
+                clusterUUID,
+                compressor,
+                STATE_TERM,
+                STATE_VERSION
+            );
+
+            assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata);
+
+            try (InputStream inputStream = remoteObjectForUpload.serialize()) {
+                remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH));
+                ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata();
+                String expectedPrefix = INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName();
+                assertThat(uploadedMetadata.getComponent(), is(expectedPrefix));
+                assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName()));
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
         });
     }
 
-    public void testRoutingTableInputStreamWithInvalidIndex() {
+    public void testSerDe() throws IOException {
+        String indexName = randomAlphaOfLength(randomIntBetween(1, 50));
+        int numberOfShards = randomIntBetween(1, 10);
+        int numberOfReplicas = randomIntBetween(1, 10);
         Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
-            .put(IndexMetadata.builder("invalid-index").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+            .put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+            )
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
-        AtomicInteger assertionError = new AtomicInteger();
-        initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> {
-            RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables);
-            try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
-                indexRouting.writeTo(streamOutput);
-                RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable(
-                    streamOutput.bytes().streamInput(),
-                    metadata.index("invalid-index").getIndex()
-                );
-            } catch (AssertionError e) {
-                assertionError.getAndIncrement();
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
+
+        initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> {
+            RemoteIndexRoutingTable remoteObjectForUpload = new RemoteIndexRoutingTable(
+                indexRoutingTable,
+                clusterUUID,
+                compressor,
+                STATE_TERM,
+                STATE_VERSION
+            );
+
+            assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata);
+
+            try (InputStream inputStream = remoteObjectForUpload.serialize()) {
+                remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath());
+                assertThat(inputStream.available(), greaterThan(0));
+                IndexRoutingTable readIndexRoutingTable = remoteObjectForUpload.deserialize(inputStream);
+                assertEquals(readIndexRoutingTable, indexRoutingTable);
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
         });
-
-        assertEquals(1, assertionError.get());
     }
-
 }