Merge branch 'opensearch-project:main' into main-repository-gcs-ADC-S…

…upport

.github/workflows/dependabot_pr.yml

@@ -27,7 +27,7 @@ jobs:
           ./gradlew updateSHAs
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Updating SHAs
           branch: ${{ github.head_ref }}
@@ -40,7 +40,7 @@ jobs:
           ./gradlew spotlessApply
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Spotless formatting
           branch: ${{ github.head_ref }}
@@ -54,7 +54,7 @@ jobs:
           version: 'Unreleased 2.x'
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: "Update changelog"
           branch: ${{ github.head_ref }}

.github/workflows/gradle-check.yml

@@ -82,10 +82,7 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :white_check_mark:
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
+            :white_check_mark: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }})
 
       - name: Extract Test Failure
         if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
@@ -108,10 +105,8 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
+            :grey_exclamation: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }}) ${{ env.test_failures }}
+
             Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
 
       - name: Create Comment Failure
@@ -120,12 +115,9 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
-            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
-            Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
+            :x: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }})
+
+            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green. Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
 
       - name: Create Issue On Push Failure
         if: ${{ github.event_name == 'push' && failure() }}

CHANGELOG.md

@@ -20,7 +20,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
 - [AdmissionControl] Added changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting ([#9286](https://github.com/opensearch-project/OpenSearch/pull/9286))
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
+- Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
+- Add back half_float BKD based sort query optimization ([#11024](https://github.com/opensearch-project/OpenSearch/pull/11024))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -88,12 +90,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Don't over-allocate in HeapBufferedAsyncEntityConsumer in order to consume the response ([#9993](https://github.com/opensearch-project/OpenSearch/pull/9993))
+- [BUG] Fix the thread context that is not properly cleared and messes up the traces ([#10873](https://github.com/opensearch-project/OpenSearch/pull/10873))
+- Handle canMatchSearchAfter for frozen context scenario ([#11249](https://github.com/opensearch-project/OpenSearch/pull/11249))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
+- Request level coordinator slow logs ([#10650](https://github.com/opensearch-project/OpenSearch/pull/10650))
 - [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567))
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
@@ -102,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
 - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
 - [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
+- Allowing pipeline processors to access index mapping info by passing ingest service ref as part of the processor factory parameters ([#10307](https://github.com/opensearch-project/OpenSearch/pull/10307))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
@@ -114,8 +120,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
-- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.1 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858), [#11000](https://github.com/opensearch-project/OpenSearch/pull/11000))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.22.0 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858), [#11000](https://github.com/opensearch-project/OpenSearch/pull/11000), [#11270](https://github.com/opensearch-project/OpenSearch/pull/11270))
 - Bump `aws-actions/configure-aws-credentials` from 2 to 4 ([#10504](https://github.com/opensearch-project/OpenSearch/pull/10504))
+- Bump `stefanzweifel/git-auto-commit-action` from 4 to 5 ([#11171](https://github.com/opensearch-project/OpenSearch/pull/11171))
+- Bump `jackson` and `jackson_databind` from 2.15.2 to 2.16.0 ([#11273](https://github.com/opensearch-project/OpenSearch/pull/11273))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -127,8 +135,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
 - [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
+- Refactor common parts from the Rounding class into a separate 'round' package ([#11023](https://github.com/opensearch-project/OpenSearch/issues/11023))
 - Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
+- Made leader/follower check timeout setting dynamic ([#10528](https://github.com/opensearch-project/OpenSearch/pull/10528))
+- Use iterative approach to evaluate Regex.simpleMatch ([#11060](https://github.com/opensearch-project/OpenSearch/pull/11060))
 
 ### Deprecated
 
@@ -143,6 +154,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
 - Fix SuggestSearch.testSkipDuplicates by forceing refresh when indexing its test documents ([#11068](https://github.com/opensearch-project/OpenSearch/pull/11068))
 - Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095))
+- Fix per request latency last phase not tracked ([#10934](https://github.com/opensearch-project/OpenSearch/pull/10934))
 
 ### Security
 

benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java → benchmarks/src/main/java/org/opensearch/common/round/RoundableBenchmark.java

@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.common;
+package org.opensearch.common.round;
 
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -27,13 +27,13 @@
 @Warmup(iterations = 3, time = 1)
 @Measurement(iterations = 1, time = 1)
 @BenchmarkMode(Mode.Throughput)
-public class ArrayRoundingBenchmark {
+public class RoundableBenchmark {
 
     @Benchmark
-    public void round(Blackhole bh, Options opts) {
-        Rounding.Prepared rounding = opts.supplier.get();
+    public void floor(Blackhole bh, Options opts) {
+        Roundable roundable = opts.supplier.get();
         for (long key : opts.queries) {
-            bh.consume(rounding.round(key));
+            bh.consume(roundable.floor(key));
         }
     }
 
@@ -90,7 +90,7 @@ public static class Options {
         public String distribution;
 
         public long[] queries;
-        public Supplier<Rounding.Prepared> supplier;
+        public Supplier<Roundable> supplier;
 
         @Setup
         public void setup() {
@@ -130,10 +130,10 @@ public void setup() {
 
             switch (type) {
                 case "binary":
-                    supplier = () -> new Rounding.BinarySearchArrayRounding(values, size, null);
+                    supplier = () -> new BinarySearcher(values, size);
                     break;
                 case "linear":
-                    supplier = () -> new Rounding.BidirectionalLinearSearchArrayRounding(values, size, null);
+                    supplier = () -> new BidirectionalLinearSearcher(values, size);
                     break;
                 default:
                     throw new IllegalArgumentException("invalid type: " + type);

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.21.1"
+  implementation "org.apache.logging.log4j:log4j-core:2.22.0"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

buildSrc/version.properties

@@ -7,8 +7,8 @@ bundled_jdk = 21.0.1+12
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.15.2
-jackson_databind  = 2.15.2
+jackson           = 2.16.0
+jackson_databind  = 2.16.0
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0

client/sniffer/licenses/jackson-core-2.16.0.jar.sha1

@@ -0,0 +1 @@
+899e5cf01be55fbf094ad72b2edb0c5df99111ee

distribution/docker/src/docker/config/log4j2.properties

@@ -34,6 +34,16 @@ logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false
 
+appender.search_request_slowlog_json_appender.type = Console
+appender.search_request_slowlog_json_appender.name = search_request_slowlog_json_appender
+appender.search_request_slowlog_json_appender.layout.type = OpenSearchJsonLayout
+appender.search_request_slowlog_json_appender.layout.type_name = search_request_slowlog
+
+logger.search_request_slowlog_logger.name = cluster.search.request.slowlog
+logger.search_request_slowlog_logger.level = trace
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_json_appender.ref = search_request_slowlog_json_appender
+logger.search_request_slowlog_logger.additivity = false
+
 appender.index_search_slowlog_rolling.type = Console
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.layout.type = OpenSearchJsonLayout

distribution/src/config/log4j2.properties

@@ -113,6 +113,47 @@ logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false
 
+######## Search Request Slowlog JSON ####################
+appender.search_request_slowlog_json_appender.type = RollingFile
+appender.search_request_slowlog_json_appender.name = search_request_slowlog_json_appender
+appender.search_request_slowlog_json_appender.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\
+  .cluster_name}_index_search_slowlog.json
+appender.search_request_slowlog_json_appender.filePermissions = rw-r-----
+appender.search_request_slowlog_json_appender.layout.type = OpenSearchJsonLayout
+appender.search_request_slowlog_json_appender.layout.type_name = search_request_slowlog
+appender.search_request_slowlog_json_appender.layout.opensearchmessagefields=message,took,took_millis,phase_took,total_hits,search_type,shards,source,id
+
+appender.search_request_slowlog_json_appender.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\
+  .cluster_name}_index_search_slowlog-%i.json.gz
+appender.search_request_slowlog_json_appender.policies.type = Policies
+appender.search_request_slowlog_json_appender.policies.size.type = SizeBasedTriggeringPolicy
+appender.search_request_slowlog_json_appender.policies.size.size = 1GB
+appender.search_request_slowlog_json_appender.strategy.type = DefaultRolloverStrategy
+appender.search_request_slowlog_json_appender.strategy.max = 4
+#################################################
+######## Search Request Slowlog Log File -  old style pattern ####
+appender.search_request_slowlog_log_appender.type = RollingFile
+appender.search_request_slowlog_log_appender.name = search_request_slowlog_log_appender
+appender.search_request_slowlog_log_appender.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\
+  _index_search_slowlog.log
+appender.search_request_slowlog_log_appender.filePermissions = rw-r-----
+appender.search_request_slowlog_log_appender.layout.type = PatternLayout
+appender.search_request_slowlog_log_appender.layout.pattern = [%d{ISO8601}][%-5p][%c{1.}] [%node_name]%marker %m%n
+
+appender.search_request_slowlog_log_appender.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\
+  _index_search_slowlog-%i.log.gz
+appender.search_request_slowlog_log_appender.policies.type = Policies
+appender.search_request_slowlog_log_appender.policies.size.type = SizeBasedTriggeringPolicy
+appender.search_request_slowlog_log_appender.policies.size.size = 1GB
+appender.search_request_slowlog_log_appender.strategy.type = DefaultRolloverStrategy
+appender.search_request_slowlog_log_appender.strategy.max = 4
+#################################################
+logger.search_request_slowlog_logger.name = cluster.search.request.slowlog
+logger.search_request_slowlog_logger.level = trace
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_json_appender.ref = search_request_slowlog_json_appender
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_log_appender.ref = search_request_slowlog_log_appender
+logger.search_request_slowlog_logger.additivity = false
+
 ######## Search slowlog JSON ####################
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.0.jar.sha1

@@ -0,0 +1 @@
+dc30995f7428c0a405eba9b8c619b20d2b3b9905

distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.0.jar.sha1

@@ -0,0 +1 @@
+3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7

libs/common/src/main/java/org/opensearch/common/CheckedBiConsumer.java

@@ -32,13 +32,16 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.function.BiConsumer;
 
 /**
  * A {@link BiConsumer}-like interface which allows throwing checked exceptions.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface CheckedBiConsumer<T, U, E extends Exception> {
     void accept(T t, U u) throws E;

libs/common/src/main/java/org/opensearch/common/CheckedFunction.java

@@ -32,13 +32,16 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.function.Function;
 
 /**
  * A {@link Function}-like interface which allows throwing checked exceptions.
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface CheckedFunction<T, R, E extends Exception> {
     R apply(T t) throws E;

libs/common/src/main/java/org/opensearch/common/Explicit.java

@@ -32,6 +32,8 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Objects;
 
 /**
@@ -43,8 +45,9 @@
  * field mapping settings it is preferable to preserve an explicit
  * choice rather than a choice made only made implicitly by defaults.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Explicit<T> {
 
     private final T value;

libs/common/src/main/java/org/opensearch/common/TriFunction.java

@@ -32,6 +32,8 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Represents a function that accepts three arguments and produces a result.
  *
@@ -40,8 +42,9 @@
  * @param <U> the type of the third argument
  * @param <R> the return type
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface TriFunction<S, T, U, R> {
     /**

libs/common/src/main/java/org/opensearch/common/collect/Tuple.java

@@ -32,12 +32,15 @@
 
 package org.opensearch.common.collect;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Java 9 Tuple
  * todo: deprecate and remove w/ min jdk upgrade to 11?
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Tuple<V1, V2> {
 
     public static <V1, V2> Tuple<V1, V2> tuple(V1 v1, V2 v2) {

libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java

@@ -7,12 +7,17 @@
  */
 package org.opensearch.common.crypto;
 
+import org.opensearch.common.annotation.ExperimentalApi;
+
 import java.io.Closeable;
 import java.util.Map;
 
 /**
  * Master key provider responsible for management of master keys.
+ *
+ * @opensearch.experimental
  */
+@ExperimentalApi
 public interface MasterKeyProvider extends Closeable {
 
     /**