From cb2e22eef91129cbfbac7ff10da4fad38ab74090 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Fri, 18 Oct 2024 02:07:23 +0000 Subject: [PATCH 1/3] fix number of shards of query index to 0 and auto expand replicas to 0-1 Signed-off-by: Subhobrata Dey --- .../alerting/util/DocLevelMonitorQueries.kt | 3 + .../alerting/MonitorDataSourcesIT.kt | 59 +++++++++++++++++++ .../transport/AlertingSingleNodeTestCase.kt | 7 +++ 3 files changed, 69 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt index d47537c7f..1d7d768f9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt @@ -31,6 +31,7 @@ import org.opensearch.alerting.MonitorRunnerService.monitorCtx import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.client.Client import org.opensearch.cluster.ClusterState +import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.common.unit.TimeValue @@ -130,6 +131,8 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ .alias(Alias(alias)) .settings( Settings.builder().put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") .build() ) return try { diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt index a4e5ad08f..a790b1736 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt @@ -18,6 +18,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest import org.opensearch.action.admin.indices.open.OpenIndexRequest import org.opensearch.action.admin.indices.refresh.RefreshRequest +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse import org.opensearch.action.bulk.BulkRequest import org.opensearch.action.bulk.BulkResponse import org.opensearch.action.fieldcaps.FieldCapabilitiesRequest @@ -30,6 +31,7 @@ import org.opensearch.alerting.transport.AlertingSingleNodeTestCase import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.DocLevelMonitorQueries.Companion.INDEX_PATTERN_SUFFIX import org.opensearch.alerting.workflow.CompositeWorkflowRunner +import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentHelper @@ -6174,4 +6176,61 @@ class MonitorDataSourcesIT : AlertingSingleNodeTestCase() { Assert.assertEquals(completedAlert1.alerts[0].state, Alert.State.COMPLETED) Assert.assertTrue(completedAlert1.alerts[0].endTime!! > acknowledgedAlert.alerts[0].lastNotificationTime!!) } + + fun `test query index created with single primary and single replica shard`() { + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf()) + val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery)) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + var monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger), + dataSources = DataSources(queryIndex = ".opensearch-alerting-custom-queries") + ) + val monitorResponse = createMonitor(monitor) + val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS)) + val testDoc = """{ + "message" : "This is an error from IAD region", + "test_strict_date_time" : "$testTime", + "test_field" : "us-west-2" + }""" + assertFalse(monitorResponse?.id.isNullOrEmpty()) + monitor = monitorResponse!!.monitor + indexDoc(index, "1", testDoc) + val id = monitorResponse.id + executeMonitor(monitor, id, false) + + var response: GetSettingsResponse? = getIndexSettings(".opensearch-alerting-custom-queries-000001") + assertEquals( + "1", + response!!.getSetting( + ".opensearch-alerting-custom-queries-000001", + IndexMetadata.SETTING_NUMBER_OF_SHARDS + ) + ) + assertEquals( + "0", + response.getSetting( + ".opensearch-alerting-custom-queries-000001", + IndexMetadata.SETTING_NUMBER_OF_REPLICAS + ) + ) + + executeMonitor(monitor, id, false) + + response = getIndexSettings(".opensearch-alerting-custom-queries-000001") + assertEquals( + "1", + response!!.getSetting( + ".opensearch-alerting-custom-queries-000001", + IndexMetadata.SETTING_NUMBER_OF_SHARDS + ) + ) + assertEquals( + "0", + response.getSetting( + ".opensearch-alerting-custom-queries-000001", + IndexMetadata.SETTING_NUMBER_OF_REPLICAS + ) + ) + } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt index 4198f0afd..9979f7969 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt @@ -13,6 +13,8 @@ import org.opensearch.action.admin.indices.get.GetIndexRequestBuilder import org.opensearch.action.admin.indices.get.GetIndexResponse import org.opensearch.action.admin.indices.refresh.RefreshAction import org.opensearch.action.admin.indices.refresh.RefreshRequest +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.WriteRequest import org.opensearch.alerting.AlertingPlugin @@ -493,6 +495,11 @@ abstract class AlertingSingleNodeTestCase : OpenSearchSingleNodeTestCase() { return client().execute(ExecuteWorkflowAction.INSTANCE, request).get() } + protected fun getIndexSettings(index: String): GetSettingsResponse? { + val request = GetSettingsRequest().indices(index) + return client().admin().indices().getSettings(request).get() + } + override fun nodeSettings(): Settings { return Settings.builder() .put(super.nodeSettings()) From d66c0e18c49854c0b1f3a762c408e2a8c145c084 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Fri, 18 Oct 2024 02:21:23 +0000 Subject: [PATCH 2/3] remove jdk-11 builds Signed-off-by: Subhobrata Dey --- .github/workflows/multi-node-test-workflow.yml | 2 +- .github/workflows/security-test-workflow.yml | 2 +- .github/workflows/test-workflow.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 7944f43db..f63ef1cc3 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -20,7 +20,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [ 11, 17 ] + java: [ 17 ] # Job name name: Build and test Alerting # This job runs on Linux diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index 5ca093c4e..0c6e3be68 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -12,7 +12,7 @@ jobs: build: strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 17, 21 ] # Job name name: Build and test Alerting # This job runs on Linux diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml index 626c44565..0e941237a 100644 --- a/.github/workflows/test-workflow.yml +++ b/.github/workflows/test-workflow.yml @@ -23,7 +23,7 @@ jobs: WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: - java: [11, 17, 21] + java: [17, 21] # Job name name: Build Alerting with JDK ${{ matrix.java }} on Linux # This job runs on Linux @@ -71,7 +71,7 @@ jobs: WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: - java: [11, 17, 21] + java: [17, 21] os: [ windows-latest, macos-latest ] include: - os: windows-latest From 537809d8f15eb511ae5718ca1faf62fdd69baf9c Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Fri, 18 Oct 2024 02:27:21 +0000 Subject: [PATCH 3/3] only support jdk-21 Signed-off-by: Subhobrata Dey --- .github/workflows/maven-publish.yml | 2 +- .github/workflows/multi-node-test-workflow.yml | 2 +- .github/workflows/security-test-workflow.yml | 2 +- .github/workflows/test-workflow.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 75f22834a..e822e15cb 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [17] + jdk: [21] platform: ["ubuntu-latest"] if: github.repository == 'opensearch-project/alerting' runs-on: ${{ matrix.platform }} diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index f63ef1cc3..7baa62806 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -20,7 +20,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [ 17 ] + java: [ 21 ] # Job name name: Build and test Alerting # This job runs on Linux diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index 0c6e3be68..fd40a62e9 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -12,7 +12,7 @@ jobs: build: strategy: matrix: - java: [ 17, 21 ] + java: [ 21 ] # Job name name: Build and test Alerting # This job runs on Linux diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml index 0e941237a..aabd8b49c 100644 --- a/.github/workflows/test-workflow.yml +++ b/.github/workflows/test-workflow.yml @@ -23,7 +23,7 @@ jobs: WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: - java: [17, 21] + java: [21] # Job name name: Build Alerting with JDK ${{ matrix.java }} on Linux # This job runs on Linux @@ -71,7 +71,7 @@ jobs: WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: - java: [17, 21] + java: [21] os: [ windows-latest, macos-latest ] include: - os: windows-latest