From 754f789c2e938b0610a3cb57db1ce0b19f815d27 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 4 Oct 2024 14:36:18 -0700 Subject: [PATCH] Add library to handle integration test failures notifications Signed-off-by: Sayali Gaikawad --- src/jenkins/ComponentBuildStatus.groovy | 56 +++- src/jenkins/ComponentIntegTestStatus.groovy | 123 ++++++++ .../CreateIntegTestMarkDownTable.groovy | 41 +++ .../TestComponentIntegTestStatus.groovy | 290 ++++++++++++++++++ .../TestCreateIntegTestMarkDownTable.groovy | 52 ++++ .../TestUpdateIntegTestFailureIssues.groovy | 248 +++++++++++++++ .../UpdateIntegTestFailureIssues_Jenkinsfile | 24 ++ ...dateIntegTestFailureIssues_Jenkinsfile.txt | 139 +++++++++ ...dateIntegTestFailureIssuesLibTester.groovy | 62 ++++ vars/publishIntegTestResults.groovy | 3 + vars/updateBuildFailureIssues.groovy | 1 - vars/updateIntegTestFailureIssues.groovy | 85 +++++ 12 files changed, 1122 insertions(+), 2 deletions(-) create mode 100644 src/jenkins/ComponentIntegTestStatus.groovy create mode 100644 src/jenkins/CreateIntegTestMarkDownTable.groovy create mode 100644 tests/jenkins/TestComponentIntegTestStatus.groovy create mode 100644 tests/jenkins/TestCreateIntegTestMarkDownTable.groovy create mode 100644 tests/jenkins/TestUpdateIntegTestFailureIssues.groovy create mode 100644 tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile create mode 100644 tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile.txt create mode 100644 tests/jenkins/lib-testers/UpdateIntegTestFailureIssuesLibTester.groovy create mode 100644 vars/updateIntegTestFailureIssues.groovy diff --git a/src/jenkins/ComponentBuildStatus.groovy b/src/jenkins/ComponentBuildStatus.groovy index 8a2f6c3a..45e3669f 100644 --- a/src/jenkins/ComponentBuildStatus.groovy +++ b/src/jenkins/ComponentBuildStatus.groovy @@ -24,6 +24,7 @@ class ComponentBuildStatus { String buildStartTimeFrom String buildStartTimeTo def script + OpenSearchMetricsQuery openSearchMetricsQuery ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, String buildStartTimeFrom, String buildStartTimeTo, def script) { this.metricsUrl = metricsUrl @@ -37,6 +38,19 @@ class ComponentBuildStatus { this.buildStartTimeFrom = buildStartTimeFrom this.buildStartTimeTo = buildStartTimeTo this.script = script + this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script) + } + + ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.indexName = indexName + this.product = product + this.version = version + this.script = script + this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script) } def getQuery(String componentBuildResult) { @@ -83,9 +97,49 @@ class ComponentBuildStatus { return query.replace('"', '\\"') } + def getLatestDistributionBuildNumberQuery() { + def queryMap = [ + size : 1, + _source: [ + "distribution_build_number", + ], + query: [ + bool: [ + filter: [ + [ + match_phrase: [ + component_category: "${this.product}" + ] + ], + [ + match_phrase: [ + version: "${this.version}" + ] + ] + ] + ] + ], + sort : [ + [ + build_start_time: [ + order: "desc" + ] + ] + ] + ] + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + def getComponents(String componentBuildResult) { - def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(componentBuildResult)) + def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(getQuery(componentBuildResult)) def components = jsonResponse.hits.hits.collect { it._source.component } return components } + + def getLatestDistributionBuildNumber(String version) { + def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(getQuery(getLatestDistributionBuildNumberQuery)) + def latestDistributionBuildNumber = jsonResponse.hits.hits[0]._source.distribution_build_number + return latestDistributionBuildNumber + } } diff --git a/src/jenkins/ComponentIntegTestStatus.groovy b/src/jenkins/ComponentIntegTestStatus.groovy new file mode 100644 index 00000000..f43a1ac6 --- /dev/null +++ b/src/jenkins/ComponentIntegTestStatus.groovy @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins + +import groovy.json.JsonOutput +import utils.OpenSearchMetricsQuery + +class ComponentIntegTestStatus { + String metricsUrl + String awsAccessKey + String awsSecretKey + String awsSessionToken + String indexName + String product + String version + String distributionBuildNumber + def script + OpenSearchMetricsQuery openSearchMetricsQuery + + ComponentIntegTestStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.indexName = indexName + this.product = product + this.version = version + this.distributionBuildNumber = distributionBuildNumber + this.script = script + this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script) + } + + def getQuery(String componentIntegTestResult) { + def queryMap = [ + size: 50, + _source: [ + "component" + ], + query: [ + bool: [ + filter: [ + [ + match_phrase: [ + version : "${this.version}" + ] + ], + [ + match_phrase: [ + component_category: "${this.product}" + ] + ], + [ + match_phrase: [ + distribution_build_number : "${this.distributionBuildNumber}" + ] + ], + [ + match_phrase: [ + component_build_result: "${componentIntegTestResult}" + ] + ] + ] + ] + ] + ] + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + + def componentIntegTestFailedDataQuery(String component) { + def queryMap = [ + _source : [ + "platform", + "architecture", + "distribution", + "test_report_manifest_yml", + "integ_test_build_url" + ], + query: [ + bool: [ + filter: [ + [ + match_phrase: [ + component: "${component}" + ] + ], + [ + match_phrase: [ + version: "${this.version}" + ] + ], + [ + match_phrase: [ + distribution_build_number: "${this.distributionBuildNumber}" + ] + ] + ] + ] + ] + ] + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + + def getComponents(String componentBuildResult) { + def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(getQuery(componentBuildResult)) + def components = jsonResponse.hits.hits.collect { it._source.component } + return components + } + + def getComponentIntegTestFailedData(String component) { + def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(componentIntegTestFailedDataQuery(component)) + return jsonResponse + } + +} diff --git a/src/jenkins/CreateIntegTestMarkDownTable.groovy b/src/jenkins/CreateIntegTestMarkDownTable.groovy new file mode 100644 index 00000000..1f71d227 --- /dev/null +++ b/src/jenkins/CreateIntegTestMarkDownTable.groovy @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins + +class CreateIntegTestMarkDownTable { + String version + ArrayList tableData + + CreateIntegTestMarkDownTable(String version, List> tableData) { + this.version = version + this.tableData = tableData + } + + def create() { + + def tableHeader = """ +### Integration Test Failed for version ${version}. See the specifications below: + +#### Details + +| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run | +|----------|--------------|--------------|----------------------|--------------| +""" + def tableRows = this.tableData.collect { row -> + "| ${row.platform} | ${row.distribution} | ${row.architecture} | ${row.test_report_manifest_yml} | ${row.integ_test_build_url}" + }.join("\n") + + def additionalInformation = """ +\nCheck out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa). +""" + return tableHeader + tableRows + additionalInformation + } + +} \ No newline at end of file diff --git a/tests/jenkins/TestComponentIntegTestStatus.groovy b/tests/jenkins/TestComponentIntegTestStatus.groovy new file mode 100644 index 00000000..f4fa6f37 --- /dev/null +++ b/tests/jenkins/TestComponentIntegTestStatus.groovy @@ -0,0 +1,290 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins + +import org.junit.* +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.mock.interceptor.MockFor + +class TestComponentIntegTestStatus { + + private ComponentIntegTestStatus componentIntegTestStatus + private final String metricsUrl = 'http://example.com' + private final String awsAccessKey = 'testAccessKey' + private final String awsSecretKey = 'testSecretKey' + private final String awsSessionToken = 'testSessionToken' + private final String indexName = 'opensearch-integration-test-results' + private final String product = "OpenSearch" + private final String version = "2.18.0" + private final String distributionBuildNumber = "4891" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "took": 10, + "timed_out": false, + "_shards": { + "total": 20, + "successful": 20, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 35, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "Tw1IS5IBpSkIYPznAxki", + "_score": 0, + "_source": { + "component": "cross-cluster-replication" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "Ug1IS5IBpSkIYPznAxki", + "_score": 0, + "_source": { + "component": "k-NN" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "pltLS5IBIpIPk1eDbs5_", + "_score": 0, + "_source": { + "component": "cross-cluster-replication" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "qFtLS5IBIpIPk1eDbs5_", + "_score": 0, + "_source": { + "component": "index-management" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "q1tLS5IBIpIPk1eDbs5_", + "_score": 0, + "_source": { + "component": "neural-search" + } + } + ] + } + } + """ + } + return "" + } + componentIntegTestStatus = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, distributionBuildNumber, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def expectedOutput = JsonOutput.toJson([ + size: 50, + _source: [ + "component", + ], + query: [ + bool: [ + filter: [ + [ + match_phrase: [ + version : "2.18.0" + ] + ], + [ + match_phrase: [ + component_category: "OpenSearch" + ] + ], + [ + match_phrase: [ + distribution_build_number : "4891" + ] + ], + [ + match_phrase: [ + component_build_result: "failed" + ] + ] + ] + ] + ] + ]).replace('"', '\\"') + + def result = componentIntegTestStatus.getQuery('failed') + assert result == expectedOutput + } + + @Test + void testComponentIntegTestFailedDataQuery() { + def expectedOutput = JsonOutput.toJson([ + _source : [ + "platform", + "architecture", + "distribution", + "test_report_manifest_yml", + "integ_test_build_url" + ], + query: [ + bool: [ + filter: [ + [ + match_phrase: [ + component: "k-NN" + ] + ], + [ + match_phrase: [ + version: "2.18.0" + ] + ], + [ + match_phrase: [ + distribution_build_number: "4891" + ] + ] + ] + ] + ] + ]).replace('"', '\\"') + def result = componentIntegTestStatus.componentIntegTestFailedDataQuery('k-NN') + assert result == expectedOutput + } + + @Test + void testGetComponents() { + def expectedOutput = ['cross-cluster-replication', 'k-NN', 'cross-cluster-replication', 'index-management', 'neural-search'] + def result = componentIntegTestStatus.getComponents('failed') + + assert result == expectedOutput + } + + @Test + void testGetComponentIntegTestFailedData() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "took": 5, + "timed_out": false, + "_shards": { + "total": 20, + "successful": 20, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "wArzVZIB2OP_jOaCFPPY", + "_score": 0, + "_source": { + "test_report_manifest_yml": "https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml", + "integ_test_build_url": "https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6561/display/redirect", + "distribution": "tar", + "platform": "linux", + "architecture": "x64" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "jVsOVpIBIpIPk1eDrdI3", + "_score": 0, + "_source": { + "test_report_manifest_yml": "https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml", + "integ_test_build_url": "https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6560/display/redirect", + "distribution": "tar", + "platform": "linux", + "architecture": "arm64" + } + } + ] + } + } + """ + } + return "" + } + componentIntegTestStatus = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, distributionBuildNumber, script) + def componentData = ''' + { + "took": 5, + "timed_out": false, + "_shards": { + "total": 20, + "successful": 20, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "wArzVZIB2OP_jOaCFPPY", + "_score": 0, + "_source": { + "test_report_manifest_yml": "https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml", + "integ_test_build_url": "https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6561/display/redirect", + "distribution": "tar", + "platform": "linux", + "architecture": "x64" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "jVsOVpIBIpIPk1eDrdI3", + "_score": 0, + "_source": { + "test_report_manifest_yml": "https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml", + "integ_test_build_url": "https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6560/display/redirect", + "distribution": "tar", + "platform": "linux", + "architecture": "arm64" + } + } + ] + } + } + ''' + def expectedOutput = new JsonSlurper().parseText(componentData) + def result = componentIntegTestStatus.getComponentIntegTestFailedData('observabilityDashboards') + + assert result == expectedOutput + } + +} diff --git a/tests/jenkins/TestCreateIntegTestMarkDownTable.groovy b/tests/jenkins/TestCreateIntegTestMarkDownTable.groovy new file mode 100644 index 00000000..32165d98 --- /dev/null +++ b/tests/jenkins/TestCreateIntegTestMarkDownTable.groovy @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins + +import org.junit.* + + +class TestCreateIntegTestMarkDownTable { + + @Test + void testCreateIntegTestMarkDownTableWithSampleData() { + def version = "2.18.0" + def tableData = [ + [ + architecture:"x64", + distribution:"tar", + integ_test_build_url:"https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6561/display/redirect", + platform:"linux", + test_report_manifest_yml:"https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml" + ], + [ + architecture:"arm64", + distribution:"tar", + integ_test_build_url:"https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6560/display/redirect", + platform:"linux", + test_report_manifest_yml:"https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml" + ] + ] + def createIntegTestMarkDownTable = new CreateIntegTestMarkDownTable(version, tableData) + def result = createIntegTestMarkDownTable.create() + def expectedOutput = """ +### Integration Test Failed for version 2.18.0. See the specifications below: + +#### Details + +| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run | +|----------|--------------|--------------|----------------------|--------------| +| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6561/display/redirect +| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/2.18.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/6560/display/redirect + +Check out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa). +""" + assert result == expectedOutput + } +} diff --git a/tests/jenkins/TestUpdateIntegTestFailureIssues.groovy b/tests/jenkins/TestUpdateIntegTestFailureIssues.groovy new file mode 100644 index 00000000..0b7d8674 --- /dev/null +++ b/tests/jenkins/TestUpdateIntegTestFailureIssues.groovy @@ -0,0 +1,248 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package jenkins.tests + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.CoreMatchers.not +import static org.hamcrest.MatcherAssert.assertThat +import groovy.json.JsonSlurper +import utils.OpenSearchMetricsQuery + +class TestUpdateIntegTestFailureIssues extends BuildPipelineTest { + + @Override + @Before + void setUp() { + this.registerLibTester(new UpdateIntegTestFailureIssuesLibTester('tests/data/opensearch-2.2.0.yml', '4891')) + super.setUp() + def unformattedResponseForPass = ''' + { + "took": 10, + "timed_out": false, + "_shards": { + "total": 20, + "successful": 20, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 35, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "Tw1IS5IBpSkIYPznAxki", + "_score": 0, + "_source": { + "component": "cross-cluster-replication" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "Ug1IS5IBpSkIYPznAxki", + "_score": 0, + "_source": { + "component": "k-NN" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "pltLS5IBIpIPk1eDbs5_", + "_score": 0, + "_source": { + "component": "cross-cluster-replication" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "qFtLS5IBIpIPk1eDbs5_", + "_score": 0, + "_source": { + "component": "index-management" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "q1tLS5IBIpIPk1eDbs5_", + "_score": 0, + "_source": { + "component": "neural-search" + } + } + ] + } + } + ''' + def unformattedResponseForFail = ''' + { + "took": 10, + "timed_out": false, + "_shards": { + "total": 20, + "successful": 20, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 35, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "Tw1IS5IBpSkIYPznAxki", + "_score": 0, + "_source": { + "component": "geospatial" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "Ug1IS5IBpSkIYPznAxki", + "_score": 0, + "_source": { + "component": "k-NN" + } + } + ] + } + } + ''' + def failedTestDataResponse = ''' + { + "took": 5, + "timed_out": false, + "_shards": { + "total": 20, + "successful": 20, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "wArzVZIB2OP_jOaCFPPY", + "_score": 0, + "_source": { + "test_report_manifest_yml": "https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml", + "integ_test_build_url": "https://build.ci.opensearch.org/job/integ-test/6561/display/redirect", + "distribution": "tar", + "platform": "linux", + "architecture": "x64" + } + }, + { + "_index": "opensearch-integration-test-results-10-2024", + "_id": "jVsOVpIBIpIPk1eDrdI3", + "_score": 0, + "_source": { + "test_report_manifest_yml": "https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml", + "integ_test_build_url": "https://build.ci.opensearch.org/job/integ-test/6560/display/redirect", + "distribution": "tar", + "platform": "linux", + "architecture": "arm64" + } + } + ] + } + } + ''' + helper.addShMock("""\n set -e\n set +x\n curl -s -XGET \"sample.url/opensearch-integration-test-results/_search\" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"abc:xyz\" -H \"x-amz-security-token:sampleToken\" -H 'Content-Type: application/json' -d \"{\\"size\\":50,\\"_source\\":[\\"component\\"],\\"query\\":{\\"bool\\":{\\"filter\\":[{\\"match_phrase\\":{\\"version\\":\\"2.2.0\\"}},{\\"match_phrase\\":{\\"component_category\\":\\"OpenSearch\\"}},{\\"match_phrase\\":{\\"distribution_build_number\\":\\"4891\\"}},{\\"match_phrase\\":{\\"component_build_result\\":\\"passed\\"}}]}}}\" | jq '.'\n """) { script -> + return [stdout: unformattedResponseForPass, exitValue: 0] + } + helper.addShMock("""\n set -e\n set +x\n curl -s -XGET \"sample.url/opensearch-integration-test-results/_search\" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"abc:xyz\" -H \"x-amz-security-token:sampleToken\" -H 'Content-Type: application/json' -d \"{\\"size\\":50,\\"_source\\":[\\"component\\"],\\"query\\":{\\"bool\\":{\\"filter\\":[{\\"match_phrase\\":{\\"version\\":\\"2.2.0\\"}},{\\"match_phrase\\":{\\"component_category\\":\\"OpenSearch\\"}},{\\"match_phrase\\":{\\"distribution_build_number\\":\\"4891\\"}},{\\"match_phrase\\":{\\"component_build_result\\":\\"failed\\"}}]}}}\" | jq '.'\n """) { script -> + return [stdout: unformattedResponseForFail, exitValue: 0] + } + helper.addShMock("""\n set -e\n set +x\n curl -s -XGET \"sample.url/opensearch-integration-test-results/_search\" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"abc:xyz\" -H \"x-amz-security-token:sampleToken\" -H 'Content-Type: application/json' -d \"{\\"_source\\":[\\"platform\\",\\"architecture\\",\\"distribution\\",\\"test_report_manifest_yml\\",\\"integ_test_build_url\\"],\\"query\\":{\\"bool\\":{\\"filter\\":[{\\"match_phrase\\":{\\"component\\":\\"geospatial\\"}},{\\"match_phrase\\":{\\"version\\":\\"2.2.0\\"}},{\\"match_phrase\\":{\\"distribution_build_number\\":\\"4891\\"}}]}}}\" | jq '.'\n """) { script -> + return [stdout: failedTestDataResponse, exitValue: 0] + } + helper.addShMock("""\n set -e\n set +x\n curl -s -XGET \"sample.url/opensearch-integration-test-results/_search\" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"abc:xyz\" -H \"x-amz-security-token:sampleToken\" -H 'Content-Type: application/json' -d \"{\\"_source\\":[\\"platform\\",\\"architecture\\",\\"distribution\\",\\"test_report_manifest_yml\\",\\"integ_test_build_url\\"],\\"query\\":{\\"bool\\":{\\"filter\\":[{\\"match_phrase\\":{\\"component\\":\\"k-NN\\"}},{\\"match_phrase\\":{\\"version\\":\\"2.2.0\\"}},{\\"match_phrase\\":{\\"distribution_build_number\\":\\"4891\\"}}]}}}\" | jq '.'\n """) { script -> + return [stdout: failedTestDataResponse, exitValue: 0] + } + } + + @Test + public void testIssueCreation() { + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/k-NN.git -S "[AUTOCUT] Integration Test Failed for geospatial-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> + return [stdout: "", exitValue: 0] + } + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/k-NN.git -S "[AUTOCUT] Integration Test Failed for geospatial-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> + return [stdout: "", exitValue: 0] + } + super.testPipeline('tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile') + assertThat(getCommands('println', ''), hasItem('Integration test failed for k-NN, creating github issue')) + assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue edit bbb\nccc --repo https://github.com/opensearch-project/k-NN.git --body \"\n### Integration Test Failed for version 2.2.0. See the specifications below:\n\n#### Details\n\n| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run |\n|----------|--------------|--------------|----------------------|--------------|\n| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6561/display/redirect\n| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6560/display/redirect\n\nCheck out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa).\n\", returnStdout=true}")) + } + + @Test + public void testGithubIssueEdit() { + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Integration Test Failed for geospatial-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> + return [stdout: "22", exitValue: 0] + } + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Integration Test Failed for geospatial-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> + return [stdout: "", exitValue: 0] + } + runScript('tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile') + assertThat(getCommands('println', ''), hasItem('Issue already exists, editing the issue body')) + assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue edit bbb\nccc --repo https://github.com/opensearch-project/geospatial.git --body \"\n### Integration Test Failed for version 2.2.0. See the specifications below:\n\n#### Details\n\n| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run |\n|----------|--------------|--------------|----------------------|--------------|\n| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6561/display/redirect\n| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6560/display/redirect\n\nCheck out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa).\n\", returnStdout=true}")) + } + + @Test + public void testClosingGithubIssueOnSuccess() { + helper.addShMock("date -d \"3 days ago\" +'%Y-%m-%d'") { script -> + return [stdout: "2023-10-24", exitValue: 0] + } + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/cross-cluster-replication.git -S "[AUTOCUT] Integration Test Failed for cross-cluster-replication-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> + return [stdout: "30", exitValue: 0] + } + runScript('tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile') + assertThat(getCommands('sh', 'cross-cluster-replication'), hasItem("{script=gh issue list --repo https://github.com/opensearch-project/cross-cluster-replication.git -S \"[AUTOCUT] Integration Test Failed for cross-cluster-replication-2.2.0 in:title\" --json number --jq '.[0].number', returnStdout=true}")) + assertThat(getCommands('sh', 'cross-cluster-replication'), hasItem("{script=gh issue close bbb\nccc -R opensearch-project/cross-cluster-replication --comment \"Closing the issue as the integration tests for cross-cluster-replication passed for version: **2.2.0**.\", returnStdout=true}")) + } + + @Test + public void testNotClosingGithubIssueOnOneFailure() { + helper.addShMock("date -d \"3 days ago\" +'%Y-%m-%d'") { script -> + return [stdout: "2023-10-24", exitValue: 0] + } + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/k-NN.git -S "[AUTOCUT] Integration Test Failed for k-NN-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> + return [stdout: "20", exitValue: 0] + } + runScript('tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile') + assertThat(getCommands('sh', 'k-NN'), not(hasItem("{script=gh issue close bbb\nccc -R opensearch-project/k-NN --comment \"Closing the issue as the integration tests for k-NN passed for version: **2.2.0**.\", returnStdout=true}"))) + assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue edit bbb\nccc --repo https://github.com/opensearch-project/geospatial.git --body \"\n### Integration Test Failed for version 2.2.0. See the specifications below:\n\n#### Details\n\n| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run |\n|----------|--------------|--------------|----------------------|--------------|\n| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6561/display/redirect\n| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6560/display/redirect\n\nCheck out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa).\n\", returnStdout=true}")) + } + + def getCommands(method, text) { + def shCommands = helper.callStack.findAll { call -> + call.methodName == method + }.collect { call -> + callArgsToString(call) + }.findAll { command -> + command.contains(text) + } + return shCommands + } + +} diff --git a/tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile b/tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile new file mode 100644 index 00000000..1880e9db --- /dev/null +++ b/tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('updateIntegTestFailureIssues') { + steps { + script { + updateIntegTestFailureIssues( + inputManifestPath: 'tests/data/opensearch-2.2.0.yml', + distributionBuildNumber: '4891' + ) + } + } + } + } +} diff --git a/tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile.txt b/tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile.txt new file mode 100644 index 00000000..111fb850 --- /dev/null +++ b/tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile.txt @@ -0,0 +1,139 @@ + UpdateIntegTestFailureIssues_Jenkinsfile.run() + UpdateIntegTestFailureIssues_Jenkinsfile.pipeline(groovy.lang.Closure) + UpdateIntegTestFailureIssues_Jenkinsfile.echo(Executing on agent [label:none]) + UpdateIntegTestFailureIssues_Jenkinsfile.stage(updateIntegTestFailureIssues, groovy.lang.Closure) + UpdateIntegTestFailureIssues_Jenkinsfile.script(groovy.lang.Closure) + UpdateIntegTestFailureIssues_Jenkinsfile.updateIntegTestFailureIssues({inputManifestPath=tests/data/opensearch-2.2.0.yml, distributionBuildNumber=4891}) + updateIntegTestFailureIssues.readYaml({file=tests/data/opensearch-2.2.0.yml}) + updateIntegTestFailureIssues.string({credentialsId=jenkins-health-metrics-account-number, variable=METRICS_HOST_ACCOUNT}) + updateIntegTestFailureIssues.string({credentialsId=jenkins-health-metrics-cluster-endpoint, variable=METRICS_HOST_URL}) + updateIntegTestFailureIssues.withCredentials([METRICS_HOST_ACCOUNT, METRICS_HOST_URL], groovy.lang.Closure) + updateIntegTestFailureIssues.withAWS({role=OpenSearchJenkinsAccessRole, roleAccount=METRICS_HOST_ACCOUNT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + ComponentIntegTestStatus.getComponents(passed) + OpenSearchMetricsQuery.fetchMetrics({\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"passed\"}}]}}}) + updateIntegTestFailureIssues.sh({script= + set -e + set +x + curl -s -XGET "sample.url/opensearch-integration-test-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"passed\"}}]}}}" | jq '.' + , returnStdout=true}) + ComponentIntegTestStatus.getComponents(failed) + OpenSearchMetricsQuery.fetchMetrics({\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"failed\"}}]}}}) + updateIntegTestFailureIssues.sh({script= + set -e + set +x + curl -s -XGET "sample.url/opensearch-integration-test-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"failed\"}}]}}}" | jq '.' + , returnStdout=true}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.println(Integration test failed for geospatial, creating github issue) + ComponentIntegTestStatus.getComponentIntegTestFailedData(geospatial) + OpenSearchMetricsQuery.fetchMetrics({\"_source\":[\"platform\",\"architecture\",\"distribution\",\"test_report_manifest_yml\",\"integ_test_build_url\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component\":\"geospatial\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}}]}}}) + updateIntegTestFailureIssues.sh({script= + set -e + set +x + curl -s -XGET "sample.url/opensearch-integration-test-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"_source\":[\"platform\",\"architecture\",\"distribution\",\"test_report_manifest_yml\",\"integ_test_build_url\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component\":\"geospatial\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}}]}}}" | jq '.' + , returnStdout=true}) + CreateIntegTestMarkDownTable.create() + updateIntegTestFailureIssues.createGithubIssue({repoUrl=https://github.com/opensearch-project/geospatial.git, issueTitle=[AUTOCUT] Integration Test Failed for geospatial-2.2.0, issueBody= +### Integration Test Failed for version 2.2.0. See the specifications below: + +#### Details + +| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run | +|----------|--------------|--------------|----------------------|--------------| +| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6561/display/redirect +| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6560/display/redirect + +Check out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa). +, label=autocut,v2.2.0, issueEdit=true}) + createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Integration Test Failed for geospatial-2.2.0 in:title" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Integration Test Failed for geospatial-2.2.0 in:title is:closed closed:>=bbb +ccc" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.println(Issue already exists, editing the issue body) + createGithubIssue.sh({script=gh issue edit bbb +ccc --repo https://github.com/opensearch-project/geospatial.git --body " +### Integration Test Failed for version 2.2.0. See the specifications below: + +#### Details + +| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run | +|----------|--------------|--------------|----------------------|--------------| +| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6561/display/redirect +| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6560/display/redirect + +Check out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa). +", returnStdout=true}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.println(Integration test failed for k-NN, creating github issue) + ComponentIntegTestStatus.getComponentIntegTestFailedData(k-NN) + OpenSearchMetricsQuery.fetchMetrics({\"_source\":[\"platform\",\"architecture\",\"distribution\",\"test_report_manifest_yml\",\"integ_test_build_url\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component\":\"k-NN\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}}]}}}) + updateIntegTestFailureIssues.sh({script= + set -e + set +x + curl -s -XGET "sample.url/opensearch-integration-test-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"_source\":[\"platform\",\"architecture\",\"distribution\",\"test_report_manifest_yml\",\"integ_test_build_url\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component\":\"k-NN\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}}]}}}" | jq '.' + , returnStdout=true}) + CreateIntegTestMarkDownTable.create() + updateIntegTestFailureIssues.createGithubIssue({repoUrl=https://github.com/opensearch-project/k-NN.git, issueTitle=[AUTOCUT] Integration Test Failed for k-NN-2.2.0, issueBody= +### Integration Test Failed for version 2.2.0. See the specifications below: + +#### Details + +| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run | +|----------|--------------|--------------|----------------------|--------------| +| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6561/display/redirect +| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6560/display/redirect + +Check out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa). +, label=autocut,v2.2.0, issueEdit=true}) + createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/k-NN.git -S "[AUTOCUT] Integration Test Failed for k-NN-2.2.0 in:title" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/k-NN.git -S "[AUTOCUT] Integration Test Failed for k-NN-2.2.0 in:title is:closed closed:>=bbb +ccc" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.println(Issue already exists, editing the issue body) + createGithubIssue.sh({script=gh issue edit bbb +ccc --repo https://github.com/opensearch-project/k-NN.git --body " +### Integration Test Failed for version 2.2.0. See the specifications below: + +#### Details + +| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run | +|----------|--------------|--------------|----------------------|--------------| +| linux | tar | x64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/x64/tar/test-results/6561/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6561/display/redirect +| linux | tar | arm64 | https://ci.opensearch.org/ci/dbc/integ-test/2.2.0/7984/linux/arm64/tar/test-results/6560/integ-test/test-report.yml | https://build.ci.opensearch.org/job/integ-test/6560/display/redirect + +Check out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa). +", returnStdout=true}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.println(Integration tests passed for cross-cluster-replication, closing github issue) + updateIntegTestFailureIssues.closeGithubIssue({repoUrl=https://github.com/opensearch-project/cross-cluster-replication.git, issueTitle=[AUTOCUT] Integration Test Failed for cross-cluster-replication-2.2.0, closeComment=Closing the issue as the integration tests for cross-cluster-replication passed for version: **2.2.0**.}) + closeGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + closeGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + closeGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/cross-cluster-replication.git -S "[AUTOCUT] Integration Test Failed for cross-cluster-replication-2.2.0 in:title" --json number --jq '.[0].number', returnStdout=true}) + closeGithubIssue.sh({script=gh issue close bbb +ccc -R opensearch-project/cross-cluster-replication --comment "Closing the issue as the integration tests for cross-cluster-replication passed for version: **2.2.0**.", returnStdout=true}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.println(Integration tests passed for index-management, closing github issue) + updateIntegTestFailureIssues.closeGithubIssue({repoUrl=https://github.com/opensearch-project/index-management.git, issueTitle=[AUTOCUT] Integration Test Failed for index-management-2.2.0, closeComment=Closing the issue as the integration tests for index-management passed for version: **2.2.0**.}) + closeGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + closeGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + closeGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/index-management.git -S "[AUTOCUT] Integration Test Failed for index-management-2.2.0 in:title" --json number --jq '.[0].number', returnStdout=true}) + closeGithubIssue.sh({script=gh issue close bbb +ccc -R opensearch-project/index-management --comment "Closing the issue as the integration tests for index-management passed for version: **2.2.0**.", returnStdout=true}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) + updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS}) diff --git a/tests/jenkins/lib-testers/UpdateIntegTestFailureIssuesLibTester.groovy b/tests/jenkins/lib-testers/UpdateIntegTestFailureIssuesLibTester.groovy new file mode 100644 index 00000000..a041e8de --- /dev/null +++ b/tests/jenkins/lib-testers/UpdateIntegTestFailureIssuesLibTester.groovy @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.CoreMatchers.nullValue +import static org.hamcrest.MatcherAssert.assertThat + +class UpdateIntegTestFailureIssuesLibTester extends LibFunctionTester{ + private String inputManifestPath + private String distributionBuildNumber = '4891' + + public UpdateIntegTestFailureIssuesLibTester(inputManifestPath, distributionBuildNumber){ + this.inputManifestPath = inputManifestPath + this.distributionBuildNumber = distributionBuildNumber + } + + public UpdateIntegTestFailureIssuesLibTester(inputManifestPath){ + this.inputManifestPath = inputManifestPath + } + + @Override + String libFunctionName() { + return 'updateIntegTestFailureIssues' + } + + @Override + void parameterInvariantsAssertions(Object call) { + assertThat(call.args.inputManifestPath.first(), notNullValue()) + assertThat(call.args.distributionBuildNumber.first(), notNullValue()) + } + + @Override + boolean expectedParametersMatcher(Object call) { + return call.args.distributionBuildNumber.first().equals(this.distributionBuildNumber) + && call.args.inputManifestPath.first().equals(this.inputManifestPath) + } + + @Override + void configure(Object helper, Object binding) { + helper.registerAllowedMethod('withCredentials', [Map]) + helper.registerAllowedMethod('sleep', [Map]) + binding.setVariable('env', [ + 'METRICS_HOST_URL': 'sample.url', + 'AWS_ACCESS_KEY_ID': 'abc', + 'AWS_SECRET_ACCESS_KEY':'xyz', + 'AWS_SESSION_TOKEN': 'sampleToken' + ]) + helper.registerAllowedMethod('withCredentials', [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('withAWS', [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + } +} diff --git a/vars/publishIntegTestResults.groovy b/vars/publishIntegTestResults.groovy index 75910e41..e3646cf2 100644 --- a/vars/publishIntegTestResults.groovy +++ b/vars/publishIntegTestResults.groovy @@ -194,6 +194,9 @@ void indexFailedTestData(indexName, testRecordsFile) { "without_security_test_stderr": { "type": "keyword" } + }, + "aliases": { + "opensearch-integration-test-results": {} } } }' diff --git a/vars/updateBuildFailureIssues.groovy b/vars/updateBuildFailureIssues.groovy index f619ca65..39bf4b2a 100644 --- a/vars/updateBuildFailureIssues.groovy +++ b/vars/updateBuildFailureIssues.groovy @@ -15,7 +15,6 @@ */ import jenkins.ComponentBuildStatus -import jenkins.InputManifest void call(Map args = [:]) { def inputManifest = readYaml(file: args.inputManifestPath) diff --git a/vars/updateIntegTestFailureIssues.groovy b/vars/updateIntegTestFailureIssues.groovy new file mode 100644 index 00000000..cb32d476 --- /dev/null +++ b/vars/updateIntegTestFailureIssues.groovy @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + /** Library to create, update and close GitHub issue across opensearch-project repositories for integration test failures. + @param Map args = [:] args A map of the following parameters + @param args.inputManifestPath - Path to build manifest. + @param args.distributionBuildNumber - Distribution build number used to run integTest. Defaults to latest build for given version. + */ + +import jenkins.ComponentBuildStatus +import jenkins.ComponentIntegTestStatus +import jenkins.CreateIntegTestMarkDownTable + +void call(Map args = [:]) { + def inputManifest = readYaml(file: args.inputManifestPath) + def version = inputManifest.build.version + def product = inputManifest.build.name + + List failedComponents = [] + List passedComponents = [] + + withCredentials([ + string(credentialsId: 'jenkins-health-metrics-account-number', variable: 'METRICS_HOST_ACCOUNT'), + string(credentialsId: 'jenkins-health-metrics-cluster-endpoint', variable: 'METRICS_HOST_URL')]) { + withAWS(role: 'OpenSearchJenkinsAccessRole', roleAccount: "${METRICS_HOST_ACCOUNT}", duration: 900, roleSessionName: 'jenkins-session') { + def metricsUrl = env.METRICS_HOST_URL + def awsAccessKey = env.AWS_ACCESS_KEY_ID + def awsSecretKey = env.AWS_SECRET_ACCESS_KEY + def awsSessionToken = env.AWS_SESSION_TOKEN + def integTestIndexName = 'opensearch-integration-test-results' + def buildIndexName = 'opensearch-distribution-build-results' + + def distributionBuildNumber = args.distributionBuildNumber ?: new ComponentBuildStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, buildIndexName, product, version, this).getLatestDistributionBuildNumber() + ComponentIntegTestStatus componentIntegTestStatus = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, integTestIndexName, product, version, distributionBuildNumber, this) + + passedComponents = componentIntegTestStatus.getComponents('passed') + failedComponents = componentIntegTestStatus.getComponents('failed') + + failedComponents = failedComponents.unique() + passedComponents = passedComponents.unique() + + for (component in inputManifest.components) { + if (failedComponents.contains(component.name)) { + println("Integration test failed for ${component.name}, creating github issue") + def testData = [] + def queryData = componentIntegTestStatus.getComponentIntegTestFailedData(component.name) + def totalHits = queryData.hits.hits.collect {it._source} + totalHits.each { hit -> + def rowData = [ + platform : hit.platform, + distribution: hit.distribution, + architecture: hit.architecture, + test_report_manifest_yml: hit.test_report_manifest_yml, + integ_test_build_url: hit.integ_test_build_url + ] + testData << rowData + } + def markdownContent = new CreateIntegTestMarkDownTable(version, testData).create() + createGithubIssue( + repoUrl: component.repository, + issueTitle: "[AUTOCUT] Integration Test Failed for ${component.name}-${version}", + issueBody: markdownContent, + label: "autocut,v${version}", + issueEdit: true + ) + } + if (passedComponents.contains(component.name) && !failedComponents.contains(component.name)) { + println("Integration tests passed for ${component.name}, closing github issue") + ghIssueBody = """Closing the issue as the integration tests for ${component.name} passed for version: **${version}**.""".stripIndent() + closeGithubIssue( + repoUrl: component.repository, + issueTitle: "[AUTOCUT] Integration Test Failed for ${component.name}-${version}", + closeComment: ghIssueBody + ) + } + sleep(time:3, unit:'SECONDS') + } + } + } +}