[Security Solution] Unskip tests for the prebuilt rules bootstrap end…

…point (#205106) **Fixes: #197108 **Fixes: #202037 **Fixes: #203632 ## Summary Builds on top of #203799 and: - simplifies the package deletion code (1 [API call](https://www.elastic.co/docs/api/doc/kibana/operation/operation-delete-fleet-epm-packages-pkgname-pkgversion) instead of 2, per package) - adds retry logic - refactors the testing utils a bit Flaky test runs: - [ESS](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7632) - 190 runs - [Serverless](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7633) - 100 runs ## Details Here's how the test logs look like: ``` info Starting tests └-: Rules Management - Prebuilt Rules - Prebuilt Rules Management └-> "before all" hook: beforeTestSuite.trigger in "Rules Management - Prebuilt Rules - Prebuilt Rules Management" └-: @ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules └-> "before all" hook: beforeTestSuite.trigger for "should install fleet packages required for detection engine to function" └-> should install fleet packages required for detection engine to function └-> "before each" hook: global before each for "should install fleet packages required for detection engine to function" └-> "before each" hook for "should install fleet packages required for detection engine to function" │ debg Deleting security_detection_engine package │ debg Deleted security_detection_engine package (was not installed) { │ statusCode: 400, │ error: 'Bad Request', │ message: 'security_detection_engine is not installed' │ } │ debg Deleting endpoint package │ debg Deleted endpoint package (was not installed) { │ statusCode: 400, │ error: 'Bad Request', │ message: 'endpoint is not installed' │ } └- ✓ pass (21.6s) └-> should skip installing fleet packages if they are already installed └-> "before each" hook: global before each for "should skip installing fleet packages if they are already installed" └-> "before each" hook for "should skip installing fleet packages if they are already installed" │ debg Deleting security_detection_engine package │ debg Deleted security_detection_engine package (was installed) │ debg Deleting endpoint package │ debg Deleted endpoint package (was installed) └- ✓ pass (22.4s) └-> "after all" hook: afterTestSuite.trigger for "should skip installing fleet packages if they are already installed" └-> "after all" hook: afterTestSuite.trigger in "Rules Management - Prebuilt Rules - Prebuilt Rules Management" 2 passing (1.0m) ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed
elastic · Dec 24, 2024 · 71144ed · 71144ed
1 parent 49956ec
commit 71144ed
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 74 deletions.
diff --git a/...t_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts b/...t_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts
@@ -22,7 +22,7 @@ export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
   const log = getService('log');
-  const retry = getService('retry');
+  const retryService = getService('retry');
 
   /* This test makes use of the mock packages created in the '/fleet_bundled_packages' folder,
   /* in order to assert that, in production environments, the latest stable version of the package
@@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext): void => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
       await deleteAllPrebuiltRuleAssets(es, log);
-      await deletePrebuiltRulesFleetPackage(supertest);
+      await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService });
     });
 
     it('should install latest stable version and ignore prerelease packages', async () => {
@@ -49,7 +49,7 @@ export default ({ getService }: FtrProviderContext): void => {
       const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI(
         es,
         supertest,
-        retry
+        retryService
       );
 
       expect(fleetPackageInstallationResponse.items.length).toBe(1);

diff --git a/...agement/prebuilt_rules/management/trial_license_complete_tier/bootstrap_prebuilt_rules.ts b/...agement/prebuilt_rules/management/trial_license_complete_tier/bootstrap_prebuilt_rules.ts
@@ -11,18 +11,24 @@ import {
 } from '@kbn/security-solution-plugin/common/detection_engine/constants';
 import expect from 'expect';
 import { FtrProviderContext } from '../../../../../../ftr_provider_context';
-import { deletePrebuiltRulesFleetPackage } from '../../../../utils';
-import { deleteEndpointFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_endpoint_fleet_package';
+import {
+  deleteAllPrebuiltRuleAssets,
+  deleteEndpointFleetPackage,
+  deletePrebuiltRulesFleetPackage,
+} from '../../../../utils';
 
 export default ({ getService }: FtrProviderContext): void => {
+  const es = getService('es');
+  const log = getService('log');
   const supertest = getService('supertest');
+  const retryService = getService('retry');
   const securitySolutionApi = getService('securitySolutionApi');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/203632
-  describe.skip('@ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules', () => {
+  describe('@ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules', () => {
     beforeEach(async () => {
-      await deletePrebuiltRulesFleetPackage(supertest);
-      await deleteEndpointFleetPackage(supertest);
+      await deleteAllPrebuiltRuleAssets(es, log);
+      await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService });
+      await deleteEndpointFleetPackage({ supertest, es, log, retryService });
     });
 
     it('should install fleet packages required for detection engine to function', async () => {

diff --git a/...les_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts b/...les_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts
@@ -13,18 +13,18 @@ import {
 import { deleteAllRules } from '../../../../../../../common/utils/security_solution';
 import { deleteAllPrebuiltRuleAssets } from '../../../../utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets';
 import { deleteAllTimelines } from '../../../../utils/rules/prebuilt_rules/delete_all_timelines';
-import { deletePrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package';
+import { deletePrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_fleet_packages';
 import { installPrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package';
 
 export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
   const log = getService('log');
-  const retry = getService('retry');
+  const retryService = getService('retry');
 
   describe('@ess @serverless @skipInServerlessMKI install_prebuilt_rules_from_real_package', () => {
     beforeEach(async () => {
-      await deletePrebuiltRulesFleetPackage(supertest);
+      await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService });
       await deleteAllRules(supertest, log);
       await deleteAllTimelines(es, log);
       await deleteAllPrebuiltRuleAssets(es, log);
@@ -48,7 +48,7 @@ export default ({ getService }: FtrProviderContext): void => {
         es,
         supertest,
         overrideExistingPackage: true,
-        retryService: retry,
+        retryService,
       });
 
       // Verify that status is updated after package installation

diff --git a/...st_suites/detections_response/utils/rules/prebuilt_rules/delete_endpoint_fleet_package.ts b/...st_suites/detections_response/utils/rules/prebuilt_rules/delete_endpoint_fleet_package.ts
diff --git a/...ation/test_suites/detections_response/utils/rules/prebuilt_rules/delete_fleet_packages.ts b/...ation/test_suites/detections_response/utils/rules/prebuilt_rules/delete_fleet_packages.ts
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+import { epmRouteService } from '@kbn/fleet-plugin/common';
+import { RetryService } from '@kbn/ftr-common-functional-services';
+import {
+  ENDPOINT_PACKAGE_NAME,
+  PREBUILT_RULES_PACKAGE_NAME,
+} from '@kbn/security-solution-plugin/common/detection_engine/constants';
+import { ToolingLog } from '@kbn/tooling-log';
+import type SuperTest from 'supertest';
+import { refreshSavedObjectIndices } from '../../refresh_index';
+
+interface DeleteFleetPackageDeps {
+  supertest: SuperTest.Agent;
+  retryService: RetryService;
+  log: ToolingLog;
+  es: Client;
+}
+
+interface DeleteFleetPackageArgs {
+  packageName: string;
+  dependencies: DeleteFleetPackageDeps;
+}
+
+/**
+ * Delete the security_detection_engine package using fleet API.
+ */
+export async function deletePrebuiltRulesFleetPackage(params: DeleteFleetPackageDeps) {
+  await deleteFleetPackage({
+    packageName: PREBUILT_RULES_PACKAGE_NAME,
+    dependencies: params,
+  });
+}
+
+/**
+ * Delete the endpoint package using fleet API.
+ */
+export async function deleteEndpointFleetPackage(params: DeleteFleetPackageDeps) {
+  await deleteFleetPackage({
+    packageName: ENDPOINT_PACKAGE_NAME,
+    dependencies: params,
+  });
+}
+
+async function deleteFleetPackage(params: DeleteFleetPackageArgs): Promise<void> {
+  const { packageName, dependencies } = params;
+  const { supertest, retryService, log, es } = dependencies;
+
+  await retryService.tryWithRetries(
+    'deleteFleetPackage',
+    async () => {
+      log.debug(`Deleting ${packageName} package`);
+
+      const response = await supertest
+        .delete(epmRouteService.getRemovePath(packageName))
+        .set('kbn-xsrf', 'true')
+        .set('elastic-api-version', '2023-10-31')
+        .send({ force: true });
+
+      if (response.status === 200) {
+        log.debug(`Deleted ${packageName} package (was installed)`);
+        return;
+      } else if (
+        response.status === 400 &&
+        response.body.message === `${packageName} is not installed`
+      ) {
+        log.debug(`Deleted ${packageName} package (was not installed)`, response.body);
+        return;
+      } else {
+        log.warning(`Error deleting ${packageName} package`, response.body);
+        throw response.error;
+      }
+    },
+    {
+      retryCount: 2, // overall max 3 attempts
+      timeout: 3 * 60000, // total timeout applied to all attempts altogether
+    }
+  );
+
+  await refreshSavedObjectIndices(es);
+}
diff --git a/...tes/detections_response/utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package.ts b/...tes/detections_response/utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package.ts
diff --git a/...ution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/index.ts b/...ution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/index.ts
@@ -7,7 +7,7 @@
 export * from './create_prebuilt_rule_saved_objects';
 export * from './delete_all_prebuilt_rule_assets';
 export * from './delete_all_timelines';
-export * from './delete_prebuilt_rules_fleet_package';
+export * from './delete_fleet_packages';
 export * from './get_installed_rules';
 export * from './get_prebuilt_rules_and_timelines_status';
 export * from './get_prebuilt_rules_status';