diff --git a/alerting/build.gradle b/alerting/build.gradle
index ba2189b54..83b3bf308 100644
--- a/alerting/build.gradle
+++ b/alerting/build.gradle
@@ -16,6 +16,7 @@ apply plugin: 'jacoco'
 
 def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
 def usingMultiNode = project.properties.containsKey('numNodes')
+String bwcVersion = "2.3.0.0"
 
 ext {
     projectSubstitutions = [:]
@@ -99,6 +100,9 @@ dependencies {
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}"
 
+    // Needed for BWC tests
+    zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${bwcVersion}-SNAPSHOT"
+
     compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}"
     api "org.opensearch.plugin:percolator-client:${opensearch_version}"
 
@@ -247,71 +251,26 @@ task integTestRemote(type: RestIntegTestTask) {
 }
 integTestRemote.enabled = System.getProperty("tests.rest.cluster") != null
 
-String bwcMinVersion = "1.1.0.0"
-String bwcBundleVersion = "1.3.2.0"
-Boolean bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null &&
-        project.properties['customDistributionDownloadType'] == "bundle");
-String bwcVersion = bwcBundleTest ? bwcBundleVersion : bwcMinVersion
-String bwcCurrentVersion = opensearch_version.replace("-SNAPSHOT", "")
 String baseName = "alertingBwcCluster"
-String bwcFilePath = "src/test/resources/bwc"
-String bwcOpenSearchPlugin = "opensearch-alerting-" + bwcVersion + ".zip"
-String bwcRemoteFile = 'https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/' + bwcOpenSearchPlugin
-
 2.times {i ->
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
+            versions = ["2.3.0-SNAPSHOT", "2.5.1-SNAPSHOT"]
             numberOfNodes = 3
-            if (bwcBundleTest) {
-                versions = ["1.3.2", bwcCurrentVersion]
-                nodes.each { node ->
-                    node.extraConfigFile("kirk.pem", file("src/test/resources/kirk.pem"))
-                    node.extraConfigFile("kirk-key.pem", file("src/test/resources/kirk-key.pem"))
-                    node.extraConfigFile("esnode.pem", file("src/test/resources/esnode.pem"))
-                    node.extraConfigFile("esnode-key.pem", file("src/test/resources/esnode-key.pem"))
-                    node.extraConfigFile("root-ca.pem", file("src/test/resources/root-ca.pem"))
-                    node.setting("plugins.security.disabled", "true")
-                    node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
-                    node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
-                    node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
-                    node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
-                    node.setting("plugins.security.ssl.http.enabled", "true")
-                    node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
-                    node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
-                    node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
-                    node.setting("plugins.security.allow_unsafe_democertificates", "true")
-                    node.setting("plugins.security.allow_default_init_securityindex", "true")
-                    node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
-                    node.setting("plugins.security.audit.type", "internal_elasticsearch")
-                    node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
-                    node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
-                    node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
-                    node.setting("plugins.security.system_indices.enabled", "true")
-                }
-            } else {
-                versions = ["1.1.0", opensearch_version]
-                plugin(provider(new Callable<RegularFile>(){
-                    @Override
-                    RegularFile call() throws Exception {
-                        return new RegularFile() {
-                            @Override
-                            File getAsFile() {
-                                File dir = new File(rootDir.path + "/alerting/" + bwcFilePath + "/alerting/" + bwcVersion)
-
-                                if (!dir.exists()) {
-                                    dir.mkdirs()
-                                }
-                                File f = new File(dir, bwcOpenSearchPlugin)
-                                if (!f.exists()) {
-                                    new URL(bwcRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
-                                }
-                                return fileTree(bwcFilePath + "/alerting/" + bwcVersion).getSingleFile()
-                            }
+            plugin(provider(new Callable<RegularFile>(){
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            return configurations.zipArchive.asFileTree.matching {
+                                include '**/alerting*'
+                            }.singleFile
                         }
                     }
-                }))
-            }
+                }
+            }))
             setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
             setting 'http.content_type.required', 'true'
         }
@@ -353,9 +312,7 @@ task prepareBwcTests {
 // Create two test clusters with 3 nodes of the old version
 2.times {i ->
     task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
-        if (!bwcBundleTest){
-            dependsOn 'prepareBwcTests'
-        }
+        dependsOn 'prepareBwcTests'
         useCluster testClusters."${baseName}$i"
         filter {
             includeTestsMatching "org.opensearch.alerting.bwc.*IT"
@@ -374,14 +331,8 @@ task prepareBwcTests {
 task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
     useCluster testClusters."${baseName}0"
     dependsOn "${baseName}#oldVersionClusterTask0"
-    if (bwcBundleTest){
-        doFirst {
-            testClusters."${baseName}0".nextNodeToNextVersion()
-        }
-    } else {
-        doFirst {
-            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
-        }
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.alerting.bwc.*IT"
@@ -399,14 +350,8 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
 task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#mixedClusterTask"
     useCluster testClusters."${baseName}0"
-    if (bwcBundleTest){
-        doFirst {
-            testClusters."${baseName}0".nextNodeToNextVersion()
-        }
-    } else {
-        doFirst {
-            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
-        }
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.alerting.bwc.*IT"
@@ -424,14 +369,8 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
 task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
     useCluster testClusters."${baseName}0"
-    if (bwcBundleTest){
-        doFirst {
-            testClusters."${baseName}0".nextNodeToNextVersion()
-        }
-    } else {
-        doFirst {
-            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
-        }
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.alerting.bwc.*IT"
@@ -449,14 +388,8 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
 task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#oldVersionClusterTask1"
     useCluster testClusters."${baseName}1"
-    if (bwcBundleTest){
-        doFirst {
-            testClusters."${baseName}1".goToNextVersion()
-        }
-    } else {
-        doFirst {
-            testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
-        }
+    doFirst {
+        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.alerting.bwc.*IT"