fix: e2e test for inconclusive analysisrun

Signed-off-by: Yurii Popivniak <ypopivniak@cargurus.com>

rollout/analysis.go

@@ -169,7 +169,10 @@ func needsNewAnalysisRun(currentAr *v1alpha1.AnalysisRun, rollout *v1alpha1.Roll
 	// is set and then seeing if the last status was inconclusive.
 	// There is an additional check for the BlueGreen Pause because the prepromotion analysis always has the BlueGreen
 	// Pause and that causes controllerPause to be set. The extra check for the BlueGreen Pause ensures that a new Analysis
-	// Run is created only when the previous AnalysisRun is inconclusive
+	// Run is created only when the previous AnalysisRun is inconclusive.
+	// Additional check for the Canary Pause prevents Canary promotion when AnalysisRun is inconclusive and reached
+	// inconclusiveLimit. Otherwise, another AnalysisRun will be spawned and can cause Success status,
+	// because of termination when the AnalysisRun is still in-flight.
 	if rollout.Status.ControllerPause &&
 		getPauseCondition(rollout, v1alpha1.PauseReasonCanaryPauseStep) == nil &&
 		getPauseCondition(rollout, v1alpha1.PauseReasonBlueGreenPause) == nil {

test/e2e/analysis_test.go

@@ -80,9 +80,9 @@ func (s *AnalysisSuite) TestCanaryInconclusiveBackgroundAnalysis() {
 		When().
 		UpdateSpec().
 		WaitForRolloutStatus("Paused").
+		WaitForBackgroundAnalysisRunPhase("Running").
 		Then().
 		ExpectAnalysisRunCount(1).
-		ExpectBackgroundAnalysisRunPhase("Running").
 		When().
 		WaitForBackgroundAnalysisRunPhase("Inconclusive").
 		WaitForRolloutMessage("InconclusiveAnalysisRun").

test/e2e/functional/analysistemplate-web-background-inconclusive.yaml

@@ -1,19 +1,22 @@
-# A dummy web metric which uses the kubernetes version endpoint as a metric provider
+# A web metric which uses the httpbin service as a metric provider
 apiVersion: argoproj.io/v1alpha1
 kind: AnalysisTemplate
 metadata:
   name: web-background-inconclusive
 spec:
   args:
     - name: url-val
-      value: "https://kubernetes.default.svc/version"
+      value: "https://httpbin.org/anything"
   metrics:
-  - name: web
-    interval: 7s
-    inconclusiveLimit: 3
-    successCondition: result.major == '2000'
-    failureCondition: result.major == '1000'
-    provider:
-      web:
-        url: "{{args.url-val}}"
-        insecure: true
+    - name: web
+      interval: 7s
+      inconclusiveLimit: 3
+      successCondition: result.status == "success"
+      failureCondition: result.status == "failure"
+      provider:
+        web:
+          url: "{{args.url-val}}"
+          method: POST
+          jsonBody:
+            status: "inconclusive"
+          jsonPath: "{$.json}"