Merge pull request #1369 from flanksource/switch-cel

Switch cel

build/full/Dockerfile

@@ -62,8 +62,7 @@ RUN curl -L https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-${JMETER_VER
 
 ENV PATH /opt/apache-jmeter-${JMETER_VERSION}/bin/:$PATH
 
-
-RUN curl -L  https://github.com/mergestat/mergestat-lite/releases/download/v0.6.1/mergestat-linux-amd64.tar.gz -o mergestat.tar.gz && \
+RUN curl -L https://github.com/flanksource/askgit/releases/download/v0.61.0-flanksource.1/mergestat-linux-amd64.tar.gz  -o mergestat.tar.gz && \
   tar zxf mergestat.tar.gz -C /usr/local/bin/ && \
   rm mergestat.tar.gz
 

chart/templates/deployment.yaml

@@ -95,7 +95,7 @@ spec:
               {{- else }}
               value: "embedded:///opt/database/"
               {{- end }}
-          {{- if .Values.upstream.enabled }}
+          {{- if .Values.upstream.secretKeyRef.name }}
           envFrom:
             - secretRef:
                 name: {{ .Values.upstream.secretKeyRef.name }}
@@ -140,11 +140,21 @@ spec:
             {{- if gt (int .Values.replicas) 1 }}
             - --enable-leader-election=true
             {{- end }}
+            {{- range $k, $v := .Values.extraArgs}}
+            - --{{$k}}={{$v}}
+            {{- end }}
+
             {{- if .Values.upstream.enabled }}
             - --agent-name={{ .Values.upstream.agentName }}
-            - --upstream-host=$UPSTREAM_HOST
-            - --upstream-user=$UPSTREAM_USER
-            - --upstream-password=$UPSTREAM_PASSWORD
+            {{- if .Values.upstream.host }}
+            - --upstream-host={{ .Values.upstream.host }}
+            {{- end}}
+            {{- if .Values.upstream.user }}
+            - --upstream-user={{ .Values.upstream.user }}
+            {{- end}}
+            {{- if .Values.upstream.password }}
+            - --upstream-password={{ .Values.upstream.password }}
+            {{- end}}
             {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}

chart/values.yaml

@@ -73,9 +73,12 @@ data:
 upstream:
   enabled: false
   agentName: default-agent
-  # Must contain: UPSTREAM_USER, UPSTREAM_PASS & UPSTREAM_HOST
+  host: ""
+  user: ""
+  password: ""
+  # Alternative to inlining values, secret must contain: UPSTREAM_NAME, UPSTREAM_USER, UPSTREAM_PASSWORD & UPSTREAM_HOST
   secretKeyRef:
-    name: canary-checker-upstream
+    name:
 
 ingress:
   enabled: false
@@ -133,6 +136,8 @@ disableChecks: {}
 # a list of db properties to update on startup
 properties: {}
 
+# a map of extra arguments to the canary-checker cli
+extraArgs: {}
 extra:
   # nodeSelector:
   #   key: value

checks/alertmanager.go

@@ -71,7 +71,11 @@ func (c *AlertManagerChecker) Check(ctx *context.Context, extConfig external.Che
 		return results
 	}
 
-	var alertMessages []map[string]any
+	type Alerts struct {
+		Alerts []map[string]interface{} `json:"alerts,omitempty"`
+	}
+
+	var alertMessages []map[string]interface{}
 	for _, alert := range alerts.Payload {
 		alertMap := map[string]any{
 			"name":        generateFullName(alert.Labels["alertname"], alert.Labels),
@@ -83,7 +87,7 @@ func (c *AlertManagerChecker) Check(ctx *context.Context, extConfig external.Che
 		alertMessages = append(alertMessages, alertMap)
 	}
 
-	result.AddDetails(alertMessages)
+	result.AddDetails(Alerts{Alerts: alertMessages})
 	return results
 }
 

checks/aws_config_rule.go

@@ -3,8 +3,8 @@
 package checks
 
 import (
-	"fmt"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/configservice"
 	"github.com/aws/aws-sdk-go-v2/service/configservice/types"
@@ -65,44 +65,82 @@ func (c *AwsConfigRuleChecker) Check(ctx *context.Context, extConfig external.Ch
 	if err != nil {
 		return results.Failf("failed to describe compliance rules: %v", err)
 	}
-	var complianceResults pkg.Results
+
+	type ConfigRuleResource struct {
+		ID         string    `json:"id"`
+		Annotation string    `json:"annotation"`
+		Type       string    `json:"type"`
+		Recorded   time.Time `json:"recorded"`
+		Mode       string    `json:"mode"`
+	}
+
+	type ComplianceResult struct {
+
+		// Supplementary information about how the evaluation determined the compliance.
+		Annotation string `json:"annotation"`
+
+		ConfigRule string `json:"rule"`
+
+		Description string `json:"description"`
+		// Indicates whether the Amazon Web Services resource complies with the Config
+		// rule that evaluated it. For the EvaluationResult data type, Config supports
+		// only the COMPLIANT , NON_COMPLIANT , and NOT_APPLICABLE values. Config does not
+		// support the INSUFFICIENT_DATA value for the EvaluationResult data type.
+		ComplianceType string `json:"type"`
+
+		Resources []ConfigRuleResource `json:"resources"`
+	}
+
+	var complianceResults []ComplianceResult
+	var failures []string
 	for _, complianceRule := range output.ComplianceByConfigRules {
 		if configRuleInRules(check.IgnoreRules, *complianceRule.ConfigRuleName) || complianceRule.Compliance.ComplianceType == "INSUFFICIENT_DATA" || complianceRule.Compliance.ComplianceType == "NOT_APPLICABLE" {
 			continue
 		}
-		if complianceRule.Compliance != nil {
-			var complianceResult *pkg.CheckResult
-			complianceCheck := check
-			complianceCheck.Description.Description = fmt.Sprintf("%s - checking compliance for config rule: %s", check.Description.Description, *complianceRule.ConfigRuleName)
-			if complianceRule.Compliance.ComplianceType != "COMPLIANT" {
-				complianceResult = pkg.Fail(complianceCheck, ctx.Canary)
-				complianceDetailsOutput, err := client.GetComplianceDetailsByConfigRule(ctx, &configservice.GetComplianceDetailsByConfigRuleInput{
-					ComplianceTypes: []types.ComplianceType{
-						"NON_COMPLIANT",
-					},
-					ConfigRuleName: complianceRule.ConfigRuleName,
+
+		if complianceRule.Compliance == nil {
+			continue
+		}
+		var data = ComplianceResult{
+			ConfigRule:     *complianceRule.ConfigRuleName,
+			ComplianceType: string(complianceRule.Compliance.ComplianceType),
+		}
+
+		if complianceRule.Compliance.ComplianceType != "COMPLIANT" {
+			failures = append(failures, *complianceRule.ConfigRuleName)
+			complianceDetailsOutput, err := client.GetComplianceDetailsByConfigRule(ctx, &configservice.GetComplianceDetailsByConfigRuleInput{
+				ComplianceTypes: []types.ComplianceType{
+					"NON_COMPLIANT",
+				},
+				ConfigRuleName: complianceRule.ConfigRuleName,
+			})
+			if err != nil {
+				result.Failf("failed to get compliance details: %v", err)
+				continue
+			}
+			for _, result := range complianceDetailsOutput.EvaluationResults {
+				id := *result.EvaluationResultIdentifier.EvaluationResultQualifier
+				data.Resources = append(data.Resources, ConfigRuleResource{
+					ID:         *id.ResourceId,
+					Type:       *id.ResourceType,
+					Mode:       string(id.EvaluationMode),
+					Recorded:   *result.ResultRecordedTime,
+					Annotation: *result.Annotation,
 				})
-				if err != nil {
-					complianceResult.Failf("failed to get compliance details: %v", err)
-					continue
-				}
-				var resources []string
-				for _, result := range complianceDetailsOutput.EvaluationResults {
-					if result.EvaluationResultIdentifier.EvaluationResultQualifier.ResourceId != nil {
-						resources = append(resources, *result.EvaluationResultIdentifier.EvaluationResultQualifier.ResourceId)
-					}
-				}
-				complianceResult.AddDetails(resources)
-				complianceResult.ResultMessage(strings.Join(resources, ","))
-			} else {
-				complianceResult = pkg.Success(complianceCheck, ctx.Canary)
-				complianceResult.AddDetails(complianceRule)
-				complianceResult.ResultMessage(fmt.Sprintf("%s rule is %v", *complianceRule.ConfigRuleName, complianceRule.Compliance.ComplianceType))
 			}
-			complianceResults = append(complianceResults, complianceResult)
 		}
+		complianceResults = append(complianceResults, data)
 	}
-	return complianceResults
+
+	if check.Test.IsEmpty() && len(failures) > 0 {
+		result.Failf(strings.Join(failures, ", "))
+	}
+	if r, err := unstructure(map[string]interface{}{"rules": complianceResults}); err != nil {
+		result.Failf(err.Error())
+	} else {
+		result.AddDetails(r)
+	}
+	return results
 }
 
 func configRuleInRules(rules []string, ruleName string) bool {

checks/cloudwatch.go

@@ -58,7 +58,11 @@ func (c *CloudWatchChecker) Check(ctx *context.Context, extConfig external.Check
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
-	result.AddDetails(alarms)
+	if o, err := unstructure(alarms); err != nil {
+		return results.ErrorMessage(err)
+	} else {
+		result.AddDetails(o)
+	}
 	firing := []string{}
 	for _, alarm := range alarms.MetricAlarms {
 		if alarm.StateValue == types.StateValueAlarm {

checks/common.go

@@ -51,6 +51,16 @@ func def(a, b string) string {
 	return b
 }
 
+// unstructure marshalls a struct to and from JSON to remove any type details
+func unstructure(o any) (out interface{}, err error) {
+	data, err := json.Marshal(o)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(data, &out)
+	return out, err
+}
+
 func template(ctx *context.Context, template v1.Template) (string, error) {
 	return gomplate.RunTemplate(ctx.Environment, template.Gomplate())
 }
@@ -107,6 +117,7 @@ func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, e
 			// We use this label to set the transformed column to true
 			// This label is used and then removed in pkg.FromV1 function
 			r.Canary.Labels["transformed"] = "true" //nolint:goconst
+			r.Labels = t.Labels
 			r.Transformed = true
 			results = append(results, &r)
 		}
@@ -144,6 +155,9 @@ func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, e
 		if t.DisplayType != "" {
 			in.DisplayType = t.DisplayType
 		}
+		if len(t.Labels) > 0 {
+			in.Labels = t.Labels
+		}
 		if len(t.Data) > 0 {
 			for k, v := range t.Data {
 				in.Data[k] = v
@@ -166,6 +180,7 @@ func GetJunitReportFromResults(canaryName string, results []*pkg.CheckResult) Ju
 		test.Name = result.Check.GetDescription()
 		test.Message = result.Message
 		test.Duration = float64(result.Duration) / 1000
+		test.Properties = result.Labels
 		testSuite.Duration += float64(result.Duration) / 1000
 		if result.Pass {
 			testSuite.Passed++

cmd/root.go

@@ -27,6 +27,10 @@ var Root = &cobra.Command{
 		if db.ConnectionString == "DB_URL" {
 			db.ConnectionString = ""
 		}
+
+		if canary.UpstreamConf.Valid() {
+			logger.Infof("Pushing checks to %s with name=%s user=%s", canary.UpstreamConf.Host, canary.UpstreamConf.AgentName, canary.UpstreamConf.Username)
+		}
 	},
 }
 
@@ -65,13 +69,12 @@ func ServerFlags(flags *pflag.FlagSet) {
 	flags.IntVar(&db.CheckRetentionDays, "check-retention-period", db.DefaultCheckRetentionDays, "Check retention period in days")
 	flags.IntVar(&db.CanaryRetentionDays, "canary-retention-period", db.DefaultCanaryRetentionDays, "Canary retention period in days")
 
-	// Flags for push/pull
-	flags.StringVar(&canary.UpstreamConf.Host, "upstream-host", "", "central canary checker instance to push/pull canaries")
-	flags.StringVar(&canary.UpstreamConf.Username, "upstream-user", "", "upstream username")
-	flags.StringVar(&canary.UpstreamConf.Password, "upstream-password", "", "upstream password")
-	flags.StringVar(&canary.UpstreamConf.AgentName, "agent-name", "", "name of this agent")
 	flags.IntVar(&canary.ReconcilePageSize, "upstream-page-size", 500, "upstream reconciliation page size")
 	flags.DurationVar(&canary.ReconcileMaxAge, "upstream-max-age", time.Hour*48, "upstream reconciliation max age")
+	flags.StringVar(&canary.UpstreamConf.Host, "upstream-host", os.Getenv("UPSTREAM_HOST"), "central canary checker instance to push/pull canaries")
+	flags.StringVar(&canary.UpstreamConf.Username, "upstream-user", os.Getenv("UPSTREAM_USER"), "upstream username")
+	flags.StringVar(&canary.UpstreamConf.Password, "upstream-password", os.Getenv("UPSTREAM_PASSWORD"), "upstream password")
+	flags.StringVar(&canary.UpstreamConf.AgentName, "agent-name", os.Getenv("UPSTREAM_NAME"), "name of this agent")
 }
 
 func readFromEnv(v string) string {

cmd/run.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -22,7 +23,7 @@ import (
 )
 
 var outputFile, dataFile, runNamespace string
-var junit, csv bool
+var junit, csv, jsonExport bool
 
 var Run = &cobra.Command{
 	Use:   "run <canary.yaml>",
@@ -112,6 +113,21 @@ var Run = &cobra.Command{
 				logger.Fatalf("error writing output file: %v", err)
 			}
 		}
+		if jsonExport {
+
+			for _, result := range results {
+				result.Name = def(result.Name, result.Check.GetName(), result.Canary.Name)
+				result.Description = def(result.Description, result.Check.GetDescription())
+				result.Labels = merge(result.Check.GetLabels(), result.Labels)
+				fmt.Println(result.Name)
+			}
+
+			data, err := json.Marshal(results)
+			if err != nil {
+				logger.Fatalf("Failed to marshall json: %s", err)
+			}
+			_ = output.HandleOutput(string(data), outputFile)
+		}
 
 		logger.Infof("%d passed, %d failed in %s", passed, failed, timer)
 
@@ -121,11 +137,32 @@ var Run = &cobra.Command{
 	},
 }
 
+func merge(m1, m2 map[string]string) map[string]string {
+	out := make(map[string]string)
+	for k, v := range m1 {
+		out[k] = v
+	}
+	for k, v := range m2 {
+		out[k] = v
+	}
+	return out
+}
+
+func def(a ...string) string {
+	for _, s := range a {
+		if s != "" {
+			return s
+		}
+	}
+	return ""
+}
+
 func init() {
 	Run.PersistentFlags().StringVarP(&dataFile, "data", "d", "", "Template out each spec using the JSON or YAML data in this file")
 	Run.PersistentFlags().StringVarP(&outputFile, "output-file", "o", "", "file to output the results in")
 	Run.Flags().StringVarP(&runNamespace, "namespace", "n", "", "Namespace to run canary checks in")
-	Run.Flags().BoolVarP(&junit, "junit", "j", false, "output results in junit format")
+	Run.Flags().BoolVar(&junit, "junit", false, "output results in junit format")
+	Run.Flags().BoolVarP(&jsonExport, "json", "j", false, "output results in json format")
 	Run.Flags().BoolVar(&csv, "csv", false, "output results in csv format")
 }
 

fixtures/aws/aws_config_rule_pass.yaml

@@ -5,7 +5,19 @@ metadata:
 spec:
   interval: 30
   awsConfigRule:
-    - description: Test config rules
-      name: non compliant rules
-      complianceTypes:
-        - NON_COMPLIANT
+    - name: AWS Config Rule
+      region: "eu-west-1"
+      complianceTypes: [NON_COMPLIANT]
+      transform:
+        expr: |
+          results.rules.map(i,
+            i.resources.map(r,
+             {
+              'name': i.rule + "/" + r.type + "/" + r.id,
+              'description': i.rule,
+              'icon': 'aws-config-alarm',
+              'duration': time.Since(timestamp(r.recorded)).getMilliseconds(),
+              'labels': {'id': r.id, 'type': r.type},
+              'message': i.description + i.annotation + r.annotation
+            })
+          ).flatten().toJSON()