diff --git a/config/crds/mission-control.flanksource.com_incidentrules.yaml b/config/crds/mission-control.flanksource.com_incidentrules.yaml index 14d3473cb..d4cece81c 100644 --- a/config/crds/mission-control.flanksource.com_incidentrules.yaml +++ b/config/crds/mission-control.flanksource.com_incidentrules.yaml @@ -44,17 +44,15 @@ spec: properties: timeout: description: How long after the health checks have been passing - before, autoclosing the incident. - format: int64 - type: integer + before, autoclosing the incident (accepts goduration format) + type: string type: object autoResolve: properties: timeout: description: How long after the health checks have been passing - before, autoclosing the incident. - format: int64 - type: integer + before, autoclosing the incident (accepts goduration format) + type: string type: object breakOnMatch: description: stop processing other incident rules, when matched diff --git a/db/notifications.go b/db/notifications.go index d7d14edaa..6ca642a96 100644 --- a/db/notifications.go +++ b/db/notifications.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + extraClausePlugin "github.com/WinterYukky/gorm-extra-clause-plugin" + "github.com/WinterYukky/gorm-extra-clause-plugin/exclause" "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" "github.com/flanksource/duty/query" @@ -111,3 +113,55 @@ func NotificationSendSummary(ctx context.Context, id string, window time.Duratio err := ctx.DB().Raw(query, id, window).Row().Scan(&earliest, &count) return earliest.Time, count, err } + +func GetMatchingNotificationSilencesCount(ctx context.Context, resources models.NotificationSilenceResource) (int64, error) { + _ = ctx.DB().Use(extraClausePlugin.New()) + + query := ctx.DB().Model(&models.NotificationSilence{}) + + // Initialize with a false condition, + // if no resources are provided, the query won't return all records + orClauses := ctx.DB().Where("1 = 0") + + if resources.ConfigID != nil { + orClauses = orClauses.Or("config_id = ?", *resources.ConfigID) + + // recursive stuff + orClauses = orClauses.Or("(recursive = true AND path_cte.path LIKE '%' || config_id::TEXT || '%')") + query = query.Clauses(exclause.NewWith( + "path_cte", + ctx.DB().Select("path").Model(&models.ConfigItem{}).Where("id = ?", *resources.ConfigID), + )) + query = query.Joins("CROSS JOIN path_cte") + } + + if resources.ComponentID != nil { + orClauses = orClauses.Or("component_id = ?", *resources.ComponentID) + + // recursive stuff + orClauses = orClauses.Or("(recursive = true AND path_cte.path LIKE '%' || component_id::TEXT || '%')") + query = query.Clauses(exclause.NewWith( + "path_cte", + ctx.DB().Select("path").Model(&models.Component{}).Where("id = ?", *resources.ComponentID), + )) + query = query.Joins("CROSS JOIN path_cte") + } + + if resources.CanaryID != nil { + orClauses = orClauses.Or("canary_id = ?", *resources.CanaryID) + } + + if resources.CheckID != nil { + orClauses = orClauses.Or("check_id = ?", *resources.CheckID) + } + + query = query.Where(orClauses) + + var count int64 + err := query.Count(&count).Where(`"from" <= NOW()`).Where("until >= NOW()").Where("deleted_at IS NULL").Error + if err != nil { + return 0, err + } + + return count, nil +} diff --git a/db/notifications_test.go b/db/notifications_test.go new file mode 100644 index 000000000..b28628840 --- /dev/null +++ b/db/notifications_test.go @@ -0,0 +1,106 @@ +package db + +import ( + "time" + + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/tests/fixtures/dummy" + "github.com/google/uuid" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" +) + +var _ = ginkgo.Describe("Notification Silence", ginkgo.Ordered, func() { + var silences []models.NotificationSilence + ginkgo.BeforeAll(func() { + silences = []models.NotificationSilence{ + { + ID: uuid.New(), + From: time.Now().Add(-time.Hour), + Until: time.Now().Add(time.Hour), + Source: models.SourceCRD, + NotificationSilenceResource: models.NotificationSilenceResource{ + ConfigID: lo.ToPtr(dummy.EKSCluster.ID.String()), + }, + }, + { + ID: uuid.New(), + From: time.Now().Add(-time.Hour), + Until: time.Now().Add(time.Hour), + Source: models.SourceCRD, + Recursive: true, + NotificationSilenceResource: models.NotificationSilenceResource{ + ConfigID: lo.ToPtr(dummy.LogisticsAPIDeployment.ID.String()), + }, + }, + { + ID: uuid.New(), + From: time.Now().Add(-time.Hour), + Until: time.Now().Add(time.Hour), + Source: models.SourceCRD, + Recursive: true, + NotificationSilenceResource: models.NotificationSilenceResource{ + ComponentID: lo.ToPtr(dummy.Logistics.ID.String()), + }, + }, + } + + err := DefaultContext.DB().Create(&silences).Error + Expect(err).To(BeNil()) + }) + + ginkgo.Context("non recursive match", func() { + ginkgo.It("should match", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.EKSCluster.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(1))) + }) + + ginkgo.It("should not match", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.KubernetesCluster.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(0))) + }) + }) + + ginkgo.Context("config recursive match", func() { + ginkgo.It("should match a child", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.LogisticsAPIReplicaSet.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(1))) + }) + + ginkgo.It("should match a grand child", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.LogisticsAPIPodConfig.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(1))) + }) + + ginkgo.It("should not match", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.LogisticsUIDeployment.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(0))) + }) + }) + + ginkgo.Context("component recursive match", func() { + ginkgo.It("should match a child", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ComponentID: lo.ToPtr(dummy.LogisticsAPI.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(1))) + }) + + ginkgo.It("should match a grand child", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ComponentID: lo.ToPtr(dummy.LogisticsWorker.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(1))) + }) + + ginkgo.It("should not match", func() { + matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ComponentID: lo.ToPtr(dummy.ClusterComponent.ID.String())}) + Expect(err).To(BeNil()) + Expect(matched).To(Equal(int64(0))) + }) + }) +}) diff --git a/db/suite_test.go b/db/suite_test.go index b2e03b77a..a3c022c74 100644 --- a/db/suite_test.go +++ b/db/suite_test.go @@ -3,6 +3,8 @@ package db import ( "testing" + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/tests/setup" ginkgo "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +13,13 @@ func TestDB(t *testing.T) { RegisterFailHandler(ginkgo.Fail) ginkgo.RunSpecs(t, "DB") } + +var ( + DefaultContext context.Context +) + +var _ = ginkgo.BeforeSuite(func() { + DefaultContext = setup.BeforeSuiteFn() +}) + +var _ = ginkgo.AfterSuite(setup.AfterSuiteFn) diff --git a/go.mod b/go.mod index b86a63633..79fcdba92 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( require ( github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b + github.com/WinterYukky/gorm-extra-clause-plugin v0.2.0 github.com/aws/aws-sdk-go-v2 v1.30.4 github.com/aws/aws-sdk-go-v2/config v1.27.29 github.com/aws/aws-sdk-go-v2/credentials v1.17.29 @@ -88,7 +89,6 @@ require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/RaveNoX/go-jsonmerge v1.0.0 // indirect github.com/Snawoot/go-http-digest-auth-client v1.1.3 // indirect - github.com/WinterYukky/gorm-extra-clause-plugin v0.2.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect diff --git a/notification/controllers.go b/notification/controllers.go index 0805f8588..7c30e1cf3 100644 --- a/notification/controllers.go +++ b/notification/controllers.go @@ -1,8 +1,11 @@ package notification import ( + "encoding/json" "net/http" + "github.com/flanksource/duty/api" + "github.com/flanksource/duty/context" echoSrv "github.com/flanksource/incident-commander/echo" "github.com/flanksource/incident-commander/rbac" "github.com/labstack/echo/v4" @@ -13,7 +16,24 @@ func init() { } func RegisterRoutes(e *echo.Echo) { - e.GET("/notification/events", func(c echo.Context) error { + g := e.Group("/notification") + + g.GET("/events", func(c echo.Context) error { return c.JSON(http.StatusOK, EventRing.Get()) }, rbac.Authorization(rbac.ObjectMonitor, rbac.ActionRead)) + + g.POST("/silence", func(c echo.Context) error { + ctx := c.Request().Context().(context.Context) + + var req SilenceSaveRequest + if err := json.NewDecoder(c.Request().Body).Decode(&req); err != nil { + return err + } + + if err := SaveNotificationSilence(ctx, req); err != nil { + return api.WriteError(c, err) + } + + return nil + }, rbac.Authorization(rbac.ObjectNotification, rbac.ActionCreate)) } diff --git a/notification/events.go b/notification/events.go index 3af92a124..6da3a5bb4 100644 --- a/notification/events.go +++ b/notification/events.go @@ -128,6 +128,12 @@ func (t *notificationHandler) addNotificationEvent(ctx context.Context, event mo t.Ring.Add(event, celEnv) + silencedResource := getSilencedResourceFromCelEnv(celEnv) + matchingSilences, err := db.GetMatchingNotificationSilencesCount(ctx, silencedResource) + if err != nil { + return err + } + for _, id := range notificationIDs { n, err := GetNotification(ctx, id) if err != nil { @@ -182,6 +188,21 @@ func (t *notificationHandler) addNotificationEvent(ctx context.Context, event mo continue } + if matchingSilences > 0 { + ctx.Logger.V(6).Infof("silencing notification for event %s due to %d matching silences", event.ID, matchingSilences) + + if err := ctx.DB().Create(&models.NotificationSendHistory{ + NotificationID: n.ID, + ResourceID: payload.ID, + SourceEvent: event.Name, + Status: "silenced", + }).Error; err != nil { + return fmt.Errorf("failed to save silenced notification history: %w", err) + } + + return nil + } + newEvent := api.Event{ Name: api.EventNotificationSend, Properties: payload.AsMap(), diff --git a/notification/silence.go b/notification/silence.go new file mode 100644 index 000000000..66b0d9e54 --- /dev/null +++ b/notification/silence.go @@ -0,0 +1,103 @@ +package notification + +import ( + "errors" + "time" + + "github.com/flanksource/duty/api" + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/db" + "github.com/flanksource/duty/models" + "github.com/samber/lo" + "github.com/timberio/go-datemath" +) + +type SilenceSaveRequest struct { + models.NotificationSilenceResource + From string `json:"from"` + Until string `json:"until"` + Description string `json:"description"` + Recursive bool `json:"recursive"` + + from time.Time + until time.Time +} + +func (t *SilenceSaveRequest) Validate() error { + if t.From == "" { + return errors.New("`from` time is required") + } + + if t.Until == "" { + return errors.New("`until` is required") + } + + if parsedTime, err := datemath.ParseAndEvaluate(t.From); err != nil { + return err + } else { + t.from = parsedTime + } + + if parsedTime, err := datemath.ParseAndEvaluate(t.Until); err != nil { + return err + } else { + t.until = parsedTime + } + + if t.from.After(t.until) { + return errors.New("`from` time must be before `until") + } + + if t.NotificationSilenceResource.Empty() { + return errors.New("at least one of `config_id`, `canary_id`, `check_id` or `component_id` is required") + } + + return nil +} + +func SaveNotificationSilence(ctx context.Context, req SilenceSaveRequest) error { + if err := req.Validate(); err != nil { + return api.Errorf(api.EINVALID, err.Error()) + } + + silence := models.NotificationSilence{ + NotificationSilenceResource: req.NotificationSilenceResource, + From: req.from, + Until: req.until, + Description: req.Description, + Recursive: req.Recursive, + Source: models.SourceUI, + CreatedBy: lo.ToPtr(ctx.User().ID), + } + + return db.ErrorDetails(ctx.DB().Create(&silence).Error) +} + +func getSilencedResourceFromCelEnv(celEnv map[string]any) models.NotificationSilenceResource { + var silencedResource models.NotificationSilenceResource + if v, ok := celEnv["config"]; ok { + if vv, ok := v.(map[string]any); ok { + silencedResource.ConfigID = lo.ToPtr(vv["id"].(string)) + } + } + + if v, ok := celEnv["check"]; ok { + if vv, ok := v.(map[string]any); ok { + silencedResource.CheckID = lo.ToPtr(vv["id"].(string)) + } + } + + if v, ok := celEnv["canary"]; ok { + if vv, ok := v.(map[string]any); ok { + silencedResource.CanaryID = lo.ToPtr(vv["id"].(string)) + } + } + + if v, ok := celEnv["component"]; ok { + if vv, ok := v.(map[string]any); ok { + silencedResource.ComponentID = lo.ToPtr(vv["id"].(string)) + } + } + + return silencedResource +} diff --git a/notification/silence_test.go b/notification/silence_test.go new file mode 100644 index 000000000..3768d04a9 --- /dev/null +++ b/notification/silence_test.go @@ -0,0 +1,91 @@ +package notification + +import ( + "testing" + "time" + + "github.com/flanksource/duty/models" + "github.com/google/uuid" + "github.com/samber/lo" +) + +func TestSilenceSaveRequest_Validate(t *testing.T) { + type fields struct { + NotificationSilenceResource models.NotificationSilenceResource + From string + Until string + Description string + from time.Time + until time.Time + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "empty from", + fields: fields{ + NotificationSilenceResource: models.NotificationSilenceResource{}, + From: "", + Until: "now+2d", + }, + wantErr: true, + }, + { + name: "empty until", + fields: fields{ + NotificationSilenceResource: models.NotificationSilenceResource{}, + From: "now", + Until: "", + }, + wantErr: true, + }, + { + name: "empty resource", + fields: fields{ + NotificationSilenceResource: models.NotificationSilenceResource{}, + From: "now", + Until: "now+2d", + }, + wantErr: true, + }, + { + name: "valid", + fields: fields{ + NotificationSilenceResource: models.NotificationSilenceResource{ + ConfigID: lo.ToPtr(uuid.NewString()), + }, + From: "now", + Until: "now+2d", + }, + }, + { + name: "complete but invalid", + fields: fields{ + NotificationSilenceResource: models.NotificationSilenceResource{ + ConfigID: lo.ToPtr(uuid.NewString()), + }, + From: "now", + Until: "now-1m", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := &SilenceSaveRequest{ + NotificationSilenceResource: tt.fields.NotificationSilenceResource, + From: tt.fields.From, + Until: tt.fields.Until, + Description: tt.fields.Description, + from: tt.fields.from, + until: tt.fields.until, + } + if err := tr.Validate(); (err != nil) != tt.wantErr { + t.Fatalf("SilenceSaveRequest.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +}