From 4360014c92492a0a211008e21e4f104724a70f19 Mon Sep 17 00:00:00 2001 From: Eliran Bivas Date: Tue, 7 Nov 2017 07:49:17 +0200 Subject: [PATCH] add metrics (#34) --- engine/actions/action.go | 12 ++++++ engine/actions/autoassign/autoassign.go | 5 +++ engine/actions/automerge/automerge.go | 9 +++- engine/actions/commenter/commenter.go | 5 +++ engine/actions/labeler/labeler.go | 6 +++ engine/actions/locker/locker.go | 11 +++-- engine/actions/sizing/sizing.go | 6 +++ engine/actions/slack/slack.go | 6 +++ engine/actions/trigger/trigger.go | 9 +++- engine/condition.go | 55 ++++++++++++++++++++++--- runner/hook_listener.go | 23 +++++++++++ runner/local/work_unit.go | 26 +++++++++++- types/builder/data.go | 43 +++++++++++++++++++ 13 files changed, 204 insertions(+), 12 deletions(-) diff --git a/engine/actions/action.go b/engine/actions/action.go index 15f1ce8..ad3d8fb 100644 --- a/engine/actions/action.go +++ b/engine/actions/action.go @@ -1,10 +1,13 @@ package actions import ( + "fmt" "strings" "github.com/bivas/rivi/util/log" + "github.com/mitchellh/multistep" + "github.com/prometheus/client_golang/prometheus" "github.com/spf13/viper" ) @@ -51,3 +54,12 @@ func BuildActionsFromConfiguration(config *viper.Viper) []Action { } return result } + +func NewCounter(name string) prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "rivi", + Subsystem: "actions", + Name: name, + Help: fmt.Sprintf("Action counter for %s", name), + }) +} diff --git a/engine/actions/autoassign/autoassign.go b/engine/actions/autoassign/autoassign.go index f789292..4466391 100644 --- a/engine/actions/autoassign/autoassign.go +++ b/engine/actions/autoassign/autoassign.go @@ -12,6 +12,7 @@ import ( "github.com/bivas/rivi/util/log" "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" + "github.com/prometheus/client_golang/prometheus" ) type action struct { @@ -67,6 +68,7 @@ func (a *action) Apply(state multistep.StateBag) { winners := a.randomUsers(conf, meta, lookupRoles) if len(winners) > 0 { + counter.Inc() meta.AddAssignees(winners...) } } @@ -121,6 +123,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &action{rule: &item, logger: logger} } +var counter = actions.NewCounter("autoassign") + func init() { actions.RegisterAction("autoassign", &factory{}) + prometheus.Register(counter) } diff --git a/engine/actions/automerge/automerge.go b/engine/actions/automerge/automerge.go index 418376b..261fad3 100644 --- a/engine/actions/automerge/automerge.go +++ b/engine/actions/automerge/automerge.go @@ -1,6 +1,7 @@ package automerge import ( + "errors" "fmt" "strings" @@ -8,8 +9,10 @@ import ( "github.com/bivas/rivi/types" "github.com/bivas/rivi/util" "github.com/bivas/rivi/util/log" + "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" + "github.com/prometheus/client_golang/prometheus" ) type action struct { @@ -33,11 +36,12 @@ type HasReviewersAPIData interface { } func (a *action) merge(meta types.Data) { + counter.Inc() if a.rule.Label == "" { mergeable, ok := meta.(MergeableData) if !ok { a.logger.Warning("Event data does not support merge. Check your configurations") - a.err = fmt.Errorf("Event data does not support merge") + a.err = errors.New("event data does not support merge") return } mergeable.Merge(a.rule.Strategy) @@ -129,6 +133,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &action{rule: &item, logger: logger} } +var counter = actions.NewCounter("automerge") + func init() { actions.RegisterAction("automerge", &factory{}) + prometheus.Register(counter) } diff --git a/engine/actions/commenter/commenter.go b/engine/actions/commenter/commenter.go index 2907990..29ab3a9 100644 --- a/engine/actions/commenter/commenter.go +++ b/engine/actions/commenter/commenter.go @@ -8,6 +8,7 @@ import ( "github.com/bivas/rivi/util/log" "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" + "github.com/prometheus/client_golang/prometheus" ) type action struct { @@ -19,6 +20,7 @@ func (a *action) String() string { } func (a *action) Apply(state multistep.StateBag) { + counter.Inc() state.Get("data").(types.Data).AddComment(a.rule.Comment) } @@ -34,6 +36,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &action{rule: &item} } +var counter = actions.NewCounter("commenter") + func init() { actions.RegisterAction("commenter", &factory{}) + prometheus.Register(counter) } diff --git a/engine/actions/labeler/labeler.go b/engine/actions/labeler/labeler.go index 54523ba..23b2824 100644 --- a/engine/actions/labeler/labeler.go +++ b/engine/actions/labeler/labeler.go @@ -8,6 +8,7 @@ import ( "github.com/bivas/rivi/util/log" "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" + "github.com/prometheus/client_golang/prometheus" ) type action struct { @@ -29,6 +30,7 @@ func (a *action) Apply(state multistep.StateBag) { log.MetaFields{log.F("issue", meta.GetShortName())}, "Skipping label '%s' as it already exists", apply) } else { + counter.Inc() meta.AddLabel(apply) } } @@ -39,6 +41,7 @@ func (a *action) Apply(state multistep.StateBag) { log.MetaFields{log.F("issue", meta.GetShortName())}, "Skipping label '%s' removal as it does not exists", remove) } else { + counter.Inc() meta.RemoveLabel(remove) } } @@ -57,6 +60,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &action{rule: &item, logger: logger} } +var counter = actions.NewCounter("labeler") + func init() { actions.RegisterAction("labeler", &factory{}) + prometheus.Register(counter) } diff --git a/engine/actions/locker/locker.go b/engine/actions/locker/locker.go index 9a2cf92..8c0a79e 100644 --- a/engine/actions/locker/locker.go +++ b/engine/actions/locker/locker.go @@ -1,13 +1,13 @@ package locker import ( - "fmt" - "github.com/bivas/rivi/engine/actions" "github.com/bivas/rivi/types" "github.com/bivas/rivi/util/log" "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" ) type LockableData interface { @@ -29,13 +29,14 @@ func (a *action) Apply(state multistep.StateBag) { a.logger.WarningWith( log.MetaFields{log.F("issue", meta.GetShortName())}, "Event data does not support locking. Check your configurations") - a.err = fmt.Errorf("Event data does not support locking") + a.err = errors.New("event data does not support locking") return } if lockable.LockState() { a.logger.Debug("Issue is locked") if a.rule.State == "unlock" || a.rule.State == "change" { a.logger.DebugWith(log.MetaFields{log.F("issue", meta.GetShortName())}, "unlocking issue") + counter.Inc() lockable.Unlock() } else if a.rule.State == "lock" { a.logger.DebugWith(log.MetaFields{log.F("issue", meta.GetShortName())}, "Issue is already locked - nothing changed") @@ -44,6 +45,7 @@ func (a *action) Apply(state multistep.StateBag) { a.logger.Debug("Issue is unlocked") if a.rule.State == "lock" || a.rule.State == "change" { a.logger.DebugWith(log.MetaFields{log.F("issue", meta.GetShortName())}, "Locking issue") + counter.Inc() lockable.Lock() } else if a.rule.State == "lock" { a.logger.DebugWith(log.MetaFields{log.F("issue", meta.GetShortName())}, "Issue is already unlocked - nothing changed") @@ -64,6 +66,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &action{rule: &item, logger: logger} } +var counter = actions.NewCounter("locker") + func init() { actions.RegisterAction("locker", &factory{}) + prometheus.Register(counter) } diff --git a/engine/actions/sizing/sizing.go b/engine/actions/sizing/sizing.go index 4ad7db2..a80eda6 100644 --- a/engine/actions/sizing/sizing.go +++ b/engine/actions/sizing/sizing.go @@ -10,6 +10,7 @@ import ( "github.com/bivas/rivi/util/log" "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" + "github.com/prometheus/client_golang/prometheus" ) type action struct { @@ -92,6 +93,7 @@ func (a *action) Apply(state multistep.StateBag) { a.logger.DebugWith( log.MetaFields{log.F("issue", meta.GetShortName())}, "Updating label from %s to %s", currentMatchedLabel, matchedLabel) + counter.Inc() meta.RemoveLabel(currentMatchedLabel) meta.AddLabel(matchedLabel) if matchedRule.Comment != "" { @@ -102,6 +104,7 @@ func (a *action) Apply(state multistep.StateBag) { log.F("issue", meta.GetShortName())}, "Updating label to %s", matchedLabel) + counter.Inc() meta.AddLabel(matchedLabel) if matchedRule.Comment != "" { meta.AddComment(matchedRule.Comment) @@ -130,6 +133,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &result } +var counter = actions.NewCounter("sizing") + func init() { actions.RegisterAction("sizing", &factory{}) + prometheus.Register(counter) } diff --git a/engine/actions/slack/slack.go b/engine/actions/slack/slack.go index 7affbb8..958ad5f 100644 --- a/engine/actions/slack/slack.go +++ b/engine/actions/slack/slack.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" api "github.com/nlopes/slack" + "github.com/prometheus/client_golang/prometheus" ) type action struct { @@ -86,6 +87,7 @@ func (a *action) sendChannelMessage(config config.Configuration, meta types.Data "Unable to get channel info") return } + counter.Inc() a.postMessage(channel.ID, meta.GetAssignees(), config, meta) } @@ -100,6 +102,7 @@ func (a *action) sendPrivateMessage(config config.Configuration, meta types.Data log.F("user.id", slacker)}, "Unable to open IM channel") continue } + counter.Inc() if err := a.postMessage(id, targets, config, meta); err != nil { continue } @@ -180,6 +183,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &action{rule: &item, logger: logger} } +var counter = actions.NewCounter("slack") + func init() { actions.RegisterAction("slack", &factory{}) + prometheus.Register(counter) } diff --git a/engine/actions/trigger/trigger.go b/engine/actions/trigger/trigger.go index 1ec1774..65bef27 100644 --- a/engine/actions/trigger/trigger.go +++ b/engine/actions/trigger/trigger.go @@ -12,6 +12,7 @@ import ( "github.com/bivas/rivi/util/log" "github.com/mitchellh/mapstructure" "github.com/mitchellh/multistep" + "github.com/prometheus/client_golang/prometheus" ) type action struct { @@ -35,13 +36,14 @@ func (a *action) Apply(state multistep.StateBag) { log.F("method", request.Method), log.F("content-length", request.ContentLength), }, "Prepared Request") + counter.Inc() response, e := a.client.Do(request) if e != nil { - a.err = fmt.Errorf("Triggering to %s, resulted in error. %s", + a.err = fmt.Errorf("triggering to %s, resulted in error. %s", a.rule.Endpoint, e) } else if response.StatusCode >= 400 { - a.err = fmt.Errorf("Triggering to %s, resulted in wrong status code. %d", + a.err = fmt.Errorf("triggering to %s, resulted in wrong status code. %d", a.rule.Endpoint, response.StatusCode) } @@ -100,6 +102,9 @@ func (*factory) BuildAction(config map[string]interface{}) actions.Action { return &action{rule: &item, client: http.DefaultClient, logger: logger} } +var counter = actions.NewCounter("trigger") + func init() { actions.RegisterAction("trigger", &factory{}) + prometheus.Register(counter) } diff --git a/engine/condition.go b/engine/condition.go index 404f56d..505a36f 100644 --- a/engine/condition.go +++ b/engine/condition.go @@ -1,13 +1,15 @@ package engine import ( + "fmt" "regexp" - "strings" - "strconv" + "strings" "github.com/bivas/rivi/types" "github.com/bivas/rivi/util/log" + + "github.com/prometheus/client_golang/prometheus" "github.com/spf13/viper" ) @@ -87,6 +89,7 @@ func (c *FilesCondition) checkExt(meta types.Data) bool { } func (c *FilesCondition) Match(meta types.Data) bool { + fileConditionCounter.Inc() return c.checkPattern(meta) || c.checkExt(meta) } @@ -101,6 +104,7 @@ func (c *TitleCondition) IsEmpty() bool { } func (c *TitleCondition) Match(meta types.Data) bool { + titleConditionCounter.Inc() title := meta.GetTitle() if c.StartsWith != "" && strings.HasPrefix(title, c.StartsWith) { lc.DebugWith( @@ -130,6 +134,7 @@ func (c *DescriptionCondition) IsEmpty() bool { } func (c *DescriptionCondition) Match(meta types.Data) bool { + descriptionConditionCounter.Inc() description := meta.GetDescription() if c.StartsWith != "" && strings.HasPrefix(description, c.StartsWith) { lc.DebugWith( @@ -162,6 +167,7 @@ func (c *RefCondition) IsEmpty() bool { } func (c *RefCondition) Match(meta types.Data) bool { + refConditionCounter.Inc() ref := meta.GetRef() if c.Equals != "" && ref == c.Equals { lc.DebugWith( @@ -197,14 +203,19 @@ func (c *CommentsCondition) IsEmpty() bool { } func (c *CommentsCondition) Match(meta types.Data) bool { + commentsConditionCounter.Inc() if commentsRegex == nil { - lc.DebugWith( + lc.WarningWith( log.MetaFields{log.F("condition", "CommentsCondition"), log.F("issue", meta.GetShortName())}, - "comments regex is nil'") + "comments regex is nil") return false } if c.Count == "" { + lc.DebugWith( + log.MetaFields{log.F("condition", "CommentsCondition"), + log.F("issue", meta.GetShortName())}, + "comments count is empty") return false } count := int64(len(meta.GetComments())) @@ -287,6 +298,7 @@ func (c *Condition) checkIfLabeled(meta types.Data) bool { if len(c.IfLabeled) == 0 { return false } else { + labelConditionCounter.Inc() for _, check := range c.IfLabeled { for _, label := range meta.GetLabels() { if check == label { @@ -312,6 +324,7 @@ func (c *Condition) checkAllEmpty(meta types.Data) bool { lc.DebugWith( log.MetaFields{log.F("issue", meta.GetShortName())}, "Condition is empty") + emptyConditionCounter.Inc() } return empty } @@ -325,7 +338,8 @@ func (c *Condition) Match(meta types.Data) bool { c.Ref.Match(meta) || c.Comments.Match(meta) - if match { + if match && len(c.SkipIfLabeled) > 0 { + blockByLabelConditionCounter.Inc() for _, check := range c.SkipIfLabeled { for _, label := range meta.GetLabels() { if check == label { @@ -340,6 +354,26 @@ func (c *Condition) Match(meta types.Data) bool { return match } +func createCounter(name string) prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "rivi", + Subsystem: "condition", + Name: name, + Help: fmt.Sprintf("Condition counter for %s", name), + }) +} + +var ( + emptyConditionCounter = createCounter("empty") + labelConditionCounter = createCounter("label") + blockByLabelConditionCounter = createCounter("not_labeled") + commentsConditionCounter = createCounter("comments") + descriptionConditionCounter = createCounter("description") + fileConditionCounter = createCounter("files") + refConditionCounter = createCounter("ref") + titleConditionCounter = createCounter("title") +) + func buildConditionFromConfiguration(config *viper.Viper) Condition { var result Condition exists := config.Get("condition") @@ -351,3 +385,14 @@ func buildConditionFromConfiguration(config *viper.Viper) Condition { } return result } + +func init() { + prometheus.Register(emptyConditionCounter) + prometheus.Register(labelConditionCounter) + prometheus.Register(blockByLabelConditionCounter) + prometheus.Register(commentsConditionCounter) + prometheus.Register(descriptionConditionCounter) + prometheus.Register(fileConditionCounter) + prometheus.Register(refConditionCounter) + prometheus.Register(titleConditionCounter) +} diff --git a/runner/hook_listener.go b/runner/hook_listener.go index c34a8f4..6c8098e 100644 --- a/runner/hook_listener.go +++ b/runner/hook_listener.go @@ -7,6 +7,7 @@ import ( "github.com/bivas/rivi/runner/types" "github.com/bivas/rivi/types/builder" "github.com/bivas/rivi/util/log" + "github.com/prometheus/client_golang/prometheus" "github.com/spf13/viper" ) @@ -18,8 +19,11 @@ type hookListener struct { } func (h *hookListener) HandleEvent(r *http.Request) *HandledEventResult { + timer := prometheus.NewTimer(incomingWebhooksHistogram) + defer timer.ObserveDuration() data, ok := builder.BuildFromHook(h.conf, r) if !ok { + skippedWebhooksCounter.Inc() return &HandledEventResult{Message: "Skipping hook processing"} } h.queue.Enqueue(types.NewMessage(h.conf, data)) @@ -44,3 +48,22 @@ func NewHookListenerWithClientConfig(config client.ClientConfig) Runner { logger: logger, } } + +var incomingWebhooksHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "rivi", + Subsystem: "webhook", + Name: "incoming", + Help: "Measure incoming webhook processing", +}) + +var skippedWebhooksCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "rivi", + Subsystem: "webhook", + Name: "skipped", + Help: "Measure skipped webhooks", +}) + +func init() { + prometheus.Register(incomingWebhooksHistogram) + prometheus.Register(skippedWebhooksCounter) +} diff --git a/runner/local/work_unit.go b/runner/local/work_unit.go index a0afaef..ab2b0b5 100644 --- a/runner/local/work_unit.go +++ b/runner/local/work_unit.go @@ -10,6 +10,7 @@ import ( "github.com/bivas/rivi/types/builder" "github.com/bivas/rivi/util/log" "github.com/bivas/rivi/util/state" + "github.com/prometheus/client_golang/prometheus" ) type workUnit struct { @@ -28,7 +29,7 @@ func (w *workUnit) internalHandle(msg *types.Message) error { defer environment.Cleanup() meta, ok := builder.BuildComplete(msg.Config, msg.Data) if !ok { - return errors.New("Nothing to process") + return errors.New("nothing to process") } if err := environment.Create(meta); err != nil { w.logger.ErrorWith( @@ -60,10 +61,33 @@ func (w *workUnit) Handle() { log.MetaFields{ log.F("data", msg.Data.GetShortName()), }, "Got data from job channel") + timer := prometheus.NewTimer(handleHistogram) if err := w.internalHandle(msg); err != nil { + handleErrorCounter.Inc() w.logger.WarningWith(log.MetaFields{ log.E(err), log.F("data", msg.Data.GetShortName())}, "Error when handling data") } + timer.ObserveDuration() } } + +var ( + handleHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "rivi", + Subsystem: "workunit", + Name: "handle", + Help: "Measure handling of event data", + }) + handleErrorCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "rivi", + Subsystem: "workunit", + Name: "failure", + Help: "Failure to handle event data", + }) +) + +func init() { + prometheus.Register(handleHistogram) + prometheus.Register(handleErrorCounter) +} diff --git a/types/builder/data.go b/types/builder/data.go index 6f5b1ff..c80340e 100644 --- a/types/builder/data.go +++ b/types/builder/data.go @@ -7,6 +7,7 @@ import ( "github.com/bivas/rivi/config/client" "github.com/bivas/rivi/types" "github.com/bivas/rivi/util/log" + "github.com/prometheus/client_golang/prometheus" ) var l log.Logger = log.Get("data.builder") @@ -40,29 +41,71 @@ func getBuilderFromRequest(r *http.Request) DataBuilder { } func BuildFromHook(config client.ClientConfig, r *http.Request) (types.HookData, bool) { + timer := prometheus.NewTimer(buildFromHookHistogram) + defer timer.ObserveDuration() builder := getBuilderFromRequest(r) if builder == nil { l.Error("No Builder to work with!") + buildFromHookFailure.Inc() return nil, false } result, process, err := builder.BuildFromHook(config, r) if err != nil { l.ErrorWith(log.MetaFields{log.E(err)}, "Unable to build from request") + buildFromHookFailure.Inc() return nil, false } return result, process } func BuildComplete(config client.ClientConfig, data types.ReadOnlyData) (types.Data, bool) { + timer := prometheus.NewTimer(buildCompleteHistogram) + defer timer.ObserveDuration() builder, exists := builders[data.GetProvider()] if !exists { l.Error("No existing builder to work with!") + buildCompleteFailure.Inc() return nil, false } result, process, err := builder.BuildFromPayload(config, data.GetRawType(), data.GetRawPayload()) if err != nil { l.ErrorWith(log.MetaFields{log.E(err)}, "Unable to build from payload.") + buildCompleteFailure.Inc() return nil, false } return result, process } + +var ( + buildFromHookHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "rivi", + Subsystem: "data", + Name: "buildFromHook", + Help: "Build from webhook histogram", + }) + buildFromHookFailure = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "rivi", + Subsystem: "data", + Name: "buildFromHookFailure", + Help: "Build from webhook failures", + }) + buildCompleteHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "rivi", + Subsystem: "data", + Name: "buildComplete", + Help: "Build from processed data histogram", + }) + buildCompleteFailure = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "rivi", + Subsystem: "data", + Name: "buildCompleteFailure", + Help: "Build from processed data failures", + }) +) + +func init() { + prometheus.Register(buildFromHookHistogram) + prometheus.Register(buildFromHookFailure) + prometheus.Register(buildCompleteHistogram) + prometheus.Register(buildCompleteFailure) +}