From c4b081c682041dc7ea1a1992c3e8b67929c2c697 Mon Sep 17 00:00:00 2001 From: Felix Breidenstein Date: Wed, 13 Mar 2019 00:53:51 +0100 Subject: [PATCH] Release v0.3.0 (#43) * Use log.Error instead of log.Fatal for json parsing errors * Added icinga2 module * Fix Hackint Webchat markdown in the README * Fix typo in log message * Added RWMutex to track connection state of IRC client * fixup! Fix Hackint Webchat markdown in the README * prometheus: Better way to determine alert name First it checks if the alert has an instance label. If not it falls back to the alert name. This fixes #26 --- Gopkg.lock | 9 + README.md | 19 +- cpthook.yml.example | 9 + input/gitlab.go | 10 +- input/icinga2.go | 358 +++++++++++++++++++++++++++++++++++ input/icinga2_test.go | 45 +++++ input/prometheus.go | 16 +- input/test_data/icinga2.json | 39 ++++ irc.go | 11 +- main.go | 2 + 10 files changed, 509 insertions(+), 9 deletions(-) create mode 100644 input/icinga2.go create mode 100644 input/icinga2_test.go create mode 100644 input/test_data/icinga2.json diff --git a/Gopkg.lock b/Gopkg.lock index cb03980..faf63df 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,14 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + digest = "1:6f9339c912bbdda81302633ad7e99a28dfa5a639c864061f1929510a9a64aa74" + name = "github.com/dustin/go-humanize" + packages = ["."] + pruneopts = "UT" + revision = "9f541cc9db5d55bce703bd99987c9d5cb8eea45e" + version = "v1.0.0" + [[projects]] digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd" name = "github.com/fsnotify/fsnotify" @@ -165,6 +173,7 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/dustin/go-humanize", "github.com/lrstanley/girc", "github.com/sirupsen/logrus", "github.com/spf13/viper", diff --git a/README.md b/README.md index 8fd084f..ddfaf9c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ different IRC channels according to the configuration. Take a look at the [input](https://github.com/fleaz/CptHook/tree/master/input) folder to find out which services are already supported by CptHook. -**If you have questions or problems visit #CptHook on HackInt -> [WebChat](*https://webirc.hackint.org/#irc://irc.hackint.org/#CptHook)** +**If you have questions or problems visit #CptHook on HackInt** -> [WebChat](https://webirc.hackint.org/#irc://irc.hackint.org/#CptHook) ## Installation @@ -86,3 +86,20 @@ This dictionary maps full project paths (groupname/projectname) to IRC-channels. Receives arbitrary messages as text via a HTTP `POST` request and forwards this message line by line to a channel. The channel can be specified by the `channel` query parameter, otherwise the `default_channel` from the config will be used. + +### Icinga2 +Receives webhooks from Icinga2. Add [icinga2-notifications-webhook] to your icinga2 installation to send the +required webhooks. + +When a webhook is received this module will first check if there is an explicit +mapping in the configuration especially for this host. If yes, this channel will be used. If not, the module will +look if there exists for the hostgroup. If yes, this channel will be used. If not, the `default` channel will be used. + +**Module specific configuration** +``` +- hostgroups +This dictionary maps Icinga2 hostgroups to IRC-channels. +- explicit +This dictionary maps hostnames to IRC-channels. +``` +[icinga2-notifications-webhook]: https://git.s7t.de/ManiacTwister/icinga2-notifications-webhook diff --git a/cpthook.yml.example b/cpthook.yml.example index 85c77a8..0ff2c62 100644 --- a/cpthook.yml.example +++ b/cpthook.yml.example @@ -36,4 +36,13 @@ modules: simple: enabled: True default_channel: "#defaultChannel" + icinga2: + enabled: True + default: "#monitoring" + hostgroups: + "webservers": + - "#monitoring-web" + explicit: + "host.example.tld": + - "#monitoring-example" diff --git a/input/gitlab.go b/input/gitlab.go index 87c62ed..504e48f 100644 --- a/input/gitlab.go +++ b/input/gitlab.go @@ -271,7 +271,7 @@ func (m GitlabModule) GetHandler() http.HandlerFunc { log.Printf("Got a Hook for a Pipeline Event") var pipelineEvent PipelineEvent if err := decoder.Decode(&pipelineEvent); err != nil { - log.Fatal(err) + log.Error(err) return } @@ -303,7 +303,7 @@ func (m GitlabModule) GetHandler() http.HandlerFunc { log.Printf("Got a Hook for a Job Event") var jobEvent JobEvent if err := decoder.Decode(&jobEvent); err != nil { - log.Fatal(err) + log.Error(err) return } @@ -328,7 +328,7 @@ func (m GitlabModule) GetHandler() http.HandlerFunc { log.Printf("Got Hook for a Merge Request") var mergeEvent MergeEvent if err := decoder.Decode(&mergeEvent); err != nil { - log.Fatal(err) + log.Error(err) return } @@ -342,7 +342,7 @@ func (m GitlabModule) GetHandler() http.HandlerFunc { log.Printf("Got Hook for an Issue") var issueEvent IssueEvent if err := decoder.Decode(&issueEvent); err != nil { - log.Fatal(err) + log.Error(err) return } @@ -356,7 +356,7 @@ func (m GitlabModule) GetHandler() http.HandlerFunc { log.Printf("Got Hook for a Push Event") var pushEvent PushEvent if err := decoder.Decode(&pushEvent); err != nil { - log.Println(err) + log.Error(err) return } diff --git a/input/icinga2.go b/input/icinga2.go new file mode 100644 index 0000000..d8a0ccf --- /dev/null +++ b/input/icinga2.go @@ -0,0 +1,358 @@ +package input + +import ( + "time" + "bytes" + "strings" + "net/http" + "text/template" + "encoding/json" + "github.com/dustin/go-humanize" + log "github.com/sirupsen/logrus" + + "github.com/spf13/viper" +) + +func JsonToTime(ts json.Number) time.Time { + t, err := ts.Float64() + if err == nil { + return time.Unix(int64(t), 0) + } + return time.Unix(0, 0) +} + +func AgoString(t time.Time) string { + return "since " + strings.Trim(humanize.RelTime(time.Now(), t, "", ""), " ") +} + +func ColorHostState(s string) string { + HostState := map[string]string{ + "UP": "\x0303Up\x03", + "DOWN": "\x0304Down\x03", + } + return HostState[s] +} + +func ColorServiceState(s string) string { + ServiceState := map[string]string{ + "UNKNOWN": "\x0313Unknown\x03", + "CRITICAL": "\x0304Critical\x03", + "WARNING": "\x0308Warning\x03", + "OK": "\x0303Ok\x03", + } + return ServiceState[s] +} + +type Host struct { + CheckAttempt float32 `json:"check_attempt"` + Name string `json:"name"` + DisplayName string `json:"display_name"` + HostGroups []string `json:"hostgroups"` + State string `json:"state"` + StateType string `json:"state_type"` + LastState string `json:"last_state"` + LastStateType string `json:"last_state_type"` + LastHardState json.Number `json:"last_hard_state"` + Output string `json:"output"` + WebURL string `json:"url"` + LastStateChangeStr json.Number `json:"last_state_change"` + LastHardStateChangeStr json.Number `json:"last_hard_state_change"` +} + +func (h Host) ColoredState() string { + return ColorHostState(h.State); +} + +func (h Host) ColoredLastState() string { + return ColorServiceState(h.LastState); +} + +func (h Host) LastStateChange() time.Time { + return JsonToTime(h.LastStateChangeStr); +} + +func (h Host) LastHardStateChange() time.Time { + return JsonToTime(h.LastHardStateChangeStr); +} + +func (h Host) AgoString() string { + return AgoString(h.LastStateChange()); +} + + +type Service struct { + CheckAttempt float32 `json:"check_attempt"` + Name string `json:"name"` + DisplayName string `json:"display_name"` + State string `json:"state"` + StateType string `json:"state_type"` + LastState string `json:"last_state"` + LastStateType string `json:"last_state_type"` + LastHardState json.Number `json:"last_hard_state"` + Output string `json:"output"` + WebURL string `json:"url"` + LastStateChangeStr json.Number `json:"last_state_change"` + LastHardStateChangeStr json.Number `json:"last_hard_state_change"` +} + +func (s Service) ColoredState() string { + return ColorServiceState(s.State); +} + +func (s Service) ColoredLastState() string { + return ColorServiceState(s.LastState); +} + +func (s Service) LastStateChange() time.Time { + return JsonToTime(s.LastStateChangeStr); +} + +func (s Service) LastHardStateChange() time.Time { + return JsonToTime(s.LastHardStateChangeStr); +} + +func (s Service) AgoString() string { + return AgoString(s.LastStateChange()); +} + + +type Notification struct { + Author string `json:"author"` + Comment string `json:"comment"` + Target string `json:"target"` + Type string `json:"type"` + Timestamp json.Number `json:"timet"` + DateTime string `json:"long_date_time"` + Host Host `json:"host"` + Service Service `json:"service"` + Channels []string `json:"channels"` +} + + +type Icinga2Module struct { + channelMapping hgmapping + channel chan IRCMessage +} + +type hgmapping struct { + DefaultChannel string `mapstructure:"default"` + HostGroupMappings map[string][]string `mapstructure:"hostgroups"` + ExplicitMappings map[string][]string `mapstructure:"explicit"` +} + +func (m *Icinga2Module) Init(c *viper.Viper, channel *chan IRCMessage) { + err := c.UnmarshalKey("default", &m.channelMapping.DefaultChannel) + if err != nil { + log.Panic(err) + } + err = c.UnmarshalKey("hostgroups", &m.channelMapping.HostGroupMappings) + if err != nil { + log.Panic(err) + } + err = c.UnmarshalKey("explicit", &m.channelMapping.ExplicitMappings) + if err != nil { + log.Panic(err) + } + m.channel = *channel +} + +func (m Icinga2Module) sendMessage(message string, notification Notification) { + var channelNames []string + var hostname = notification.Host.Name + if contains(m.channelMapping.ExplicitMappings, hostname) { // Check if explicit mapping exists + for _, channelName := range m.channelMapping.ExplicitMappings[hostname] { + channelNames = append(channelNames, channelName) + } + } else { + var found = false + for _, hostgroup := range notification.Host.HostGroups { // Check if hostgroup mapping exists + if contains(m.channelMapping.HostGroupMappings, hostgroup) { + for _, channelName := range m.channelMapping.HostGroupMappings[hostgroup] { + channelNames = append(channelNames, channelName) + found = true + } + } + } + if !found { // Fall back to default channel + channelNames = append(channelNames, m.channelMapping.DefaultChannel) + } + } + + for _, channelName := range channelNames { + var event IRCMessage + event.Messages = append(event.Messages, message) + event.Channel = channelName + m.channel <- event + } + +} + +func (m Icinga2Module) GetChannelList() []string { + var all []string + + for _, v := range m.channelMapping.ExplicitMappings { + for _, name := range v { + all = append(all, name) + } + } + for _, v := range m.channelMapping.HostGroupMappings { + for _, name := range v { + all = append(all, name) + } + } + all = append(all, m.channelMapping.DefaultChannel) + return all +} + +func (m Icinga2Module) GetEndpoint() string { + return "/icinga2" +} + +func (m Icinga2Module) GetHandler() http.HandlerFunc { + + const serviceStateChangeString = "Service \x0312{{ .Service.DisplayName }}\x03 (\x0314{{ .Host.DisplayName }}\x03) transitioned from state {{ .Service.ColoredLastState }} to {{ .Service.ColoredState }}" + const serviceStateEnteredString = "Service \x0312{{ .Service.DisplayName }}\x03 (\x0314{{ .Host.DisplayName }}\x03) entered state {{ .Service.ColoredState }}" + const serviceStateString = "Service \x0312{{ .Service.DisplayName }}\x03 (\x0314{{ .Host.DisplayName }}\x03) is still in state {{ .Service.ColoredState }} ({{ .Service.AgoString }})" + const serviceAckString = "{{ .Author }} acknowledged service \x0312{{ .Service.DisplayName }}\x03 (State {{ .Service.ColoredState }} {{ .Service.AgoString }})" + const serviceRecoveryString = "Service \x0312{{ .Service.DisplayName }}\x03 (\x0314{{ .Host.DisplayName }}\x03) \x0303recovered\x03 from state {{ .Service.ColoredLastState }}" + const serviceOutputString = "→ {{ .Service.Output }}" + + const hostStateChangeString = "Host \x0312{{ .Host.DisplayName }}\x03 transitioned from state {{ .Host.ColoredLastState }} to {{ .Host.ColoredState }}" + const hostStateEnteredString = "Host \x0312{{ .Host.DisplayName }}\x03 entered state {{ .Host.ColoredState }}" + const hostStateString = "Host \x0312{{ .Host.DisplayName }}\x03 is still in state {{ .Host.ColoredState }} ({{ .Host.AgoString }})" + const hostAckString = "{{ .Author }} acknowledged host \x0312{{ .Host.DisplayName }}\x03 (State {{ .Host.ColoredState }} {{ .Host.AgoString }})" + const hostRecoveryString = "Host \x0312{{ .Host.DisplayName }}\x03 \x0303recovered\x03 from state {{ .Host.ColoredLastState }}" + const hostOutputString = "→ {{ .Host.Output }}" + + serviceStateChangeTemplate, err := template.New("hostOutput").Parse(serviceStateChangeString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse serviceStateChange template: %v", err) + } + + serviceStateEnteredTemplate, err := template.New("hostOutput").Parse(serviceStateEnteredString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse serviceStateEntered template: %v", err) + } + + serviceStateTemplate, err := template.New("serviceState").Parse(serviceStateString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse serviceState template: %v", err) + } + + serviceAckTemplate, err := template.New("serviceState").Parse(serviceAckString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse serviceAck template: %v", err) + } + + serviceRecoveryTemplate, err := template.New("serviceState").Parse(serviceRecoveryString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse serviceRecovery template: %v", err) + } + + serviceOutputTemplate, err := template.New("serviceOutput").Parse(serviceOutputString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse serviceOutput template: %v", err) + } + + hostStateChangeTemplate, err := template.New("hostOutput").Parse(hostStateChangeString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse hostStateChange template: %v", err) + } + + hostStateEnteredTemplate, err := template.New("hostOutput").Parse(hostStateEnteredString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse hostStateEntered template: %v", err) + } + + hostStateTemplate, err := template.New("hostState").Parse(hostStateString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse hostState template: %v", err) + } + + hostAckTemplate, err := template.New("serviceState").Parse(hostAckString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse hostAck template: %v", err) + } + + hostRecoveryTemplate, err := template.New("serviceState").Parse(hostRecoveryString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse hostRecovery template: %v", err) + } + + hostOutputTemplate, err := template.New("hostOutput").Parse(hostOutputString) + if err != nil { + log.Fatalf("[icinga2] Failed to parse hostOutput template: %v", err) + } + + return func(wr http.ResponseWriter, req *http.Request) { + log.Debug("[icinga2] Got a request for the Icinga2Module") + defer req.Body.Close() + decoder := json.NewDecoder(req.Body) + + var buf bytes.Buffer + var notification Notification + if err := decoder.Decode(¬ification); err != nil { + log.Panic(err) + return + } + + switch notification.Target { + + case "service": + log.Printf("[icinga2] Got a Hook for a Service Event") + + if notification.Type == "ACKNOWLEDGEMENT" { // Acknowledge + err = serviceAckTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else if notification.Type == "RECOVERY" { // Recovery + err = serviceRecoveryTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else if notification.Service.LastStateType != notification.Service.StateType { // State entered + err = serviceStateEnteredTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + buf.Reset() + err = serviceOutputTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else if notification.Service.LastState == notification.Service.State { // Renotification + err = serviceStateTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else { // State changed + err = serviceStateChangeTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + buf.Reset() + err = serviceOutputTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } + + case "host": + log.Printf("[icinga2] Got a Hook for a Host Event") + + if notification.Type == "ACKNOWLEDGEMENT" { // Acknowledge + err = hostAckTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else if notification.Type == "RECOVERY" { // Recovery + err = hostRecoveryTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else if notification.Host.LastStateType != notification.Host.StateType { // State entered + err = hostStateEnteredTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + buf.Reset() + err = hostOutputTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else if notification.Host.LastState == notification.Host.State { // Renotification + err = hostStateTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } else { // State changed + err = hostStateChangeTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + buf.Reset() + err = hostOutputTemplate.Execute(&buf, ¬ification) + m.sendMessage(buf.String(), notification) + } + default: + log.Printf("[icinga2] Unknown event: %s", notification.Target) + } + + } + +} diff --git a/input/icinga2_test.go b/input/icinga2_test.go new file mode 100644 index 0000000..4f7e053 --- /dev/null +++ b/input/icinga2_test.go @@ -0,0 +1,45 @@ +package input + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + + log "github.com/sirupsen/logrus" + + "github.com/spf13/viper" +) + +func TestIcinga2Handler(t *testing.T) { + viper.SetConfigName("cpthook") + viper.AddConfigPath("../") + err := viper.ReadInConfig() + if err != nil { + log.Fatal(err) + } + + file, e := os.Open("./test_data/icinga2.json") + if e != nil { + log.Fatal(e) + } + + req, err := http.NewRequest("POST", "/", file) + req.Header.Set("Content-Type", "application/json") + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + var icinga2Module Module = &Icinga2Module{} + c := make(chan IRCMessage, 10) + icinga2Module.Init(viper.Sub("modules.icinga2"), &c) + handler := http.HandlerFunc(icinga2Module.GetHandler()) + + handler.ServeHTTP(rr, req) + + if status := rr.Code; status != http.StatusOK { + t.Errorf("Handler returned wrong status code: got %v wanted %v", + status, http.StatusOK) + } +} diff --git a/input/prometheus.go b/input/prometheus.go index 1ca8373..87f6a75 100644 --- a/input/prometheus.go +++ b/input/prometheus.go @@ -154,8 +154,7 @@ func (m PrometheusModule) GetHandler() http.HandlerFunc { instanceList = instanceList[:0] for _, alert := range alertList { - name := alert.Labels["instance"].(string) - name = shortenInstanceName(name, m.hostnameFilter) + name := getNameFromLabels(&alert, m.hostnameFilter) value, ok := alert.Annotations["value"].(string) if ok { inst = instance{Name: name, Value: value} @@ -194,3 +193,16 @@ func (m PrometheusModule) GetHandler() http.HandlerFunc { } } + +// getNameFromLabels tries to determine a meaningful name for an alert +// If the alert has no 'instance' label, we use the 'alertname' which should always +// be present in an alert +func getNameFromLabels(alert *alert, filter string) string { + if instance, ok := alert.Labels["instance"]; ok { + return shortenInstanceName(instance.(string), filter) + } else if alertName, ok := alert.Labels["alertname"]; ok { + return alertName.(string) + } else { + return "unknown" + } +} diff --git a/input/test_data/icinga2.json b/input/test_data/icinga2.json new file mode 100644 index 0000000..77cee6e --- /dev/null +++ b/input/test_data/icinga2.json @@ -0,0 +1,39 @@ +{ + "author":"admin", + "comment":"test", + "host":{ + "check_attempt":1.0, + "display_name":"test.host.tld", + "hostgroups":[ + "testhost" + ], + "last_hard_state":0.0, + "last_hard_state_change":1547331012.0, + "last_state":"UP", + "last_state_change":1547331012.0, + "last_state_type":"HARD", + "name":"test.host.tld", + "output":"PING =OK - Packet loss = 0%, RTA = 5.13 ms", + "state":"UP", + "state_type":"HARD", + "url":"https://monitoring.bla.tld/monitoring/host/show?host=test.host.tld" + }, + "long_date_time":"2019-01-13 20:19:45 +0100", + "service":{ + "check_attempt":1.0, + "display_name":"test-service-check", + "last_hard_state":3.0, + "last_hard_state_change":1547386923.0, + "last_state":"UNKNOWN", + "last_state_change":1547386745.0, + "last_state_type":"HARD", + "name":"test-service-check", + "output":"", + "state":"UNKNOWN", + "state_type":"HARD", + "url":"https://monitoring.bla.tld/monitoring/host/show?host=test.host.tld&service=test-service-check" + }, + "target":"service", + "timet":1547407185.0, + "type":"CUSTOM" +} diff --git a/irc.go b/irc.go index 21e2947..a52cffc 100644 --- a/irc.go +++ b/irc.go @@ -5,6 +5,7 @@ import ( "crypto/x509" "io/ioutil" "strings" + "sync" "time" log "github.com/sirupsen/logrus" @@ -14,6 +15,7 @@ import ( ) var client *girc.Client +var clientLock = &sync.RWMutex{} func ircConnection(config *viper.Viper, channelList []string) { clientConfig := girc.Config{ @@ -77,9 +79,11 @@ func ircConnection(config *viper.Viper, channelList []string) { } } + clientLock.Lock() client = girc.New(clientConfig) client.Handlers.Add(girc.CONNECTED, func(c *girc.Client, e girc.Event) { + clientLock.Unlock() for _, name := range channelList { joinChannel(name) } @@ -90,8 +94,9 @@ func ircConnection(config *viper.Viper, channelList []string) { for { if err := client.Connect(); err != nil { + clientLock.Lock() log.Warnf("Connection to %s terminated: %s", client.Server(), err) - log.Warn("Reconnecting to in 30 seconds...") + log.Warn("Reconnecting in 30 seconds...") time.Sleep(30 * time.Second) } } @@ -105,7 +110,9 @@ func channelReceiver() { log.Debug("Took IRC event out of channel.") joinChannel(elem.Channel) for _, message := range elem.Messages { + clientLock.RLock() client.Cmd.Message(elem.Channel, message) + clientLock.RUnlock() } } } @@ -121,5 +128,7 @@ func joinChannel(newChannel string) { "channel": newChannel, }).Debug("Need to join new channel") + clientLock.RLock() client.Cmd.Join(newChannel) + clientLock.RUnlock() } diff --git a/main.go b/main.go index f1d5658..3e1a755 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,8 @@ func createModuleObject(name string) (input.Module, error) { m = &input.PrometheusModule{} case "simple": m = &input.SimpleModule{} + case "icinga2": + m = &input.Icinga2Module{} default: e = fmt.Errorf("found configuration for unknown module: %q", name) }