Skip to content

Commit

Permalink
send notifications to slack
Browse files Browse the repository at this point in the history
  • Loading branch information
teleivo committed Sep 7, 2021
1 parent f53ccbf commit 560043c
Show file tree
Hide file tree
Showing 6 changed files with 146 additions and 38 deletions.
46 changes: 34 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
[![Release](https://img.shields.io/github/release/teleivo/diskmon.svg)](https://github.com/teleivo/diskmon/releases/latest)
[![Powered By: GoReleaser](https://img.shields.io/badge/powered%20by-goreleaser-green.svg)](https://github.com/goreleaser)

Diskmon will notify you if a disk has reached a configurable size limit.
Diskmon will notify you via [Slack](https://slack.com) if a disk has reached a configurable size limit.

## Design Rationale

Expand All @@ -18,34 +18,56 @@ It provides mount point monitoring and much more!

## Get started

### Using Binary

Build the binary or run directly using
Dowload a [pre-built binary](https://github.com/teleivo/diskmon/releases) or build the binary yourself

```sh
go run main.go -basedir /home
go build -o /usr/local/bin/diskmon
```

### Using Docker
Change the destination to a path of your choice and make sure it can be found
by your shell via the $PATH variable.

Build the image
Or run it directly

```sh
docker build -t dockermon .
go run main.go -basedir <directory> -limit 65
```

Run the image
which will write usage reports to stdout.

### Notifications

Notifications can be sent to
* stdout (by default)
* or Slack using a [Slack App](https://api.slack.com/start/building)

For Slack please follow the Slack documentation on how to create a Slack App Bot.
You can also follow this YouTube tutorial [Golang Tutorial: Build a Slack Bot](https://youtu.be/n-7l-N541u0).

You will then need to pass the Slack Bot User OAuth Token and the channel ID to
the binary via CLI flags.

Prefer passing credentials for example like so

```sh
docker run --volume /home:/home:ro diskmon -basedir /hom
diskmon -basedir <directory> -limit 65 \
-slackToken $SLACK_TOKEN \
-slackChannel $SLACK_CHANNEL
```

so that the credentials are not in your shell history.

## Limitations

The diskmon is not a general purpose disk monitor. It is specifically designed
for the use case we had ([see Design Rationale](#design-rationale))
for the use case we had ([see Design Rationale](#design-rationale)).

If you have prometheus already please use the [node exporter](https://github.com/prometheus/node_exporter).
It provides mount point monitoring and much more!


* Notifications can only be sent to Slack using a [Slack App](https://api.slack.com/start/building)
or to stdout
You can of course adapt the code to send notifications anywhere :smile:
* It will not discover all mount points for you like the [node exporter](https://github.com/prometheus/node_exporter).
You can only provide one directory in which your mount points should be.
For our use case the volumes we wanted to monitor were all in one directory.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,7 @@ go 1.16

require (
github.com/dustin/go-humanize v1.0.0
github.com/pkg/errors v0.9.1 // indirect
github.com/slack-go/slack v0.9.4 // indirect
golang.org/x/sys v0.0.0-20210903071746-97244b99971b
)
11 changes: 11 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,4 +1,15 @@
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/slack-go/slack v0.9.4 h1:C+FC3zLxLxUTQjDy2RZeMHYon005zsCROiZNWVo+opQ=
github.com/slack-go/slack v0.9.4/go.mod h1:wWL//kk0ho+FcQXcBTmEafUI5dz4qz5f4mMk8oIkioQ=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/sys v0.0.0-20210903071746-97244b99971b h1:3Dq0eVHn0uaQJmPO+/aYPI/fRMqdrVDbu7MQcku54gg=
golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
31 changes: 22 additions & 9 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os"
"time"

"github.com/teleivo/diskmon/slack"
"github.com/teleivo/diskmon/usage"
)

Expand All @@ -22,35 +23,47 @@ func main() {
func run(args []string, out io.Writer) error {

flags := flag.NewFlagSet(args[0], flag.ExitOnError)
basedir := flags.String("basedir", "", "statfs syscall information will be printed for each directory (depth 1) in this base directory")
basedir := flags.String("basedir", "", "statfs syscall information will be gathered for each directory (depth 1) in this base directory")
limit := flags.Uint64("limit", 80, "percentage of disk usage at which notification should be sent")
interval := flags.Uint("interval", 60, "interval in minutes at which the disk usage will be checked")
slackToken := flags.String("slackToken", "", "Slack Bot User OAuth Token used to post notifications to Slack")
slackChannel := flags.String("slackChannel", "", "Slack channel ID where notifications are posted to")
err := flags.Parse(args[1:])
if err != nil {
return err
}
if *basedir == "" {
return errors.New("basedir must be provided")
}
if *slackToken != "" && *slackChannel == "" || *slackChannel != "" && *slackToken == "" {
return errors.New("both slackChannel and slackApiToken must be either provided or not")
}

logger := log.New(out, args[0]+" ", log.LUTC)
var notifier usage.Notifier
if *slackToken != "" && *slackChannel != "" {
notifier = slack.New(*slackToken, *slackChannel, logger)
} else {
notifier = usage.WriteNotifier(out)
}

t := time.NewTicker(time.Minute * time.Duration(*interval))
defer t.Stop()

logger := log.New(out, args[0]+" ", log.LUTC)

// check usage once right after starting up
err = usage.Check(*basedir, *limit, logger, out)
err = usage.Check(*basedir, *limit, logger, notifier)
if err != nil {
return err
}
for {
select {
case <-t.C:
// TODO what if there is an error here on reading the basedir?
// I think its a good idea to call ReadDir every time we check
// usage since one could add a volume to a droplet after the
// diskmon has started
usage.Check(*basedir, *limit, logger, out)
err := usage.Check(*basedir, *limit, logger, notifier)
// TODO what if there is an error here on reading the basedir? At
// least log it for now
if err != nil {
log.Println(err.Error())
}
}
}
}
39 changes: 39 additions & 0 deletions slack/slack.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package slack

import (
"log"
"strings"

"github.com/slack-go/slack"
"github.com/teleivo/diskmon/usage"
)

type Notifier struct {
channel string
client *slack.Client
logger *log.Logger
}

func New(token, channel string, logger *log.Logger) *Notifier {
return &Notifier{
channel: channel,
client: slack.New(token),
logger: logger,
}
}

func (n *Notifier) Notify(r usage.Report) error {
var sb strings.Builder
l := strings.Join(r.Limits, "\n")
if l != "" {
sb.WriteString(l)
}
for _, e := range r.Errors {
sb.WriteString(e.Error())
}
_, _, err := n.client.PostMessage(
n.channel, slack.MsgOptionText(sb.String(), false),
)
log.Printf("Posted slack message on channel")
return err
}
55 changes: 38 additions & 17 deletions usage/usage.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,57 @@ import (
"github.com/teleivo/diskmon/fstat"
)

type Notification struct {
type Report struct {
Limits []string
Errors []error
}

func Check(basedir string, limit uint64, logger *log.Logger, out io.Writer) error {
type Notifier interface {
Notify(Report) error
}

type writeNotifier struct {
io.Writer
}

func (wn writeNotifier) Notify(r Report) error {
for _, l := range r.Limits {
wn.Write([]byte(l))
wn.Write([]byte("\n"))
}
for _, e := range r.Errors {
wn.Write([]byte(e.Error()))
wn.Write([]byte("\n"))
}

return nil
}

func WriteNotifier(w io.Writer) Notifier {
return writeNotifier{w}
}

func Check(basedir string, limit uint64, logger *log.Logger, nt Notifier) error {
logger.Print("Checking disk usage")

n, err := checkDiskUsage(basedir, limit)
r, err := checkDiskUsage(basedir, limit)
if err != nil {
return err
}

for _, l := range n.Limits {
out.Write([]byte(l))
out.Write([]byte("\n"))
}
for _, e := range n.Errors {
out.Write([]byte(e.Error()))
out.Write([]byte("\n"))
if len(r.Limits) == 0 && len(r.Errors) == 0 {
logger.Print("No limits exceeded and no errors found")
return nil
}

return nil
return nt.Notify(r)
}

func checkDiskUsage(basedir string, limit uint64) (Notification, error) {
n := Notification{}
func checkDiskUsage(basedir string, limit uint64) (Report, error) {
r := Report{}
files, err := ioutil.ReadDir(basedir)
if err != nil {
return n, fmt.Errorf("error reading basedir: %w", err)
return r, fmt.Errorf("error reading basedir: %w", err)
}

for _, file := range files {
Expand All @@ -50,13 +71,13 @@ func checkDiskUsage(basedir string, limit uint64) (Notification, error) {

fstat, err := fstat.GetFilesystemStat(filepath.Join(basedir, file.Name()))
if err != nil {
n.Errors = append(n.Errors, fmt.Errorf("error getting filesystem stats from %q: %w", file.Name(), err))
r.Errors = append(r.Errors, fmt.Errorf("error getting filesystem stats from %q: %w", file.Name(), err))
continue
}

if fstat.IsExceedingLimit(limit) {
n.Limits = append(n.Limits, fmt.Sprintf("Free/Total %s/%s %q - reached limit of %d%%", humanize.Bytes(fstat.Free()), humanize.Bytes(fstat.Total()), file.Name(), limit))
r.Limits = append(r.Limits, fmt.Sprintf("Free/Total %s/%s %q - reached limit of %d%%", humanize.Bytes(fstat.Free()), humanize.Bytes(fstat.Total()), file.Name(), limit))
}
}
return n, nil
return r, nil
}

0 comments on commit 560043c

Please sign in to comment.