Skip to content

Commit

Permalink
feat: added db-search plugin
Browse files Browse the repository at this point in the history
- added proper logic using goqu plugin for sanitising input
- updated example to make it work
  • Loading branch information
yusufhm committed May 24, 2024
1 parent d5f23a8 commit a7a1d6d
Show file tree
Hide file tree
Showing 15 changed files with 289 additions and 109 deletions.
39 changes: 25 additions & 14 deletions examples/domain-in-db-tables.yml
Original file line number Diff line number Diff line change
@@ -1,32 +1,43 @@
connections:
docker-mariadb:
docker.exec:
container: mariadb
drupal-db:
mysql:
connection: docker-mariadb
username: drupal
host: mariadb
user: drupal
password: drupal
database: drupal

collect:
table-names:
database.query:
connection: drupal-db
query: "SHOW TABLES"
format: list
domain-in-tables:
database.query:
database.db-search:
connection: drupal-db
input: table_names
query: "SELECT domain FROM {{ .Item }} WHERE domain LIKE %.example.com%"
format: map-list
search: "%.example.com%"
id-field: entity_id
# Not specifying any tables might be very time-consuming, depending on
# the database being scanned. Uncomment below to limit to specific tables
# and fields.
# tables:
# block_content__body:
# - body_value
# block_content_revision__body:
# - body_value

analyse:
domain-found-in-tables:
not-empty:
description: Domain found in table
input: domain-in-tables
# We want a result in the following format:
# # Breaches were detected
#
# ### Domain found in table
# -- [Table: block_content__body] [Column: body_value]: Entity IDs: 40,41,48,62
# -- [Table: block_content_revision__body] [Column: body_value]: Entity IDs: 1,40,41,44,45,48,62
breach-format:
type: key-value
key-label: Table
key: ' {{ .Breach.Key }}'
value-label: '[Column: {{ .Breach.ValueLabel }}]'
value: 'Entity IDs: {{ .Breach.Value }}'

output:
stdout:
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ module github.com/salsadigitalauorg/shipshape
go 1.21

require (
github.com/doug-martin/goqu/v9 v9.19.0
github.com/go-sql-driver/mysql v1.6.0
github.com/goccy/go-json v0.10.2
github.com/gocolly/colly v1.2.0
github.com/hashicorp/go-version v1.6.0
Expand Down
15 changes: 15 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
Expand All @@ -14,6 +16,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/doug-martin/goqu/v9 v9.19.0 h1:PD7t1X3tRcUiSdc5TEyOFKujZA5gs3VSA7wxSvBx7qo=
github.com/doug-martin/goqu/v9 v9.19.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ=
github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 h1:aRd8M7HJVZOqn/vhOzrGcQH0lNAMkqMn+pXUYkatmcA=
github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
Expand All @@ -36,6 +41,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
Expand All @@ -50,6 +57,7 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -102,8 +110,11 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/minio/selfupdate v0.4.0 h1:A7t07pN4Ch1tBTIRStW0KhUVyykz+2muCqFsITQeEW8=
github.com/minio/selfupdate v0.4.0/go.mod h1:mcDkzMgq8PRcpCRJo/NlPY7U45O5dfYl2Y0Rg7IustY=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down Expand Up @@ -139,6 +150,8 @@ github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyh
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
Expand All @@ -157,6 +170,8 @@ github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN
go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI=
go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
Expand Down
2 changes: 1 addition & 1 deletion modd.conf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
**/*.go {
prep: go test -v -race @dirmods
# prep: go test -v -race @dirmods
prep: CGO_ENABLED=0 go build -ldflags="-s -w" -o ~/go/bin/shipshape . && ls -lh ~/go/bin/shipshape
prep: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o ~/go/bin/shipshape_linux_arm64 . && ls -lh ~/go/bin/shipshape_linux_arm64
prep: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o ~/go/bin/shipshape_linux_amd64 . && ls -lh ~/go/bin/shipshape_linux_amd64
Expand Down
51 changes: 51 additions & 0 deletions pkg/analyse/notempty.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package analyse

import (
"github.com/salsadigitalauorg/shipshape/pkg/breach"
"github.com/salsadigitalauorg/shipshape/pkg/data"
"github.com/salsadigitalauorg/shipshape/pkg/fact"
"github.com/salsadigitalauorg/shipshape/pkg/result"
log "github.com/sirupsen/logrus"
)

type NotEmpty struct {
// Common fields.
Id string `yaml:"name"`
Description string `yaml:"description"`
InputName string `yaml:"input"`
Severity string `yaml:"severity"`
breach.BreachTemplate `yaml:"breach-format"`
Result result.Result
input fact.Facter
}

//go:generate go run ../../cmd/gen.go analyse-plugin --plugin=NotEmpty --package=analyse

func init() {
Registry["not-empty"] = func(id string) Analyser { return NewNotEmpty(id) }
}

func (p *NotEmpty) PluginName() string {
return "not-empty"
}

func (p *NotEmpty) Analyse() {
log.WithField("input-format", p.input.GetFormat()).Debug("analysing")
switch p.input.GetFormat() {
case fact.FormatMapNestedString:
inputData := data.AsNestedStringMap(p.input.GetData())
log.WithField("inputData", inputData).Debug("analysing")
if len(inputData) == 0 {
return
}
for k, kvs := range inputData {
for subK, v := range kvs {
breach.EvaluateTemplate(p, &breach.KeyValueBreach{
Key: k,
ValueLabel: subK,
Value: v,
})
}
}
}
}
7 changes: 6 additions & 1 deletion pkg/breach/breachtemplate.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,13 @@ import (
)

func EvaluateTemplate(bt BreachTemplater, b Breach) {

t := bt.GetBreachTemplate()

if t.Type == "" {
bt.AddBreach(b)
return
}

rendered := BreachTemplate{
Type: b.GetType(),
ValueLabel: BreachGetValueLabel(b),
Expand Down
1 change: 1 addition & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func ParseConfigData(configData [][]byte) (bool, Config, ConfigV2, error) {
}

if len(cfgV2.Collect) > 0 {
log.WithField("config", cfgV2).Debug("v2-config parsed")
return true, Config{}, cfgV2, nil
}

Expand Down
1 change: 1 addition & 0 deletions pkg/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ var Errors = []error{}

func ParseConfig(raw map[string]map[string]interface{}) {
count := 0
log.WithField("registry", Registry).Debug("available connections")
for name, pluginConf := range raw {
for pluginName, pluginMap := range pluginConf {
f, ok := Registry[pluginName]
Expand Down
4 changes: 4 additions & 0 deletions pkg/connection/dockerexec.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ type DockerExec struct {

//go:generate go run ../../cmd/gen.go connection-plugin --plugin=DockerExec

func init() {
Registry["docker.exec"] = func(n string) Connectioner { return &DockerExec{Name: n} }
}

func (p *DockerExec) PluginName() string {
return "docker.exec"
}
Expand Down
56 changes: 29 additions & 27 deletions pkg/connection/mysql.go
Original file line number Diff line number Diff line change
@@ -1,52 +1,54 @@
package connection

import (
"errors"
"database/sql"
"fmt"

"github.com/salsadigitalauorg/shipshape/pkg/command"
"github.com/doug-martin/goqu/v9"
_ "github.com/doug-martin/goqu/v9/dialect/mysql"
"github.com/go-sql-driver/mysql"
)

type Mysql struct {
// Common fields.
Name string `yaml:"name"`

// Plugin fields.
Connection string `yaml:"connection"`
DbHost string `yaml:"db-host"`
DbPort string `yaml:"db-port"`
DbUser string `yaml:"db-user"`
DbPass string `yaml:"db-pass"`
DbName string `yaml:"db-name"`
Query string `yaml:"query"`
Host string `yaml:"host"`
Port string `yaml:"port"`
User string `yaml:"user"`
Password string `yaml:"password"`
Database string `yaml:"database"`
Db *goqu.Database
}

//go:generate go run ../../cmd/gen.go connection-plugin --plugin=Mysql

func init() {
Registry["mysql"] = func(n string) Connectioner { return &Mysql{Name: n} }
}

func (p *Mysql) PluginName() string {
return "mysql"
}

func (p *Mysql) Run() ([]byte, error) {
cmdArgs := []string{
"-h", p.DbHost,
"-P", p.DbPort,
"-u", p.DbUser,
"-p" + p.DbPass,
p.DbName,
p.Query,
if p.Port == "" {
p.Port = "3306"
}

cn := GetInstance(p.Connection)
if cn != nil {
// We currently only support docker connections.
if cn.PluginName() != "docker.exec" {
return nil, errors.New("unsupported connection")
}
cfg := mysql.NewConfig()
cfg.User = p.User
cfg.Passwd = p.Password
cfg.Net = "tcp"
cfg.Addr = fmt.Sprintf("%s:%s", p.Host, p.Port)
cfg.DBName = p.Database

dockerConn := cn.(*DockerExec)
dockerConn.Command = append([]string{"mysql"}, cmdArgs...)
return dockerConn.Run()
mysqlDb, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
return nil, err
}

return command.ShellCommander("mysql", cmdArgs...).Output()
dialect := goqu.Dialect("mysql")
p.Db = dialect.DB(mysqlDb)
return nil, nil
}
Loading

0 comments on commit a7a1d6d

Please sign in to comment.