Merge pull request #9 from Matrix86/json_filter

new: added basic json filter

Makefile

@@ -13,6 +13,10 @@ test-coverage:
 	@go test -short -coverprofile cover.out -covermode=atomic ${PKG_LIST}
 	@cat cover.out >> coverage.txt
 
+test-coverage-html:
+	@go test -short -coverprofile cover.out -covermode=atomic ${PKG_LIST}
+	@go tool cover -html=cover.out
+
 lint:
 	@golint -set_exit_status ${PKG_LIST}
 

filters/json.go

@@ -0,0 +1,100 @@
+package filters
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/Matrix86/driplane/data"
+
+	"github.com/antchfx/jsonquery"
+	"github.com/evilsocket/islazy/log"
+	"github.com/evilsocket/islazy/str"
+)
+
+// JSON is a filter to parse the JSON format
+type JSON struct {
+	Base
+
+	selector string
+	target   string
+
+	params map[string]string
+}
+
+// NewJSONFilter is the registered method to instantiate a JSONFilter
+func NewJSONFilter(p map[string]string) (Filter, error) {
+	f := &JSON{
+		params:   p,
+		target:   "main",
+		selector: "",
+	}
+	f.cbFilter = f.DoFilter
+
+	if v, ok := f.params["selector"]; ok {
+		f.selector = v
+	}
+
+	if f.selector == "" {
+		return nil, errors.New("no selector specified for JSON filter")
+	}
+	if v, ok := f.params["target"]; ok {
+		f.target = v
+	}
+
+	return f, nil
+}
+
+// DoFilter is the mandatory method used to "filter" the input data.Message
+func (f *JSON) DoFilter(msg *data.Message) (bool, error) {
+	//var err error
+	var text string
+
+	if v, ok := msg.GetTarget(f.target).(string); ok {
+		text = str.Trim(v)
+	} else if v, ok := msg.GetTarget(f.target).([]byte); ok {
+		text = string(v)
+	} else {
+		// ERROR this filter can't be used with different types
+		return false, fmt.Errorf("received data is not a string")
+	}
+
+	if len(text) > 0 {
+		var jsonData string
+
+		if text[0] == '{' {
+			// json text
+			jsonData = str.Trim(text)
+		} else {
+			log.Error("'%v' is not a json document", text)
+			return false, nil
+		}
+
+		if doc, err := jsonquery.Parse(strings.NewReader(jsonData)); err == nil {
+			atLeastOne := false
+			for _, node := range jsonquery.Find(doc, f.selector) {
+				atLeastOne = true
+				clone := msg.Clone()
+				clone.SetMessage(node.Value())
+				f.Propagate(clone)
+			}
+
+			return atLeastOne, nil
+
+		} else {
+			log.Debug("'%v' could not be parsed as JSON: %v", text, err)
+			return false, nil
+		}
+
+	}
+
+	return false, nil
+}
+
+// OnEvent is called when an event occurs
+func (f *JSON) OnEvent(event *data.Event) {}
+
+// Set the name of the filter
+func init() {
+	register("json", NewJSONFilter)
+}

filters/json_test.go

@@ -0,0 +1,244 @@
+package filters
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/Matrix86/driplane/data"
+
+	"github.com/asaskevich/EventBus"
+)
+
+const jsonTest = `{"top": { "inside": "value"}}`
+
+func TestNewJSONFilter(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{"none": "none", "selector": "#selector"})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		if e.target != "main" {
+			t.Errorf("target should be 'main' if not specified")
+		}
+		if e.selector != "#selector" {
+			t.Errorf("selector should be '#selector'")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestNewJSONFilterParams(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "#selector",
+		"target":   "othertarget",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		if e.target != "othertarget" {
+			t.Errorf("target should be 'othertarget'")
+		}
+		if e.selector != "#selector" {
+			t.Errorf("selector should be '#selector'")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestNewJSONWithoutFilterParams(t *testing.T) {
+	_, err := NewJSONFilter(map[string]string{})
+	if err == nil {
+		t.Errorf("The test should return an error")
+	}
+}
+
+func TestJSON_DoFilter(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "top/inside",
+		"target":   "main",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		fb := NewFakeBus()
+		filter.setBus(EventBus.Bus(fb))
+
+		msg := jsonTest
+		m := data.NewMessage(msg)
+		_, err := e.DoFilter(m)
+		if err != nil {
+			t.Errorf("DoFilter returned an error '%s'", err)
+		}
+		if m.GetMessage() != msg {
+			t.Errorf("the message has been altered by the filter")
+		}
+		if len(fb.Collected) == 0 || fb.Collected[0].GetMessage() != "value" {
+			t.Errorf("tags have not been extracted correctly")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestJSON_DoFilterOnByte(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "top/inside",
+		"target":   "main",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		fb := NewFakeBus()
+		filter.setBus(EventBus.Bus(fb))
+
+		msg := []byte(jsonTest)
+		m := data.NewMessage(msg)
+		_, err := e.DoFilter(m)
+		if err != nil {
+			t.Errorf("DoFilter returned an error '%s'", err)
+		}
+		if !bytes.Equal(m.GetMessage().([]byte), msg) {
+			t.Errorf("the message has been altered by the filter")
+		}
+		if len(fb.Collected) == 0 || fb.Collected[0].GetMessage() != "value" {
+			t.Errorf("tags have not been extracted correctly")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestJSON_DoFilterOnWrongType(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "top/inside",
+		"target":   "main",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		fb := NewFakeBus()
+		filter.setBus(EventBus.Bus(fb))
+
+		msg := 2
+		m := data.NewMessage(msg)
+		x, err := e.DoFilter(m)
+		if err == nil {
+			t.Errorf("DoFilter should return an error")
+		}
+		if x {
+			t.Errorf("DoFilter should return false")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestJSON_DoFilterIfNotJSON(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "top/inside",
+		"target":   "main",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		fb := NewFakeBus()
+		filter.setBus(EventBus.Bus(fb))
+
+		msg := "not a JSON"
+		m := data.NewMessage(msg)
+		x, err := e.DoFilter(m)
+		if err != nil {
+			t.Errorf("DoFilter should return nil")
+		}
+		if x {
+			t.Errorf("DoFilter should return false")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestJSON_DoFilterOnWrongSelector(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "top/ops",
+		"target":   "main",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		fb := NewFakeBus()
+		filter.setBus(EventBus.Bus(fb))
+
+		msg := jsonTest
+		m := data.NewMessage(msg)
+		x, err := e.DoFilter(m)
+		if err != nil {
+			t.Errorf("DoFilter should return nil")
+		}
+		if x {
+			t.Errorf("DoFilter should return false")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestJSON_DoFilterOnEmptyJSON(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "top/ops",
+		"target":   "main",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		fb := NewFakeBus()
+		filter.setBus(EventBus.Bus(fb))
+
+		msg := ""
+		m := data.NewMessage(msg)
+		x, err := e.DoFilter(m)
+		if err != nil {
+			t.Errorf("DoFilter should return nil")
+		}
+		if x {
+			t.Errorf("DoFilter should return false")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}
+
+func TestJSON_DoFilterOnBadJSON(t *testing.T) {
+	filter, err := NewJSONFilter(map[string]string{
+		"selector": "top/ops",
+		"target":   "main",
+	})
+	if err != nil {
+		t.Errorf("constructor returned '%s'", err)
+	}
+	if e, ok := filter.(*JSON); ok {
+		fb := NewFakeBus()
+		filter.setBus(EventBus.Bus(fb))
+
+		msg := jsonTest[:len(jsonTest)-1]
+		m := data.NewMessage(msg)
+		x, err := e.DoFilter(m)
+		if err != nil {
+			t.Errorf("DoFilter should return nil")
+		}
+		if x {
+			t.Errorf("DoFilter should return false")
+		}
+	} else {
+		t.Errorf("cannot cast to proper Filter...")
+	}
+}

go.mod

@@ -26,6 +26,8 @@ require (
 )
 
 require (
+	github.com/antchfx/jsonquery v1.3.3 // indirect
+	github.com/antchfx/xpath v1.2.5 // indirect
 	github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 // indirect
 	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect

go.sum

@@ -42,6 +42,11 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/antchfx/jsonquery v1.3.3 h1:zjZpbnZhYng3uOAbIfdNq81A9mMEeuDJeYIpeKpZ4es=
+github.com/antchfx/jsonquery v1.3.3/go.mod h1:1JG4DqRlRCHgVYDPY1ioYFAGSXGfWHzNgrbiGQHsWck=
+github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
+github.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY=
+github.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=