Add Blob encoder and parser for moving data un-parsed

The blob encoder and parser are a matching pair and can be used to move
data between protocols or endpoints without implementing support for the
underlying format.

It can also be forwarded securely as well, since the regular skogul
(json) encoder will base64-encode the data. So you can have:

syslog -(udp+blob)-> skogul -(tcp+blob)-> log-server

or:

syslog -(udp+blob)-> skogul -(https+blob)-> skogul -(udp+blob)-> log-server

An other example useful for capturing debugging data:

switch -(udp+gpb) -> (udp+blob) -> skogul
					--> skogul (udp+protobuf)
					`-> disk (file+json)

Or for that matter:

x -(https+unknown format,blob)-> skogul -> disk

The latter here isn't easily captured with tcpdump due to encryption.

In short: While the blob encoder/parser is really simple, it is very
valuable and largely makes the dummy-parser obsolete.

encoder/auto.go

@@ -60,6 +60,12 @@ func init() {
 		Help:     "Encodes the GOB format.",
 		AutoMake: true,
 	})
+	Auto.Add(skogul.Module{
+		Name:     "blob",
+		Alloc:    func() interface{} { return &Blob{} },
+		Help:     "Use data[\"data\"] as the raw message, unaltered. Optionally with a delimiter between metrics. Useful for transparently moving data in conjunction with the blob parser.",
+		AutoMake: true,
+	})
 	Auto.Add(skogul.Module{
 		Name:  "avro",
 		Alloc: func() interface{} { return &AVRO{} },

encoder/blob.go

@@ -0,0 +1,69 @@
+package encoder
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/telenornms/skogul"
+	"sync"
+)
+
+/*
+TODO:
+- handle multiple metrics
+- add separator
+- test cases
+*/
+type Blob struct {
+	Field     string `doc:"Data field to read data from, defaults to 'data'."`
+	Delimiter []byte `doc:"Base64 encoded delimiter. Defaults to no delimiter."`
+	once      sync.Once
+	err       error
+}
+
+func (x *Blob) init() {
+	if x.Field == "" {
+		x.Field = "data"
+	}
+}
+
+func (x *Blob) Encode(c *skogul.Container) ([]byte, error) {
+	x.once.Do(func() {
+		x.init()
+	})
+	b := bytes.Buffer{}
+	for i := 0; i < len(c.Metrics); i++ {
+		if i > 0 && len(x.Delimiter) > 0 {
+			n, err := b.Write(x.Delimiter)
+			if n != len(x.Delimiter) {
+				return nil, fmt.Errorf("unable to write whole delimiter, wrote %d of %d bytes", n, len(x.Delimiter))
+			}
+			if err != nil {
+				return nil, fmt.Errorf("unable to append to encoding buffer: %w", err)
+			}
+		}
+		b2, ok := c.Metrics[i].Data[x.Field].([]byte)
+		if !ok {
+			return nil, fmt.Errorf("field is not a byte array")
+		}
+		n, err := b.Write(b2)
+		if n != len(b2) {
+			return nil, fmt.Errorf("unable to write whole metric, wrote %d of %d bytes", n, len(x.Delimiter))
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to append to encoding buffer: %w", err)
+		}
+	}
+	rb := b.Bytes()[:]
+	return rb, nil
+}
+
+func (x *Blob) EncodeMetric(m *skogul.Metric) ([]byte, error) {
+	x.once.Do(func() {
+		x.init()
+	})
+	b, ok := m.Data[x.Field].([]byte)
+	if !ok {
+		return nil, fmt.Errorf("field is not a byte array")
+	}
+	return b, nil
+}

encoder/blob_test.go

@@ -0,0 +1,77 @@
+/*
+ * skogul, test blob encoder
+ *
+ * Copyright (c) 2019-2020 Telenor Norge AS
+ * Author(s):
+ *  - Kristian Lyngstøl <kly@kly.no>
+ *  - Håkon Solbjørg <hakon.solbjorg@telenor.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+package encoder_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/telenornms/skogul"
+	"github.com/telenornms/skogul/encoder"
+)
+
+// TestBlobEncode performs a shallow test of encoding of a skogul container
+// using blob.
+// Tests single-metric encoding, encoding something that isn't a byte
+// array and multiple (two) metrics.
+func TestBlobEncode(t *testing.T) {
+	c := skogul.Container{}
+	c.Metrics = make([]*skogul.Metric, 1, 1)
+	raw := []byte(`hei faderullan`)
+	raw2 := []byte(`kjell magne bondevik uten mellomnavn`)
+	raw3 := []byte(`hei faderullan:kjell magne bondevik uten mellomnavn`)
+	m := skogul.Metric{}
+	m.Data = make(map[string]interface{})
+	m.Data["data"] = raw
+	m2 := skogul.Metric{}
+	m2.Data = make(map[string]interface{})
+	m2.Data["data"] = raw2
+	c.Metrics[0] = &m
+
+	enc := encoder.Blob{}
+	enc.Delimiter = []byte(`:`)
+	b, err := enc.Encode(&c)
+	if err != nil {
+		t.Errorf("Encoding failed: %s", err)
+	}
+	if bytes.Compare(b, raw) != 0 {
+		t.Errorf("Encoding failed, new and old not the same: %v vs %v", b, raw)
+	}
+	m.Data["data"] = "not a byte array"
+	b, err = enc.Encode(&c)
+	if err == nil {
+		t.Errorf("Encoding failed: %s", err)
+	}
+
+	m.Data["data"] = raw
+	c.Metrics = append(c.Metrics, &m2)
+	b, err = enc.Encode(&c)
+	if err != nil {
+		t.Errorf("Encoding failed: %s", err)
+	}
+	if bytes.Compare(b, raw3) != 0 {
+		t.Errorf("Encoding failed, the two don't compare")
+	}
+}

parser/auto.go

@@ -115,4 +115,11 @@ func init() {
 		Help:     "Parse a prometheus formatted document into a skogul container, one metric per line.",
 		AutoMake: true,
 	})
+	Auto.Add(skogul.Module{
+		Name:     "blob",
+		Aliases:  []string{},
+		Alloc:    func() interface{} { return &Blob{} },
+		Help:     "Store unparsed data as a byte string in a Skogul metric. Can be used to parse arbitrary data from A to B without implementing support for it. See the blob encoder for the oposite end.",
+		AutoMake: true,
+	})
 }

parser/blob.go

@@ -0,0 +1,42 @@
+/*
+ * skogul, blob parser
+ *
+ * Copyright (c) 2023 Telenor Norge AS
+ * Author(s):
+ *  - Kristian Lyngstøl <kly@kly.no>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+package parser
+
+import (
+	"github.com/telenornms/skogul"
+)
+
+type Blob struct{}
+
+// Parse accepts a byte slice of arbitrary data and stores it on
+// data["data"] unprocessed
+func (x Blob) Parse(b []byte) (*skogul.Container, error) {
+	container := skogul.Container{}
+	container.Metrics = make([]*skogul.Metric, 1, 1)
+	m := skogul.Metric{}
+	container.Metrics[0] = &m
+	container.Metrics[0].Data = make(map[string]interface{})
+	container.Metrics[0].Data["data"] = b
+	return &container, nil
+}

parser/blob_test.go

@@ -0,0 +1,54 @@
+/*
+ * skogul, test blob parser
+ *
+ * Copyright (c) 2023 Telenor Norge AS
+ * Author(s):
+ *  - Kristian Lyngstøl <kly@kly.no>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+package parser_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/telenornms/skogul/parser"
+)
+
+// TestJSONParse tests parsing of a simple JSON document to skogul
+// container
+func TestBlobParse(t *testing.T) {
+	b := []byte("kjell magne bondevik uten mellomnavn")
+	x := parser.Blob{}
+	c, err := x.Parse(b)
+	if err != nil {
+		t.Errorf("Blob.Parse(b) failed: %s", err)
+	}
+	if len(c.Metrics) != 1 {
+		t.Errorf("Blob.Parse(b) returned ok, but length is not 1")
+	}
+	b2, ok := c.Metrics[0].Data["data"].([]byte)
+	if !ok {
+		t.Errorf("data[\"data\"] is not a byte slice")
+	}
+
+	if bytes.Compare(b2, b) != 0 {
+		t.Errorf("data[\"data\"] is not %v!", b)
+	}
+
+}