Improvements in integration tests (vmware#325)

* Use more meaningful byte counts in test records (count and
  reverseCount have no reason to be the same).
* Generate test data packets automatically using a function, instead of
  having to generate packet data manually.
* Add integration build tag for linters

Signed-off-by: Antonin Bas <abas@vmware.com>

.golangci.yml

@@ -3,6 +3,8 @@ run:
   tests: true
   timeout: 10m
   skip-dirs-use-default: true
+  build-tags:
+    - integration
 
 linters-settings:
   goimports:

pkg/collector/process_test.go

@@ -29,7 +29,7 @@ import (
 
 	"github.com/vmware/go-ipfix/pkg/entities"
 	"github.com/vmware/go-ipfix/pkg/registry"
-	"github.com/vmware/go-ipfix/pkg/test"
+	testcerts "github.com/vmware/go-ipfix/pkg/test/certs"
 )
 
 var validTemplatePacket = []byte{0, 10, 0, 40, 95, 154, 107, 127, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 24, 1, 0, 0, 3, 0, 8, 0, 4, 0, 12, 0, 4, 128, 101, 255, 255, 0, 0, 220, 186}
@@ -376,11 +376,11 @@ func TestTLSCollectingProcess(t *testing.T) {
 	var config *tls.Config
 	go func() {
 		roots := x509.NewCertPool()
-		ok := roots.AppendCertsFromPEM([]byte(test.FakeCACert))
+		ok := roots.AppendCertsFromPEM([]byte(testcerts.FakeCACert))
 		if !ok {
 			t.Error("Failed to parse root certificate")
 		}
-		cert, err := tls.X509KeyPair([]byte(test.FakeClientCert), []byte(test.FakeClientKey))
+		cert, err := tls.X509KeyPair([]byte(testcerts.FakeClientCert), []byte(testcerts.FakeClientKey))
 		if err != nil {
 			t.Error(err)
 		}
@@ -424,7 +424,7 @@ func TestDTLSCollectingProcess(t *testing.T) {
 	collectorAddr, _ := net.ResolveUDPAddr("udp", cp.GetAddress().String())
 	go func() {
 		roots := x509.NewCertPool()
-		ok := roots.AppendCertsFromPEM([]byte(test.FakeCert2))
+		ok := roots.AppendCertsFromPEM([]byte(testcerts.FakeCert2))
 		if !ok {
 			t.Error("Failed to parse root certificate")
 		}
@@ -517,9 +517,9 @@ func getCollectorInput(network string, isEncrypted bool, isIPv6 bool) CollectorI
 				MaxBufferSize: 1024,
 				TemplateTTL:   0,
 				IsEncrypted:   true,
-				CACert:        []byte(test.FakeCACert),
-				ServerCert:    []byte(test.FakeCert),
-				ServerKey:     []byte(test.FakeKey),
+				CACert:        []byte(testcerts.FakeCACert),
+				ServerCert:    []byte(testcerts.FakeCert),
+				ServerKey:     []byte(testcerts.FakeKey),
 			}
 		} else {
 			return CollectorInput{
@@ -542,8 +542,8 @@ func getCollectorInput(network string, isEncrypted bool, isIPv6 bool) CollectorI
 				MaxBufferSize: 1024,
 				TemplateTTL:   0,
 				IsEncrypted:   true,
-				ServerCert:    []byte(test.FakeCert2),
-				ServerKey:     []byte(test.FakeKey2),
+				ServerCert:    []byte(testcerts.FakeCert2),
+				ServerKey:     []byte(testcerts.FakeKey2),
 			}
 		} else {
 			return CollectorInput{

pkg/exporter/msg.go

@@ -0,0 +1,56 @@
+// Copyright 2023 VMware, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exporter
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/vmware/go-ipfix/pkg/entities"
+)
+
+func CreateIPFIXMsg(set entities.Set, obsDomainID uint32, seqNumber uint32, exportTime time.Time) ([]byte, error) {
+	// Create a new message and use it to send the set.
+	msg := entities.NewMessage(false)
+
+	// Check if message is exceeding the limit after adding the set. Include message
+	// header length too.
+	msgLen := entities.MsgHeaderLength + set.GetSetLength()
+	if msgLen > entities.MaxSocketMsgSize {
+		// This is applicable for both TCP and UDP sockets.
+		return nil, fmt.Errorf("message size exceeds max socket buffer size")
+	}
+
+	// Set the fields in the message header.
+	// IPFIX version number is 10.
+	// https://www.iana.org/assignments/ipfix/ipfix.xhtml#ipfix-version-numbers
+	msg.SetVersion(10)
+	msg.SetObsDomainID(obsDomainID)
+	msg.SetMessageLen(uint16(msgLen))
+	msg.SetExportTime(uint32(exportTime.Unix()))
+	msg.SetSequenceNum(seqNumber)
+
+	bytesSlice := make([]byte, msgLen)
+	copy(bytesSlice[:entities.MsgHeaderLength], msg.GetMsgHeader())
+	copy(bytesSlice[entities.MsgHeaderLength:entities.MsgHeaderLength+entities.SetHeaderLen], set.GetHeaderBuffer())
+	index := entities.MsgHeaderLength + entities.SetHeaderLen
+	for _, record := range set.GetRecords() {
+		len := record.GetRecordLength()
+		copy(bytesSlice[index:index+len], record.GetBuffer())
+		index += len
+	}
+
+	return bytesSlice, nil
+}

pkg/exporter/process.go

@@ -285,45 +285,20 @@ func (ep *ExportingProcess) NewTemplateID() uint16 {
 // createAndSendIPFIXMsg takes in a set as input, creates the IPFIX message, and sends it out.
 // TODO: This method will change when we support sending multiple sets.
 func (ep *ExportingProcess) createAndSendIPFIXMsg(set entities.Set) (int, error) {
-	// Create a new message and use it to send the set.
-	msg := entities.NewMessage(false)
-
-	// Check if message is exceeding the limit after adding the set. Include message
-	// header length too.
-	msgLen := entities.MsgHeaderLength + set.GetSetLength()
-	if msgLen > entities.MaxSocketMsgSize {
-		// This is applicable for both TCP and UDP sockets.
-		return 0, fmt.Errorf("message size exceeds max socket buffer size")
-	}
-
-	// Set the fields in the message header.
-	// IPFIX version number is 10.
-	// https://www.iana.org/assignments/ipfix/ipfix.xhtml#ipfix-version-numbers
-	msg.SetVersion(10)
-	msg.SetObsDomainID(ep.obsDomainID)
-	msg.SetMessageLen(uint16(msgLen))
-	msg.SetExportTime(uint32(time.Now().Unix()))
 	if set.GetSetType() == entities.Data {
 		ep.seqNumber = ep.seqNumber + set.GetNumberOfRecords()
 	}
-	msg.SetSequenceNum(ep.seqNumber)
-
-	bytesSlice := make([]byte, msgLen)
-	copy(bytesSlice[:entities.MsgHeaderLength], msg.GetMsgHeader())
-	copy(bytesSlice[entities.MsgHeaderLength:entities.MsgHeaderLength+entities.SetHeaderLen], set.GetHeaderBuffer())
-	index := entities.MsgHeaderLength + entities.SetHeaderLen
-	for _, record := range set.GetRecords() {
-		len := record.GetRecordLength()
-		copy(bytesSlice[index:index+len], record.GetBuffer())
-		index += len
+	bytesSlice, err := CreateIPFIXMsg(set, ep.obsDomainID, ep.seqNumber, time.Now())
+	if err != nil {
+		return 0, err
 	}
 
 	// Send the message on the exporter connection.
 	bytesSent, err := ep.connToCollector.Write(bytesSlice)
 
 	if err != nil {
 		return bytesSent, fmt.Errorf("error when sending message on the connection: %v", err)
-	} else if bytesSent != msgLen {
+	} else if bytesSent != len(bytesSlice) {
 		return bytesSent, fmt.Errorf("could not send the complete message on the connection")
 	}
 

pkg/exporter/process_test.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/vmware/go-ipfix/pkg/entities"
 	"github.com/vmware/go-ipfix/pkg/registry"
-	"github.com/vmware/go-ipfix/pkg/test"
+	testcerts "github.com/vmware/go-ipfix/pkg/test/certs"
 )
 
 func init() {
@@ -396,14 +396,14 @@ func TestExportingProcess_SendingDataRecordToLocalUDPServer(t *testing.T) {
 }
 
 func TestInitExportingProcessWithTLS(t *testing.T) {
-	caCert, caKey, caData, err := test.GenerateCACert()
+	caCert, caKey, caData, err := testcerts.GenerateCACert()
 	require.NoError(t, err, "Error when generating CA cert")
-	clientCertData, clientKeyData, err := test.GenerateClientCert(caCert, caKey)
+	clientCertData, clientKeyData, err := testcerts.GenerateClientCert(caCert, caKey)
 	require.NoError(t, err, "Error when generating client cert")
 
 	testCases := []struct {
 		name              string
-		serverCertOptions []test.CertificateOption
+		serverCertOptions []testcerts.CertificateOption
 		withClientAuth    bool
 		tlsClientConfig   *ExporterTLSClientConfig
 		expectedErr       string
@@ -419,22 +419,22 @@ func TestInitExportingProcessWithTLS(t *testing.T) {
 		},
 		{
 			name:              "IP SAN",
-			serverCertOptions: []test.CertificateOption{test.AddIPAddress(net.ParseIP("127.0.0.1"))},
+			serverCertOptions: []testcerts.CertificateOption{testcerts.AddIPAddress(net.ParseIP("127.0.0.1"))},
 			tlsClientConfig: &ExporterTLSClientConfig{
 				CAData: caData,
 			},
 		},
 		{
 			name:              "name SAN with matching ServerName",
-			serverCertOptions: []test.CertificateOption{test.AddDNSName("foobar")},
+			serverCertOptions: []testcerts.CertificateOption{testcerts.AddDNSName("foobar")},
 			tlsClientConfig: &ExporterTLSClientConfig{
 				CAData:     caData,
 				ServerName: "foobar",
 			},
 		},
 		{
 			name:              "name SAN with mismatching ServerName",
-			serverCertOptions: []test.CertificateOption{test.AddDNSName("foobar")},
+			serverCertOptions: []testcerts.CertificateOption{testcerts.AddDNSName("foobar")},
 			tlsClientConfig: &ExporterTLSClientConfig{
 				CAData:     caData,
 				ServerName: "badname",
@@ -444,7 +444,7 @@ func TestInitExportingProcessWithTLS(t *testing.T) {
 		},
 		{
 			name:              "name SAN without ServerName",
-			serverCertOptions: []test.CertificateOption{test.AddDNSName("foobar")},
+			serverCertOptions: []testcerts.CertificateOption{testcerts.AddDNSName("foobar")},
 			tlsClientConfig: &ExporterTLSClientConfig{
 				CAData: caData,
 			},
@@ -453,7 +453,7 @@ func TestInitExportingProcessWithTLS(t *testing.T) {
 		},
 		{
 			name:              "client auth with no cert",
-			serverCertOptions: []test.CertificateOption{test.AddIPAddress(net.ParseIP("127.0.0.1"))},
+			serverCertOptions: []testcerts.CertificateOption{testcerts.AddIPAddress(net.ParseIP("127.0.0.1"))},
 			withClientAuth:    true,
 			tlsClientConfig: &ExporterTLSClientConfig{
 				CAData: caData,
@@ -462,7 +462,7 @@ func TestInitExportingProcessWithTLS(t *testing.T) {
 		},
 		{
 			name:              "client auth with cert",
-			serverCertOptions: []test.CertificateOption{test.AddIPAddress(net.ParseIP("127.0.0.1"))},
+			serverCertOptions: []testcerts.CertificateOption{testcerts.AddIPAddress(net.ParseIP("127.0.0.1"))},
 			withClientAuth:    true,
 			tlsClientConfig: &ExporterTLSClientConfig{
 				CAData:   caData,
@@ -472,7 +472,7 @@ func TestInitExportingProcessWithTLS(t *testing.T) {
 		},
 		{
 			name:              "client auth and ServerName",
-			serverCertOptions: []test.CertificateOption{test.AddDNSName("foobar")},
+			serverCertOptions: []testcerts.CertificateOption{testcerts.AddDNSName("foobar")},
 			withClientAuth:    true,
 			tlsClientConfig: &ExporterTLSClientConfig{
 				CAData:     caData,
@@ -489,7 +489,7 @@ func TestInitExportingProcessWithTLS(t *testing.T) {
 			// Create local server for testing
 			address, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0")
 			require.NoError(t, err)
-			serverCertData, serverKeyData, err := test.GenerateServerCert(caCert, caKey, tc.serverCertOptions...)
+			serverCertData, serverKeyData, err := testcerts.GenerateServerCert(caCert, caKey, tc.serverCertOptions...)
 			require.NoError(t, err, "Error when generating server cert")
 			serverCert, err := tls.X509KeyPair(serverCertData, serverKeyData)
 			require.NoError(t, err)
@@ -551,7 +551,7 @@ func TestExportingProcessWithTLS(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Got error when resolving tcp address: %v", err)
 	}
-	cer, err := tls.X509KeyPair([]byte(test.FakeCert), []byte(test.FakeKey))
+	cer, err := tls.X509KeyPair([]byte(testcerts.FakeCert), []byte(testcerts.FakeKey))
 	if err != nil {
 		t.Error(err)
 		return
@@ -592,7 +592,7 @@ func TestExportingProcessWithTLS(t *testing.T) {
 		ObservationDomainID: 1,
 		TempRefTimeout:      0,
 		TLSClientConfig: &ExporterTLSClientConfig{
-			CAData:     []byte(test.FakeCACert),
+			CAData:     []byte(testcerts.FakeCACert),
 			ServerName: "127.0.0.1",
 		},
 	}
@@ -638,7 +638,7 @@ func TestExportingProcessWithDTLS(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Got error when resolving udp address: %v", err)
 	}
-	cert, err := tls.X509KeyPair([]byte(test.FakeCert2), []byte(test.FakeKey2))
+	cert, err := tls.X509KeyPair([]byte(testcerts.FakeCert2), []byte(testcerts.FakeKey2))
 	if err != nil {
 		t.Error(err)
 		return
@@ -679,7 +679,7 @@ func TestExportingProcessWithDTLS(t *testing.T) {
 		ObservationDomainID: 1,
 		TempRefTimeout:      0,
 		TLSClientConfig: &ExporterTLSClientConfig{
-			CAData: []byte(test.FakeCert2),
+			CAData: []byte(testcerts.FakeCert2),
 		},
 	}
 	exporter, err := InitExportingProcess(input)

pkg/kafka/producer/kafka_test.go

@@ -22,21 +22,21 @@ import (
 	"github.com/Shopify/sarama"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/vmware/go-ipfix/pkg/test"
+	testcerts "github.com/vmware/go-ipfix/pkg/test/certs"
 )
 
 func TestInitKafkaProducerWithTLS(t *testing.T) {
-	caCertFile := createTmpFileAndWrite(t, test.FakeCACert, "ca-")
-	certFile := createTmpFileAndWrite(t, test.FakeCert, "cert-")
-	keyFile := createTmpFileAndWrite(t, test.FakeKey, "key-")
+	caCertFile := createTmpFileAndWrite(t, testcerts.FakeCACert, "ca-")
+	certFile := createTmpFileAndWrite(t, testcerts.FakeCert, "cert-")
+	keyFile := createTmpFileAndWrite(t, testcerts.FakeKey, "key-")
 	defer closeTmpFile(t, caCertFile)
 	defer closeTmpFile(t, certFile)
 	defer closeTmpFile(t, keyFile)
 
 	serverTLSConfig, err := setupTLSConfig(caCertFile.Name(), certFile.Name(), keyFile.Name(), false)
 	assert.NoError(t, err)
 
-	doListenerTLSTest(t, serverTLSConfig, test.FakeCACert, test.FakeClientCert, test.FakeClientKey)
+	doListenerTLSTest(t, serverTLSConfig, testcerts.FakeCACert, testcerts.FakeClientCert, testcerts.FakeClientKey)
 }
 
 func doListenerTLSTest(t *testing.T, serverTLSConfig *tls.Config, caCert, clientCert, clientKey string) {
@@ -49,9 +49,9 @@ func doListenerTLSTest(t *testing.T, serverTLSConfig *tls.Config, caCert, client
 
 	seedBroker.Returns(new(sarama.MetadataResponse))
 
-	caCertFile := createTmpFileAndWrite(t, test.FakeCACert, "ca-")
-	certFile := createTmpFileAndWrite(t, test.FakeClientCert, "clientCert-")
-	keyFile := createTmpFileAndWrite(t, test.FakeClientKey, "clientKey-")
+	caCertFile := createTmpFileAndWrite(t, testcerts.FakeCACert, "ca-")
+	certFile := createTmpFileAndWrite(t, testcerts.FakeClientCert, "clientCert-")
+	keyFile := createTmpFileAndWrite(t, testcerts.FakeClientKey, "clientKey-")
 	defer closeTmpFile(t, caCertFile)
 	defer closeTmpFile(t, certFile)
 	defer closeTmpFile(t, keyFile)

pkg/test/certs.go → pkg/test/certs/certs.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package test
+package certs
 
 import (
 	"bytes"