Added HTTP_SKIP_CERT_VALIDATION and http implementation to http util (#…

…147)

Co-authored-by: jp <jan-paul.konijn@tno.nl>

.env.example

@@ -16,4 +16,6 @@ LOG_FORMAT: "json"
 
 ENABLE_FINS: false
 MQTT_BROKER: "localhost"
-MQTT_PORT: 1883
+MQTT_PORT: 1883
+
+HTTP_SKIP_CERT_VALIDATION: false

.github/workflows/ci.yml

@@ -45,23 +45,28 @@ jobs:
   test:
     name: Run ci-tests
     runs-on: ubuntu-latest
-    container:
-      image: golang:latest
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22.2' #
       - name: Make repo safe
         run: git config --global --add safe.directory /__w/SOARCA/SOARCA
       - name: Install swaggo
-        run:  go install github.com/swaggo/swag/cmd/swag@latest   
+        run:  go install github.com/swaggo/swag/cmd/swag@latest  
+        timeout-minutes: 12
+      - name: Start docker containers for test
+        run: docker-compose -f "docker/testing/httpbin-test/docker-compose.yml" up -d --build
       - name: Run tests
         run: |
-          apt update
-          apt install openssh-server -y
-          useradd sshtest
-          echo "sshtest:pdKY77qNxpI5MAizirtjCVOcm0KFKIs" | chpasswd
-          service ssh start
+          sudo apt update
+          sudo apt install openssh-server -y
+          sudo useradd sshtest
+          echo "sshtest:pdKY77qNxpI5MAizirtjCVOcm0KFKIs" | sudo chpasswd
+          sudo service ssh start
           make ci-test
 

docker-compose.yaml

@@ -27,6 +27,7 @@ services:
       DB_PASSWORD: "rootpassword"
       PLAYBOOK_API_LOG_LEVEL: trace
       DATABASE: "false"
+      HTTP_SKIP_CERT_VALIDATION: false
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

docker/soarca/docker-compose.yml

@@ -58,6 +58,7 @@ services:
       ENABLE_FINS: true
       MQTT_BROKER: "mosquitto"
       MQTT_PORT: 1883
+      HTTP_SKIP_CERT_VALIDATION: false
     networks:
       - db-net
     ports:

docker/testing/httpbin-test/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '2'
+
+services:
+    httpbin:
+      image: kennethreitz/httpbin
+    proxy:
+      image: fsouza/docker-ssl-proxy
+      environment:
+        DOMAIN: localhost
+        TARGET_HOST: httpbin
+      links:
+        - httpbin
+      ports:
+        - 443:443

internal/controller/controller.go

@@ -51,7 +51,10 @@ func (controller *Controller) NewDecomposer() decomposer.IDecomposer {
 	ssh := new(ssh.SshCapability)
 	capabilities := map[string]capability.ICapability{ssh.GetType(): ssh}
 
+	skip, _ := strconv.ParseBool(utils.GetEnv("HTTP_SKIP_CERT_VALIDATION", "false"))
+
 	httpUtil := new(httpUtil.HttpRequest)
+	httpUtil.SkipCertificateValidation(skip)
 	http := http.New(httpUtil)
 	capabilities[http.GetType()] = http
 

test/integration/capability/http/http_integration_test.go

@@ -2,12 +2,14 @@ package http_integrations_test
 
 import (
 	"fmt"
+	"testing"
+
 	"soarca/internal/capability/http"
 	"soarca/models/cacao"
 	"soarca/models/execution"
 	httpUtil "soarca/utils/http"
-	"testing"
 
+	"github.com/go-playground/assert/v2"
 	"github.com/google/uuid"
 )
 
@@ -26,15 +28,15 @@ func TestHttpConnection(t *testing.T) {
 		Headers: map[string][]string{"accept": {"application/json"}},
 	}
 
-	var variable1 = cacao.Variable{
+	variable1 := cacao.Variable{
 		Type:  "string",
 		Name:  "test_auth",
 		Value: "",
 	}
 
-	var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
-	var playbookId, _ = uuid.Parse("playbook--d09351a2-a075-40c8-8054-0b7c423db83f")
-	var stepId, _ = uuid.Parse("action--81eff59f-d084-4324-9e0a-59e353dbd28f")
+	executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
+	playbookId, _ := uuid.Parse("playbook--d09351a2-a075-40c8-8054-0b7c423db83f")
+	stepId, _ := uuid.Parse("action--81eff59f-d084-4324-9e0a-59e353dbd28f")
 
 	metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()}
 	// But what to do if there is no target and no AuthInfo?
@@ -74,9 +76,9 @@ func TestHttpOAuth2(t *testing.T) {
 		Headers: map[string][]string{"accept": {"application/json"}},
 	}
 
-	var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
-	var playbookId, _ = uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f")
-	var stepId, _ = uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f")
+	executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
+	playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f")
+	stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f")
 	metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()}
 	results, err := httpCapability.Execute(
 		metadata,
@@ -100,7 +102,7 @@ func TestHttpBasicAuth(t *testing.T) {
 
 	target := cacao.AgentTarget{
 		Address: map[cacao.NetAddressType][]string{
-			"url": []string{url},
+			"url": {url},
 		},
 		AuthInfoIdentifier: "d0c7e6a0-f7fe-464e-9935-e6b3443f5b91",
 	}
@@ -117,9 +119,9 @@ func TestHttpBasicAuth(t *testing.T) {
 		Command: "GET / HTTP/1.1",
 		Headers: map[string][]string{"accept": {"application/json"}},
 	}
-	var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
-	var playbookId, _ = uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f")
-	var stepId, _ = uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f")
+	executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
+	playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f")
+	stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f")
 	metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()}
 	results, err := httpCapability.Execute(
 		metadata,
@@ -133,3 +135,53 @@ func TestHttpBasicAuth(t *testing.T) {
 	}
 	fmt.Println(results)
 }
+
+func TestInsecureHTTPConnection(t *testing.T) {
+	httpRequest := httpUtil.HttpRequest{}
+
+	target := cacao.AgentTarget{
+		Address: map[cacao.NetAddressType][]string{
+			"url": {"https://localhost/get"},
+		},
+	}
+	command := cacao.Command{
+		Type:    "http-api",
+		Command: "GET / HTTP/1.1",
+		Headers: map[string][]string{"accept": {"application/json"}},
+	}
+	httpOptions := httpUtil.HttpOptions{
+		Command: &command,
+		Target:  &target,
+	}
+	httpRequest.SkipCertificateValidation(true)
+	response, err := httpRequest.Request(httpOptions)
+	assert.Equal(t, err, nil)
+	t.Log(string(response))
+	if len(response) == 0 {
+		t.Error("empty response")
+	}
+	t.Log(string(response))
+}
+
+func TestInsecureHTTPConnectionWithFailure(t *testing.T) {
+	httpRequest := httpUtil.HttpRequest{}
+
+	target := cacao.AgentTarget{
+		Address: map[cacao.NetAddressType][]string{
+			"url": {"https://localhost/get"},
+		},
+	}
+	command := cacao.Command{
+		Type:    "http-api",
+		Command: "GET / HTTP/1.1",
+		Headers: map[string][]string{"accept": {"application/json"}},
+	}
+	httpOptions := httpUtil.HttpOptions{
+		Command: &command,
+		Target:  &target,
+	}
+
+	response, err := httpRequest.Request(httpOptions)
+	assert.NotEqual(t, err, nil)
+	t.Log(string(response))
+}

test/unittest/utils/http/http_test.go

@@ -34,6 +34,7 @@ type httpBinResponseBody struct {
 }
 
 // Test general http options, we do not check responses body, as these are variable for the general connection tests
+
 func TestHttpGetConnection(t *testing.T) {
 	httpRequest := http.HttpRequest{}
 

utils/http/http.go

@@ -2,6 +2,7 @@ package http
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -35,7 +36,9 @@ type IHttpRequest interface {
 	Request(httpOptions HttpOptions) ([]byte, error)
 }
 
-type HttpRequest struct{}
+type HttpRequest struct {
+	skipCertificateValidation bool
+}
 
 // https://gist.githubusercontent.com/ahmetozer/ffa4cd0b319aff32ea9ed0068c8b81cf/raw/fc8742e6e087451e954bf0da214794a620356a4d/IPv4-IPv6-domain-regex.go
 const (
@@ -44,14 +47,22 @@ const (
 	domainRegex = `^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$`
 )
 
+func (httpRequest *HttpRequest) SkipCertificateValidation(skip bool) {
+	httpRequest.skipCertificateValidation = skip
+}
+
 func (httpRequest *HttpRequest) Request(httpOptions HttpOptions) ([]byte, error) {
 	log = logger.Logger(component, logger.Info, "", logger.Json)
 	request, err := httpOptions.setupRequest()
 	if err != nil {
 		return []byte{}, err
 	}
 
-	client := &http.Client{}
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: httpRequest.skipCertificateValidation},
+	}
+
+	client := &http.Client{Transport: transport}
 	log.Trace(request)
 	response, err := client.Do(request)
 	if err != nil {
@@ -112,6 +123,8 @@ func (httpRequest *HttpOptions) handleResponse(response *http.Response) ([]byte,
 		return []byte{}, err
 	}
 	sc := response.StatusCode
+	log.Trace(fmt.Sprint(sc))
+	log.Trace(string(responseBytes))
 	if sc < 200 || sc > 299 {
 		return []byte{}, errors.New(string(responseBytes))
 	}