uploader: add mTLS support for connection to clickhouse (#132)

README.md

@@ -31,11 +31,11 @@ make
 2. Create tables.
 
 ```sql
-CREATE TABLE graphite ( 
-  Path String,  
-  Value Float64,  
-  Time UInt32,  
-  Date Date,  
+CREATE TABLE graphite (
+  Path String,
+  Value Float64,
+  Time UInt32,
+  Date Date,
   Timestamp UInt32
 ) ENGINE = GraphiteMergeTree('graphite_rollup')
 PARTITION BY toYYYYMM(Date)
@@ -135,7 +135,7 @@ url = "http://localhost:8123/"
 compress-data = true
 timeout = "1m0s"
 # save zero value to Timestamp column (for point and posts-reverse tables)
-zero-timestamp = false 
+zero-timestamp = false
 
 [upload.graphite_index]
 type = "index"
@@ -181,6 +181,23 @@ disable-daily-index = false
 #     "a.b.c.d",  # all tags (but __name__) will be ignored for metrics like a.b.c.d?tagName1=tagValue1&tagName2=tagValue2...
 #     "*",  # all tags (but __name__) will be ignored for all metrics; this is the only special case with wildcards
 # ]
+#
+# It is possible to connect to clickhouse with OpenSSL certificates (mTLS) like below:
+# [upload.graphite]
+# type = "points"
+# table = "graphite"
+# threads = 1
+# compress-data = true
+# zero-timestamp = false
+# timeout = "1m0s"
+# url = "https://localhost:8443/" # url is https
+# [upload.graphite.tls]
+# ca-cert = [ "<path/to/rootCA.crt>", "<path/to/other/rootCA.crt>" ]
+# server-name = "<server-name>"
+# insecure-skip-verify = false # if true, server certificates will not be validated
+# [[upload.graphite.tls.certificates]]
+# key = "<path/to/client.key>"
+# cert = "<path/to/client.crt>"
 
 [udp]
 listen = ":2003"
@@ -239,7 +256,7 @@ concat = "."
 # /debug/receive/grpc/dropped/
 # /debug/receive/prometheus/dropped/
 # /debug/receive/telegraf_http_json/dropped/
-[pprof] 
+[pprof]
 listen = "localhost:7007"
 enabled = false
 

cmd/e2e-test/carbon_clickhouse.go

@@ -14,14 +14,15 @@ import (
 type CarbonClickhouse struct {
 	Binary    string `toml:"binary"`
 	ConfigTpl string `toml:"template"`
+	TestDir   string `toml:"-"`
 
 	storeDir   string    `toml:"-"`
 	configFile string    `toml:"-"`
 	address    string    `toml:"-"`
 	cmd        *exec.Cmd `toml:"-"`
 }
 
-func (c *CarbonClickhouse) Start(clickhouseURL string) error {
+func (c *CarbonClickhouse) Start(clickhouseURL string, clickhouseTLSURL string) error {
 	if c.cmd != nil {
 		return fmt.Errorf("carbon-clickhouse already started")
 	}
@@ -52,13 +53,17 @@ func (c *CarbonClickhouse) Start(clickhouseURL string) error {
 		return err
 	}
 	param := struct {
-		CLICKHOUSE_URL string
-		CCH_STORE_DIR  string
-		CCH_ADDR       string
+		CLICKHOUSE_URL     string
+		CLICKHOUSE_TLS_URL string
+		CCH_STORE_DIR      string
+		CCH_ADDR           string
+		TEST_DIR           string
 	}{
-		CLICKHOUSE_URL: clickhouseURL,
-		CCH_STORE_DIR:  c.storeDir,
-		CCH_ADDR:       c.address,
+		CLICKHOUSE_URL:     clickhouseURL,
+		CLICKHOUSE_TLS_URL: clickhouseTLSURL,
+		CCH_STORE_DIR:      c.storeDir,
+		CCH_ADDR:           c.address,
+		TEST_DIR:           c.TestDir,
 	}
 
 	c.configFile = path.Join(c.storeDir, "carbon-clickhouse.conf")

cmd/e2e-test/clickhouse.go

@@ -10,7 +10,6 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/msaf1980/go-stringutils"
 )
@@ -20,16 +19,19 @@ var ClickhouseOldImage = "yandex/clickhouse-server"
 var ClickhouseDefaultImage = "clickhouse/clickhouse-server"
 
 type Clickhouse struct {
-	Version string `toml:"version"`
-	Dir     string `toml:"dir"`
+	Version    string `toml:"version"`
+	Dir        string `toml:"dir"`
+	TLSEnabled bool   `toml:"tls"`
 
 	DockerImage string `toml:"image"`
 
 	TZ string `toml:"tz"` // override timezone
 
-	httpAddress string `toml:"-"`
-	url         string `toml:"-"`
-	container   string `toml:"-"`
+	httpAddress  string `toml:"-"`
+	httpsAddress string `toml:"-"`
+	url          string `toml:"-"`
+	tlsurl       string `toml:"-"`
+	container    string `toml:"-"`
 }
 
 func (c *Clickhouse) CheckConfig(rootDir string) error {
@@ -71,6 +73,7 @@ func (c *Clickhouse) Start() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	port := strings.Split(c.httpAddress, ":")[1]
 	c.url = "http://" + c.httpAddress
 
 	c.container = ClickhouseContainerName
@@ -80,13 +83,27 @@ func (c *Clickhouse) Start() (string, error) {
 	chStart := []string{"run", "-d",
 		"--name", c.container,
 		"--ulimit", "nofile=262144:262144",
-		"-p", c.httpAddress + ":8123",
+		"-p", port + ":8123",
 		// "-e", "TZ=" + tz, // workaround for TZ=":/etc/localtime"
 		"-v", c.Dir + "/config.xml:/etc/clickhouse-server/config.xml",
 		"-v", c.Dir + "/users.xml:/etc/clickhouse-server/users.xml",
 		"-v", c.Dir + "/rollup.xml:/etc/clickhouse-server/config.d/rollup.xml",
 		"-v", c.Dir + "/init.sql:/docker-entrypoint-initdb.d/init.sql",
 	}
+	if c.TLSEnabled {
+		c.httpsAddress, err = getFreeTCPPort("")
+		if err != nil {
+			return "", err
+		}
+		port = strings.Split(c.httpsAddress, ":")[1]
+		c.tlsurl = "https://" + c.httpsAddress
+		chStart = append(chStart,
+			"-v", c.Dir+"/server.crt:/etc/clickhouse-server/server.crt",
+			"-v", c.Dir+"/server.key:/etc/clickhouse-server/server.key",
+			"-v", c.Dir+"/rootCA.crt:/etc/clickhouse-server/rootCA.crt",
+			"-p", port+":8443",
+		)
+	}
 	if c.TZ != "" {
 		chStart = append(chStart, "-e", "TZ="+c.TZ)
 	}
@@ -136,6 +153,10 @@ func (c *Clickhouse) URL() string {
 	return c.url
 }
 
+func (c *Clickhouse) TLSURL() string {
+	return c.tlsurl
+}
+
 func (c *Clickhouse) Container() string {
 	return c.container
 }
@@ -151,10 +172,7 @@ func (c *Clickhouse) Query(sql string) (string, error) {
 		return "", err
 	}
 
-	httpClient := http.Client{
-		Timeout: time.Minute,
-	}
-	resp, err := httpClient.Do(request)
+	resp, err := http.DefaultClient.Do(request)
 	if err != nil {
 		return "", err
 	}
@@ -172,7 +190,7 @@ func (c *Clickhouse) Alive() bool {
 	if len(c.container) == 0 {
 		return false
 	}
-	req, err := http.DefaultClient.Get(c.url)
+	req, err := http.DefaultClient.Get(c.URL())
 	if err != nil {
 		return false
 	}

cmd/e2e-test/e2etesting.go

@@ -9,12 +9,14 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/BurntSushi/toml"
-	"github.com/lomik/carbon-clickhouse/helper/tests"
 	"go.uber.org/zap"
+
+	"github.com/lomik/carbon-clickhouse/helper/tests"
 )
 
 type InputType int
@@ -71,6 +73,10 @@ type TestSchema struct {
 	chVersions map[string]bool `toml:"-"`
 }
 
+func (schema *TestSchema) HasTLSSettings() bool {
+	return strings.Contains(schema.dir, "tls")
+}
+
 func getFreeTCPPort(name string) (string, error) {
 	if len(name) == 0 {
 		name = "127.0.0.1:0"
@@ -104,14 +110,16 @@ func sendPlain(network, address string, input []string) error {
 	}
 }
 
-func verifyOut(chURL string, verify Verify) []string {
+func verifyOut(ch *Clickhouse, verify Verify) []string {
+	chURL := ch.URL()
 	var errs []string
 
 	q := []byte(verify.Query)
 	req, err := http.NewRequest("POST", chURL, bytes.NewBuffer(q))
-
-	client := &http.Client{Timeout: time.Second * 5}
-	resp, err := client.Do(req)
+	if err != nil {
+		return []string{err.Error()}
+	}
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return []string{err.Error()}
 	}
@@ -155,14 +163,27 @@ func testCarbonClickhouse(
 
 	err := clickhouse.CheckConfig(rootDir)
 	if err != nil {
-		testSuccess = false
-		return
+		return false
+	}
+	absoluteDir, err := filepath.Abs(test.dir)
+	if err != nil {
+		return false
 	}
-
 	cch := CarbonClickhouse{
 		ConfigTpl: testDir + "/" + test.ConfigTpl,
+		TestDir:   absoluteDir,
+	}
+	var (
+		tlsurl string
+	)
+	if test.HasTLSSettings() {
+		tlsurl = clickhouse.TLSURL()
+		if tlsurl == "" {
+			logger.Error("test has tls settings but there is no clickhouse tls url")
+			return false
+		}
 	}
-	err = cch.Start(clickhouse.URL())
+	err = cch.Start(clickhouse.URL(), tlsurl)
 	if err != nil {
 		logger.Error("starting carbon-clickhouse",
 			zap.String("config", test.name),
@@ -210,7 +231,7 @@ func testCarbonClickhouse(
 			verifyFailed := 0
 			time.Sleep(10 * time.Second)
 			for _, verify := range test.Verify {
-				if errs := verifyOut(clickhouse.URL(), verify); len(errs) > 0 {
+				if errs := verifyOut(clickhouse, verify); len(errs) > 0 {
 					testSuccess = false
 					verifyFailed++
 					for _, e := range errs {

cmd/e2e-test/main.go

@@ -133,8 +133,7 @@ func main() {
 
 	failed := 0
 	total := 0
-	for chVersion := range chVersions {
-		ch := chVersions[chVersion]
+	for chVersion, ch := range chVersions {
 		if exist, out := containerExist(ClickhouseContainerName); exist {
 			logger.Error("clickhouse already exist",
 				zap.String("container", ClickhouseContainerName),