Add support Disabling TLS Certificate Verification for Secure S3 Storage

Signed-off-by: Anisur Rahman <anisur@appscode.com>

apis/constants.go

@@ -78,6 +78,7 @@ const (
 	RepositoryPrefix          = "REPOSITORY_PREFIX"
 	RepositoryEndpoint        = "REPOSITORY_ENDPOINT"
 	RepositoryRegion          = "REPOSITORY_REGION"
+	RepositoryInsecureTLS     = "REPOSITORY_INSECURE_TLS"
 
 	Hostname       = "HOSTNAME"
 	SourceHostname = "SOURCE_HOSTNAME"

crds/stash.appscode.com_backupblueprints.yaml

@@ -1794,6 +1794,8 @@ spec:
                         type: string
                       endpoint:
                         type: string
+                      insecureTLS:
+                        type: boolean
                       prefix:
                         type: string
                       region:

crds/stash.appscode.com_repositories.yaml

@@ -1801,6 +1801,8 @@ spec:
                         type: string
                       endpoint:
                         type: string
+                      insecureTLS:
+                        type: boolean
                       prefix:
                         type: string
                       region:

go.mod

@@ -24,7 +24,7 @@ require (
 	kmodules.xyz/client-go v0.29.3
 	kmodules.xyz/crd-schema-fuzz v0.29.1
 	kmodules.xyz/custom-resources v0.29.0
-	kmodules.xyz/objectstore-api v0.29.0
+	kmodules.xyz/objectstore-api v0.29.1-0.20240205052451-a5cf0aa669f1
 	kmodules.xyz/offshoot-api v0.29.0
 	kmodules.xyz/prober v0.29.0
 	sigs.k8s.io/controller-runtime v0.16.3
@@ -137,3 +137,5 @@ require (
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
+
+replace kmodules.xyz/objectstore-api => github.com/anisurrahman75/objectstore-api v0.0.0-20240202113030-1e50488736e0

go.sum

@@ -12,6 +12,8 @@ github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmy
 github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/anisurrahman75/objectstore-api v0.0.0-20240202113030-1e50488736e0 h1:jakz69uoxTtUQb1Zv9piW5MZdbfrqJEZfhqIRtiu2JI=
+github.com/anisurrahman75/objectstore-api v0.0.0-20240202113030-1e50488736e0/go.mod h1:Kxmv6F7Kd/7EoKX3X2xIzhHT++zlj2qdXLcp/8avUYI=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
 github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs=
@@ -389,8 +391,6 @@ kmodules.xyz/crd-schema-fuzz v0.29.1 h1:zJTlWYOrT5dsVVHW8HGcnR/vaWfxQfNh11QwTtkY
 kmodules.xyz/crd-schema-fuzz v0.29.1/go.mod h1:n708z9YQqLMP2KNLQVgBcRJw1QpSWLvpNCEi+KJDOYE=
 kmodules.xyz/custom-resources v0.29.0 h1:RaDM2+wSVXiwIvLqmkTVYpwoH83AC8wruXe2p2rOZNY=
 kmodules.xyz/custom-resources v0.29.0/go.mod h1:MzZyXtxdg1PDxGk3RTTO1Xv3KiVqZnIonSwmxVbagOY=
-kmodules.xyz/objectstore-api v0.29.0 h1:dK53fQXdoboyW/EyBBAMjykT8u7jstKrM1DS4RJvhEU=
-kmodules.xyz/objectstore-api v0.29.0/go.mod h1:Kxmv6F7Kd/7EoKX3X2xIzhHT++zlj2qdXLcp/8avUYI=
 kmodules.xyz/offshoot-api v0.29.0 h1:GHLhxxT9jU1N8+FvOCCeJNyU5g0duYS46UGrs6AHNLY=
 kmodules.xyz/offshoot-api v0.29.0/go.mod h1:5NxhBblXoDHWStx9HCDJR2KFTwYjEZ7i1Id3jelIunw=
 kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg=

openapi/swagger.json

@@ -12303,6 +12303,9 @@
           "type": "string",
           "default": ""
         },
+        "insecureTLS": {
+          "type": "boolean"
+        },
         "prefix": {
           "type": "string"
         },

pkg/restic/commands.go

@@ -80,6 +80,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error)
 	result := make([]Snapshot, 0)
 	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--quiet", "--no-lock"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	for _, id := range snapshotIDs {
 		args = append(args, id)
@@ -95,6 +96,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error)
 func (w *ResticWrapper) deleteSnapshots(snapshotIDs []string) ([]byte, error) {
 	args := w.appendCacheDirFlag([]interface{}{"forget", "--quiet", "--prune"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	for _, id := range snapshotIDs {
 		args = append(args, id)
@@ -107,6 +109,7 @@ func (w *ResticWrapper) repositoryExist() bool {
 	klog.Infoln("Checking whether the backend repository exist or not....")
 	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--no-lock"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	if _, err := w.run(Command{Name: ResticCMD, Args: args}); err == nil {
 		return true
@@ -122,6 +125,7 @@ func (w *ResticWrapper) initRepository() error {
 
 	args := w.appendCacheDirFlag([]interface{}{"init"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	_, err := w.run(Command{Name: ResticCMD, Args: args})
 	return err
@@ -151,6 +155,7 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCleanupCacheFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
@@ -174,6 +179,7 @@ func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCleanupCacheFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	commands = append(commands, Command{Name: ResticCMD, Args: args})
@@ -248,6 +254,7 @@ func (w *ResticWrapper) tryCleanup(retentionPolicy v1alpha1.RetentionPolicy, hos
 	if len(args) > 1 {
 		args = w.appendCacheDirFlag(args)
 		args = w.appendCaCertFlag(args)
+		args = w.appendInsecureTLSFlag(args)
 		args = w.appendMaxConnectionsFlag(args)
 
 		return w.run(Command{Name: ResticCMD, Args: args})
@@ -294,6 +301,7 @@ func (w *ResticWrapper) restore(params restoreParams) ([]byte, error) {
 	}
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
@@ -325,6 +333,7 @@ func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) {
 
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	// first add restic command, then add StdoutPipeCommands
@@ -339,6 +348,7 @@ func (w *ResticWrapper) check() ([]byte, error) {
 	klog.Infoln("Checking integrity of repository")
 	args := w.appendCacheDirFlag([]interface{}{"check", "--no-lock"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
@@ -353,6 +363,7 @@ func (w *ResticWrapper) stats(snapshotID string) ([]byte, error) {
 	args = w.appendMaxConnectionsFlag(args)
 	args = append(args, "--quiet", "--json", "--mode", "raw-data", "--no-lock")
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -362,6 +373,7 @@ func (w *ResticWrapper) unlock() ([]byte, error) {
 	args := w.appendCacheDirFlag([]interface{}{"unlock", "--remove-all"})
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -515,6 +527,7 @@ func (w *ResticWrapper) addKey(params keyParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -527,6 +540,7 @@ func (w *ResticWrapper) listKey() ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -543,6 +557,7 @@ func (w *ResticWrapper) updateKey(params keyParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -555,6 +570,14 @@ func (w *ResticWrapper) removeKey(params keyParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
+
+func (w *ResticWrapper) appendInsecureTLSFlag(args []interface{}) []interface{} {
+	if w.config.InsecureTLS {
+		return append(args, "--insecure-tls")
+	}
+	return args
+}

pkg/restic/config.go

@@ -87,6 +87,7 @@ type SetupOptions struct {
 	CacertFile     string
 	ScratchDir     string
 	EnableCache    bool
+	InsecureTLS    bool
 	MaxConnections int64
 	StorageSecret  *core.Secret
 	Nice           *ofst.NiceSettings