feat: use aws client from duty and set proper endpoint on each clients (

#2078)

* feat: use aws client from duty and set proper endpoint on each clients

* feat: use artifacts for smb & SFTP connection

* chore: Update go.mod

* chore: bump duty & artifacts

* chore: bump golang version in Docker image

* fix: s3 test and bump duty

* chore: set s3 pager size

* chore: set limit on aws max objects

---------

Co-authored-by: Moshe Immerman <moshe@flanksource.com>

build/full/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:6d71b7c3f884e7b9552bffa852d938315ecca843dcc75a86ee7000567da0923d AS builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:f020456572fc292e9627b3fb435c6de5dfb8020fbcef1fd7b65dd092c0ac56bb AS builder
 WORKDIR /app
 
 ARG TARGETOS

build/minimal/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:6d71b7c3f884e7b9552bffa852d938315ecca843dcc75a86ee7000567da0923d AS builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:f020456572fc292e9627b3fb435c6de5dfb8020fbcef1fd7b65dd092c0ac56bb AS builder
 WORKDIR /app
 
 ARG TARGETOS

checks/aws_config.go

@@ -4,9 +4,7 @@ package checks
 
 import (
 	"github.com/aws/aws-sdk-go-v2/service/configservice"
-	awsUtil "github.com/flanksource/artifacts/clients/aws"
 	"github.com/flanksource/canary-checker/api/context"
-	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
 	"github.com/flanksource/duty/connection"
@@ -30,8 +28,7 @@ func (c *AwsConfigChecker) Type() string {
 	return "awsconfig"
 }
 
-func (c *AwsConfigChecker) Check(ctx *context.Context, extConfig external.Check) pkg.Results {
-	check := extConfig.(v1.AwsConfigCheck)
+func (c *AwsConfigChecker) Check(ctx *context.Context, check v1.AwsConfigCheck) pkg.Results {
 	result := pkg.Success(check, ctx.Canary)
 	var results pkg.Results
 	results = append(results, result)
@@ -44,12 +41,17 @@ func (c *AwsConfigChecker) Check(ctx *context.Context, extConfig external.Check)
 		}
 	}
 
-	cfg, err := awsUtil.NewSession(ctx.Context, *check.AWSConnection)
+	cfg, err := check.AWSConnection.Client(ctx.Context)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
 
-	client := configservice.NewFromConfig(*cfg)
+	client := configservice.NewFromConfig(cfg, func(o *configservice.Options) {
+		if check.AWSConnection.Endpoint != "" {
+			o.BaseEndpoint = &check.AWSConnection.Endpoint
+		}
+	})
+
 	if check.AggregatorName != nil {
 		output, err := client.SelectAggregateResourceConfig(ctx, &configservice.SelectAggregateResourceConfigInput{
 			ConfigurationAggregatorName: check.AggregatorName,

checks/aws_config_rule.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/configservice"
 	"github.com/aws/aws-sdk-go-v2/service/configservice/types"
-	awsUtil "github.com/flanksource/artifacts/clients/aws"
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
@@ -45,15 +44,16 @@ func (c *AwsConfigRuleChecker) Check(ctx *context.Context, extConfig external.Ch
 		return results.Failf("failed to populate aws connection: %v", err)
 	}
 
-	cfg, err := awsUtil.NewSession(ctx.Context, *check.AWSConnection)
+	cfg, err := check.AWSConnection.Client(ctx.Context)
 	if err != nil {
 		return results.Failf("failed to create a session: %v", err)
 	}
 
-	client := configservice.NewFromConfig(*cfg)
-	if err != nil {
-		return results.Failf("failed to describe compliance rules: %v", err)
-	}
+	client := configservice.NewFromConfig(cfg, func(o *configservice.Options) {
+		if check.AWSConnection.Endpoint != "" {
+			o.BaseEndpoint = &check.AWSConnection.Endpoint
+		}
+	})
 
 	var complianceTypes = []types.ComplianceType{}
 	for _, i := range check.ComplianceTypes {

checks/cloudwatch.go

@@ -7,9 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
-	awsUtil "github.com/flanksource/artifacts/clients/aws"
 	"github.com/flanksource/canary-checker/api/context"
-	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
 )
@@ -32,8 +30,7 @@ func (c *CloudWatchChecker) Type() string {
 	return "cloudwatch"
 }
 
-func (c *CloudWatchChecker) Check(ctx *context.Context, extConfig external.Check) pkg.Results {
-	check := extConfig.(v1.CloudWatchCheck)
+func (c *CloudWatchChecker) Check(ctx *context.Context, check v1.CloudWatchCheck) pkg.Results {
 	result := pkg.Success(check, ctx.Canary)
 	var results pkg.Results
 	results = append(results, result)
@@ -42,11 +39,17 @@ func (c *CloudWatchChecker) Check(ctx *context.Context, extConfig external.Check
 		return results.Failf("failed to populate aws connection: %v", err)
 	}
 
-	cfg, err := awsUtil.NewSession(ctx.Context, check.AWSConnection)
+	cfg, err := check.AWSConnection.Client(ctx.Context)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
-	client := cloudwatch.NewFromConfig(*cfg)
+
+	client := cloudwatch.NewFromConfig(cfg, func(o *cloudwatch.Options) {
+		if check.AWSConnection.Endpoint != "" {
+			o.BaseEndpoint = &check.AWSConnection.Endpoint
+		}
+	})
+
 	maxRecords := int32(100)
 	alarms, err := client.DescribeAlarms(ctx, &cloudwatch.DescribeAlarmsInput{
 		AlarmNames:      check.CloudWatchFilter.Alarms,

checks/folder.go

@@ -6,9 +6,10 @@ import (
 	"os"
 	"strings"
 
+	"github.com/flanksource/artifacts"
+	artifactFS "github.com/flanksource/artifacts/fs"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"github.com/flanksource/artifacts"
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
@@ -111,18 +112,18 @@ func checkLocalFolder(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	return results
 }
 
-func genericFolderCheck(dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func genericFolderCheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	return _genericFolderCheck(true, dirFS, path, recursive, filter)
 }
 
 // genericFolderCheckWithoutPrecheck is used for those filesystems that do not support fetching the stat of a directory.
 // Eg: s3, gcs.
 // It will not pre check whether the given path is a directory.
-func genericFolderCheckWithoutPrecheck(dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func genericFolderCheckWithoutPrecheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	return _genericFolderCheck(false, dirFS, path, recursive, filter)
 }
 
-func _genericFolderCheck(supportsDirStat bool, dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func _genericFolderCheck(supportsDirStat bool, dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	result := FolderCheck{}
 	_filter, err := filter.New()
 	if err != nil {
@@ -170,7 +171,7 @@ func _genericFolderCheck(supportsDirStat bool, dirFS artifacts.Filesystem, path
 
 // getFolderContents walks the folder and returns all files.
 // Also supports recursively fetching contents
-func getFolderContents(dirFs artifacts.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) {
+func getFolderContents(dirFs artifactFS.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) {
 	files, err := dirFs.ReadDir(path)
 	if err != nil {
 		return nil, err

checks/folder_s3.go

@@ -8,10 +8,10 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/flanksource/artifacts"
+	artifactFS "github.com/flanksource/artifacts/fs"
 	"github.com/flanksource/canary-checker/api/context"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
-	"github.com/flanksource/duty/models"
 )
 
 type S3 struct {
@@ -31,26 +31,22 @@ func CheckS3Bucket(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	var bucket string
 	bucket, check.Path = parseS3Path(check.Path)
 
-	connection, err := ctx.HydrateConnectionByURL(check.AWSConnection.ConnectionName)
-	if err != nil {
-		return results.Failf("failed to populate AWS connection: %v", err)
-	} else if connection == nil {
-		connection = &models.Connection{Type: models.ConnectionTypeS3}
-		if check.S3Connection.Bucket == "" {
-			check.S3Connection.Bucket = bucket
-		}
-
-		connection, err = connection.Merge(ctx, check.S3Connection)
-		if err != nil {
-			return results.Failf("failed to populate AWS connection: %v", err)
-		}
+	if err := check.S3Connection.Populate(ctx); err != nil {
+		return results.ErrorMessage(err)
 	}
 
-	fs, err := artifacts.GetFSForConnection(ctx.Context, *connection)
+	conn := check.S3Connection.ToModel()
+	conn.SetProperty("bucket", bucket)
+
+	fs, err := artifacts.GetFSForConnection(ctx.Context, conn)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
 
+	if limitFS, ok := fs.(artifactFS.ListItemLimiter); ok {
+		limitFS.SetMaxListItems(ctx.Properties().Int("s3.list.max-objects", 50_000))
+	}
+
 	folders, err := genericFolderCheckWithoutPrecheck(fs, check.Path, check.Recursive, check.Filter)
 	if err != nil {
 		return results.ErrorMessage(err)

checks/folder_sftp.go

@@ -1,11 +1,7 @@
 package checks
 
 import (
-	"fmt"
-
 	"github.com/flanksource/artifacts"
-	"github.com/flanksource/artifacts/clients/sftp"
-
 	"github.com/flanksource/canary-checker/api/context"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
@@ -16,19 +12,16 @@ func CheckSFTP(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	var results pkg.Results
 	results = append(results, result)
 
-	err := check.SFTPConnection.HydrateConnection(ctx)
-	if err != nil {
+	if err := check.SFTPConnection.HydrateConnection(ctx); err != nil {
 		return results.Failf("failed to populate SFTP connection: %v", err)
 	}
 
-	client, err := sftp.SSHConnect(fmt.Sprintf("%s:%d", check.SFTPConnection.Host, check.SFTPConnection.GetPort()), check.SFTPConnection.GetUsername(), check.SFTPConnection.GetPassword())
+	fs, err := artifacts.GetFSForConnection(ctx.Context, check.SFTPConnection.ToModel())
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
-	defer client.Close()
 
-	session := artifacts.Filesystem(client)
-	folders, err := genericFolderCheck(session, check.Path, check.Recursive, check.Filter)
+	folders, err := genericFolderCheck(fs, check.Path, check.Recursive, check.Filter)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}

checks/folder_smb.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/flanksource/artifacts/clients/smb"
+	"github.com/flanksource/artifacts"
 	"github.com/flanksource/canary-checker/api/context"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
@@ -15,26 +15,30 @@ func CheckSmb(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	var results pkg.Results
 	results = append(results, result)
 
-	var serverPath = strings.TrimPrefix(check.Path, "smb://")
-	server, sharename, path, err := extractServerDetails(serverPath)
+	serverPath := strings.TrimPrefix(check.Path, "smb://")
+	server, share, path, err := extractServerDetails(serverPath)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
 
-	err = check.SMBConnection.Populate(ctx)
-	if err != nil {
+	if err := check.SMBConnection.Populate(ctx); err != nil {
 		return results.Failf("failed to populate SMB connection: %v", err)
 	}
 
-	session, err := smb.SMBConnect(server, fmt.Sprintf("%d", check.SMBConnection.GetPort()), sharename, check.SMBConnection.Authentication)
+	if server != "" {
+		check.SMBConnection.Domain = server
+	}
+
+	if share != "" {
+		check.SMBConnection.Share = share
+	}
+
+	fs, err := artifacts.GetFSForConnection(ctx.Context, check.SMBConnection.ToModel())
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
-	if session != nil {
-		defer session.Close()
-	}
 
-	folders, err := genericFolderCheck(session, path, check.Recursive, check.Filter)
+	folders, err := genericFolderCheck(fs, path, check.Recursive, check.Filter)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}