diff --git a/cloud/examples/object_storage/object_storage.go b/cloud/examples/object_storage/object_storage.go index 7c64d09..fd02b80 100644 --- a/cloud/examples/object_storage/object_storage.go +++ b/cloud/examples/object_storage/object_storage.go @@ -1,9 +1,14 @@ package main import ( + "bytes" "context" "errors" "fmt" + "github.com/aws/aws-sdk-go/aws" + "io" + "log" + "net/http" "time" "github.com/byte-power/gorich/cloud" @@ -29,11 +34,16 @@ func main() { object_storage_examples("aws_bucket_name_xxx", optionForAWS) optionForAliOSS := object_storage.AliCloudStorageOption{ - CredentialType: "oidc_role_arn", - EndPoint: "oss-cn-zhangjiakou.aliyuncs.com", - SessionName: "test-rrsa-oidc-token", + //CredentialType: "oidc_role_arn", + //EndPoint: "oss-cn-zhangjiakou.aliyuncs.com", + //SessionName: "test-rrsa-oidc-token", + + CredentialType: "ak", + EndPoint: "oss-cn-beijing.aliyuncs.com", + AccessKeyID: "alicloud_access_key_id_xxx", + AccessKeySecret: "alicloud_access_key_secret_xxx", } - object_storage_examples("my-bucket", optionForAliOSS) + object_storage_examples("alicloud_bucket_name_xxx", optionForAliOSS) } func object_storage_examples(bucketName string, option cloud.Option) { @@ -186,4 +196,80 @@ func object_storage_examples(bucketName string, option cloud.Option) { fmt.Printf("GetSignedURLForExistedKey %s %s\n", name, url) } } + + // PutSignedURL examples + for name, item := range files { + // get pre-signed put url + opt := object_storage.PutHeaderOption{ + ContentType: aws.String(item.ContentType), + } + url, err := service.PutSignedURL(name, 1*time.Hour, opt) + if err != nil { + fmt.Printf("GetSignedURL for object %s error %s\n", name, err) + return + } + fmt.Printf("GetSignedURL for put object %s %s\n", name, url) + + // put content to s3 with signed-url + if err := uploadContent(url, string(item.Body), item.ContentType); err != nil { + fmt.Printf("Error uploading content: %v\n", err) + return + } + + // get pre-signed url for download and check + getSignedURL, err := service.GetSignedURL(name, 1*time.Hour) + if err != nil { + fmt.Printf("GetSignedURL for object %s error %s\n", name, err) + return + } + fmt.Printf("GetSignedURL for get object %s %s\n", name, getSignedURL) + + // check content + content, err := accessContentBySignedURL(getSignedURL) + if err != nil { + log.Fatalf("Get content err: %v", err) + } + fmt.Println("Get content:", content) + } +} + +// uploadContent uploads content to S3 using the provided presigned URL. +func uploadContent(signedURL, content, contentType string) error { + req, err := http.NewRequest("PUT", signedURL, bytes.NewBuffer([]byte(content))) + if err != nil { + return fmt.Errorf("error creating PUT request: %w", err) + } + req.Header.Set("Content-Type", contentType) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error executing PUT request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("upload failed with status code %d", resp.StatusCode) + } + fmt.Println("Content uploaded successfully") + return nil +} + +// accessContentBySignedURL access content by Signed URL +func accessContentBySignedURL(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("access content failed: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("access content failed, status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("get content failed: %v", err) + } + return string(body), nil } diff --git a/cloud/object_storage/alicloud_object_storage.go b/cloud/object_storage/alicloud_object_storage.go index b5f500d..3c8a078 100644 --- a/cloud/object_storage/alicloud_object_storage.go +++ b/cloud/object_storage/alicloud_object_storage.go @@ -17,15 +17,21 @@ import ( var ossClientMap = make(map[string]*oss.Client) var ( - ErrAliCloudStorageServiceCredentialTypeEmpty = errors.New("credential_type for alicloud storage service is empty") - ErrAliCloudStorageServiceEndPointEmpty = errors.New("endpoint for alicloud storage service is empty") - ErrAliCloudStorageServiceSessionNameEmpty = errors.New("session_name for alicloud storage service is empty") + ErrAliCloudStorageServiceCredentialTypeEmpty = errors.New("credential_type for alicloud storage service is empty") + ErrAliCloudStorageServiceEndPointEmpty = errors.New("endpoint for alicloud storage service is empty") + ErrAliCloudStorageServiceSessionNameEmpty = errors.New("session_name for alicloud storage service is empty") + ErrAliCloudStorageServiceAccessKeyIDEmpty = errors.New("access_key_id for alicloud storage service is empty") + ErrAliCloudStorageServiceAccessKeySecretEmpty = errors.New("access_key_secret for alicloud storage service is empty") ) type AliCloudStorageOption struct { - CredentialType string // eg: "oidc_role_arn" + CredentialType string // eg: "oidc_role_arn" or "ak" EndPoint string // eg: "oss-cn-zhangjiakou.aliyuncs.com" SessionName string // eg: "test-rrsa-oidc-token" + + // "ak" required + AccessKeyID string + AccessKeySecret string } func (option AliCloudStorageOption) GetProvider() cloud.Provider { @@ -33,11 +39,11 @@ func (option AliCloudStorageOption) GetProvider() cloud.Provider { } func (option AliCloudStorageOption) GetSecretID() string { - return "" + return option.AccessKeyID } func (option AliCloudStorageOption) GetSecretKey() string { - return "" + return option.AccessKeySecret } func (option AliCloudStorageOption) GetAssumeRoleArn() string { @@ -79,8 +85,18 @@ func (option AliCloudStorageOption) check() error { if option.EndPoint == "" { return ErrAliCloudStorageServiceEndPointEmpty } - if option.SessionName == "" { - return ErrAliCloudStorageServiceSessionNameEmpty + + if option.CredentialType == "oidc_role_arn" { + if option.SessionName == "" { + return ErrAliCloudStorageServiceSessionNameEmpty + } + } else if option.CredentialType == "ak" { + if option.AccessKeyID == "" { + return ErrAliCloudStorageServiceAccessKeyIDEmpty + } + if option.AccessKeySecret == "" { + return ErrAliCloudStorageServiceAccessKeySecretEmpty + } } return nil } @@ -109,17 +125,28 @@ func GetAliCloudObjectService(bucketName string, option cloud.Option) (ObjectSto return &AliCloudObjectStorageService{client: client, bucketName: bucketName}, nil } - cred, err := newOidcCredential(storageOption.CredentialType, storageOption.SessionName) - if err != nil { - return nil, err - } - - provider := &aliCloudCredentialsProvider{ - cred: cred, - } - client, err := oss.New(storageOption.EndPoint, "", "", oss.SetCredentialsProvider(provider)) - if err != nil { - return nil, err + var client *oss.Client + if storageOption.CredentialType == "oidc_role_arn" { + cred, err := newOidcCredential(storageOption.CredentialType, storageOption.SessionName) + if err != nil { + return nil, err + } + provider := &aliCloudCredentialsProvider{ + cred: cred, + } + ossClient, err := oss.New(storageOption.EndPoint, "", "", oss.SetCredentialsProvider(provider)) + if err != nil { + return nil, err + } + client = ossClient + } else if storageOption.CredentialType == "ak" { + ossClient, err := oss.New(storageOption.EndPoint, storageOption.AccessKeyID, storageOption.AccessKeySecret) + if err != nil { + return nil, err + } + client = ossClient + } else { + return nil, fmt.Errorf("credential type '%s' unsupported", storageOption.CredentialType) } // cache client @@ -319,3 +346,15 @@ func (service *AliCloudObjectStorageService) GetSignedURLForExistedKey(ctx conte } return service.GetSignedURL(key, duration) } + +func (service *AliCloudObjectStorageService) PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error) { + if key == "" { + return "", ErrObjectKeyEmpty + } + bucket, err := service.client.Bucket(service.bucketName) + if err != nil { + return "", err + } + options := option.ToAliCloudOptions() + return bucket.SignURL(key, oss.HTTPPut, int64(duration.Seconds()), options...) +} diff --git a/cloud/object_storage/aws_object_storage.go b/cloud/object_storage/aws_object_storage.go index aa8437d..5b0ddaa 100644 --- a/cloud/object_storage/aws_object_storage.go +++ b/cloud/object_storage/aws_object_storage.go @@ -204,6 +204,28 @@ func (service *AWSObjectStorageService) GetSignedURLForExistedKey(ctx context.Co return service.GetSignedURL(key, duration) } +func (service *AWSObjectStorageService) PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error) { + if key == "" { + return "", ErrObjectKeyEmpty + } + + request, _ := service.client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: &service.bucketName, + Key: &key, + ContentDisposition: option.ContentDisposition, + ContentEncoding: option.ContentEncoding, + ContentMD5: option.ContentMD5, + ContentType: option.ContentType, + ContentLength: option.ContentLength, + Tagging: option.Tagging, + }) + url, err := request.Presign(duration) + if err != nil { + return "", err + } + return url, nil +} + func isNotFoundErrorForAWS(err error) bool { awsErr, ok := err.(awserr.Error) if !ok { diff --git a/cloud/object_storage/object_storage.go b/cloud/object_storage/object_storage.go index cd35f51..beb303f 100644 --- a/cloud/object_storage/object_storage.go +++ b/cloud/object_storage/object_storage.go @@ -3,6 +3,11 @@ package object_storage import ( "context" "errors" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/tencentyun/cos-go-sdk-v5" + "net/http" + "net/url" + "strconv" "time" "github.com/byte-power/gorich/cloud" @@ -24,6 +29,7 @@ type ObjectStorageService interface { GetSignedURL(key string, duration time.Duration) (string, error) // GetSignedURLForExistedKey generates signed url if key exists. If key does not exist, return error GetSignedURLForExistedKey(ctx context.Context, key string, duration time.Duration) (string, error) + PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error) } type PutObjectInput struct { @@ -75,3 +81,64 @@ func GetObjectStorageService(bucketName string, option cloud.Option) (ObjectStor } return nil, cloud.ErrUnsupportedCloudProvider } + +type PutHeaderOption struct { + ContentDisposition *string + ContentEncoding *string + ContentMD5 *string + ContentType *string + ContentLength *int64 + Tagging *string +} + +func (o *PutHeaderOption) ToAliCloudOptions() []oss.Option { + + options := make([]oss.Option, 0) + if o.ContentDisposition != nil { + options = append(options, oss.ContentDisposition(*o.ContentDisposition)) + } + if o.ContentEncoding != nil { + options = append(options, oss.ContentEncoding(*o.ContentEncoding)) + } + if o.ContentMD5 != nil { + options = append(options, oss.ContentMD5(*o.ContentMD5)) + } + if o.ContentType != nil { + options = append(options, oss.ContentType(*o.ContentType)) + } + if o.ContentLength != nil { + options = append(options, oss.ContentLength(*o.ContentLength)) + } + if o.Tagging != nil { + options = append(options, oss.SetHeader(oss.HTTPHeaderOssTagging, *o.Tagging)) + } + return options +} + +func (o *PutHeaderOption) ToTencentCloudOptions() *cos.PresignedURLOptions { + + opt := &cos.PresignedURLOptions{ + Query: &url.Values{}, + Header: &http.Header{}, + } + + if o.ContentDisposition != nil { + opt.Header.Add("Content-Disposition", *o.ContentDisposition) + } + if o.ContentEncoding != nil { + opt.Header.Add("Content-Encoding", *o.ContentEncoding) + } + if o.ContentMD5 != nil { + opt.Header.Add("Content-MD5", *o.ContentMD5) + } + if o.ContentType != nil { + opt.Header.Add("Content-Type", *o.ContentType) + } + if o.ContentLength != nil { + opt.Header.Add("Content-Length", strconv.FormatInt(*o.ContentLength, 10)) + } + if o.Tagging != nil { + opt.Header.Add("x-cos-tagging", *o.Tagging) + } + return opt +} diff --git a/cloud/object_storage/tencentcloud_object_storage.go b/cloud/object_storage/tencentcloud_object_storage.go index 7e4495d..44e615e 100644 --- a/cloud/object_storage/tencentcloud_object_storage.go +++ b/cloud/object_storage/tencentcloud_object_storage.go @@ -257,3 +257,22 @@ func (service *TencentCloudObjectStorageService) GetSignedURLForExistedKey(ctx c } return service.GetSignedURL(key, duration) } + +func (service *TencentCloudObjectStorageService) PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error) { + if key == "" { + return "", ErrObjectKeyEmpty + } + + options := option.ToTencentCloudOptions() + + url, err := service.client.Object.GetPresignedURL( + context.Background(), http.MethodPut, key, + service.client.GetCredential().SecretID, + service.client.GetCredential().SecretKey, + duration, options, + ) + if err != nil { + return "", err + } + return url.String(), nil +}