Trying a map.

jadudm · Mar 5, 2024 · 2e58e9f · 2e58e9f
1 parent d158dbd
commit 2e58e9f
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 40 deletions.
diff --git a/cmd/bucket.go b/cmd/bucket.go
@@ -51,7 +51,7 @@ func bucket_local_tables(source_creds *structs.CredentialsRDS, up structs.UserPr
 	}
 }
 
-func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up *structs.CredentialsS3) {
+func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up map[string]string) {
 	table_to_schema := util.Get_table_and_schema_names(source_creds)
 	for table, schema := range table_to_schema {
 		s3_pipe := pipes.S3(

diff --git a/internal/pipes/s3.go b/internal/pipes/s3.go
@@ -2,32 +2,33 @@ package pipes
 
 import (
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/bitfield/script"
 	"gov.gsa.fac.cgov-util/internal/logging"
-	"gov.gsa.fac.cgov-util/internal/structs"
 	"gov.gsa.fac.cgov-util/internal/util"
 )
 
 // For reasons that are unclear, the access key id and secret key
 // are coming through from VCAP empty. But, the endpoint is not.
 // This makes no sense.
 func S3(in_pipe *script.Pipe,
-	up *structs.CredentialsS3,
+	up map[string]string,
 	prefix string,
 	source_db string,
 	schema string, table string) *script.Pipe {
+	os.Setenv("AWS_SECRET_ACCESS_KEY", up["secret_access_key"])
+	os.Setenv("AWS_ACCESS_KEY_ID", up["access_key_id"])
+	os.Setenv("AWS_DEFAULT_REGION", up["region"])
 	// https://serverfault.com/questions/886562/streaming-postgresql-pg-dump-to-s3
 	cmd := []string{
 		"aws",
 		"s3",
 		"cp",
-		"--endpoint-url",
-		up.Uri,
 		"-",
 		fmt.Sprintf("s3://%s/backups/%s-%s_%s.dump",
-			up.Bucket,
+			up["bucket"],
 			prefix,
 			schema, table),
 	}

diff --git a/internal/structs/vcap.go b/internal/structs/vcap.go
@@ -22,20 +22,21 @@ type CredentialsS3 struct {
 	Endpoint           string   `json:"endpoint"`
 	FipsEndpoint       string   `json:"fips_endpoint"`
 	AdditionalBuckets  []string `json:"additional_buckets"`
-	SyslogDrainUrl     string   `json:"syslog_drain_url"`
-	VolumeMounts       []string `json:"volume_mounts`
 }
 
 type InstanceS3 struct {
-	Label        string        `json:"label"`
-	Plan         string        `json:"plan"`
-	Name         string        `json:"name"`
-	Tags         []string      `json:"tags"`
-	InstanceGuid string        `json:"instance_guid"`
-	InstanceName string        `json:"instance_name"`
-	BindingGuid  string        `json:"binding_guid"`
-	BindingName  string        `json:"binding_name"`
-	Credentials  CredentialsS3 `json:"credentials"`
+	Label          string            `json:"label"`
+	Provider       string            `json:"provider"`
+	Plan           string            `json:"plan"`
+	Name           string            `json:"name"`
+	Tags           []string          `json:"tags"`
+	InstanceGuid   string            `json:"instance_guid"`
+	InstanceName   string            `json:"instance_name"`
+	BindingGuid    string            `json:"binding_guid"`
+	BindingName    string            `json:"binding_name"`
+	Credentials    map[string]string `json:"credentials"`
+	SyslogDrainUrl string            `json:"syslog_drain_url"`
+	VolumeMounts   []string          `json:"volume_mounts"`
 }
 
 type InstanceRDS struct {

diff --git a/internal/vcap/example.json b/internal/vcap/example.json
@@ -2,7 +2,7 @@
     "s3": [
         {
             "label": "s3",
-            "provider": null,
+            "provider": "alpha-provider",
             "plan": "basic",
             "name": "backups",
             "tags": [
@@ -18,15 +18,15 @@
                 "uri": "s3://ACCESSKEYIDALPHA:SECRETACCESSKEYALPHA@s3-us-gov-alpha-1.amazonaws.com/BROKEREDBUCKETALPHA",
                 "insecure_skip_verify": false,
                 "access_key_id": "ACCESSKEYIDALPHA",
-                "secret_access_key": "SECRETACCESSKEYALPHA",
+                "secret_access_key": "SECRETACCESSKEY+ALPHA",
                 "region": "us-gov-west-1",
                 "bucket": "BROKEREDBUCKETALPHA",
                 "endpoint": "s3-us-gov-alpha-1.amazonaws.com",
                 "fips_endpoint": "s3-fips.us-gov-alpha-1.amazonaws.com",
                 "additional_buckets": []
             },
-            "syslog_drain_url": null,
-            "volume_mounts": []
+            "syslog_drain_url": "https://ALPHA.drain.url",
+            "volume_mounts": ["no_mounts"]
         },
         {
             "label": "s3",

diff --git a/internal/vcap/vcap.go b/internal/vcap/vcap.go
@@ -59,33 +59,20 @@ func GetUserProvidedCredentials(label string) (structs.UserProvidedCredentials,
 	return nil, errors.Errorf("No credentials found for '%s'", label)
 }
 
-func GetS3Credentials(name string) (*structs.CredentialsS3, error) {
+func GetS3Credentials(name string) (map[string]string, error) {
 	var instanceSlice []structs.InstanceS3
 	err := viper.UnmarshalKey("s3", &instanceSlice)
 	if err != nil {
 		logging.Logger.Println("Could not unmarshal s3 from VCAP_SERVICES")
 	}
 	for _, instance := range instanceSlice {
 		if instance.Name == name {
-			all_looks_good := false
-
-			// We have to have an endpoint, or the two key bits.
-			if (len(instance.Credentials.AccessKeyId) > 0) &&
-				(len(instance.Credentials.SecretAccessKey) > 0) {
-				all_looks_good = true
-			} else if len(instance.Credentials.Uri) > 0 {
-				all_looks_good = true
-			}
+			fmt.Println("INST", instance)
+			fmt.Println("AKI", instance.Credentials["access_key_id"])
+			fmt.Println("SAK", instance.Credentials["secret_access_key"])
+			fmt.Println("REG", instance.Credentials["region"])
 
-			if len(instance.Credentials.Region) < 1 {
-				logging.Logger.Println("BACKUPS region is empty")
-				os.Exit(-1)
-			}
-			if all_looks_good {
-				return &instance.Credentials, nil
-			} else {
-				return nil, fmt.Errorf("BACKUPS no access key or endpoint")
-			}
+			return instance.Credentials, nil
 		}
 	}
 

diff --git a/internal/vcap/vcap_test.go b/internal/vcap/vcap_test.go
@@ -1,6 +1,7 @@
 package vcap
 
 import (
+	"io/ioutil"
 	"os"
 	"testing"
 )
@@ -117,3 +118,23 @@ func TestReadUserProvided(t *testing.T) {
 		t.Error("Could not find a username")
 	}
 }
+
+func TestReadS3(t *testing.T) {
+	buffer, err := ioutil.ReadFile("example.json")
+	if err != nil {
+		t.Error("Could not read example.json")
+	}
+	os.Setenv("VCAP_SERVICES", string(buffer))
+	ReadVCAPConfig()
+
+	creds, err := GetS3Credentials("backups")
+	if err != nil {
+		t.Error("Could not read backups credentials from s3.")
+	}
+	if creds["access_key_id"] != "ACCESSKEYIDALPHA" {
+		t.Error("Did not get s3 access key ACCESSKEYIDALPHA")
+	}
+	if creds["secret_access_key"] != "SECRETACCESSKEY+ALPHA" {
+		t.Error("Did not get s3 secret key SECRETACCESSKEY+ALPHA")
+	}
+}