fix: explain better s3_bucket_partition_prefix func & script for movi…

…ng object between paths & trim / at the end of the bucket_prefix before writing object.

athena_query/fix_path.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# In releases previous to 2.0.0, Elevator didn't check for double slashes in the path, which could cause Athena to fail.
+# If your S3 bucket has double slashes in the path, you can run this script to fix the path.
+# Replace SOURCE_PREFIX, DESTINATION_PREFIX, and BUCKET_NAME with your values.
+
+set -x  # Enable debugging
+
+SOURCE_PREFIX="logs//2024/"
+DESTINATION_PREFIX="logs/2024/"
+BUCKET_NAME=""
+
+# Set the number of parallel jobs (adjust based on your system's resources)
+NUM_JOBS=10
+
+# Function to move a single file
+move_file() {
+  FILE_PATH="$1"
+  BUCKET_NAME="$2"
+  NEW_FILE_PATH=$(echo "$FILE_PATH" | sed "s|//|/|g")
+
+  # Copy the file to the new path with encryption
+  aws s3 cp "s3://$BUCKET_NAME/$FILE_PATH" "s3://$BUCKET_NAME/$NEW_FILE_PATH" --sse AES256
+
+  # Delete the original file if the copy was successful
+  if [[ $? -eq 0 ]]; then
+    aws s3 rm "s3://$BUCKET_NAME/$FILE_PATH"
+    echo "Moved $FILE_PATH to $NEW_FILE_PATH"
+  else
+    echo "Error copying $FILE_PATH"
+  fi
+}
+
+export -f move_file  # Export the function to be used by parallel
+
+# Step 1: List all objects in the first path and move them asynchronously
+aws s3 ls "s3://$BUCKET_NAME/$SOURCE_PREFIX" --recursive | awk '{print $4}' | \
+  grep -v '^$' | xargs -P "$NUM_JOBS" -I {} bash -c 'move_file "$@"' _ {} "$BUCKET_NAME"

src/config.py

@@ -118,6 +118,7 @@ def get_accounts_and_permission_sets(cls, values: dict) -> dict:  # noqa: ANN101
         groups = get_groups_from_statements(group_statements)
         permission_sets = set()
         accounts = set()
+        s3_bucket_prefix_for_partitions = values.get("s3_bucket_prefix_for_partitions", "").rstrip("/")
         for statement in statements:
             permission_sets.update(statement.permission_set)
             if statement.resource_type == "Account":
@@ -128,6 +129,7 @@ def get_accounts_and_permission_sets(cls, values: dict) -> dict:  # noqa: ANN101
             "statements": frozenset(statements),
             "group_statements": frozenset(group_statements),
             "groups": groups,
+            "s3_bucket_prefix_for_partitions": s3_bucket_prefix_for_partitions,
         }
 
 

vars.tf

@@ -143,7 +143,11 @@ variable "s3_bucket_name_for_audit_entry" {
 }
 
 variable "s3_bucket_partition_prefix" {
-  description = "The prefix for the S3 bucket partitions"
+  description = <<EOT
+  The prefix for the S3 audit bucket object partitions.
+  Don't use slashes (/) in the prefix, as it will be added automatically, e.g. "logs" will be transformed to "logs/".
+  If you want to use the root of the bucket, leave this empty.
+  EOT
   type        = string
   default     = "logs"
 }