fix(amethyst): add Ceph bucket and bucket policy in terraform

- also add mimir objectStoreUser
timtorChen · Feb 21, 2024 · 6d86505 · 6d86505
1 parent 98f347e
commit 6d86505
Show file tree

Hide file tree

Showing 4 changed files with 96 additions and 0 deletions.
diff --git a/amethyst/kubernetes/rook-ceph/objectuser.yaml b/amethyst/kubernetes/rook-ceph/objectuser.yaml
@@ -19,3 +19,12 @@ metadata:
 spec:
   store: fast
   displayName: loki
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: mimir
+  namespace: rook-ceph
+spec:
+  store: fast
+  displayName: mimir
diff --git a/amethyst/terraform/aws-data.tf b/amethyst/terraform/aws-data.tf
@@ -5,3 +5,13 @@ data "aws_region" "main" {}
 data "tls_certificate" "main" {
   url = local.oidc_issuer_url
 }
+
+# Parameter store secrets
+data "aws_ssm_parameter" "ceph-admin" {
+  name = "/amethyst/ceph-admin"
+}
+
+locals {
+  ceph_s3_access_key = jsondecode(data.aws_ssm_parameter.ceph-admin.value)["access_key"]
+  ceph_s3_secret_key = jsondecode(data.aws_ssm_parameter.ceph-admin.value)["secret_key"]
+}
diff --git a/amethyst/terraform/ceph-s3.tf b/amethyst/terraform/ceph-s3.tf
@@ -0,0 +1,62 @@
+resource "aws_s3_bucket" "loki" {
+  provider = aws.ceph-fast
+  bucket   = "${local.project}-loki"
+}
+
+resource "aws_s3_bucket_policy" "loki" {
+  provider = aws.ceph-fast
+  bucket   = aws_s3_bucket.loki.id
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Principal" : {
+          "AWS" : ["arn:aws:iam:::user/loki"]
+        }
+        "Action" : [
+          "s3:ListBucket",
+          "s3:PutObject",
+          "s3:GetObject",
+          "s3:DeleteObject"
+        ],
+        "Effect" : "Allow"
+        "Resource" : [
+          "${aws_s3_bucket.loki.arn}",
+          "${aws_s3_bucket.loki.arn}/*"
+        ]
+      }
+    ]
+  })
+}
+
+resource "aws_s3_bucket" "mimir" {
+  provider = aws.ceph-fast
+  bucket   = "${local.project}-mimir"
+}
+
+resource "aws_s3_bucket_policy" "mimir" {
+  provider = aws.ceph-fast
+  bucket   = aws_s3_bucket.mimir.id
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Principal" : {
+          "AWS" : "arn:aws:iam:::user/mimir"
+        }
+        "Action" : [
+          "s3:ListBucket",
+          "s3:PutObject",
+          "s3:GetObject",
+          "s3:DeleteObject"
+        ],
+        "Effect" : "Allow"
+        "Resource" : [
+          "${aws_s3_bucket.mimir.arn}",
+          "${aws_s3_bucket.mimir.arn}/*"
+        ]
+      }
+    ]
+  })
+}
+
diff --git a/amethyst/terraform/main.tf b/amethyst/terraform/main.tf
@@ -20,6 +20,21 @@ provider "aws" {
   region              = "us-west-2"
 }
 
+provider "aws" {
+  alias  = "ceph-fast"
+  region = "us-east-1" # it just works
+  endpoints {
+    s3 = "https://s3-fast.timtor.dev"
+  }
+  access_key                  = local.ceph_s3_access_key
+  secret_key                  = local.ceph_s3_secret_key
+  skip_credentials_validation = true
+  skip_requesting_account_id  = true
+  skip_metadata_api_check     = true
+  skip_region_validation      = true
+  s3_use_path_style           = true
+}
+
 locals {
   project         = "amethyst"
   oidc_issuer_url = "https://raw.githubusercontent.com/timtorChen/homelab/main/amethyst"