Fixed problems with old version of EKS Blueprints #260

awslabs · Aug 28, 2023 · 3b89aed · 3b89aed
1 parent 95dd541
commit 3b89aed
Show file tree

Hide file tree

Showing 7 changed files with 156 additions and 115 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,3 +20,6 @@ hugo.linux
 # Temporary lock file while building
 /.hugo_build.lock
 **/nohup.out
+
+.terraform*
+terraform.tfstate*
diff --git a/...using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf b/...using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf
@@ -22,21 +22,21 @@ provider "aws" {
 }
 
 provider "kubernetes" {
-  host                   = module.eks_blueprints.eks_cluster_endpoint
-  cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data)
+  host                   = module.eks.cluster_endpoint
+  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
   token                  = data.aws_eks_cluster_auth.this.token
 }
 
 provider "helm" {
   kubernetes {
-    host                   = module.eks_blueprints.eks_cluster_endpoint
-    cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data)
+    host                   = module.eks.cluster_endpoint
+    cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
     token                  = data.aws_eks_cluster_auth.this.token
   }
 }
 
 data "aws_eks_cluster_auth" "this" {
-  name = module.eks_blueprints.eks_cluster_id
+  name = module.eks.cluster_name
 }
 
 data "aws_ami" "amazonlinux2eks" {
@@ -66,68 +66,65 @@ locals {
   }
 }
 
-#---------------------------------------------------------------
-# EKS Blueprints
-#---------------------------------------------------------------
-module "eks_blueprints" {
-  source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.21.0"
-
-  cluster_name    = local.name
-  cluster_version = local.cluster_version
-
-  vpc_id             = module.vpc.vpc_id
-  private_subnet_ids = module.vpc.private_subnets
-
-  node_security_group_additional_rules = {
-    # Extend node-to-node security group rules. Recommended and required for the Add-ons
-    ingress_self_all = {
-      description = "Node to node all ports/protocols"
-      protocol    = "-1"
-      from_port   = 0
-      to_port     = 0
-      type        = "ingress"
-      self        = true
-    }
-    # Recommended outbound traffic for Node groups
-    egress_all = {
-      description      = "Node all egress"
-      protocol         = "-1"
-      from_port        = 0
-      to_port          = 0
-      type             = "egress"
-      cidr_blocks      = ["0.0.0.0/0"]
-      ipv6_cidr_blocks = ["::/0"]
-    }
-    # Allows Control Plane Nodes to talk to Worker nodes on all ports. Added this to simplify the example and further avoid issues with Add-ons communication with Control plane.
-    # This can be restricted further to specific port based on the requirement for each Add-on e.g., metrics-server 4443, spark-operator 8080, karpenter 8443 etc.
-    # Change this according to your security requirements if needed
-    ingress_cluster_to_node_all_traffic = {
-      description                   = "Cluster API to Nodegroup all traffic"
-      protocol                      = "-1"
-      from_port                     = 0
-      to_port                       = 0
-      type                          = "ingress"
-      source_cluster_security_group = true
+################################################################################
+# Cluster
+################################################################################
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "19.16.0"
+
+  cluster_name                   = local.name
+  cluster_version                = local.cluster_version
+  cluster_endpoint_public_access = true
+
+  cluster_addons = {
+    # aws-ebs-csi-driver = { most_recent = true }
+    kube-proxy = { most_recent = true }
+    coredns    = { most_recent = true }
+
+    vpc-cni = {
+      most_recent    = true
+      before_compute = true
+      configuration_values = jsonencode({
+        env = {
+          ENABLE_PREFIX_DELEGATION = "true"
+          WARM_PREFIX_TARGET       = "1"
+        }
+      })
     }
   }
 
-  managed_node_groups = {
-    # Managed Node groups with minimum config
-    mg5 = {
+  vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
+
+  create_cloudwatch_log_group = false
+
+  manage_aws_auth_configmap = true
+
+  eks_managed_node_groups = {
+    mg_5 = {
       node_group_name = "mg5"
       instance_types  = ["m4.xlarge", "m5.xlarge", "m5a.xlarge", "m5ad.xlarge", "m5d.xlarge", "t2.xlarge", "t3.xlarge", "t3a.xlarge"]
-      min_size        = 2
+
+      create_security_group = false
+
+      subnet_ids   = module.vpc.private_subnets
+      max_size     = 2
+      desired_size = 2
+      min_size     = 2
+
       create_iam_role = false
       iam_role_arn    = aws_iam_role.managed_ng.arn
       disk_size       = 100
-      update_config = [{
-        max_unavailable_percentage = 30
-      }]
 
-      k8s_labels = {
+      # Launch template configuration
+      create_launch_template = true              # false will use the default launch template
+      launch_template_os     = "amazonlinux2eks" # amazonlinux2eks or bottlerocket`
+
+      labels = {
         intent = "control-apps"
       }
-    },
+    }
 
     // ### -->> SPOT NODE GROUPS GO HERE <<--- ###
   }
@@ -137,29 +134,28 @@ module "eks_blueprints" {
   tags = local.tags
 }
 
-module "eks_blueprints_kubernetes_addons" {
-  source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.21.0"
+module "eks_blueprints_addons" {
+  source  = "aws-ia/eks-blueprints-addons/aws"
+  version = "1.7.0"
 
-  eks_cluster_id       = module.eks_blueprints.eks_cluster_id
-  eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint
-  eks_oidc_provider    = module.eks_blueprints.oidc_provider
-  eks_cluster_version  = module.eks_blueprints.eks_cluster_version
+  cluster_name      = module.eks.cluster_name
+  cluster_endpoint  = module.eks.cluster_endpoint
+  cluster_version   = module.eks.cluster_version
+  oidc_provider_arn = module.eks.oidc_provider_arn
+
+  create_delay_dependencies = [for prof in module.eks.eks_managed_node_groups : prof.node_group_arn]
 
   enable_metrics_server = true
 
   tags = local.tags
-
-  depends_on = [
-    module.eks_blueprints
-  ]
 }
 
 #---------------------------------------------------------------
 # Supporting Resources
 #---------------------------------------------------------------
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
-  version = "4.0.2"
+  version = "5.0.0"
 
   name = local.name
   cidr = local.vpc_cidr
@@ -240,5 +236,5 @@ resource "aws_iam_instance_profile" "managed_ng" {
 
 output "configure_kubectl" {
   description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig"
-  value       = module.eks_blueprints.configure_kubectl
-}
+  value       = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}"
+}
diff --git a/...ances_with_eks/010_prerequisites/prerequisites.files/eks-spot-workshop-quickstart-cnf.yml b/...ances_with_eks/010_prerequisites/prerequisites.files/eks-spot-workshop-quickstart-cnf.yml
@@ -28,22 +28,22 @@ Parameters:
   C9KubectlVersion:
     Description: Cloud9 instance kubectl version
     Type: String
-    Default: v1.23.7
+    Default: v1.27.3
     ConstraintDescription: Must be a valid kubectl version
   C9KubectlVersionTEST:
     Description: Cloud9 instance kubectl version
     Type: String
-    Default: v1.23.7
+    Default: v1.27.3
     ConstraintDescription: Must be a valid kubectl version
   C9EKSctlVersion:
     Description: Cloud9 instance eksctl version
     Type: String
-    Default: v0.110.0
+    Default: v0.153.0
     ConstraintDescription: Must be a valid eksctl version
   EKSClusterVersion:
     Description: EKS Cluster Version
     Type: String
-    Default: 1.24
+    Default: 1.27
     ConstraintDescription: Must be a valid eks version
   EKSClusterName:
     Description: EKS Cluster Name
@@ -272,7 +272,7 @@ Resources:
             - echo '=== Install Terraform ==='
             - sudo yum install -y yum-utils
             - sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo
-            - sudo yum -y install terraform-1.3.7-1
+            - sudo yum -y install terraform-1.5.5
             - echo '=== Create EKS Cluster ==='        
             - sudo -H -u ec2-user mkdir -p /home/ec2-user/environment/eksworkshop/
             - sudo -H -u ec2-user curl --silent --location -o /home/ec2-user/environment/eksworkshop/main.tf "https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf"

diff --git a/...2_spot_instances_with_eks/040_eksmanagednodegroupswithspot/workers_terraform.md b/...2_spot_instances_with_eks/040_eksmanagednodegroupswithspot/workers_terraform.md
@@ -27,11 +27,17 @@ Now copy the following code snippet, and paste it just below the `// ### -->> SP
       desired_size    = 2
       min_size        = 0
 
-      subnet_type = "private"
-      subnet_ids  = []
+      subnet_ids = module.vpc.private_subnets
+
+      taints = {
+        spotInstance = {
+          key    = "spotInstance"
+          value  = "true"
+          effect = "PREFER_NO_SCHEDULE"
+        }
+      }
 
-      k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }]
-      k8s_labels = {
+      labels = {
         intent = "apps"
       }
     },
@@ -44,20 +50,22 @@ Now copy the following code snippet, and paste it just below the `// ### -->> SP
       desired_size    = 1
       min_size        = 0
 
-      subnet_type = "private"
-      subnet_ids  = []
+      subnet_ids = module.vpc.private_subnets
+
+      taints = {
+        spotInstance = {
+          key    = "spotInstance"
+          value  = "true"
+          effect = "PREFER_NO_SCHEDULE"
+        }
+      }
 
-      k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }]
-      k8s_labels = {
+      labels = {
         intent = "apps"
       }
     }
 ```
 
-Now your `main.tf` file should look like this:
-
-![EKS Blueprints - Spot Node Groups Block](/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups_final.png)
-
 Run the following command to fix any identation or configuration problem (if any):
 
 ```

diff --git a/..._spot_instances_with_eks/070_selfmanagednodegroupswithspot/workers_terraform.md b/..._spot_instances_with_eks/070_selfmanagednodegroupswithspot/workers_terraform.md
@@ -34,43 +34,77 @@ Just below that line, paste the following code snippet to create two Spot self-m
 ```
   self_managed_node_groups = {
     smng_spot_4vcpu_16mem = {
-      node_group_name    = "smng-spot-4vcpu-16mem"
-      capacity_type      = "spot"
-      capacity_rebalance = true
-      instance_types     = ["m4.xlarge", "m5.xlarge", "m5a.xlarge", "m5ad.xlarge", "m5d.xlarge", "t2.xlarge", "t3.xlarge", "t3a.xlarge"]
-      
-      max_size           = 4
-      desired_size       = 2
-      min_size           = 0
+      node_group_name            = "smng-spot-4vcpu-16mem"
+      capacity_rebalance         = true
+      use_mixed_instances_policy = true      
+      create_iam_role            = false
+      iam_role_arn               = aws_iam_role.managed_ng.arn
+      instance_type              = "m5.xlarge"
+
+      bootstrap_extra_args = "--kubelet-extra-args '--node-labels=eks.amazonaws.com/capacityType=SPOT,intent=apps,type=self-managed-spot --register-with-taints=spotInstance=true:PreferNoSchedule'"
+
+      mixed_instances_policy = {
+        instances_distribution = {
+          on_demand_base_capacity                  = 0
+          on_demand_percentage_above_base_capacity = 0
+          spot_allocation_strategy                 = "price-capacity-optimized"
+        }
+
+        override = [
+          { instance_type = "m4.xlarge" },
+          { instance_type = "m5.xlarge" },
+          { instance_type = "m5a.xlarge" },
+          { instance_type = "m5ad.xlarge" },
+          { instance_type = "m5d.xlarge" },
+          { instance_type = "t2.xlarge" },
+          { instance_type = "t3.xlarge" },
+          { instance_type = "t3a.xlarge" }
+        ]
+      }
+
+      max_size     = 4
+      desired_size = 2
+      min_size     = 0
 
       subnet_ids         = module.vpc.private_subnets
       launch_template_os = "amazonlinux2eks"
-
-      k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }]
-      k8s_labels = {
-        intent = "apps"
-        type   = "self-managed-spot"
-      }
     }
 
     smng_spot_8vcpu_32mem = {
-      node_group_name    = "smng-spot-8vcpu-32mem"
-      capacity_type      = "spot"
-      capacity_rebalance = true
-      instance_types     = ["m4.2xlarge", "m5.2xlarge", "m5a.2xlarge", "m5ad.2xlarge", "m5d.2xlarge", "t2.2xlarge", "t3.2xlarge", "t3a.2xlarge"]
-      
-      max_size           = 2
-      desired_size       = 1
-      min_size           = 0
+      node_group_name            = "smng-spot-8vcpu-32mem"
+      capacity_rebalance         = true
+      use_mixed_instances_policy = true      
+      create_iam_role            = false
+      iam_role_arn               = aws_iam_role.managed_ng.arn
+      instance_type              = "m5.2xlarge"
+
+      bootstrap_extra_args = "--kubelet-extra-args '--node-labels=eks.amazonaws.com/capacityType=SPOT,intent=apps,type=self-managed-spot --register-with-taints=spotInstance=true:PreferNoSchedule'"
+
+      mixed_instances_policy = {
+        instances_distribution = {
+          on_demand_base_capacity                  = 0
+          on_demand_percentage_above_base_capacity = 0
+          spot_allocation_strategy                 = "price-capacity-optimized"
+        }
+
+        override = [
+          { instance_type = "m4.2xlarge" },
+          { instance_type = "m5.2xlarge" },
+          { instance_type = "m5a.2xlarge" },
+          { instance_type = "m5ad.2xlarge" },
+          { instance_type = "m5d.2xlarge" },
+          { instance_type = "t2.2xlarge" },
+          { instance_type = "t3.2xlarge" },
+          { instance_type = "t3a.2xlarge" }
+        ]
+      }
 
-      subnet_ids         = module.vpc.private_subnets
-      launch_template_os = "amazonlinux2eks"
+      max_size     = 2
+      desired_size = 1
+      min_size     = 0
 
-      k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }]
-      k8s_labels = {
-        intent = "apps"
-        type   = "self-managed-spot"
-      }
+      subnet_ids         = module.vpc.private_subnets
+      launch_template_os = "amazonlinux2eks"      
     }
   }
 ```

diff --git a/...g_ec2_spot_instances_with_eks/prerequisites/eksblueprints_clusterautoscaler.png b/...g_ec2_spot_instances_with_eks/prerequisites/eksblueprints_clusterautoscaler.png
diff --git a/...ing_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups.png b/...ing_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups.png