diff --git a/README.md b/README.md
index db639640..c8c24339 100644
--- a/README.md
+++ b/README.md
@@ -109,12 +109,12 @@ aws s3 rb s3://"${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" --force
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
-| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | map(object({
ami = optional(string)
instance_types = list(string)
spot = optional(bool, false)
min_per_az = number
max_per_az = number
desired_per_az = number
labels = map(string)
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = object({
size = string
type = string
})
})) | `{}` | no |
+| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | map(object({
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = list(string)
spot = optional(bool, false)
min_per_az = number
max_per_az = number
desired_per_az = number
labels = map(string)
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = object({
size = string
type = string
})
})) | `{}` | no |
| [availability\_zones](#input\_availability\_zones) | List of Availibility zones to distribute the deployment, EKS needs at least 2,https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html.
Note that setting this variable bypasses validation of the status of the zones data 'aws\_availability\_zones' 'available'.
Caller is responsible for validating status of these zones. | `list(string)` | `[]` | no |
| [bastion\_ami\_id](#input\_bastion\_ami\_id) | AMI ID for the bastion EC2 instance, otherwise we will use the latest 'amazon\_linux\_2' ami | `string` | `""` | no |
| [cidr](#input\_cidr) | The IPv4 CIDR block for the VPC. | `string` | `"10.0.0.0/16"` | no |
| [create\_bastion](#input\_create\_bastion) | Create bastion toggle. | `bool` | `false` | no |
-| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | object(
{
compute = object(
{
ami = optional(string)
instance_types = optional(list(string), ["m5.2xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
platform = object(
{
ami = optional(string)
instance_types = optional(list(string), ["m5.4xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 1)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "platform"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
gpu = object(
{
ami = optional(string)
instance_types = optional(list(string), ["g4dn.xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 0)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default-gpu"
"nvidia.com/gpu" = true
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [
{ key = "nvidia.com/gpu", value = "true", effect = "NO_SCHEDULE" }
])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
})
})
| {
"compute": {},
"gpu": {},
"platform": {}
} | no |
+| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | object(
{
compute = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["m5.2xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
platform = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["m5.4xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 1)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "platform"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
gpu = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["g4dn.xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 0)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default-gpu"
"nvidia.com/gpu" = true
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [
{ key = "nvidia.com/gpu", value = "true", effect = "NO_SCHEDULE" }
])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
})
})
| {
"compute": {},
"gpu": {},
"platform": {}
} | no |
| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"domino-eks"` | no |
| [efs\_access\_point\_path](#input\_efs\_access\_point\_path) | Filesystem path for efs. | `string` | `"/domino"` | no |
| [eks\_master\_role\_names](#input\_eks\_master\_role\_names) | IAM role names to be added as masters in eks. | `list(string)` | `[]` | no |
diff --git a/submodules/eks/README.md b/submodules/eks/README.md
index 7c918762..24d750b8 100644
--- a/submodules/eks/README.md
+++ b/submodules/eks/README.md
@@ -60,10 +60,10 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
-| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | map(object({
ami = optional(string)
instance_types = list(string)
spot = optional(bool, false)
min_per_az = number
max_per_az = number
desired_per_az = number
labels = map(string)
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = object({
size = string
type = string
})
})) | `{}` | no |
+| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | map(object({
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = list(string)
spot = optional(bool, false)
min_per_az = number
max_per_az = number
desired_per_az = number
labels = map(string)
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = object({
size = string
type = string
})
})) | `{}` | no |
| [bastion\_security\_group\_id](#input\_bastion\_security\_group\_id) | Bastion security group id. | `string` | `""` | no |
| [create\_bastion\_sg](#input\_create\_bastion\_sg) | Create bastion access rules toggle. | `bool` | n/a | yes |
-| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | object(
{
compute = object(
{
ami = optional(string)
instance_types = optional(list(string), ["m5.2xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
platform = object(
{
ami = optional(string)
instance_types = optional(list(string), ["m5.4xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "platform"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
gpu = object(
{
ami = optional(string)
instance_types = optional(list(string), ["g4dn.xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 0)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default-gpu"
"nvidia.com/gpu" = true
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
})
})
| {
"compute": {},
"gpu": {},
"platform": {}
} | no |
+| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | object(
{
compute = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["m5.2xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
platform = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["m5.4xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "platform"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
gpu = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["g4dn.xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 0)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default-gpu"
"nvidia.com/gpu" = true
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
})
})
| {
"compute": {},
"gpu": {},
"platform": {}
} | no |
| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | n/a | yes |
| [efs\_security\_group](#input\_efs\_security\_group) | Security Group ID for EFS | `string` | n/a | yes |
| [eks\_cluster\_addons](#input\_eks\_cluster\_addons) | EKS cluster addons. | `list(string)` | [
"vpc-cni",
"kube-proxy",
"coredns"
]
| no |
diff --git a/submodules/eks/node-group.tf b/submodules/eks/node-group.tf
index bdf22f4e..d2b9b816 100644
--- a/submodules/eks/node-group.tf
+++ b/submodules/eks/node-group.tf
@@ -87,8 +87,22 @@ resource "aws_launch_template" "node_groups" {
name = "${local.eks_cluster_name}-${each.key}"
disable_api_termination = false
key_name = var.ssh_pvt_key_path
- vpc_security_group_ids = [aws_security_group.eks_nodes.id]
- image_id = each.value.ami
+ user_data = each.value.ami == null ? null : base64encode(templatefile(
+ "${path.module}/templates/linux_user_data.tpl",
+ {
+ # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami
+ # Required to bootstrap node
+ cluster_name = aws_eks_cluster.this.name
+ cluster_endpoint = aws_eks_cluster.this.endpoint
+ cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data
+ # Optional
+ cluster_service_ipv4_cidr = aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr != null ? aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr : ""
+ bootstrap_extra_args = each.value.bootstrap_extra_args
+ pre_bootstrap_user_data = ""
+ post_bootstrap_user_data = ""
+ }))
+ vpc_security_group_ids = [aws_security_group.eks_nodes.id]
+ image_id = each.value.ami
block_device_mappings {
device_name = "/dev/xvda"
diff --git a/submodules/eks/templates/linux_user_data.tpl b/submodules/eks/templates/linux_user_data.tpl
new file mode 100644
index 00000000..065a60d5
--- /dev/null
+++ b/submodules/eks/templates/linux_user_data.tpl
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+${pre_bootstrap_user_data ~}
+%{ if length(cluster_service_ipv4_cidr) > 0 ~}
+export SERVICE_IPV4_CIDR=${cluster_service_ipv4_cidr}
+%{ endif ~}
+B64_CLUSTER_CA=${cluster_auth_base64}
+API_SERVER_URL=${cluster_endpoint}
+/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL
+${post_bootstrap_user_data ~}
diff --git a/submodules/eks/variables.tf b/submodules/eks/variables.tf
index fb79829b..a7a12dbd 100755
--- a/submodules/eks/variables.tf
+++ b/submodules/eks/variables.tf
@@ -24,12 +24,13 @@ variable "default_node_groups" {
{
compute = object(
{
- ami = optional(string)
- instance_types = optional(list(string), ["m5.2xlarge"])
- spot = optional(bool, false)
- min_per_az = optional(number, 0)
- max_per_az = optional(number, 10)
- desired_per_az = optional(number, 1)
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = optional(list(string), ["m5.2xlarge"])
+ spot = optional(bool, false)
+ min_per_az = optional(number, 0)
+ max_per_az = optional(number, 10)
+ desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default"
})
@@ -48,12 +49,13 @@ variable "default_node_groups" {
}),
platform = object(
{
- ami = optional(string)
- instance_types = optional(list(string), ["m5.4xlarge"])
- spot = optional(bool, false)
- min_per_az = optional(number, 0)
- max_per_az = optional(number, 10)
- desired_per_az = optional(number, 1)
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = optional(list(string), ["m5.4xlarge"])
+ spot = optional(bool, false)
+ min_per_az = optional(number, 0)
+ max_per_az = optional(number, 10)
+ desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "platform"
})
@@ -72,12 +74,13 @@ variable "default_node_groups" {
}),
gpu = object(
{
- ami = optional(string)
- instance_types = optional(list(string), ["g4dn.xlarge"])
- spot = optional(bool, false)
- min_per_az = optional(number, 0)
- max_per_az = optional(number, 10)
- desired_per_az = optional(number, 0)
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = optional(list(string), ["g4dn.xlarge"])
+ spot = optional(bool, false)
+ min_per_az = optional(number, 0)
+ max_per_az = optional(number, 10)
+ desired_per_az = optional(number, 0)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default-gpu"
"nvidia.com/gpu" = true
@@ -106,15 +109,16 @@ variable "default_node_groups" {
variable "additional_node_groups" {
description = "Additional EKS managed node groups definition."
type = map(object({
- ami = optional(string)
- instance_types = list(string)
- spot = optional(bool, false)
- min_per_az = number
- max_per_az = number
- desired_per_az = number
- labels = map(string)
- taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
- tags = optional(map(string), {})
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = list(string)
+ spot = optional(bool, false)
+ min_per_az = number
+ max_per_az = number
+ desired_per_az = number
+ labels = map(string)
+ taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
+ tags = optional(map(string), {})
volume = object({
size = string
type = string
diff --git a/tests/README.md b/tests/README.md
index dba1d9d7..f15e3e78 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -5,11 +5,14 @@
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | >= 1.2.0 |
+| [terraform](#requirement\_terraform) | >= 1.3.0 |
+| [aws](#requirement\_aws) | >= 4.0 |
## Providers
-No providers.
+| Name | Version |
+|------|---------|
+| [aws.local](#provider\_aws.local) | 4.40.0 |
## Modules
@@ -19,7 +22,9 @@ No providers.
## Resources
-No resources.
+| Name | Type |
+|------|------|
+| [aws_ami.eks_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
## Inputs
diff --git a/tests/main.tf b/tests/main.tf
index 7f4146a0..9ee766e0 100755
--- a/tests/main.tf
+++ b/tests/main.tf
@@ -1,3 +1,14 @@
+data "aws_ami" "eks_node" {
+ provider = aws.local
+ most_recent = true
+ owners = ["amazon"]
+
+ filter {
+ name = "name"
+ values = ["amazon-eks-node-${var.k8s_version}-*"]
+ }
+}
+
module "domino_eks" {
source = "./.."
deploy_id = var.deploy_id
@@ -13,13 +24,9 @@ module "domino_eks" {
default_node_groups = {
compute = {
spot = true
- labels = {
- label1 = "value1"
- }
+ ami = data.aws_ami.eks_node.image_id
}
platform = {}
- gpu = {
- taints = [{ key = "abc", effect = "NO_SCHEDULE" }]
- }
+ gpu = {}
}
}
diff --git a/tests/versions.tf b/tests/versions.tf
index 3e6ba018..7fce60d3 100644
--- a/tests/versions.tf
+++ b/tests/versions.tf
@@ -1,3 +1,14 @@
terraform {
- required_version = ">= 1.2.0"
+ required_version = ">= 1.3.0"
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 4.0"
+ }
+ }
+}
+
+provider "aws" {
+ alias = "local"
+ region = var.region
}
diff --git a/variables.tf b/variables.tf
index f3a13b31..4e965b16 100755
--- a/variables.tf
+++ b/variables.tf
@@ -82,12 +82,13 @@ variable "default_node_groups" {
{
compute = object(
{
- ami = optional(string)
- instance_types = optional(list(string), ["m5.2xlarge"])
- spot = optional(bool, false)
- min_per_az = optional(number, 0)
- max_per_az = optional(number, 10)
- desired_per_az = optional(number, 1)
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = optional(list(string), ["m5.2xlarge"])
+ spot = optional(bool, false)
+ min_per_az = optional(number, 0)
+ max_per_az = optional(number, 10)
+ desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default"
})
@@ -106,12 +107,13 @@ variable "default_node_groups" {
}),
platform = object(
{
- ami = optional(string)
- instance_types = optional(list(string), ["m5.4xlarge"])
- spot = optional(bool, false)
- min_per_az = optional(number, 1)
- max_per_az = optional(number, 10)
- desired_per_az = optional(number, 1)
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = optional(list(string), ["m5.4xlarge"])
+ spot = optional(bool, false)
+ min_per_az = optional(number, 1)
+ max_per_az = optional(number, 10)
+ desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "platform"
})
@@ -130,12 +132,13 @@ variable "default_node_groups" {
}),
gpu = object(
{
- ami = optional(string)
- instance_types = optional(list(string), ["g4dn.xlarge"])
- spot = optional(bool, false)
- min_per_az = optional(number, 0)
- max_per_az = optional(number, 10)
- desired_per_az = optional(number, 0)
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = optional(list(string), ["g4dn.xlarge"])
+ spot = optional(bool, false)
+ min_per_az = optional(number, 0)
+ max_per_az = optional(number, 10)
+ desired_per_az = optional(number, 0)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default-gpu"
"nvidia.com/gpu" = true
@@ -166,15 +169,16 @@ variable "default_node_groups" {
variable "additional_node_groups" {
description = "Additional EKS managed node groups definition."
type = map(object({
- ami = optional(string)
- instance_types = list(string)
- spot = optional(bool, false)
- min_per_az = number
- max_per_az = number
- desired_per_az = number
- labels = map(string)
- taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
- tags = optional(map(string), {})
+ ami = optional(string)
+ bootstrap_extra_args = optional(string, "")
+ instance_types = list(string)
+ spot = optional(bool, false)
+ min_per_az = number
+ max_per_az = number
+ desired_per_az = number
+ labels = map(string)
+ taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
+ tags = optional(map(string), {})
volume = object({
size = string
type = string