[WIP] update suggestions for add features in aws mutlti zone support

ci/infra/aws/cloud-init.tf

@@ -1,25 +1,24 @@
-data "template_file" "register_rmt" {
-  template = file("cloud-init/register-rmt.tpl")
-  count    = var.rmt_server_name == "" ? 0 : 1
+data "template_file" "register_scc" {
+  count    = var.caasp_registry_code != "" && var.rmt_server_name == "" ? 1 : 0
+  template = file("${path.module}/cloud-init/register-scc.tpl")
 
   vars = {
-    rmt_server_name = var.rmt_server_name
+    caasp_registry_code = var.caasp_registry_code
   }
 }
 
-data "template_file" "register_scc" {
-  # register with SCC iff an RMT has not been provided
-  count    = var.caasp_registry_code != "" && var.rmt_server_name == "" ? 1 : 0
-  template = file("cloud-init/register-scc.tpl")
+data "template_file" "register_rmt" {
+  count    = var.rmt_server_name == "" ? 0 : 1
+  template = file("${path.module}/cloud-init/register-rmt.tpl")
 
   vars = {
-    caasp_registry_code = var.caasp_registry_code
+    rmt_server_name = var.rmt_server_name
   }
 }
 
 data "template_file" "register_suma" {
-  template = file("cloud-init/register-suma.tpl")
   count    = var.suma_server_name == "" ? 0 : 1
+  template = file("${path.module}/cloud-init/register-suma.tpl")
 
   vars = {
     suma_server_name = var.suma_server_name
@@ -28,43 +27,43 @@ data "template_file" "register_suma" {
 
 data "template_file" "repositories" {
   count    = length(var.repositories)
-  template = file("cloud-init/repository.tpl")
+  template = file("${path.module}/cloud-init/repository.tpl")
 
   vars = {
-    repository_url  = var.repositories[count.index]
-    repository_name = var.repositories[count.index]
+    repository_url  = element(values(var.repositories), count.index)
+    repository_name = element(keys(var.repositories), count.index)
   }
 }
 
 data "template_file" "commands" {
-  template = file("cloud-init/commands.tpl")
-  count    = length(var.packages) == 0 ? 0 : 1
+  count    = join("", var.packages) == "" ? 0 : 1
+  template = file("${path.module}/cloud-init/commands.tpl")
 
   vars = {
     packages = join(", ", var.packages)
   }
 }
 
 data "template_file" "cloud-init" {
-  template = file("cloud-init/cloud-init.yaml.tpl")
+  template = file("${path.module}/cloud-init/cloud-init.yaml.tpl")
 
   vars = {
     authorized_keys = join("\n", formatlist("  - %s", var.authorized_keys))
-    commands        = join("\n", data.template_file.commands.*.rendered)
-    repositories    = length(var.repositories) == 0 ? "\n" : join("\n", data.template_file.repositories.*.rendered)
     register_scc    = var.caasp_registry_code != "" && var.rmt_server_name == "" ? join("\n", data.template_file.register_scc.*.rendered) : ""
     register_rmt    = var.rmt_server_name != "" ? join("\n", data.template_file.register_rmt.*.rendered) : ""
     register_suma   = var.suma_server_name != "" ? join("\n", data.template_file.register_suma.*.rendered) : ""
+    username        = var.username
+    repositories    = length(var.repositories) == 0 ? "\n" : join("\n", data.template_file.repositories.*.rendered)
+    commands = join("\n", data.template_file.commands.*.rendered)
   }
 }
 
 data "template_cloudinit_config" "cfg" {
   gzip          = false
-  base64_encode = false
+  base64_encode = true
 
   part {
     content_type = "text/cloud-config"
     content      = data.template_file.cloud-init.rendered
   }
 }
-

ci/infra/aws/cloud-init/cloud-init.yaml.tpl

@@ -1,22 +1,50 @@
 #cloud-config
+# vim: syntax=yaml
+#
+# ***********************
+#   ---- for more examples look at: ------
+# ---> https://cloudinit.readthedocs.io/en/latest/topics/examples.html
+# ---> https://www.terraform.io/docs/providers/template/d/cloudinit_config.html
+# ******************************
+#
+# This is the configuration syntax that the write_files module
+# will know how to understand. encoding can be given b64 or gzip or (gz+b64).
+# The content will be decoded accordingly and then written to the path that is
+# provided.
+#
+# Note: Content strings here are truncated for example purposes.
 
 # set locale
 locale: en_US.UTF-8
 
 # set timezone
 timezone: Etc/UTC
 
+users:
+  - name: ${username}
+    ssh-authorized-keys:
+      ${authorized_keys}
+    sudo: ['ALL=(ALL) NOPASSWD:ALL']
+    groups: sudo
+    shell: /bin/bash
+
+# Inject the public keys
 ssh_authorized_keys:
 ${authorized_keys}
 
+# WARNING!!! Do not use cloud-init packages module when SUSE CaaSP Registration
+# Code is provided. In this case, repositories will be added in runcmd module
+# with SUSEConnect command after packages module is ran
+#packages:
+
 bootcmd:
   - ip link set dev eth0 mtu 1500
 
 runcmd:
 ${register_scc}
 ${register_rmt}
 ${register_suma}
+${repositories}
 ${commands}
 
 final_message: "The system is finally up, after $UPTIME seconds"
-

ci/infra/aws/cloud-init/commands.tpl

@@ -1,6 +1,6 @@
-  - echo "solver.onlyRequires = true" >> /etc/zypp/zypp.conf
-  - [ zypper, -n, install, ${packages} ]
-  - [ ip, link, delete, docker0 ]
+  - ip link delete docker0
   - iptables -L | grep DOCKER | awk {'print $2'} | xargs -d "\n" -i iptables -X {}
   - iptables-save | awk '/^[*]/ { print $1 "\nCOMMIT" }' | iptables-restore
   - lsmod | egrep ^iptable_ | awk '{print $1}' | xargs -rd\\n modprobe -r
+  - echo "solver.onlyRequires = true" >> /etc/zypp/zypp.conf
+  - zypper -n install ${packages}

ci/infra/aws/cloud-init/register-rmt.tpl

@@ -1,4 +1,4 @@
   - curl --tlsv1.2 --silent --insecure --connect-timeout 10 https://${rmt_server_name}/rmt.crt --output /etc/pki/trust/anchors/rmt-server.pem && /usr/sbin/update-ca-certificates &> /dev/null
   - SUSEConnect --url https://${rmt_server_name}
   - SUSEConnect -p sle-module-containers/15.2/x86_64
-  - SUSEConnect -p caasp/4.5/x86_64
+  - SUSEConnect -p caasp/4.5/x86_64 -r ${caasp_registry_code}

ci/infra/aws/cloud-init/repository.tpl

@@ -1,6 +1,2 @@
-    - id: ${repository_name}
-      name: ${repository_name}
-      baseurl: ${repository_url}
-      enabled: 1
-      autorefresh: 1
-      gpgcheck: 0
+  - zypper addrepo --refresh ${repository_url} ${repository_name}
+  - zypper --gpg-auto-import-keys refresh

ci/infra/aws/container-openrc.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+
+export AWS_ACCESS_KEY_ID=""
+export AWS_SECRET_ACCESS_KEY=""

ci/infra/aws/iam_policies.tf → ci/infra/aws/iam.tf

@@ -7,16 +7,16 @@ locals {
 }
 
 resource "aws_iam_instance_profile" "master" {
+  count = length(var.iam_profile_master) == 0 ? 1 : 0
   name  = local.aws_iam_instance_profile_master_terraform
   role  = aws_iam_role.master[count.index].name
-  count = length(var.iam_profile_master) == 0 ? 1 : 0
 }
 
 resource "aws_iam_role" "master" {
+  count       = length(var.iam_profile_master) == 0 ? 1 : 0
   name        = local.aws_iam_instance_profile_master_terraform
   description = "IAM role needed by CPI on master nodes"
   path        = "/"
-  count       = length(var.iam_profile_master) == 0 ? 1 : 0
 
   assume_role_policy = <<EOF
 {
@@ -36,9 +36,9 @@ EOF
 }
 
 resource "aws_iam_role_policy" "master" {
+  count = length(var.iam_profile_master) == 0 ? 1 : 0
   name  = local.aws_iam_instance_profile_master_terraform
   role  = aws_iam_role.master[count.index].id
-  count = length(var.iam_profile_master) == 0 ? 1 : 0
 
   policy = <<EOF
 {
@@ -112,16 +112,16 @@ EOF
 }
 
 resource "aws_iam_instance_profile" "worker" {
+  count = length(var.iam_profile_worker) == 0 ? 1 : 0
   name  = local.aws_iam_instance_profile_worker_terraform
   role  = aws_iam_role.worker[count.index].name
-  count = length(var.iam_profile_worker) == 0 ? 1 : 0
 }
 
 resource "aws_iam_role" "worker" {
+  count       = length(var.iam_profile_worker) == 0 ? 1 : 0
   name        = local.aws_iam_instance_profile_worker_terraform
   description = "IAM role needed by CPI on worker nodes"
   path        = "/"
-  count       = length(var.iam_profile_worker) == 0 ? 1 : 0
 
   assume_role_policy = <<EOF
 {
@@ -140,11 +140,10 @@ resource "aws_iam_role" "worker" {
 EOF
 }
 
-
 resource "aws_iam_role_policy" "worker" {
+  count = length(var.iam_profile_worker) == 0 ? 1 : 0
   name  = local.aws_iam_instance_profile_worker_terraform
   role  = aws_iam_role.worker[count.index].id
-  count = length(var.iam_profile_worker) == 0 ? 1 : 0
 
   policy = <<EOF
 {

ci/infra/aws/load-balancer.tf

@@ -1,19 +1,63 @@
-resource "aws_elb" "kube_api" {
-  connection_draining       = false
-  cross_zone_load_balancing = true
-  idle_timeout              = 400
-  instances                 = aws_instance.control_plane.*.id
-  name                      = "${var.stack_name}-elb"
-  subnets                   = [aws_subnet.public.id]
+# A security group for the ELB so it is accessible via the web
+resource "aws_security_group" "elb" {
+  name        = "${var.stack_name}-elb"
+  description = "give access to kube api server"
+  vpc_id      = aws_vpc.platform.id
+
+  # # HTTP access from anywhere
+  # ingress {
+  #   from_port   = 80
+  #   to_port     = 80
+  #   protocol    = "tcp"
+  #   cidr_blocks = ["0.0.0.0/0"]
+  # }
+
+  # # HTTPS access from anywhere
+  # ingress {
+  #   from_port   = 443
+  #   to_port     = 443
+  #   protocol    = "tcp"
+  #   cidr_blocks = ["0.0.0.0/0"]
+  # }
+
+  ingress {
+    from_port   = 6443
+    to_port     = 6443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "kubernetes API server"
+  }
+
+  # Allow access to dex (32000) and gangway (32001)
+  ingress {
+    from_port   = 32000
+    to_port     = 32001
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "dex and gangway"
+  }
 
   tags = merge(
-    local.basic_tags,
+    local.tags,
     {
       "Name"  = "${var.stack_name}-elb"
-      "Class" = "ElasticLoadBalancer"
+      "Class" = "SecurityGroup"
     },
   )
+}
 
+# https://www.terraform.io/docs/providers/aws/r/elb.html
+resource "aws_elb" "elb" {
+  name = "${var.stack_name}-elb"
+  # https://www.terraform.io/docs/providers/aws/r/elb.html
+  # Exactly one of availability_zones or subnets must be specified: this determines if the ELB exists in a VPC or in EC2-classic.
+  # availability_zones        = tolist(aws_subnet.public.*.availability_zone)
+  subnets                   = tolist(aws_subnet.public.*.id)
+  instances                 = aws_instance.master.*.id
+  cross_zone_load_balancing = true
+  idle_timeout              = 400
+  connection_draining       = false
+  # connection_draining_timeout = 400
   security_groups = [
     aws_security_group.elb.id,
     aws_security_group.egress.id,
@@ -52,9 +96,12 @@ resource "aws_elb" "kube_api" {
     timeout             = 3
     unhealthy_threshold = 6
   }
-}
 
-output "elb_address" {
-  value = aws_elb.kube_api.dns_name
+  tags = merge(
+    local.basic_tags,
+    {
+      Name    = "${var.stack_name}-elb"
+      "Class" = "ElasticLoadBalancer"
+    },
+  )
 }
-

ci/infra/aws/main.tf

@@ -0,0 +1,53 @@
+locals {
+  # Do not add the special `kubernetes.io/cluster<cluster-name>` here,
+  # this tag cannot be added to all our resources otherwise the CPI
+  # will get confused when dealing with security rules objects.
+  basic_tags = merge(
+    {
+      "Name"        = var.stack_name
+      "Environment" = var.stack_name
+    },
+    var.tags,
+  )
+
+  # tags = local.basic_tags
+  tags = merge(
+    local.basic_tags,
+    {
+      format("kubernetes.io/cluster/%v", var.stack_name) = "SUSE-terraform"
+    },
+  )
+}
+
+# https://www.terraform.io/docs/providers/aws/index.html
+provider "aws" {
+  profile = "default"
+  region  = var.aws_region
+}
+
+data "susepubliccloud_image_ids" "sles15sp2_chost_byos" {
+  cloud  = "amazon"
+  region = var.aws_region
+  state  = "active"
+
+  # USE SLES 15 SP2 Container host AMI - this is needed to avoid issues like bsc#1146774
+  name_regex = "suse-sles-15-sp2-chost-byos.*-hvm-ssd-x86_64"
+}
+
+resource "aws_key_pair" "kube" {
+  key_name   = "${var.stack_name}-keypair"
+  public_key = element(var.authorized_keys, 0)
+
+  tags = merge(
+    local.basic_tags,
+    {
+      "Name"  = "${var.stack_name}-keypair"
+      "Class" = "KeyPair"
+    },
+  )
+}
+
+# list of availability_zones which can be access from the current region
+data "aws_availability_zones" "availability_zones" {
+  state = "available"
+}