feat: add HAProxy TLS Termination (#24)

---------

Co-authored-by: Stamatis Katsaounis <stamatis.katsaounis@canonical.com>

README.md

@@ -41,6 +41,12 @@ You will be asked for a `max_connections` during installation of the PostgreSQL
 - If number of MAAS region nodes is known beforehand, you can calculate the desired max_connections and set them, based on the formula: `max_connections = max(100, 10 + 50 * number_of_region_nodes)`.
 - If number of MAAS region nodes is not known, you can set `max_connections` to `dynamic` and let MAAS Anvil recalculate the appropriate PostgreSQL `max_connections` every time a region node is joining or leaving the Anvil cluster. **This options includes a database restart with every modification.**
 
+#### TLS Termination at HAProxy
+
+While deploying HAProxy, MAAS Anvil will ask you for filepaths pointing to an SSL certificate and private key. If passed, HAProxy will be configured to use the given certificate and key for TLS termination. To skip TLS configuration, enter nothing when prompted for the certificate and key files (this is the behavior if `--accept-defaults` is passed).
+
+Note that the certificate and key must be accessible by the `maas-anvil` snap; please make sure these files are in a directory that can be accessed, such as `$HOME/.config/anvil`.
+
 ### Add new nodes to the MAAS cluster
 
 ```bash
@@ -100,6 +106,7 @@ This allows passing a new manifest file with `--manifest` for updating configura
 ## Juju permission denied
 
 If you get an error message such as:
+
 ```bash
 please enter password for $node on anvil-controller:
 ```

anvil-python/anvil/commands/haproxy.py

@@ -15,10 +15,12 @@
 
 import ipaddress
 import logging
+import os.path
 from typing import Any, List
 
 from rich.console import Console
 from sunbeam.clusterd.client import Client
+from sunbeam.commands.juju import BOOTSTRAP_CONFIG_KEY
 from sunbeam.commands.terraform import TerraformInitStep
 from sunbeam.jobs import questions
 from sunbeam.jobs.common import BaseStep, ResultType
@@ -36,21 +38,37 @@
 APPLICATION = "haproxy"
 CONFIG_KEY = "TerraformVarsHaproxyPlan"
 HAPROXY_CONFIG_KEY = "TerraformVarsHaproxy"
-KEEPALIVED_CONFIG_KEY = "TerraformVarsKeepalived"
 HAPROXY_APP_TIMEOUT = 180  # 3 minutes, managing the application should be fast
 HAPROXY_UNIT_TIMEOUT = (
     1200  # 15 minutes, adding / removing units can take a long time
 )
+LOG = logging.getLogger(__name__)
 
 
-def keepalived_questions() -> dict[str, questions.PromptQuestion]:
-    return {
-        "virtual_ip": questions.PromptQuestion(
-            "Virtual IP to use for the Cluster in HA",
-            default_value="",
-            validation_function=validate_virtual_ip,
-        ),
-    }
+def validate_cert_file(filepath: str | None) -> None:
+    if filepath is None:
+        return
+    if not os.path.isfile(filepath):
+        raise ValueError(f"{filepath} does not exist")
+    try:
+        with open(filepath) as f:
+            if "BEGIN CERTIFICATE" not in f.read():
+                raise ValueError("Invalid certificate file")
+    except PermissionError:
+        raise ValueError(f"Permission denied when trying to read {filepath}")
+
+
+def validate_key_file(filepath: str | None) -> None:
+    if filepath is None:
+        return
+    if not os.path.isfile(filepath):
+        raise ValueError(f"{filepath} does not exist")
+    try:
+        with open(filepath) as f:
+            if "BEGIN PRIVATE KEY" not in f.read():
+                raise ValueError("Invalid key file")
+    except PermissionError:
+        raise ValueError(f"Permission denied when trying to read {filepath}")
 
 
 def validate_virtual_ip(value: str) -> str:
@@ -63,10 +81,30 @@ def validate_virtual_ip(value: str) -> str:
         raise ValueError(f"{value} is not a valid IP address: {e}")
 
 
+def haproxy_questions() -> dict[str, questions.PromptQuestion]:
+    return {
+        "virtual_ip": questions.PromptQuestion(
+            "Virtual IP to use for the Cluster in HA",
+            default_value="",
+            validation_function=validate_virtual_ip,
+        ),
+        "ssl_cert": questions.PromptQuestion(
+            "Path to SSL Certificate for HAProxy (enter nothing to skip TLS)",
+            default_value=None,
+            validation_function=validate_cert_file,
+        ),
+        "ssl_key": questions.PromptQuestion(
+            "Path to private key for the SSL certificate (enter nothing to skip TLS)",
+            default_value=None,
+            validation_function=validate_key_file,
+        ),
+    }
+
+
 class DeployHAProxyApplicationStep(DeployMachineApplicationStep):
     """Deploy HAProxy application using Terraform"""
 
-    _KEEPALIVED_CONFIG = KEEPALIVED_CONFIG_KEY
+    _HAPROXY_CONFIG = HAPROXY_CONFIG_KEY
 
     def __init__(
         self,
@@ -105,44 +143,100 @@ def has_prompts(self) -> bool:
         # No need to prompt for questions in case of refresh
         if self.refresh:
             return False
-
         skip_result = self.is_skip()
         if skip_result.result_type == ResultType.SKIPPED:
             return False
         else:
             return True
 
     def prompt(self, console: Console | None = None) -> None:
-        variables = questions.load_answers(
-            self.client, self._KEEPALIVED_CONFIG
-        )
+        variables = questions.load_answers(self.client, self._HAPROXY_CONFIG)
         variables.setdefault("virtual_ip", "")
+        variables.setdefault("ssl_cert", None)
+        variables.setdefault("ssl_key", None)
 
         # Set defaults
         self.preseed.setdefault("virtual_ip", "")
+        self.preseed.setdefault("ssl_cert", None)
+        self.preseed.setdefault("ssl_key", None)
 
-        keepalived_config_bank = questions.QuestionBank(
-            questions=keepalived_questions(),
+        haproxy_config_bank = questions.QuestionBank(
+            questions=haproxy_questions(),
             console=console,
             preseed=self.preseed.get("haproxy"),
             previous_answers=variables,
             accept_defaults=self.accept_defaults,
         )
 
-        variables["virtual_ip"] = keepalived_config_bank.virtual_ip.ask()
+        cert_filepath = haproxy_config_bank.ssl_cert.ask()
+        variables["ssl_cert"] = cert_filepath
+        key_filepath = haproxy_config_bank.ssl_key.ask()
+        variables["ssl_key"] = key_filepath
+        virtual_ip = haproxy_config_bank.virtual_ip.ask()
+        variables["virtual_ip"] = virtual_ip
 
         LOG.debug(variables)
-        questions.write_answers(
-            self.client, self._KEEPALIVED_CONFIG, variables
-        )
+        questions.write_answers(self.client, self._HAPROXY_CONFIG, variables)
 
     def extra_tfvars(self) -> dict[str, Any]:
         variables: dict[str, Any] = questions.load_answers(
-            self.client, self._KEEPALIVED_CONFIG
+            self.client, self._HAPROXY_CONFIG
         )
-        variables["haproxy_port"] = 80
+
+        cert_filepath = variables["ssl_cert"]
+        key_filepath = variables["ssl_key"]
+        if cert_filepath is not None and key_filepath is not None:
+            with open(cert_filepath) as cert_file:
+                variables["ssl_cert_content"] = cert_file.read()
+            with open(key_filepath) as key_file:
+                variables["ssl_key_content"] = key_file.read()
+            variables["haproxy_port"] = 443
+            variables["haproxy_services_yaml"] = self.get_tls_services_yaml()
+        else:
+            variables["haproxy_port"] = 80
+
+        # Terraform does not need the content of these answers
+        variables.pop("ssl_cert", None)
+        variables.pop("ssl_key", None)
+
+        LOG.debug(f"extra tfvars: {variables}")
         return variables
 
+    def get_management_cidrs(self) -> list[str]:
+        """Retrieve the Management CIDRs shared by hosts"""
+        answers: dict[str, dict[str, str]] = questions.load_answers(
+            self.client, BOOTSTRAP_CONFIG_KEY
+        )
+        return answers["bootstrap"]["management_cidr"].split(",")
+
+    def get_tls_services_yaml(self) -> str:
+        """Get the HAProxy services.yaml for TLS, inserting the VIP for the frontend bind"""
+        cidrs = self.get_management_cidrs()
+        services: str = (
+            """- service_name: haproxy_service
+  service_host: 0.0.0.0
+  service_port: 443
+  service_options:
+    - balance leastconn
+    - cookie SRVNAME insert
+    - http-request redirect scheme https unless { ssl_fc }
+  server_options: maxconn 100 cookie S{i} check
+  crts: [DEFAULT]
+- service_name: agent_service
+  service_host: 0.0.0.0
+  service_port: 80
+  service_options:
+    - balance leastconn
+    - cookie SRVNAME insert
+    - acl is-internal src """
+            + " ".join(cidrs)
+            + """
+    - http-request deny if !is-internal
+  server_options: maxconn 100 cookie S{i} check
+"""
+        )
+        return services
+
 
 class AddHAProxyUnitsStep(AddMachineUnitsStep):
     """Add HAProxy Unit."""

anvil-python/anvil/commands/maas_region.py

@@ -17,13 +17,15 @@
 
 from sunbeam.clusterd.client import Client
 from sunbeam.commands.terraform import TerraformInitStep
+from sunbeam.jobs import questions
 from sunbeam.jobs.common import BaseStep
 from sunbeam.jobs.juju import JujuHelper
 from sunbeam.jobs.steps import (
     AddMachineUnitsStep,
     DeployMachineApplicationStep,
 )
 
+from anvil.commands.haproxy import HAPROXY_CONFIG_KEY
 from anvil.jobs.manifest import Manifest
 from anvil.jobs.steps import RemoveMachineUnitStep
 
@@ -70,7 +72,13 @@ def extra_tfvars(self) -> dict[str, Any]:
             if self.client.cluster.list_nodes_by_role("haproxy")
             else False
         )
-        return {"enable_haproxy": enable_haproxy}
+        variables: dict[str, Any] = {"enable_haproxy": enable_haproxy}
+        haproxy_vars: dict[str, Any] = questions.load_answers(
+            self.client, HAPROXY_CONFIG_KEY
+        )
+        if enable_haproxy and "ssl_cert" in haproxy_vars:
+            variables["tls_mode"] = "termination"
+        return variables
 
 
 class AddMAASRegionUnitsStep(AddMachineUnitsStep):

anvil-python/anvil/provider/local/deployment.py

@@ -26,7 +26,7 @@
     LocalDeployment as SunbeamLocalDeployment,
 )
 
-from anvil.commands.haproxy import KEEPALIVED_CONFIG_KEY, keepalived_questions
+from anvil.commands.haproxy import HAPROXY_CONFIG_KEY, haproxy_questions
 from anvil.commands.postgresql import (
     POSTGRESQL_CONFIG_KEY,
     postgresql_questions,
@@ -73,11 +73,11 @@ def generate_preseed(self, console: Console) -> str:
 
         # HAProxy questions
         try:
-            variables = load_answers(client, KEEPALIVED_CONFIG_KEY)
+            variables = load_answers(client, HAPROXY_CONFIG_KEY)
         except ClusterServiceUnavailableException:
             variables = {}
         keepalived_config_bank = QuestionBank(
-            questions=keepalived_questions(),
+            questions=haproxy_questions(),
             console=console,
             previous_answers=variables,
         )

cloud/etc/deploy-haproxy/main.tf

@@ -32,6 +32,9 @@ data "juju_model" "machine_model" {
 
 locals {
   virtual_ip = var.virtual_ip != "" ? { virtual_ip = var.virtual_ip } : {}
+  services   = var.haproxy_services_yaml != "" ? { services = var.haproxy_services_yaml } : {}
+  ssl_cert   = var.ssl_cert_content != "" ? { ssl_cert = base64encode(var.ssl_cert_content) } : {}
+  ssl_key    = var.ssl_key_content != "" ? { ssl_key = base64encode(var.ssl_key_content) } : {}
 }
 
 resource "juju_application" "haproxy" {
@@ -46,7 +49,12 @@ resource "juju_application" "haproxy" {
     base     = "ubuntu@22.04"
   }
 
-  config = var.charm_haproxy_config
+  config = merge(
+    local.services,
+    local.ssl_cert,
+    local.ssl_key,
+    var.charm_haproxy_config,
+  )
 }
 
 resource "juju_application" "keepalived" {

cloud/etc/deploy-haproxy/variables.tf

@@ -60,6 +60,24 @@ variable "virtual_ip" {
   default     = ""
 }
 
+variable "haproxy_services_yaml" {
+  description = "yaml-formatted services definition for HAProxy charm"
+  type        = string
+  default     = ""
+}
+
+variable "ssl_cert_content" {
+  description = "base64 encoded SSL Certificate content for HAProxy charm"
+  type        = string
+  default     = ""
+}
+
+variable "ssl_key_content" {
+  description = "base64 encoded SSL Key content for HAProxy charm"
+  type        = string
+  default     = ""
+}
+
 variable "haproxy_port" {
   description = "The port that HAProxy listens on"
   type        = string

cloud/etc/deploy-maas-region/main.tf

@@ -30,6 +30,10 @@ data "juju_model" "machine_model" {
   name = var.machine_model
 }
 
+locals {
+  tls_mode = var.tls_mode != "" ? { tls_mode = var.tls_mode } : {}
+}
+
 resource "juju_application" "maas-region" {
   name  = "maas-region"
   model = data.juju_model.machine_model.name
@@ -42,7 +46,10 @@ resource "juju_application" "maas-region" {
     base     = "ubuntu@22.04"
   }
 
-  config = var.charm_maas_region_config
+  config = merge(
+    local.tls_mode,
+    var.charm_maas_region_config,
+  )
 }
 
 resource "juju_application" "pgbouncer" {

cloud/etc/deploy-maas-region/variables.tf

@@ -71,3 +71,9 @@ variable "max_connections_per_region" {
   type        = number
   default     = 50
 }
+
+variable "tls_mode" {
+  description = "TLS Mode for MAAS Region charm ('', 'termination', or 'passthrough')"
+  type        = string
+  default     = ""
+}