Consul connect error when bootstrapping Envoy config -> TLS error: 268436526:SSL routines:OPENSSL_internal:TLSV1_ALERT_PROTOCOL_VERSION

[joel-vaz]

Hello,

I’m trying to update my service mesh (Consul - Envoy) to use TLS minimum version 1.3 on my cluster, updating from version 1.2.

Consul Version: 1.16.6
Envoy Version: 1.26.8

I confirmed that both the Consul server and Consul agent are correctly configured to use the minimum version of TLS 1.3, but the Envoy proxy that I use as a sidecar for my services is in an unhealthy status with the log:

DeltaAggregatedResources gRPC config stream to local_agent closed since 97s ago: 14, upstream connect error or disconnect/reset before headers. reset reason: connection failure, transport failure reason: TLS error: 268436526:SSL routines:OPENSSL_internal:TLSV1_ALERT_PROTOCOL_VERSION
Consul Agent Configuration:

{
  "acl": {
    "enabled": true,
    "down_policy": "async-cache",
    "default_policy": "deny",
    "tokens": {
      "default": ""
    }
  },
  "enable_central_service_config": false,
  "datacenter": "",
  "encrypt": "",
  "encrypt_verify_incoming": true,
  "encrypt_verify_outgoing": true,
  "server": false,
  "log_level": "INFO",
  "advertise_addr": "",
  "bind_addr": "0.0.0.0",
  "client_addr": "0.0.0.0",
  "data_dir": "/consul/data",
  "retry_join": [
    ""
  ],
  "auto_encrypt": {
    "tls": true,
    "ip_san": [
      ""
    ]
  },
  "tls": {  
    "defaults": {
      "ca_file": "/consul/ca.pem",
      "verify_outgoing": true,
      "verify_incoming": false,
      "tls_min_version": "TLSv1_3"
    },
    "internal_rpc": {
      "verify_server_hostname": true
    }
  },
  "leave_on_terminate": true,
  "ports": {
    "https": 8501,
    "http": -1,
    "grpc": 8502,
    "grpc_tls": 8503
  },
  "domain": "consul",
  "node_meta": {
    "env": "",
    "version": ""
  }
}
Envoy Service Configuration:

{
  "service": {
    "name": "",
    "id": "",
    "token": "",
    "address": "",
    "port": 0,
    "meta": {
      "env": "",
      "version": ""
    },
    "check": {
      "deregister_critical_service_after": "30m",
      "http": "",
      "method": "GET",
      "interval": "",
      "timeout": ""
    },
    "connect": {
      "sidecar_service": {
        "port": 21000,
        "checks": [
          {
            "name": "Connect Envoy Sidecar",
            "tcp": "",
            "interval": "10s"
          },
          {
            "id": "",
            "alias_service": ""
          }
        ],
        "proxy": {
          "config": {
            "envoy_stats_bind_addr": "0.0.0.0:19001",
            "envoy_tracing_json": "{\"http\":{\"name\":\"envoy.tracers.datadog\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.config.trace.v3.DatadogConfig\",\"collector_cluster\":\"datadog_8126\",\"service_name\":\"%NAME%\"}}}",
            "envoy_extra_static_clusters_json": "{\"connect_timeout\":\"3.000s\",\"dns_lookup_family\":\"V4_ONLY\",\"lb_policy\":\"ROUND_ROBIN\",\"load_assignment\":{\"cluster_name\":\"datadog_8126\",\"endpoints\":[{\"lb_endpoints\":[{\"endpoint\":{\"address\":{\"socket_address\":{\"address\":\"%ADDRESS%\",\"port_value\":8126,\"protocol\":\"TCP\"}}}}]}]},\"name\":\"datadog_8126\",\"type\":\"STRICT_DNS\"}"
          },
          "upstreams": []
        }
      }
    }
  }
}

Can I get some help on this issue, please? Did anyone go through the same? 🙏

Additional information, I use dockerfile entrypoint script to generate the service file for my proxy envoy and consul connect envoy command to bootstrap it.

set_proxy_configuration()
{
  ## Env variables code
  ##

  base_renderers=$(jq '.service.connect.sidecar_service.proxy.upstreams = '"${CONSUL_SERVICE_UPSTREAMS}"' |
      .service.name = "'${SERVICE_NAME}'" |
      .service.id = "'${SERVICE_ID}'" |
      .service.token = "'${CONSUL_HTTP_TOKEN}'" |
      .service.address = "'${CONTAINER_IP}'" |
      .service.port = '${SERVICE_PORT}' |
      .service.meta.env = "'${DD_ENV}'" |
      .service.meta.version = "'${DD_VERSION}'" |
      .service.connect.sidecar_service.port = '${SIDECAR_PORT}' |
      .service.check.http = "'${SERVICE_HEALTH_CHECK}'" |
      .service.check.interval = "'${SERVICE_HEALTH_CHECK_INTERVAL}'" |
      .service.check.timeout = "'${SERVICE_HEALTH_CHECK_TIMEOUT}'" |
      .service.connect.sidecar_service.checks[0].tcp = "'${SIDECAR_HEALTH_CHECK}'" |
      .service.connect.sidecar_service.checks[1].id = "'${SERVICE_ID}'-alias" |
      .service.connect.sidecar_service.checks[1].alias_service = "'${SERVICE_ID}'" |
      .service.connect.sidecar_service.proxy.config.envoy_tracing_json |=gsub("%NAME%";"'$DD_SERVICE'") |
      .service.connect.sidecar_service.proxy.config.envoy_extra_static_clusters_json |= gsub("%ADDRESS%"; "'$EC2_HOST_ADDRESS'") |
      .service.connect.sidecar_service.proxy.config.common_tls_context.tls_params.tls_minimum_protocol_version = "TLSv1_3"' ./service_config.json)

echo "Base Renderers configuration: $base_renderers"

  # Wait until Consul can be contacted
  until curl -s -k ${CONSUL_HTTP_ADDR}/v1/status/leader | grep ***; do
    echo "Waiting for Consul to start at ${CONSUL_HTTP_ADDR}."
    sleep 1
  done

  echo "Registering service with consul ${SERVICE_CONFIG_FILE}."
  consul services register ${SERVICE_CONFIG_FILE}

  consul connect envoy -sidecar-for=${SERVICE_ID} -grpc-ca-file=${CONSUL_CACERT} $ENVOY_DEBUG &
}

Config_Dump snippet:

     "clusters": [
      {
       "name": "local_agent",
       "type": "STATIC",
       "connect_timeout": "1s",
       "transport_socket": {
        "name": "tls",
        "typed_config": {
         "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
         "common_tls_context": {
          "validation_context": {
           "trusted_ca": {
            "inline_string": "-----BEGIN CERTIFICATE-----<value>-----END CERTIFICATE-----\n"
           }
          }
         }
        }
       },
       "load_assignment": {
        "cluster_name": "local_agent",
        "endpoints": [
         {
          "lb_endpoints": [
           {
            "endpoint": {
             "address": {
              "socket_address": {
               "address": "<value>",
               "port_value": <value>
              }
             }
            }
           }
          ]
         }
        ]
       },
       "typed_extension_protocol_options": {
        "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
         "explicit_http_config": {
          "http2_protocol_options": {}
         }
        }
       }
      },

Additional info: issue on the Envoy repository: envoyproxy/envoy#36181

Kind Regards,
Joel Vaz