Add support for using nginx as a reverse proxy for prometheus_proxy (#85

)
pambrose · Nov 29, 2022 · b4f646d · b4f646d
1 parent 4d2d15b
commit b4f646d
Show file tree

Hide file tree

Showing 27 changed files with 206 additions and 64 deletions.
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-VERSION=1.14.2
+VERSION=1.15.0-b1
 
 default: versioncheck
 

diff --git a/README.md b/README.md
@@ -112,8 +112,8 @@ scrape_configs:
 The docker images are available via:
 
 ```bash
-docker pull pambrose/prometheus-proxy:1.14.2
-docker pull pambrose/prometheus-agent:1.14.2
+docker pull pambrose/prometheus-proxy:1.15.0-b1
+docker pull pambrose/prometheus-agent:1.15.0-b1
 ```
 
 Start a proxy container with:
@@ -122,15 +122,15 @@ Start a proxy container with:
 docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \
         --env ADMIN_ENABLED=true \
         --env METRICS_ENABLED=true \
-        pambrose/prometheus-proxy:1.14.2
+        pambrose/prometheus-proxy:1.15.0-b1
 ```
 
 Start an agent container with:
 
 ```bash
 docker run --rm -p 8083:8083 -p 8093:8093 \
         --env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
-        pambrose/prometheus-agent:1.14.2
+        pambrose/prometheus-agent:1.15.0-b1
 ```
 
 Using the config
@@ -148,7 +148,7 @@ is in your current directory, run an agent container with:
 docker run --rm -p 8083:8083 -p 8093:8093 \
     --mount type=bind,source="$(pwd)"/prom-agent.conf,target=/app/prom-agent.conf \
     --env AGENT_CONFIG=prom-agent.conf \
-    pambrose/prometheus-agent:1.14.2
+    pambrose/prometheus-agent:1.15.0-b1
 ```
 
 **Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure to use `/app` as the base directory in the
@@ -193,6 +193,7 @@ argument is an agent config value, which should have an `agent.pathConfigs` valu
 | --sd_enabled       | SD_ENABLED       <br> proxy.service.discovery.enabled                      | false                    | Service discovery endpoint enabled          |
 | --sd_path          | SD_PATH          <br> proxy.service.discovery.path                         | "discovery"              | Service discovery endpoint path             |
 | --sd_target_prefix | SD_TARGET_PREFIX <br> proxy.service.discovery.targetPrefix                 | "http://localhost:8080/" | Service discovery target prefix             |
+| --tf-disabled      | TRANSPORT_FILTER_DISABLED <br> proxy.transportFilterDisabled               | false                    | Transport filter disabled                   |
 | --cert, -t         | CERT_CHAIN_FILE_PATH <br> proxy.tls.certChainFilePath                      |                          | Certificate chain file path                 |
 | --key, -k          | PRIVATE_KEY_FILE_PATH <br> proxy.tls.privateKeyFilePath                    |                          | Private key file path                       |
 | --trust, -s        | TRUST_CERT_COLLECTION_FILE_PATH <br> proxy.tls.trustCertCollectionFilePath |                          | Trust certificate collection file path      |
@@ -217,6 +218,7 @@ argument is an agent config value, which should have an `agent.pathConfigs` valu
 | --max_retries      | SCRAPE_MAX_RETRIES <br> agent.scrapeMaxRetries                             | 0       | Scrape maximum retries (0 disables scrape retries)         |
 | --chunk            | CHUNK_CONTENT_SIZE_KBS <br> agent.chunkContentSizeKbs                      | 32      | Threshold for chunking data to Proxy and buffer size (KBs) |
 | --gzip             | MIN_GZIP_SIZE_BYTES <br> agent.minGzipSizeBytes                            | 1024    | Minimum size for content to be gzipped (bytes)             |
+| --tf-disabled      | TRANSPORT_FILTER_DISABLED <br> proxy.transportFilterDisabled               | false   | Transport filter disabled                                  |
 | --trust_all_x509   | TRUST_ALL_X509_CERTIFICATES <br> agent.http.enableTrustAllX509Certificates | false   | Disable SSL verification for agent https endpoints         |
 | --cert, -t         | CERT_CHAIN_FILE_PATH <br> agent.tls.certChainFilePath                      |         | Certificate chain file path                                |
 | --key, -k          | PRIVATE_KEY_FILE_PATH <br> agent.tls.privateKeyFilePath                    |         | Private key file path                                      |
@@ -294,15 +296,15 @@ docker run --rm -p 8082:8082 -p 8092:8092 -p 50440:50440 -p 8080:8080 \
     --env PROXY_CONFIG=tls-no-mutual-auth.conf \
     --env ADMIN_ENABLED=true \
     --env METRICS_ENABLED=true \
-    pambrose/prometheus-proxy:1.14.2
+    pambrose/prometheus-proxy:1.15.0-b1
 
 docker run --rm -p 8083:8083 -p 8093:8093 \
     --mount type=bind,source="$(pwd)"/testing/certs,target=/app/testing/certs \
     --mount type=bind,source="$(pwd)"/examples/tls-no-mutual-auth.conf,target=/app/tls-no-mutual-auth.conf \
     --env AGENT_CONFIG=tls-no-mutual-auth.conf \
     --env PROXY_HOSTNAME=mymachine.lan:50440 \
     --name docker-agent \
-    pambrose/prometheus-agent:1.14.2
+    pambrose/prometheus-agent:1.15.0-b1
 ```
 
 **Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure to use `/app` as the base directory in the
@@ -321,15 +323,26 @@ This enables using the existing service discovery features already built into Pr
 An example config can be found in
 [federate.conf](https://github.com/pambrose/prometheus-proxy/blob/master/examples/federate.conf).
 
+## Nginx Support
+
+To use the prometheus_proxy with nginx as a reverse proxy, disable the transport filter with the
+`TRANSPORT_FILTER_DISABLED` environment var, the `--tf-disabled` CLI option, or the `agent.transportFilterDisabled`/
+`proxy.transportFilterDisabled` properties. Agents and the Proxy must run with the same `transporFilterDisabled` value.
+
+When using `transporFilterDisabled`, you will not see agent contexts immediately removed
+from the proxy when agents are terminated. Instead, agent contexts will be removed from the proxy
+after they age out from inactivity. The maximum age is controlled by the `proxy.internal.maxAgentInactivitySecs` value.
+The default value is 1 minute.
+
+An example nginx conf file is [here](https://github.com/pambrose/prometheus-proxy/tree/master/nginx/docker/nginx.conf)
+and an example agent/proxy conf file
+is [here](https://github.com/pambrose/prometheus-proxy/tree/master/nginx/nginx-proxy.conf)
+
 ## Grafana
 
 [Grafana](https://grafana.com) dashboards for the proxy and agent
 are [here](https://github.com/pambrose/prometheus-proxy/tree/master/grafana).
 
-## Nginx Support
-
-Nginx does not work as a reverse proxy for prometheus-proxy.
-
 ## Related Links
 
 * [Prometheus.io](http://prometheus.io)

diff --git a/bin/docker-agent.sh b/bin/docker-agent.sh
@@ -3,4 +3,4 @@
 docker run --rm -p 8083:8083 -p 8093:8093 \
   --env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
   --env PROXY_HOSTNAME=mymachine.lan \
-  pambrose/prometheus-agent:1.14.2
+  pambrose/prometheus-agent:1.15.0-b1
diff --git a/bin/docker-proxy.sh b/bin/docker-proxy.sh
@@ -2,4 +2,4 @@
 
 docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \
   --env PROXY_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
-  pambrose/prometheus-proxy:1.14.2
+  pambrose/prometheus-proxy:1.15.0-b1
diff --git a/build.gradle b/build.gradle
@@ -15,13 +15,20 @@ plugins {
 }
 
 group = 'io.prometheus'
-version = '1.14.2'
+version = '1.15.0-b1'
 
 sourceCompatibility = 11
 targetCompatibility = 11
 
+buildConfig {
+    packageName("io.prometheus")
+
+    buildConfigField('String', 'APP_NAME', "\"${project.name}\"")
+    buildConfigField('String', 'APP_VERSION', "\"${project.version}\"")
+    buildConfigField('String', 'APP_RELEASE_DATE', "\"11/29/22\"")
+}
+
 repositories {
-    //maven { url "https://maven.pkg.jetbrains.space/public/p/ktor/eap" }
     google()
     mavenCentral()
     maven { url = 'https://jitpack.io' }
@@ -85,14 +92,6 @@ dependencies {
     testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit_version"
 }
 
-buildConfig {
-    packageName("io.prometheus")
-
-    buildConfigField('String', 'APP_NAME', "\"${project.name}\"")
-    buildConfigField('String', 'APP_VERSION', "\"${project.version}\"")
-    buildConfigField('String', 'APP_RELEASE_DATE', "\"11/26/22\"")
-}
-
 publishing {
     publications {
         mavenJava(MavenPublication) {

diff --git a/etc/compose/proxy.yml b/etc/compose/proxy.yml
@@ -1,6 +1,6 @@
 prometheus-proxy:
   autoredeploy: true
-  image: 'pambrose/prometheus-proxy:1.14.2'
+  image: 'pambrose/prometheus-proxy:1.15.0-b1'
   ports:
     - '8080:8080'
     - '8082:8082'

diff --git a/etc/config/config.conf b/etc/config/config.conf
@@ -3,6 +3,8 @@ proxy {
 
   agent.port = 50051                              // Listen port for agent connections
 
+  transportFilterDisabled = false                 // Assign to true if using nginx as a reverse proxy
+
   service.discovery {
     enabled = false                               // Enable service discovery
     path = "discovery"                            // Service discovery path
@@ -83,6 +85,8 @@ proxy {
 agent {
   name = ""                                       // Agent name used in metrics reporting
 
+  transportFilterDisabled = false                 // Assign to true if using nginx as a reverse proxy
+
   consolidated = false
 
   // See: https://github.com/grpc/grpc.github.io/issues/371

diff --git a/nginx/docker/Dockerfile b/nginx/docker/Dockerfile
@@ -0,0 +1,4 @@
+FROM nginx
+COPY ./nginx.conf /etc/nginx/conf.d/default.conf
+
+EXPOSE 50440
diff --git a/nginx/docker/nginx.conf b/nginx/docker/nginx.conf
@@ -0,0 +1,16 @@
+server {
+  # 50440 is the agent.proxy.port value
+  listen 50440 http2;
+
+  # Prevent nginx from closing the gRPC connections (not working for me)
+  # https://stackoverflow.com/questions/67430437/grpc-send-timeout-doesnt-work-nginx-closes-grpc-streams-unexpectedly
+  client_header_timeout 1d;
+  client_body_timeout 1d;
+
+  location / {
+    # The nginx gRPX options: https://nginx.org/en/docs/http/ngx_http_grpc_module.html
+    # 50051 is the proxy.agent.port value
+    grpc_pass grpc://alta.lan:50051;
+      grpc_socket_keepalive on;
+  }
+}
diff --git a/nginx/docker/run.sh b/nginx/docker/run.sh
@@ -0,0 +1,2 @@
+docker build -t pambrose/nginx2 .
+docker run --rm -p 50440:50440 pambrose/nginx2
diff --git a/nginx/nginx-proxy.conf b/nginx/nginx-proxy.conf
@@ -0,0 +1,32 @@
+proxy {
+  # Required for use with nginx reverse proxy
+  transportFilterDisabled = true
+}
+
+agent {
+  # Required for use with nginx reverse proxy
+  transportFilterDisabled = true
+
+  proxy {
+    # nginx http2 port specified in nginx.conf
+    port = 50440
+  }
+
+  pathConfigs: [
+    {
+      name: "App1 metrics"
+      path: app1_metrics
+      url: "http://localhost:8082/metrics"
+    },
+    {
+      name: "App2 metrics"
+      path: app2_metrics
+      url: "http://app2.local:9100/metrics"
+    },
+    {
+      name: "App3 metrics"
+      path: app3_metrics
+      url: "http://app3.local:9100/metrics"
+    }
+  ]
+}
diff --git a/src/main/java/io/prometheus/common/ConfigVals.java b/src/main/java/io/prometheus/common/ConfigVals.java
@@ -1,4 +1,4 @@
-// generated by tscfg 0.9.997 on Thu Oct 13 20:14:33 EDT 2022
+// generated by tscfg 0.9.997 on Mon Nov 28 23:31:57 PST 2022
 // source: etc/config/config.conf
 
 package io.prometheus.common;
@@ -28,6 +28,7 @@ public static class Agent {
     public final int scrapeMaxRetries;
     public final int scrapeTimeoutSecs;
     public final Agent.Tls tls;
+    public final boolean transportFilterDisabled;
     public Agent(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
       this.admin = c.hasPathOrNull("admin") ? new Agent.Admin(c.getConfig("admin"), parentPath + "admin.", $tsCfgValidator) : new Agent.Admin(com.typesafe.config.ConfigFactory.parseString("admin{}"), parentPath + "admin.", $tsCfgValidator);
       this.chunkContentSizeKbs = c.hasPathOrNull("chunkContentSizeKbs") ? c.getInt("chunkContentSizeKbs") : 32;
@@ -42,6 +43,7 @@ public Agent(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgVa
       this.scrapeMaxRetries = c.hasPathOrNull("scrapeMaxRetries") ? c.getInt("scrapeMaxRetries") : 0;
       this.scrapeTimeoutSecs = c.hasPathOrNull("scrapeTimeoutSecs") ? c.getInt("scrapeTimeoutSecs") : 15;
       this.tls = c.hasPathOrNull("tls") ? new Agent.Tls(c.getConfig("tls"), parentPath + "tls.", $tsCfgValidator) : new Agent.Tls(com.typesafe.config.ConfigFactory.parseString("tls{}"), parentPath + "tls.", $tsCfgValidator);
+      this.transportFilterDisabled = c.hasPathOrNull("transportFilterDisabled") && c.getBoolean("transportFilterDisabled");
     }
 
     private static java.util.List<Agent.PathConfigs$Elm> $_LAgent_PathConfigs$Elm(com.typesafe.config.ConfigList cl, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
@@ -88,7 +90,6 @@ public static class Internal {
       public final int reconnectPauseSecs;
       public final int scrapeRequestBacklogUnhealthySize;
       public final Internal.Zipkin zipkin;
-
       public Internal(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
         this.cioTimeoutSecs = c.hasPathOrNull("cioTimeoutSecs") ? c.getInt("cioTimeoutSecs") : 90;
         this.heartbeatCheckPauseMillis = c.hasPathOrNull("heartbeatCheckPauseMillis") ? c.getInt("heartbeatCheckPauseMillis") : 500;
@@ -208,6 +209,7 @@ public static class Proxy2 {
     public final Proxy2.Metrics2 metrics;
     public final Proxy2.Service service;
     public final Proxy2.Tls2 tls;
+    public final boolean transportFilterDisabled;
     public Proxy2(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
       this.admin = c.hasPathOrNull("admin") ? new Proxy2.Admin2(c.getConfig("admin"), parentPath + "admin.", $tsCfgValidator) : new Proxy2.Admin2(com.typesafe.config.ConfigFactory.parseString("admin{}"), parentPath + "admin.", $tsCfgValidator);
       this.agent = c.hasPathOrNull("agent") ? new Proxy2.Agent2(c.getConfig("agent"), parentPath + "agent.", $tsCfgValidator) : new Proxy2.Agent2(com.typesafe.config.ConfigFactory.parseString("agent{}"), parentPath + "agent.", $tsCfgValidator);
@@ -216,6 +218,7 @@ public Proxy2(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgV
       this.metrics = c.hasPathOrNull("metrics") ? new Proxy2.Metrics2(c.getConfig("metrics"), parentPath + "metrics.", $tsCfgValidator) : new Proxy2.Metrics2(com.typesafe.config.ConfigFactory.parseString("metrics{}"), parentPath + "metrics.", $tsCfgValidator);
       this.service = c.hasPathOrNull("service") ? new Proxy2.Service(c.getConfig("service"), parentPath + "service.", $tsCfgValidator) : new Proxy2.Service(com.typesafe.config.ConfigFactory.parseString("service{}"), parentPath + "service.", $tsCfgValidator);
       this.tls = c.hasPathOrNull("tls") ? new Proxy2.Tls2(c.getConfig("tls"), parentPath + "tls.", $tsCfgValidator) : new Proxy2.Tls2(com.typesafe.config.ConfigFactory.parseString("tls{}"), parentPath + "tls.", $tsCfgValidator);
+      this.transportFilterDisabled = c.hasPathOrNull("transportFilterDisabled") && c.getBoolean("transportFilterDisabled");
     }
 
     public static class Admin2 {

diff --git a/src/main/kotlin/io/prometheus/Agent.kt b/src/main/kotlin/io/prometheus/Agent.kt
@@ -153,7 +153,7 @@ class Agent(
       scrapeRequestBacklogSize.set(0)
       lastMsgSentMark = clock.markNow()
 
-      if (grpcService.connectAgent()) {
+      if (grpcService.connectAgent(configVals.agent.transportFilterDisabled)) {
         grpcService.registerAgent(initialConnectionLatch)
         pathManager.registerPaths()
 

diff --git a/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt b/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt
@@ -25,7 +25,7 @@ import com.github.pambrose.common.util.simpleClassName
 import com.github.pambrose.common.utils.TlsContext
 import com.github.pambrose.common.utils.TlsContext.Companion.PLAINTEXT_CONTEXT
 import com.github.pambrose.common.utils.TlsUtils.buildClientTlsContext
-import com.google.protobuf.Empty
+import io.grpc.ClientInterceptor
 import io.grpc.ClientInterceptors
 import io.grpc.ManagedChannel
 import io.grpc.Status
@@ -34,6 +34,7 @@ import io.prometheus.Agent
 import io.prometheus.common.BaseOptions.Companion.HTTPS_PREFIX
 import io.prometheus.common.BaseOptions.Companion.HTTP_PREFIX
 import io.prometheus.common.GrpcObjects
+import io.prometheus.common.GrpcObjects.EMPTY_INSTANCE
 import io.prometheus.common.GrpcObjects.newAgentInfo
 import io.prometheus.common.GrpcObjects.newRegisterAgentRequest
 import io.prometheus.common.GrpcObjects.newScrapeResponseChunk
@@ -143,15 +144,23 @@ internal class AgentGrpcService(
           intercept(grpcTracing.newClientInterceptor())
       }
 
-    val interceptors = listOf(AgentClientInterceptor(agent))
+    val interceptors =
+      buildList<ClientInterceptor> {
+        if (!options.transportFilterDisabled)
+          add(AgentClientInterceptor(agent))
+      }
     stub = ProxyServiceGrpcKt.ProxyServiceCoroutineStub(ClientInterceptors.intercept(channel, interceptors))
   }
 
   // If successful, this will create an agentContext on the Proxy and an interceptor will add an agent_id to the headers`
-  suspend fun connectAgent() =
+  suspend fun connectAgent(transportFilterDisabled: Boolean) =
     try {
       logger.info { "Connecting to proxy at ${agent.proxyHost} using ${tlsContext.desc()}..." }
-      stub.connectAgent(Empty.getDefaultInstance())
+      if (transportFilterDisabled)
+        stub.connectAgentWithTransportFilterDisabled(EMPTY_INSTANCE).also { agent.agentId = it.agentId }
+      else
+        stub.connectAgent(EMPTY_INSTANCE)
+
       logger.info { "Connected to proxy at ${agent.proxyHost} using ${tlsContext.desc()}" }
       agent.metrics { connectCount.labels(agent.launchId, "success").inc() }
       true

diff --git a/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt b/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt
@@ -33,7 +33,6 @@ import io.ktor.client.plugins.auth.providers.*
 import io.ktor.client.request.*
 import io.ktor.client.statement.*
 import io.ktor.http.*
-import io.ktor.http.HttpStatusCode.Companion.NotFound
 import io.ktor.network.sockets.*
 import io.prometheus.Agent
 import io.prometheus.common.ScrapeResults
@@ -109,7 +108,7 @@ internal class AgentHttpService(val agent: Agent) {
                 else {
                   retryOnException(maxRetries)
                   retryIf(maxRetries) { _, response ->
-                    !response.status.isSuccess() && response.status != NotFound
+                    !response.status.isSuccess() && response.status != HttpStatusCode.NotFound
                   }
                   modifyRequest { it.headers.append("x-retry-count", retryCount.toString()) }
                   exponentialDelay()

diff --git a/src/main/kotlin/io/prometheus/agent/AgentOptions.kt b/src/main/kotlin/io/prometheus/agent/AgentOptions.kt
@@ -128,14 +128,17 @@ class AgentOptions(argv: Array<String>, exitOnMissingConfig: Boolean) :
         assignAdminPort(agentConfigVals.admin.port)
         assignMetricsEnabled(agentConfigVals.metrics.enabled)
         assignMetricsPort(agentConfigVals.metrics.port)
+        assignTransportFilterDisabled(agentConfigVals.transportFilterDisabled)
         assignDebugEnabled(agentConfigVals.admin.debugEnabled)
 
         assignCertChainFilePath(agentConfigVals.tls.certChainFilePath)
         assignPrivateKeyFilePath(agentConfigVals.tls.privateKeyFilePath)
         assignTrustCertCollectionFilePath(agentConfigVals.tls.trustCertCollectionFilePath)
 
-        logger.info { "agent.internal.cioTimeoutSecs: ${agentConfigVals.internal.cioTimeoutSecs.seconds}" }
         logger.info { "agent.scrapeTimeoutSecs: ${agentConfigVals.scrapeTimeoutSecs.seconds}" }
+        logger.info { "agent.internal.cioTimeoutSecs: ${agentConfigVals.internal.cioTimeoutSecs.seconds}" }
+        logger.info { "agent.internal.heartbeatCheckPauseMillis: ${agentConfigVals.internal.heartbeatCheckPauseMillis}" }
+        logger.info { "agent.internal.heartbeatMaxInactivitySecs: ${agentConfigVals.internal.heartbeatMaxInactivitySecs}" }
       }
   }