From b4f646da20f0449e451fc75756c9c12a47a66ea0 Mon Sep 17 00:00:00 2001 From: Paul Ambrose Date: Mon, 28 Nov 2022 23:36:08 -0800 Subject: [PATCH] Add support for using nginx as a reverse proxy for prometheus_proxy (#85) --- Makefile | 2 +- README.md | 35 +++++++++++------ bin/docker-agent.sh | 2 +- bin/docker-proxy.sh | 2 +- build.gradle | 19 +++++----- etc/compose/proxy.yml | 2 +- etc/config/config.conf | 4 ++ nginx/docker/Dockerfile | 4 ++ nginx/docker/nginx.conf | 16 ++++++++ nginx/docker/run.sh | 2 + nginx/nginx-proxy.conf | 32 ++++++++++++++++ .../java/io/prometheus/common/ConfigVals.java | 7 +++- src/main/kotlin/io/prometheus/Agent.kt | 2 +- .../io/prometheus/agent/AgentGrpcService.kt | 17 +++++++-- .../io/prometheus/agent/AgentHttpService.kt | 3 +- .../io/prometheus/agent/AgentOptions.kt | 5 ++- .../io/prometheus/common/BaseOptions.kt | 27 +++++++++---- .../kotlin/io/prometheus/common/EnvVars.kt | 1 + .../io/prometheus/common/GrpcObjects.kt | 3 ++ .../proxy/AgentContextCleanupService.kt | 2 +- .../prometheus/proxy/AgentContextManager.kt | 4 +- .../io/prometheus/proxy/ProxyGrpcService.kt | 15 ++++++-- .../io/prometheus/proxy/ProxyHttpConfig.kt | 14 +++---- .../io/prometheus/proxy/ProxyOptions.kt | 6 ++- .../proxy/ProxyServerTransportFilter.kt | 3 +- .../io/prometheus/proxy/ProxyServiceImpl.kt | 38 +++++++++++++++++-- src/main/proto/proxy_service.proto | 3 ++ 27 files changed, 206 insertions(+), 64 deletions(-) create mode 100644 nginx/docker/Dockerfile create mode 100644 nginx/docker/nginx.conf create mode 100755 nginx/docker/run.sh create mode 100644 nginx/nginx-proxy.conf diff --git a/Makefile b/Makefile index 25af2978..26b56327 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VERSION=1.14.2 +VERSION=1.15.0-b1 default: versioncheck diff --git a/README.md b/README.md index 6dbd8e46..2fc9f2a5 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,8 @@ scrape_configs: The docker images are available via: ```bash -docker pull pambrose/prometheus-proxy:1.14.2 -docker pull pambrose/prometheus-agent:1.14.2 +docker pull pambrose/prometheus-proxy:1.15.0-b1 +docker pull pambrose/prometheus-agent:1.15.0-b1 ``` Start a proxy container with: @@ -122,7 +122,7 @@ Start a proxy container with: docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \ --env ADMIN_ENABLED=true \ --env METRICS_ENABLED=true \ - pambrose/prometheus-proxy:1.14.2 + pambrose/prometheus-proxy:1.15.0-b1 ``` Start an agent container with: @@ -130,7 +130,7 @@ Start an agent container with: ```bash docker run --rm -p 8083:8083 -p 8093:8093 \ --env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \ - pambrose/prometheus-agent:1.14.2 + pambrose/prometheus-agent:1.15.0-b1 ``` Using the config @@ -148,7 +148,7 @@ is in your current directory, run an agent container with: docker run --rm -p 8083:8083 -p 8093:8093 \ --mount type=bind,source="$(pwd)"/prom-agent.conf,target=/app/prom-agent.conf \ --env AGENT_CONFIG=prom-agent.conf \ - pambrose/prometheus-agent:1.14.2 + pambrose/prometheus-agent:1.15.0-b1 ``` **Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure to use `/app` as the base directory in the @@ -193,6 +193,7 @@ argument is an agent config value, which should have an `agent.pathConfigs` valu | --sd_enabled | SD_ENABLED
proxy.service.discovery.enabled | false | Service discovery endpoint enabled | | --sd_path | SD_PATH
proxy.service.discovery.path | "discovery" | Service discovery endpoint path | | --sd_target_prefix | SD_TARGET_PREFIX
proxy.service.discovery.targetPrefix | "http://localhost:8080/" | Service discovery target prefix | +| --tf-disabled | TRANSPORT_FILTER_DISABLED
proxy.transportFilterDisabled | false | Transport filter disabled | | --cert, -t | CERT_CHAIN_FILE_PATH
proxy.tls.certChainFilePath | | Certificate chain file path | | --key, -k | PRIVATE_KEY_FILE_PATH
proxy.tls.privateKeyFilePath | | Private key file path | | --trust, -s | TRUST_CERT_COLLECTION_FILE_PATH
proxy.tls.trustCertCollectionFilePath | | Trust certificate collection file path | @@ -217,6 +218,7 @@ argument is an agent config value, which should have an `agent.pathConfigs` valu | --max_retries | SCRAPE_MAX_RETRIES
agent.scrapeMaxRetries | 0 | Scrape maximum retries (0 disables scrape retries) | | --chunk | CHUNK_CONTENT_SIZE_KBS
agent.chunkContentSizeKbs | 32 | Threshold for chunking data to Proxy and buffer size (KBs) | | --gzip | MIN_GZIP_SIZE_BYTES
agent.minGzipSizeBytes | 1024 | Minimum size for content to be gzipped (bytes) | +| --tf-disabled | TRANSPORT_FILTER_DISABLED
proxy.transportFilterDisabled | false | Transport filter disabled | | --trust_all_x509 | TRUST_ALL_X509_CERTIFICATES
agent.http.enableTrustAllX509Certificates | false | Disable SSL verification for agent https endpoints | | --cert, -t | CERT_CHAIN_FILE_PATH
agent.tls.certChainFilePath | | Certificate chain file path | | --key, -k | PRIVATE_KEY_FILE_PATH
agent.tls.privateKeyFilePath | | Private key file path | @@ -294,7 +296,7 @@ docker run --rm -p 8082:8082 -p 8092:8092 -p 50440:50440 -p 8080:8080 \ --env PROXY_CONFIG=tls-no-mutual-auth.conf \ --env ADMIN_ENABLED=true \ --env METRICS_ENABLED=true \ - pambrose/prometheus-proxy:1.14.2 + pambrose/prometheus-proxy:1.15.0-b1 docker run --rm -p 8083:8083 -p 8093:8093 \ --mount type=bind,source="$(pwd)"/testing/certs,target=/app/testing/certs \ @@ -302,7 +304,7 @@ docker run --rm -p 8083:8083 -p 8093:8093 \ --env AGENT_CONFIG=tls-no-mutual-auth.conf \ --env PROXY_HOSTNAME=mymachine.lan:50440 \ --name docker-agent \ - pambrose/prometheus-agent:1.14.2 + pambrose/prometheus-agent:1.15.0-b1 ``` **Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure to use `/app` as the base directory in the @@ -321,15 +323,26 @@ This enables using the existing service discovery features already built into Pr An example config can be found in [federate.conf](https://github.com/pambrose/prometheus-proxy/blob/master/examples/federate.conf). +## Nginx Support + +To use the prometheus_proxy with nginx as a reverse proxy, disable the transport filter with the +`TRANSPORT_FILTER_DISABLED` environment var, the `--tf-disabled` CLI option, or the `agent.transportFilterDisabled`/ +`proxy.transportFilterDisabled` properties. Agents and the Proxy must run with the same `transporFilterDisabled` value. + +When using `transporFilterDisabled`, you will not see agent contexts immediately removed +from the proxy when agents are terminated. Instead, agent contexts will be removed from the proxy +after they age out from inactivity. The maximum age is controlled by the `proxy.internal.maxAgentInactivitySecs` value. +The default value is 1 minute. + +An example nginx conf file is [here](https://github.com/pambrose/prometheus-proxy/tree/master/nginx/docker/nginx.conf) +and an example agent/proxy conf file +is [here](https://github.com/pambrose/prometheus-proxy/tree/master/nginx/nginx-proxy.conf) + ## Grafana [Grafana](https://grafana.com) dashboards for the proxy and agent are [here](https://github.com/pambrose/prometheus-proxy/tree/master/grafana). -## Nginx Support - -Nginx does not work as a reverse proxy for prometheus-proxy. - ## Related Links * [Prometheus.io](http://prometheus.io) diff --git a/bin/docker-agent.sh b/bin/docker-agent.sh index 8d7cd131..cbb01de5 100755 --- a/bin/docker-agent.sh +++ b/bin/docker-agent.sh @@ -3,4 +3,4 @@ docker run --rm -p 8083:8083 -p 8093:8093 \ --env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \ --env PROXY_HOSTNAME=mymachine.lan \ - pambrose/prometheus-agent:1.14.2 + pambrose/prometheus-agent:1.15.0-b1 diff --git a/bin/docker-proxy.sh b/bin/docker-proxy.sh index b884ec6d..e245f8ce 100755 --- a/bin/docker-proxy.sh +++ b/bin/docker-proxy.sh @@ -2,4 +2,4 @@ docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \ --env PROXY_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \ - pambrose/prometheus-proxy:1.14.2 + pambrose/prometheus-proxy:1.15.0-b1 diff --git a/build.gradle b/build.gradle index 3da8bc0a..298939ef 100644 --- a/build.gradle +++ b/build.gradle @@ -15,13 +15,20 @@ plugins { } group = 'io.prometheus' -version = '1.14.2' +version = '1.15.0-b1' sourceCompatibility = 11 targetCompatibility = 11 +buildConfig { + packageName("io.prometheus") + + buildConfigField('String', 'APP_NAME', "\"${project.name}\"") + buildConfigField('String', 'APP_VERSION', "\"${project.version}\"") + buildConfigField('String', 'APP_RELEASE_DATE', "\"11/29/22\"") +} + repositories { - //maven { url "https://maven.pkg.jetbrains.space/public/p/ktor/eap" } google() mavenCentral() maven { url = 'https://jitpack.io' } @@ -85,14 +92,6 @@ dependencies { testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit_version" } -buildConfig { - packageName("io.prometheus") - - buildConfigField('String', 'APP_NAME', "\"${project.name}\"") - buildConfigField('String', 'APP_VERSION', "\"${project.version}\"") - buildConfigField('String', 'APP_RELEASE_DATE', "\"11/26/22\"") -} - publishing { publications { mavenJava(MavenPublication) { diff --git a/etc/compose/proxy.yml b/etc/compose/proxy.yml index 6da760f0..d5e226d7 100644 --- a/etc/compose/proxy.yml +++ b/etc/compose/proxy.yml @@ -1,6 +1,6 @@ prometheus-proxy: autoredeploy: true - image: 'pambrose/prometheus-proxy:1.14.2' + image: 'pambrose/prometheus-proxy:1.15.0-b1' ports: - '8080:8080' - '8082:8082' diff --git a/etc/config/config.conf b/etc/config/config.conf index f695122b..0c0f5952 100644 --- a/etc/config/config.conf +++ b/etc/config/config.conf @@ -3,6 +3,8 @@ proxy { agent.port = 50051 // Listen port for agent connections + transportFilterDisabled = false // Assign to true if using nginx as a reverse proxy + service.discovery { enabled = false // Enable service discovery path = "discovery" // Service discovery path @@ -83,6 +85,8 @@ proxy { agent { name = "" // Agent name used in metrics reporting + transportFilterDisabled = false // Assign to true if using nginx as a reverse proxy + consolidated = false // See: https://github.com/grpc/grpc.github.io/issues/371 diff --git a/nginx/docker/Dockerfile b/nginx/docker/Dockerfile new file mode 100644 index 00000000..6619805f --- /dev/null +++ b/nginx/docker/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx +COPY ./nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 50440 diff --git a/nginx/docker/nginx.conf b/nginx/docker/nginx.conf new file mode 100644 index 00000000..3c57744a --- /dev/null +++ b/nginx/docker/nginx.conf @@ -0,0 +1,16 @@ +server { + # 50440 is the agent.proxy.port value + listen 50440 http2; + + # Prevent nginx from closing the gRPC connections (not working for me) + # https://stackoverflow.com/questions/67430437/grpc-send-timeout-doesnt-work-nginx-closes-grpc-streams-unexpectedly + client_header_timeout 1d; + client_body_timeout 1d; + + location / { + # The nginx gRPX options: https://nginx.org/en/docs/http/ngx_http_grpc_module.html + # 50051 is the proxy.agent.port value + grpc_pass grpc://alta.lan:50051; + grpc_socket_keepalive on; + } +} diff --git a/nginx/docker/run.sh b/nginx/docker/run.sh new file mode 100755 index 00000000..95dc9976 --- /dev/null +++ b/nginx/docker/run.sh @@ -0,0 +1,2 @@ +docker build -t pambrose/nginx2 . +docker run --rm -p 50440:50440 pambrose/nginx2 \ No newline at end of file diff --git a/nginx/nginx-proxy.conf b/nginx/nginx-proxy.conf new file mode 100644 index 00000000..611b8e19 --- /dev/null +++ b/nginx/nginx-proxy.conf @@ -0,0 +1,32 @@ +proxy { + # Required for use with nginx reverse proxy + transportFilterDisabled = true +} + +agent { + # Required for use with nginx reverse proxy + transportFilterDisabled = true + + proxy { + # nginx http2 port specified in nginx.conf + port = 50440 + } + + pathConfigs: [ + { + name: "App1 metrics" + path: app1_metrics + url: "http://localhost:8082/metrics" + }, + { + name: "App2 metrics" + path: app2_metrics + url: "http://app2.local:9100/metrics" + }, + { + name: "App3 metrics" + path: app3_metrics + url: "http://app3.local:9100/metrics" + } + ] +} \ No newline at end of file diff --git a/src/main/java/io/prometheus/common/ConfigVals.java b/src/main/java/io/prometheus/common/ConfigVals.java index f695acc0..be536256 100644 --- a/src/main/java/io/prometheus/common/ConfigVals.java +++ b/src/main/java/io/prometheus/common/ConfigVals.java @@ -1,4 +1,4 @@ -// generated by tscfg 0.9.997 on Thu Oct 13 20:14:33 EDT 2022 +// generated by tscfg 0.9.997 on Mon Nov 28 23:31:57 PST 2022 // source: etc/config/config.conf package io.prometheus.common; @@ -28,6 +28,7 @@ public static class Agent { public final int scrapeMaxRetries; public final int scrapeTimeoutSecs; public final Agent.Tls tls; + public final boolean transportFilterDisabled; public Agent(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) { this.admin = c.hasPathOrNull("admin") ? new Agent.Admin(c.getConfig("admin"), parentPath + "admin.", $tsCfgValidator) : new Agent.Admin(com.typesafe.config.ConfigFactory.parseString("admin{}"), parentPath + "admin.", $tsCfgValidator); this.chunkContentSizeKbs = c.hasPathOrNull("chunkContentSizeKbs") ? c.getInt("chunkContentSizeKbs") : 32; @@ -42,6 +43,7 @@ public Agent(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgVa this.scrapeMaxRetries = c.hasPathOrNull("scrapeMaxRetries") ? c.getInt("scrapeMaxRetries") : 0; this.scrapeTimeoutSecs = c.hasPathOrNull("scrapeTimeoutSecs") ? c.getInt("scrapeTimeoutSecs") : 15; this.tls = c.hasPathOrNull("tls") ? new Agent.Tls(c.getConfig("tls"), parentPath + "tls.", $tsCfgValidator) : new Agent.Tls(com.typesafe.config.ConfigFactory.parseString("tls{}"), parentPath + "tls.", $tsCfgValidator); + this.transportFilterDisabled = c.hasPathOrNull("transportFilterDisabled") && c.getBoolean("transportFilterDisabled"); } private static java.util.List $_LAgent_PathConfigs$Elm(com.typesafe.config.ConfigList cl, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) { @@ -88,7 +90,6 @@ public static class Internal { public final int reconnectPauseSecs; public final int scrapeRequestBacklogUnhealthySize; public final Internal.Zipkin zipkin; - public Internal(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) { this.cioTimeoutSecs = c.hasPathOrNull("cioTimeoutSecs") ? c.getInt("cioTimeoutSecs") : 90; this.heartbeatCheckPauseMillis = c.hasPathOrNull("heartbeatCheckPauseMillis") ? c.getInt("heartbeatCheckPauseMillis") : 500; @@ -208,6 +209,7 @@ public static class Proxy2 { public final Proxy2.Metrics2 metrics; public final Proxy2.Service service; public final Proxy2.Tls2 tls; + public final boolean transportFilterDisabled; public Proxy2(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) { this.admin = c.hasPathOrNull("admin") ? new Proxy2.Admin2(c.getConfig("admin"), parentPath + "admin.", $tsCfgValidator) : new Proxy2.Admin2(com.typesafe.config.ConfigFactory.parseString("admin{}"), parentPath + "admin.", $tsCfgValidator); this.agent = c.hasPathOrNull("agent") ? new Proxy2.Agent2(c.getConfig("agent"), parentPath + "agent.", $tsCfgValidator) : new Proxy2.Agent2(com.typesafe.config.ConfigFactory.parseString("agent{}"), parentPath + "agent.", $tsCfgValidator); @@ -216,6 +218,7 @@ public Proxy2(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgV this.metrics = c.hasPathOrNull("metrics") ? new Proxy2.Metrics2(c.getConfig("metrics"), parentPath + "metrics.", $tsCfgValidator) : new Proxy2.Metrics2(com.typesafe.config.ConfigFactory.parseString("metrics{}"), parentPath + "metrics.", $tsCfgValidator); this.service = c.hasPathOrNull("service") ? new Proxy2.Service(c.getConfig("service"), parentPath + "service.", $tsCfgValidator) : new Proxy2.Service(com.typesafe.config.ConfigFactory.parseString("service{}"), parentPath + "service.", $tsCfgValidator); this.tls = c.hasPathOrNull("tls") ? new Proxy2.Tls2(c.getConfig("tls"), parentPath + "tls.", $tsCfgValidator) : new Proxy2.Tls2(com.typesafe.config.ConfigFactory.parseString("tls{}"), parentPath + "tls.", $tsCfgValidator); + this.transportFilterDisabled = c.hasPathOrNull("transportFilterDisabled") && c.getBoolean("transportFilterDisabled"); } public static class Admin2 { diff --git a/src/main/kotlin/io/prometheus/Agent.kt b/src/main/kotlin/io/prometheus/Agent.kt index fdfe65fc..0527a1de 100644 --- a/src/main/kotlin/io/prometheus/Agent.kt +++ b/src/main/kotlin/io/prometheus/Agent.kt @@ -153,7 +153,7 @@ class Agent( scrapeRequestBacklogSize.set(0) lastMsgSentMark = clock.markNow() - if (grpcService.connectAgent()) { + if (grpcService.connectAgent(configVals.agent.transportFilterDisabled)) { grpcService.registerAgent(initialConnectionLatch) pathManager.registerPaths() diff --git a/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt b/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt index 73a5f6cf..34df0d51 100644 --- a/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt +++ b/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt @@ -25,7 +25,7 @@ import com.github.pambrose.common.util.simpleClassName import com.github.pambrose.common.utils.TlsContext import com.github.pambrose.common.utils.TlsContext.Companion.PLAINTEXT_CONTEXT import com.github.pambrose.common.utils.TlsUtils.buildClientTlsContext -import com.google.protobuf.Empty +import io.grpc.ClientInterceptor import io.grpc.ClientInterceptors import io.grpc.ManagedChannel import io.grpc.Status @@ -34,6 +34,7 @@ import io.prometheus.Agent import io.prometheus.common.BaseOptions.Companion.HTTPS_PREFIX import io.prometheus.common.BaseOptions.Companion.HTTP_PREFIX import io.prometheus.common.GrpcObjects +import io.prometheus.common.GrpcObjects.EMPTY_INSTANCE import io.prometheus.common.GrpcObjects.newAgentInfo import io.prometheus.common.GrpcObjects.newRegisterAgentRequest import io.prometheus.common.GrpcObjects.newScrapeResponseChunk @@ -143,15 +144,23 @@ internal class AgentGrpcService( intercept(grpcTracing.newClientInterceptor()) } - val interceptors = listOf(AgentClientInterceptor(agent)) + val interceptors = + buildList { + if (!options.transportFilterDisabled) + add(AgentClientInterceptor(agent)) + } stub = ProxyServiceGrpcKt.ProxyServiceCoroutineStub(ClientInterceptors.intercept(channel, interceptors)) } // If successful, this will create an agentContext on the Proxy and an interceptor will add an agent_id to the headers` - suspend fun connectAgent() = + suspend fun connectAgent(transportFilterDisabled: Boolean) = try { logger.info { "Connecting to proxy at ${agent.proxyHost} using ${tlsContext.desc()}..." } - stub.connectAgent(Empty.getDefaultInstance()) + if (transportFilterDisabled) + stub.connectAgentWithTransportFilterDisabled(EMPTY_INSTANCE).also { agent.agentId = it.agentId } + else + stub.connectAgent(EMPTY_INSTANCE) + logger.info { "Connected to proxy at ${agent.proxyHost} using ${tlsContext.desc()}" } agent.metrics { connectCount.labels(agent.launchId, "success").inc() } true diff --git a/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt b/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt index e03b46cc..9eaa8a30 100644 --- a/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt +++ b/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt @@ -33,7 +33,6 @@ import io.ktor.client.plugins.auth.providers.* import io.ktor.client.request.* import io.ktor.client.statement.* import io.ktor.http.* -import io.ktor.http.HttpStatusCode.Companion.NotFound import io.ktor.network.sockets.* import io.prometheus.Agent import io.prometheus.common.ScrapeResults @@ -109,7 +108,7 @@ internal class AgentHttpService(val agent: Agent) { else { retryOnException(maxRetries) retryIf(maxRetries) { _, response -> - !response.status.isSuccess() && response.status != NotFound + !response.status.isSuccess() && response.status != HttpStatusCode.NotFound } modifyRequest { it.headers.append("x-retry-count", retryCount.toString()) } exponentialDelay() diff --git a/src/main/kotlin/io/prometheus/agent/AgentOptions.kt b/src/main/kotlin/io/prometheus/agent/AgentOptions.kt index 55ff82cf..bcce54f8 100644 --- a/src/main/kotlin/io/prometheus/agent/AgentOptions.kt +++ b/src/main/kotlin/io/prometheus/agent/AgentOptions.kt @@ -128,14 +128,17 @@ class AgentOptions(argv: Array, exitOnMissingConfig: Boolean) : assignAdminPort(agentConfigVals.admin.port) assignMetricsEnabled(agentConfigVals.metrics.enabled) assignMetricsPort(agentConfigVals.metrics.port) + assignTransportFilterDisabled(agentConfigVals.transportFilterDisabled) assignDebugEnabled(agentConfigVals.admin.debugEnabled) assignCertChainFilePath(agentConfigVals.tls.certChainFilePath) assignPrivateKeyFilePath(agentConfigVals.tls.privateKeyFilePath) assignTrustCertCollectionFilePath(agentConfigVals.tls.trustCertCollectionFilePath) - logger.info { "agent.internal.cioTimeoutSecs: ${agentConfigVals.internal.cioTimeoutSecs.seconds}" } logger.info { "agent.scrapeTimeoutSecs: ${agentConfigVals.scrapeTimeoutSecs.seconds}" } + logger.info { "agent.internal.cioTimeoutSecs: ${agentConfigVals.internal.cioTimeoutSecs.seconds}" } + logger.info { "agent.internal.heartbeatCheckPauseMillis: ${agentConfigVals.internal.heartbeatCheckPauseMillis}" } + logger.info { "agent.internal.heartbeatMaxInactivitySecs: ${agentConfigVals.internal.heartbeatMaxInactivitySecs}" } } } diff --git a/src/main/kotlin/io/prometheus/common/BaseOptions.kt b/src/main/kotlin/io/prometheus/common/BaseOptions.kt index 1f950c56..88b54f89 100644 --- a/src/main/kotlin/io/prometheus/common/BaseOptions.kt +++ b/src/main/kotlin/io/prometheus/common/BaseOptions.kt @@ -28,6 +28,7 @@ import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigParseOptions import com.typesafe.config.ConfigResolveOptions import com.typesafe.config.ConfigSyntax +import io.prometheus.common.EnvVars.* import mu.KLogging import java.io.File import java.io.FileNotFoundException @@ -67,6 +68,10 @@ abstract class BaseOptions protected constructor( var debugEnabled = false private set + @Parameter(names = ["--tf-disabled"], description = "Transport filter disabled") + var transportFilterDisabled = false + private set + @Parameter(names = ["-t", "--cert"], description = "Certificate chain file path") var certChainFilePath = "" private set @@ -130,49 +135,55 @@ abstract class BaseOptions protected constructor( protected fun assignAdminEnabled(defaultVal: Boolean) { if (!adminEnabled) - adminEnabled = EnvVars.ADMIN_ENABLED.getEnv(defaultVal) + adminEnabled = ADMIN_ENABLED.getEnv(defaultVal) logger.info { "adminEnabled: $adminEnabled" } } protected fun assignAdminPort(defaultVal: Int) { if (adminPort == -1) - adminPort = EnvVars.ADMIN_PORT.getEnv(defaultVal) + adminPort = ADMIN_PORT.getEnv(defaultVal) logger.info { "adminPort: $adminPort" } } protected fun assignMetricsEnabled(defaultVal: Boolean) { if (!metricsEnabled) - metricsEnabled = EnvVars.METRICS_ENABLED.getEnv(defaultVal) + metricsEnabled = METRICS_ENABLED.getEnv(defaultVal) logger.info { "metricsEnabled: $metricsEnabled" } } protected fun assignDebugEnabled(defaultVal: Boolean) { if (!debugEnabled) - debugEnabled = EnvVars.DEBUG_ENABLED.getEnv(defaultVal) + debugEnabled = DEBUG_ENABLED.getEnv(defaultVal) logger.info { "debugEnabled: $debugEnabled" } } protected fun assignMetricsPort(defaultVal: Int) { if (metricsPort == -1) - metricsPort = EnvVars.METRICS_PORT.getEnv(defaultVal) + metricsPort = METRICS_PORT.getEnv(defaultVal) logger.info { "metricsPort: $metricsPort" } } + protected fun assignTransportFilterDisabled(defaultVal: Boolean) { + if (!transportFilterDisabled) + transportFilterDisabled = TRANSPORT_FILTER_DISABLED.getEnv(defaultVal) + logger.info { "transportFilterDisabled: $transportFilterDisabled" } + } + protected fun assignCertChainFilePath(defaultVal: String) { if (certChainFilePath.isEmpty()) - certChainFilePath = EnvVars.CERT_CHAIN_FILE_PATH.getEnv(defaultVal) + certChainFilePath = CERT_CHAIN_FILE_PATH.getEnv(defaultVal) logger.info { "certChainFilePath: $certChainFilePath" } } protected fun assignPrivateKeyFilePath(defaultVal: String) { if (privateKeyFilePath.isEmpty()) - privateKeyFilePath = EnvVars.PRIVATE_KEY_FILE_PATH.getEnv(defaultVal) + privateKeyFilePath = PRIVATE_KEY_FILE_PATH.getEnv(defaultVal) logger.info { "privateKeyFilePath: $privateKeyFilePath" } } protected fun assignTrustCertCollectionFilePath(defaultVal: String) { if (trustCertCollectionFilePath.isEmpty()) - trustCertCollectionFilePath = EnvVars.TRUST_CERT_COLLECTION_FILE_PATH.getEnv(defaultVal) + trustCertCollectionFilePath = TRUST_CERT_COLLECTION_FILE_PATH.getEnv(defaultVal) logger.info { "trustCertCollectionFilePath: $trustCertCollectionFilePath" } } diff --git a/src/main/kotlin/io/prometheus/common/EnvVars.kt b/src/main/kotlin/io/prometheus/common/EnvVars.kt index 58e2daf2..25985acb 100644 --- a/src/main/kotlin/io/prometheus/common/EnvVars.kt +++ b/src/main/kotlin/io/prometheus/common/EnvVars.kt @@ -34,6 +34,7 @@ enum class EnvVars { AGENT_CONFIG, PROXY_HOSTNAME, AGENT_NAME, + TRANSPORT_FILTER_DISABLED, CONSOLIDATED, SCRAPE_TIMEOUT_SECS, SCRAPE_MAX_RETRIES, diff --git a/src/main/kotlin/io/prometheus/common/GrpcObjects.kt b/src/main/kotlin/io/prometheus/common/GrpcObjects.kt index 322621b8..d8840e93 100644 --- a/src/main/kotlin/io/prometheus/common/GrpcObjects.kt +++ b/src/main/kotlin/io/prometheus/common/GrpcObjects.kt @@ -19,6 +19,7 @@ package io.prometheus.common import com.google.protobuf.ByteString +import com.google.protobuf.Empty import io.prometheus.grpc.AgentInfo import io.prometheus.grpc.ChunkData import io.prometheus.grpc.ChunkedScrapeResponse @@ -346,4 +347,6 @@ internal object GrpcObjects { block() build() } + + internal val EMPTY_INSTANCE = Empty.getDefaultInstance() } diff --git a/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt b/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt index e52cb8af..04a502cd 100644 --- a/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt +++ b/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt @@ -47,7 +47,7 @@ internal class AgentContextCleanupService( .forEach { (agentId, agentContext) -> val inactivityDuration = agentContext.inactivityDuration if (inactivityDuration > maxAgentInactivityTime) { - logger.info { "Evicting agent after $inactivityDuration of inactivity $agentContext" } + logger.info { "Evicting agentId ${agentContext.agentId} after $inactivityDuration (max $maxAgentInactivityTime) of inactivity: $agentContext" } proxy.removeAgentContext(agentId, "Eviction") proxy.metrics { agentEvictionCount.inc() } } diff --git a/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt b/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt index 6dfa91e0..0dca4050 100644 --- a/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt +++ b/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt @@ -35,7 +35,7 @@ internal class AgentContextManager(private val isTestMode: Boolean) { val totalAgentScrapeRequestBacklogSize: Int get() = agentContextMap.values.sumOf { it.scrapeRequestBacklogSize } fun addAgentContext(agentContext: AgentContext): AgentContext? { - logger.debug { "Registering agentId: ${agentContext.agentId}" } + logger.info { "Registering agentId: ${agentContext.agentId}" } return agentContextMap.put(agentContext.agentId, agentContext) } @@ -48,7 +48,7 @@ internal class AgentContextManager(private val isTestMode: Boolean) { logger.warn { "Missing AgentContext for agentId: $agentId ($reason)" } else { if (!isTestMode) - logger.info { "Removed AgentContext $agentContext for agentId: $agentId ($reason)" } + logger.info { "Removed $agentContext for agentId: $agentId ($reason)" } agentContext.invalidate() } agentContext diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt b/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt index 766ca08a..438b2b7a 100644 --- a/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt +++ b/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt @@ -75,11 +75,18 @@ internal class ProxyGrpcService( inProcessServerName = inProcessName ) { val proxyService = ProxyServiceImpl(proxy) - val interceptors = mutableListOf(ProxyServerInterceptor()) - if (proxy.isZipkinEnabled) - interceptors += grpcTracing.newServerInterceptor() + val interceptors = + buildList { + if (!options.transportFilterDisabled) + add(ProxyServerInterceptor()) + if (proxy.isZipkinEnabled) + add(grpcTracing.newServerInterceptor()) + } + addService(ServerInterceptors.intercept(proxyService.bindService(), interceptors)) - addTransportFilter(ProxyServerTransportFilter(proxy)) + + if (!options.transportFilterDisabled) + addTransportFilter(ProxyServerTransportFilter(proxy)) } grpcServer.shutdownWithJvm(2.seconds) diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt b/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt index 205b1e94..83960f8c 100644 --- a/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt +++ b/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt @@ -38,7 +38,6 @@ import io.ktor.server.response.* import io.ktor.server.routing.* import io.prometheus.Proxy import kotlinx.coroutines.async -import kotlinx.coroutines.delay import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonPrimitive @@ -68,6 +67,7 @@ internal object ProxyHttpConfig : KLogging() { HttpStatusCode.Found -> { "$status: ${call.request.toLogString()} -> ${call.response.headers[HttpHeaders.Location]} - ${call.request.origin.remoteHost}" } + else -> "$status: ${call.request.toLogString()} - ${call.request.origin.remoteHost}" } } @@ -97,10 +97,10 @@ internal object ProxyHttpConfig : KLogging() { } routing { - get("/__test__") { - delay(30.seconds) - call.respondWith("Test value", Plain, OK) - } +// get("/__test__") { +// delay(30.seconds) +// call.respondWith("Test value", Plain, OK) +// } if (proxy.options.sdEnabled) { logger.info { "Adding /${proxy.options.sdPath} service discovery endpoint" } @@ -117,8 +117,8 @@ internal object ProxyHttpConfig : KLogging() { put("__metrics_path__", JsonPrimitive(path)) val agentContexts = proxy.pathManager.getAgentContextInfo(path)?.agentContexts - put("agentName", JsonPrimitive(agentContexts?.map { it.agentName }?.joinToString())) - put("hostName", JsonPrimitive(agentContexts?.map { it.hostName }?.joinToString())) + put("agentName", JsonPrimitive(agentContexts?.joinToString { it.agentName })) + put("hostName", JsonPrimitive(agentContexts?.joinToString { it.hostName })) } } } diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt b/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt index d608622a..9e8f9d5b 100644 --- a/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt +++ b/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt @@ -22,7 +22,6 @@ import com.beust.jcommander.Parameter import io.prometheus.Proxy import io.prometheus.common.BaseOptions import io.prometheus.common.EnvVars.* -import kotlin.time.Duration.Companion.seconds class ProxyOptions(argv: Array) : BaseOptions(Proxy::class.java.simpleName, argv, PROXY_CONFIG.name) { @@ -86,13 +85,16 @@ class ProxyOptions(argv: Array) : BaseOptions(Proxy::class.java.simpleNa assignAdminPort(proxyConfigVals.admin.port) assignMetricsEnabled(proxyConfigVals.metrics.enabled) assignMetricsPort(proxyConfigVals.metrics.port) + assignTransportFilterDisabled(proxyConfigVals.transportFilterDisabled) assignDebugEnabled(proxyConfigVals.admin.debugEnabled) assignCertChainFilePath(proxyConfigVals.tls.certChainFilePath) assignPrivateKeyFilePath(proxyConfigVals.tls.privateKeyFilePath) assignTrustCertCollectionFilePath(proxyConfigVals.tls.trustCertCollectionFilePath) - logger.info { "proxy.internal.scrapeRequestTimeoutSecs: ${proxyConfigVals.internal.scrapeRequestTimeoutSecs.seconds}" } + logger.info { "proxy.internal.scrapeRequestTimeoutSecs: ${proxyConfigVals.internal.scrapeRequestTimeoutSecs}" } + logger.info { "proxy.internal.staleAgentCheckPauseSecs: ${proxyConfigVals.internal.staleAgentCheckPauseSecs}" } + logger.info { "proxy.internal.maxAgentInactivitySecs: ${proxyConfigVals.internal.maxAgentInactivitySecs}" } } } } \ No newline at end of file diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt b/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt index 040fb968..210ea18f 100644 --- a/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt +++ b/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt @@ -23,12 +23,13 @@ import com.github.pambrose.common.util.isNotNull import io.grpc.Attributes import io.grpc.ServerTransportFilter import io.prometheus.Proxy +import io.prometheus.proxy.ProxyServiceImpl.Companion.UNKNOWN_ADDRESS import mu.KLogging internal class ProxyServerTransportFilter(private val proxy: Proxy) : ServerTransportFilter() { override fun transportReady(attributes: Attributes): Attributes { - val remoteAddress = attributes.get(REMOTE_ADDR_KEY)?.toString() ?: "Unknown" + val remoteAddress = attributes.get(REMOTE_ADDR_KEY)?.toString() ?: UNKNOWN_ADDRESS val agentContext = AgentContext(remoteAddress) proxy.agentContextManager.addAgentContext(agentContext) diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt b/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt index dc68a24d..9c4f6671 100644 --- a/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt +++ b/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt @@ -23,6 +23,9 @@ import com.github.pambrose.common.util.isNull import com.google.protobuf.Empty import io.grpc.Status import io.prometheus.Proxy +import io.prometheus.agent.RequestFailureException +import io.prometheus.common.GrpcObjects.EMPTY_INSTANCE +import io.prometheus.common.GrpcObjects.newAgentInfo import io.prometheus.common.GrpcObjects.newHeartBeatResponse import io.prometheus.common.GrpcObjects.newPathMapSizeResponse import io.prometheus.common.GrpcObjects.newRegisterAgentResponse @@ -52,8 +55,31 @@ import java.util.concurrent.atomic.AtomicLong internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.ProxyServiceCoroutineImplBase() { - override suspend fun connectAgent(request: Empty): Empty = - proxy.metrics { connectCount.inc() }.let { Empty.getDefaultInstance() } + override suspend fun connectAgent(request: Empty): Empty { + if (proxy.options.transportFilterDisabled) { + "Agent (false) and Proxy (true) do not have matching transportFilterDisabled config values".also { msg -> + logger.error { msg } + throw RequestFailureException(msg) + } + } + + proxy.metrics { connectCount.inc() } + return EMPTY_INSTANCE + } + + override suspend fun connectAgentWithTransportFilterDisabled(request: Empty): AgentInfo { + if (!proxy.options.transportFilterDisabled) { + "Agent (true) and Proxy (false) do not have matching transportFilterDisabled config values".also { msg -> + logger.error { msg } + throw RequestFailureException(msg) + } + } + + proxy.metrics { connectCount.inc() } + val agentContext = AgentContext(UNKNOWN_ADDRESS) + proxy.agentContextManager.addAgentContext(agentContext) + return newAgentInfo(agentContext.agentId) + } override suspend fun registerAgent(request: RegisterAgentRequest): RegisterAgentResponse { val agentId = request.agentId @@ -136,7 +162,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P logger.error(throwable) { "Error in writeResponsesToProxy(): $arg" } } } - return Empty.getDefaultInstance() + return EMPTY_INSTANCE } override suspend fun writeChunkedResponsesToProxy(requests: Flow): Empty { @@ -150,6 +176,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P logger.debug { "Reading header for scrapeId: $scrapeId}" } chunkedContextMap[scrapeId] = ChunkedContext(response) } + "chunk" -> { response.chunk .apply { @@ -159,6 +186,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P context.applyChunk(chunkBytes.toByteArray(), chunkByteCount, chunkCount, chunkChecksum) } } + "summary" -> { response.summary .apply { @@ -169,6 +197,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P proxy.scrapeRequestManager.assignScrapeResults(context.scrapeResults) } } + else -> throw IllegalStateException("Invalid field name in writeChunkedResponsesToProxy()") } } @@ -180,10 +209,11 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P logger.error(throwable) { "Error in writeChunkedResponsesToProxy(): $arg" } } } - return Empty.getDefaultInstance() + return EMPTY_INSTANCE } companion object : KLogging() { private val PATH_ID_GENERATOR = AtomicLong(0L) + internal const val UNKNOWN_ADDRESS = "Unknown" } } \ No newline at end of file diff --git a/src/main/proto/proxy_service.proto b/src/main/proto/proxy_service.proto index f307902a..38b0425f 100644 --- a/src/main/proto/proxy_service.proto +++ b/src/main/proto/proxy_service.proto @@ -126,6 +126,9 @@ service ProxyService { rpc connectAgent (google.protobuf.Empty) returns (google.protobuf.Empty) { } + rpc connectAgentWithTransportFilterDisabled (google.protobuf.Empty) returns (AgentInfo) { + } + rpc registerAgent (RegisterAgentRequest) returns (RegisterAgentResponse) { }