From 17fc45a31dba7f82dc4599041165fd6695816e90 Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Fri, 22 Sep 2023 12:29:17 +0200 Subject: [PATCH 1/9] Record Prometheus exemplars Augmenting the efforts in #121, add exemplar recording to the metric collection. Users can pass an effect that determines the optional exemplar, for example from their favourite tracing library, and it will be sampled via the Prometheus client library. --- .../metrics/prometheus/Prometheus.scala | 136 ++++++++++++------ .../prometheus/PrometheusExemplarsSuite.scala | 62 ++++++++ 2 files changed, 153 insertions(+), 45 deletions(-) create mode 100644 prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala diff --git a/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala b/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala index 9b270d2..d744214 100644 --- a/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala +++ b/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala @@ -16,10 +16,13 @@ package org.http4s.metrics.prometheus +import cats.Applicative import cats.data.NonEmptyList import cats.effect.Resource import cats.effect.Sync import cats.syntax.apply._ +import cats.syntax.flatMap._ +import cats.syntax.functor._ import io.prometheus.client._ import org.http4s.Method import org.http4s.Status @@ -30,6 +33,8 @@ import org.http4s.metrics.TerminationType.Canceled import org.http4s.metrics.TerminationType.Error import org.http4s.metrics.TerminationType.Timeout +import scala.jdk.CollectionConverters._ + /** [[MetricsOps]] algebra capable of recording Prometheus metrics * * For example, the following code would wrap a [[org.http4s.HttpRoutes]] with a `org.http4s.server.middleware.Metrics` @@ -95,10 +100,21 @@ object Prometheus { ): Resource[F, MetricsOps[F]] = for { metrics <- createMetricsCollection(registry, prefix, responseDurationSecondsHistogramBuckets) - } yield createMetricsOps(metrics) + } yield createMetricsOps(metrics, Applicative[F].pure(None)) + + def metricsOpsWithExemplars[F[_]: Sync]( + registry: CollectorRegistry, + sampleExemplar: F[Option[Map[String, String]]], + prefix: String = "org_http4s_server", + responseDurationSecondsHistogramBuckets: NonEmptyList[Double] = DefaultHistogramBuckets, + ): Resource[F, MetricsOps[F]] = + for { + metrics <- createMetricsCollection(registry, prefix, responseDurationSecondsHistogramBuckets) + } yield createMetricsOps(metrics, sampleExemplar.map(_.map(_.asJava))) private def createMetricsOps[F[_]]( - metrics: MetricsCollection + metrics: MetricsCollection, + exemplarLabels: F[Option[java.util.Map[String, String]]], )(implicit F: Sync[F]): MetricsOps[F] = new MetricsOps[F] { override def increaseActiveRequests(classifier: Option[String]): F[Unit] = @@ -120,10 +136,15 @@ object Prometheus { elapsed: Long, classifier: Option[String], ): F[Unit] = - F.delay { - metrics.responseDuration - .labels(label(classifier), reportMethod(method), Phase.report(Phase.Headers)) - .observe(SimpleTimer.elapsedSecondsFromNanos(0, elapsed)) + exemplarLabels.flatMap { exemplarOpt => + F.delay { + metrics.responseDuration + .labels(label(classifier), reportMethod(method), Phase.report(Phase.Headers)) + .observeWithExemplar( + SimpleTimer.elapsedSecondsFromNanos(0, elapsed), + exemplarOpt.orNull, + ) + } } override def recordTotalTime( @@ -132,13 +153,18 @@ object Prometheus { elapsed: Long, classifier: Option[String], ): F[Unit] = - F.delay { - metrics.responseDuration - .labels(label(classifier), reportMethod(method), Phase.report(Phase.Body)) - .observe(SimpleTimer.elapsedSecondsFromNanos(0, elapsed)) - metrics.requests - .labels(label(classifier), reportMethod(method), reportStatus(status)) - .inc() + exemplarLabels.flatMap { exemplarOpt => + F.delay { + metrics.responseDuration + .labels(label(classifier), reportMethod(method), Phase.report(Phase.Body)) + .observeWithExemplar( + SimpleTimer.elapsedSecondsFromNanos(0, elapsed), + exemplarOpt.orNull, + ) + metrics.requests + .labels(label(classifier), reportMethod(method), reportStatus(status)) + .incWithExemplar(exemplarOpt.orNull) + } } override def recordAbnormalTermination( @@ -154,14 +180,19 @@ object Prometheus { } private def recordCanceled(elapsed: Long, classifier: Option[String]): F[Unit] = - F.delay { - metrics.abnormalTerminations - .labels( - label(classifier), - AbnormalTermination.report(AbnormalTermination.Canceled), - label(Option.empty), - ) - .observe(SimpleTimer.elapsedSecondsFromNanos(0, elapsed)) + exemplarLabels.flatMap { exemplarOpt => + F.delay { + metrics.abnormalTerminations + .labels( + label(classifier), + AbnormalTermination.report(AbnormalTermination.Canceled), + label(Option.empty), + ) + .observeWithExemplar( + SimpleTimer.elapsedSecondsFromNanos(0, elapsed), + exemplarOpt.orNull, + ) + } } private def recordAbnormal( @@ -169,14 +200,19 @@ object Prometheus { classifier: Option[String], cause: Throwable, ): F[Unit] = - F.delay { - metrics.abnormalTerminations - .labels( - label(classifier), - AbnormalTermination.report(AbnormalTermination.Abnormal), - label(Option(cause.getClass.getName)), - ) - .observe(SimpleTimer.elapsedSecondsFromNanos(0, elapsed)) + exemplarLabels.flatMap { exemplarOpt => + F.delay { + metrics.abnormalTerminations + .labels( + label(classifier), + AbnormalTermination.report(AbnormalTermination.Abnormal), + label(Option(cause.getClass.getName)), + ) + .observeWithExemplar( + SimpleTimer.elapsedSecondsFromNanos(0, elapsed), + exemplarOpt.orNull, + ) + } } private def recordError( @@ -184,25 +220,35 @@ object Prometheus { classifier: Option[String], cause: Throwable, ): F[Unit] = - F.delay { - metrics.abnormalTerminations - .labels( - label(classifier), - AbnormalTermination.report(AbnormalTermination.Error), - label(Option(cause.getClass.getName)), - ) - .observe(SimpleTimer.elapsedSecondsFromNanos(0, elapsed)) + exemplarLabels.flatMap { exemplarOpt => + F.delay { + metrics.abnormalTerminations + .labels( + label(classifier), + AbnormalTermination.report(AbnormalTermination.Error), + label(Option(cause.getClass.getName)), + ) + .observeWithExemplar( + SimpleTimer.elapsedSecondsFromNanos(0, elapsed), + exemplarOpt.orNull, + ) + } } private def recordTimeout(elapsed: Long, classifier: Option[String]): F[Unit] = - F.delay { - metrics.abnormalTerminations - .labels( - label(classifier), - AbnormalTermination.report(AbnormalTermination.Timeout), - label(Option.empty), - ) - .observe(SimpleTimer.elapsedSecondsFromNanos(0, elapsed)) + exemplarLabels.flatMap { exemplarOpt => + F.delay { + metrics.abnormalTerminations + .labels( + label(classifier), + AbnormalTermination.report(AbnormalTermination.Timeout), + label(Option.empty), + ) + .observeWithExemplar( + SimpleTimer.elapsedSecondsFromNanos(0, elapsed), + exemplarOpt.orNull, + ) + } } private def label(value: Option[String]): String = value.getOrElse("") diff --git a/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala new file mode 100644 index 0000000..b746289 --- /dev/null +++ b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2018 http4s.org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.http4s.metrics.prometheus + +import cats.effect._ +import io.prometheus.client.CollectorRegistry +import munit.CatsEffectSuite +import org.http4s.HttpApp +import org.http4s.client.Client +import org.http4s.client.middleware.Metrics +import org.http4s.metrics.prometheus.util._ + +class PrometheusExemplarsSuite extends CatsEffectSuite { + val client: Client[IO] = Client.fromHttpApp[IO](HttpApp[IO](stub)) + + meteredClient(exemplar = Map("trace_id" -> "123")).test( + "A http client with a prometheus metrics middleware should sample an exemplar" + ) { case (registry, client) => + client.expect[String]("/ok").attempt.map { resp => + val exemplar = registry + .filteredMetricFamilySamples(java.util.Set.of("exemplars_request_count_total")) + .nextElement() + .samples + .get(0) + .exemplar + assertEquals(exemplar.getLabelName(0), "trace_id") + assertEquals(exemplar.getLabelValue(0), "123") + assertEquals(resp, Right("200 OK")) + } + } + + private def buildMeteredClient( + exemplar: Map[String, String] + ): Resource[IO, (CollectorRegistry, Client[IO])] = { + implicit val clock: Clock[IO] = FakeClock[IO] + + for { + registry <- Prometheus.collectorRegistry[IO] + metrics <- Prometheus + .metricsOpsWithExemplars[IO](registry, IO.pure(Some(exemplar)), "exemplars") + } yield (registry, Metrics(metrics)(client)) + } + + def meteredClient( + exemplar: Map[String, String] + ): SyncIO[FunFixture[(CollectorRegistry, Client[IO])]] = + ResourceFixture(buildMeteredClient(exemplar)) +} From d938767253d84d053665bc70bf20cc80be1830ce Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Fri, 22 Sep 2023 13:17:52 +0200 Subject: [PATCH 2/9] Fix tests on older Scala & Java versions --- .../metrics/prometheus/Prometheus.scala | 32 ++++++++++++------- .../prometheus/PrometheusExemplarsSuite.scala | 5 ++- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala b/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala index d744214..fb5dcc4 100644 --- a/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala +++ b/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala @@ -33,8 +33,6 @@ import org.http4s.metrics.TerminationType.Canceled import org.http4s.metrics.TerminationType.Error import org.http4s.metrics.TerminationType.Timeout -import scala.jdk.CollectionConverters._ - /** [[MetricsOps]] algebra capable of recording Prometheus metrics * * For example, the following code would wrap a [[org.http4s.HttpRoutes]] with a `org.http4s.server.middleware.Metrics` @@ -110,11 +108,11 @@ object Prometheus { ): Resource[F, MetricsOps[F]] = for { metrics <- createMetricsCollection(registry, prefix, responseDurationSecondsHistogramBuckets) - } yield createMetricsOps(metrics, sampleExemplar.map(_.map(_.asJava))) + } yield createMetricsOps(metrics, sampleExemplar.map(_.map(toFlatArray))) private def createMetricsOps[F[_]]( metrics: MetricsCollection, - exemplarLabels: F[Option[java.util.Map[String, String]]], + exemplarLabels: F[Option[Array[String]]], )(implicit F: Sync[F]): MetricsOps[F] = new MetricsOps[F] { override def increaseActiveRequests(classifier: Option[String]): F[Unit] = @@ -142,7 +140,7 @@ object Prometheus { .labels(label(classifier), reportMethod(method), Phase.report(Phase.Headers)) .observeWithExemplar( SimpleTimer.elapsedSecondsFromNanos(0, elapsed), - exemplarOpt.orNull, + exemplarOpt.orNull: _* ) } } @@ -159,11 +157,11 @@ object Prometheus { .labels(label(classifier), reportMethod(method), Phase.report(Phase.Body)) .observeWithExemplar( SimpleTimer.elapsedSecondsFromNanos(0, elapsed), - exemplarOpt.orNull, + exemplarOpt.orNull: _* ) metrics.requests .labels(label(classifier), reportMethod(method), reportStatus(status)) - .incWithExemplar(exemplarOpt.orNull) + .incWithExemplar(exemplarOpt.orNull: _*) } } @@ -190,7 +188,7 @@ object Prometheus { ) .observeWithExemplar( SimpleTimer.elapsedSecondsFromNanos(0, elapsed), - exemplarOpt.orNull, + exemplarOpt.orNull: _* ) } } @@ -210,7 +208,7 @@ object Prometheus { ) .observeWithExemplar( SimpleTimer.elapsedSecondsFromNanos(0, elapsed), - exemplarOpt.orNull, + exemplarOpt.orNull: _* ) } } @@ -230,7 +228,7 @@ object Prometheus { ) .observeWithExemplar( SimpleTimer.elapsedSecondsFromNanos(0, elapsed), - exemplarOpt.orNull, + exemplarOpt.orNull: _* ) } } @@ -246,7 +244,7 @@ object Prometheus { ) .observeWithExemplar( SimpleTimer.elapsedSecondsFromNanos(0, elapsed), - exemplarOpt.orNull, + exemplarOpt.orNull: _* ) } } @@ -338,6 +336,18 @@ object Prometheus { // https://github.com/prometheus/client_java/blob/parent-0.6.0/simpleclient/src/main/java/io/prometheus/client/Histogram.java#L73 private val DefaultHistogramBuckets: NonEmptyList[Double] = NonEmptyList(.005, List(.01, .025, .05, .075, .1, .25, .5, .75, 1, 2.5, 5, 7.5, 10)) + + // Prometheus expects exemplars as alternating key-value strings: k1, v1, k2, v2, ... + private def toFlatArray(m: Map[String, String]): Array[String] = { + val arr = new Array[String](m.size * 2) + var i = 0 + m.foreach { case (key, value) => + arr(i) = key + arr(i + 1) = value + i += 2 + } + arr + } } final case class MetricsCollection( diff --git a/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala index b746289..70a5024 100644 --- a/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala +++ b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala @@ -31,12 +31,15 @@ class PrometheusExemplarsSuite extends CatsEffectSuite { "A http client with a prometheus metrics middleware should sample an exemplar" ) { case (registry, client) => client.expect[String]("/ok").attempt.map { resp => + val filter = new java.util.HashSet[String]() + filter.add("exemplars_request_count_total") val exemplar = registry - .filteredMetricFamilySamples(java.util.Set.of("exemplars_request_count_total")) + .filteredMetricFamilySamples(filter) .nextElement() .samples .get(0) .exemplar + assertEquals(exemplar.getLabelName(0), "trace_id") assertEquals(exemplar.getLabelValue(0), "123") assertEquals(resp, Right("200 OK")) From b25ba9823462772df66fd56887e92ee5c68fabfe Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Fri, 22 Sep 2023 13:20:55 +0200 Subject: [PATCH 3/9] Add section to docs --- docs/index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/index.md b/docs/index.md index 21c12d6..081efdf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -49,3 +49,8 @@ val prefixedClient: Resource[IO, Client[IO]] = metrics <- Prometheus.metricsOps[IO](registry, "prefix") } yield Metrics[IO](metrics, classifier)(httpClient) ``` + +## Exemplars + +You can add Prometheus exemplars to most of the metrics (except gauges) recorded by `http4s-prometheus-metrics` +by using `Prometheus.metricsOpsWithExemplars` and passing an effect that captures the related exemplar labels. From c3ee73926005e9ddbca78c2cb9148ba254cdfb4d Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Mon, 9 Oct 2023 13:21:12 +0200 Subject: [PATCH 4/9] Add validity warning to Scala doc --- .../org/http4s/metrics/prometheus/Prometheus.scala | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala b/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala index fb5dcc4..8fe411c 100644 --- a/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala +++ b/prometheus-metrics/src/main/scala/org/http4s/metrics/prometheus/Prometheus.scala @@ -100,6 +100,17 @@ object Prometheus { metrics <- createMetricsCollection(registry, prefix, responseDurationSecondsHistogramBuckets) } yield createMetricsOps(metrics, Applicative[F].pure(None)) + /** Creates a [[MetricsOps]] that supports Prometheus metrics and records exemplars. + * + * Warning: The sampler effect is responsible for producing exemplar labels that are valid for the underlying + * implementation as errors happening during metric recording will not be handled! For Prometheus version < 1.0, + * this means the combined length of keys and values may not exceed 128 characters and the parts must adhere + * to the label regex Prometheus defines. + * + * @param registry a metrics collector registry + * @param sampleExemplar an effect that returns the corresponding exemplar labels + * @param prefix a prefix that will be added to all metrics + */ def metricsOpsWithExemplars[F[_]: Sync]( registry: CollectorRegistry, sampleExemplar: F[Option[Map[String, String]]], From 30c56a833aa2a9d9f46e5de311f707b700fddef0 Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Mon, 9 Oct 2023 13:37:43 +0200 Subject: [PATCH 5/9] Add jvmopts to increase stack size Scala 2.12 compiler fails with StackOverflow. --- .jvmopts | 1 + 1 file changed, 1 insertion(+) create mode 100644 .jvmopts diff --git a/.jvmopts b/.jvmopts new file mode 100644 index 0000000..23d300a --- /dev/null +++ b/.jvmopts @@ -0,0 +1 @@ +-Xss4M From d69e8593d94962a4c1616637e498b14c1955369d Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Mon, 9 Oct 2023 13:45:44 +0200 Subject: [PATCH 6/9] Upgrade build tooling --- project/build.properties | 2 +- project/plugins.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project/build.properties b/project/build.properties index 52413ab..2743082 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.3 +sbt.version=1.9.6 diff --git a/project/plugins.sbt b/project/plugins.sbt index c0a460f..2b5ce22 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1 @@ -addSbtPlugin("org.http4s" % "sbt-http4s-org" % "0.14.13") +addSbtPlugin("org.http4s" % "sbt-http4s-org" % "0.15.3") From 390714f3cca1ee5af63cc7b117e0093fc6c744bb Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Mon, 9 Oct 2023 13:46:36 +0200 Subject: [PATCH 7/9] githubWorkflowGenerate --- .github/workflows/ci.yml | 289 ++++++++++++++++++++------------------- 1 file changed, 148 insertions(+), 141 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e94c25d..bf152b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,13 +15,13 @@ on: tags: [v*] env: - PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }} - SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} - PGP_SECRET: ${{ secrets.PGP_SECRET }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +concurrency: + group: ${{ github.workflow }} @ ${{ github.ref }} + cancel-in-progress: true + jobs: build: name: Build and Test @@ -29,83 +29,63 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - scala: [2.12.18, 2.13.11, 3.3.0] + scala: [2.12, 2.13, 3] java: [temurin@8, temurin@11, temurin@17] exclude: - - scala: 2.12.18 + - scala: 2.12 java: temurin@11 - - scala: 2.12.18 + - scala: 2.12 java: temurin@17 - - scala: 3.3.0 + - scala: 3 java: temurin@11 - - scala: 3.3.0 + - scala: 3 java: temurin@17 runs-on: ${{ matrix.os }} + timeout-minutes: 60 steps: - name: Checkout current branch (full) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Download Java (temurin@8) - id: download-java-temurin-8 - if: matrix.java == 'temurin@8' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 8 - - name: Setup Java (temurin@8) + id: setup-java-temurin-8 if: matrix.java == 'temurin@8' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 8 - jdkFile: ${{ steps.download-java-temurin-8.outputs.jdkFile }} + cache: sbt - - name: Download Java (temurin@11) - id: download-java-temurin-11 - if: matrix.java == 'temurin@11' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 11 + - name: sbt update + if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false' + run: sbt +update - name: Setup Java (temurin@11) + id: setup-java-temurin-11 if: matrix.java == 'temurin@11' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 11 - jdkFile: ${{ steps.download-java-temurin-11.outputs.jdkFile }} + cache: sbt - - name: Download Java (temurin@17) - id: download-java-temurin-17 - if: matrix.java == 'temurin@17' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 17 + - name: sbt update + if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false' + run: sbt +update - name: Setup Java (temurin@17) + id: setup-java-temurin-17 if: matrix.java == 'temurin@17' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 17 - jdkFile: ${{ steps.download-java-temurin-17.outputs.jdkFile }} + cache: sbt - - name: Cache sbt - uses: actions/cache@v3 - with: - path: | - ~/.sbt - ~/.ivy2/cache - ~/.coursier/cache/v1 - ~/.cache/coursier/v1 - ~/AppData/Local/Coursier/Cache/v1 - ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} + - name: sbt update + if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' + run: sbt +update - name: Check that workflows are up to date run: sbt githubWorkflowCheck @@ -126,7 +106,7 @@ jobs: run: sbt '++ ${{ matrix.scala }}' doc - name: Check scalafix lints - if: matrix.java == 'temurin@8' && !startsWith(matrix.scala, '3.') + if: matrix.java == 'temurin@8' && !startsWith(matrix.scala, '3') run: sbt '++ ${{ matrix.scala }}' 'scalafixAll --check' - name: Check unused compile dependencies @@ -135,11 +115,11 @@ jobs: - name: Make target directories if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') - run: mkdir -p target prometheus-metrics/target site/target project/target + run: mkdir -p prometheus-metrics/target project/target - name: Compress target directories if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') - run: tar cf targets.tar target prometheus-metrics/target site/target project/target + run: tar cf targets.tar prometheus-metrics/target project/target - name: Upload target directories if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') @@ -159,116 +139,106 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Download Java (temurin@8) - id: download-java-temurin-8 - if: matrix.java == 'temurin@8' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 8 - - name: Setup Java (temurin@8) + id: setup-java-temurin-8 if: matrix.java == 'temurin@8' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 8 - jdkFile: ${{ steps.download-java-temurin-8.outputs.jdkFile }} + cache: sbt - - name: Download Java (temurin@11) - id: download-java-temurin-11 - if: matrix.java == 'temurin@11' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 11 + - name: sbt update + if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false' + run: sbt +update - name: Setup Java (temurin@11) + id: setup-java-temurin-11 if: matrix.java == 'temurin@11' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 11 - jdkFile: ${{ steps.download-java-temurin-11.outputs.jdkFile }} + cache: sbt - - name: Download Java (temurin@17) - id: download-java-temurin-17 - if: matrix.java == 'temurin@17' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 17 + - name: sbt update + if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false' + run: sbt +update - name: Setup Java (temurin@17) + id: setup-java-temurin-17 if: matrix.java == 'temurin@17' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 17 - jdkFile: ${{ steps.download-java-temurin-17.outputs.jdkFile }} + cache: sbt - - name: Cache sbt - uses: actions/cache@v3 - with: - path: | - ~/.sbt - ~/.ivy2/cache - ~/.coursier/cache/v1 - ~/.cache/coursier/v1 - ~/AppData/Local/Coursier/Cache/v1 - ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Download target directories (2.12.18) + - name: sbt update + if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' + run: sbt +update + + - name: Download target directories (2.12) uses: actions/download-artifact@v3 with: - name: target-${{ matrix.os }}-${{ matrix.java }}-2.12.18 + name: target-${{ matrix.os }}-${{ matrix.java }}-2.12 - - name: Inflate target directories (2.12.18) + - name: Inflate target directories (2.12) run: | tar xf targets.tar rm targets.tar - - name: Download target directories (2.13.11) + - name: Download target directories (2.13) uses: actions/download-artifact@v3 with: - name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.11 + name: target-${{ matrix.os }}-${{ matrix.java }}-2.13 - - name: Inflate target directories (2.13.11) + - name: Inflate target directories (2.13) run: | tar xf targets.tar rm targets.tar - - name: Download target directories (3.3.0) + - name: Download target directories (3) uses: actions/download-artifact@v3 with: - name: target-${{ matrix.os }}-${{ matrix.java }}-3.3.0 + name: target-${{ matrix.os }}-${{ matrix.java }}-3 - - name: Inflate target directories (3.3.0) + - name: Inflate target directories (3) run: | tar xf targets.tar rm targets.tar - name: Import signing key if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == '' + env: + PGP_SECRET: ${{ secrets.PGP_SECRET }} + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} run: echo $PGP_SECRET | base64 -di | gpg --import - name: Import signing key and strip passphrase if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE != '' + env: + PGP_SECRET: ${{ secrets.PGP_SECRET }} + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} run: | echo "$PGP_SECRET" | base64 -di > /tmp/signing-key.gpg echo "$PGP_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg (echo "$PGP_PASSPHRASE"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1) - name: Publish + env: + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }} run: sbt tlCiRelease - site: - name: Generate Site + dependency-submission: + name: Submit Dependencies + if: github.event_name != 'pull_request' strategy: matrix: os: [ubuntu-latest] @@ -276,76 +246,113 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Download Java (temurin@8) - id: download-java-temurin-8 + - name: Setup Java (temurin@8) + id: setup-java-temurin-8 if: matrix.java == 'temurin@8' - uses: typelevel/download-java@v2 + uses: actions/setup-java@v3 with: distribution: temurin java-version: 8 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false' + run: sbt +update + + - name: Setup Java (temurin@11) + id: setup-java-temurin-11 + if: matrix.java == 'temurin@11' + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 11 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false' + run: sbt +update + + - name: Setup Java (temurin@17) + id: setup-java-temurin-17 + if: matrix.java == 'temurin@17' + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 17 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' + run: sbt +update + + - name: Submit Dependencies + uses: scalacenter/sbt-dependency-submission@v2 + with: + modules-ignore: root_2.12 root_2.13 root_3 docs_2.12 docs_2.13 docs_3 sbt-http4s-org-scalafix-internal_2.12 sbt-http4s-org-scalafix-internal_2.13 sbt-http4s-org-scalafix-internal_3 + configs-ignore: test scala-tool scala-doc-tool test-internal + + site: + name: Generate Site + strategy: + matrix: + os: [ubuntu-latest] + java: [temurin@8] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout current branch (full) + uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Setup Java (temurin@8) + id: setup-java-temurin-8 if: matrix.java == 'temurin@8' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 8 - jdkFile: ${{ steps.download-java-temurin-8.outputs.jdkFile }} + cache: sbt - - name: Download Java (temurin@11) - id: download-java-temurin-11 - if: matrix.java == 'temurin@11' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 11 + - name: sbt update + if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false' + run: sbt +update - name: Setup Java (temurin@11) + id: setup-java-temurin-11 if: matrix.java == 'temurin@11' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 11 - jdkFile: ${{ steps.download-java-temurin-11.outputs.jdkFile }} + cache: sbt - - name: Download Java (temurin@17) - id: download-java-temurin-17 - if: matrix.java == 'temurin@17' - uses: typelevel/download-java@v2 - with: - distribution: temurin - java-version: 17 + - name: sbt update + if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false' + run: sbt +update - name: Setup Java (temurin@17) + id: setup-java-temurin-17 if: matrix.java == 'temurin@17' uses: actions/setup-java@v3 with: - distribution: jdkfile + distribution: temurin java-version: 17 - jdkFile: ${{ steps.download-java-temurin-17.outputs.jdkFile }} + cache: sbt - - name: Cache sbt - uses: actions/cache@v3 - with: - path: | - ~/.sbt - ~/.ivy2/cache - ~/.coursier/cache/v1 - ~/.cache/coursier/v1 - ~/AppData/Local/Coursier/Cache/v1 - ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} + - name: sbt update + if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' + run: sbt +update - name: Generate site run: sbt docs/tlSite - name: Publish site if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - uses: peaceiris/actions-gh-pages@v3.9.0 + uses: peaceiris/actions-gh-pages@v3.9.3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: site/target/docs/site From 8b514572230278d581a6efb07778d7068e930a0c Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Mon, 9 Oct 2023 14:02:15 +0200 Subject: [PATCH 8/9] Try 16m stack for Scala 2.12 compilation --- .jvmopts | 2 +- .../metrics/prometheus/PrometheusExemplarsSuite.scala | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.jvmopts b/.jvmopts index 23d300a..a852161 100644 --- a/.jvmopts +++ b/.jvmopts @@ -1 +1 @@ --Xss4M +-Xss16M diff --git a/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala index 70a5024..cc6e153 100644 --- a/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala +++ b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala @@ -16,13 +16,14 @@ package org.http4s.metrics.prometheus -import cats.effect._ +import cats.effect.* import io.prometheus.client.CollectorRegistry +import io.prometheus.client.exemplars.Exemplar import munit.CatsEffectSuite import org.http4s.HttpApp import org.http4s.client.Client import org.http4s.client.middleware.Metrics -import org.http4s.metrics.prometheus.util._ +import org.http4s.metrics.prometheus.util.* class PrometheusExemplarsSuite extends CatsEffectSuite { val client: Client[IO] = Client.fromHttpApp[IO](HttpApp[IO](stub)) @@ -30,10 +31,10 @@ class PrometheusExemplarsSuite extends CatsEffectSuite { meteredClient(exemplar = Map("trace_id" -> "123")).test( "A http client with a prometheus metrics middleware should sample an exemplar" ) { case (registry, client) => - client.expect[String]("/ok").attempt.map { resp => + client.expect[String]("/ok").map { resp => val filter = new java.util.HashSet[String]() filter.add("exemplars_request_count_total") - val exemplar = registry + val exemplar: Exemplar = registry .filteredMetricFamilySamples(filter) .nextElement() .samples @@ -42,7 +43,7 @@ class PrometheusExemplarsSuite extends CatsEffectSuite { assertEquals(exemplar.getLabelName(0), "trace_id") assertEquals(exemplar.getLabelValue(0), "123") - assertEquals(resp, Right("200 OK")) + assertEquals(resp, "200 OK") } } From d78e52153bf43ebbba0f2ceca76b7681aa84e377 Mon Sep 17 00:00:00 2001 From: Yannick Heiber Date: Mon, 9 Oct 2023 14:28:44 +0200 Subject: [PATCH 9/9] Fix 2.12 test compilation Remove stack size override as it didn't help --- .jvmopts | 1 - .../http4s/metrics/prometheus/PrometheusExemplarsSuite.scala | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 .jvmopts diff --git a/.jvmopts b/.jvmopts deleted file mode 100644 index a852161..0000000 --- a/.jvmopts +++ /dev/null @@ -1 +0,0 @@ --Xss16M diff --git a/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala index cc6e153..1a34d06 100644 --- a/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala +++ b/prometheus-metrics/src/test/scala/org/http4s/metrics/prometheus/PrometheusExemplarsSuite.scala @@ -56,7 +56,7 @@ class PrometheusExemplarsSuite extends CatsEffectSuite { registry <- Prometheus.collectorRegistry[IO] metrics <- Prometheus .metricsOpsWithExemplars[IO](registry, IO.pure(Some(exemplar)), "exemplars") - } yield (registry, Metrics(metrics)(client)) + } yield (registry, Metrics[IO](metrics)(client)) } def meteredClient(