Merge pull request #63 from FINTLabs/feature/metrics

FLA-512 Add metrics support

build.gradle.kts

@@ -38,10 +38,13 @@ dependencies {
     implementation(platform(libs.koin.bom))
     implementation(libs.koin.core)
     implementation(libs.bundles.fabric8)
-    implementation(libs.operator.framework.core)
+    implementation(libs.bundles.operator)
     implementation(libs.bundles.hoplite)
     implementation(libs.bundles.logging)
     implementation(libs.jackson.module.kotlin)
+    implementation(platform(libs.http4k.bom))
+    implementation(libs.bundles.http4k)
+    implementation(libs.micrometer.registry.prometheus)
 
     testImplementation(kotlin("test"))
     testImplementation(libs.mockk)

charts/flaiserator/templates/deployment.yaml

@@ -12,10 +12,7 @@ spec:
   template:
     metadata:
       annotations:
-        prometheus.io/scrape: "false"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-        {{- with .Values.podLabels }}
+        {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
       labels:
@@ -37,6 +34,10 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          ports:
+            - name: metrics
+              containerPort: 8080
+              protocol: TCP
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           {{- with .Values.volumeMounts }}

charts/flaiserator/templates/networkpolicy.yaml

@@ -0,0 +1,20 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ include "flaiserator.fullname" . }}-prometheus
+  labels:
+    {{- include "flaiserator.labels" . | nindent 4 }}
+spec:
+  podSelector:
+    matchLabels:
+      {{- include "flaiserator.selectorLabels" . | nindent 6 }}
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+  policyTypes:
+    - Ingress

charts/flaiserator/templates/podmonitor.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    {{- include "flaiserator.labels" . | nindent 4 }}
+  name: {{ include "flaiserator.fullname" . }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "flaiserator.selectorLabels" . | nindent 6 }}
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

gradle/libs.versions.toml

@@ -10,15 +10,24 @@ logback = "1.5.8"
 logstashEncoder = "8.0"
 jackson = "2.17.0"
 logunit = "2.0.0"
+http4k = "5.32.4.0"
+micrometer = "1.13.6"
 
 [libraries]
 fabric8-kubernetes-client = { module = "io.fabric8:kubernetes-client", version.ref = "fabric8" }
 fabric8-crd-generator-apt = { module = "io.fabric8:crd-generator-apt", version.ref = "fabric8" }
 operator-framework-core = { module = "io.javaoperatorsdk:operator-framework-core", version.ref = "operatorSdk" }
+operator-micrometer-support = { module = "io.javaoperatorsdk:micrometer-support", version.ref = "operatorSdk" }
 hoplite-core = { module = "com.sksamuel.hoplite:hoplite-core", version.ref = "hoplite" }
 hoplite-yaml = { module = "com.sksamuel.hoplite:hoplite-yaml", version.ref = "hoplite" }
 jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" }
 
+http4k-bom = { module = "org.http4k:http4k-bom", version.ref = "http4k" }
+http4k-server-jetty = { module = "org.http4k:http4k-server-jetty", version.ref = "http4k" }
+http4k-metrics-micrometer = { module = "org.http4k:http4k-metrics-micrometer", version.ref = "http4k" }
+
+micrometer-registry-prometheus = { module = "io.micrometer:micrometer-registry-prometheus", version.ref = "micrometer" }
+
 logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
 logstash-logback-encoder = { module = "net.logstash.logback:logstash-logback-encoder", version.ref = "logstashEncoder" }
 
@@ -41,9 +50,11 @@ kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" }
 fabric8-generator = { id = "io.fabric8.java-generator", version.ref = "fabric8" }
 
 [bundles]
+operator = ["operator-framework-core", "operator-micrometer-support"]
 koinTest = ["koin-test", "koin-test-junit5"]
 fabric8 = ["fabric8-kubernetes-client", "fabric8-crd-generator-apt"]
 fabric8test = ["fabric8-kubernetes-server-mock", "fabric8-kube-api-test"]
 logging = ["logback-classic", "logstash-logback-encoder"]
 hoplite = ["hoplite-core", "hoplite-yaml"]
-logunit = ["logunit-core", "logunit-logback"]
+logunit = ["logunit-core", "logunit-logback"]
+http4k = ["http4k-server-jetty", "http4k-metrics-micrometer"]

src/main/kotlin/no/fintlabs/Application.kt

@@ -7,7 +7,21 @@ import io.fabric8.kubernetes.client.utils.KubernetesSerialization
 import io.javaoperatorsdk.operator.Operator
 import io.javaoperatorsdk.operator.api.config.ConfigurationService
 import io.javaoperatorsdk.operator.api.reconciler.Reconciler
+import io.javaoperatorsdk.operator.monitoring.micrometer.MicrometerMetrics
+import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics
+import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
+import io.micrometer.core.instrument.binder.system.ProcessorMetrics
+import io.micrometer.prometheusmetrics.PrometheusConfig
+import io.micrometer.prometheusmetrics.PrometheusMeterRegistry
 import no.fintlabs.operator.applicationReconcilerModule
+import org.http4k.core.HttpHandler
+import org.http4k.core.Method
+import org.http4k.core.Response
+import org.http4k.core.Status
+import org.http4k.routing.bind
+import org.http4k.routing.routes
+import org.http4k.server.Jetty
+import org.http4k.server.asServer
 import org.koin.core.context.startKoin
 import org.koin.dsl.module
 import org.koin.mp.KoinPlatform.getKoin
@@ -20,11 +34,20 @@ fun main() {
             baseModule
         )
     }
+
+    startHttpServer()
     startOperator()
 }
 
 val baseModule = module {
     single(createdAtStart = true) { defaultConfig() }
+    single {
+        PrometheusMeterRegistry(PrometheusConfig.DEFAULT).apply {
+            JvmMemoryMetrics().bindTo(this)
+            JvmGcMetrics().bindTo(this)
+            ProcessorMetrics().bindTo(this)
+        }
+    }
     single {
         ObjectMapper().apply {
             registerKotlinModule()
@@ -41,7 +64,12 @@ val baseModule = module {
         }
     }
     single {
-        OperatorConfigHandler { config -> config.withKubernetesClient(get()) }
+        OperatorConfigHandler { config ->
+            config.withKubernetesClient(get())
+            get<PrometheusMeterRegistry>().let { registry ->
+                config.withMetrics (MicrometerMetrics.withoutPerResourceMetrics(registry))
+            }
+        }
     }
     single {
         val config = ConfigurationService.newOverriddenConfigurationService { config ->
@@ -52,6 +80,20 @@ val baseModule = module {
             get<OperatorPostConfigHandler>().accept(this)
         }
     }
+    single<HttpHandler> {
+        val prometheusRegistry: PrometheusMeterRegistry = get()
+        routes(
+            "/metrics" bind Method.GET to { Response(Status.OK).body(prometheusRegistry.scrape()) },
+        )
+    }
+}
+
+fun startHttpServer() {
+    val service: HttpHandler = getKoin().get()
+    val server = service.asServer(Jetty(8080)).start()
+    Runtime.getRuntime().addShutdownHook(Thread {
+        server.stop()
+    })
 }
 
 fun startOperator() {

src/test/integration/kotlin/no/fintlabs/HttpTests.kt

@@ -0,0 +1,29 @@
+package no.fintlabs
+
+import no.fintlabs.operator.Utils.createKoinTestExtension
+import org.http4k.core.HttpHandler
+import org.http4k.core.Method
+import org.http4k.core.Request
+import org.http4k.core.Status
+import org.junit.jupiter.api.extension.RegisterExtension
+import org.koin.test.KoinTest
+import org.koin.test.inject
+import kotlin.test.Test
+import kotlin.test.assertContains
+import kotlin.test.assertEquals
+
+class HttpTests : KoinTest {
+    @Test
+    fun `should return metrics`() {
+        val client: HttpHandler by inject()
+
+        val response = client(Request(Method.GET, "/metrics"))
+        assertEquals(Status.OK, response.status)
+        assertContains(response.bodyString(), "jvm_memory_used_bytes")
+    }
+
+    companion object {
+        @RegisterExtension
+        val koinTestExtension = createKoinTestExtension()
+    }
+}