feat: add metrics document generation

Makefile

@@ -24,6 +24,7 @@ all: clean build test
 
 clean:
 	./gradlew clean
+	rm -f docs/*.rst
 
 checkstyle:
 	./gradlew checkstyleMain checkstyleTest checkstyleIntegrationTest
@@ -43,9 +44,14 @@ storage/azure/build/distributions/azure-$(VERSION).tgz:
 	./gradlew build :storage:azure:distTar -x test -x integrationTest -x e2e:test
 
 .PHONY: docs
-docs:
+docs: config.rst metrics.rst
+
+config.rst:
 	./gradlew :docs:genConfigDocs
 
+metrics.rst:
+	./gradlew :docs:genMetricsDocs
+
 test: build
 	./gradlew test -x e2e:test
 

docs/build.gradle

@@ -28,6 +28,12 @@ dependencies {
 
 tasks.register('genConfigDocs', JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
-    mainClass = 'io.aiven.kafka.tieredstorage.misc.ConfigDocs'
-    standardOutput = new File("config.rst").newOutputStream()
+    mainClass = 'io.aiven.kafka.tieredstorage.misc.ConfigsDocs'
+    standardOutput = new File("docs/configs.rst").newOutputStream()
 }
+
+tasks.register('genMetricsDocs', JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    mainClass = 'io.aiven.kafka.tieredstorage.misc.MetricsDocs'
+    standardOutput = new File("docs/metrics.rst").newOutputStream()
+}

config.rst → docs/configs.rst

@@ -1,6 +1,9 @@
 =================
-RemoteStorageManagerConfig
+Core components
 =================
+-----------------
+RemoteStorageManagerConfig
+-----------------
 ``chunk.size``
   Segment files are chunked into smaller parts to allow for faster processing (e.g. encryption, compression) and for range-fetching. It is recommended to benchmark this value, starting with 4MiB.
 
@@ -91,9 +94,9 @@ RemoteStorageManagerConfig
   * Importance: low
 
 
-=================
+-----------------
 SegmentManifestCacheConfig
-=================
+-----------------
 Under ``fetch.manifest.cache.``
 
 ``retention.ms``
@@ -129,9 +132,9 @@ Under ``fetch.manifest.cache.``
   * Importance: low
 
 
-=================
+-----------------
 SegmentIndexesCacheConfig
-=================
+-----------------
 Under ``fetch.indexes.cache.``
 
 ``retention.ms``
@@ -167,9 +170,9 @@ Under ``fetch.indexes.cache.``
   * Importance: low
 
 
-=================
+-----------------
 ChunkManagerFactoryConfig
-=================
+-----------------
 ``fetch.chunk.cache.class``
   Chunk cache implementation. There are 2 implementations included: io.aiven.kafka.tieredstorage.fetch.cache.MemoryChunkCache and io.aiven.kafka.tieredstorage.fetch.cache.DiskChunkCache
 
@@ -179,9 +182,9 @@ ChunkManagerFactoryConfig
   * Importance: medium
 
 
-=================
+-----------------
 MemoryChunkCacheConfig
-=================
+-----------------
 Under ``fetch.chunk.cache.``
 
 ``size``
@@ -224,9 +227,9 @@ Under ``fetch.chunk.cache.``
   * Importance: low
 
 
-=================
+-----------------
 DiskChunkCacheConfig
-=================
+-----------------
 Under ``fetch.chunk.cache.``
 
 ``path``
@@ -452,15 +455,15 @@ S3StorageConfig
   * Importance: low
 
 ``s3.api.call.attempt.timeout``
-  AWS S3 API call attempt timeout in milliseconds
+  AWS S3 API call attempt (single retry) timeout in milliseconds
 
   * Type: long
   * Default: null
   * Valid Values: null or [1,...,9223372036854775807]
   * Importance: low
 
 ``s3.api.call.timeout``
-  AWS S3 API call timeout in milliseconds
+  AWS S3 API call timeout in milliseconds, including all retries
 
   * Type: long
   * Default: null

docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigDocs.java → docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigsDocs.java

@@ -37,32 +37,34 @@
 /**
  * Gather all config definitions across the project and generate a documentation page
  **/
-public class ConfigDocs {
+public class ConfigsDocs {
     public static void main(final String[] args) {
-        printSectionTitle("RemoteStorageManagerConfig");
+        printSectionTitle("Core components");
+
+        printSubsectionTitle("RemoteStorageManagerConfig");
         final var rsmConfigDef = RemoteStorageManagerConfig.configDef();
         System.out.println(rsmConfigDef.toEnrichedRst());
 
-        printSectionTitle("SegmentManifestCacheConfig");
+        printSubsectionTitle("SegmentManifestCacheConfig");
         System.out.println("Under ``" + SEGMENT_MANIFEST_CACHE_PREFIX + "``\n");
         final var segmentManifestCacheDef = MemorySegmentManifestCache.configDef();
         System.out.println(segmentManifestCacheDef.toEnrichedRst());
 
-        printSectionTitle("SegmentIndexesCacheConfig");
+        printSubsectionTitle("SegmentIndexesCacheConfig");
         System.out.println("Under ``" + FETCH_INDEXES_CACHE_PREFIX + "``\n");
         final var segmentIndexesCacheDef = MemorySegmentIndexesCache.configDef();
         System.out.println(segmentIndexesCacheDef.toEnrichedRst());
 
-        printSectionTitle("ChunkManagerFactoryConfig");
+        printSubsectionTitle("ChunkManagerFactoryConfig");
         final var chunkCacheFactoryDef = ChunkManagerFactoryConfig.configDef();
         System.out.println(chunkCacheFactoryDef.toEnrichedRst());
 
-        printSectionTitle("MemoryChunkCacheConfig");
+        printSubsectionTitle("MemoryChunkCacheConfig");
         System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
         final var memChunkCacheDef = ChunkCacheConfig.configDef(new ConfigDef());
         System.out.println(memChunkCacheDef.toEnrichedRst());
 
-        printSectionTitle("DiskChunkCacheConfig");
+        printSubsectionTitle("DiskChunkCacheConfig");
         System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
         final var diskChunkCacheDef = DiskChunkCacheConfig.configDef();
         System.out.println(diskChunkCacheDef.toEnrichedRst());

docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricsDocs.java

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2024 Aiven Oy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.aiven.kafka.tieredstorage.misc;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.MetricNameTemplate;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.utils.Sanitizer;
+
+import io.aiven.kafka.tieredstorage.fetch.cache.ChunkCache;
+import io.aiven.kafka.tieredstorage.fetch.index.MemorySegmentIndexesCache;
+import io.aiven.kafka.tieredstorage.fetch.manifest.MemorySegmentManifestCache;
+import io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry;
+import io.aiven.kafka.tieredstorage.metrics.MetricsRegistry;
+import io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry;
+
+public class MetricsDocs {
+    public static void main(final String[] args) {
+        printSectionTitle("Core components metrics");
+        System.out.println();
+        printSubsectionTitle("RemoteStorageManager metrics");
+        System.out.println();
+        System.out.println(toRstTable(MetricsRegistry.METRIC_CONTEXT, new MetricsRegistry().all()));
+
+        System.out.println();
+        printSubsectionTitle("SegmentManifestCache metrics");
+        System.out.println();
+        System.out.println(toRstTable(
+            CaffeineMetricsRegistry.METRIC_CONTEXT,
+            new CaffeineMetricsRegistry(MemorySegmentManifestCache.METRIC_GROUP).all()));
+        System.out.println();
+        System.out.println(toRstTable(
+            ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG,
+            new ThreadPoolMonitorMetricsRegistry(MemorySegmentManifestCache.THREAD_POOL_METRIC_GROUP).all()));
+
+        System.out.println();
+        printSubsectionTitle("SegmentIndexesCache metrics");
+        System.out.println(toRstTable(
+            CaffeineMetricsRegistry.METRIC_CONTEXT,
+            new CaffeineMetricsRegistry(MemorySegmentIndexesCache.METRIC_GROUP).all()));
+        System.out.println(toRstTable(
+            ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG,
+            new ThreadPoolMonitorMetricsRegistry(MemorySegmentIndexesCache.THREAD_POOL_METRIC_GROUP).all()));
+        System.out.println();
+        printSubsectionTitle("ChunkCache metrics");
+        System.out.println();
+        System.out.println(toRstTable(
+            CaffeineMetricsRegistry.METRIC_CONTEXT,
+            new CaffeineMetricsRegistry(ChunkCache.METRIC_GROUP).all()));
+        System.out.println();
+        System.out.println(toRstTable(
+            ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG,
+            new ThreadPoolMonitorMetricsRegistry(ChunkCache.THREAD_POOL_METRIC_GROUP).all()));
+
+        System.out.println();
+        printSectionTitle("Storage Backend metrics");
+        System.out.println();
+        printSubsectionTitle("AzureBlobStorage metrics");
+        System.out.println();
+        System.out.println(toRstTable(
+            io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.METRIC_CONTEXT,
+            new io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry().all()));
+        System.out.println();
+        printSubsectionTitle("GcsStorage metrics");
+        System.out.println();
+        System.out.println(toRstTable(
+            io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT,
+            new io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry().all()));
+        System.out.println();
+        printSubsectionTitle("S3Storage metrics");
+        System.out.println();
+        System.out.println(toRstTable(
+            io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.METRIC_CONTEXT,
+            new io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry().all()));
+    }
+
+    // o.a.k.common.metrics.Metrics does only have generation of Html documentation.
+    // as there is no plans to publish HTML docs, this util method is added to generate RST.
+    // may be upstreamed.
+    static String toRstTable(final String domain, final Iterable<MetricNameTemplate> allMetrics) {
+        final Map<String, Map<String, String>> beansAndAttributes = new TreeMap<>();
+
+        try (final Metrics metrics = new Metrics()) {
+            for (final MetricNameTemplate template : allMetrics) {
+                final Map<String, String> tags = new LinkedHashMap<>();
+                for (final String s : template.tags()) {
+                    tags.put(s, "{" + s + "}");
+                }
+
+                final MetricName metricName = metrics.metricName(
+                    template.name(),
+                    template.group(),
+                    template.description(),
+                    tags
+                );
+                final String beanName = getMBeanName(domain, metricName);
+                beansAndAttributes.computeIfAbsent(beanName, k -> new TreeMap<>());
+                final Map<String, String> attrAndDesc = beansAndAttributes.get(beanName);
+                if (!attrAndDesc.containsKey(template.name())) {
+                    attrAndDesc.put(template.name(), template.description());
+                } else {
+                    throw new IllegalArgumentException(
+                        "mBean '" + beanName
+                            + "' attribute '"
+                            + template.name()
+                            + "' is defined twice."
+                    );
+                }
+            }
+        }
+
+        final StringBuilder b = new StringBuilder();
+
+        for (final Map.Entry<String, Map<String, String>> e : beansAndAttributes.entrySet()) {
+            // Add mBean name as a section title
+            b.append(e.getKey()).append("\n");
+            b.append("=".repeat(e.getKey().length())).append("\n\n");
+
+            // Determine the maximum lengths for each column
+            final int maxAttrLength = Math.max("Attribute name".length(),
+                    e.getValue().keySet().stream().mapToInt(String::length).max().orElse(0));
+            final int maxDescLength = Math.max("Description".length(),
+                    e.getValue().values().stream().mapToInt(String::length).max().orElse(0));
+
+            // Create the table header
+            final String headerFormat = "%-" + maxAttrLength + "s   %-" + maxDescLength + "s\n";
+            final String separatorLine = "=" + "=".repeat(maxAttrLength) + "  " + "=".repeat(maxDescLength) + "\n";
+
+            b.append(separatorLine);
+            b.append(String.format(headerFormat, "Attribute name", "Description"));
+            b.append(separatorLine);
+
+            // Add table rows
+            for (final Map.Entry<String, String> e2 : e.getValue().entrySet()) {
+                b.append(String.format(headerFormat, e2.getKey(), e2.getValue()));
+            }
+
+            // Close the table
+            b.append(separatorLine);
+            b.append("\n");  // Add an empty line between tables
+        }
+
+        return b.toString();
+    }
+
+    // same as o.a.k.common.metrics.JmxReporter#getMBeanName but copy/pasted
+    // to avoid adding another dependency to this module.
+    static String getMBeanName(final String prefix, final MetricName metricName) {
+        final StringBuilder beanName = new StringBuilder();
+        beanName.append(prefix);
+        beanName.append(":type=");
+        beanName.append(metricName.group());
+        for (final Map.Entry<String, String> entry : metricName.tags().entrySet()) {
+            if (entry.getKey().length() <= 0 || entry.getValue().length() <= 0) {
+                continue;
+            }
+            beanName.append(",");
+            beanName.append(entry.getKey());
+            beanName.append("=");
+            beanName.append(Sanitizer.jmxSanitize(entry.getValue()));
+        }
+        return beanName.toString();
+    }
+
+    static void printSectionTitle(final String title) {
+        System.out.println("=================\n"
+            + title + "\n"
+            + "=================");
+    }
+
+    static void printSubsectionTitle(final String title) {
+        System.out.println("-----------------\n"
+            + title + "\n"
+            + "-----------------");
+    }
+}

storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/S3StorageConfig.java

@@ -63,9 +63,11 @@ public class S3StorageConfig extends AbstractConfig {
     static final int S3_MULTIPART_UPLOAD_PART_SIZE_DEFAULT = S3_MULTIPART_UPLOAD_PART_SIZE_MIN;
 
     private static final String S3_API_CALL_TIMEOUT_CONFIG = "s3.api.call.timeout";
-    private static final String S3_API_CALL_TIMEOUT_DOC = "AWS S3 API call timeout in milliseconds";
+    private static final String S3_API_CALL_TIMEOUT_DOC = "AWS S3 API call timeout in milliseconds, "
+        + "including all retries";
     private static final String S3_API_CALL_ATTEMPT_TIMEOUT_CONFIG = "s3.api.call.attempt.timeout";
-    private static final String S3_API_CALL_ATTEMPT_TIMEOUT_DOC = "AWS S3 API call attempt timeout in milliseconds";
+    private static final String S3_API_CALL_ATTEMPT_TIMEOUT_DOC = "AWS S3 API call attempt "
+        + "(single retry) timeout in milliseconds";
     public static final String AWS_CREDENTIALS_PROVIDER_CLASS_CONFIG = "aws.credentials.provider.class";
     private static final String AWS_CREDENTIALS_PROVIDER_CLASS_DOC = "AWS credentials provider. "
         + "If not set, AWS SDK uses the default "