docs: Add documentation module

Add module to generate config documentation

config.rst

@@ -0,0 +1,277 @@
+=================
+RemoteStorageManagerConfig
+=================
+``chunk.size``
+  Segment files are chunked into smaller parts to allow for faster processing (e.g. encryption, compression) and for range-fetching. It is recommended to benchmark this value, starting with 4MiB.
+
+  * Type: int
+  * Valid Values: [1,...,1073741823]
+  * Importance: high
+
+``storage.backend.class``
+  The storage backend implementation class
+
+  * Type: class
+  * Importance: high
+
+``compression.enabled``
+  Segments can be further compressed to optimize storage usage. Disabled by default.
+
+  * Type: boolean
+  * Default: false
+  * Importance: high
+
+``compression.heuristic.enabled``
+  Only compress segments where native compression has not been enabled. This is currently validated by looking into the first batch header. Only enabled if compression.enabled is enabled.
+
+  * Type: boolean
+  * Default: false
+  * Importance: high
+
+``encryption.enabled``
+  Segments and indexes can be encrypted, so objects are not accessible by accessing the remote storage. Disabled by default.
+
+  * Type: boolean
+  * Default: false
+  * Importance: high
+
+``key.prefix``
+  The object storage path prefix
+
+  * Type: string
+  * Default: ""
+  * Valid Values: non-null string
+  * Importance: high
+
+``upload.rate.limit.bytes.per.second``
+  Upper bound on bytes to upload (therefore read from disk) per second. Rate limit must be equal or larger than 1 MiB/sec as minimal upload throughput.
+
+  * Type: int
+  * Default: null
+  * Valid Values: io.aiven.kafka.tieredstorage.config.validators.Null@49097b5d
+  * Importance: medium
+
+``custom.metadata.fields.include``
+  Custom Metadata to be stored along Remote Log Segment metadata on Remote Log Metadata Manager back-end. Allowed values: [REMOTE_SIZE, OBJECT_PREFIX, OBJECT_KEY]
+
+  * Type: list
+  * Default: ""
+  * Valid Values: [REMOTE_SIZE, OBJECT_PREFIX, OBJECT_KEY]
+  * Importance: low
+
+``key.prefix.mask``
+  Whether to mask path prefix in logs
+
+  * Type: boolean
+  * Default: false
+  * Importance: low
+
+``metrics.num.samples``
+  The number of samples maintained to compute metrics.
+
+  * Type: int
+  * Default: 2
+  * Valid Values: [1,...]
+  * Importance: low
+
+``metrics.recording.level``
+  The highest recording level for metrics.
+
+  * Type: string
+  * Default: INFO
+  * Valid Values: [INFO, DEBUG, TRACE]
+  * Importance: low
+
+``metrics.sample.window.ms``
+  The window of time a metrics sample is computed over.
+
+  * Type: long
+  * Default: 30000 (30 seconds)
+  * Valid Values: [1,...]
+  * Importance: low
+
+
+=================
+SegmentManifestCacheConfig
+=================
+Under ``fetch.manifest.cache.``
+
+``retention.ms``
+  Cache retention time ms, where "-1" represents infinite retention
+
+  * Type: long
+  * Default: 3600000 (1 hour)
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``size``
+  Cache size in bytes, where "-1" represents unbounded cache
+
+  * Type: long
+  * Default: 1000
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``get.timeout.ms``
+  When getting an object from the fetch, how long to wait before timing out. Defaults to 10 sec.
+
+  * Type: long
+  * Default: 10000 (10 seconds)
+  * Valid Values: [1,...,9223372036854775807]
+  * Importance: low
+
+``thread.pool.size``
+  Size for the thread pool used to schedule asynchronous fetching tasks, default to number of processors.
+
+  * Type: int
+  * Default: 0
+  * Valid Values: [0,...,1024]
+  * Importance: low
+
+
+=================
+SegmentIndexesCacheConfig
+=================
+Under ``fetch.indexes.cache.``
+
+``retention.ms``
+  Cache retention time ms, where "-1" represents infinite retention
+
+  * Type: long
+  * Default: 600000 (10 minutes)
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``size``
+  Cache size in bytes, where "-1" represents unbounded cache
+
+  * Type: long
+  * Default: 10485760
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``get.timeout.ms``
+  When getting an object from the fetch, how long to wait before timing out. Defaults to 10 sec.
+
+  * Type: long
+  * Default: 10000 (10 seconds)
+  * Valid Values: [1,...,9223372036854775807]
+  * Importance: low
+
+``thread.pool.size``
+  Size for the thread pool used to schedule asynchronous fetching tasks, default to number of processors.
+
+  * Type: int
+  * Default: 0
+  * Valid Values: [0,...,1024]
+  * Importance: low
+
+
+=================
+ChunkManagerFactoryConfig
+=================
+``fetch.chunk.cache.class``
+  Chunk cache implementation. There are 2 implementations included: io.aiven.kafka.tieredstorage.fetch.cache.MemoryChunkCache and io.aiven.kafka.tieredstorage.fetch.cache.DiskChunkCache
+
+  * Type: class
+  * Default: null
+  * Valid Values: io.aiven.kafka.tieredstorage.config.validators.Subclass@736e9adb
+  * Importance: medium
+
+
+=================
+MemoryChunkCacheConfig
+=================
+Under ``fetch.chunk.cache.``
+
+``size``
+  Cache size in bytes, where "-1" represents unbounded cache
+
+  * Type: long
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``prefetch.max.size``
+  The amount of data that should be eagerly prefetched and cached
+
+  * Type: int
+  * Default: 0
+  * Valid Values: [0,...,2147483647]
+  * Importance: medium
+
+``retention.ms``
+  Cache retention time ms, where "-1" represents infinite retention
+
+  * Type: long
+  * Default: 600000 (10 minutes)
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``get.timeout.ms``
+  When getting an object from the fetch, how long to wait before timing out. Defaults to 10 sec.
+
+  * Type: long
+  * Default: 10000 (10 seconds)
+  * Valid Values: [1,...,9223372036854775807]
+  * Importance: low
+
+``thread.pool.size``
+  Size for the thread pool used to schedule asynchronous fetching tasks, default to number of processors.
+
+  * Type: int
+  * Default: 0
+  * Valid Values: [0,...,1024]
+  * Importance: low
+
+
+=================
+DiskChunkCacheConfig
+=================
+Under ``fetch.chunk.cache.``
+
+``path``
+  Cache base directory. It is required to exist and be writable prior to the execution of the plugin.
+
+  * Type: string
+  * Importance: high
+
+``size``
+  Cache size in bytes, where "-1" represents unbounded cache
+
+  * Type: long
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``prefetch.max.size``
+  The amount of data that should be eagerly prefetched and cached
+
+  * Type: int
+  * Default: 0
+  * Valid Values: [0,...,2147483647]
+  * Importance: medium
+
+``retention.ms``
+  Cache retention time ms, where "-1" represents infinite retention
+
+  * Type: long
+  * Default: 600000 (10 minutes)
+  * Valid Values: [-1,...,9223372036854775807]
+  * Importance: medium
+
+``get.timeout.ms``
+  When getting an object from the fetch, how long to wait before timing out. Defaults to 10 sec.
+
+  * Type: long
+  * Default: 10000 (10 seconds)
+  * Valid Values: [1,...,9223372036854775807]
+  * Importance: low
+
+``thread.pool.size``
+  Size for the thread pool used to schedule asynchronous fetching tasks, default to number of processors.
+
+  * Type: int
+  * Default: 0
+  * Valid Values: [0,...,1024]
+  * Importance: low
+
+

docs/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2024 Aiven Oy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id 'java-library'
+}
+
+dependencies {
+    implementation "org.apache.kafka:kafka-clients:$kafkaVersion"
+    implementation project(":core")
+}
+
+tasks.register('genConfigDocs', JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    mainClass = 'io.aiven.kafka.tieredstorage.misc.ConfigDocs'
+    standardOutput = new File("config.rst").newOutputStream()
+}

docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigDocs.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2024 Aiven Oy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.aiven.kafka.tieredstorage.misc;
+
+import org.apache.kafka.common.config.ConfigDef;
+
+import io.aiven.kafka.tieredstorage.config.ChunkCacheConfig;
+import io.aiven.kafka.tieredstorage.config.ChunkManagerFactoryConfig;
+import io.aiven.kafka.tieredstorage.config.DiskChunkCacheConfig;
+import io.aiven.kafka.tieredstorage.config.RemoteStorageManagerConfig;
+import io.aiven.kafka.tieredstorage.fetch.index.MemorySegmentIndexesCache;
+import io.aiven.kafka.tieredstorage.fetch.manifest.MemorySegmentManifestCache;
+
+public class ConfigDocs {
+    public static void main(final String[] args) {
+        System.out.println("=================\n"
+            + "RemoteStorageManagerConfig\n"
+            + "=================");
+        final var rsmConfigDef = RemoteStorageManagerConfig.configDef();
+        System.out.println(rsmConfigDef.toEnrichedRst());
+
+        System.out.println("=================\n"
+            + "SegmentManifestCacheConfig\n"
+            + "=================");
+        System.out.println("Under ``" + RemoteStorageManagerConfig.SEGMENT_MANIFEST_CACHE_PREFIX + "``\n");
+        final var segmentManifestCacheDef = MemorySegmentManifestCache.configDef();
+        System.out.println(segmentManifestCacheDef.toEnrichedRst());
+
+        System.out.println("=================\n"
+            + "SegmentIndexesCacheConfig\n"
+            + "=================");
+        System.out.println("Under ``" + RemoteStorageManagerConfig.FETCH_INDEXES_CACHE_PREFIX + "``\n");
+        final var segmentIndexesCacheDef = MemorySegmentIndexesCache.configDef();
+        System.out.println(segmentIndexesCacheDef.toEnrichedRst());
+
+        System.out.println("=================\n"
+            + "ChunkManagerFactoryConfig\n"
+            + "=================");
+        final var chunkCacheFactoryDef = ChunkManagerFactoryConfig.configDef();
+        System.out.println(chunkCacheFactoryDef.toEnrichedRst());
+
+        System.out.println("=================\n"
+            + "MemoryChunkCacheConfig\n"
+            + "=================");
+        System.out.println("Under ``" + ChunkManagerFactoryConfig.FETCH_CHUNK_CACHE_PREFIX + "``\n");
+        final var memChunkCacheDef = ChunkCacheConfig.configDef(new ConfigDef());
+        System.out.println(memChunkCacheDef.toEnrichedRst());
+
+        System.out.println("=================\n"
+            + "DiskChunkCacheConfig\n"
+            + "=================");
+        System.out.println("Under ``" + ChunkManagerFactoryConfig.FETCH_CHUNK_CACHE_PREFIX + "``\n");
+        final var diskChunkCacheDef = DiskChunkCacheConfig.configDef();
+        System.out.println(diskChunkCacheDef.toEnrichedRst());
+    }
+}

settings.gradle

@@ -24,3 +24,5 @@ include 'storage:gcs'
 include 'storage:s3'
 include 'e2e'
 include 'commons'
+include 'docs'
+