From 880411b4947addbab98cfbea981df1e7b1b5dc20 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Thu, 29 Aug 2024 12:47:02 -0400 Subject: [PATCH 01/13] Dynamically generate metrics documentation --- .../apache/accumulo/core/metrics/Metric.java | 67 ++++++++++++ .../accumulo/core/metrics/MetricsDocGen.java | 101 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 core/src/main/java/org/apache/accumulo/core/metrics/Metric.java create mode 100644 core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java new file mode 100644 index 00000000000..8cf4a8154ad --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.metrics; + +public enum Metric { + SERVER_IDLE("accumulo.server.idle", MetricType.GAUGE, + "Indicates if the server is idle or not. The value will be 1 when idle and 0 when not idle.", + MetricCategory.GENERAL_SERVER), + LOW_MEMORY("accumulo.detected.low.memory", MetricType.GAUGE, + "reports 1 when process memory usage is above threshold, 0 when memory is okay", + MetricCategory.GENERAL_SERVER), + SCAN_BUSY_TIMEOUT_COUNT("accumulo.scan.busy.timeout.count", MetricType.COUNTER, + "Count of the scans where a busy timeout happened", MetricCategory.SCAN_SERVER); + + private final String name; + private final MetricType type; + private final String description; + private final MetricCategory category; + + Metric(String name, MetricType type, String description, MetricCategory category) { + this.name = name; + this.type = type; + this.description = description; + this.category = category; + } + + public String getName() { + return name; + } + + public MetricType getType() { + return type; + } + + public String getDescription() { + return description; + } + + public MetricCategory getCategory() { + return category; + } + + public enum MetricType { + GAUGE, COUNTER, TIMER, FUNCTION_COUNTER, LONG_TASK_TIMER + } + + public enum MetricCategory { + GENERAL_SERVER, COMPACTOR, SCAN_SERVER, FATE + } + +} diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java new file mode 100644 index 00000000000..bc0d489a41c --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.metrics; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.TreeMap; + +/** + * This class generates documentation to inform users of the available metrics in a presentable + * form. + */ +public class MetricsDocGen { + private final PrintStream doc; + private final TreeMap sortedMetrics = new TreeMap<>(); + + void generate() { + pageHeader(); + + // Generate sections for each category of metrics + generateCategorySection(Metric.MetricCategory.GENERAL_SERVER, "General Server Metrics"); + generateCategorySection(Metric.MetricCategory.COMPACTOR, "Compactor Metrics"); + generateCategorySection(Metric.MetricCategory.SCAN_SERVER, "Scan Server Metrics"); + generateCategorySection(Metric.MetricCategory.FATE, "Fate Metrics"); + + doc.close(); + } + + void pageHeader() { + doc.println("---"); + doc.println("title: Metrics Documentation (3.x)"); + doc.println("category: metrics"); + doc.println("order: 6"); + doc.println("---\n"); + doc.println("\n"); + doc.println("Below are the metrics used to monitor various components of Accumulo.\n"); + } + + void generateCategorySection(Metric.MetricCategory category, String sectionTitle) { + beginSection(sectionTitle); + + for (Metric metric : sortedMetrics.values()) { + if (metric.getCategory() == category) { + generateMetricSubsection(metric); + } + } + } + + void beginSection(String sectionTitle) { + doc.println("\n## " + sectionTitle + "\n"); + } + + void generateMetricSubsection(Metric metric) { + doc.println("### " + metric.getName() + "\n"); + doc.println("**Type:** " + metric.getType().name() + "\n"); + doc.println("**Description:** " + metric.getDescription() + "\n"); + } + + private MetricsDocGen(PrintStream doc) { + this.doc = doc; + for (Metric metric : Metric.values()) { + this.sortedMetrics.put(metric.getName(), metric); + } + } + + /** + * Generates documentation for Accumulo metrics. Arguments are: "--generate-markdown filename" + * + * @param args command-line arguments + * @throws IllegalArgumentException if args is invalid + */ + public static void main(String[] args) throws IOException { + if (args.length == 2 && args[0].equals("--generate-markdown")) { + try (var printStream = new PrintStream(args[1], UTF_8)) { + new MetricsDocGen(printStream).generate(); + } + } else { + throw new IllegalArgumentException( + "Usage: " + MetricsDocGen.class.getName() + " --generate-markdown "); + } + } +} From bc1511feb4d8af40306f002541731bfd2a059c50 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Thu, 29 Aug 2024 15:39:36 -0400 Subject: [PATCH 02/13] Add all metrics from MetricsProducer docs --- .../apache/accumulo/core/metrics/Metric.java | 180 +++++++++++++++++- .../core/metrics/MetricsProducer.java | 6 +- 2 files changed, 179 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java index 8cf4a8154ad..1caecf1016f 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -19,14 +19,178 @@ package org.apache.accumulo.core.metrics; public enum Metric { + // General Server Metrics SERVER_IDLE("accumulo.server.idle", MetricType.GAUGE, "Indicates if the server is idle or not. The value will be 1 when idle and 0 when not idle.", MetricCategory.GENERAL_SERVER), LOW_MEMORY("accumulo.detected.low.memory", MetricType.GAUGE, - "reports 1 when process memory usage is above threshold, 0 when memory is okay", + "Reports 1 when process memory usage is above threshold, 0 when memory is okay.", MetricCategory.GENERAL_SERVER), + + // Compactor Metrics + COMPACTOR_MAJC_STUCK("accumulo.compactor.majc.stuck", MetricType.LONG_TASK_TIMER, "", + MetricCategory.COMPACTOR), + COMPACTOR_ENTRIES_READ("accumulo.compactor.entries.read", MetricType.FUNCTION_COUNTER, + "Number of entries read by all threads performing compactions.", MetricCategory.COMPACTOR), + COMPACTOR_ENTRIES_WRITTEN("accumulo.compactor.entries.written", MetricType.FUNCTION_COUNTER, + "Number of entries written by all threads performing compactions.", MetricCategory.COMPACTOR), + + // Fate Metrics + FATE_TYPE_IN_PROGRESS("accumulo.fate.ops.in.progress.by.type", MetricType.GAUGE, + "Number of FATE operations in progress. The type is designated by the `op.type` tag.", + MetricCategory.FATE), + FATE_OPS("accumulo.fate.ops", MetricType.GAUGE, "Tracks all the current FATE ops in any state.", + MetricCategory.FATE), + FATE_OPS_ACTIVITY("accumulo.fate.ops.activity", MetricType.GAUGE, "", MetricCategory.FATE), + FATE_ERRORS("accumulo.fate.errors", MetricType.GAUGE, "", MetricCategory.FATE), + FATE_TX("accumulo.fate.tx", MetricType.GAUGE, + "The state is now in a tag (e.g., state=new, state=in.progress, state=failed, etc.).", + MetricCategory.FATE), + + // Garbage Collection Metrics + GC_STARTED("accumulo.gc.started", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_FINISHED("accumulo.gc.finished", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_CANDIDATES("accumulo.gc.candidates", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_IN_USE("accumulo.gc.in.use", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_DELETED("accumulo.gc.deleted", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_ERRORS("accumulo.gc.errors", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_WAL_STARTED("accumulo.gc.wal.started", MetricType.GAUGE, "", + MetricCategory.GARBAGE_COLLECTION), + GC_WAL_FINISHED("accumulo.gc.wal.finished", MetricType.GAUGE, "", + MetricCategory.GARBAGE_COLLECTION), + GC_WAL_CANDIDATES("accumulo.gc.wal.candidates", MetricType.GAUGE, "", + MetricCategory.GARBAGE_COLLECTION), + GC_WAL_IN_USE("accumulo.gc.wal.in.use", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_WAL_DELETED("accumulo.gc.wal.deleted", MetricType.GAUGE, "", + MetricCategory.GARBAGE_COLLECTION), + GC_WAL_ERRORS("accumulo.gc.wal.errors", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + GC_POST_OP_DURATION("accumulo.gc.post.op.duration", MetricType.GAUGE, "", + MetricCategory.GARBAGE_COLLECTION), + GC_RUN_CYCLE("accumulo.gc.run.cycle", MetricType.GAUGE, "", MetricCategory.GARBAGE_COLLECTION), + + // Tablet Server Metrics + TSERVER_ENTRIES("accumulo.tserver.entries", MetricType.GAUGE, "", MetricCategory.TABLET_SERVER), + TSERVER_MEM_ENTRIES("accumulo.tserver.entries.mem", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_MAJC_RUNNING("accumulo.tserver.majc.running", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_MAJC_STUCK("accumulo.tserver.majc.stuck", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_MAJC_QUEUED("accumulo.tserver.majc.queued", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_MINC_QUEUED("accumulo.tserver.minc.queued", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_MINC_RUNNING("accumulo.tserver.minc.running", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_MINC_TOTAL("accumulo.tserver.minc.total", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_TABLETS_ONLINE("accumulo.tserver.tablets.online", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_TABLETS_ASSIGNMENTS_WARNING("accumulo.tserver.tablets.assignments.warning", + MetricType.GAUGE, "", MetricCategory.TABLET_SERVER), + TSERVER_TABLETS_OPENING("accumulo.tserver.tablets.opening", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_TABLETS_UNOPENED("accumulo.tserver.tablets.unopened", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_TABLETS_FILES("accumulo.tserver.tablets.files", MetricType.GAUGE, "", + MetricCategory.TABLET_SERVER), + TSERVER_INGEST_MUTATIONS("accumulo.tserver.ingest.mutations", MetricType.GAUGE, + "Prior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be derived.", + MetricCategory.TABLET_SERVER), + TSERVER_INGEST_BYTES("accumulo.tserver.ingest.bytes", MetricType.GAUGE, + "Prior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be derived.", + MetricCategory.TABLET_SERVER), + TSERVER_HOLD("accumulo.tserver.hold", MetricType.GAUGE, "", MetricCategory.TABLET_SERVER), + + // Scan Metrics + SCAN_RESERVATION_TOTAL_TIMER("accumulo.scan.reservation.total.timer", MetricType.TIMER, + "Time to reserve a tablet's files for scan.", MetricCategory.SCAN_SERVER), SCAN_BUSY_TIMEOUT_COUNT("accumulo.scan.busy.timeout.count", MetricType.COUNTER, - "Count of the scans where a busy timeout happened", MetricCategory.SCAN_SERVER); + "Count of the scans where a busy timeout happened.", MetricCategory.SCAN_SERVER), + SCAN_TABLET_METADATA_CACHE("accumulo.scan.tablet.metadata.cache", MetricType.CACHE, + "Scan server tablet cache metrics.", MetricCategory.SCAN_SERVER), + SCAN_TIMES("accumulo.scan.times", MetricType.TIMER, "", MetricCategory.SCAN_SERVER), + SCAN_OPEN_FILES("accumulo.scan.files.open", MetricType.GAUGE, "", MetricCategory.SCAN_SERVER), + SCAN_RESULTS("accumulo.scan.result", MetricType.GAUGE, "", MetricCategory.SCAN_SERVER), + SCAN_YIELDS("accumulo.scan.yields", MetricType.GAUGE, "", MetricCategory.SCAN_SERVER), + SCAN_START("accumulo.scan.start", MetricType.COUNTER, "", MetricCategory.SCAN_SERVER), + SCAN_CONTINUE("accumulo.scan.continue", MetricType.COUNTER, "", MetricCategory.SCAN_SERVER), + SCAN_CLOSE("accumulo.scan.close", MetricType.COUNTER, "", MetricCategory.SCAN_SERVER), + SCAN_QUERIES("accumulo.scan.queries", MetricType.GAUGE, "", MetricCategory.SCAN_SERVER), + SCAN_SCANNED_ENTRIES("accumulo.scan.query.scanned.entries", MetricType.GAUGE, + "Prior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be derived.", + MetricCategory.SCAN_SERVER), + SCAN_QUERY_SCAN_RESULTS("accumulo.scan.query.results", MetricType.GAUGE, + "Prior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be derived.", + MetricCategory.SCAN_SERVER), + SCAN_QUERY_SCAN_RESULTS_BYTES("accumulo.scan.query.results.bytes", MetricType.GAUGE, + "Prior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be derived.", + MetricCategory.SCAN_SERVER), + SCAN_ZOMBIE_THREADS("accumulo.scan.zombie.threads", MetricType.GAUGE, "", + MetricCategory.SCAN_SERVER), + + // Major Compaction Metrics + MAJC_QUEUED("accumulo.tserver.compactions.majc.queued", MetricType.GAUGE, + "The compaction service information is in a tag: id={i|e}_{compactionServiceName}_{executor_name}.", + MetricCategory.TABLET_SERVER), + MAJC_RUNNING("accumulo.tserver.compactions.majc.running", MetricType.GAUGE, + "The compaction service information is in a tag: id={i|e}_{compactionServiceName}_{executor_name}.", + MetricCategory.TABLET_SERVER), + MAJC_PAUSED("accumulo.tserver.compactions.majc.paused", MetricType.COUNTER, "", + MetricCategory.TABLET_SERVER), + + // Minor Compaction Metrics + MINC_QUEUED("accumulo.tserver.compactions.minc.queued", MetricType.TIMER, "", + MetricCategory.TABLET_SERVER), + MINC_RUNNING("accumulo.tserver.compactions.minc.running", MetricType.TIMER, "", + MetricCategory.TABLET_SERVER), + MINC_PAUSED("accumulo.tserver.compactions.minc.paused", MetricType.COUNTER, "", + MetricCategory.TABLET_SERVER), + + // Updates (Ingest) Metrics + UPDATE_ERRORS("accumulo.tserver.updates.error", MetricType.GAUGE, + "Type is stored in a tag (e.g., type=permission, type=unknown.tablet, type=constraint.violation).", + MetricCategory.TABLET_SERVER), + UPDATE_COMMIT("accumulo.tserver.updates.commit", MetricType.TIMER, "", + MetricCategory.TABLET_SERVER), + UPDATE_COMMIT_PREP("accumulo.tserver.updates.commit.prep", MetricType.TIMER, "", + MetricCategory.TABLET_SERVER), + UPDATE_WALOG_WRITE("accumulo.tserver.updates.walog.write", MetricType.TIMER, "", + MetricCategory.TABLET_SERVER), + UPDATE_MUTATION_ARRAY_SIZE("accumulo.tserver.updates.mutation.arrays.size", + MetricType.DISTRIBUTION_SUMMARY, "", MetricCategory.TABLET_SERVER), + + // Thrift Metrics + + THRIFT_IDLE("accumulo.thrift.idle", MetricType.DISTRIBUTION_SUMMARY, "", MetricCategory.THRIFT), + THRIFT_EXECUTE("accumulo.thrift.execute", MetricType.DISTRIBUTION_SUMMARY, "", + MetricCategory.THRIFT), + + // Block Cache Metrics + BLOCKCACHE_INDEX_HITCOUNT("accumulo.blockcache.index.hitcount", MetricType.FUNCTION_COUNTER, "", + MetricCategory.BLOCK_CACHE), + BLOCKCACHE_INDEX_REQUESTCOUNT("accumulo.blockcache.index.requestcount", + MetricType.FUNCTION_COUNTER, "", MetricCategory.BLOCK_CACHE), + BLOCKCACHE_INDEX_EVICTIONCOUNT("accumulo.blockcache.index.evictioncount", + MetricType.FUNCTION_COUNTER, "", MetricCategory.BLOCK_CACHE), + BLOCKCACHE_DATA_HITCOUNT("accumulo.blockcache.data.hitcount", MetricType.FUNCTION_COUNTER, "", + MetricCategory.BLOCK_CACHE), + BLOCKCACHE_DATA_REQUESTCOUNT("accumulo.blockcache.data.requestcount", MetricType.FUNCTION_COUNTER, + "", MetricCategory.BLOCK_CACHE), + BLOCKCACHE_DATA_EVICTIONCOUNT("accumulo.blockcache.data.evictioncount", + MetricType.FUNCTION_COUNTER, "", MetricCategory.BLOCK_CACHE), + BLOCKCACHE_SUMMARY_HITCOUNT("accumulo.blockcache.summary.hitcount", MetricType.FUNCTION_COUNTER, + "", MetricCategory.BLOCK_CACHE), + BLOCKCACHE_SUMMARY_REQUESTCOUNT("accumulo.blockcache.summary.requestcount", + MetricType.FUNCTION_COUNTER, "", MetricCategory.BLOCK_CACHE), + BLOCKCACHE_SUMMARY_EVICTIONCOUNT("accumulo.blockcache.summary.evictioncount", + MetricType.FUNCTION_COUNTER, "", MetricCategory.BLOCK_CACHE), + + // Manager Metrics + MANAGER_BALANCER_MIGRATIONS_NEEDED("accumulo.manager.balancer.migrations.needed", + MetricType.GAUGE, + "The number of migrations that need to complete before the system is balanced.", + MetricCategory.MANAGER); private final String name; private final MetricType type; @@ -57,11 +221,19 @@ public MetricCategory getCategory() { } public enum MetricType { - GAUGE, COUNTER, TIMER, FUNCTION_COUNTER, LONG_TASK_TIMER + GAUGE, COUNTER, TIMER, LONG_TASK_TIMER, DISTRIBUTION_SUMMARY, FUNCTION_COUNTER, CACHE } public enum MetricCategory { - GENERAL_SERVER, COMPACTOR, SCAN_SERVER, FATE + GENERAL_SERVER, + COMPACTOR, + FATE, + GARBAGE_COLLECTION, + TABLET_SERVER, + SCAN_SERVER, + THRIFT, + BLOCK_CACHE, + MANAGER } } diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java index dd1e493e6c7..fafeab6a33f 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java @@ -51,14 +51,14 @@ * {@value #METRICS_SERVER_IDLE} * Gauge * Indicates if the server is idle or not. The value will be 1 when idle and 0 when not idle. - * * * N/A * N/A * {@value #METRICS_LOW_MEMORY} - * Guage + * Gauge * reports 1 when process memory usage is above threshold, 0 when memory is okay * + * * * N/A * N/A @@ -497,7 +497,7 @@ * * * - * {@link #METRICS_MAJC_PAUSED} + * {@value #METRICS_MAJC_PAUSED} * Counter * * From f9deacd60f32b63c023e9d122f8bb85279fa1c75 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Tue, 3 Sep 2024 14:33:49 -0400 Subject: [PATCH 03/13] Replace Strings from MetricsProducer with Metric Enum --- .../apache/accumulo/core/metrics/Metric.java | 6 + .../core/metrics/MetricsProducer.java | 674 +----------------- .../compaction/PausedCompactionMetrics.java | 7 +- .../server/metrics/ProcessMetrics.java | 7 +- .../server/metrics/ThriftMetrics.java | 8 +- .../server/metrics/ThriftMetricsTest.java | 8 +- .../apache/accumulo/compactor/Compactor.java | 9 +- .../apache/accumulo/gc/metrics/GcMetrics.java | 43 +- .../manager/metrics/BalancerMetrics.java | 4 +- .../manager/metrics/fate/FateMetrics.java | 30 +- .../accumulo/tserver/BlockCacheMetrics.java | 29 +- .../accumulo/tserver/ScanServerMetrics.java | 16 +- .../metrics/CompactionExecutorsMetrics.java | 13 +- .../tserver/metrics/TabletServerMetrics.java | 59 +- .../metrics/TabletServerMinCMetrics.java | 7 +- .../metrics/TabletServerScanMetrics.java | 48 +- .../metrics/TabletServerUpdateMetrics.java | 22 +- .../apache/accumulo/test/ZombieScanIT.java | 4 +- .../ExternalCompactionMetricsIT.java | 4 +- .../ExternalCompactionProgressIT.java | 21 +- .../test/functional/IdleProcessMetricsIT.java | 9 +- .../test/functional/MemoryStarvedMajCIT.java | 4 +- .../test/functional/MemoryStarvedMinCIT.java | 14 +- .../test/functional/MemoryStarvedScanIT.java | 11 +- .../accumulo/test/metrics/MetricsIT.java | 29 +- 25 files changed, 265 insertions(+), 821 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java index 1caecf1016f..57fe19de5b6 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -105,6 +105,8 @@ public enum Metric { // Scan Metrics SCAN_RESERVATION_TOTAL_TIMER("accumulo.scan.reservation.total.timer", MetricType.TIMER, "Time to reserve a tablet's files for scan.", MetricCategory.SCAN_SERVER), + SCAN_RESERVATION_WRITEOUT_TIMER("accumulo.scan.reservation.writeout.timer", MetricType.TIMER, + "Time to write out a tablets file reservations for scan", MetricCategory.SCAN_SERVER), SCAN_BUSY_TIMEOUT_COUNT("accumulo.scan.busy.timeout.count", MetricType.COUNTER, "Count of the scans where a busy timeout happened.", MetricCategory.SCAN_SERVER), SCAN_TABLET_METADATA_CACHE("accumulo.scan.tablet.metadata.cache", MetricType.CACHE, @@ -126,6 +128,10 @@ public enum Metric { SCAN_QUERY_SCAN_RESULTS_BYTES("accumulo.scan.query.results.bytes", MetricType.GAUGE, "Prior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be derived.", MetricCategory.SCAN_SERVER), + SCAN_PAUSED_FOR_MEM("accumulo.scan.paused.for.memory", MetricType.COUNTER, "", + MetricCategory.SCAN_SERVER), + SCAN_RETURN_FOR_MEM("accumulo.scan.return.early.for.memory", MetricType.COUNTER, "", + MetricCategory.SCAN_SERVER), SCAN_ZOMBIE_THREADS("accumulo.scan.zombie.threads", MetricType.GAUGE, "", MetricCategory.SCAN_SERVER), diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java index fafeab6a33f..26ec8e5a038 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java @@ -34,569 +34,7 @@ * Metrics2 framework. In 2.1.0 Accumulo migrated away from the Metrics2 framework to * Micrometer. Micrometer suggests using a particular * naming convention for the - * metrics. The table below contains a mapping of the old to new metric names. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Summary of Metric Changes
Old NameHadoop Metrics2 TypeNew NameMicrometer TypeNotes
N/AN/A{@value #METRICS_SERVER_IDLE}GaugeIndicates if the server is idle or not. The value will be 1 when idle and 0 when not idle. - *
N/AN/A{@value #METRICS_LOW_MEMORY}Gaugereports 1 when process memory usage is above threshold, 0 when memory is okay
N/AN/A{@value #METRICS_COMPACTOR_MAJC_STUCK}LongTaskTimer
N/AN/A{@value #METRICS_COMPACTOR_ENTRIES_READ}FunctionCounterNumber of entries read by all threads performing compactions
N/AN/A{@value #METRICS_COMPACTOR_ENTRIES_WRITTEN}FunctionCounterNumber of entries written by all threads performing compactions
currentFateOpsGauge{@value #METRICS_FATE_OPS}GaugeWas previously named "accumulo.fate.ops.in.progress". Changed to better reflect what the - * gauge is actually tracking which is all the current fate ops in any state.
FateTxOpType_{name}Gauge{@value #METRICS_FATE_TYPE_IN_PROGRESS}GaugePreviously there was a metric per operation type with the count of in-progress transactions - * of that type. Now there is one metric and the type is in the tag op.type
totalFateOpsGauge{@value #METRICS_FATE_OPS_ACTIVITY}Gauge
totalZkConnErrorsGauge{@value #METRICS_FATE_ERRORS}Gauge
FateTxState_NEWGauge{@value #METRICS_FATE_TX}GaugeThe state is now in a tag: state=new
FateTxState_IN_PROGRESSGauge{@value #METRICS_FATE_TX}GaugeThe state is now in a tag: state=in.progress
FateTxState_FAILED_IN_PROGRESSGauge{@value #METRICS_FATE_TX}GaugeThe state is now in a tag: state=failed.in.progress
FateTxState_FAILEDGauge{@value #METRICS_FATE_TX}GaugeThe state is now in a tag: state=failed
FateTxState_SUCCESSFULGauge{@value #METRICS_FATE_TX}GaugeThe state is now in a tag: state=successful
FateTxState_UNKNOWNGauge{@value #METRICS_FATE_TX}GaugeThe state is now in a tag: state=unknown
AccGcStartedGauge{@value #METRICS_GC_STARTED}Gauge
AccGcFinishedGauge{@value #METRICS_GC_FINISHED}Gauge
AccGcCandidatesGauge{@value #METRICS_GC_CANDIDATES}Gauge
AccGcInUseGauge{@value #METRICS_GC_IN_USE}Gauge
AccGcDeletedGauge{@value #METRICS_GC_DELETED}Gauge
AccGcErrorsGauge{@value #METRICS_GC_ERRORS}Gauge
AccGcWalStartedGauge{@value #METRICS_GC_WAL_STARTED}Gauge
AccGcWalFinishedGauge{@value #METRICS_GC_WAL_FINISHED}Gauge
AccGcWalCandidatesGauge{@value #METRICS_GC_WAL_CANDIDATES}Gauge
AccGcWalInUseGauge{@value #METRICS_GC_WAL_IN_USE}Gauge
AccGcWalDeletedGauge{@value #METRICS_GC_WAL_DELETED}Gauge
AccGcWalErrorsGauge{@value #METRICS_GC_WAL_ERRORS}Gauge
AccGcPosOpDurationGauge{@value #METRICS_GC_POST_OP_DURATION}Gauge
AccGcRunCycleCountGauge{@value #METRICS_GC_RUN_CYCLE}Gauge
entriesGauge{@value #METRICS_TSERVER_ENTRIES}Gauge
entriesInMemGauge{@value #METRICS_TSERVER_MEM_ENTRIES}Gauge
activeMajCsGauge{@value #METRICS_TSERVER_MAJC_RUNNING}Gauge
N/AN/A{@value #METRICS_TSERVER_MAJC_STUCK}LongTaskTimer
queuedMajCsGauge{@value #METRICS_TSERVER_MAJC_QUEUED}Gauge
activeMinCsGauge{@value #METRICS_TSERVER_MINC_RUNNING}Gauge
queuedMinCsGauge{@value #METRICS_TSERVER_MINC_QUEUED}Gauge
totalMinCsGauge{@value #METRICS_TSERVER_MINC_TOTAL}Gauge
onlineTabletsGauge{@value #METRICS_TSERVER_TABLETS_ONLINE}Gauge
N/AN/A{@value #METRICS_TSERVER_TABLETS_LONG_ASSIGNMENTS}Gauge
openingTabletsGauge{@value #METRICS_TSERVER_TABLETS_OPENING}Gauge
unopenedTabletsGauge{@value #METRICS_TSERVER_TABLETS_UNOPENED}Gauge
filesPerTabletGauge{@value #METRICS_TSERVER_TABLETS_FILES}Gauge
ingestRateGauge{@value #METRICS_TSERVER_INGEST_MUTATIONS}GaugePrior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be - * derived
ingestByteRateGauge{@value #METRICS_TSERVER_INGEST_BYTES}GaugePrior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be - * derived
holdTimeGauge{@value #METRICS_TSERVER_HOLD}Gauge
N/AN/A{@value #METRICS_SCAN_RESERVATION_TOTAL_TIMER}TimerTime to reserve a tablets files for scan
N/AN/A{@value #METRICS_SCAN_BUSY_TIMEOUT_COUNTER}CounterCount of the scans where a busy timeout happened
N/AN/A{@value #METRICS_SCAN_TABLET_METADATA_CACHE}Cachescan server tablet cache metrics
scanStat{@value #METRICS_SCAN_TIMES}Timer
N/AN/A{@value #METRICS_SCAN_OPEN_FILES}Gauge
resultStat{@value #METRICS_SCAN_RESULTS}Gauge
yieldStat{@value #METRICS_SCAN_YIELDS}Gauge
N/AN/A{@value #METRICS_SCAN_START}Counter
N/AN/A{@value #METRICS_SCAN_CONTINUE}Counter
N/AN/A{@value #METRICS_SCAN_CLOSE}Counter
queriesGauge{@value #METRICS_SCAN_QUERIES}Gauge
scannedRateGauge{@value #METRICS_SCAN_SCANNED_ENTRIES}GaugePrior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be - * derived
queryRateGauge{@value #METRICS_SCAN_QUERY_SCAN_RESULTS}GaugePrior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be - * derived
queryByteRateGauge{@value #METRICS_SCAN_QUERY_SCAN_RESULTS_BYTES}GaugePrior to 2.1.0 this metric was reported as a rate, it is now the count and the rate can be - * derived
N/AN/A{@value #METRICS_SCAN_ZOMBIE_THREADS}Gauge
{i|e}_{compactionServiceName}_{executor_name}_queuedGauge{@value #METRICS_MAJC_QUEUED}GaugeThe compaction service information is in a tag: - * id={i|e}_{compactionServiceName}_{executor_name}
{i|e}_{compactionServiceName}_{executor_name}_runningGauge{@value #METRICS_MAJC_RUNNING}GaugeThe compaction service information is in a tag: - * id={i|e}_{compactionServiceName}_{executor_name}
{@value #METRICS_MAJC_PAUSED}Counter
QueueStat{@value #METRICS_MINC_QUEUED}Timer
MincStat{@value #METRICS_MINC_RUNNING}Timer
{@value #METRICS_MINC_PAUSED}Counter
permissionErrorsCounter{@value #METRICS_UPDATE_ERRORS}GaugeType is stored in tag: type=permission
unknownTabletErrorsCounter{@value #METRICS_UPDATE_ERRORS}GaugeType is stored in tag: type=unknown.tablet
constraintViolationsCounter{@value #METRICS_UPDATE_ERRORS}GaugeType is stored in tag: type=constraint.violation
commitPrepStat{@value #METRICS_UPDATE_COMMIT_PREP}Timer
commitTimeStat{@value #METRICS_UPDATE_COMMIT}Timer
waLogWriteTimeStat{@value #METRICS_UPDATE_WALOG_WRITE}Timer
mutationArraysSizeStat{@value #METRICS_UPDATE_MUTATION_ARRAY_SIZE}Distribution Summary
idleStat{@value #METRICS_THRIFT_IDLE}Distribution Summary
executeStat{@value #METRICS_THRIFT_EXECUTE}Distribution Summary
N/AN/A{@value METRICS_MANAGER_BALANCER_MIGRATIONS_NEEDED}GaugeThe number of migrations that need to complete before the system is balanced
+ * metrics. * * @since 2.1.0 */ @@ -604,116 +42,6 @@ public interface MetricsProducer { Logger LOG = LoggerFactory.getLogger(MetricsProducer.class); - String METRICS_LOW_MEMORY = "accumulo.detected.low.memory"; - String METRICS_SERVER_IDLE = "accumulo.server.idle"; - - String METRICS_COMPACTOR_PREFIX = "accumulo.compactor."; - String METRICS_COMPACTOR_MAJC_STUCK = METRICS_COMPACTOR_PREFIX + "majc.stuck"; - String METRICS_COMPACTOR_ENTRIES_READ = METRICS_COMPACTOR_PREFIX + "entries.read"; - String METRICS_COMPACTOR_ENTRIES_WRITTEN = METRICS_COMPACTOR_PREFIX + "entries.written"; - - String METRICS_FATE_PREFIX = "accumulo.fate."; - String METRICS_FATE_TYPE_IN_PROGRESS = METRICS_FATE_PREFIX + "ops.in.progress.by.type"; - String METRICS_FATE_OPS = METRICS_FATE_PREFIX + "ops"; - String METRICS_FATE_OPS_ACTIVITY = METRICS_FATE_PREFIX + "ops.activity"; - String METRICS_FATE_ERRORS = METRICS_FATE_PREFIX + "errors"; - String METRICS_FATE_TX = METRICS_FATE_PREFIX + "tx"; - - String METRICS_GC_PREFIX = "accumulo.gc."; - String METRICS_GC_STARTED = METRICS_GC_PREFIX + "started"; - String METRICS_GC_FINISHED = METRICS_GC_PREFIX + "finished"; - String METRICS_GC_CANDIDATES = METRICS_GC_PREFIX + "candidates"; - String METRICS_GC_IN_USE = METRICS_GC_PREFIX + "in.use"; - String METRICS_GC_DELETED = METRICS_GC_PREFIX + "deleted"; - String METRICS_GC_ERRORS = METRICS_GC_PREFIX + "errors"; - String METRICS_GC_WAL_STARTED = METRICS_GC_PREFIX + "wal.started"; - String METRICS_GC_WAL_FINISHED = METRICS_GC_PREFIX + "wal.finished"; - String METRICS_GC_WAL_CANDIDATES = METRICS_GC_PREFIX + "wal.candidates"; - String METRICS_GC_WAL_IN_USE = METRICS_GC_PREFIX + "wal.in.use"; - String METRICS_GC_WAL_DELETED = METRICS_GC_PREFIX + "wal.deleted"; - String METRICS_GC_WAL_ERRORS = METRICS_GC_PREFIX + "wal.errors"; - String METRICS_GC_POST_OP_DURATION = METRICS_GC_PREFIX + "post.op.duration"; - String METRICS_GC_RUN_CYCLE = METRICS_GC_PREFIX + "run.cycle"; - - String METRICS_MAJC_PREFIX = "accumulo.tserver.compactions.majc."; - String METRICS_MAJC_QUEUED = METRICS_MAJC_PREFIX + "queued"; - String METRICS_MAJC_RUNNING = METRICS_MAJC_PREFIX + "running"; - String METRICS_MAJC_PAUSED = METRICS_MAJC_PREFIX + "paused"; - - String METRICS_MINC_PREFIX = "accumulo.tserver.compactions.minc."; - String METRICS_MINC_QUEUED = METRICS_MINC_PREFIX + "queued"; - String METRICS_MINC_RUNNING = METRICS_MINC_PREFIX + "running"; - String METRICS_MINC_PAUSED = METRICS_MINC_PREFIX + "paused"; - - String METRICS_SCAN_PREFIX = "accumulo.scan."; - String METRICS_SCAN_TIMES = METRICS_SCAN_PREFIX + "times"; - String METRICS_SCAN_OPEN_FILES = METRICS_SCAN_PREFIX + "files.open"; - String METRICS_SCAN_RESULTS = METRICS_SCAN_PREFIX + "result"; - String METRICS_SCAN_YIELDS = METRICS_SCAN_PREFIX + "yields"; - String METRICS_SCAN_START = METRICS_SCAN_PREFIX + "start"; - String METRICS_SCAN_CONTINUE = METRICS_SCAN_PREFIX + "continue"; - String METRICS_SCAN_CLOSE = METRICS_SCAN_PREFIX + "close"; - String METRICS_SCAN_RESERVATION_TOTAL_TIMER = METRICS_SCAN_PREFIX + "reservation.total.timer"; - String METRICS_SCAN_RESERVATION_WRITEOUT_TIMER = - METRICS_SCAN_PREFIX + "reservation.writeout.timer"; - String METRICS_SCAN_BUSY_TIMEOUT_COUNTER = METRICS_SCAN_PREFIX + "busy.timeout.count"; - String METRICS_SCAN_RESERVATION_CONFLICT_COUNTER = - METRICS_SCAN_PREFIX + "reservation.conflict.count"; - String METRICS_SCAN_QUERIES = METRICS_SCAN_PREFIX + "queries"; - String METRICS_SCAN_QUERY_SCAN_RESULTS = METRICS_SCAN_PREFIX + "query.results"; - String METRICS_SCAN_QUERY_SCAN_RESULTS_BYTES = METRICS_SCAN_PREFIX + "query.results.bytes"; - String METRICS_SCAN_SCANNED_ENTRIES = METRICS_SCAN_PREFIX + "query.scanned.entries"; - String METRICS_SCAN_PAUSED_FOR_MEM = METRICS_SCAN_PREFIX + "paused.for.memory"; - String METRICS_SCAN_RETURN_FOR_MEM = METRICS_SCAN_PREFIX + "return.early.for.memory"; - - String METRICS_SCAN_ZOMBIE_THREADS = METRICS_SCAN_PREFIX + "zombie.threads"; - String METRICS_SCAN_TABLET_METADATA_CACHE = METRICS_SCAN_PREFIX + "tablet.metadata.cache"; - - String METRICS_TSERVER_PREFIX = "accumulo.tserver."; - String METRICS_TSERVER_ENTRIES = METRICS_TSERVER_PREFIX + "entries"; - String METRICS_TSERVER_MEM_ENTRIES = METRICS_TSERVER_PREFIX + "entries.mem"; - String METRICS_TSERVER_MAJC_QUEUED = METRICS_TSERVER_PREFIX + "majc.queued"; - String METRICS_TSERVER_MAJC_RUNNING = METRICS_TSERVER_PREFIX + "majc.running"; - String METRICS_TSERVER_MAJC_STUCK = METRICS_TSERVER_PREFIX + "majc.stuck"; - String METRICS_TSERVER_MINC_QUEUED = METRICS_TSERVER_PREFIX + "minc.queued"; - String METRICS_TSERVER_MINC_RUNNING = METRICS_TSERVER_PREFIX + "minc.running"; - String METRICS_TSERVER_MINC_TOTAL = METRICS_TSERVER_PREFIX + "minc.total"; - String METRICS_TSERVER_TABLETS_LONG_ASSIGNMENTS = - METRICS_TSERVER_PREFIX + "tablets.assignments.warning"; - String METRICS_TSERVER_TABLETS_ONLINE = METRICS_TSERVER_PREFIX + "tablets.online"; - String METRICS_TSERVER_TABLETS_OPENING = METRICS_TSERVER_PREFIX + "tablets.opening"; - String METRICS_TSERVER_TABLETS_UNOPENED = METRICS_TSERVER_PREFIX + "tablets.unopened"; - String METRICS_TSERVER_TABLETS_FILES = METRICS_TSERVER_PREFIX + "tablets.files"; - String METRICS_TSERVER_HOLD = METRICS_TSERVER_PREFIX + "hold"; - String METRICS_TSERVER_INGEST_MUTATIONS = METRICS_TSERVER_PREFIX + "ingest.mutations"; - String METRICS_TSERVER_INGEST_BYTES = METRICS_TSERVER_PREFIX + "ingest.bytes"; - - String METRICS_THRIFT_PREFIX = "accumulo.thrift."; - String METRICS_THRIFT_EXECUTE = METRICS_THRIFT_PREFIX + "execute"; - String METRICS_THRIFT_IDLE = METRICS_THRIFT_PREFIX + "idle"; - - String METRICS_UPDATE_PREFIX = "accumulo.tserver.updates."; - String METRICS_UPDATE_ERRORS = METRICS_UPDATE_PREFIX + "error"; - String METRICS_UPDATE_COMMIT = METRICS_UPDATE_PREFIX + "commit"; - String METRICS_UPDATE_COMMIT_PREP = METRICS_UPDATE_COMMIT + ".prep"; - String METRICS_UPDATE_WALOG_WRITE = METRICS_UPDATE_PREFIX + "walog.write"; - String METRICS_UPDATE_MUTATION_ARRAY_SIZE = METRICS_UPDATE_PREFIX + "mutation.arrays.size"; - - String METRICS_BLOCKCACHE_PREFIX = "accumulo.blockcache."; - String METRICS_BLOCKCACHE_INDEX_HITCOUNT = METRICS_BLOCKCACHE_PREFIX + "index.hitcount"; - String METRICS_BLOCKCACHE_INDEX_REQUESTCOUNT = METRICS_BLOCKCACHE_PREFIX + "index.requestcount"; - String METRICS_BLOCKCACHE_INDEX_EVICTIONCOUNT = METRICS_BLOCKCACHE_PREFIX + "index.evictioncount"; - String METRICS_BLOCKCACHE_DATA_HITCOUNT = METRICS_BLOCKCACHE_PREFIX + "data.hitcount"; - String METRICS_BLOCKCACHE_DATA_REQUESTCOUNT = METRICS_BLOCKCACHE_PREFIX + "data.requestcount"; - String METRICS_BLOCKCACHE_DATA_EVICTIONCOUNT = METRICS_BLOCKCACHE_PREFIX + "data.evictioncount"; - String METRICS_BLOCKCACHE_SUMMARY_HITCOUNT = METRICS_BLOCKCACHE_PREFIX + "summary.hitcount"; - String METRICS_BLOCKCACHE_SUMMARY_REQUESTCOUNT = - METRICS_BLOCKCACHE_PREFIX + "summary.requestcount"; - String METRICS_BLOCKCACHE_SUMMARY_EVICTIONCOUNT = - METRICS_BLOCKCACHE_PREFIX + "summary.evictioncount"; - - String METRICS_MANAGER_BALANCER_MIGRATIONS_NEEDED = "accumulo.manager.balancer.migrations.needed"; - /** * Build Micrometer Meter objects and register them with the registry */ diff --git a/server/base/src/main/java/org/apache/accumulo/server/compaction/PausedCompactionMetrics.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/PausedCompactionMetrics.java index 29e6da06d0e..4c0f69f8663 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/compaction/PausedCompactionMetrics.java +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/PausedCompactionMetrics.java @@ -18,6 +18,9 @@ */ package org.apache.accumulo.server.compaction; +import static org.apache.accumulo.core.metrics.Metric.MAJC_PAUSED; +import static org.apache.accumulo.core.metrics.Metric.MINC_PAUSED; + import java.util.concurrent.atomic.AtomicLong; import org.apache.accumulo.core.metrics.MetricsProducer; @@ -40,9 +43,9 @@ public void incrementMajCPause() { @Override public void registerMetrics(MeterRegistry registry) { - FunctionCounter.builder(METRICS_MAJC_PAUSED, majcPauseCount, AtomicLong::get) + FunctionCounter.builder(MAJC_PAUSED.getName(), majcPauseCount, AtomicLong::get) .description("major compaction pause count").register(registry); - FunctionCounter.builder(METRICS_MINC_PAUSED, mincPauseCount, AtomicLong::get) + FunctionCounter.builder(MINC_PAUSED.getName(), mincPauseCount, AtomicLong::get) .description("minor compactor pause count").register(registry); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java b/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java index 69de547e0b9..83def192286 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java @@ -18,6 +18,9 @@ */ package org.apache.accumulo.server.metrics; +import static org.apache.accumulo.core.metrics.Metric.LOW_MEMORY; +import static org.apache.accumulo.core.metrics.Metric.SERVER_IDLE; + import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -38,8 +41,8 @@ public ProcessMetrics(final ServerContext context) { @Override public void registerMetrics(MeterRegistry registry) { - registry.gauge(METRICS_LOW_MEMORY, List.of(), this, this::lowMemDetected); - registry.gauge(METRICS_SERVER_IDLE, isIdle, AtomicInteger::get); + registry.gauge(LOW_MEMORY.getName(), List.of(), this, this::lowMemDetected); + registry.gauge(SERVER_IDLE.getName(), isIdle, AtomicInteger::get); } private int lowMemDetected(ProcessMetrics processMetrics) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/metrics/ThriftMetrics.java b/server/base/src/main/java/org/apache/accumulo/server/metrics/ThriftMetrics.java index c6288752ad9..9fdf98a61c2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metrics/ThriftMetrics.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/ThriftMetrics.java @@ -18,6 +18,9 @@ */ package org.apache.accumulo.server.metrics; +import static org.apache.accumulo.core.metrics.Metric.THRIFT_EXECUTE; +import static org.apache.accumulo.core.metrics.Metric.THRIFT_IDLE; + import org.apache.accumulo.core.metrics.MetricsProducer; import io.micrometer.core.instrument.DistributionSummary; @@ -40,8 +43,9 @@ public void addExecute(long time) { @Override public void registerMetrics(MeterRegistry registry) { - idle = DistributionSummary.builder(METRICS_THRIFT_IDLE).baseUnit("ms").register(registry); - execute = DistributionSummary.builder(METRICS_THRIFT_EXECUTE).baseUnit("ms").register(registry); + idle = DistributionSummary.builder(THRIFT_IDLE.getName()).baseUnit("ms").register(registry); + execute = + DistributionSummary.builder(THRIFT_EXECUTE.getName()).baseUnit("ms").register(registry); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/metrics/ThriftMetricsTest.java b/server/base/src/test/java/org/apache/accumulo/server/metrics/ThriftMetricsTest.java index 91730d0582e..c116bebac64 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/metrics/ThriftMetricsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/metrics/ThriftMetricsTest.java @@ -18,8 +18,8 @@ */ package org.apache.accumulo.server.metrics; -import static org.apache.accumulo.core.metrics.MetricsProducer.METRICS_THRIFT_EXECUTE; -import static org.apache.accumulo.core.metrics.MetricsProducer.METRICS_THRIFT_IDLE; +import static org.apache.accumulo.core.metrics.Metric.THRIFT_EXECUTE; +import static org.apache.accumulo.core.metrics.Metric.THRIFT_IDLE; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -57,7 +57,7 @@ void registerMetrics() { assertInstanceOf(DistributionSummary.class, m); assertFalse(m instanceof NoopDistributionSummary); }); - assertTrue(registry.get(METRICS_THRIFT_IDLE).summary().count() > 0); - assertTrue(registry.get(METRICS_THRIFT_EXECUTE).summary().count() > 0); + assertTrue(registry.get(THRIFT_IDLE.getName()).summary().count() > 0); + assertTrue(registry.get(THRIFT_EXECUTE.getName()).summary().count() > 0); } } diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index b7b0470d58e..4658bd8a5a7 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -21,6 +21,9 @@ import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_READ; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_WRITTEN; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_MAJC_STUCK; import static org.apache.accumulo.core.util.LazySingletons.RANDOM; import java.io.IOException; @@ -179,14 +182,14 @@ private long getTotalEntriesWritten() { @Override public void registerMetrics(MeterRegistry registry) { super.registerMetrics(registry); - FunctionCounter.builder(METRICS_COMPACTOR_ENTRIES_READ, this, Compactor::getTotalEntriesRead) + FunctionCounter.builder(COMPACTOR_ENTRIES_READ.getName(), this, Compactor::getTotalEntriesRead) .description("Number of entries read by all compactions that have run on this compactor") .register(registry); FunctionCounter - .builder(METRICS_COMPACTOR_ENTRIES_WRITTEN, this, Compactor::getTotalEntriesWritten) + .builder(COMPACTOR_ENTRIES_WRITTEN.getName(), this, Compactor::getTotalEntriesWritten) .description("Number of entries written by all compactions that have run on this compactor") .register(registry); - LongTaskTimer timer = LongTaskTimer.builder(METRICS_COMPACTOR_MAJC_STUCK) + LongTaskTimer timer = LongTaskTimer.builder(COMPACTOR_MAJC_STUCK.getName()) .description("Number and duration of stuck major compactions").register(registry); CompactionWatcher.setTimer(timer); } diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/metrics/GcMetrics.java b/server/gc/src/main/java/org/apache/accumulo/gc/metrics/GcMetrics.java index 7a9ca9afcd0..9a88f853164 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/metrics/GcMetrics.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/metrics/GcMetrics.java @@ -18,6 +18,21 @@ */ package org.apache.accumulo.gc.metrics; +import static org.apache.accumulo.core.metrics.Metric.GC_CANDIDATES; +import static org.apache.accumulo.core.metrics.Metric.GC_DELETED; +import static org.apache.accumulo.core.metrics.Metric.GC_ERRORS; +import static org.apache.accumulo.core.metrics.Metric.GC_FINISHED; +import static org.apache.accumulo.core.metrics.Metric.GC_IN_USE; +import static org.apache.accumulo.core.metrics.Metric.GC_POST_OP_DURATION; +import static org.apache.accumulo.core.metrics.Metric.GC_RUN_CYCLE; +import static org.apache.accumulo.core.metrics.Metric.GC_STARTED; +import static org.apache.accumulo.core.metrics.Metric.GC_WAL_CANDIDATES; +import static org.apache.accumulo.core.metrics.Metric.GC_WAL_DELETED; +import static org.apache.accumulo.core.metrics.Metric.GC_WAL_ERRORS; +import static org.apache.accumulo.core.metrics.Metric.GC_WAL_FINISHED; +import static org.apache.accumulo.core.metrics.Metric.GC_WAL_IN_USE; +import static org.apache.accumulo.core.metrics.Metric.GC_WAL_STARTED; + import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.metrics.MetricsProducer; @@ -37,40 +52,40 @@ public GcMetrics(SimpleGarbageCollector gc) { @Override public void registerMetrics(MeterRegistry registry) { - Gauge.builder(METRICS_GC_STARTED, metricValues, v -> v.getLastCollect().getStarted()) + Gauge.builder(GC_STARTED.getName(), metricValues, v -> v.getLastCollect().getStarted()) .description("Timestamp GC file collection cycle started").register(registry); - Gauge.builder(METRICS_GC_FINISHED, metricValues, v -> v.getLastCollect().getFinished()) + Gauge.builder(GC_FINISHED.getName(), metricValues, v -> v.getLastCollect().getFinished()) .description("Timestamp GC file collect cycle finished").register(registry); - Gauge.builder(METRICS_GC_CANDIDATES, metricValues, v -> v.getLastCollect().getCandidates()) + Gauge.builder(GC_CANDIDATES.getName(), metricValues, v -> v.getLastCollect().getCandidates()) .description("Number of files that are candidates for deletion").register(registry); - Gauge.builder(METRICS_GC_IN_USE, metricValues, v -> v.getLastCollect().getInUse()) + Gauge.builder(GC_IN_USE.getName(), metricValues, v -> v.getLastCollect().getInUse()) .description("Number of candidate files still in use").register(registry); - Gauge.builder(METRICS_GC_DELETED, metricValues, v -> v.getLastCollect().getDeleted()) + Gauge.builder(GC_DELETED.getName(), metricValues, v -> v.getLastCollect().getDeleted()) .description("Number of candidate files deleted").register(registry); - Gauge.builder(METRICS_GC_ERRORS, metricValues, v -> v.getLastCollect().getErrors()) + Gauge.builder(GC_ERRORS.getName(), metricValues, v -> v.getLastCollect().getErrors()) .description("Number of candidate deletion errors").register(registry); // WAL metrics Gauges - Gauge.builder(METRICS_GC_WAL_STARTED, metricValues, v -> v.getLastWalCollect().getStarted()) + Gauge.builder(GC_WAL_STARTED.getName(), metricValues, v -> v.getLastWalCollect().getStarted()) .description("Timestamp GC WAL collection cycle started").register(registry); - Gauge.builder(METRICS_GC_WAL_FINISHED, metricValues, v -> v.getLastWalCollect().getFinished()) + Gauge.builder(GC_WAL_FINISHED.getName(), metricValues, v -> v.getLastWalCollect().getFinished()) .description("Timestamp GC WAL collect cycle finished").register(registry); Gauge - .builder(METRICS_GC_WAL_CANDIDATES, metricValues, + .builder(GC_WAL_CANDIDATES.getName(), metricValues, v -> v.getLastWalCollect().getCandidates()) .description("Number of files that are candidates for deletion").register(registry); - Gauge.builder(METRICS_GC_WAL_IN_USE, metricValues, v -> v.getLastWalCollect().getInUse()) + Gauge.builder(GC_WAL_IN_USE.getName(), metricValues, v -> v.getLastWalCollect().getInUse()) .description("Number of wal file candidates that are still in use").register(registry); - Gauge.builder(METRICS_GC_WAL_DELETED, metricValues, v -> v.getLastWalCollect().getDeleted()) + Gauge.builder(GC_WAL_DELETED.getName(), metricValues, v -> v.getLastWalCollect().getDeleted()) .description("Number of candidate wal files deleted").register(registry); - Gauge.builder(METRICS_GC_WAL_ERRORS, metricValues, v -> v.getLastWalCollect().getErrors()) + Gauge.builder(GC_WAL_ERRORS.getName(), metricValues, v -> v.getLastWalCollect().getErrors()) .description("Number candidate wal file deletion errors").register(registry); Gauge - .builder(METRICS_GC_POST_OP_DURATION, metricValues, + .builder(GC_POST_OP_DURATION.getName(), metricValues, v -> TimeUnit.NANOSECONDS.toMillis(v.getPostOpDurationNanos())) .description("GC metadata table post operation duration in milliseconds") .register(registry); - Gauge.builder(METRICS_GC_RUN_CYCLE, metricValues, GcCycleMetrics::getRunCycleCount) + Gauge.builder(GC_RUN_CYCLE.getName(), metricValues, GcCycleMetrics::getRunCycleCount) .description("gauge incremented each gc cycle run, rest on process start") .register(registry); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java index 01aa9e30527..820f49b139e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java @@ -18,6 +18,8 @@ */ package org.apache.accumulo.manager.metrics; +import static org.apache.accumulo.core.metrics.Metric.MANAGER_BALANCER_MIGRATIONS_NEEDED; + import java.util.function.LongSupplier; import org.apache.accumulo.core.metrics.MetricsProducer; @@ -44,7 +46,7 @@ public long getMigratingCount() { @Override public void registerMetrics(MeterRegistry registry) { Gauge - .builder(METRICS_MANAGER_BALANCER_MIGRATIONS_NEEDED, this, + .builder(MANAGER_BALANCER_MIGRATIONS_NEEDED.getName(), this, BalancerMetrics::getMigratingCount) .description("Overall total migrations that need to complete").register(registry); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java index 2f66aef2563..ebbbec43166 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java @@ -18,6 +18,12 @@ */ package org.apache.accumulo.manager.metrics.fate; +import static org.apache.accumulo.core.metrics.Metric.FATE_ERRORS; +import static org.apache.accumulo.core.metrics.Metric.FATE_OPS; +import static org.apache.accumulo.core.metrics.Metric.FATE_OPS_ACTIVITY; +import static org.apache.accumulo.core.metrics.Metric.FATE_TX; +import static org.apache.accumulo.core.metrics.Metric.FATE_TYPE_IN_PROGRESS; + import java.util.List; import java.util.Map.Entry; import java.util.concurrent.ScheduledExecutorService; @@ -122,35 +128,35 @@ private void update() { } metricValues.getOpTypeCounters().forEach((name, count) -> { - Metrics.gauge(METRICS_FATE_TYPE_IN_PROGRESS, Tags.of(OP_TYPE_TAG, name), count); + Metrics.gauge(FATE_TYPE_IN_PROGRESS.getName(), Tags.of(OP_TYPE_TAG, name), count); }); } @Override public void registerMetrics(final MeterRegistry registry) { - totalCurrentOpsGauge = registry.gauge(METRICS_FATE_OPS, new AtomicLong(0)); - totalOpsGauge = registry.gauge(METRICS_FATE_OPS_ACTIVITY, new AtomicLong(0)); - fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS, List.of(Tag.of("type", "zk.connection")), - new AtomicLong(0)); - newTxGauge = registry.gauge(METRICS_FATE_TX, + totalCurrentOpsGauge = registry.gauge(FATE_OPS.getName(), new AtomicLong(0)); + totalOpsGauge = registry.gauge(FATE_OPS_ACTIVITY.getName(), new AtomicLong(0)); + fateErrorsGauge = registry.gauge(FATE_ERRORS.getName(), + List.of(Tag.of("type", "zk.connection")), new AtomicLong(0)); + newTxGauge = registry.gauge(FATE_TX.getName(), List.of(Tag.of("state", ReadOnlyTStore.TStatus.NEW.name().toLowerCase())), new AtomicLong(0)); - submittedTxGauge = registry.gauge(METRICS_FATE_TX, + submittedTxGauge = registry.gauge(FATE_TX.getName(), List.of(Tag.of("state", ReadOnlyTStore.TStatus.SUBMITTED.name().toLowerCase())), new AtomicLong(0)); - inProgressTxGauge = registry.gauge(METRICS_FATE_TX, + inProgressTxGauge = registry.gauge(FATE_TX.getName(), List.of(Tag.of("state", ReadOnlyTStore.TStatus.IN_PROGRESS.name().toLowerCase())), new AtomicLong(0)); - failedInProgressTxGauge = registry.gauge(METRICS_FATE_TX, + failedInProgressTxGauge = registry.gauge(FATE_TX.getName(), List.of(Tag.of("state", ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase())), new AtomicLong(0)); - failedTxGauge = registry.gauge(METRICS_FATE_TX, + failedTxGauge = registry.gauge(FATE_TX.getName(), List.of(Tag.of("state", ReadOnlyTStore.TStatus.FAILED.name().toLowerCase())), new AtomicLong(0)); - successfulTxGauge = registry.gauge(METRICS_FATE_TX, + successfulTxGauge = registry.gauge(FATE_TX.getName(), List.of(Tag.of("state", ReadOnlyTStore.TStatus.SUCCESSFUL.name().toLowerCase())), new AtomicLong(0)); - unknownTxGauge = registry.gauge(METRICS_FATE_TX, + unknownTxGauge = registry.gauge(FATE_TX.getName(), List.of(Tag.of("state", ReadOnlyTStore.TStatus.UNKNOWN.name().toLowerCase())), new AtomicLong(0)); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java index 6444e74ac2d..b6325654d85 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java @@ -18,6 +18,16 @@ */ package org.apache.accumulo.tserver; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_DATA_EVICTIONCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_DATA_HITCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_DATA_REQUESTCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_INDEX_EVICTIONCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_INDEX_HITCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_INDEX_REQUESTCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_SUMMARY_EVICTIONCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_SUMMARY_HITCOUNT; +import static org.apache.accumulo.core.metrics.Metric.BLOCKCACHE_SUMMARY_REQUESTCOUNT; + import java.util.function.ToDoubleFunction; import org.apache.accumulo.core.metrics.MetricsProducer; @@ -44,26 +54,27 @@ public void registerMetrics(MeterRegistry registry) { ToDoubleFunction getRequestCount = cache -> cache.getStats().requestCount(); ToDoubleFunction getEvictionCount = cache -> cache.getStats().evictionCount(); - FunctionCounter.builder(METRICS_BLOCKCACHE_INDEX_HITCOUNT, indexCache, getHitCount) + FunctionCounter.builder(BLOCKCACHE_INDEX_HITCOUNT.getName(), indexCache, getHitCount) .description("Index block cache hit count").register(registry); - FunctionCounter.builder(METRICS_BLOCKCACHE_INDEX_REQUESTCOUNT, indexCache, getRequestCount) + FunctionCounter.builder(BLOCKCACHE_INDEX_REQUESTCOUNT.getName(), indexCache, getRequestCount) .description("Index block cache request count").register(registry); - FunctionCounter.builder(METRICS_BLOCKCACHE_INDEX_EVICTIONCOUNT, indexCache, getEvictionCount) + FunctionCounter.builder(BLOCKCACHE_INDEX_EVICTIONCOUNT.getName(), indexCache, getEvictionCount) .description("Index block cache eviction count").register(registry); - FunctionCounter.builder(METRICS_BLOCKCACHE_DATA_HITCOUNT, dataCache, getHitCount) + FunctionCounter.builder(BLOCKCACHE_DATA_HITCOUNT.getName(), dataCache, getHitCount) .description("Data block cache hit count").register(registry); - FunctionCounter.builder(METRICS_BLOCKCACHE_DATA_REQUESTCOUNT, dataCache, getRequestCount) + FunctionCounter.builder(BLOCKCACHE_DATA_REQUESTCOUNT.getName(), dataCache, getRequestCount) .description("Data block cache request count").register(registry); - FunctionCounter.builder(METRICS_BLOCKCACHE_DATA_EVICTIONCOUNT, dataCache, getEvictionCount) + FunctionCounter.builder(BLOCKCACHE_DATA_EVICTIONCOUNT.getName(), dataCache, getEvictionCount) .description("Data block cache eviction count").register(registry); - FunctionCounter.builder(METRICS_BLOCKCACHE_SUMMARY_HITCOUNT, summaryCache, getHitCount) + FunctionCounter.builder(BLOCKCACHE_SUMMARY_HITCOUNT.getName(), summaryCache, getHitCount) .description("Summary block cache hit count").register(registry); - FunctionCounter.builder(METRICS_BLOCKCACHE_SUMMARY_REQUESTCOUNT, summaryCache, getRequestCount) + FunctionCounter + .builder(BLOCKCACHE_SUMMARY_REQUESTCOUNT.getName(), summaryCache, getRequestCount) .description("Summary block cache request count").register(registry); FunctionCounter - .builder(METRICS_BLOCKCACHE_SUMMARY_EVICTIONCOUNT, summaryCache, getEvictionCount) + .builder(BLOCKCACHE_SUMMARY_EVICTIONCOUNT.getName(), summaryCache, getEvictionCount) .description("Summary block cache eviction count").register(registry); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java index 46a3793861c..b91a410f88d 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java @@ -18,6 +18,11 @@ */ package org.apache.accumulo.tserver; +import static org.apache.accumulo.core.metrics.Metric.SCAN_BUSY_TIMEOUT_COUNT; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_TOTAL_TIMER; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_WRITEOUT_TIMER; +import static org.apache.accumulo.core.metrics.Metric.SCAN_TABLET_METADATA_CACHE; + import java.time.Duration; import java.util.concurrent.atomic.AtomicLong; @@ -49,15 +54,14 @@ public ScanServerMetrics(final LoadingCache tabletMeta @Override public void registerMetrics(MeterRegistry registry) { - totalReservationTimer = Timer.builder(MetricsProducer.METRICS_SCAN_RESERVATION_TOTAL_TIMER) + totalReservationTimer = Timer.builder(SCAN_RESERVATION_TOTAL_TIMER.getName()) .description("Time to reserve a tablets files for scan").register(registry); - writeOutReservationTimer = Timer - .builder(MetricsProducer.METRICS_SCAN_RESERVATION_WRITEOUT_TIMER) + writeOutReservationTimer = Timer.builder(SCAN_RESERVATION_WRITEOUT_TIMER.getName()) .description("Time to write out a tablets file reservations for scan").register(registry); - FunctionCounter.builder(METRICS_SCAN_BUSY_TIMEOUT_COUNTER, busyTimeoutCount, AtomicLong::get) + FunctionCounter.builder(SCAN_BUSY_TIMEOUT_COUNT.getName(), busyTimeoutCount, AtomicLong::get) .description("The number of scans where a busy timeout happened").register(registry); FunctionCounter - .builder(METRICS_SCAN_RESERVATION_CONFLICT_COUNTER, reservationConflictCount, + .builder(SCAN_RESERVATION_WRITEOUT_TIMER.getName(), reservationConflictCount, AtomicLong::get) .description( "Counts instances where file reservation attempts for scans encountered conflicts") @@ -67,7 +71,7 @@ public void registerMetrics(MeterRegistry registry) { Preconditions.checkState(tabletMetadataCache.policy().isRecordingStats(), "Attempted to instrument cache that is not recording stats."); CaffeineCacheMetrics.monitor(registry, tabletMetadataCache, - METRICS_SCAN_TABLET_METADATA_CACHE); + SCAN_TABLET_METADATA_CACHE.getName()); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/CompactionExecutorsMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/CompactionExecutorsMetrics.java index 59c66e1d35c..7f5a5b96af2 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/CompactionExecutorsMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/CompactionExecutorsMetrics.java @@ -18,6 +18,9 @@ */ package org.apache.accumulo.tserver.metrics; +import static org.apache.accumulo.core.metrics.Metric.MAJC_QUEUED; +import static org.apache.accumulo.core.metrics.Metric.MAJC_RUNNING; + import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -88,9 +91,9 @@ public synchronized CeMetrics addExecutor(CompactionExecutorId ceid, IntSupplier CeMetrics cem = ceMetricsMap.computeIfAbsent(ceid, id -> { CeMetrics m = new CeMetrics(); if (registry != null) { - m.queued = registry.gauge(METRICS_MAJC_QUEUED, Tags.of("id", ceid.canonical()), + m.queued = registry.gauge(MAJC_QUEUED.getName(), Tags.of("id", ceid.canonical()), new AtomicInteger(0)); - m.running = registry.gauge(METRICS_MAJC_RUNNING, Tags.of("id", ceid.canonical()), + m.running = registry.gauge(MAJC_RUNNING.getName(), Tags.of("id", ceid.canonical()), new AtomicInteger(0)); } return m; @@ -119,10 +122,10 @@ public void update() { ExMetrics exm = exCeMetricsMap.computeIfAbsent(ecm.ceid, id -> { ExMetrics m = new ExMetrics(); if (registry != null) { - m.queued = registry.gauge(METRICS_MAJC_QUEUED, Tags.of("id", ecm.ceid.canonical()), - new AtomicInteger(0)); - m.running = registry.gauge(METRICS_MAJC_RUNNING, Tags.of("id", ecm.ceid.canonical()), + m.queued = registry.gauge(MAJC_QUEUED.getName(), Tags.of("id", ecm.ceid.canonical()), new AtomicInteger(0)); + m.running = registry.gauge(MAJC_RUNNING.getName(), + Tags.of("id", ecm.ceid.canonical()), new AtomicInteger(0)); } return m; }); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java index 70b0c4980ba..f3f4ba2943e 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java @@ -18,6 +18,25 @@ */ package org.apache.accumulo.tserver.metrics; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_READ; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_WRITTEN; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_ENTRIES; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_HOLD; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_INGEST_BYTES; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_INGEST_MUTATIONS; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MAJC_QUEUED; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MAJC_RUNNING; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MAJC_STUCK; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MEM_ENTRIES; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MINC_QUEUED; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MINC_RUNNING; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MINC_TOTAL; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_ASSIGNMENTS_WARNING; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_FILES; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_ONLINE; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_OPENING; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_UNOPENED; + import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.server.compaction.CompactionWatcher; import org.apache.accumulo.server.compaction.FileCompactor; @@ -47,58 +66,62 @@ private long getTotalEntriesWritten() { @Override public void registerMetrics(MeterRegistry registry) { FunctionCounter - .builder(METRICS_COMPACTOR_ENTRIES_READ, this, TabletServerMetrics::getTotalEntriesRead) + .builder(COMPACTOR_ENTRIES_READ.getName(), this, TabletServerMetrics::getTotalEntriesRead) .description("Number of entries read by all compactions that have run on this tserver") .register(registry); FunctionCounter - .builder(METRICS_COMPACTOR_ENTRIES_WRITTEN, this, + .builder(COMPACTOR_ENTRIES_WRITTEN.getName(), this, TabletServerMetrics::getTotalEntriesWritten) .description("Number of entries written by all compactions that have run on this tserver") .register(registry); - LongTaskTimer timer = LongTaskTimer.builder(METRICS_TSERVER_MAJC_STUCK) + LongTaskTimer timer = LongTaskTimer.builder(TSERVER_MAJC_STUCK.getName()) .description("Number and duration of stuck major compactions").register(registry); CompactionWatcher.setTimer(timer); Gauge - .builder(METRICS_TSERVER_TABLETS_LONG_ASSIGNMENTS, util, + .builder(TSERVER_TABLETS_ASSIGNMENTS_WARNING.getName(), util, TabletServerMetricsUtil::getLongTabletAssignments) .description("Number of tablet assignments that are taking a long time").register(registry); - Gauge.builder(METRICS_TSERVER_ENTRIES, util, TabletServerMetricsUtil::getEntries) + Gauge.builder(TSERVER_ENTRIES.getName(), util, TabletServerMetricsUtil::getEntries) .description("Number of entries").register(registry); - Gauge.builder(METRICS_TSERVER_MEM_ENTRIES, util, TabletServerMetricsUtil::getEntriesInMemory) + Gauge.builder(TSERVER_MEM_ENTRIES.getName(), util, TabletServerMetricsUtil::getEntriesInMemory) .description("Number of entries in memory").register(registry); - Gauge.builder(METRICS_TSERVER_MAJC_RUNNING, util, TabletServerMetricsUtil::getMajorCompactions) + Gauge + .builder(TSERVER_MAJC_RUNNING.getName(), util, TabletServerMetricsUtil::getMajorCompactions) .description("Number of active major compactions").register(registry); Gauge - .builder(METRICS_TSERVER_MAJC_QUEUED, util, + .builder(TSERVER_MAJC_QUEUED.getName(), util, TabletServerMetricsUtil::getMajorCompactionsQueued) .description("Number of queued major compactions").register(registry); - Gauge.builder(METRICS_TSERVER_MINC_RUNNING, util, TabletServerMetricsUtil::getMinorCompactions) + Gauge + .builder(TSERVER_MINC_RUNNING.getName(), util, TabletServerMetricsUtil::getMinorCompactions) .description("Number of active minor compactions").register(registry); Gauge - .builder(METRICS_TSERVER_MINC_QUEUED, util, + .builder(TSERVER_MINC_QUEUED.getName(), util, TabletServerMetricsUtil::getMinorCompactionsQueued) .description("Number of queued minor compactions").register(registry); - Gauge.builder(METRICS_TSERVER_TABLETS_ONLINE, util, TabletServerMetricsUtil::getOnlineCount) + Gauge.builder(TSERVER_TABLETS_ONLINE.getName(), util, TabletServerMetricsUtil::getOnlineCount) .description("Number of online tablets").register(registry); - Gauge.builder(METRICS_TSERVER_TABLETS_OPENING, util, TabletServerMetricsUtil::getOpeningCount) + Gauge.builder(TSERVER_TABLETS_OPENING.getName(), util, TabletServerMetricsUtil::getOpeningCount) .description("Number of opening tablets").register(registry); - Gauge.builder(METRICS_TSERVER_TABLETS_UNOPENED, util, TabletServerMetricsUtil::getUnopenedCount) + Gauge + .builder(TSERVER_TABLETS_UNOPENED.getName(), util, + TabletServerMetricsUtil::getUnopenedCount) .description("Number of unopened tablets").register(registry); Gauge - .builder(METRICS_TSERVER_MINC_TOTAL, util, + .builder(TSERVER_MINC_TOTAL.getName(), util, TabletServerMetricsUtil::getTotalMinorCompactions) .description("Total number of minor compactions performed").register(registry); Gauge - .builder(METRICS_TSERVER_TABLETS_FILES, util, + .builder(TSERVER_TABLETS_FILES.getName(), util, TabletServerMetricsUtil::getAverageFilesPerTablet) .description("Number of files per tablet").register(registry); - Gauge.builder(METRICS_TSERVER_HOLD, util, TabletServerMetricsUtil::getHoldTime) + Gauge.builder(TSERVER_HOLD.getName(), util, TabletServerMetricsUtil::getHoldTime) .description("Time commits held").register(registry); - Gauge.builder(METRICS_TSERVER_INGEST_MUTATIONS, util, TabletServerMetricsUtil::getIngestCount) + Gauge.builder(TSERVER_INGEST_MUTATIONS.getName(), util, TabletServerMetricsUtil::getIngestCount) .description("Ingest rate (entries/sec)").register(registry); - Gauge.builder(METRICS_TSERVER_INGEST_BYTES, util, TabletServerMetricsUtil::getIngestByteCount) + Gauge.builder(TSERVER_INGEST_BYTES.getName(), util, TabletServerMetricsUtil::getIngestByteCount) .description("Ingest rate (bytes/sec)").register(registry); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java index bf95b002620..7bc3e5bc856 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java @@ -18,6 +18,9 @@ */ package org.apache.accumulo.tserver.metrics; +import static org.apache.accumulo.core.metrics.Metric.MINC_QUEUED; +import static org.apache.accumulo.core.metrics.Metric.MINC_RUNNING; + import java.time.Duration; import org.apache.accumulo.core.metrics.MetricsProducer; @@ -41,10 +44,10 @@ public void addQueued(long value) { @Override public void registerMetrics(MeterRegistry registry) { - activeMinc = Timer.builder(METRICS_MINC_RUNNING).description("Minor compactions time active") + activeMinc = Timer.builder(MINC_RUNNING.getName()).description("Minor compactions time active") .register(registry); - queuedMinc = Timer.builder(METRICS_MINC_QUEUED) + queuedMinc = Timer.builder(MINC_QUEUED.getName()) .description("Queued minor compactions time queued").register(registry); } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java index 612045cb90d..4270fa3efa2 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java @@ -18,6 +18,21 @@ */ package org.apache.accumulo.tserver.metrics; +import static org.apache.accumulo.core.metrics.Metric.SCAN_BUSY_TIMEOUT_COUNT; +import static org.apache.accumulo.core.metrics.Metric.SCAN_CLOSE; +import static org.apache.accumulo.core.metrics.Metric.SCAN_CONTINUE; +import static org.apache.accumulo.core.metrics.Metric.SCAN_OPEN_FILES; +import static org.apache.accumulo.core.metrics.Metric.SCAN_PAUSED_FOR_MEM; +import static org.apache.accumulo.core.metrics.Metric.SCAN_QUERIES; +import static org.apache.accumulo.core.metrics.Metric.SCAN_QUERY_SCAN_RESULTS; +import static org.apache.accumulo.core.metrics.Metric.SCAN_QUERY_SCAN_RESULTS_BYTES; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESULTS; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RETURN_FOR_MEM; +import static org.apache.accumulo.core.metrics.Metric.SCAN_START; +import static org.apache.accumulo.core.metrics.Metric.SCAN_TIMES; +import static org.apache.accumulo.core.metrics.Metric.SCAN_YIELDS; +import static org.apache.accumulo.core.metrics.Metric.SCAN_ZOMBIE_THREADS; + import java.time.Duration; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -121,36 +136,39 @@ public long getZombieThreadsCount() { @Override public void registerMetrics(MeterRegistry registry) { - Gauge.builder(METRICS_SCAN_OPEN_FILES, openFiles::get) + Gauge.builder(SCAN_OPEN_FILES.getName(), openFiles::get) .description("Number of files open for scans").register(registry); - scans = Timer.builder(METRICS_SCAN_TIMES).description("Scans").register(registry); - resultsPerScan = DistributionSummary.builder(METRICS_SCAN_RESULTS) + scans = Timer.builder(SCAN_TIMES.getName()).description("Scans").register(registry); + resultsPerScan = DistributionSummary.builder(SCAN_RESULTS.getName()) .description("Results per scan").register(registry); yields = - DistributionSummary.builder(METRICS_SCAN_YIELDS).description("yields").register(registry); - FunctionCounter.builder(METRICS_SCAN_START, this.startScanCalls, AtomicLong::get) + DistributionSummary.builder(SCAN_YIELDS.getName()).description("yields").register(registry); + FunctionCounter.builder(SCAN_START.getName(), this.startScanCalls, AtomicLong::get) .description("calls to start a scan / multiscan").register(registry); - FunctionCounter.builder(METRICS_SCAN_CONTINUE, this.continueScanCalls, AtomicLong::get) + FunctionCounter.builder(SCAN_CONTINUE.getName(), this.continueScanCalls, AtomicLong::get) .description("calls to continue a scan / multiscan").register(registry); - FunctionCounter.builder(METRICS_SCAN_CLOSE, this.closeScanCalls, AtomicLong::get) + FunctionCounter.builder(SCAN_CLOSE.getName(), this.closeScanCalls, AtomicLong::get) .description("calls to close a scan / multiscan").register(registry); FunctionCounter - .builder(METRICS_SCAN_BUSY_TIMEOUT_COUNTER, this.busyTimeoutCount, AtomicLong::get) + .builder(SCAN_BUSY_TIMEOUT_COUNT.getName(), this.busyTimeoutCount, AtomicLong::get) .description("The number of scans where a busy timeout happened").register(registry); - FunctionCounter.builder(METRICS_SCAN_QUERIES, this.lookupCount, LongAdder::sum) + FunctionCounter.builder(SCAN_QUERIES.getName(), this.lookupCount, LongAdder::sum) .description("Number of queries").register(registry); - FunctionCounter.builder(METRICS_SCAN_SCANNED_ENTRIES, this.scannedCount, LongAdder::sum) + FunctionCounter.builder(SCAN_QUERIES.getName(), this.scannedCount, LongAdder::sum) .description("Scanned rate").register(registry); - FunctionCounter.builder(METRICS_SCAN_PAUSED_FOR_MEM, this.pausedForMemory, AtomicLong::get) + FunctionCounter.builder(SCAN_PAUSED_FOR_MEM.getName(), this.pausedForMemory, AtomicLong::get) .description("scan paused due to server being low on memory").register(registry); - FunctionCounter.builder(METRICS_SCAN_RETURN_FOR_MEM, this.earlyReturnForMemory, AtomicLong::get) + FunctionCounter + .builder(SCAN_RETURN_FOR_MEM.getName(), this.earlyReturnForMemory, AtomicLong::get) .description("scan returned results early due to server being low on memory") .register(registry); - Gauge.builder(METRICS_SCAN_QUERY_SCAN_RESULTS, this.queryResultCount, LongAdder::sum) + Gauge.builder(SCAN_QUERY_SCAN_RESULTS.getName(), this.queryResultCount, LongAdder::sum) .description("Query rate (entries/sec)").register(registry); - Gauge.builder(METRICS_SCAN_QUERY_SCAN_RESULTS_BYTES, this.queryResultBytes, LongAdder::sum) + Gauge.builder(SCAN_QUERY_SCAN_RESULTS_BYTES.getName(), this.queryResultBytes, LongAdder::sum) .description("Query rate (bytes/sec)").register(registry); - Gauge.builder(METRICS_SCAN_ZOMBIE_THREADS, this, TabletServerScanMetrics::getZombieThreadsCount) + Gauge + .builder(SCAN_ZOMBIE_THREADS.getName(), this, + TabletServerScanMetrics::getZombieThreadsCount) .description("Number of scan threads that have no associated client session") .register(registry); } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerUpdateMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerUpdateMetrics.java index 1c30517e325..37beff251c3 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerUpdateMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerUpdateMetrics.java @@ -18,6 +18,12 @@ */ package org.apache.accumulo.tserver.metrics; +import static org.apache.accumulo.core.metrics.Metric.UPDATE_COMMIT; +import static org.apache.accumulo.core.metrics.Metric.UPDATE_COMMIT_PREP; +import static org.apache.accumulo.core.metrics.Metric.UPDATE_ERRORS; +import static org.apache.accumulo.core.metrics.Metric.UPDATE_MUTATION_ARRAY_SIZE; +import static org.apache.accumulo.core.metrics.Metric.UPDATE_WALOG_WRITE; + import java.time.Duration; import java.util.concurrent.atomic.AtomicLong; @@ -69,21 +75,21 @@ public void addMutationArraySize(long value) { @Override public void registerMetrics(MeterRegistry registry) { - FunctionCounter.builder(METRICS_UPDATE_ERRORS, permissionErrorsCount, AtomicLong::get) + FunctionCounter.builder(UPDATE_ERRORS.getName(), permissionErrorsCount, AtomicLong::get) .tags("type", "permission").description("Counts permission errors").register(registry); - FunctionCounter.builder(METRICS_UPDATE_ERRORS, unknownTabletErrorsCount, AtomicLong::get) + FunctionCounter.builder(UPDATE_ERRORS.getName(), unknownTabletErrorsCount, AtomicLong::get) .tags("type", "unknown.tablet").description("Counts unknown tablet errors") .register(registry); - FunctionCounter.builder(METRICS_UPDATE_ERRORS, constraintViolationsCount, AtomicLong::get) + FunctionCounter.builder(UPDATE_ERRORS.getName(), constraintViolationsCount, AtomicLong::get) .tags("type", "constraint.violation").description("Counts constraint violations") .register(registry); - commitPrepStat = Timer.builder(METRICS_UPDATE_COMMIT_PREP) + commitPrepStat = Timer.builder(UPDATE_COMMIT_PREP.getName()) .description("preparing to commit mutations").register(registry); - walogWriteTimeStat = Timer.builder(METRICS_UPDATE_WALOG_WRITE) + walogWriteTimeStat = Timer.builder(UPDATE_WALOG_WRITE.getName()) .description("writing mutations to WAL").register(registry); - commitTimeStat = - Timer.builder(METRICS_UPDATE_COMMIT).description("committing mutations").register(registry); - mutationArraySizeStat = DistributionSummary.builder(METRICS_UPDATE_MUTATION_ARRAY_SIZE) + commitTimeStat = Timer.builder(UPDATE_COMMIT.getName()).description("committing mutations") + .register(registry); + mutationArraySizeStat = DistributionSummary.builder(UPDATE_MUTATION_ARRAY_SIZE.getName()) .description("mutation array").register(registry); } diff --git a/test/src/main/java/org/apache/accumulo/test/ZombieScanIT.java b/test/src/main/java/org/apache/accumulo/test/ZombieScanIT.java index e8c3a47a3e4..adb2f04947e 100644 --- a/test/src/main/java/org/apache/accumulo/test/ZombieScanIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ZombieScanIT.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.test; +import static org.apache.accumulo.core.metrics.Metric.SCAN_ZOMBIE_THREADS; import static org.apache.accumulo.test.functional.ScannerIT.countActiveScans; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -45,7 +46,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.iterators.WrappingIterator; import org.apache.accumulo.core.metadata.schema.TabletMetadata; -import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory; import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; @@ -381,7 +381,7 @@ private Future startStuckBatchScan(AccumuloClient c, String table, private int getZombieScansMetric() { return sink.getLines().stream().map(TestStatsDSink::parseStatsDMetric) - .filter(metric -> metric.getName().equals(MetricsProducer.METRICS_SCAN_ZOMBIE_THREADS)) + .filter(metric -> metric.getName().equals(SCAN_ZOMBIE_THREADS.getName())) .mapToInt(metric -> Integer.parseInt(metric.getValue())).max().orElse(-1); } } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java index ff542538694..9bee84352db 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.test.compaction; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_MAJC_QUEUED; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE2; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; @@ -40,7 +41,6 @@ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; -import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.threads.Threads; @@ -127,7 +127,7 @@ public void testMetrics() throws Exception { if (shutdownTailer.get()) { break; } - if (s.startsWith(MetricsProducer.METRICS_MAJC_QUEUED)) { + if (s.startsWith(TSERVER_MAJC_QUEUED.getName())) { queueMetrics.add(TestStatsDSink.parseStatsDMetric(s)); } } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java index fb8e2d4cccd..a80299a6e69 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java @@ -20,6 +20,8 @@ import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_READ; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_WRITTEN; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable; @@ -54,7 +56,6 @@ import org.apache.accumulo.core.iterators.IteratorUtil; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; -import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.compaction.RunningCompactionInfo; import org.apache.accumulo.core.util.threads.Threads; @@ -282,18 +283,14 @@ private static Thread getMetricsCheckerThread(AtomicLong totalEntriesRead, break out; } TestStatsDSink.Metric metric = TestStatsDSink.parseStatsDMetric(s); - if (!metric.getName().startsWith(MetricsProducer.METRICS_COMPACTOR_PREFIX)) { - continue; - } + final String metricName = metric.getName(); int value = Integer.parseInt(metric.getValue()); - log.debug("Found metric: {} with value: {}", metric.getName(), value); - switch (metric.getName()) { - case MetricsProducer.METRICS_COMPACTOR_ENTRIES_READ: - totalEntriesRead.addAndGet(value); - break; - case MetricsProducer.METRICS_COMPACTOR_ENTRIES_WRITTEN: - totalEntriesWritten.addAndGet(value); - break; + if (metricName.equals(COMPACTOR_ENTRIES_READ.getName())) { + totalEntriesRead.set(value); + log.debug("Found metric: {} with value: {}", metricName, value); + } else if (metricName.equals(COMPACTOR_ENTRIES_WRITTEN.getName())) { + totalEntriesWritten.set(value); + log.debug("Found metric: {} with value: {}", metricName, value); } } sleepUninterruptibly(CHECKER_THREAD_SLEEP_MS, TimeUnit.MILLISECONDS); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java index ca54665742c..8ffa54fa5f7 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.test.functional; +import static org.apache.accumulo.core.metrics.Metric.SERVER_IDLE; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.MAX_DATA; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; @@ -42,7 +43,6 @@ import org.apache.accumulo.core.client.ScannerBase; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.iterators.IteratorUtil; -import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.harness.MiniClusterConfigurationCallback; import org.apache.accumulo.harness.SharedMiniClusterBase; @@ -128,8 +128,8 @@ public void testIdleStopMetrics() throws Exception { AtomicBoolean sawTServer = new AtomicBoolean(false); Wait.waitFor(() -> { List statsDMetrics = sink.getLines(); - statsDMetrics.stream().filter(line -> line.startsWith(MetricsProducer.METRICS_SERVER_IDLE)) - .peek(log::info).map(TestStatsDSink::parseStatsDMetric).forEach(a -> { + statsDMetrics.stream().filter(line -> line.startsWith(SERVER_IDLE.getName())).peek(log::info) + .map(TestStatsDSink::parseStatsDMetric).forEach(a -> { String processName = a.getTags().get("process.name"); int value = Integer.parseInt(a.getValue()); assertTrue(value == 0 || value == 1 || value == -1, "Unexpected value " + value); @@ -235,8 +235,7 @@ public void idleScanServerTest() throws Exception { private static void waitForIdleMetricValueToBe(int expectedValue, String processName) { Wait.waitFor( - () -> sink.getLines().stream() - .filter(line -> line.startsWith(MetricsProducer.METRICS_SERVER_IDLE)) + () -> sink.getLines().stream().filter(line -> line.startsWith(SERVER_IDLE.getName())) .map(TestStatsDSink::parseStatsDMetric) .filter(a -> a.getTags().get("process.name").equals(processName)) .peek(a -> log.info("Idle metric: {}", a)) diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java index 771d74d5881..b74e461b708 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.test.functional; +import static org.apache.accumulo.core.metrics.Metric.MINC_PAUSED; import static org.apache.accumulo.test.util.Wait.waitFor; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -34,7 +35,6 @@ import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.harness.MiniClusterConfigurationCallback; import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.minicluster.MemoryUnit; @@ -89,7 +89,7 @@ public static void start() throws Exception { } if (line.startsWith("accumulo")) { Metric metric = TestStatsDSink.parseStatsDMetric(line); - if (MetricsProducer.METRICS_MAJC_PAUSED.equals(metric.getName())) { + if (MINC_PAUSED.getName().equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); MAJC_PAUSED.add(val); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java index edb4cd03db8..d6ca8346ff4 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.test.functional; +import static org.apache.accumulo.core.metrics.Metric.MINC_PAUSED; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -32,7 +33,6 @@ import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.harness.MiniClusterConfigurationCallback; import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.minicluster.MemoryUnit; @@ -71,7 +71,7 @@ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreS } } - private static final DoubleAdder MINC_PAUSED = new DoubleAdder(); + private static final DoubleAdder MINC_PAUSED_COUNT = new DoubleAdder(); private static TestStatsDSink sink; private static Thread metricConsumer; @@ -87,9 +87,9 @@ public static void start() throws Exception { } if (line.startsWith("accumulo")) { Metric metric = TestStatsDSink.parseStatsDMetric(line); - if (MetricsProducer.METRICS_MINC_PAUSED.equals(metric.getName())) { + if (MINC_PAUSED.getName().equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); - MINC_PAUSED.add(val); + MINC_PAUSED_COUNT.add(val); } } } @@ -111,7 +111,7 @@ public static void stop() throws Exception { @BeforeEach public void beforeEach() { // Reset the client side counters - MINC_PAUSED.reset(); + MINC_PAUSED_COUNT.reset(); } @Test @@ -141,14 +141,14 @@ public void testMinCPauses() throws Exception { MemoryStarvedScanIT.consumeServerMemory(scanner); - int paused = MINC_PAUSED.intValue(); + int paused = MINC_PAUSED_COUNT.intValue(); assertEquals(0, paused); ingestThread.start(); while (paused <= 0) { Thread.sleep(1000); - paused = MINC_PAUSED.intValue(); + paused = MINC_PAUSED_COUNT.intValue(); } MemoryStarvedScanIT.freeServerMemory(client); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java index d2cb595b3d5..8e4da36bb94 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java @@ -20,7 +20,9 @@ import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.apache.accumulo.core.metrics.MetricsProducer.METRICS_LOW_MEMORY; +import static org.apache.accumulo.core.metrics.Metric.LOW_MEMORY; +import static org.apache.accumulo.core.metrics.Metric.SCAN_PAUSED_FOR_MEM; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RETURN_FOR_MEM; import static org.apache.accumulo.test.util.Wait.waitFor; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -44,7 +46,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.WrappingIterator; -import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory; import org.apache.accumulo.harness.MiniClusterConfigurationCallback; import org.apache.accumulo.harness.SharedMiniClusterBase; @@ -109,13 +110,13 @@ public static void start() throws Exception { } if (line.startsWith("accumulo")) { Metric metric = TestStatsDSink.parseStatsDMetric(line); - if (MetricsProducer.METRICS_SCAN_PAUSED_FOR_MEM.equals(metric.getName())) { + if (SCAN_PAUSED_FOR_MEM.getName().equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); SCAN_START_DELAYED.add(val); - } else if (MetricsProducer.METRICS_SCAN_RETURN_FOR_MEM.equals(metric.getName())) { + } else if (SCAN_RETURN_FOR_MEM.getName().equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); SCAN_RETURNED_EARLY.add(val); - } else if (metric.getName().equals(METRICS_LOW_MEMORY)) { + } else if (metric.getName().equals(LOW_MEMORY.getName())) { String process = metric.getTags().get("process.name"); if (process != null && process.contains("tserver")) { int val = Integer.parseInt(metric.getValue()); diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java index 3c1302cc1b7..78b0c32f0df 100644 --- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java @@ -18,6 +18,15 @@ */ package org.apache.accumulo.test.metrics; +import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_MAJC_STUCK; +import static org.apache.accumulo.core.metrics.Metric.FATE_TYPE_IN_PROGRESS; +import static org.apache.accumulo.core.metrics.Metric.MANAGER_BALANCER_MIGRATIONS_NEEDED; +import static org.apache.accumulo.core.metrics.Metric.SCAN_BUSY_TIMEOUT_COUNT; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_TOTAL_TIMER; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_WRITEOUT_TIMER; +import static org.apache.accumulo.core.metrics.Metric.SCAN_TABLET_METADATA_CACHE; +import static org.apache.accumulo.core.metrics.Metric.SCAN_YIELDS; +import static org.apache.accumulo.core.metrics.Metric.SERVER_IDLE; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -101,18 +110,18 @@ public void confirmMetricsPublished() throws Exception { // meter names sorted and formatting disabled to make it easier to diff changes // @formatter:off Set unexpectedMetrics = - Set.of(METRICS_COMPACTOR_MAJC_STUCK, - METRICS_SCAN_YIELDS); + Set.of(COMPACTOR_MAJC_STUCK.getName(), + SCAN_YIELDS.getName()); // add sserver as flaky until scan server included in mini tests. - Set flakyMetrics = Set.of(METRICS_FATE_TYPE_IN_PROGRESS, - METRICS_MANAGER_BALANCER_MIGRATIONS_NEEDED, - METRICS_SCAN_BUSY_TIMEOUT_COUNTER, - METRICS_SCAN_RESERVATION_CONFLICT_COUNTER, - METRICS_SCAN_RESERVATION_TOTAL_TIMER, - METRICS_SCAN_RESERVATION_WRITEOUT_TIMER, - METRICS_SCAN_TABLET_METADATA_CACHE, - METRICS_SERVER_IDLE); + Set flakyMetrics = Set.of(FATE_TYPE_IN_PROGRESS.getName(), + MANAGER_BALANCER_MIGRATIONS_NEEDED.getName(), + SCAN_BUSY_TIMEOUT_COUNT.getName(), + SCAN_RESERVATION_WRITEOUT_TIMER.getName(), + SCAN_RESERVATION_TOTAL_TIMER.getName(), + SCAN_RESERVATION_WRITEOUT_TIMER.getName(), + SCAN_TABLET_METADATA_CACHE.getName(), + SERVER_IDLE.getName()); // @formatter:on Map expectedMetricNames = this.getMetricFields(); From 1eab80b96cacf8d3a374133482561225aecce4f2 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Tue, 3 Sep 2024 15:44:59 -0400 Subject: [PATCH 04/13] Add fromName() method to Metric --- .../apache/accumulo/core/metrics/Metric.java | 21 ++++++ .../core/metrics/MetricsProducer.java | 26 -------- .../accumulo/tserver/ScanServerMetrics.java | 3 +- .../metrics/TabletServerScanMetrics.java | 3 +- .../accumulo/test/metrics/MetricsIT.java | 64 ++++++++++--------- 5 files changed, 60 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java index 57fe19de5b6..34095c16e62 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -18,6 +18,9 @@ */ package org.apache.accumulo.core.metrics; +import java.util.HashMap; +import java.util.Map; + public enum Metric { // General Server Metrics SERVER_IDLE("accumulo.server.idle", MetricType.GAUGE, @@ -107,6 +110,8 @@ public enum Metric { "Time to reserve a tablet's files for scan.", MetricCategory.SCAN_SERVER), SCAN_RESERVATION_WRITEOUT_TIMER("accumulo.scan.reservation.writeout.timer", MetricType.TIMER, "Time to write out a tablets file reservations for scan", MetricCategory.SCAN_SERVER), + SCAN_RESERVATION_CONFLICT_TIMER("accumulo.scan.reservation.conflict.timer", MetricType.TIMER, "", + MetricCategory.SCAN_SERVER), SCAN_BUSY_TIMEOUT_COUNT("accumulo.scan.busy.timeout.count", MetricType.COUNTER, "Count of the scans where a busy timeout happened.", MetricCategory.SCAN_SERVER), SCAN_TABLET_METADATA_CACHE("accumulo.scan.tablet.metadata.cache", MetricType.CACHE, @@ -203,6 +208,14 @@ public enum Metric { private final String description; private final MetricCategory category; + private static final Map NAME_TO_ENUM_MAP = new HashMap<>(); + + static { + for (Metric metric : values()) { + NAME_TO_ENUM_MAP.put(metric.name, metric); + } + } + Metric(String name, MetricType type, String description, MetricCategory category) { this.name = name; this.type = type; @@ -242,4 +255,12 @@ public enum MetricCategory { MANAGER } + public static Metric fromName(String name) { + Metric metric = NAME_TO_ENUM_MAP.get(name); + if (metric == null) { + throw new IllegalArgumentException("No enum constant for metric name: " + name); + } + return metric; + } + } diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java index 26ec8e5a038..48e1e759ff5 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java @@ -18,11 +18,6 @@ */ package org.apache.accumulo.core.metrics; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.HashMap; -import java.util.Map; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,25 +42,4 @@ public interface MetricsProducer { */ void registerMetrics(MeterRegistry registry); - /** - * Returns a new mutable mapping of metric field value to metric field name. - * - * @return map of field names to variable names. - */ - default Map getMetricFields() { - Map fields = new HashMap<>(); - for (Field f : MetricsProducer.class.getDeclaredFields()) { - if (Modifier.isStatic(f.getModifiers()) && f.getType().equals(String.class) - && !f.getName().contains("PREFIX")) { - try { - - fields.put((String) f.get(MetricsProducer.class), f.getName()); - } catch (IllegalArgumentException | IllegalAccessException e) { - // this shouldn't happen, but let's log it anyway - LOG.error("Error getting metric value for field: {}", f.getName()); - } - } - } - return fields; - } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java index b91a410f88d..8ebf51e1445 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java @@ -19,6 +19,7 @@ package org.apache.accumulo.tserver; import static org.apache.accumulo.core.metrics.Metric.SCAN_BUSY_TIMEOUT_COUNT; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_CONFLICT_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_TOTAL_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_WRITEOUT_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_TABLET_METADATA_CACHE; @@ -61,7 +62,7 @@ public void registerMetrics(MeterRegistry registry) { FunctionCounter.builder(SCAN_BUSY_TIMEOUT_COUNT.getName(), busyTimeoutCount, AtomicLong::get) .description("The number of scans where a busy timeout happened").register(registry); FunctionCounter - .builder(SCAN_RESERVATION_WRITEOUT_TIMER.getName(), reservationConflictCount, + .builder(SCAN_RESERVATION_CONFLICT_TIMER.getName(), reservationConflictCount, AtomicLong::get) .description( "Counts instances where file reservation attempts for scans encountered conflicts") diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java index 4270fa3efa2..d5848722cba 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java @@ -28,6 +28,7 @@ import static org.apache.accumulo.core.metrics.Metric.SCAN_QUERY_SCAN_RESULTS_BYTES; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESULTS; import static org.apache.accumulo.core.metrics.Metric.SCAN_RETURN_FOR_MEM; +import static org.apache.accumulo.core.metrics.Metric.SCAN_SCANNED_ENTRIES; import static org.apache.accumulo.core.metrics.Metric.SCAN_START; import static org.apache.accumulo.core.metrics.Metric.SCAN_TIMES; import static org.apache.accumulo.core.metrics.Metric.SCAN_YIELDS; @@ -154,7 +155,7 @@ public void registerMetrics(MeterRegistry registry) { .description("The number of scans where a busy timeout happened").register(registry); FunctionCounter.builder(SCAN_QUERIES.getName(), this.lookupCount, LongAdder::sum) .description("Number of queries").register(registry); - FunctionCounter.builder(SCAN_QUERIES.getName(), this.scannedCount, LongAdder::sum) + FunctionCounter.builder(SCAN_SCANNED_ENTRIES.getName(), this.scannedCount, LongAdder::sum) .description("Scanned rate").register(registry); FunctionCounter.builder(SCAN_PAUSED_FOR_MEM.getName(), this.pausedForMemory, AtomicLong::get) .description("scan paused due to server being low on memory").register(registry); diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java index 78b0c32f0df..91404625953 100644 --- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java @@ -22,6 +22,7 @@ import static org.apache.accumulo.core.metrics.Metric.FATE_TYPE_IN_PROGRESS; import static org.apache.accumulo.core.metrics.Metric.MANAGER_BALANCER_MIGRATIONS_NEEDED; import static org.apache.accumulo.core.metrics.Metric.SCAN_BUSY_TIMEOUT_COUNT; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_CONFLICT_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_TOTAL_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_WRITEOUT_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_TABLET_METADATA_CACHE; @@ -34,13 +35,13 @@ import static org.junit.jupiter.api.Assertions.fail; import java.time.Duration; -import java.util.HashMap; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; -import java.util.function.Predicate; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; @@ -51,11 +52,11 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metrics.Metric; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.test.functional.ConfigurableMacBase; -import org.apache.accumulo.test.metrics.TestStatsDSink.Metric; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.AfterAll; @@ -109,50 +110,55 @@ public void confirmMetricsPublished() throws Exception { cluster.stop(); // meter names sorted and formatting disabled to make it easier to diff changes // @formatter:off - Set unexpectedMetrics = - Set.of(COMPACTOR_MAJC_STUCK.getName(), - SCAN_YIELDS.getName()); + Set unexpectedMetrics = Set.of( + COMPACTOR_MAJC_STUCK, + SCAN_YIELDS + ); // add sserver as flaky until scan server included in mini tests. - Set flakyMetrics = Set.of(FATE_TYPE_IN_PROGRESS.getName(), - MANAGER_BALANCER_MIGRATIONS_NEEDED.getName(), - SCAN_BUSY_TIMEOUT_COUNT.getName(), - SCAN_RESERVATION_WRITEOUT_TIMER.getName(), - SCAN_RESERVATION_TOTAL_TIMER.getName(), - SCAN_RESERVATION_WRITEOUT_TIMER.getName(), - SCAN_TABLET_METADATA_CACHE.getName(), - SERVER_IDLE.getName()); + Set flakyMetrics = Set.of( + FATE_TYPE_IN_PROGRESS, + MANAGER_BALANCER_MIGRATIONS_NEEDED, + SCAN_BUSY_TIMEOUT_COUNT, + SCAN_RESERVATION_CONFLICT_TIMER, + SCAN_RESERVATION_TOTAL_TIMER, + SCAN_RESERVATION_WRITEOUT_TIMER, + SCAN_TABLET_METADATA_CACHE, + SERVER_IDLE + ); // @formatter:on - Map expectedMetricNames = this.getMetricFields(); - flakyMetrics.forEach(expectedMetricNames::remove); // might not see these - unexpectedMetrics.forEach(expectedMetricNames::remove); // definitely shouldn't see these - assertFalse(expectedMetricNames.isEmpty()); // make sure we didn't remove everything + Set expectedMetrics = new HashSet<>(Arrays.asList(Metric.values())); + expectedMetrics.removeAll(flakyMetrics); // might not see these + expectedMetrics.removeAll(unexpectedMetrics); // definitely shouldn't see these + assertFalse(expectedMetrics.isEmpty()); // make sure we didn't remove everything - Map seenMetricNames = new HashMap<>(); + Set seenMetrics = new HashSet<>(); List statsDMetrics; // loop until we run out of lines or until we see all expected metrics - while (!(statsDMetrics = sink.getLines()).isEmpty() && !expectedMetricNames.isEmpty()) { + while (!(statsDMetrics = sink.getLines()).isEmpty() && !expectedMetrics.isEmpty()) { // for each metric name not yet seen, check if it is expected, flaky, or unknown statsDMetrics.stream().filter(line -> line.startsWith("accumulo")) - .map(TestStatsDSink::parseStatsDMetric).map(Metric::getName) - .filter(Predicate.not(seenMetricNames::containsKey)).forEach(name -> { - if (expectedMetricNames.containsKey(name)) { - // record expected metric names as seen, along with the value seen - seenMetricNames.put(name, expectedMetricNames.remove(name)); - } else if (flakyMetrics.contains(name)) { + .map(TestStatsDSink::parseStatsDMetric).map(metric -> Metric.fromName(metric.getName())) + .filter(metric -> !seenMetrics.contains(metric)).forEach(metric -> { + if (expectedMetrics.contains(metric)) { + // record expected Metric as seen + seenMetrics.add(metric); + expectedMetrics.remove(metric); + } else if (flakyMetrics.contains(metric)) { // ignore any flaky metric names seen // these aren't always expected, but we shouldn't be surprised if we see them } else { // completely unexpected metric - fail("Found accumulo metric not in expectedMetricNames or flakyMetricNames: " + name); + fail("Found accumulo metric not in expectedMetricNames or flakyMetricNames: " + + metric); } }); } - assertTrue(expectedMetricNames.isEmpty(), - "Did not see all expected metric names, missing: " + expectedMetricNames.values()); + assertTrue(expectedMetrics.isEmpty(), + "Did not see all expected metric names, missing: " + expectedMetrics); } private void doWorkToGenerateMetrics() throws Exception { From bb2365aab6cf688ad66b900d369a1e31f4c9baaf Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Tue, 3 Sep 2024 16:13:02 -0400 Subject: [PATCH 05/13] Simplify map to set --- .../accumulo/core/metrics/MetricsDocGen.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java index bc0d489a41c..bc627d14962 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.io.PrintStream; -import java.util.TreeMap; +import java.util.Arrays; +import java.util.Comparator; +import java.util.TreeSet; /** * This class generates documentation to inform users of the available metrics in a presentable @@ -30,7 +32,12 @@ */ public class MetricsDocGen { private final PrintStream doc; - private final TreeMap sortedMetrics = new TreeMap<>(); + private final TreeSet sortedMetrics = + new TreeSet<>(Comparator.comparing(Metric::getName)) { + { + addAll(Arrays.asList(Metric.values())); + } + }; void generate() { pageHeader(); @@ -58,7 +65,7 @@ void pageHeader() { void generateCategorySection(Metric.MetricCategory category, String sectionTitle) { beginSection(sectionTitle); - for (Metric metric : sortedMetrics.values()) { + for (Metric metric : sortedMetrics) { if (metric.getCategory() == category) { generateMetricSubsection(metric); } @@ -77,9 +84,6 @@ void generateMetricSubsection(Metric metric) { private MetricsDocGen(PrintStream doc) { this.doc = doc; - for (Metric metric : Metric.values()) { - this.sortedMetrics.put(metric.getName(), metric); - } } /** From 3a265ed1fa98eee646d4a305c76374b699c274cf Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Tue, 3 Sep 2024 16:41:25 -0400 Subject: [PATCH 06/13] fix error prone error --- .../org/apache/accumulo/core/metrics/MetricsDocGen.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java index bc627d14962..bd8080a7513 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java @@ -33,11 +33,7 @@ public class MetricsDocGen { private final PrintStream doc; private final TreeSet sortedMetrics = - new TreeSet<>(Comparator.comparing(Metric::getName)) { - { - addAll(Arrays.asList(Metric.values())); - } - }; + new TreeSet<>(Comparator.comparing(Metric::getName)); void generate() { pageHeader(); @@ -84,6 +80,7 @@ void generateMetricSubsection(Metric metric) { private MetricsDocGen(PrintStream doc) { this.doc = doc; + this.sortedMetrics.addAll(Arrays.asList(Metric.values())); } /** From 139ff5c45f8205f6afbab6eecf0cb9f4688a3751 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Wed, 4 Sep 2024 15:07:55 -0400 Subject: [PATCH 07/13] Add metrics markdown generate section to pom --- core/pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/core/pom.xml b/core/pom.xml index 990f853a598..06d2d8511c8 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -387,6 +387,21 @@ + + metrics-markdown + + java + + package + + org.apache.accumulo.core.metrics.MetricsDocGen + test + + --generate-markdown + ${project.build.directory}/generated-docs/metrics3.md + + + From 7a40eecb9065d822eadcad7c497f05f3fff32dfb Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Thu, 5 Sep 2024 16:44:07 -0400 Subject: [PATCH 08/13] Code review --- .../java/org/apache/accumulo/core/metrics/Metric.java | 8 ++++---- .../apache/accumulo/core/metrics/MetricsDocGen.java | 6 ++---- .../org/apache/accumulo/tserver/ScanServerMetrics.java | 4 ++-- .../accumulo/tserver/metrics/TabletServerMetrics.java | 4 ++-- .../test/compaction/ExternalCompactionMetricsIT.java | 4 ++-- .../test/compaction/ExternalCompactionProgressIT.java | 10 ++++++---- .../accumulo/test/functional/MemoryStarvedMinCIT.java | 4 ++-- .../org/apache/accumulo/test/metrics/MetricsIT.java | 4 ++-- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java index 34095c16e62..8042da71ab9 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -89,8 +89,8 @@ public enum Metric { MetricCategory.TABLET_SERVER), TSERVER_TABLETS_ONLINE("accumulo.tserver.tablets.online", MetricType.GAUGE, "", MetricCategory.TABLET_SERVER), - TSERVER_TABLETS_ASSIGNMENTS_WARNING("accumulo.tserver.tablets.assignments.warning", - MetricType.GAUGE, "", MetricCategory.TABLET_SERVER), + TSERVER_TABLETS_LONG_ASSIGNMENTS("accumulo.tserver.tablets.assignments.warning", MetricType.GAUGE, + "", MetricCategory.TABLET_SERVER), TSERVER_TABLETS_OPENING("accumulo.tserver.tablets.opening", MetricType.GAUGE, "", MetricCategory.TABLET_SERVER), TSERVER_TABLETS_UNOPENED("accumulo.tserver.tablets.unopened", MetricType.GAUGE, "", @@ -110,8 +110,8 @@ public enum Metric { "Time to reserve a tablet's files for scan.", MetricCategory.SCAN_SERVER), SCAN_RESERVATION_WRITEOUT_TIMER("accumulo.scan.reservation.writeout.timer", MetricType.TIMER, "Time to write out a tablets file reservations for scan", MetricCategory.SCAN_SERVER), - SCAN_RESERVATION_CONFLICT_TIMER("accumulo.scan.reservation.conflict.timer", MetricType.TIMER, "", - MetricCategory.SCAN_SERVER), + SCAN_RESERVATION_CONFLICT_COUNTER("accumulo.scan.reservation.conflict.timer", MetricType.COUNTER, + "", MetricCategory.SCAN_SERVER), SCAN_BUSY_TIMEOUT_COUNT("accumulo.scan.busy.timeout.count", MetricType.COUNTER, "Count of the scans where a busy timeout happened.", MetricCategory.SCAN_SERVER), SCAN_TABLET_METADATA_CACHE("accumulo.scan.tablet.metadata.cache", MetricType.CACHE, diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java index bd8080a7513..ab542773773 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java @@ -43,15 +43,13 @@ void generate() { generateCategorySection(Metric.MetricCategory.COMPACTOR, "Compactor Metrics"); generateCategorySection(Metric.MetricCategory.SCAN_SERVER, "Scan Server Metrics"); generateCategorySection(Metric.MetricCategory.FATE, "Fate Metrics"); - - doc.close(); } void pageHeader() { doc.println("---"); doc.println("title: Metrics Documentation (3.x)"); - doc.println("category: metrics"); - doc.println("order: 6"); + doc.println("category: configuration"); + doc.println("order: 7"); doc.println("---\n"); doc.println("\n"); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java index 8ebf51e1445..12d85f1d22d 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java @@ -19,7 +19,7 @@ package org.apache.accumulo.tserver; import static org.apache.accumulo.core.metrics.Metric.SCAN_BUSY_TIMEOUT_COUNT; -import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_CONFLICT_TIMER; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_CONFLICT_COUNTER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_TOTAL_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_WRITEOUT_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_TABLET_METADATA_CACHE; @@ -62,7 +62,7 @@ public void registerMetrics(MeterRegistry registry) { FunctionCounter.builder(SCAN_BUSY_TIMEOUT_COUNT.getName(), busyTimeoutCount, AtomicLong::get) .description("The number of scans where a busy timeout happened").register(registry); FunctionCounter - .builder(SCAN_RESERVATION_CONFLICT_TIMER.getName(), reservationConflictCount, + .builder(SCAN_RESERVATION_CONFLICT_COUNTER.getName(), reservationConflictCount, AtomicLong::get) .description( "Counts instances where file reservation attempts for scans encountered conflicts") diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java index f3f4ba2943e..26bbd11adbf 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java @@ -31,8 +31,8 @@ import static org.apache.accumulo.core.metrics.Metric.TSERVER_MINC_QUEUED; import static org.apache.accumulo.core.metrics.Metric.TSERVER_MINC_RUNNING; import static org.apache.accumulo.core.metrics.Metric.TSERVER_MINC_TOTAL; -import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_ASSIGNMENTS_WARNING; import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_FILES; +import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_LONG_ASSIGNMENTS; import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_ONLINE; import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_OPENING; import static org.apache.accumulo.core.metrics.Metric.TSERVER_TABLETS_UNOPENED; @@ -78,7 +78,7 @@ public void registerMetrics(MeterRegistry registry) { .description("Number and duration of stuck major compactions").register(registry); CompactionWatcher.setTimer(timer); Gauge - .builder(TSERVER_TABLETS_ASSIGNMENTS_WARNING.getName(), util, + .builder(TSERVER_TABLETS_LONG_ASSIGNMENTS.getName(), util, TabletServerMetricsUtil::getLongTabletAssignments) .description("Number of tablet assignments that are taking a long time").register(registry); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java index 9bee84352db..3ebd198b703 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java @@ -18,7 +18,7 @@ */ package org.apache.accumulo.test.compaction; -import static org.apache.accumulo.core.metrics.Metric.TSERVER_MAJC_QUEUED; +import static org.apache.accumulo.core.metrics.Metric.MAJC_QUEUED; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE2; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; @@ -127,7 +127,7 @@ public void testMetrics() throws Exception { if (shutdownTailer.get()) { break; } - if (s.startsWith(TSERVER_MAJC_QUEUED.getName())) { + if (s.startsWith(MAJC_QUEUED.getName())) { queueMetrics.add(TestStatsDSink.parseStatsDMetric(s)); } } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java index a80299a6e69..278c8a4e96c 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java @@ -284,13 +284,15 @@ private static Thread getMetricsCheckerThread(AtomicLong totalEntriesRead, } TestStatsDSink.Metric metric = TestStatsDSink.parseStatsDMetric(s); final String metricName = metric.getName(); + if (!metricName.startsWith("accumulo.compactor.entries")) { + continue; + } int value = Integer.parseInt(metric.getValue()); + log.debug("Found metric: {} with value: {}", metricName, value); if (metricName.equals(COMPACTOR_ENTRIES_READ.getName())) { - totalEntriesRead.set(value); - log.debug("Found metric: {} with value: {}", metricName, value); + totalEntriesRead.addAndGet(value); } else if (metricName.equals(COMPACTOR_ENTRIES_WRITTEN.getName())) { - totalEntriesWritten.set(value); - log.debug("Found metric: {} with value: {}", metricName, value); + totalEntriesWritten.addAndGet(value); } } sleepUninterruptibly(CHECKER_THREAD_SLEEP_MS, TimeUnit.MILLISECONDS); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java index d6ca8346ff4..faf746fbc9e 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java @@ -18,7 +18,7 @@ */ package org.apache.accumulo.test.functional; -import static org.apache.accumulo.core.metrics.Metric.MINC_PAUSED; +import static org.apache.accumulo.core.metrics.Metric.MAJC_PAUSED; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -87,7 +87,7 @@ public static void start() throws Exception { } if (line.startsWith("accumulo")) { Metric metric = TestStatsDSink.parseStatsDMetric(line); - if (MINC_PAUSED.getName().equals(metric.getName())) { + if (MAJC_PAUSED.getName().equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); MINC_PAUSED_COUNT.add(val); } diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java index 91404625953..c77c41d049e 100644 --- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java @@ -22,7 +22,7 @@ import static org.apache.accumulo.core.metrics.Metric.FATE_TYPE_IN_PROGRESS; import static org.apache.accumulo.core.metrics.Metric.MANAGER_BALANCER_MIGRATIONS_NEEDED; import static org.apache.accumulo.core.metrics.Metric.SCAN_BUSY_TIMEOUT_COUNT; -import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_CONFLICT_TIMER; +import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_CONFLICT_COUNTER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_TOTAL_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_RESERVATION_WRITEOUT_TIMER; import static org.apache.accumulo.core.metrics.Metric.SCAN_TABLET_METADATA_CACHE; @@ -120,7 +120,7 @@ public void confirmMetricsPublished() throws Exception { FATE_TYPE_IN_PROGRESS, MANAGER_BALANCER_MIGRATIONS_NEEDED, SCAN_BUSY_TIMEOUT_COUNT, - SCAN_RESERVATION_CONFLICT_TIMER, + SCAN_RESERVATION_CONFLICT_COUNTER, SCAN_RESERVATION_TOTAL_TIMER, SCAN_RESERVATION_WRITEOUT_TIMER, SCAN_TABLET_METADATA_CACHE, From 5f91018b9e667ecb4a8312ac4975cfdced2ff1d4 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Thu, 5 Sep 2024 16:51:46 -0400 Subject: [PATCH 09/13] Code review --- .../apache/accumulo/test/functional/MemoryStarvedMajCIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java index b74e461b708..0fb0d9e4b57 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java @@ -18,7 +18,6 @@ */ package org.apache.accumulo.test.functional; -import static org.apache.accumulo.core.metrics.Metric.MINC_PAUSED; import static org.apache.accumulo.test.util.Wait.waitFor; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -89,7 +88,8 @@ public static void start() throws Exception { } if (line.startsWith("accumulo")) { Metric metric = TestStatsDSink.parseStatsDMetric(line); - if (MINC_PAUSED.getName().equals(metric.getName())) { + if (org.apache.accumulo.core.metrics.Metric.MAJC_PAUSED.getName() + .equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); MAJC_PAUSED.add(val); } From dcf41183f083fe96f336e23c72adb0a91e88c661 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Fri, 6 Sep 2024 11:44:41 -0400 Subject: [PATCH 10/13] Generate sections for all metric categories --- .../apache/accumulo/core/metrics/Metric.java | 28 +++++++++++++------ .../accumulo/core/metrics/MetricsDocGen.java | 8 ++---- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java index 8042da71ab9..7a0f1daddea 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -244,15 +244,25 @@ public enum MetricType { } public enum MetricCategory { - GENERAL_SERVER, - COMPACTOR, - FATE, - GARBAGE_COLLECTION, - TABLET_SERVER, - SCAN_SERVER, - THRIFT, - BLOCK_CACHE, - MANAGER + GENERAL_SERVER("General Server Metrics"), + COMPACTOR("Compactor Metrics"), + FATE("Fate Metrics"), + GARBAGE_COLLECTION("Garbage Collection Metrics"), + TABLET_SERVER("Tablet Server Metrics"), + SCAN_SERVER("Scan Server Metrics"), + THRIFT("Thrift Metrics"), + BLOCK_CACHE("Block Cache Metrics"), + MANAGER("Manager Metrics"); + + private final String sectionTitle; + + MetricCategory(String sectionTitle) { + this.sectionTitle = sectionTitle; + } + + public String getSectionTitle() { + return sectionTitle; + } } public static Metric fromName(String name) { diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java index ab542773773..9b75092b231 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java @@ -38,11 +38,9 @@ public class MetricsDocGen { void generate() { pageHeader(); - // Generate sections for each category of metrics - generateCategorySection(Metric.MetricCategory.GENERAL_SERVER, "General Server Metrics"); - generateCategorySection(Metric.MetricCategory.COMPACTOR, "Compactor Metrics"); - generateCategorySection(Metric.MetricCategory.SCAN_SERVER, "Scan Server Metrics"); - generateCategorySection(Metric.MetricCategory.FATE, "Fate Metrics"); + for (var category : Metric.MetricCategory.values()) { + generateCategorySection(category, category.getSectionTitle()); + } } void pageHeader() { From 7f68e4137b6bd0af071a33aab10815887af429c2 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Mon, 9 Sep 2024 12:43:12 -0400 Subject: [PATCH 11/13] Fix typos when converting to new Metrics Co-authored-by: Kevin Rathbun --- .../org/apache/accumulo/core/metrics/Metric.java | 2 +- .../test/functional/MemoryStarvedMajCIT.java | 14 +++++++------- .../test/functional/MemoryStarvedMinCIT.java | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java index 7a0f1daddea..8f11fc763fe 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -110,7 +110,7 @@ public enum Metric { "Time to reserve a tablet's files for scan.", MetricCategory.SCAN_SERVER), SCAN_RESERVATION_WRITEOUT_TIMER("accumulo.scan.reservation.writeout.timer", MetricType.TIMER, "Time to write out a tablets file reservations for scan", MetricCategory.SCAN_SERVER), - SCAN_RESERVATION_CONFLICT_COUNTER("accumulo.scan.reservation.conflict.timer", MetricType.COUNTER, + SCAN_RESERVATION_CONFLICT_COUNTER("accumulo.scan.reservation.conflict.count", MetricType.COUNTER, "", MetricCategory.SCAN_SERVER), SCAN_BUSY_TIMEOUT_COUNT("accumulo.scan.busy.timeout.count", MetricType.COUNTER, "Count of the scans where a busy timeout happened.", MetricCategory.SCAN_SERVER), diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java index 0fb0d9e4b57..eeb95017a0a 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.test.functional; +import static org.apache.accumulo.core.metrics.Metric.MAJC_PAUSED; import static org.apache.accumulo.test.util.Wait.waitFor; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -72,7 +73,7 @@ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreS } } - private static final DoubleAdder MAJC_PAUSED = new DoubleAdder(); + private static final DoubleAdder MAJC_PAUSED_COUNT = new DoubleAdder(); private static TestStatsDSink sink; private static Thread metricConsumer; @@ -88,10 +89,9 @@ public static void start() throws Exception { } if (line.startsWith("accumulo")) { Metric metric = TestStatsDSink.parseStatsDMetric(line); - if (org.apache.accumulo.core.metrics.Metric.MAJC_PAUSED.getName() - .equals(metric.getName())) { + if (MAJC_PAUSED.getName().equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); - MAJC_PAUSED.add(val); + MAJC_PAUSED_COUNT.add(val); } } } @@ -113,7 +113,7 @@ public static void stop() throws Exception { @BeforeEach public void beforeEach() { // Reset the client side counters - MAJC_PAUSED.reset(); + MAJC_PAUSED_COUNT.reset(); } @Test @@ -141,13 +141,13 @@ public void testMajCPauses() throws Exception { MemoryStarvedScanIT.consumeServerMemory(scanner); - int paused = MAJC_PAUSED.intValue(); + int paused = MAJC_PAUSED_COUNT.intValue(); assertEquals(0, paused); ReadWriteIT.ingest(client, 100, 100, 100, 0, table); compactionThread.start(); - waitFor(() -> MAJC_PAUSED.intValue() > 0); + waitFor(() -> MAJC_PAUSED_COUNT.intValue() > 0); MemoryStarvedScanIT.freeServerMemory(client); compactionThread.interrupt(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java index faf746fbc9e..d6ca8346ff4 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java @@ -18,7 +18,7 @@ */ package org.apache.accumulo.test.functional; -import static org.apache.accumulo.core.metrics.Metric.MAJC_PAUSED; +import static org.apache.accumulo.core.metrics.Metric.MINC_PAUSED; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -87,7 +87,7 @@ public static void start() throws Exception { } if (line.startsWith("accumulo")) { Metric metric = TestStatsDSink.parseStatsDMetric(line); - if (MAJC_PAUSED.getName().equals(metric.getName())) { + if (MINC_PAUSED.getName().equals(metric.getName())) { double val = Double.parseDouble(metric.getValue()); MINC_PAUSED_COUNT.add(val); } From 8b74eec3c4a318046007a21eb17f03aa06248da7 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Wed, 11 Sep 2024 13:42:48 -0400 Subject: [PATCH 12/13] Fix markdown formatting for underlines --- .../main/java/org/apache/accumulo/core/metrics/Metric.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java index 8f11fc763fe..9850d1a0646 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java @@ -142,10 +142,10 @@ public enum Metric { // Major Compaction Metrics MAJC_QUEUED("accumulo.tserver.compactions.majc.queued", MetricType.GAUGE, - "The compaction service information is in a tag: id={i|e}_{compactionServiceName}_{executor_name}.", + "The compaction service information is in a tag: `id={i|e}_{compactionServiceName}_{executor_name}`.", MetricCategory.TABLET_SERVER), MAJC_RUNNING("accumulo.tserver.compactions.majc.running", MetricType.GAUGE, - "The compaction service information is in a tag: id={i|e}_{compactionServiceName}_{executor_name}.", + "The compaction service information is in a tag: `id={i|e}_{compactionServiceName}_{executor_name}`.", MetricCategory.TABLET_SERVER), MAJC_PAUSED("accumulo.tserver.compactions.majc.paused", MetricType.COUNTER, "", MetricCategory.TABLET_SERVER), From e818f6d6f2f0e9c8c7a3ad3e9de2f95a128f0c6a Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Thu, 12 Sep 2024 14:48:46 -0400 Subject: [PATCH 13/13] Add styling to metric sections --- .../accumulo/core/metrics/MetricsDocGen.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java index 9b75092b231..e07bd10748c 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsDocGen.java @@ -57,11 +57,17 @@ void pageHeader() { void generateCategorySection(Metric.MetricCategory category, String sectionTitle) { beginSection(sectionTitle); + // Enable block-level HTML parsing + doc.println("{::options parse_block_html=\"true\" /}"); + for (Metric metric : sortedMetrics) { if (metric.getCategory() == category) { generateMetricSubsection(metric); } } + + // Disable block-level HTML parsing after the section + doc.println("{::options parse_block_html=\"false\" /}\n"); } void beginSection(String sectionTitle) { @@ -69,9 +75,15 @@ void beginSection(String sectionTitle) { } void generateMetricSubsection(Metric metric) { - doc.println("### " + metric.getName() + "\n"); - doc.println("**Type:** " + metric.getType().name() + "\n"); - doc.println("**Description:** " + metric.getDescription() + "\n"); + // Open the div block with markdown enabled + doc.println("
"); + + // Metric details + doc.println("### " + metric.getName()); + doc.println("**Type:** " + metric.getType().name() + " "); // Ensuring a line break in Markdown + doc.println("**Description:** " + metric.getDescription()); + + doc.println("
"); } private MetricsDocGen(PrintStream doc) {