From f8238bf3dfddba7673566993277671815556950a Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Mon, 23 Sep 2024 19:56:51 +0530 Subject: [PATCH 01/12] URI path filtering support in cluster stats API Signed-off-by: Swetha Guptha --- .../admin/cluster/stats/ClusterStatsIT.java | 113 ++++++++++++++++ .../cluster/stats/ClusterStatsIndices.java | 117 ++++++++++++---- .../cluster/stats/ClusterStatsNodes.java | 109 +++++++++------ .../cluster/stats/ClusterStatsRequest.java | 128 ++++++++++++++++++ .../stats/ClusterStatsRequestBuilder.java | 14 ++ .../cluster/stats/ClusterStatsResponse.java | 61 +++++++-- .../stats/TransportClusterStatsAction.java | 71 +++++----- .../admin/cluster/RestClusterStatsAction.java | 126 ++++++++++++++++- .../ClusterStatsRequestBuilderTests.java | 70 ++++++++++ 9 files changed, 699 insertions(+), 110 deletions(-) create mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index f23cdbb50b37a..ed28fffb05c75 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -37,6 +37,8 @@ import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetrics; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.client.Client; import org.opensearch.client.Requests; import org.opensearch.cluster.health.ClusterHealthStatus; @@ -505,6 +507,117 @@ public void testNodeRolesWithDataNodeLegacySettings() throws ExecutionException, assertEquals(expectedNodesRoles, Set.of(getNodeRoles(client, 0), getNodeRoles(client, 1))); } + public void testClusterStatsMetricFiltering() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.FS.metricName(), Metric.JVM.metricName(), Metric.PLUGINS.metricName(), Metric.OS.metricName())) + .get(); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getNodesStats().getJvm()); + assertNotNull(response.getNodesStats().getOs()); + assertNotNull(response.getNodesStats().getPlugins()); + assertNotNull(response.getNodesStats().getFs()); + assertNull(response.getIndicesStats()); + + response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES.metricName())) + .get(); + assertNotNull(response.getIndicesStats()); + assertNotNull(response.getIndicesStats().getMappings()); + assertNotNull(response.getIndicesStats().getAnalysis()); + assertNotNull(response.getIndicesStats().getCompletion()); + assertNotNull(response.getIndicesStats().getQueryCache()); + assertNotNull(response.getIndicesStats().getShards()); + assertNotNull(response.getIndicesStats().getSegments()); + assertNotNull(response.getIndicesStats().getStore()); + assertNotNull(response.getIndicesStats().getFieldData()); + assertNotNull(response.getIndicesStats().getIndexCount()); + assertNull(response.getNodesStats()); + + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(randomBoolean()).get(); + assertNotNull(response.getIndicesStats()); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats().getMappings()); + assertNotNull(response.getIndicesStats().getAnalysis()); + assertNotNull(response.getNodesStats().getJvm()); + assertNotNull(response.getNodesStats().getOs()); + + response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES.metricName())) + .indexMetrics( + Set.of( + IndexMetrics.SHARDS.metricName(), + IndexMetrics.DOCS.metricName(), + IndexMetrics.MAPPINGS.metricName(), + IndexMetrics.ANALYSIS.metricName() + ) + ) + .get(); + assertNotNull(response.getIndicesStats()); + assertNotNull(response.getIndicesStats().getShards()); + assertNotNull(response.getIndicesStats().getDocs()); + assertNotNull(response.getIndicesStats().getMappings()); + assertNotNull(response.getIndicesStats().getAnalysis()); + assertNull(response.getIndicesStats().getStore()); + assertNull(response.getIndicesStats().getSegments()); + assertNull(response.getIndicesStats().getCompletion()); + assertNull(response.getIndicesStats().getQueryCache()); + assertNull(response.getNodesStats()); + + response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.OS.metricName(), Metric.PROCESS.metricName(), Metric.INDICES.metricName())) + .indexMetrics(Set.of(IndexMetrics.SHARDS.metricName(), IndexMetrics.MAPPINGS.metricName())) + .get(); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getNodesStats().getOs()); + assertNotNull(response.getNodesStats().getProcess()); + assertNull(response.getNodesStats().getPlugins()); + assertNull(response.getNodesStats().getFs()); + assertNotNull(response.getIndicesStats()); + assertNotNull(response.getIndicesStats().getShards()); + assertNotNull(response.getIndicesStats().getMappings()); + assertNull(response.getIndicesStats().getAnalysis()); + assertNull(response.getIndicesStats().getFieldData()); + + assertThrows( + IllegalStateException.class, + () -> client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of("random_metric")) + .get() + ); + + assertThrows( + IllegalStateException.class, + () -> client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Metric.allMetrics()) + .indexMetrics(Set.of("random_metric")) + .get() + ); + + } + private Map getExpectedCounts( int dataRoleCount, int masterRoleCount, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 03a73f45ffe81..7bece7f014b95 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.cluster.stats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetrics; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.xcontent.ToXContentFragment; @@ -47,6 +48,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; /** * Cluster Stats per index @@ -68,14 +71,56 @@ public class ClusterStatsIndices implements ToXContentFragment { private MappingStats mappings; public ClusterStatsIndices(List nodeResponses, MappingStats mappingStats, AnalysisStats analysisStats) { + this(IndexMetrics.allIndicesMetrics(), nodeResponses, mappingStats, analysisStats); + + } + + public ClusterStatsIndices( + Set indicesMetrics, + List nodeResponses, + MappingStats mappingStats, + AnalysisStats analysisStats + ) { Map countsPerIndex = new HashMap<>(); + Consumer docsStatsConsumer = (docs) -> { + if (IndexMetrics.DOCS.containedIn(indicesMetrics)) { + if (this.docs == null) this.docs = new DocsStats(); + this.docs.add(docs); + } + }; + Consumer storeStatsConsumer = (store) -> { + if (IndexMetrics.STORE.containedIn(indicesMetrics)) { + if (this.store == null) this.store = new StoreStats(); + this.store.add(store); + } + }; + Consumer fieldDataConsumer = (fieldDataStats) -> { + if (IndexMetrics.FIELDDATA.containedIn(indicesMetrics)) { + if (this.fieldData == null) this.fieldData = new FieldDataStats(); + this.fieldData.add(fieldDataStats); + } + }; - this.docs = new DocsStats(); - this.store = new StoreStats(); - this.fieldData = new FieldDataStats(); - this.queryCache = new QueryCacheStats(); - this.completion = new CompletionStats(); - this.segments = new SegmentsStats(); + Consumer queryCacheStatsConsumer = (queryCacheStats) -> { + if (IndexMetrics.QUERY_CACHE.containedIn(indicesMetrics)) { + if (this.queryCache == null) this.queryCache = new QueryCacheStats(); + this.queryCache.add(queryCacheStats); + } + }; + + Consumer completionStatsConsumer = (completionStats) -> { + if (IndexMetrics.COMPLETION.containedIn(indicesMetrics)) { + if (this.completion == null) this.completion = new CompletionStats(); + this.completion.add(completionStats); + } + }; + + Consumer segmentsStatsConsumer = (segmentsStats) -> { + if (IndexMetrics.SEGMENTS.containedIn(indicesMetrics)) { + if (this.segments == null) this.segments = new SegmentsStats(); + this.segments.add(segmentsStats); + } + }; for (ClusterStatsNodeResponse r : nodeResponses) { // Aggregated response from the node @@ -92,12 +137,12 @@ public ClusterStatsIndices(List nodeResponses, Mapping } } - docs.add(r.getAggregatedNodeLevelStats().commonStats.docs); - store.add(r.getAggregatedNodeLevelStats().commonStats.store); - fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData); - queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache); - completion.add(r.getAggregatedNodeLevelStats().commonStats.completion); - segments.add(r.getAggregatedNodeLevelStats().commonStats.segments); + docsStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.docs); + storeStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.store); + fieldDataConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.fieldData); + queryCacheStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.queryCache); + completionStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.completion); + segmentsStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.segments); } else { // Default response from the node for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { @@ -113,21 +158,23 @@ public ClusterStatsIndices(List nodeResponses, Mapping if (shardStats.getShardRouting().primary()) { indexShardStats.primaries++; - docs.add(shardCommonStats.docs); + docsStatsConsumer.accept(shardCommonStats.docs); } - store.add(shardCommonStats.store); - fieldData.add(shardCommonStats.fieldData); - queryCache.add(shardCommonStats.queryCache); - completion.add(shardCommonStats.completion); - segments.add(shardCommonStats.segments); + storeStatsConsumer.accept(shardCommonStats.store); + fieldDataConsumer.accept(shardCommonStats.fieldData); + queryCacheStatsConsumer.accept(shardCommonStats.queryCache); + completionStatsConsumer.accept(shardCommonStats.completion); + segmentsStatsConsumer.accept(shardCommonStats.segments); } } } - shards = new ShardStats(); indexCount = countsPerIndex.size(); - for (final ShardStats indexCountsCursor : countsPerIndex.values()) { - shards.addIndexShardCount(indexCountsCursor); + if (IndexMetrics.SHARDS.containedIn(indicesMetrics)) { + shards = new ShardStats(); + for (final ShardStats indexCountsCursor : countsPerIndex.values()) { + shards.addIndexShardCount(indexCountsCursor); + } } this.mappings = mappingStats; @@ -186,13 +233,27 @@ static final class Fields { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.COUNT, indexCount); - shards.toXContent(builder, params); - docs.toXContent(builder, params); - store.toXContent(builder, params); - fieldData.toXContent(builder, params); - queryCache.toXContent(builder, params); - completion.toXContent(builder, params); - segments.toXContent(builder, params); + if (shards != null) { + shards.toXContent(builder, params); + } + if (docs != null) { + docs.toXContent(builder, params); + } + if (store != null) { + store.toXContent(builder, params); + } + if (fieldData != null) { + fieldData.toXContent(builder, params); + } + if (queryCache != null) { + queryCache.toXContent(builder, params); + } + if (completion != null) { + completion.toXContent(builder, params); + } + if (segments != null) { + segments.toXContent(builder, params); + } if (mappings != null) { mappings.toXContent(builder, params); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index b44e9cfc5c74a..224948027d8f2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -36,6 +36,7 @@ import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.annotation.PublicApi; @@ -90,37 +91,51 @@ public class ClusterStatsNodes implements ToXContentFragment { private final IngestStats ingestStats; ClusterStatsNodes(List nodeResponses) { + this(Metric.allMetrics(), nodeResponses); + } + + ClusterStatsNodes(Set requestedMetrics, List nodeResponses) { this.versions = new HashSet<>(); - this.fs = new FsInfo.Path(); - this.plugins = new HashSet<>(); + boolean isFSInfoRequested = ClusterStatsRequest.Metric.FS.containedIn(requestedMetrics); + boolean isPluginsInfoRequested = ClusterStatsRequest.Metric.PLUGINS.containedIn(requestedMetrics); + this.fs = isFSInfoRequested ? new FsInfo.Path() : null; + this.plugins = isPluginsInfoRequested ? new HashSet<>() : null; Set seenAddresses = new HashSet<>(nodeResponses.size()); List nodeInfos = new ArrayList<>(nodeResponses.size()); - List nodeStats = new ArrayList<>(nodeResponses.size()); + List nodesStats = new ArrayList<>(nodeResponses.size()); for (ClusterStatsNodeResponse nodeResponse : nodeResponses) { - nodeInfos.add(nodeResponse.nodeInfo()); - nodeStats.add(nodeResponse.nodeStats()); - this.versions.add(nodeResponse.nodeInfo().getVersion()); - this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos()); + NodeInfo nodeInfo = nodeResponse.nodeInfo(); + NodeStats nodeStats = nodeResponse.nodeStats(); + nodeInfos.add(nodeInfo); + nodesStats.add(nodeStats); + this.versions.add(nodeInfo.getVersion()); + if (isPluginsInfoRequested) { + this.plugins.addAll(nodeInfo.getInfo(PluginsAndModules.class).getPluginInfos()); + } // now do the stats that should be deduped by hardware (implemented by ip deduping) - TransportAddress publishAddress = nodeResponse.nodeInfo().getInfo(TransportInfo.class).address().publishAddress(); + TransportAddress publishAddress = nodeInfo.getInfo(TransportInfo.class).address().publishAddress(); final InetAddress inetAddress = publishAddress.address().getAddress(); if (!seenAddresses.add(inetAddress)) { continue; } - if (nodeResponse.nodeStats().getFs() != null) { - this.fs.add(nodeResponse.nodeStats().getFs().getTotal()); + if (isFSInfoRequested && nodeStats.getFs() != null) { + this.fs.add(nodeStats.getFs().getTotal()); } } this.counts = new Counts(nodeInfos); - this.os = new OsStats(nodeInfos, nodeStats); - this.process = new ProcessStats(nodeStats); - this.jvm = new JvmStats(nodeInfos, nodeStats); - this.networkTypes = new NetworkTypes(nodeInfos); - this.discoveryTypes = new DiscoveryTypes(nodeInfos); - this.packagingTypes = new PackagingTypes(nodeInfos); - this.ingestStats = new IngestStats(nodeStats); + this.os = ClusterStatsRequest.Metric.OS.containedIn(requestedMetrics) ? new OsStats(nodeInfos, nodesStats) : null; + this.process = ClusterStatsRequest.Metric.PROCESS.containedIn(requestedMetrics) ? new ProcessStats(nodesStats) : null; + this.jvm = ClusterStatsRequest.Metric.JVM.containedIn(requestedMetrics) ? new JvmStats(nodeInfos, nodesStats) : null; + this.networkTypes = ClusterStatsRequest.Metric.NETWORK_TYPES.containedIn(requestedMetrics) ? new NetworkTypes(nodeInfos) : null; + this.discoveryTypes = ClusterStatsRequest.Metric.DISCOVERY_TYPES.containedIn(requestedMetrics) + ? new DiscoveryTypes(nodeInfos) + : null; + this.packagingTypes = ClusterStatsRequest.Metric.PACKAGING_TYPES.containedIn(requestedMetrics) + ? new PackagingTypes(nodeInfos) + : null; + this.ingestStats = ClusterStatsRequest.Metric.INGEST.containedIn(requestedMetrics) ? new IngestStats(nodesStats) : null; } public Counts getCounts() { @@ -179,36 +194,54 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); - builder.startObject(Fields.OS); - os.toXContent(builder, params); - builder.endObject(); + if (os != null) { + builder.startObject(Fields.OS); + os.toXContent(builder, params); + builder.endObject(); + } - builder.startObject(Fields.PROCESS); - process.toXContent(builder, params); - builder.endObject(); + if (process != null) { + builder.startObject(Fields.PROCESS); + process.toXContent(builder, params); + builder.endObject(); + } - builder.startObject(Fields.JVM); - jvm.toXContent(builder, params); - builder.endObject(); + if (jvm != null) { + builder.startObject(Fields.JVM); + jvm.toXContent(builder, params); + builder.endObject(); + } - builder.field(Fields.FS); - fs.toXContent(builder, params); + if (fs != null) { + builder.field(Fields.FS); + fs.toXContent(builder, params); + } - builder.startArray(Fields.PLUGINS); - for (PluginInfo pluginInfo : plugins) { - pluginInfo.toXContent(builder, params); + if (plugins != null) { + builder.startArray(Fields.PLUGINS); + for (PluginInfo pluginInfo : plugins) { + pluginInfo.toXContent(builder, params); + } + builder.endArray(); } - builder.endArray(); - builder.startObject(Fields.NETWORK_TYPES); - networkTypes.toXContent(builder, params); - builder.endObject(); + if (networkTypes != null) { + builder.startObject(Fields.NETWORK_TYPES); + networkTypes.toXContent(builder, params); + builder.endObject(); + } - discoveryTypes.toXContent(builder, params); + if (discoveryTypes != null) { + discoveryTypes.toXContent(builder, params); + } - packagingTypes.toXContent(builder, params); + if (packagingTypes != null) { + packagingTypes.toXContent(builder, params); + } - ingestStats.toXContent(builder, params); + if (ingestStats != null) { + ingestStats.toXContent(builder, params); + } return builder; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index b82a9d256a134..5abd69265f3bb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -39,6 +39,10 @@ import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; /** * A request to get cluster level stats. @@ -48,11 +52,20 @@ @PublicApi(since = "1.0.0") public class ClusterStatsRequest extends BaseNodesRequest { + private final Set requestedMetrics = new HashSet<>(Metric.allMetrics()); + private final Set indicesMetrics = new HashSet<>(IndexMetrics.allIndicesMetrics()); + public ClusterStatsRequest(StreamInput in) throws IOException { super(in); if (in.getVersion().onOrAfter(Version.V_2_16_0)) { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + requestedMetrics.clear(); + indicesMetrics.clear(); + requestedMetrics.addAll(in.readStringList()); + indicesMetrics.addAll(in.readStringList()); + } } private Boolean useAggregatedNodeLevelResponses = false; @@ -73,12 +86,127 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon this.useAggregatedNodeLevelResponses = useAggregatedNodeLevelResponses; } + /** + * Add Metric + */ + public ClusterStatsRequest addMetric(String metric) { + if (Metric.allMetrics().contains(metric) == false) { + throw new IllegalStateException("Used an illegal Metric: " + metric); + } + requestedMetrics.add(metric); + return this; + } + + /** + * Get the names of requested metrics + */ + public Set requestedMetrics() { + return new HashSet<>(requestedMetrics); + } + + /** + * Add IndexMetric + */ + public ClusterStatsRequest addIndexMetric(String indexMetric) { + if (IndexMetrics.allIndicesMetrics().contains(indexMetric) == false) { + throw new IllegalStateException("Used an illegal index metric: " + indexMetric); + } + indicesMetrics.add(indexMetric); + return this; + } + + public Set indicesMetrics() { + return new HashSet<>(indicesMetrics); + } + + public void clearRequestedMetrics() { + requestedMetrics.clear(); + } + + public void clearIndicesMetrics() { + indicesMetrics.clear(); + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); if (out.getVersion().onOrAfter(Version.V_2_16_0)) { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeStringArray(requestedMetrics.toArray(new String[0])); + out.writeStringArray(indicesMetrics.toArray(new String[0])); + } } + /** + * An enumeration of the "core" sections of metrics that may be requested + * from the cluster stats endpoint. + */ + @PublicApi(since = "3.0.0") + public enum Metric { + OS("os"), + PROCESS("process"), + JVM("jvm"), + FS("fs"), + PLUGINS("plugins"), + INGEST("ingest"), + NETWORK_TYPES("network_types"), + DISCOVERY_TYPES("discovery_types"), + PACKAGING_TYPES("packaging_types"), + INDICES("indices"); + + private String metricName; + + Metric(String name) { + this.metricName = name; + } + + public String metricName() { + return this.metricName; + } + + public static Set allMetrics() { + return Arrays.stream(values()).map(Metric::metricName).collect(Collectors.toSet()); + } + + public boolean containedIn(Set metricNames) { + return metricNames.contains(this.metricName()); + } + + } + + /** + * An enumeration of the "core" sections of indices metrics that may be requested + * from the cluster stats endpoint. + */ + public enum IndexMetrics { + SHARDS("shards"), + DOCS("docs"), + STORE("store"), + FIELDDATA("fielddata"), + QUERY_CACHE("query_cache"), + COMPLETION("completion"), + SEGMENTS("segments"), + ANALYSIS("analysis"), + MAPPINGS("mappings"); + + private String metricName; + + IndexMetrics(String name) { + this.metricName = name; + } + + public String metricName() { + return this.metricName; + } + + public boolean containedIn(Set metricNames) { + return metricNames.contains(this.metricName()); + } + + public static Set allIndicesMetrics() { + return Arrays.stream(values()).map(IndexMetrics::metricName).collect(Collectors.toSet()); + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index 4d0932bd3927d..619cca950b786 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -36,6 +36,8 @@ import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; +import java.util.Set; + /** * Transport request builder for obtaining cluster stats * @@ -55,4 +57,16 @@ public final ClusterStatsRequestBuilder useAggregatedNodeLevelResponses(boolean request.useAggregatedNodeLevelResponses(useAggregatedNodeLevelResponses); return this; } + + public final ClusterStatsRequestBuilder requestMetrics(Set requestMetrics) { + request.clearRequestedMetrics(); + requestMetrics.forEach(request::addMetric); + return this; + } + + public final ClusterStatsRequestBuilder indexMetrics(Set indexMetrics) { + request.clearIndicesMetrics(); + indexMetrics.forEach(request::addIndexMetric); + return this; + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java index cc002b689a2a5..fd2a72992b4d5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -33,6 +33,7 @@ package org.opensearch.action.admin.cluster.stats; import org.opensearch.action.FailedNodeException; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -47,6 +48,7 @@ import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Set; /** * Transport response for obtaining cluster stats @@ -88,12 +90,42 @@ public ClusterStatsResponse( List nodes, List failures, ClusterState state + ) { + this( + timestamp, + clusterUUID, + clusterName, + nodes, + failures, + state, + ClusterStatsRequest.Metric.allMetrics(), + ClusterStatsRequest.IndexMetrics.allIndicesMetrics() + ); + } + + public ClusterStatsResponse( + long timestamp, + String clusterUUID, + ClusterName clusterName, + List nodes, + List failures, + ClusterState state, + Set requestedMetrics, + Set indicesMetrics ) { super(clusterName, nodes, failures); this.clusterUUID = clusterUUID; this.timestamp = timestamp; - nodesStats = new ClusterStatsNodes(nodes); - indicesStats = new ClusterStatsIndices(nodes, MappingStats.of(state), AnalysisStats.of(state)); + nodesStats = !requestedMetrics.isEmpty() && !requestedMetrics.equals(Set.of(Metric.INDICES.metricName())) + ? new ClusterStatsNodes(requestedMetrics, nodes) + : null; + MappingStats mappingStats = ClusterStatsRequest.IndexMetrics.MAPPINGS.containedIn(indicesMetrics) ? MappingStats.of(state) : null; + AnalysisStats analysisStats = ClusterStatsRequest.IndexMetrics.ANALYSIS.containedIn(indicesMetrics) + ? AnalysisStats.of(state) + : null; + indicesStats = Metric.INDICES.containedIn(requestedMetrics) + ? new ClusterStatsIndices(indicesMetrics, nodes, mappingStats, analysisStats) + : null; ClusterHealthStatus status = null; for (ClusterStatsNodeResponse response : nodes) { // only the cluster-manager node populates the status @@ -131,8 +163,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(timestamp); out.writeOptionalWriteable(status); out.writeOptionalString(clusterUUID); - out.writeOptionalWriteable(indicesStats.getMappings()); - out.writeOptionalWriteable(indicesStats.getAnalysis()); + if (indicesStats != null) { + out.writeOptionalWriteable(indicesStats.getMappings()); + out.writeOptionalWriteable(indicesStats.getAnalysis()); + } else { + out.writeOptionalWriteable(null); + out.writeOptionalWriteable(null); + } } @Override @@ -153,12 +190,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (status != null) { builder.field("status", status.name().toLowerCase(Locale.ROOT)); } - builder.startObject("indices"); - indicesStats.toXContent(builder, params); - builder.endObject(); - builder.startObject("nodes"); - nodesStats.toXContent(builder, params); - builder.endObject(); + if (indicesStats != null) { + builder.startObject("indices"); + indicesStats.toXContent(builder, params); + builder.endObject(); + } + if (nodesStats != null) { + builder.startObject("nodes"); + nodesStats.toXContent(builder, params); + builder.endObject(); + } return builder; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index c4b3524cf6da5..aa11821c7032f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -37,6 +37,7 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.ShardStats; @@ -64,6 +65,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * Transport action for obtaining cluster state @@ -130,7 +132,9 @@ protected ClusterStatsResponse newResponse( clusterService.getClusterName(), responses, failures, - state + state, + request.requestedMetrics(), + request.indicesMetrics() ); } @@ -147,19 +151,20 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce @Override protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) { NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false, false); + Set requestedMetrics = nodeRequest.request.requestedMetrics(); NodeStats nodeStats = nodeService.stats( CommonStatsFlags.NONE, - true, - true, - true, + Metric.OS.containedIn(requestedMetrics), + Metric.PROCESS.containedIn(requestedMetrics), + Metric.JVM.containedIn(requestedMetrics), false, - true, + Metric.FS.containedIn(requestedMetrics), false, false, false, false, false, - true, + Metric.INGEST.containedIn(requestedMetrics), false, false, false, @@ -178,33 +183,35 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false ); List shardsStats = new ArrayList<>(); - for (IndexService indexService : indicesService) { - for (IndexShard indexShard : indexService) { - if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { - // only report on fully started shards - CommitStats commitStats; - SeqNoStats seqNoStats; - RetentionLeaseStats retentionLeaseStats; - try { - commitStats = indexShard.commitStats(); - seqNoStats = indexShard.seqNoStats(); - retentionLeaseStats = indexShard.getRetentionLeaseStats(); - } catch (final AlreadyClosedException e) { - // shard is closed - no stats is fine - commitStats = null; - seqNoStats = null; - retentionLeaseStats = null; + if (Metric.INDICES.containedIn(requestedMetrics)) { + for (IndexService indexService : indicesService) { + for (IndexShard indexShard : indexService) { + if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { + // only report on fully started shards + CommitStats commitStats; + SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; + try { + commitStats = indexShard.commitStats(); + seqNoStats = indexShard.seqNoStats(); + retentionLeaseStats = indexShard.getRetentionLeaseStats(); + } catch (final AlreadyClosedException e) { + // shard is closed - no stats is fine + commitStats = null; + seqNoStats = null; + retentionLeaseStats = null; + } + shardsStats.add( + new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), + commitStats, + seqNoStats, + retentionLeaseStats + ) + ); } - shardsStats.add( - new ShardStats( - indexShard.routingEntry(), - indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), - commitStats, - seqNoStats, - retentionLeaseStats - ) - ); } } } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index ee33bd18db05d..398b75de38e06 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -34,12 +34,21 @@ import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest; import org.opensearch.client.node.NodeClient; +import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestActions.NodesResponseRestListener; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Consumer; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; @@ -54,7 +63,34 @@ public class RestClusterStatsAction extends BaseRestHandler { @Override public List routes() { - return unmodifiableList(asList(new Route(GET, "/_cluster/stats"), new Route(GET, "/_cluster/stats/nodes/{nodeId}"))); + return unmodifiableList( + asList( + new Route(GET, "/_cluster/stats"), + new Route(GET, "/_cluster/stats/nodes/{nodeId}"), + new Route(GET, "/_cluster/stats/{metric}/nodes/{nodeId}"), + new Route(GET, "/_cluster/stats/{metric}/{index_metric}/nodes/{nodeId}") + ) + ); + } + + static final Map> INDEX_METRIC_TO_REQUEST_CONSUMER_MAP; + + static final Map> METRIC_REQUEST_CONSUMER_MAP; + + static { + Map> metricRequestConsumerMap = new HashMap<>(); + for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { + metricRequestConsumerMap.put(metric.metricName(), request -> request.addMetric(metric.metricName())); + } + METRIC_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricRequestConsumerMap); + } + + static { + Map> metricMap = new HashMap<>(); + for (ClusterStatsRequest.IndexMetrics indexMetric : ClusterStatsRequest.IndexMetrics.values()) { + metricMap.put(indexMetric.metricName(), request -> request.addIndexMetric(indexMetric.metricName())); + } + INDEX_METRIC_TO_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricMap); } @Override @@ -64,9 +100,95 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null)); + String[] nodeIds = request.paramAsStringArray("nodeId", null); + Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", null)); + logger.info("Request URI path params, metrics : {}, index_metrics : {}, nodes : {}", metrics, indexMetrics, nodeIds); + + ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); clusterStatsRequest.timeout(request.param("timeout")); clusterStatsRequest.useAggregatedNodeLevelResponses(true); + + if (metrics.size() > 1 && metrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ); + } else if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual index metrics [%s]", + request.path(), + request.param("sub_metric") + ) + ); + } else { + clusterStatsRequest.clearRequestedMetrics(); + clusterStatsRequest.clearIndicesMetrics(); + + final Set metricsRequested = new HashSet<>(); + if (metrics.contains("_all")) { + metricsRequested.addAll(METRIC_REQUEST_CONSUMER_MAP.keySet()); + } else { + metricsRequested.addAll(metrics); + } + final Set invalidMetrics = new TreeSet<>(); + for (String metric : metricsRequested) { + Consumer clusterStatsRequestConsumer = METRIC_REQUEST_CONSUMER_MAP.get(metric); + if (clusterStatsRequestConsumer != null) { + clusterStatsRequestConsumer.accept(clusterStatsRequest); + } else { + invalidMetrics.add(metric); + } + } + if (!invalidMetrics.isEmpty()) { + throw new IllegalArgumentException(unrecognized(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric")); + } + if (!metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName()) && !indexMetrics.isEmpty()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains index metrics [%s] but indices stats not requested", + request.path(), + request.param("index_metric") + ) + ); + } + + logger.info("Computed metrics: {}", metricsRequested); + if (ClusterStatsRequest.Metric.INDICES.containedIn(metricsRequested)) { + logger.info("Stats request contains indices metric"); + final Set indexMetricsRequested = new HashSet<>(); + if (indexMetrics.isEmpty() || indexMetrics.contains("_all")) { + indexMetricsRequested.addAll(ClusterStatsRequest.IndexMetrics.allIndicesMetrics()); + } else { + indexMetricsRequested.addAll(indexMetrics); + } + logger.info("Computed index metrics: {}", indexMetricsRequested); + final Set invalidIndexMetrics = new TreeSet<>(); + for (String indexMetric : indexMetricsRequested) { + Consumer clusterStatsRequestConsumer = INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.get(indexMetric); + if (clusterStatsRequestConsumer != null) { + clusterStatsRequestConsumer.accept(clusterStatsRequest); + } else { + invalidIndexMetrics.add(indexMetric); + } + } + + if (!invalidIndexMetrics.isEmpty()) { + throw new IllegalArgumentException( + unrecognized(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index_metric") + ); + } + } + + } + return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java new file mode 100644 index 0000000000000..c167cf0bbe46b --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.stats; + +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetrics; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.client.NoOpClient; +import org.junit.After; +import org.junit.Before; + +import java.util.Set; + +public class ClusterStatsRequestBuilderTests extends OpenSearchTestCase { + + private NoOpClient testClient; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + this.testClient = new NoOpClient(getTestName()); + } + + @Override + @After + public void tearDown() throws Exception { + this.testClient.close(); + super.tearDown(); + } + + public void testUseAggregatedNodeLevelResponses() { + ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( + this.testClient, + ClusterStatsAction.INSTANCE + ); + clusterStatsRequestBuilder.useAggregatedNodeLevelResponses(false); + assertFalse(clusterStatsRequestBuilder.request().useAggregatedNodeLevelResponses()); + } + + public void testRequestedMetrics() { + ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( + this.testClient, + ClusterStatsAction.INSTANCE + ); + clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.OS.metricName(), Metric.JVM.metricName())); + assertEquals(Set.of(Metric.OS.metricName(), Metric.JVM.metricName()), clusterStatsRequestBuilder.request().requestedMetrics()); + } + + public void testIndicesMetrics() { + ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( + this.testClient, + ClusterStatsAction.INSTANCE + ); + clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.INDICES.metricName(), Metric.JVM.metricName())); + clusterStatsRequestBuilder.indexMetrics(Set.of(IndexMetrics.MAPPINGS.metricName(), IndexMetrics.ANALYSIS.metricName())); + assertEquals(Set.of(Metric.INDICES.metricName(), Metric.JVM.metricName()), clusterStatsRequestBuilder.request().requestedMetrics()); + assertEquals( + Set.of(IndexMetrics.MAPPINGS.metricName(), IndexMetrics.ANALYSIS.metricName()), + clusterStatsRequestBuilder.request().indicesMetrics() + ); + } + +} From 2571ea3d2c7aa5384621457d8fede1c0a32bfd24 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Thu, 26 Sep 2024 00:01:24 +0530 Subject: [PATCH 02/12] Handle backward compatibility. Signed-off-by: Swetha Guptha --- .../admin/cluster/stats/ClusterStatsIT.java | 7 +++ .../cluster/stats/ClusterStatsNodes.java | 46 ++++++++------ .../cluster/stats/ClusterStatsRequest.java | 28 ++++++--- .../stats/ClusterStatsRequestBuilder.java | 7 ++- .../cluster/stats/ClusterStatsResponse.java | 2 +- .../stats/TransportClusterStatsAction.java | 62 +++++++++++++------ .../admin/cluster/RestClusterStatsAction.java | 54 ++++++++-------- .../ClusterStatsRequestBuilderTests.java | 10 +++ 8 files changed, 139 insertions(+), 77 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index ed28fffb05c75..35f3b58813794 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -518,6 +518,7 @@ public void testClusterStatsMetricFiltering() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.FS.metricName(), Metric.JVM.metricName(), Metric.PLUGINS.metricName(), Metric.OS.metricName())) + .applyMetricFiltering(true) .get(); assertNotNull(response.getNodesStats()); assertNotNull(response.getNodesStats().getJvm()); @@ -531,6 +532,8 @@ public void testClusterStatsMetricFiltering() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.INDICES.metricName())) + .indexMetrics(IndexMetrics.allIndicesMetrics()) + .applyMetricFiltering(true) .get(); assertNotNull(response.getIndicesStats()); assertNotNull(response.getIndicesStats().getMappings()); @@ -565,6 +568,7 @@ public void testClusterStatsMetricFiltering() { IndexMetrics.ANALYSIS.metricName() ) ) + .applyMetricFiltering(true) .get(); assertNotNull(response.getIndicesStats()); assertNotNull(response.getIndicesStats().getShards()); @@ -583,6 +587,7 @@ public void testClusterStatsMetricFiltering() { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.OS.metricName(), Metric.PROCESS.metricName(), Metric.INDICES.metricName())) .indexMetrics(Set.of(IndexMetrics.SHARDS.metricName(), IndexMetrics.MAPPINGS.metricName())) + .applyMetricFiltering(true) .get(); assertNotNull(response.getNodesStats()); assertNotNull(response.getNodesStats().getOs()); @@ -602,6 +607,7 @@ public void testClusterStatsMetricFiltering() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of("random_metric")) + .applyMetricFiltering(true) .get() ); @@ -613,6 +619,7 @@ public void testClusterStatsMetricFiltering() { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Metric.allMetrics()) .indexMetrics(Set.of("random_metric")) + .applyMetricFiltering(true) .get() ); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 224948027d8f2..9ebda488d3ac7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -90,44 +90,52 @@ public class ClusterStatsNodes implements ToXContentFragment { private final PackagingTypes packagingTypes; private final IngestStats ingestStats; + public static final Set NODE_STATS_METRICS = Set.of( + Metric.OS.metricName(), + Metric.PROCESS.metricName(), + Metric.JVM.metricName(), + Metric.FS.metricName(), + Metric.PLUGINS.metricName(), + Metric.INGEST.metricName(), + Metric.NETWORK_TYPES.metricName(), + Metric.DISCOVERY_TYPES.metricName(), + Metric.PACKAGING_TYPES.metricName() + ); + ClusterStatsNodes(List nodeResponses) { this(Metric.allMetrics(), nodeResponses); } ClusterStatsNodes(Set requestedMetrics, List nodeResponses) { this.versions = new HashSet<>(); - boolean isFSInfoRequested = ClusterStatsRequest.Metric.FS.containedIn(requestedMetrics); - boolean isPluginsInfoRequested = ClusterStatsRequest.Metric.PLUGINS.containedIn(requestedMetrics); - this.fs = isFSInfoRequested ? new FsInfo.Path() : null; - this.plugins = isPluginsInfoRequested ? new HashSet<>() : null; + this.fs = ClusterStatsRequest.Metric.FS.containedIn(requestedMetrics) ? new FsInfo.Path() : null; + this.plugins = ClusterStatsRequest.Metric.PLUGINS.containedIn(requestedMetrics) ? new HashSet<>() : null; Set seenAddresses = new HashSet<>(nodeResponses.size()); List nodeInfos = new ArrayList<>(nodeResponses.size()); - List nodesStats = new ArrayList<>(nodeResponses.size()); + List nodeStats = new ArrayList<>(nodeResponses.size()); for (ClusterStatsNodeResponse nodeResponse : nodeResponses) { - NodeInfo nodeInfo = nodeResponse.nodeInfo(); - NodeStats nodeStats = nodeResponse.nodeStats(); - nodeInfos.add(nodeInfo); - nodesStats.add(nodeStats); - this.versions.add(nodeInfo.getVersion()); - if (isPluginsInfoRequested) { - this.plugins.addAll(nodeInfo.getInfo(PluginsAndModules.class).getPluginInfos()); + nodeInfos.add(nodeResponse.nodeInfo()); + nodeStats.add(nodeResponse.nodeStats()); + this.versions.add(nodeResponse.nodeInfo().getVersion()); + if (this.plugins != null) { + this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos()); } // now do the stats that should be deduped by hardware (implemented by ip deduping) - TransportAddress publishAddress = nodeInfo.getInfo(TransportInfo.class).address().publishAddress(); + TransportAddress publishAddress = nodeResponse.nodeInfo().getInfo(TransportInfo.class).address().publishAddress(); final InetAddress inetAddress = publishAddress.address().getAddress(); if (!seenAddresses.add(inetAddress)) { continue; } - if (isFSInfoRequested && nodeStats.getFs() != null) { - this.fs.add(nodeStats.getFs().getTotal()); + if (this.fs != null && nodeResponse.nodeStats().getFs() != null) { + this.fs.add(nodeResponse.nodeStats().getFs().getTotal()); } } this.counts = new Counts(nodeInfos); - this.os = ClusterStatsRequest.Metric.OS.containedIn(requestedMetrics) ? new OsStats(nodeInfos, nodesStats) : null; - this.process = ClusterStatsRequest.Metric.PROCESS.containedIn(requestedMetrics) ? new ProcessStats(nodesStats) : null; - this.jvm = ClusterStatsRequest.Metric.JVM.containedIn(requestedMetrics) ? new JvmStats(nodeInfos, nodesStats) : null; + this.os = ClusterStatsRequest.Metric.OS.containedIn(requestedMetrics) ? new OsStats(nodeInfos, nodeStats) : null; + this.process = ClusterStatsRequest.Metric.PROCESS.containedIn(requestedMetrics) ? new ProcessStats(nodeStats) : null; + this.jvm = ClusterStatsRequest.Metric.JVM.containedIn(requestedMetrics) ? new JvmStats(nodeInfos, nodeStats) : null; this.networkTypes = ClusterStatsRequest.Metric.NETWORK_TYPES.containedIn(requestedMetrics) ? new NetworkTypes(nodeInfos) : null; this.discoveryTypes = ClusterStatsRequest.Metric.DISCOVERY_TYPES.containedIn(requestedMetrics) ? new DiscoveryTypes(nodeInfos) @@ -135,7 +143,7 @@ public class ClusterStatsNodes implements ToXContentFragment { this.packagingTypes = ClusterStatsRequest.Metric.PACKAGING_TYPES.containedIn(requestedMetrics) ? new PackagingTypes(nodeInfos) : null; - this.ingestStats = ClusterStatsRequest.Metric.INGEST.containedIn(requestedMetrics) ? new IngestStats(nodesStats) : null; + this.ingestStats = ClusterStatsRequest.Metric.INGEST.containedIn(requestedMetrics) ? new IngestStats(nodeStats) : null; } public Counts getCounts() { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 5abd69265f3bb..1526c817ddbdb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -52,8 +52,9 @@ @PublicApi(since = "1.0.0") public class ClusterStatsRequest extends BaseNodesRequest { - private final Set requestedMetrics = new HashSet<>(Metric.allMetrics()); - private final Set indicesMetrics = new HashSet<>(IndexMetrics.allIndicesMetrics()); + private final Set requestedMetrics = new HashSet<>(); + private final Set indexMetricsRequested = new HashSet<>(); + private Boolean applyMetricFiltering = false; public ClusterStatsRequest(StreamInput in) throws IOException { super(in); @@ -61,10 +62,9 @@ public ClusterStatsRequest(StreamInput in) throws IOException { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } if (in.getVersion().onOrAfter(Version.V_3_0_0)) { - requestedMetrics.clear(); - indicesMetrics.clear(); + applyMetricFiltering = in.readOptionalBoolean(); requestedMetrics.addAll(in.readStringList()); - indicesMetrics.addAll(in.readStringList()); + indexMetricsRequested.addAll(in.readStringList()); } } @@ -86,6 +86,14 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon this.useAggregatedNodeLevelResponses = useAggregatedNodeLevelResponses; } + public boolean applyMetricFiltering() { + return applyMetricFiltering; + } + + public void applyMetricFiltering(boolean honourMetricFiltering) { + this.applyMetricFiltering = honourMetricFiltering; + } + /** * Add Metric */ @@ -111,12 +119,12 @@ public ClusterStatsRequest addIndexMetric(String indexMetric) { if (IndexMetrics.allIndicesMetrics().contains(indexMetric) == false) { throw new IllegalStateException("Used an illegal index metric: " + indexMetric); } - indicesMetrics.add(indexMetric); + indexMetricsRequested.add(indexMetric); return this; } public Set indicesMetrics() { - return new HashSet<>(indicesMetrics); + return new HashSet<>(indexMetricsRequested); } public void clearRequestedMetrics() { @@ -124,7 +132,7 @@ public void clearRequestedMetrics() { } public void clearIndicesMetrics() { - indicesMetrics.clear(); + indexMetricsRequested.clear(); } @Override @@ -134,8 +142,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalBoolean(applyMetricFiltering); out.writeStringArray(requestedMetrics.toArray(new String[0])); - out.writeStringArray(indicesMetrics.toArray(new String[0])); + out.writeStringArray(indexMetricsRequested.toArray(new String[0])); } } @@ -180,6 +189,7 @@ public boolean containedIn(Set metricNames) { * An enumeration of the "core" sections of indices metrics that may be requested * from the cluster stats endpoint. */ + @PublicApi(since = "3.0.0") public enum IndexMetrics { SHARDS("shards"), DOCS("docs"), diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index 619cca950b786..63305825d3ed3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -58,14 +58,17 @@ public final ClusterStatsRequestBuilder useAggregatedNodeLevelResponses(boolean return this; } + public final ClusterStatsRequestBuilder applyMetricFiltering(boolean applyMetricFiltering) { + request.applyMetricFiltering(applyMetricFiltering); + return this; + } + public final ClusterStatsRequestBuilder requestMetrics(Set requestMetrics) { - request.clearRequestedMetrics(); requestMetrics.forEach(request::addMetric); return this; } public final ClusterStatsRequestBuilder indexMetrics(Set indexMetrics) { - request.clearIndicesMetrics(); indexMetrics.forEach(request::addIndexMetric); return this; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java index fd2a72992b4d5..f994b0ddb78b1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -116,7 +116,7 @@ public ClusterStatsResponse( super(clusterName, nodes, failures); this.clusterUUID = clusterUUID; this.timestamp = timestamp; - nodesStats = !requestedMetrics.isEmpty() && !requestedMetrics.equals(Set.of(Metric.INDICES.metricName())) + nodesStats = requestedMetrics.stream().anyMatch(ClusterStatsNodes.NODE_STATS_METRICS::contains) ? new ClusterStatsNodes(requestedMetrics, nodes) : null; MappingStats mappingStats = ClusterStatsRequest.IndexMetrics.MAPPINGS.containedIn(indicesMetrics) ? MappingStats.of(state) : null; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index aa11821c7032f..3ebb501a5e7e1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -65,6 +65,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -78,13 +79,20 @@ public class TransportClusterStatsAction extends TransportNodesAction< TransportClusterStatsAction.ClusterStatsNodeRequest, ClusterStatsNodeResponse> { - private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags( + private static final Map INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP = Map.of( + ClusterStatsRequest.IndexMetrics.DOCS.metricName(), CommonStatsFlags.Flag.Docs, + ClusterStatsRequest.IndexMetrics.STORE.metricName(), CommonStatsFlags.Flag.Store, + ClusterStatsRequest.IndexMetrics.FIELDDATA.metricName(), CommonStatsFlags.Flag.FieldData, + ClusterStatsRequest.IndexMetrics.QUERY_CACHE.metricName(), CommonStatsFlags.Flag.QueryCache, + ClusterStatsRequest.IndexMetrics.COMPLETION.metricName(), CommonStatsFlags.Flag.Completion, + ClusterStatsRequest.IndexMetrics.SEGMENTS.metricName(), CommonStatsFlags.Flag.Segments + ); private final NodeService nodeService; @@ -126,16 +134,27 @@ protected ClusterStatsResponse newResponse( + " the cluster state that are too slow for a transport thread" ); ClusterState state = clusterService.state(); - return new ClusterStatsResponse( - System.currentTimeMillis(), - state.metadata().clusterUUID(), - clusterService.getClusterName(), - responses, - failures, - state, - request.requestedMetrics(), - request.indicesMetrics() - ); + if (request.applyMetricFiltering()) { + return new ClusterStatsResponse( + System.currentTimeMillis(), + state.metadata().clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + state, + request.requestedMetrics(), + request.indicesMetrics() + ); + } else { + return new ClusterStatsResponse( + System.currentTimeMillis(), + state.metadata().clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + state + ); + } } @Override @@ -151,20 +170,21 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce @Override protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) { NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false, false); + boolean applyMetricFiltering = nodeRequest.request.applyMetricFiltering(); Set requestedMetrics = nodeRequest.request.requestedMetrics(); NodeStats nodeStats = nodeService.stats( CommonStatsFlags.NONE, - Metric.OS.containedIn(requestedMetrics), - Metric.PROCESS.containedIn(requestedMetrics), - Metric.JVM.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.OS.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.PROCESS.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.JVM.containedIn(requestedMetrics), false, - Metric.FS.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.FS.containedIn(requestedMetrics), false, false, false, false, false, - Metric.INGEST.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.INGEST.containedIn(requestedMetrics), false, false, false, @@ -183,7 +203,13 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false ); List shardsStats = new ArrayList<>(); - if (Metric.INDICES.containedIn(requestedMetrics)) { + if (!applyMetricFiltering || Metric.INDICES.containedIn(requestedMetrics)) { + CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); + for (String metric : nodeRequest.request.indicesMetrics()) { + if (INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.containsKey(metric)) { + commonStatsFlags.set(INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.get(metric), true); + } + } for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { @@ -205,7 +231,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq new ShardStats( indexShard.routingEntry(), indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, commonStatsFlags), commitStats, seqNoStats, retentionLeaseStats diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index 398b75de38e06..e3858c27815ab 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -100,36 +100,38 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - String[] nodeIds = request.paramAsStringArray("nodeId", null); - Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", null)); Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", null)); - logger.info("Request URI path params, metrics : {}, index_metrics : {}, nodes : {}", metrics, indexMetrics, nodeIds); + String[] nodeIds = request.paramAsStringArray("nodeId", null); ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); clusterStatsRequest.timeout(request.param("timeout")); clusterStatsRequest.useAggregatedNodeLevelResponses(true); - if (metrics.size() > 1 && metrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ); - } else if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual index metrics [%s]", - request.path(), - request.param("sub_metric") - ) - ); - } else { - clusterStatsRequest.clearRequestedMetrics(); - clusterStatsRequest.clearIndicesMetrics(); + if (!metrics.isEmpty()) { + if (metrics.size() > 1 && metrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ); + } + + if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual index metrics [%s]", + request.path(), + request.param("sub_metric") + ) + ); + } + + clusterStatsRequest.applyMetricFiltering(true); final Set metricsRequested = new HashSet<>(); if (metrics.contains("_all")) { @@ -160,16 +162,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } - logger.info("Computed metrics: {}", metricsRequested); if (ClusterStatsRequest.Metric.INDICES.containedIn(metricsRequested)) { - logger.info("Stats request contains indices metric"); final Set indexMetricsRequested = new HashSet<>(); if (indexMetrics.isEmpty() || indexMetrics.contains("_all")) { indexMetricsRequested.addAll(ClusterStatsRequest.IndexMetrics.allIndicesMetrics()); } else { indexMetricsRequested.addAll(indexMetrics); } - logger.info("Computed index metrics: {}", indexMetricsRequested); final Set invalidIndexMetrics = new TreeSet<>(); for (String indexMetric : indexMetricsRequested) { Consumer clusterStatsRequestConsumer = INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.get(indexMetric); @@ -186,7 +185,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } } - } return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java index c167cf0bbe46b..1c64c01eb0e49 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java @@ -44,6 +44,16 @@ public void testUseAggregatedNodeLevelResponses() { assertFalse(clusterStatsRequestBuilder.request().useAggregatedNodeLevelResponses()); } + public void testApplyMetricFiltering() { + ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( + this.testClient, + ClusterStatsAction.INSTANCE + ); + assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + clusterStatsRequestBuilder.applyMetricFiltering(true); + assertTrue(clusterStatsRequestBuilder.request().applyMetricFiltering()); + } + public void testRequestedMetrics() { ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( this.testClient, From 727ecf98696052c2d5bcd1b02d82ef9f9a499e41 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Sun, 29 Sep 2024 12:07:07 +0530 Subject: [PATCH 03/12] Serialise metrics/index_metrics using bit mask. Signed-off-by: Swetha Guptha --- .../admin/cluster/stats/ClusterStatsIT.java | 237 +++++++++++------- .../cluster/stats/ClusterStatsIndices.java | 20 +- .../cluster/stats/ClusterStatsNodes.java | 50 ++-- .../cluster/stats/ClusterStatsRequest.java | 120 ++++----- .../stats/ClusterStatsRequestBuilder.java | 7 +- .../cluster/stats/ClusterStatsResponse.java | 24 +- .../stats/TransportClusterStatsAction.java | 39 +-- .../admin/cluster/RestClusterStatsAction.java | 16 +- .../ClusterStatsRequestBuilderTests.java | 21 +- .../cluster/RestClusterStatsActionTests.java | 85 +++++++ 10 files changed, 383 insertions(+), 236 deletions(-) create mode 100644 server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index 35f3b58813794..9cddcc4b3eeca 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetrics; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.client.Client; import org.opensearch.client.Requests; @@ -232,6 +232,7 @@ public void testIndicesShardStatsWithoutNodeLevelAggregations() { } public void testIndicesShardStatsWithNodeLevelAggregations() { + internalCluster().startNode(); ensureGreen(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(true).get(); @@ -507,122 +508,186 @@ public void testNodeRolesWithDataNodeLegacySettings() throws ExecutionException, assertEquals(expectedNodesRoles, Set.of(getNodeRoles(client, 0), getNodeRoles(client, 1))); } - public void testClusterStatsMetricFiltering() { + public void testClusterStatsApplyMetricFilterDisabled() throws IOException { internalCluster().startNode(); ensureGreen(); client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); - ClusterStatsResponse response = client().admin() + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats()); + + ClusterStatsResponse statsResponseWithMetricFilterDisabled = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.FS.metricName(), Metric.JVM.metricName(), Metric.PLUGINS.metricName(), Metric.OS.metricName())) - .applyMetricFiltering(true) + .applyMetricFiltering(false) + .requestMetrics(Set.of(Metric.FS, Metric.JVM, Metric.PLUGINS, Metric.OS)) .get(); + + assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats()); + assertEquals( + response.getNodesStats().getCounts().getTotal(), + statsResponseWithMetricFilterDisabled.getNodesStats().getCounts().getTotal() + ); + assertEquals( + response.getNodesStats().getCounts().getRoles(), + statsResponseWithMetricFilterDisabled.getNodesStats().getCounts().getRoles() + ); + assertEquals(response.getNodesStats().getVersions(), statsResponseWithMetricFilterDisabled.getNodesStats().getVersions()); + assertEquals(response.getNodesStats().getPlugins(), statsResponseWithMetricFilterDisabled.getNodesStats().getPlugins()); + assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getOs()); + assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getJvm()); + assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getProcess()); + assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getFs()); + + assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats()); + assertEquals(response.getIndicesStats().getFieldData(), statsResponseWithMetricFilterDisabled.getIndicesStats().getFieldData()); + assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithMetricFilterDisabled.getIndicesStats().getAnalysis()); + assertEquals(response.getIndicesStats().getMappings(), statsResponseWithMetricFilterDisabled.getIndicesStats().getMappings()); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithMetricFilterDisabled.getIndicesStats().getIndexCount()); + assertEquals( + response.getIndicesStats().getShards().getTotal(), + statsResponseWithMetricFilterDisabled.getIndicesStats().getShards().getTotal() + ); + assertEquals( + response.getIndicesStats().getShards().getPrimaries(), + statsResponseWithMetricFilterDisabled.getIndicesStats().getShards().getPrimaries() + ); + assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getSegments()); + assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getDocs()); + assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getQueryCache()); + assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getCompletion()); + assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getStore()); + } + + public void testClusterStatsWithMetricsFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); assertNotNull(response.getNodesStats()); - assertNotNull(response.getNodesStats().getJvm()); - assertNotNull(response.getNodesStats().getOs()); - assertNotNull(response.getNodesStats().getPlugins()); - assertNotNull(response.getNodesStats().getFs()); - assertNull(response.getIndicesStats()); + assertNotNull(response.getIndicesStats()); - response = client().admin() + ClusterStatsResponse statsResponseWithAllNodeStatsMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.INDICES.metricName())) - .indexMetrics(IndexMetrics.allIndicesMetrics()) + .requestMetrics(ClusterStatsNodes.NODE_STATS_METRICS) .applyMetricFiltering(true) .get(); - assertNotNull(response.getIndicesStats()); - assertNotNull(response.getIndicesStats().getMappings()); - assertNotNull(response.getIndicesStats().getAnalysis()); - assertNotNull(response.getIndicesStats().getCompletion()); - assertNotNull(response.getIndicesStats().getQueryCache()); - assertNotNull(response.getIndicesStats().getShards()); - assertNotNull(response.getIndicesStats().getSegments()); - assertNotNull(response.getIndicesStats().getStore()); - assertNotNull(response.getIndicesStats().getFieldData()); - assertNotNull(response.getIndicesStats().getIndexCount()); - assertNull(response.getNodesStats()); + assertNotNull(statsResponseWithAllNodeStatsMetrics); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats()); + assertNull(statsResponseWithAllNodeStatsMetrics.getIndicesStats()); + assertEquals( + response.getNodesStats().getCounts().getTotal(), + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getTotal() + ); + assertEquals( + response.getNodesStats().getCounts().getRoles(), + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getRoles() + ); + assertEquals(response.getNodesStats().getVersions(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getVersions()); + assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getOs()); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getJvm()); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getProcess()); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getFs()); + + ClusterStatsResponse statsResponseWithIndicesRequestMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.PLUGINS, Metric.INDICES)) + .applyMetricFiltering(true) + .get(); + assertNotNull(statsResponseWithIndicesRequestMetrics); + assertNotNull(statsResponseWithIndicesRequestMetrics.getNodesStats()); + assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); + assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getOs()); + assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getFs()); + assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getProcess()); + assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getJvm()); + assertNotNull(statsResponseWithIndicesRequestMetrics.getIndicesStats()); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithIndicesRequestMetrics.getIndicesStats().getIndexCount()); + } - response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(randomBoolean()).get(); - assertNotNull(response.getIndicesStats()); + public void testClusterStatsWithIndexMetricFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); assertNotNull(response.getNodesStats()); - assertNotNull(response.getIndicesStats().getMappings()); - assertNotNull(response.getIndicesStats().getAnalysis()); - assertNotNull(response.getNodesStats().getJvm()); - assertNotNull(response.getNodesStats().getOs()); + assertNotNull(response.getIndicesStats()); - response = client().admin() + ClusterStatsResponse statsResponseWithSpecificIndicesMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.INDICES.metricName())) - .indexMetrics( - Set.of( - IndexMetrics.SHARDS.metricName(), - IndexMetrics.DOCS.metricName(), - IndexMetrics.MAPPINGS.metricName(), - IndexMetrics.ANALYSIS.metricName() - ) - ) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)) .applyMetricFiltering(true) .get(); - assertNotNull(response.getIndicesStats()); - assertNotNull(response.getIndicesStats().getShards()); - assertNotNull(response.getIndicesStats().getDocs()); - assertNotNull(response.getIndicesStats().getMappings()); - assertNotNull(response.getIndicesStats().getAnalysis()); - assertNull(response.getIndicesStats().getStore()); - assertNull(response.getIndicesStats().getSegments()); - assertNull(response.getIndicesStats().getCompletion()); - assertNull(response.getIndicesStats().getQueryCache()); - assertNull(response.getNodesStats()); - - response = client().admin() + assertNotNull(statsResponseWithSpecificIndicesMetrics); + assertNull(statsResponseWithSpecificIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithSpecificIndicesMetrics.getIndicesStats()); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getIndexCount()); + assertEquals(response.getIndicesStats().getMappings(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getMappings()); + assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getAnalysis()); + + ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.OS.metricName(), Metric.PROCESS.metricName(), Metric.INDICES.metricName())) - .indexMetrics(Set.of(IndexMetrics.SHARDS.metricName(), IndexMetrics.MAPPINGS.metricName())) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.values())) .applyMetricFiltering(true) .get(); - assertNotNull(response.getNodesStats()); - assertNotNull(response.getNodesStats().getOs()); - assertNotNull(response.getNodesStats().getProcess()); - assertNull(response.getNodesStats().getPlugins()); - assertNull(response.getNodesStats().getFs()); - assertNotNull(response.getIndicesStats()); - assertNotNull(response.getIndicesStats().getShards()); - assertNotNull(response.getIndicesStats().getMappings()); - assertNull(response.getIndicesStats().getAnalysis()); - assertNull(response.getIndicesStats().getFieldData()); - - assertThrows( - IllegalStateException.class, - () -> client().admin() - .cluster() - .prepareClusterStats() - .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of("random_metric")) - .applyMetricFiltering(true) - .get() + assertNotNull(statsResponseWithAllIndicesMetrics); + assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + assertEquals(response.getIndicesStats().getFieldData(), statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData()); + assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithAllIndicesMetrics.getIndicesStats().getAnalysis()); + assertEquals(response.getIndicesStats().getMappings(), statsResponseWithAllIndicesMetrics.getIndicesStats().getMappings()); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithAllIndicesMetrics.getIndicesStats().getIndexCount()); + assertEquals( + response.getIndicesStats().getShards().getTotal(), + statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getTotal() ); - - assertThrows( - IllegalStateException.class, - () -> client().admin() - .cluster() - .prepareClusterStats() - .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Metric.allMetrics()) - .indexMetrics(Set.of("random_metric")) - .applyMetricFiltering(true) - .get() + assertEquals( + response.getIndicesStats().getShards().getPrimaries(), + statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getPrimaries() ); - + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getQueryCache()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getCompletion()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getStore()); } private Map getExpectedCounts( diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 7bece7f014b95..3d3e022a142a0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.stats; -import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetrics; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.xcontent.ToXContentFragment; @@ -71,52 +71,52 @@ public class ClusterStatsIndices implements ToXContentFragment { private MappingStats mappings; public ClusterStatsIndices(List nodeResponses, MappingStats mappingStats, AnalysisStats analysisStats) { - this(IndexMetrics.allIndicesMetrics(), nodeResponses, mappingStats, analysisStats); + this(Set.of(IndexMetric.values()), nodeResponses, mappingStats, analysisStats); } public ClusterStatsIndices( - Set indicesMetrics, + Set indicesMetrics, List nodeResponses, MappingStats mappingStats, AnalysisStats analysisStats ) { Map countsPerIndex = new HashMap<>(); Consumer docsStatsConsumer = (docs) -> { - if (IndexMetrics.DOCS.containedIn(indicesMetrics)) { + if (indicesMetrics.contains(IndexMetric.DOCS)) { if (this.docs == null) this.docs = new DocsStats(); this.docs.add(docs); } }; Consumer storeStatsConsumer = (store) -> { - if (IndexMetrics.STORE.containedIn(indicesMetrics)) { + if (indicesMetrics.contains(IndexMetric.STORE)) { if (this.store == null) this.store = new StoreStats(); this.store.add(store); } }; Consumer fieldDataConsumer = (fieldDataStats) -> { - if (IndexMetrics.FIELDDATA.containedIn(indicesMetrics)) { + if (indicesMetrics.contains(IndexMetric.FIELDDATA)) { if (this.fieldData == null) this.fieldData = new FieldDataStats(); this.fieldData.add(fieldDataStats); } }; Consumer queryCacheStatsConsumer = (queryCacheStats) -> { - if (IndexMetrics.QUERY_CACHE.containedIn(indicesMetrics)) { + if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) { if (this.queryCache == null) this.queryCache = new QueryCacheStats(); this.queryCache.add(queryCacheStats); } }; Consumer completionStatsConsumer = (completionStats) -> { - if (IndexMetrics.COMPLETION.containedIn(indicesMetrics)) { + if (indicesMetrics.contains(IndexMetric.COMPLETION)) { if (this.completion == null) this.completion = new CompletionStats(); this.completion.add(completionStats); } }; Consumer segmentsStatsConsumer = (segmentsStats) -> { - if (IndexMetrics.SEGMENTS.containedIn(indicesMetrics)) { + if (indicesMetrics.contains(IndexMetric.SEGMENTS)) { if (this.segments == null) this.segments = new SegmentsStats(); this.segments.add(segmentsStats); } @@ -170,7 +170,7 @@ public ClusterStatsIndices( } indexCount = countsPerIndex.size(); - if (IndexMetrics.SHARDS.containedIn(indicesMetrics)) { + if (indicesMetrics.contains(IndexMetric.SHARDS)) { shards = new ShardStats(); for (final ShardStats indexCountsCursor : countsPerIndex.values()) { shards.addIndexShardCount(indexCountsCursor); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 9ebda488d3ac7..657e3b425db21 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -90,26 +90,29 @@ public class ClusterStatsNodes implements ToXContentFragment { private final PackagingTypes packagingTypes; private final IngestStats ingestStats; - public static final Set NODE_STATS_METRICS = Set.of( - Metric.OS.metricName(), - Metric.PROCESS.metricName(), - Metric.JVM.metricName(), - Metric.FS.metricName(), - Metric.PLUGINS.metricName(), - Metric.INGEST.metricName(), - Metric.NETWORK_TYPES.metricName(), - Metric.DISCOVERY_TYPES.metricName(), - Metric.PACKAGING_TYPES.metricName() + public static final Set NODE_STATS_METRICS = Set.of( + // Stats computed from node info and node stat + Metric.OS, + Metric.JVM, + // Stats computed from node stat + Metric.FS, + Metric.PROCESS, + Metric.INGEST, + // Stats computed from node info + Metric.PLUGINS, + Metric.NETWORK_TYPES, + Metric.DISCOVERY_TYPES, + Metric.PACKAGING_TYPES ); ClusterStatsNodes(List nodeResponses) { - this(Metric.allMetrics(), nodeResponses); + this(Set.of(Metric.values()), nodeResponses); } - ClusterStatsNodes(Set requestedMetrics, List nodeResponses) { + ClusterStatsNodes(Set requestedMetrics, List nodeResponses) { this.versions = new HashSet<>(); - this.fs = ClusterStatsRequest.Metric.FS.containedIn(requestedMetrics) ? new FsInfo.Path() : null; - this.plugins = ClusterStatsRequest.Metric.PLUGINS.containedIn(requestedMetrics) ? new HashSet<>() : null; + this.fs = requestedMetrics.contains(ClusterStatsRequest.Metric.FS) ? new FsInfo.Path() : null; + this.plugins = requestedMetrics.contains(ClusterStatsRequest.Metric.PLUGINS) ? new HashSet<>() : null; Set seenAddresses = new HashSet<>(nodeResponses.size()); List nodeInfos = new ArrayList<>(nodeResponses.size()); @@ -132,18 +135,15 @@ public class ClusterStatsNodes implements ToXContentFragment { this.fs.add(nodeResponse.nodeStats().getFs().getTotal()); } } + this.counts = new Counts(nodeInfos); - this.os = ClusterStatsRequest.Metric.OS.containedIn(requestedMetrics) ? new OsStats(nodeInfos, nodeStats) : null; - this.process = ClusterStatsRequest.Metric.PROCESS.containedIn(requestedMetrics) ? new ProcessStats(nodeStats) : null; - this.jvm = ClusterStatsRequest.Metric.JVM.containedIn(requestedMetrics) ? new JvmStats(nodeInfos, nodeStats) : null; - this.networkTypes = ClusterStatsRequest.Metric.NETWORK_TYPES.containedIn(requestedMetrics) ? new NetworkTypes(nodeInfos) : null; - this.discoveryTypes = ClusterStatsRequest.Metric.DISCOVERY_TYPES.containedIn(requestedMetrics) - ? new DiscoveryTypes(nodeInfos) - : null; - this.packagingTypes = ClusterStatsRequest.Metric.PACKAGING_TYPES.containedIn(requestedMetrics) - ? new PackagingTypes(nodeInfos) - : null; - this.ingestStats = ClusterStatsRequest.Metric.INGEST.containedIn(requestedMetrics) ? new IngestStats(nodeStats) : null; + this.networkTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.NETWORK_TYPES) ? new NetworkTypes(nodeInfos) : null; + this.discoveryTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.DISCOVERY_TYPES) ? new DiscoveryTypes(nodeInfos) : null; + this.packagingTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.PACKAGING_TYPES) ? new PackagingTypes(nodeInfos) : null; + this.ingestStats = requestedMetrics.contains(ClusterStatsRequest.Metric.INGEST) ? new IngestStats(nodeStats) : null; + this.process = requestedMetrics.contains(ClusterStatsRequest.Metric.PROCESS) ? new ProcessStats(nodeStats) : null; + this.os = requestedMetrics.contains(ClusterStatsRequest.Metric.OS) ? new OsStats(nodeInfos, nodeStats) : null; + this.jvm = requestedMetrics.contains(ClusterStatsRequest.Metric.JVM) ? new JvmStats(nodeInfos, nodeStats) : null; } public Counts getCounts() { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 1526c817ddbdb..25ab3e933a686 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -39,10 +39,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; -import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; /** * A request to get cluster level stats. @@ -52,8 +50,8 @@ @PublicApi(since = "1.0.0") public class ClusterStatsRequest extends BaseNodesRequest { - private final Set requestedMetrics = new HashSet<>(); - private final Set indexMetricsRequested = new HashSet<>(); + private final Set requestedMetrics = new HashSet<>(); + private final Set indexMetricsRequested = new HashSet<>(); private Boolean applyMetricFiltering = false; public ClusterStatsRequest(StreamInput in) throws IOException { @@ -63,8 +61,18 @@ public ClusterStatsRequest(StreamInput in) throws IOException { } if (in.getVersion().onOrAfter(Version.V_3_0_0)) { applyMetricFiltering = in.readOptionalBoolean(); - requestedMetrics.addAll(in.readStringList()); - indexMetricsRequested.addAll(in.readStringList()); + final long longMetricsFlags = in.readLong(); + for (Metric metric : Metric.values()) { + if ((longMetricsFlags & (1 << metric.getIndex())) != 0) { + requestedMetrics.add(metric); + } + } + final long longIndexMetricFlags = in.readLong(); + for (IndexMetric indexMetric : IndexMetric.values()) { + if ((longIndexMetricFlags & (1 << indexMetric.getIndex())) != 0) { + indexMetricsRequested.add(indexMetric); + } + } } } @@ -97,10 +105,7 @@ public void applyMetricFiltering(boolean honourMetricFiltering) { /** * Add Metric */ - public ClusterStatsRequest addMetric(String metric) { - if (Metric.allMetrics().contains(metric) == false) { - throw new IllegalStateException("Used an illegal Metric: " + metric); - } + public ClusterStatsRequest addMetric(Metric metric) { requestedMetrics.add(metric); return this; } @@ -108,33 +113,22 @@ public ClusterStatsRequest addMetric(String metric) { /** * Get the names of requested metrics */ - public Set requestedMetrics() { + public Set requestedMetrics() { return new HashSet<>(requestedMetrics); } /** * Add IndexMetric */ - public ClusterStatsRequest addIndexMetric(String indexMetric) { - if (IndexMetrics.allIndicesMetrics().contains(indexMetric) == false) { - throw new IllegalStateException("Used an illegal index metric: " + indexMetric); - } + public ClusterStatsRequest addIndexMetric(IndexMetric indexMetric) { indexMetricsRequested.add(indexMetric); return this; } - public Set indicesMetrics() { + public Set indicesMetrics() { return new HashSet<>(indexMetricsRequested); } - public void clearRequestedMetrics() { - requestedMetrics.clear(); - } - - public void clearIndicesMetrics() { - indexMetricsRequested.clear(); - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -143,8 +137,16 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getVersion().onOrAfter(Version.V_3_0_0)) { out.writeOptionalBoolean(applyMetricFiltering); - out.writeStringArray(requestedMetrics.toArray(new String[0])); - out.writeStringArray(indexMetricsRequested.toArray(new String[0])); + long longMetricFlags = 0; + for (Metric metric : requestedMetrics) { + longMetricFlags |= (1 << metric.getIndex()); + } + out.writeLong(longMetricFlags); + long longIndexMetricFlags = 0; + for (IndexMetric indexMetric : indexMetricsRequested) { + longIndexMetricFlags |= (1 << indexMetric.getIndex()); + } + out.writeLong(longIndexMetricFlags); } } @@ -154,33 +156,32 @@ public void writeTo(StreamOutput out) throws IOException { */ @PublicApi(since = "3.0.0") public enum Metric { - OS("os"), - PROCESS("process"), - JVM("jvm"), - FS("fs"), - PLUGINS("plugins"), - INGEST("ingest"), - NETWORK_TYPES("network_types"), - DISCOVERY_TYPES("discovery_types"), - PACKAGING_TYPES("packaging_types"), - INDICES("indices"); + OS("os", 0), + JVM("jvm", 1), + FS("fs", 2), + PROCESS("process", 3), + INGEST("ingest", 4), + PLUGINS("plugins", 5), + NETWORK_TYPES("network_types", 6), + DISCOVERY_TYPES("discovery_types", 7), + PACKAGING_TYPES("packaging_types", 8), + INDICES("indices", 9); private String metricName; - Metric(String name) { + private int index; + + Metric(String name, int index) { this.metricName = name; + this.index = index; } public String metricName() { return this.metricName; } - public static Set allMetrics() { - return Arrays.stream(values()).map(Metric::metricName).collect(Collectors.toSet()); - } - - public boolean containedIn(Set metricNames) { - return metricNames.contains(this.metricName()); + public int getIndex() { + return index; } } @@ -190,33 +191,32 @@ public boolean containedIn(Set metricNames) { * from the cluster stats endpoint. */ @PublicApi(since = "3.0.0") - public enum IndexMetrics { - SHARDS("shards"), - DOCS("docs"), - STORE("store"), - FIELDDATA("fielddata"), - QUERY_CACHE("query_cache"), - COMPLETION("completion"), - SEGMENTS("segments"), - ANALYSIS("analysis"), - MAPPINGS("mappings"); + public enum IndexMetric { + SHARDS("shards", 0), + DOCS("docs", 1), + STORE("store", 2), + FIELDDATA("fielddata", 3), + QUERY_CACHE("query_cache", 4), + COMPLETION("completion", 5), + SEGMENTS("segments", 6), + ANALYSIS("analysis", 7), + MAPPINGS("mappings", 8); private String metricName; - IndexMetrics(String name) { + private int index; + + IndexMetric(String name, int index) { this.metricName = name; + this.index = index; } public String metricName() { return this.metricName; } - public boolean containedIn(Set metricNames) { - return metricNames.contains(this.metricName()); - } - - public static Set allIndicesMetrics() { - return Arrays.stream(values()).map(IndexMetrics::metricName).collect(Collectors.toSet()); + public int getIndex() { + return this.index; } } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index 63305825d3ed3..fb2ba86531a32 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -38,6 +38,9 @@ import java.util.Set; +import static org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import static org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; + /** * Transport request builder for obtaining cluster stats * @@ -63,12 +66,12 @@ public final ClusterStatsRequestBuilder applyMetricFiltering(boolean applyMetric return this; } - public final ClusterStatsRequestBuilder requestMetrics(Set requestMetrics) { + public final ClusterStatsRequestBuilder requestMetrics(Set requestMetrics) { requestMetrics.forEach(request::addMetric); return this; } - public final ClusterStatsRequestBuilder indexMetrics(Set indexMetrics) { + public final ClusterStatsRequestBuilder indexMetrics(Set indexMetrics) { indexMetrics.forEach(request::addIndexMetric); return this; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java index f994b0ddb78b1..870996bb61b23 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -33,6 +33,7 @@ package org.opensearch.action.admin.cluster.stats; import org.opensearch.action.FailedNodeException; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; @@ -91,16 +92,7 @@ public ClusterStatsResponse( List failures, ClusterState state ) { - this( - timestamp, - clusterUUID, - clusterName, - nodes, - failures, - state, - ClusterStatsRequest.Metric.allMetrics(), - ClusterStatsRequest.IndexMetrics.allIndicesMetrics() - ); + this(timestamp, clusterUUID, clusterName, nodes, failures, state, Set.of(Metric.values()), Set.of(IndexMetric.values())); } public ClusterStatsResponse( @@ -110,8 +102,8 @@ public ClusterStatsResponse( List nodes, List failures, ClusterState state, - Set requestedMetrics, - Set indicesMetrics + Set requestedMetrics, + Set indicesMetrics ) { super(clusterName, nodes, failures); this.clusterUUID = clusterUUID; @@ -119,11 +111,9 @@ public ClusterStatsResponse( nodesStats = requestedMetrics.stream().anyMatch(ClusterStatsNodes.NODE_STATS_METRICS::contains) ? new ClusterStatsNodes(requestedMetrics, nodes) : null; - MappingStats mappingStats = ClusterStatsRequest.IndexMetrics.MAPPINGS.containedIn(indicesMetrics) ? MappingStats.of(state) : null; - AnalysisStats analysisStats = ClusterStatsRequest.IndexMetrics.ANALYSIS.containedIn(indicesMetrics) - ? AnalysisStats.of(state) - : null; - indicesStats = Metric.INDICES.containedIn(requestedMetrics) + MappingStats mappingStats = indicesMetrics.contains(IndexMetric.MAPPINGS) ? MappingStats.of(state) : null; + AnalysisStats analysisStats = indicesMetrics.contains(IndexMetric.ANALYSIS) ? AnalysisStats.of(state) : null; + indicesStats = requestedMetrics.contains(Metric.INDICES) ? new ClusterStatsIndices(indicesMetrics, nodes, mappingStats, analysisStats) : null; ClusterHealthStatus status = null; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 3ebb501a5e7e1..099837e3e7ca5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -79,20 +79,19 @@ public class TransportClusterStatsAction extends TransportNodesAction< TransportClusterStatsAction.ClusterStatsNodeRequest, ClusterStatsNodeResponse> { - private static final Map INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP = Map.of( - ClusterStatsRequest.IndexMetrics.DOCS.metricName(), + private static final Map INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP = Map.of( + ClusterStatsRequest.IndexMetric.DOCS, CommonStatsFlags.Flag.Docs, - ClusterStatsRequest.IndexMetrics.STORE.metricName(), + ClusterStatsRequest.IndexMetric.STORE, CommonStatsFlags.Flag.Store, - ClusterStatsRequest.IndexMetrics.FIELDDATA.metricName(), + ClusterStatsRequest.IndexMetric.FIELDDATA, CommonStatsFlags.Flag.FieldData, - ClusterStatsRequest.IndexMetrics.QUERY_CACHE.metricName(), + ClusterStatsRequest.IndexMetric.QUERY_CACHE, CommonStatsFlags.Flag.QueryCache, - ClusterStatsRequest.IndexMetrics.COMPLETION.metricName(), + ClusterStatsRequest.IndexMetric.COMPLETION, CommonStatsFlags.Flag.Completion, - ClusterStatsRequest.IndexMetrics.SEGMENTS.metricName(), + ClusterStatsRequest.IndexMetric.SEGMENTS, CommonStatsFlags.Flag.Segments - ); private final NodeService nodeService; @@ -171,20 +170,20 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) { NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false, false); boolean applyMetricFiltering = nodeRequest.request.applyMetricFiltering(); - Set requestedMetrics = nodeRequest.request.requestedMetrics(); + Set requestedMetrics = nodeRequest.request.requestedMetrics(); NodeStats nodeStats = nodeService.stats( CommonStatsFlags.NONE, - !applyMetricFiltering || Metric.OS.containedIn(requestedMetrics), - !applyMetricFiltering || Metric.PROCESS.containedIn(requestedMetrics), - !applyMetricFiltering || Metric.JVM.containedIn(requestedMetrics), + isMetricRequired(applyMetricFiltering, Metric.OS, requestedMetrics), + isMetricRequired(applyMetricFiltering, Metric.PROCESS, requestedMetrics), + isMetricRequired(applyMetricFiltering, Metric.JVM, requestedMetrics), false, - !applyMetricFiltering || Metric.FS.containedIn(requestedMetrics), + isMetricRequired(applyMetricFiltering, Metric.FS, requestedMetrics), false, false, false, false, false, - !applyMetricFiltering || Metric.INGEST.containedIn(requestedMetrics), + isMetricRequired(applyMetricFiltering, Metric.INGEST, requestedMetrics), false, false, false, @@ -203,11 +202,11 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false ); List shardsStats = new ArrayList<>(); - if (!applyMetricFiltering || Metric.INDICES.containedIn(requestedMetrics)) { + if (isMetricRequired(applyMetricFiltering, Metric.INDICES, requestedMetrics)) { CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); - for (String metric : nodeRequest.request.indicesMetrics()) { - if (INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.containsKey(metric)) { - commonStatsFlags.set(INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.get(metric), true); + for (ClusterStatsRequest.IndexMetric indexMetric : nodeRequest.request.indicesMetrics()) { + if (INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.containsKey(indexMetric)) { + commonStatsFlags.set(INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.get(indexMetric), true); } } for (IndexService indexService : indicesService) { @@ -257,6 +256,10 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq ); } + private boolean isMetricRequired(boolean applyMetricFilter, Metric metric, Set requestedMetrics) { + return !applyMetricFilter || requestedMetrics.contains(metric); + } + /** * Inner Cluster Stats Node Request * diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index e3858c27815ab..3874e043eea30 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -80,15 +80,15 @@ public List routes() { static { Map> metricRequestConsumerMap = new HashMap<>(); for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { - metricRequestConsumerMap.put(metric.metricName(), request -> request.addMetric(metric.metricName())); + metricRequestConsumerMap.put(metric.metricName(), request -> request.addMetric(metric)); } METRIC_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricRequestConsumerMap); } static { Map> metricMap = new HashMap<>(); - for (ClusterStatsRequest.IndexMetrics indexMetric : ClusterStatsRequest.IndexMetrics.values()) { - metricMap.put(indexMetric.metricName(), request -> request.addIndexMetric(indexMetric.metricName())); + for (ClusterStatsRequest.IndexMetric indexMetric : ClusterStatsRequest.IndexMetric.values()) { + metricMap.put(indexMetric.metricName(), request -> request.addIndexMetric(indexMetric)); } INDEX_METRIC_TO_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricMap); } @@ -100,7 +100,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", null)); + Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", null)); String[] nodeIds = request.paramAsStringArray("nodeId", null); @@ -126,7 +126,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Locale.ROOT, "request [%s] contains _all and individual index metrics [%s]", request.path(), - request.param("sub_metric") + request.param("index_metric") ) ); } @@ -162,10 +162,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } - if (ClusterStatsRequest.Metric.INDICES.containedIn(metricsRequested)) { + if (metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName())) { final Set indexMetricsRequested = new HashSet<>(); if (indexMetrics.isEmpty() || indexMetrics.contains("_all")) { - indexMetricsRequested.addAll(ClusterStatsRequest.IndexMetrics.allIndicesMetrics()); + indexMetricsRequested.addAll(INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()); } else { indexMetricsRequested.addAll(indexMetrics); } @@ -181,7 +181,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (!invalidIndexMetrics.isEmpty()) { throw new IllegalArgumentException( - unrecognized(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index_metric") + unrecognized(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") ); } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java index 1c64c01eb0e49..df77590879e87 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java @@ -8,7 +8,7 @@ package org.opensearch.action.admin.cluster.stats; -import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetrics; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpClient; @@ -59,8 +59,10 @@ public void testRequestedMetrics() { this.testClient, ClusterStatsAction.INSTANCE ); - clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.OS.metricName(), Metric.JVM.metricName())); - assertEquals(Set.of(Metric.OS.metricName(), Metric.JVM.metricName()), clusterStatsRequestBuilder.request().requestedMetrics()); + clusterStatsRequestBuilder.applyMetricFiltering(true); + clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.OS, Metric.JVM)); + assertTrue(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertEquals(Set.of(Metric.OS, Metric.JVM), clusterStatsRequestBuilder.request().requestedMetrics()); } public void testIndicesMetrics() { @@ -68,13 +70,12 @@ public void testIndicesMetrics() { this.testClient, ClusterStatsAction.INSTANCE ); - clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.INDICES.metricName(), Metric.JVM.metricName())); - clusterStatsRequestBuilder.indexMetrics(Set.of(IndexMetrics.MAPPINGS.metricName(), IndexMetrics.ANALYSIS.metricName())); - assertEquals(Set.of(Metric.INDICES.metricName(), Metric.JVM.metricName()), clusterStatsRequestBuilder.request().requestedMetrics()); - assertEquals( - Set.of(IndexMetrics.MAPPINGS.metricName(), IndexMetrics.ANALYSIS.metricName()), - clusterStatsRequestBuilder.request().indicesMetrics() - ); + clusterStatsRequestBuilder.applyMetricFiltering(true); + clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.INDICES, Metric.JVM)); + clusterStatsRequestBuilder.indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)); + assertTrue(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertEquals(Set.of(Metric.INDICES, Metric.JVM), clusterStatsRequestBuilder.request().requestedMetrics()); + assertEquals(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS), clusterStatsRequestBuilder.request().indicesMetrics()); } } diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java new file mode 100644 index 0000000000000..9ec04dfc55ee2 --- /dev/null +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java @@ -0,0 +1,85 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.admin.cluster; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; + +import java.util.HashMap; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.object.HasToString.hasToString; +import static org.mockito.Mockito.mock; + +public class RestClusterStatsActionTests extends OpenSearchTestCase { + + private RestClusterStatsAction action; + + @Override + public void setUp() throws Exception { + super.setUp(); + action = new RestClusterStatsAction(); + } + + public void testUnrecognizedMetric() { + final HashMap params = new HashMap<>(); + final String metric = randomAlphaOfLength(64); + params.put("metric", metric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat(e, hasToString(containsString("request [/_cluster/stats] contains unrecognized metric: [" + metric + "]"))); + } + + public void testUnrecognizedIndexMetric() { + final HashMap params = new HashMap<>(); + params.put("metric", "_all,"); + final String indexMetric = randomAlphaOfLength(64); + params.put("index_metric", indexMetric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat(e, hasToString(containsString("request [/_cluster/stats] contains unrecognized index metric: [" + indexMetric + "]"))); + } + + public void testAllMetricsRequestWithOtherMetric() { + final HashMap params = new HashMap<>(); + final String metric = randomSubsetOf(1, RestClusterStatsAction.METRIC_REQUEST_CONSUMER_MAP.keySet()).get(0); + params.put("metric", "_all," + metric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat(e, hasToString(containsString("request [/_cluster/stats] contains _all and individual metrics [_all," + metric + "]"))); + } + + public void testAllIndexMetricsRequestWithOtherIndicesMetric() { + final HashMap params = new HashMap<>(); + params.put("metric", "_all,"); + final String indexMetric = randomSubsetOf(1, RestClusterStatsAction.INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()).get(0); + params.put("index_metric", "_all," + indexMetric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat( + e, + hasToString(containsString("request [/_cluster/stats] contains _all and individual index metrics [_all," + indexMetric + "]")) + ); + } + +} From 30521cc30a19503ac2175fec0d08c897bbd19ea5 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Mon, 30 Sep 2024 18:55:46 +0530 Subject: [PATCH 04/12] Add test cases for cluster stats request/response serde. Signed-off-by: Swetha Guptha --- .../admin/cluster/stats/ClusterStatsIT.java | 94 ++++++- .../cluster/stats/ClusterStatsRequest.java | 16 +- .../stats/ClusterStatsRequestBuilder.java | 4 +- .../stats/TransportClusterStatsAction.java | 72 ++++-- .../index/cache/query/QueryCacheStats.java | 3 + .../index/fielddata/FieldDataStats.java | 3 + .../admin/cluster/RestClusterStatsAction.java | 38 +-- .../ClusterStatsRequestBuilderTests.java | 14 +- .../stats/ClusterStatsRequestTests.java | 49 ++++ .../stats/ClusterStatsResponseTests.java | 231 ++++++++++++++++++ .../cluster/RestClusterStatsActionTests.java | 74 ++++++ 11 files changed, 539 insertions(+), 59 deletions(-) create mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index 9cddcc4b3eeca..aa4f799db4083 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -39,6 +39,7 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; +import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Client; import org.opensearch.client.Requests; import org.opensearch.cluster.health.ClusterHealthStatus; @@ -320,6 +321,52 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent()); } + public void testValuesSmokeScreenWithMetricFilter() throws IOException, ExecutionException, InterruptedException { + internalCluster().startNodes(randomIntBetween(1, 3)); + index("test1", "type", "1", "f", "f"); + + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .computeAllMetrics(false) + .requestMetrics(Set.of(Metric.values())) + .indexMetrics(Set.of(IndexMetric.values())) + .get(); + String msg = response.toString(); + assertThat(msg, response.getTimestamp(), Matchers.greaterThan(946681200000L)); // 1 Jan 2000 + assertThat(msg, response.indicesStats.getStore().getSizeInBytes(), Matchers.greaterThan(0L)); + + assertThat(msg, response.nodesStats.getFs().getTotal().getBytes(), Matchers.greaterThan(0L)); + assertThat(msg, response.nodesStats.getJvm().getVersions().size(), Matchers.greaterThan(0)); + + assertThat(msg, response.nodesStats.getVersions().size(), Matchers.greaterThan(0)); + assertThat(msg, response.nodesStats.getVersions().contains(Version.CURRENT), Matchers.equalTo(true)); + assertThat(msg, response.nodesStats.getPlugins().size(), Matchers.greaterThanOrEqualTo(0)); + + assertThat(msg, response.nodesStats.getProcess().count, Matchers.greaterThan(0)); + // 0 happens when not supported on platform + assertThat(msg, response.nodesStats.getProcess().getAvgOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(0L)); + // these can be -1 if not supported on platform + assertThat(msg, response.nodesStats.getProcess().getMinOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L)); + assertThat(msg, response.nodesStats.getProcess().getMaxOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L)); + + NodesStatsResponse nodesStatsResponse = client().admin().cluster().prepareNodesStats().addMetric(OS.metricName()).get(); + long total = 0; + long free = 0; + long used = 0; + for (NodeStats nodeStats : nodesStatsResponse.getNodes()) { + total += nodeStats.getOs().getMem().getTotal().getBytes(); + free += nodeStats.getOs().getMem().getFree().getBytes(); + used += nodeStats.getOs().getMem().getUsed().getBytes(); + } + assertEquals(msg, free, response.nodesStats.getOs().getMem().getFree().getBytes()); + assertEquals(msg, total, response.nodesStats.getOs().getMem().getTotal().getBytes()); + assertEquals(msg, used, response.nodesStats.getOs().getMem().getUsed().getBytes()); + assertEquals(msg, OsStats.calculatePercentage(used, total), response.nodesStats.getOs().getMem().getUsedPercent()); + assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent()); + } + public void testAllocatedProcessors() throws Exception { // start one node with 7 processors. internalCluster().startNode(Settings.builder().put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 7).build()); @@ -360,6 +407,9 @@ public void testFieldTypes() { .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) + .computeAllMetrics(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.MAPPINGS)) .get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); @@ -518,7 +568,7 @@ public void testClusterStatsApplyMetricFilterDisabled() throws IOException { .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()); - assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); ClusterStatsResponse response = clusterStatsRequestBuilder.get(); assertNotNull(response.getNodesStats()); @@ -528,7 +578,7 @@ public void testClusterStatsApplyMetricFilterDisabled() throws IOException { .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .applyMetricFiltering(false) + .computeAllMetrics(true) .requestMetrics(Set.of(Metric.FS, Metric.JVM, Metric.PLUGINS, Metric.OS)) .get(); @@ -578,7 +628,7 @@ public void testClusterStatsWithMetricsFilter() { .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()); - assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); ClusterStatsResponse response = clusterStatsRequestBuilder.get(); assertNotNull(response); @@ -590,7 +640,7 @@ public void testClusterStatsWithMetricsFilter() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(ClusterStatsNodes.NODE_STATS_METRICS) - .applyMetricFiltering(true) + .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithAllNodeStatsMetrics); assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats()); @@ -615,7 +665,7 @@ public void testClusterStatsWithMetricsFilter() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.PLUGINS, Metric.INDICES)) - .applyMetricFiltering(true) + .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithIndicesRequestMetrics); assertNotNull(statsResponseWithIndicesRequestMetrics.getNodesStats()); @@ -633,12 +683,14 @@ public void testClusterStatsWithIndexMetricFilter() { ensureGreen(); client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source("{\"test_type\" : \"metrics_filter\"}"); + client().index(indexRequest); ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()); - assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); ClusterStatsResponse response = clusterStatsRequestBuilder.get(); assertNotNull(response); @@ -651,7 +703,7 @@ public void testClusterStatsWithIndexMetricFilter() { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.INDICES)) .indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)) - .applyMetricFiltering(true) + .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithSpecificIndicesMetrics); assertNull(statsResponseWithSpecificIndicesMetrics.getNodesStats()); @@ -666,7 +718,7 @@ public void testClusterStatsWithIndexMetricFilter() { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.INDICES)) .indexMetrics(Set.of(IndexMetric.values())) - .applyMetricFiltering(true) + .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithAllIndicesMetrics); assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); @@ -690,6 +742,32 @@ public void testClusterStatsWithIndexMetricFilter() { assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getStore()); } + public void testClusterStatsWithIndexMetricWithDocsFilter() throws IOException { + internalCluster().startNode(); + createIndex("test1"); + + client().prepareIndex("test1").setId(Integer.toString(1)).setSource("field1", "value1").execute().actionGet(); + client().prepareIndex("test1").setId(Integer.toString(2)).setSource("field2", "value2").execute().actionGet(); + refreshAndWaitForReplication(); + + ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.DOCS)) + .computeAllMetrics(true) + .get(); + assertNotNull(statsResponseWithAllIndicesMetrics); + assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + assertNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getShards()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs()); + assertEquals(2, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getCount()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getDeleted()); + assertTrue(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getAverageSizeInBytes() > 0); + } + private Map getExpectedCounts( int dataRoleCount, int masterRoleCount, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 25ab3e933a686..4b0a32517f48e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -52,7 +52,7 @@ public class ClusterStatsRequest extends BaseNodesRequest { private final Set requestedMetrics = new HashSet<>(); private final Set indexMetricsRequested = new HashSet<>(); - private Boolean applyMetricFiltering = false; + private Boolean computeAllMetric = true; public ClusterStatsRequest(StreamInput in) throws IOException { super(in); @@ -60,7 +60,7 @@ public ClusterStatsRequest(StreamInput in) throws IOException { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } if (in.getVersion().onOrAfter(Version.V_3_0_0)) { - applyMetricFiltering = in.readOptionalBoolean(); + computeAllMetric = in.readOptionalBoolean(); final long longMetricsFlags = in.readLong(); for (Metric metric : Metric.values()) { if ((longMetricsFlags & (1 << metric.getIndex())) != 0) { @@ -94,12 +94,12 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon this.useAggregatedNodeLevelResponses = useAggregatedNodeLevelResponses; } - public boolean applyMetricFiltering() { - return applyMetricFiltering; + public boolean computeAllMetrics() { + return computeAllMetric; } - public void applyMetricFiltering(boolean honourMetricFiltering) { - this.applyMetricFiltering = honourMetricFiltering; + public void computeAllMetrics(boolean computeAllMetrics) { + this.computeAllMetric = computeAllMetrics; } /** @@ -136,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } if (out.getVersion().onOrAfter(Version.V_3_0_0)) { - out.writeOptionalBoolean(applyMetricFiltering); + out.writeOptionalBoolean(computeAllMetric); long longMetricFlags = 0; for (Metric metric : requestedMetrics) { longMetricFlags |= (1 << metric.getIndex()); @@ -192,6 +192,7 @@ public int getIndex() { */ @PublicApi(since = "3.0.0") public enum IndexMetric { + // Metrics computed from ShardStats SHARDS("shards", 0), DOCS("docs", 1), STORE("store", 2), @@ -199,6 +200,7 @@ public enum IndexMetric { QUERY_CACHE("query_cache", 4), COMPLETION("completion", 5), SEGMENTS("segments", 6), + // Metrics computed from ClusterState ANALYSIS("analysis", 7), MAPPINGS("mappings", 8); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index fb2ba86531a32..34fd9ea06235e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -61,8 +61,8 @@ public final ClusterStatsRequestBuilder useAggregatedNodeLevelResponses(boolean return this; } - public final ClusterStatsRequestBuilder applyMetricFiltering(boolean applyMetricFiltering) { - request.applyMetricFiltering(applyMetricFiltering); + public final ClusterStatsRequestBuilder computeAllMetrics(boolean applyMetricFiltering) { + request.computeAllMetrics(applyMetricFiltering); return this; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 099837e3e7ca5..b7a645240f407 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -64,6 +64,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -94,6 +95,21 @@ public class TransportClusterStatsAction extends TransportNodesAction< CommonStatsFlags.Flag.Segments ); + private static final Map SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC = Map.of( + CommonStatsFlags.Flag.Docs, + ClusterStatsRequest.IndexMetric.DOCS, + CommonStatsFlags.Flag.Store, + ClusterStatsRequest.IndexMetric.STORE, + CommonStatsFlags.Flag.FieldData, + ClusterStatsRequest.IndexMetric.FIELDDATA, + CommonStatsFlags.Flag.QueryCache, + ClusterStatsRequest.IndexMetric.QUERY_CACHE, + CommonStatsFlags.Flag.Completion, + ClusterStatsRequest.IndexMetric.COMPLETION, + CommonStatsFlags.Flag.Segments, + ClusterStatsRequest.IndexMetric.SEGMENTS + ); + private final NodeService nodeService; private final IndicesService indicesService; @@ -133,16 +149,14 @@ protected ClusterStatsResponse newResponse( + " the cluster state that are too slow for a transport thread" ); ClusterState state = clusterService.state(); - if (request.applyMetricFiltering()) { + if (request.computeAllMetrics()) { return new ClusterStatsResponse( System.currentTimeMillis(), state.metadata().clusterUUID(), clusterService.getClusterName(), responses, failures, - state, - request.requestedMetrics(), - request.indicesMetrics() + state ); } else { return new ClusterStatsResponse( @@ -151,7 +165,9 @@ protected ClusterStatsResponse newResponse( clusterService.getClusterName(), responses, failures, - state + state, + request.requestedMetrics(), + request.indicesMetrics() ); } } @@ -169,21 +185,19 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce @Override protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) { NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false, false); - boolean applyMetricFiltering = nodeRequest.request.applyMetricFiltering(); - Set requestedMetrics = nodeRequest.request.requestedMetrics(); NodeStats nodeStats = nodeService.stats( CommonStatsFlags.NONE, - isMetricRequired(applyMetricFiltering, Metric.OS, requestedMetrics), - isMetricRequired(applyMetricFiltering, Metric.PROCESS, requestedMetrics), - isMetricRequired(applyMetricFiltering, Metric.JVM, requestedMetrics), + isMetricRequired(Metric.OS, nodeRequest.request), + isMetricRequired(Metric.PROCESS, nodeRequest.request), + isMetricRequired(Metric.JVM, nodeRequest.request), false, - isMetricRequired(applyMetricFiltering, Metric.FS, requestedMetrics), + isMetricRequired(Metric.FS, nodeRequest.request), false, false, false, false, false, - isMetricRequired(applyMetricFiltering, Metric.INGEST, requestedMetrics), + isMetricRequired(Metric.INGEST, nodeRequest.request), false, false, false, @@ -202,13 +216,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false ); List shardsStats = new ArrayList<>(); - if (isMetricRequired(applyMetricFiltering, Metric.INDICES, requestedMetrics)) { - CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); - for (ClusterStatsRequest.IndexMetric indexMetric : nodeRequest.request.indicesMetrics()) { - if (INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.containsKey(indexMetric)) { - commonStatsFlags.set(INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.get(indexMetric), true); - } - } + if (isMetricRequired(Metric.INDICES, nodeRequest.request)) { + CommonStatsFlags commonStatsFlags = getCommonStatsFlags(nodeRequest); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { @@ -256,8 +265,29 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq ); } - private boolean isMetricRequired(boolean applyMetricFilter, Metric metric, Set requestedMetrics) { - return !applyMetricFilter || requestedMetrics.contains(metric); + /** + * A metric is required when: all cluster stats are required (OR) if the metric was requested + * @param metric + * @param clusterStatsRequest + * @return + */ + private boolean isMetricRequired(Metric metric, ClusterStatsRequest clusterStatsRequest) { + return clusterStatsRequest.computeAllMetrics() || clusterStatsRequest.requestedMetrics().contains(metric); + } + + private static CommonStatsFlags getCommonStatsFlags(ClusterStatsNodeRequest nodeRequest) { + Set requestedCommonStatsFlags = new HashSet<>(); + if (nodeRequest.request.computeAllMetrics()) { + requestedCommonStatsFlags.addAll(INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.values()); + } else { + for (Map.Entry entry : SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC + .entrySet()) { + if (nodeRequest.request.indicesMetrics().contains(entry.getValue())) { + requestedCommonStatsFlags.add(entry.getKey()); + } + } + } + return new CommonStatsFlags(requestedCommonStatsFlags.toArray(new CommonStatsFlags.Flag[0])); } /** diff --git a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java index d844e5cbb8897..8ca3157d9f775 100644 --- a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java +++ b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java @@ -77,6 +77,9 @@ public QueryCacheStats(long ramBytesUsed, long hitCount, long missCount, long ca } public void add(QueryCacheStats stats) { + if (stats == null) { + return; + } ramBytesUsed += stats.ramBytesUsed; hitCount += stats.hitCount; missCount += stats.missCount; diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java index 85b435e969bfa..1bd81c840a3d4 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java @@ -80,6 +80,9 @@ public FieldDataStats(long memorySize, long evictions, @Nullable FieldMemoryStat } public void add(FieldDataStats stats) { + if (stats == null) { + return; + } this.memorySize += stats.memorySize; this.evictions += stats.evictions; if (stats.fields != null) { diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index 3874e043eea30..daa4a75285823 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -100,8 +100,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + ClusterStatsRequest clusterStatsRequest = fromRequest(request); + return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); + } + + public static ClusterStatsRequest fromRequest(final RestRequest request) { Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); - Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", null)); String[] nodeIds = request.paramAsStringArray("nodeId", null); ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); @@ -120,6 +124,17 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } + final Set metricsRequested = new HashSet<>(); + if (metrics.contains("_all")) { + metricsRequested.addAll(METRIC_REQUEST_CONSUMER_MAP.keySet()); + } else { + metricsRequested.addAll(metrics); + } + + Set indexMetrics = Strings.tokenizeByCommaToSet( + request.param("index_metric", metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName()) ? "_all" : null) + ); + if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { throw new IllegalArgumentException( String.format( @@ -131,14 +146,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } - clusterStatsRequest.applyMetricFiltering(true); - - final Set metricsRequested = new HashSet<>(); - if (metrics.contains("_all")) { - metricsRequested.addAll(METRIC_REQUEST_CONSUMER_MAP.keySet()); - } else { - metricsRequested.addAll(metrics); - } final Set invalidMetrics = new TreeSet<>(); for (String metric : metricsRequested) { Consumer clusterStatsRequestConsumer = METRIC_REQUEST_CONSUMER_MAP.get(metric); @@ -149,8 +156,11 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } } if (!invalidMetrics.isEmpty()) { - throw new IllegalArgumentException(unrecognized(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric")); + throw new IllegalArgumentException( + unrecognizedStrings(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric") + ); } + if (!metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName()) && !indexMetrics.isEmpty()) { throw new IllegalArgumentException( String.format( @@ -164,7 +174,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName())) { final Set indexMetricsRequested = new HashSet<>(); - if (indexMetrics.isEmpty() || indexMetrics.contains("_all")) { + if (indexMetrics.contains("_all")) { indexMetricsRequested.addAll(INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()); } else { indexMetricsRequested.addAll(indexMetrics); @@ -181,13 +191,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (!invalidIndexMetrics.isEmpty()) { throw new IllegalArgumentException( - unrecognized(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") + unrecognizedStrings(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") ); } } + clusterStatsRequest.computeAllMetrics(false); } - - return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); + return clusterStatsRequest; } @Override diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java index df77590879e87..8bc14d4617171 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java @@ -49,9 +49,9 @@ public void testApplyMetricFiltering() { this.testClient, ClusterStatsAction.INSTANCE ); - assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); - clusterStatsRequestBuilder.applyMetricFiltering(true); - assertTrue(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); + clusterStatsRequestBuilder.computeAllMetrics(false); + assertFalse(clusterStatsRequestBuilder.request().computeAllMetrics()); } public void testRequestedMetrics() { @@ -59,9 +59,9 @@ public void testRequestedMetrics() { this.testClient, ClusterStatsAction.INSTANCE ); - clusterStatsRequestBuilder.applyMetricFiltering(true); + clusterStatsRequestBuilder.computeAllMetrics(false); clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.OS, Metric.JVM)); - assertTrue(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertFalse(clusterStatsRequestBuilder.request().computeAllMetrics()); assertEquals(Set.of(Metric.OS, Metric.JVM), clusterStatsRequestBuilder.request().requestedMetrics()); } @@ -70,10 +70,10 @@ public void testIndicesMetrics() { this.testClient, ClusterStatsAction.INSTANCE ); - clusterStatsRequestBuilder.applyMetricFiltering(true); + clusterStatsRequestBuilder.computeAllMetrics(false); clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.INDICES, Metric.JVM)); clusterStatsRequestBuilder.indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)); - assertTrue(clusterStatsRequestBuilder.request().applyMetricFiltering()); + assertFalse(clusterStatsRequestBuilder.request().computeAllMetrics()); assertEquals(Set.of(Metric.INDICES, Metric.JVM), clusterStatsRequestBuilder.request().requestedMetrics()); assertEquals(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS), clusterStatsRequestBuilder.request().indicesMetrics()); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java new file mode 100644 index 0000000000000..cd452693779f2 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.stats; + +import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; + +public class ClusterStatsRequestTests extends OpenSearchTestCase { + + public void testSerialization() throws Exception { + ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(); + clusterStatsRequest.computeAllMetrics(randomBoolean()); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.OS); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.PLUGINS); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.INDICES); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.SHARDS); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.QUERY_CACHE); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.MAPPINGS); + clusterStatsRequest.useAggregatedNodeLevelResponses(randomBoolean()); + + Version testVersion = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT); + + BytesStreamOutput output = new BytesStreamOutput(); + output.setVersion(testVersion); + clusterStatsRequest.writeTo(output); + + StreamInput streamInput = output.bytes().streamInput(); + streamInput.setVersion(testVersion); + ClusterStatsRequest deserializedClusterStatsRequest = new ClusterStatsRequest(streamInput); + + assertEquals(clusterStatsRequest.computeAllMetrics(), deserializedClusterStatsRequest.computeAllMetrics()); + assertEquals(clusterStatsRequest.requestedMetrics(), deserializedClusterStatsRequest.requestedMetrics()); + assertEquals(clusterStatsRequest.indicesMetrics(), deserializedClusterStatsRequest.indicesMetrics()); + assertEquals( + clusterStatsRequest.useAggregatedNodeLevelResponses(), + deserializedClusterStatsRequest.useAggregatedNodeLevelResponses() + ); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java new file mode 100644 index 0000000000000..850460d2f86d7 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java @@ -0,0 +1,231 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.stats; + +import org.opensearch.Build; +import org.opensearch.Version; +import org.opensearch.action.admin.cluster.node.info.NodeInfo; +import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; +import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.indices.stats.CommonStats; +import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.index.Index; +import org.opensearch.index.cache.query.QueryCacheStats; +import org.opensearch.index.engine.SegmentsStats; +import org.opensearch.index.fielddata.FieldDataStats; +import org.opensearch.index.flush.FlushStats; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.shard.IndexingStats; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.StoreStats; +import org.opensearch.monitor.jvm.JvmInfo; +import org.opensearch.monitor.os.OsInfo; +import org.opensearch.search.suggest.completion.CompletionStats; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.TransportInfo; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class ClusterStatsResponseTests extends OpenSearchTestCase { + + public void testSerializationWithIndicesMappingAndAnalysisStats() throws Exception { + List defaultClusterStatsNodeResponses = new ArrayList<>(); + + int numberOfNodes = randomIntBetween(1, 4); + Index testIndex = new Index("test-index", "_na_"); + + for (int i = 0; i < numberOfNodes; i++) { + DiscoveryNode node = new DiscoveryNode("node-" + i, buildNewFakeTransportAddress(), Version.CURRENT); + CommonStats commonStats = createRandomCommonStats(); + ShardStats[] shardStats = createShardStats(node, testIndex, commonStats); + ClusterStatsNodeResponse customClusterStatsResponse = createClusterStatsNodeResponse(node, shardStats); + defaultClusterStatsNodeResponses.add(customClusterStatsResponse); + } + ClusterStatsResponse clusterStatsResponse = new ClusterStatsResponse( + 1l, + "UUID", + new ClusterName("cluster_name"), + defaultClusterStatsNodeResponses, + List.of(), + ClusterState.EMPTY_STATE, + Set.of(ClusterStatsRequest.Metric.INDICES), + Set.of(ClusterStatsRequest.IndexMetric.MAPPINGS, ClusterStatsRequest.IndexMetric.ANALYSIS) + ); + BytesStreamOutput output = new BytesStreamOutput(); + clusterStatsResponse.writeTo(output); + + StreamInput streamInput = output.bytes().streamInput(); + ClusterStatsResponse deserializedClusterStatsResponse = new ClusterStatsResponse(streamInput); + assertEquals(clusterStatsResponse.timestamp, deserializedClusterStatsResponse.timestamp); + assertEquals(clusterStatsResponse.status, deserializedClusterStatsResponse.status); + assertEquals(clusterStatsResponse.clusterUUID, deserializedClusterStatsResponse.clusterUUID); + assertNotNull(clusterStatsResponse.indicesStats); + assertEquals(clusterStatsResponse.indicesStats.getMappings(), deserializedClusterStatsResponse.indicesStats.getMappings()); + assertEquals(clusterStatsResponse.indicesStats.getAnalysis(), deserializedClusterStatsResponse.indicesStats.getAnalysis()); + } + + public void testSerializationWithoutIndicesMappingAndAnalysisStats() throws Exception { + List defaultClusterStatsNodeResponses = new ArrayList<>(); + + int numberOfNodes = randomIntBetween(1, 4); + Index testIndex = new Index("test-index", "_na_"); + + for (int i = 0; i < numberOfNodes; i++) { + DiscoveryNode node = new DiscoveryNode("node-" + i, buildNewFakeTransportAddress(), Version.CURRENT); + CommonStats commonStats = createRandomCommonStats(); + ShardStats[] shardStats = createShardStats(node, testIndex, commonStats); + ClusterStatsNodeResponse customClusterStatsResponse = createClusterStatsNodeResponse(node, shardStats); + defaultClusterStatsNodeResponses.add(customClusterStatsResponse); + } + ClusterStatsResponse clusterStatsResponse = new ClusterStatsResponse( + 1l, + "UUID", + new ClusterName("cluster_name"), + defaultClusterStatsNodeResponses, + List.of(), + ClusterState.EMPTY_STATE, + Set.of(), + Set.of() + ); + BytesStreamOutput output = new BytesStreamOutput(); + clusterStatsResponse.writeTo(output); + + StreamInput streamInput = output.bytes().streamInput(); + ClusterStatsResponse deserializedClusterStatsResponse = new ClusterStatsResponse(streamInput); + assertEquals(clusterStatsResponse.timestamp, deserializedClusterStatsResponse.timestamp); + assertEquals(clusterStatsResponse.status, deserializedClusterStatsResponse.status); + assertEquals(clusterStatsResponse.clusterUUID, deserializedClusterStatsResponse.clusterUUID); + assertNull(clusterStatsResponse.indicesStats); + } + + private ClusterStatsNodeResponse createClusterStatsNodeResponse(DiscoveryNode node, ShardStats[] shardStats) throws IOException { + NodeInfo nodeInfo = new NodeInfo( + Version.CURRENT, + Build.CURRENT, + node, + Settings.EMPTY, + new OsInfo(randomLong(), randomInt(), randomInt(), "name", "pretty_name", "arch", "version"), + null, + JvmInfo.jvmInfo(), + null, + new TransportInfo( + new BoundTransportAddress(new TransportAddress[] { buildNewFakeTransportAddress() }, buildNewFakeTransportAddress()), + null + ), + null, + new PluginsAndModules(Collections.emptyList(), Collections.emptyList()), + null, + null, + null, + null + ); + + NodeStats nodeStats = new NodeStats( + node, + randomNonNegativeLong(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats); + + } + + private CommonStats createRandomCommonStats() { + CommonStats commonStats = new CommonStats(CommonStatsFlags.NONE); + commonStats.docs = new DocsStats(randomLongBetween(0, 10000), randomLongBetween(0, 100), randomLongBetween(0, 1000)); + commonStats.store = new StoreStats(randomLongBetween(0, 100), randomLongBetween(0, 1000)); + commonStats.indexing = new IndexingStats(); + commonStats.completion = new CompletionStats(); + commonStats.flush = new FlushStats(randomLongBetween(0, 100), randomLongBetween(0, 100), randomLongBetween(0, 100)); + commonStats.fieldData = new FieldDataStats(randomLongBetween(0, 100), randomLongBetween(0, 100), null); + commonStats.queryCache = new QueryCacheStats( + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100) + ); + commonStats.segments = new SegmentsStats(); + + return commonStats; + } + + private ShardStats[] createShardStats(DiscoveryNode localNode, Index index, CommonStats commonStats) { + List shardStatsList = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + index.getName(), + i, + localNode.getId(), + randomBoolean(), + shardRoutingState + ); + + Path path = createTempDir().resolve("indices") + .resolve(shardRouting.shardId().getIndex().getUUID()) + .resolve(String.valueOf(shardRouting.shardId().id())); + + ShardStats shardStats = new ShardStats( + shardRouting, + new ShardPath(false, path, path, shardRouting.shardId()), + commonStats, + null, + null, + null + ); + shardStatsList.add(shardStats); + } + + return shardStatsList.toArray(new ShardStats[0]); + } + +} diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java index 9ec04dfc55ee2..e3eb525696272 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java @@ -8,12 +8,15 @@ package org.opensearch.rest.action.admin.cluster; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import java.util.HashMap; +import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.object.HasToString.hasToString; @@ -29,6 +32,77 @@ public void setUp() throws Exception { action = new RestClusterStatsAction(); } + public void testFromRequestBasePath() { + final HashMap params = new HashMap<>(); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); + assertNotNull(clusterStatsRequest); + assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); + assertTrue(clusterStatsRequest.computeAllMetrics()); + assertNotNull(clusterStatsRequest.requestedMetrics()); + assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); + for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { + assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); + } + assertNotNull(clusterStatsRequest.indicesMetrics()); + assertFalse(clusterStatsRequest.indicesMetrics().isEmpty()); + for (ClusterStatsRequest.IndexMetric indexMetric : ClusterStatsRequest.IndexMetric.values()) { + assertTrue(clusterStatsRequest.indicesMetrics().contains(indexMetric)); + } + } + + public void testFromRequestWithNodeStatsMetricsFilter() { + Set metricsRequested = Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.FS); + String metricParam = metricsRequested.stream().map(ClusterStatsRequest.Metric::metricName).collect(Collectors.joining(",")); + final HashMap params = new HashMap<>(); + params.put("metric", metricParam); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); + assertNotNull(clusterStatsRequest); + assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); + assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); + assertEquals(2, clusterStatsRequest.requestedMetrics().size()); + for (ClusterStatsRequest.Metric metric : metricsRequested) { + assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); + } + assertTrue(clusterStatsRequest.indicesMetrics().isEmpty()); + } + + public void testFromRequestWithIndicesStatsMetricsFilter() { + Set metricsRequested = Set.of( + ClusterStatsRequest.Metric.OS, + ClusterStatsRequest.Metric.FS, + ClusterStatsRequest.Metric.INDICES + ); + Set indicesMetricsRequested = Set.of( + ClusterStatsRequest.IndexMetric.SHARDS, + ClusterStatsRequest.IndexMetric.SEGMENTS + ); + String metricParam = metricsRequested.stream().map(ClusterStatsRequest.Metric::metricName).collect(Collectors.joining(",")); + String indicesMetricParam = indicesMetricsRequested.stream() + .map(ClusterStatsRequest.IndexMetric::metricName) + .collect(Collectors.joining(",")); + final HashMap params = new HashMap<>(); + params.put("metric", metricParam); + params.put("index_metric", indicesMetricParam); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); + assertNotNull(clusterStatsRequest); + assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); + assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); + assertEquals(3, clusterStatsRequest.requestedMetrics().size()); + for (ClusterStatsRequest.Metric metric : metricsRequested) { + assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); + } + assertFalse(clusterStatsRequest.indicesMetrics().isEmpty()); + assertEquals(2, clusterStatsRequest.indicesMetrics().size()); + for (ClusterStatsRequest.IndexMetric indexMetric : indicesMetricsRequested) { + assertTrue(clusterStatsRequest.indicesMetrics().contains(indexMetric)); + } + } + public void testUnrecognizedMetric() { final HashMap params = new HashMap<>(); final String metric = randomAlphaOfLength(64); From 05b6b8cb26c9af06a5d65e5b1e0b7a046addd92c Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Fri, 4 Oct 2024 16:49:43 +0530 Subject: [PATCH 05/12] Test case changes and refactoring. Signed-off-by: Swetha Guptha --- .../admin/cluster/stats/ClusterStatsIT.java | 273 +++++++++++------- .../cluster/stats/ClusterStatsRequest.java | 10 +- .../stats/TransportClusterStatsAction.java | 2 +- .../admin/cluster/RestClusterStatsAction.java | 144 ++++----- .../stats/ClusterStatsRequestTests.java | 89 ++++-- .../stats/ClusterStatsResponseTests.java | 61 +++- .../cluster/RestClusterStatsActionTests.java | 18 +- 7 files changed, 385 insertions(+), 212 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index aa4f799db4083..9116fe8f2b300 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -321,7 +321,8 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent()); } - public void testValuesSmokeScreenWithMetricFilter() throws IOException, ExecutionException, InterruptedException { + public void testValuesSmokeScreenWithNodeStatsAndIndicesStatsMetricsFilter() throws IOException, ExecutionException, + InterruptedException { internalCluster().startNodes(randomIntBetween(1, 3)); index("test1", "type", "1", "f", "f"); @@ -401,6 +402,40 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception { } public void testFieldTypes() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + client().admin() + .indices() + .prepareCreate("test2") + .setMapping( + "{\"properties\":{\"foo\":{\"type\": \"keyword\"},\"bar\":{\"properties\":{\"baz\":{\"type\":\"keyword\"}," + + "\"eggplant\":{\"type\":\"integer\"}}}}}" + ) + .get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(randomBoolean()).get(); + assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3)); + Set stats = response.getIndicesStats().getMappings().getFieldTypeStats(); + for (IndexFeatureStats stat : stats) { + if (stat.getName().equals("integer")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } else if (stat.getName().equals("keyword")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(3)); + } else if (stat.getName().equals("object")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } + } + } + + public void testFieldTypesWithMappingsFilter() { internalCluster().startNode(); ensureGreen(); ClusterStatsResponse response = client().admin() @@ -558,7 +593,7 @@ public void testNodeRolesWithDataNodeLegacySettings() throws ExecutionException, assertEquals(expectedNodesRoles, Set.of(getNodeRoles(client, 0), getNodeRoles(client, 1))); } - public void testClusterStatsApplyMetricFilterDisabled() throws IOException { + public void testClusterStatsWithNodeMetricsFilter() { internalCluster().startNode(); ensureGreen(); @@ -571,54 +606,34 @@ public void testClusterStatsApplyMetricFilterDisabled() throws IOException { assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); assertNotNull(response.getNodesStats()); assertNotNull(response.getIndicesStats()); - ClusterStatsResponse statsResponseWithMetricFilterDisabled = client().admin() + ClusterStatsResponse statsResponseWithAllNodeStatsMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .computeAllMetrics(true) - .requestMetrics(Set.of(Metric.FS, Metric.JVM, Metric.PLUGINS, Metric.OS)) + .requestMetrics(ClusterStatsNodes.NODE_STATS_METRICS) + .computeAllMetrics(false) .get(); - - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats()); + assertNotNull(statsResponseWithAllNodeStatsMetrics); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats()); + assertNull(statsResponseWithAllNodeStatsMetrics.getIndicesStats()); + validateNodeStatsOutput(ClusterStatsNodes.NODE_STATS_METRICS, statsResponseWithAllNodeStatsMetrics); assertEquals( response.getNodesStats().getCounts().getTotal(), - statsResponseWithMetricFilterDisabled.getNodesStats().getCounts().getTotal() + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getTotal() ); assertEquals( response.getNodesStats().getCounts().getRoles(), - statsResponseWithMetricFilterDisabled.getNodesStats().getCounts().getRoles() - ); - assertEquals(response.getNodesStats().getVersions(), statsResponseWithMetricFilterDisabled.getNodesStats().getVersions()); - assertEquals(response.getNodesStats().getPlugins(), statsResponseWithMetricFilterDisabled.getNodesStats().getPlugins()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getOs()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getJvm()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getProcess()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getFs()); - - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats()); - assertEquals(response.getIndicesStats().getFieldData(), statsResponseWithMetricFilterDisabled.getIndicesStats().getFieldData()); - assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithMetricFilterDisabled.getIndicesStats().getAnalysis()); - assertEquals(response.getIndicesStats().getMappings(), statsResponseWithMetricFilterDisabled.getIndicesStats().getMappings()); - assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithMetricFilterDisabled.getIndicesStats().getIndexCount()); - assertEquals( - response.getIndicesStats().getShards().getTotal(), - statsResponseWithMetricFilterDisabled.getIndicesStats().getShards().getTotal() - ); - assertEquals( - response.getIndicesStats().getShards().getPrimaries(), - statsResponseWithMetricFilterDisabled.getIndicesStats().getShards().getPrimaries() + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getRoles() ); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getSegments()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getDocs()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getQueryCache()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getCompletion()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getStore()); + assertEquals(response.getNodesStats().getVersions(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getVersions()); + assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); } - public void testClusterStatsWithMetricsFilter() { + public void testClusterStatsWithIndicesOnlyMetricsFilter() { internalCluster().startNode(); ensureGreen(); @@ -635,55 +650,26 @@ public void testClusterStatsWithMetricsFilter() { assertNotNull(response.getNodesStats()); assertNotNull(response.getIndicesStats()); - ClusterStatsResponse statsResponseWithAllNodeStatsMetrics = client().admin() - .cluster() - .prepareClusterStats() - .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(ClusterStatsNodes.NODE_STATS_METRICS) - .computeAllMetrics(false) - .get(); - assertNotNull(statsResponseWithAllNodeStatsMetrics); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats()); - assertNull(statsResponseWithAllNodeStatsMetrics.getIndicesStats()); - assertEquals( - response.getNodesStats().getCounts().getTotal(), - statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getTotal() - ); - assertEquals( - response.getNodesStats().getCounts().getRoles(), - statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getRoles() - ); - assertEquals(response.getNodesStats().getVersions(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getVersions()); - assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getOs()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getJvm()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getProcess()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getFs()); - ClusterStatsResponse statsResponseWithIndicesRequestMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.PLUGINS, Metric.INDICES)) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.values())) .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithIndicesRequestMetrics); - assertNotNull(statsResponseWithIndicesRequestMetrics.getNodesStats()); - assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getOs()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getFs()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getProcess()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getJvm()); + assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats()); assertNotNull(statsResponseWithIndicesRequestMetrics.getIndicesStats()); - assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithIndicesRequestMetrics.getIndicesStats().getIndexCount()); + validateIndicesStatsOutput(Set.of(IndexMetric.values()), statsResponseWithIndicesRequestMetrics); } - public void testClusterStatsWithIndexMetricFilter() { + public void testClusterStatsWithSelectiveNodeMetricAndIndexMetricsFilter() { internalCluster().startNode(); ensureGreen(); client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); - IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source("{\"test_type\" : \"metrics_filter\"}"); + IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source(Map.of("test_type", "metrics_filter")); client().index(indexRequest); ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() @@ -697,35 +683,23 @@ public void testClusterStatsWithIndexMetricFilter() { assertNotNull(response.getNodesStats()); assertNotNull(response.getIndicesStats()); - ClusterStatsResponse statsResponseWithSpecificIndicesMetrics = client().admin() - .cluster() - .prepareClusterStats() - .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.INDICES)) - .indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)) - .computeAllMetrics(false) - .get(); - assertNotNull(statsResponseWithSpecificIndicesMetrics); - assertNull(statsResponseWithSpecificIndicesMetrics.getNodesStats()); - assertNotNull(statsResponseWithSpecificIndicesMetrics.getIndicesStats()); - assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getIndexCount()); - assertEquals(response.getIndicesStats().getMappings(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getMappings()); - assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getAnalysis()); - ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.INDICES)) - .indexMetrics(Set.of(IndexMetric.values())) + .requestMetrics(Set.of(Metric.OS, Metric.FS, Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE)) .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithAllIndicesMetrics); - assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getNodesStats()); assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + validateNodeStatsOutput(Set.of(Metric.FS, Metric.OS), statsResponseWithAllIndicesMetrics); + validateIndicesStatsOutput( + Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE), + statsResponseWithAllIndicesMetrics + ); assertEquals(response.getIndicesStats().getFieldData(), statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData()); - assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithAllIndicesMetrics.getIndicesStats().getAnalysis()); - assertEquals(response.getIndicesStats().getMappings(), statsResponseWithAllIndicesMetrics.getIndicesStats().getMappings()); assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithAllIndicesMetrics.getIndicesStats().getIndexCount()); assertEquals( response.getIndicesStats().getShards().getTotal(), @@ -735,11 +709,42 @@ public void testClusterStatsWithIndexMetricFilter() { response.getIndicesStats().getShards().getPrimaries(), statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getPrimaries() ); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getQueryCache()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getCompletion()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getStore()); + } + + public void testClusterStatsWithMappingsAndAnalysisStatsIndexMetricsFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source(Map.of("test_type", "metrics_filter")); + client().index(indexRequest); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats()); + + ClusterStatsResponse statsResponseWithSpecificIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithSpecificIndicesMetrics); + assertNull(statsResponseWithSpecificIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithSpecificIndicesMetrics.getIndicesStats()); + validateIndicesStatsOutput(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS), statsResponseWithSpecificIndicesMetrics); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getIndexCount()); + assertEquals(response.getIndicesStats().getMappings(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getMappings()); + assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getAnalysis()); } public void testClusterStatsWithIndexMetricWithDocsFilter() throws IOException { @@ -756,18 +761,90 @@ public void testClusterStatsWithIndexMetricWithDocsFilter() throws IOException { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.INDICES)) .indexMetrics(Set.of(IndexMetric.DOCS)) - .computeAllMetrics(true) + .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithAllIndicesMetrics); assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); - assertNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getShards()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs()); + validateIndicesStatsOutput(Set.of(IndexMetric.DOCS), statsResponseWithAllIndicesMetrics); assertEquals(2, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getCount()); assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getDeleted()); assertTrue(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getAverageSizeInBytes() > 0); } + private void validateNodeStatsOutput(Set expectedMetrics, ClusterStatsResponse clusterStatsResponse) { + // Ingest, network types, discovery types and packaging types stats are not included here as they don't have a get method exposed. + Set NodeMetrics = Set.of(Metric.OS, Metric.JVM, Metric.FS, Metric.PROCESS, Metric.PLUGINS); + for (Metric metric : NodeMetrics) { + Object object = null; + switch (metric) { + case OS: + object = clusterStatsResponse.getNodesStats().getOs(); + break; + case JVM: + object = clusterStatsResponse.getNodesStats().getJvm(); + break; + case FS: + object = clusterStatsResponse.getNodesStats().getFs(); + break; + case PROCESS: + object = clusterStatsResponse.getNodesStats().getProcess(); + break; + case PLUGINS: + object = clusterStatsResponse.getNodesStats().getPlugins(); + break; + } + if (expectedMetrics.contains(metric)) { + assertNotNull(object); + } else { + assertNull(object); + } + } + } + + private void validateIndicesStatsOutput( + Set expectedMetrics, + ClusterStatsResponse clusterStatsResponse + ) { + for (IndexMetric indexMetric : IndexMetric.values()) { + Object object = null; + switch (indexMetric) { + case SHARDS: + object = clusterStatsResponse.getIndicesStats().getShards(); + break; + case DOCS: + object = clusterStatsResponse.getIndicesStats().getDocs(); + break; + case STORE: + object = clusterStatsResponse.getIndicesStats().getStore(); + break; + case FIELDDATA: + object = clusterStatsResponse.getIndicesStats().getFieldData(); + break; + case QUERY_CACHE: + object = clusterStatsResponse.getIndicesStats().getQueryCache(); + break; + case COMPLETION: + object = clusterStatsResponse.getIndicesStats().getCompletion(); + break; + case SEGMENTS: + object = clusterStatsResponse.getIndicesStats().getSegments(); + break; + case ANALYSIS: + object = clusterStatsResponse.getIndicesStats().getAnalysis(); + break; + case MAPPINGS: + object = clusterStatsResponse.getIndicesStats().getMappings(); + break; + } + if (expectedMetrics.contains(indexMetric)) { + assertNotNull(object); + } else { + assertNull(object); + } + } + } + private Map getExpectedCounts( int dataRoleCount, int masterRoleCount, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 4b0a32517f48e..1bfacd7cc6451 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -52,7 +52,7 @@ public class ClusterStatsRequest extends BaseNodesRequest { private final Set requestedMetrics = new HashSet<>(); private final Set indexMetricsRequested = new HashSet<>(); - private Boolean computeAllMetric = true; + private Boolean computeAllMetrics = true; public ClusterStatsRequest(StreamInput in) throws IOException { super(in); @@ -60,7 +60,7 @@ public ClusterStatsRequest(StreamInput in) throws IOException { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } if (in.getVersion().onOrAfter(Version.V_3_0_0)) { - computeAllMetric = in.readOptionalBoolean(); + computeAllMetrics = in.readOptionalBoolean(); final long longMetricsFlags = in.readLong(); for (Metric metric : Metric.values()) { if ((longMetricsFlags & (1 << metric.getIndex())) != 0) { @@ -95,11 +95,11 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon } public boolean computeAllMetrics() { - return computeAllMetric; + return computeAllMetrics; } public void computeAllMetrics(boolean computeAllMetrics) { - this.computeAllMetric = computeAllMetrics; + this.computeAllMetrics = computeAllMetrics; } /** @@ -136,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } if (out.getVersion().onOrAfter(Version.V_3_0_0)) { - out.writeOptionalBoolean(computeAllMetric); + out.writeOptionalBoolean(computeAllMetrics); long longMetricFlags = 0; for (Metric metric : requestedMetrics) { longMetricFlags |= (1 << metric.getIndex()); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index b7a645240f407..342a949480be8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -266,7 +266,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq } /** - * A metric is required when: all cluster stats are required (OR) if the metric was requested + * A metric is required when: all cluster stats are required (OR) if the metric is requested * @param metric * @param clusterStatsRequest * @return diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index daa4a75285823..f10ac9e4a1f9b 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -33,6 +33,8 @@ package org.opensearch.rest.action.admin.cluster; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; @@ -79,7 +81,7 @@ public List routes() { static { Map> metricRequestConsumerMap = new HashMap<>(); - for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { + for (Metric metric : Metric.values()) { metricRequestConsumerMap.put(metric.metricName(), request -> request.addMetric(metric)); } METRIC_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricRequestConsumerMap); @@ -87,7 +89,7 @@ public List routes() { static { Map> metricMap = new HashMap<>(); - for (ClusterStatsRequest.IndexMetric indexMetric : ClusterStatsRequest.IndexMetric.values()) { + for (IndexMetric indexMetric : IndexMetric.values()) { metricMap.put(indexMetric.metricName(), request -> request.addIndexMetric(indexMetric)); } INDEX_METRIC_TO_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricMap); @@ -106,6 +108,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC public static ClusterStatsRequest fromRequest(final RestRequest request) { Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + String indicesMetricsDefaultValue = metrics.contains(Metric.INDICES.metricName()) || metrics.contains("_all") ? "_all" : null; + Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", indicesMetricsDefaultValue)); String[] nodeIds = request.paramAsStringArray("nodeId", null); ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); @@ -113,82 +117,25 @@ public static ClusterStatsRequest fromRequest(final RestRequest request) { clusterStatsRequest.useAggregatedNodeLevelResponses(true); if (!metrics.isEmpty()) { - if (metrics.size() > 1 && metrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ); - } - - final Set metricsRequested = new HashSet<>(); - if (metrics.contains("_all")) { - metricsRequested.addAll(METRIC_REQUEST_CONSUMER_MAP.keySet()); - } else { - metricsRequested.addAll(metrics); - } - - Set indexMetrics = Strings.tokenizeByCommaToSet( - request.param("index_metric", metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName()) ? "_all" : null) - ); - - if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual index metrics [%s]", - request.path(), - request.param("index_metric") - ) - ); - } - - final Set invalidMetrics = new TreeSet<>(); - for (String metric : metricsRequested) { - Consumer clusterStatsRequestConsumer = METRIC_REQUEST_CONSUMER_MAP.get(metric); - if (clusterStatsRequestConsumer != null) { - clusterStatsRequestConsumer.accept(clusterStatsRequest); - } else { - invalidMetrics.add(metric); - } - } + paramValidations(metrics, indexMetrics, request); + final Set metricsRequested = metrics.contains("_all") + ? new HashSet<>(METRIC_REQUEST_CONSUMER_MAP.keySet()) + : new HashSet<>(metrics); + Set invalidMetrics = validateAndSetRequestedMetrics(metricsRequested, METRIC_REQUEST_CONSUMER_MAP, clusterStatsRequest); if (!invalidMetrics.isEmpty()) { throw new IllegalArgumentException( unrecognizedStrings(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric") ); } - - if (!metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName()) && !indexMetrics.isEmpty()) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains index metrics [%s] but indices stats not requested", - request.path(), - request.param("index_metric") - ) + if (metricsRequested.contains(Metric.INDICES.metricName())) { + final Set indexMetricsRequested = indexMetrics.contains("_all") + ? INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet() + : new HashSet<>(indexMetrics); + Set invalidIndexMetrics = validateAndSetRequestedMetrics( + indexMetricsRequested, + INDEX_METRIC_TO_REQUEST_CONSUMER_MAP, + clusterStatsRequest ); - } - - if (metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName())) { - final Set indexMetricsRequested = new HashSet<>(); - if (indexMetrics.contains("_all")) { - indexMetricsRequested.addAll(INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()); - } else { - indexMetricsRequested.addAll(indexMetrics); - } - final Set invalidIndexMetrics = new TreeSet<>(); - for (String indexMetric : indexMetricsRequested) { - Consumer clusterStatsRequestConsumer = INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.get(indexMetric); - if (clusterStatsRequestConsumer != null) { - clusterStatsRequestConsumer.accept(clusterStatsRequest); - } else { - invalidIndexMetrics.add(indexMetric); - } - } - if (!invalidIndexMetrics.isEmpty()) { throw new IllegalArgumentException( unrecognizedStrings(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") @@ -197,9 +144,62 @@ public static ClusterStatsRequest fromRequest(final RestRequest request) { } clusterStatsRequest.computeAllMetrics(false); } + return clusterStatsRequest; } + private static void paramValidations(Set metrics, Set indexMetrics, RestRequest request) { + if (metrics.size() > 1 && metrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ); + } + + if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual index metrics [%s]", + request.path(), + request.param("index_metric") + ) + ); + } + + if (!metrics.contains(Metric.INDICES.metricName()) && !metrics.contains("_all") && !indexMetrics.isEmpty()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains index metrics [%s] but indices stats not requested", + request.path(), + request.param("index_metric") + ) + ); + } + } + + private static Set validateAndSetRequestedMetrics( + Set metrics, + Map> metricConsumerMap, + ClusterStatsRequest clusterStatsRequest + ) { + final Set invalidMetrics = new TreeSet<>(); + for (String metric : metrics) { + Consumer clusterStatsRequestConsumer = metricConsumerMap.get(metric); + if (clusterStatsRequestConsumer != null) { + clusterStatsRequestConsumer.accept(clusterStatsRequest); + } else { + invalidMetrics.add(metric); + } + } + return invalidMetrics; + } + @Override public boolean canTripCircuitBreaker() { return false; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java index cd452693779f2..7d2c109bad83a 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java @@ -12,22 +12,41 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.test.VersionUtils; + +import java.util.Set; public class ClusterStatsRequestTests extends OpenSearchTestCase { - public void testSerialization() throws Exception { - ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(); - clusterStatsRequest.computeAllMetrics(randomBoolean()); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.OS); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.PLUGINS); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.INDICES); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.SHARDS); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.QUERY_CACHE); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.MAPPINGS); - clusterStatsRequest.useAggregatedNodeLevelResponses(randomBoolean()); + public void testSerializationWithVersion3x() throws Exception { + ClusterStatsRequest clusterStatsRequest = getClusterStatsRequest(); + + Version testVersion = Version.V_3_0_0; + + BytesStreamOutput output = new BytesStreamOutput(); + output.setVersion(testVersion); + clusterStatsRequest.writeTo(output); + + StreamInput streamInput = output.bytes().streamInput(); + streamInput.setVersion(testVersion); + ClusterStatsRequest deserializedClusterStatsRequest = new ClusterStatsRequest(streamInput); + + validateClusterStatsRequest( + Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.PLUGINS, ClusterStatsRequest.Metric.INDICES), + Set.of( + ClusterStatsRequest.IndexMetric.SHARDS, + ClusterStatsRequest.IndexMetric.QUERY_CACHE, + ClusterStatsRequest.IndexMetric.MAPPINGS + ), + Version.V_3_0_0, + deserializedClusterStatsRequest + ); + assertEquals(-1, streamInput.read()); + } - Version testVersion = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT); + public void testSerializationOnVersionBelow3x() throws Exception { + ClusterStatsRequest clusterStatsRequest = getClusterStatsRequest(); + + Version testVersion = Version.V_2_17_0; BytesStreamOutput output = new BytesStreamOutput(); output.setVersion(testVersion); @@ -37,13 +56,47 @@ public void testSerialization() throws Exception { streamInput.setVersion(testVersion); ClusterStatsRequest deserializedClusterStatsRequest = new ClusterStatsRequest(streamInput); - assertEquals(clusterStatsRequest.computeAllMetrics(), deserializedClusterStatsRequest.computeAllMetrics()); - assertEquals(clusterStatsRequest.requestedMetrics(), deserializedClusterStatsRequest.requestedMetrics()); - assertEquals(clusterStatsRequest.indicesMetrics(), deserializedClusterStatsRequest.indicesMetrics()); - assertEquals( - clusterStatsRequest.useAggregatedNodeLevelResponses(), - deserializedClusterStatsRequest.useAggregatedNodeLevelResponses() + validateClusterStatsRequest( + Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.PLUGINS, ClusterStatsRequest.Metric.INDICES), + Set.of( + ClusterStatsRequest.IndexMetric.SHARDS, + ClusterStatsRequest.IndexMetric.QUERY_CACHE, + ClusterStatsRequest.IndexMetric.MAPPINGS + ), + Version.V_2_17_0, + deserializedClusterStatsRequest ); + assertEquals(-1, streamInput.read()); + } + + private ClusterStatsRequest getClusterStatsRequest() { + ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(); + clusterStatsRequest.computeAllMetrics(true); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.OS); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.PLUGINS); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.INDICES); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.SHARDS); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.QUERY_CACHE); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.MAPPINGS); + clusterStatsRequest.useAggregatedNodeLevelResponses(true); + return clusterStatsRequest; + } + + private void validateClusterStatsRequest( + Set metrics, + Set indexMetrics, + Version version, + ClusterStatsRequest deserializedClusterStatsRequest + ) { + if (version.before(Version.V_3_0_0)) { + assertEquals(true, deserializedClusterStatsRequest.computeAllMetrics()); + assertTrue(deserializedClusterStatsRequest.requestedMetrics().isEmpty()); + assertTrue(deserializedClusterStatsRequest.indicesMetrics().isEmpty()); + } else { + assertEquals(true, deserializedClusterStatsRequest.computeAllMetrics()); + assertEquals(metrics, deserializedClusterStatsRequest.requestedMetrics()); + assertEquals(indexMetrics, deserializedClusterStatsRequest.indicesMetrics()); + } } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java index 850460d2f86d7..abd607d5787dc 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java @@ -13,6 +13,7 @@ import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.ShardStats; @@ -37,7 +38,9 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.StoreStats; import org.opensearch.monitor.jvm.JvmInfo; +import org.opensearch.monitor.jvm.JvmStats; import org.opensearch.monitor.os.OsInfo; +import org.opensearch.monitor.process.ProcessStats; import org.opensearch.search.suggest.completion.CompletionStats; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportInfo; @@ -72,7 +75,7 @@ public void testSerializationWithIndicesMappingAndAnalysisStats() throws Excepti List.of(), ClusterState.EMPTY_STATE, Set.of(ClusterStatsRequest.Metric.INDICES), - Set.of(ClusterStatsRequest.IndexMetric.MAPPINGS, ClusterStatsRequest.IndexMetric.ANALYSIS) + Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS) ); BytesStreamOutput output = new BytesStreamOutput(); clusterStatsResponse.writeTo(output); @@ -107,8 +110,15 @@ public void testSerializationWithoutIndicesMappingAndAnalysisStats() throws Exce defaultClusterStatsNodeResponses, List.of(), ClusterState.EMPTY_STATE, - Set.of(), - Set.of() + Set.of(ClusterStatsRequest.Metric.INDICES, ClusterStatsRequest.Metric.PROCESS, ClusterStatsRequest.Metric.JVM), + Set.of( + IndexMetric.DOCS, + IndexMetric.STORE, + IndexMetric.SEGMENTS, + IndexMetric.QUERY_CACHE, + IndexMetric.FIELDDATA, + IndexMetric.COMPLETION + ) ); BytesStreamOutput output = new BytesStreamOutput(); clusterStatsResponse.writeTo(output); @@ -118,10 +128,28 @@ public void testSerializationWithoutIndicesMappingAndAnalysisStats() throws Exce assertEquals(clusterStatsResponse.timestamp, deserializedClusterStatsResponse.timestamp); assertEquals(clusterStatsResponse.status, deserializedClusterStatsResponse.status); assertEquals(clusterStatsResponse.clusterUUID, deserializedClusterStatsResponse.clusterUUID); - assertNull(clusterStatsResponse.indicesStats); + assertNotNull(deserializedClusterStatsResponse.nodesStats); + assertNotNull(deserializedClusterStatsResponse.nodesStats.getProcess()); + assertNotNull(deserializedClusterStatsResponse.nodesStats.getJvm()); + assertNotNull(deserializedClusterStatsResponse.indicesStats); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getDocs()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getStore()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getSegments()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getQueryCache()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getFieldData()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getCompletion()); + assertNull(deserializedClusterStatsResponse.indicesStats.getMappings()); + assertNull(deserializedClusterStatsResponse.indicesStats.getAnalysis()); } private ClusterStatsNodeResponse createClusterStatsNodeResponse(DiscoveryNode node, ShardStats[] shardStats) throws IOException { + JvmStats.GarbageCollector[] garbageCollectorsArray = new JvmStats.GarbageCollector[1]; + garbageCollectorsArray[0] = new JvmStats.GarbageCollector( + randomAlphaOfLengthBetween(3, 10), + randomNonNegativeLong(), + randomNonNegativeLong() + ); + JvmStats.GarbageCollectors garbageCollectors = new JvmStats.GarbageCollectors(garbageCollectorsArray); NodeInfo nodeInfo = new NodeInfo( Version.CURRENT, Build.CURRENT, @@ -148,8 +176,29 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse(DiscoveryNode no randomNonNegativeLong(), null, null, - null, - null, + new ProcessStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + new ProcessStats.Cpu(randomShort(), randomNonNegativeLong()), + new ProcessStats.Mem(randomNonNegativeLong()) + ), + new JvmStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + new JvmStats.Mem( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + Collections.emptyList() + ), + new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), + garbageCollectors, + Collections.emptyList(), + new JvmStats.Classes(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) + ), null, null, null, diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java index e3eb525696272..8c8284fa36211 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java @@ -38,7 +38,7 @@ public void testFromRequestBasePath() { ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); assertNotNull(clusterStatsRequest); assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); - assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.computeAllMetrics()); assertNotNull(clusterStatsRequest.requestedMetrics()); assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { @@ -60,12 +60,10 @@ public void testFromRequestWithNodeStatsMetricsFilter() { ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); assertNotNull(clusterStatsRequest); assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); - assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.computeAllMetrics()); assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); assertEquals(2, clusterStatsRequest.requestedMetrics().size()); - for (ClusterStatsRequest.Metric metric : metricsRequested) { - assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); - } + assertEquals(metricsRequested, clusterStatsRequest.requestedMetrics()); assertTrue(clusterStatsRequest.indicesMetrics().isEmpty()); } @@ -90,17 +88,13 @@ public void testFromRequestWithIndicesStatsMetricsFilter() { ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); assertNotNull(clusterStatsRequest); assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); - assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.computeAllMetrics()); assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); assertEquals(3, clusterStatsRequest.requestedMetrics().size()); - for (ClusterStatsRequest.Metric metric : metricsRequested) { - assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); - } + assertEquals(metricsRequested, clusterStatsRequest.requestedMetrics()); assertFalse(clusterStatsRequest.indicesMetrics().isEmpty()); assertEquals(2, clusterStatsRequest.indicesMetrics().size()); - for (ClusterStatsRequest.IndexMetric indexMetric : indicesMetricsRequested) { - assertTrue(clusterStatsRequest.indicesMetrics().contains(indexMetric)); - } + assertEquals(indicesMetricsRequested, clusterStatsRequest.indicesMetrics()); } public void testUnrecognizedMetric() { From ba22cae0536142f2b4bc674065d350bb1ff7d1b3 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Fri, 4 Oct 2024 19:52:03 +0530 Subject: [PATCH 06/12] Add comments. Signed-off-by: Swetha Guptha --- .../cluster/stats/ClusterStatsRequest.java | 2 + .../admin/cluster/RestClusterStatsAction.java | 49 ++++++----- .../ClusterStatsRequestBuilderTests.java | 81 ------------------- .../stats/ClusterStatsResponseTests.java | 1 + 4 files changed, 27 insertions(+), 106 deletions(-) delete mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 1bfacd7cc6451..1c929881b898b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -189,6 +189,8 @@ public int getIndex() { /** * An enumeration of the "core" sections of indices metrics that may be requested * from the cluster stats endpoint. + * + * When no value is provided for param index_metric, default filter is set to _all. */ @PublicApi(since = "3.0.0") public enum IndexMetric { diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index f10ac9e4a1f9b..47f3e048c516a 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -108,6 +108,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC public static ClusterStatsRequest fromRequest(final RestRequest request) { Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + // Value for param index_metric defaults to _all when indices metric or all metrics are requested. String indicesMetricsDefaultValue = metrics.contains(Metric.INDICES.metricName()) || metrics.contains("_all") ? "_all" : null; Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", indicesMetricsDefaultValue)); String[] nodeIds = request.paramAsStringArray("nodeId", null); @@ -115,34 +116,32 @@ public static ClusterStatsRequest fromRequest(final RestRequest request) { ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); clusterStatsRequest.timeout(request.param("timeout")); clusterStatsRequest.useAggregatedNodeLevelResponses(true); - - if (!metrics.isEmpty()) { - paramValidations(metrics, indexMetrics, request); - final Set metricsRequested = metrics.contains("_all") - ? new HashSet<>(METRIC_REQUEST_CONSUMER_MAP.keySet()) - : new HashSet<>(metrics); - Set invalidMetrics = validateAndSetRequestedMetrics(metricsRequested, METRIC_REQUEST_CONSUMER_MAP, clusterStatsRequest); - if (!invalidMetrics.isEmpty()) { + clusterStatsRequest.computeAllMetrics(false); + + paramValidations(metrics, indexMetrics, request); + final Set metricsRequested = metrics.contains("_all") + ? new HashSet<>(METRIC_REQUEST_CONSUMER_MAP.keySet()) + : new HashSet<>(metrics); + Set invalidMetrics = validateAndSetRequestedMetrics(metricsRequested, METRIC_REQUEST_CONSUMER_MAP, clusterStatsRequest); + if (!invalidMetrics.isEmpty()) { + throw new IllegalArgumentException( + unrecognizedStrings(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric") + ); + } + if (metricsRequested.contains(Metric.INDICES.metricName())) { + final Set indexMetricsRequested = indexMetrics.contains("_all") + ? INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet() + : new HashSet<>(indexMetrics); + Set invalidIndexMetrics = validateAndSetRequestedMetrics( + indexMetricsRequested, + INDEX_METRIC_TO_REQUEST_CONSUMER_MAP, + clusterStatsRequest + ); + if (!invalidIndexMetrics.isEmpty()) { throw new IllegalArgumentException( - unrecognizedStrings(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric") - ); - } - if (metricsRequested.contains(Metric.INDICES.metricName())) { - final Set indexMetricsRequested = indexMetrics.contains("_all") - ? INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet() - : new HashSet<>(indexMetrics); - Set invalidIndexMetrics = validateAndSetRequestedMetrics( - indexMetricsRequested, - INDEX_METRIC_TO_REQUEST_CONSUMER_MAP, - clusterStatsRequest + unrecognizedStrings(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") ); - if (!invalidIndexMetrics.isEmpty()) { - throw new IllegalArgumentException( - unrecognizedStrings(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") - ); - } } - clusterStatsRequest.computeAllMetrics(false); } return clusterStatsRequest; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java deleted file mode 100644 index 8bc14d4617171..0000000000000 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.action.admin.cluster.stats; - -import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; -import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; -import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.test.client.NoOpClient; -import org.junit.After; -import org.junit.Before; - -import java.util.Set; - -public class ClusterStatsRequestBuilderTests extends OpenSearchTestCase { - - private NoOpClient testClient; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - this.testClient = new NoOpClient(getTestName()); - } - - @Override - @After - public void tearDown() throws Exception { - this.testClient.close(); - super.tearDown(); - } - - public void testUseAggregatedNodeLevelResponses() { - ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( - this.testClient, - ClusterStatsAction.INSTANCE - ); - clusterStatsRequestBuilder.useAggregatedNodeLevelResponses(false); - assertFalse(clusterStatsRequestBuilder.request().useAggregatedNodeLevelResponses()); - } - - public void testApplyMetricFiltering() { - ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( - this.testClient, - ClusterStatsAction.INSTANCE - ); - assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); - clusterStatsRequestBuilder.computeAllMetrics(false); - assertFalse(clusterStatsRequestBuilder.request().computeAllMetrics()); - } - - public void testRequestedMetrics() { - ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( - this.testClient, - ClusterStatsAction.INSTANCE - ); - clusterStatsRequestBuilder.computeAllMetrics(false); - clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.OS, Metric.JVM)); - assertFalse(clusterStatsRequestBuilder.request().computeAllMetrics()); - assertEquals(Set.of(Metric.OS, Metric.JVM), clusterStatsRequestBuilder.request().requestedMetrics()); - } - - public void testIndicesMetrics() { - ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( - this.testClient, - ClusterStatsAction.INSTANCE - ); - clusterStatsRequestBuilder.computeAllMetrics(false); - clusterStatsRequestBuilder.requestMetrics(Set.of(Metric.INDICES, Metric.JVM)); - clusterStatsRequestBuilder.indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)); - assertFalse(clusterStatsRequestBuilder.request().computeAllMetrics()); - assertEquals(Set.of(Metric.INDICES, Metric.JVM), clusterStatsRequestBuilder.request().requestedMetrics()); - assertEquals(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS), clusterStatsRequestBuilder.request().indicesMetrics()); - } - -} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java index abd607d5787dc..ad7706292d93c 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java @@ -221,6 +221,7 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse(DiscoveryNode no null, null, null, + null, null ); return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats); From 96f5e72b5bf910f9b8241a1bfda72ff846051245 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Sat, 5 Oct 2024 00:30:23 +0530 Subject: [PATCH 07/12] Initialise indices stats by default. Signed-off-by: Swetha Guptha --- .../cluster/stats/ClusterStatsIndices.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 3d3e022a142a0..55fd62940fb86 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -82,42 +82,44 @@ public ClusterStatsIndices( AnalysisStats analysisStats ) { Map countsPerIndex = new HashMap<>(); + this.docs = indicesMetrics.contains(IndexMetric.DOCS) ? new DocsStats() : null; Consumer docsStatsConsumer = (docs) -> { - if (indicesMetrics.contains(IndexMetric.DOCS)) { - if (this.docs == null) this.docs = new DocsStats(); + if (this.docs != null) { this.docs.add(docs); } }; + + this.store = indicesMetrics.contains(IndexMetric.STORE) ? new StoreStats() : null; Consumer storeStatsConsumer = (store) -> { - if (indicesMetrics.contains(IndexMetric.STORE)) { - if (this.store == null) this.store = new StoreStats(); + if (this.store != null) { this.store.add(store); } }; + + this.fieldData = indicesMetrics.contains(IndexMetric.FIELDDATA) ? new FieldDataStats() : null; Consumer fieldDataConsumer = (fieldDataStats) -> { - if (indicesMetrics.contains(IndexMetric.FIELDDATA)) { - if (this.fieldData == null) this.fieldData = new FieldDataStats(); + if (this.fieldData != null) { this.fieldData.add(fieldDataStats); } }; + this.queryCache = indicesMetrics.contains(IndexMetric.QUERY_CACHE) ? new QueryCacheStats() : null; Consumer queryCacheStatsConsumer = (queryCacheStats) -> { - if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) { - if (this.queryCache == null) this.queryCache = new QueryCacheStats(); + if (this.queryCache != null) { this.queryCache.add(queryCacheStats); } }; + this.completion = indicesMetrics.contains(IndexMetric.COMPLETION) ? new CompletionStats() : null; Consumer completionStatsConsumer = (completionStats) -> { - if (indicesMetrics.contains(IndexMetric.COMPLETION)) { - if (this.completion == null) this.completion = new CompletionStats(); + if (this.completion != null) { this.completion.add(completionStats); } }; + this.segments = indicesMetrics.contains(IndexMetric.SEGMENTS) ? new SegmentsStats() : null; Consumer segmentsStatsConsumer = (segmentsStats) -> { - if (indicesMetrics.contains(IndexMetric.SEGMENTS)) { - if (this.segments == null) this.segments = new SegmentsStats(); + if (this.segments != null) { this.segments.add(segmentsStats); } }; From 7341f61573e9c6a3c41067ed6c1667c44783bbc0 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Sat, 5 Oct 2024 12:07:16 +0530 Subject: [PATCH 08/12] Remove indices stats consumers. Signed-off-by: Swetha Guptha --- .../admin/cluster/stats/ClusterStatsIT.java | 36 ++++++++ .../cluster/stats/ClusterStatsIndices.java | 84 ++++++++----------- .../stats/TransportClusterStatsAction.java | 17 +--- .../cluster/RestClusterStatsActionTests.java | 18 ++++ 4 files changed, 91 insertions(+), 64 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index 9116fe8f2b300..5f00ba35c7b69 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -47,6 +47,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.monitor.os.OsStats; import org.opensearch.node.NodeRoleSettings; import org.opensearch.test.OpenSearchIntegTestCase; @@ -772,6 +773,41 @@ public void testClusterStatsWithIndexMetricWithDocsFilter() throws IOException { assertTrue(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getAverageSizeInBytes() > 0); } + public void testClusterStatsWithSelectiveMetricsFilterAndNoIndex() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.OS, Metric.FS, Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE)) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithAllIndicesMetrics); + assertNotNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + validateNodeStatsOutput(Set.of(Metric.FS, Metric.OS), statsResponseWithAllIndicesMetrics); + validateIndicesStatsOutput( + Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE), + statsResponseWithAllIndicesMetrics + ); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getIndices()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getTotal()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getPrimaries()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getCount()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getDeleted()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getTotalSizeInBytes()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getStore().getSizeInBytes()); + assertEquals(new ByteSizeValue(0), statsResponseWithAllIndicesMetrics.getIndicesStats().getStore().getReservedSize()); + assertEquals(new ByteSizeValue(0), statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getMemorySize()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getEvictions()); + assertNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getFields()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getCount()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getIndexWriterMemoryInBytes()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getVersionMapMemoryInBytes()); + } + private void validateNodeStatsOutput(Set expectedMetrics, ClusterStatsResponse clusterStatsResponse) { // Ingest, network types, discovery types and packaging types stats are not included here as they don't have a get method exposed. Set NodeMetrics = Set.of(Metric.OS, Metric.JVM, Metric.FS, Metric.PROCESS, Metric.PLUGINS); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 55fd62940fb86..9620c682e746f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -49,7 +49,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Consumer; /** * Cluster Stats per index @@ -83,46 +82,11 @@ public ClusterStatsIndices( ) { Map countsPerIndex = new HashMap<>(); this.docs = indicesMetrics.contains(IndexMetric.DOCS) ? new DocsStats() : null; - Consumer docsStatsConsumer = (docs) -> { - if (this.docs != null) { - this.docs.add(docs); - } - }; - this.store = indicesMetrics.contains(IndexMetric.STORE) ? new StoreStats() : null; - Consumer storeStatsConsumer = (store) -> { - if (this.store != null) { - this.store.add(store); - } - }; - this.fieldData = indicesMetrics.contains(IndexMetric.FIELDDATA) ? new FieldDataStats() : null; - Consumer fieldDataConsumer = (fieldDataStats) -> { - if (this.fieldData != null) { - this.fieldData.add(fieldDataStats); - } - }; - this.queryCache = indicesMetrics.contains(IndexMetric.QUERY_CACHE) ? new QueryCacheStats() : null; - Consumer queryCacheStatsConsumer = (queryCacheStats) -> { - if (this.queryCache != null) { - this.queryCache.add(queryCacheStats); - } - }; - this.completion = indicesMetrics.contains(IndexMetric.COMPLETION) ? new CompletionStats() : null; - Consumer completionStatsConsumer = (completionStats) -> { - if (this.completion != null) { - this.completion.add(completionStats); - } - }; - this.segments = indicesMetrics.contains(IndexMetric.SEGMENTS) ? new SegmentsStats() : null; - Consumer segmentsStatsConsumer = (segmentsStats) -> { - if (this.segments != null) { - this.segments.add(segmentsStats); - } - }; for (ClusterStatsNodeResponse r : nodeResponses) { // Aggregated response from the node @@ -139,12 +103,24 @@ public ClusterStatsIndices( } } - docsStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.docs); - storeStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.store); - fieldDataConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.fieldData); - queryCacheStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.queryCache); - completionStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.completion); - segmentsStatsConsumer.accept(r.getAggregatedNodeLevelStats().commonStats.segments); + if (docs != null) { + docs.add(r.getAggregatedNodeLevelStats().commonStats.docs); + } + if (store != null) { + store.add(r.getAggregatedNodeLevelStats().commonStats.store); + } + if (fieldData != null) { + fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData); + } + if (queryCache != null) { + queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache); + } + if (completion != null) { + completion.add(r.getAggregatedNodeLevelStats().commonStats.completion); + } + if (segments != null) { + segments.add(r.getAggregatedNodeLevelStats().commonStats.segments); + } } else { // Default response from the node for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { @@ -160,13 +136,25 @@ public ClusterStatsIndices( if (shardStats.getShardRouting().primary()) { indexShardStats.primaries++; - docsStatsConsumer.accept(shardCommonStats.docs); + if (docs != null) { + docs.add(shardCommonStats.docs); + } + } + if (store != null) { + store.add(shardCommonStats.store); + } + if (fieldData != null) { + fieldData.add(shardCommonStats.fieldData); + } + if (queryCache != null) { + queryCache.add(shardCommonStats.queryCache); + } + if (completion != null) { + completion.add(shardCommonStats.completion); + } + if (segments != null) { + segments.add(shardCommonStats.segments); } - storeStatsConsumer.accept(shardCommonStats.store); - fieldDataConsumer.accept(shardCommonStats.fieldData); - queryCacheStatsConsumer.accept(shardCommonStats.queryCache); - completionStatsConsumer.accept(shardCommonStats.completion); - segmentsStatsConsumer.accept(shardCommonStats.segments); } } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 342a949480be8..c6581b99eb559 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -80,21 +80,6 @@ public class TransportClusterStatsAction extends TransportNodesAction< TransportClusterStatsAction.ClusterStatsNodeRequest, ClusterStatsNodeResponse> { - private static final Map INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP = Map.of( - ClusterStatsRequest.IndexMetric.DOCS, - CommonStatsFlags.Flag.Docs, - ClusterStatsRequest.IndexMetric.STORE, - CommonStatsFlags.Flag.Store, - ClusterStatsRequest.IndexMetric.FIELDDATA, - CommonStatsFlags.Flag.FieldData, - ClusterStatsRequest.IndexMetric.QUERY_CACHE, - CommonStatsFlags.Flag.QueryCache, - ClusterStatsRequest.IndexMetric.COMPLETION, - CommonStatsFlags.Flag.Completion, - ClusterStatsRequest.IndexMetric.SEGMENTS, - CommonStatsFlags.Flag.Segments - ); - private static final Map SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC = Map.of( CommonStatsFlags.Flag.Docs, ClusterStatsRequest.IndexMetric.DOCS, @@ -278,7 +263,7 @@ private boolean isMetricRequired(Metric metric, ClusterStatsRequest clusterStats private static CommonStatsFlags getCommonStatsFlags(ClusterStatsNodeRequest nodeRequest) { Set requestedCommonStatsFlags = new HashSet<>(); if (nodeRequest.request.computeAllMetrics()) { - requestedCommonStatsFlags.addAll(INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.values()); + requestedCommonStatsFlags.addAll(SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC.keySet()); } else { for (Map.Entry entry : SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC .entrySet()) { diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java index 8c8284fa36211..8b46f676636fd 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java @@ -150,4 +150,22 @@ public void testAllIndexMetricsRequestWithOtherIndicesMetric() { ); } + public void testIndexMetricsRequestWithoutMetricIndices() { + final HashMap params = new HashMap<>(); + params.put("metric", "os"); + final String indexMetric = randomSubsetOf(1, RestClusterStatsAction.INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()).get(0); + params.put("index_metric", indexMetric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat( + e, + hasToString( + containsString("request [/_cluster/stats] contains index metrics [" + indexMetric + "] but indices stats not requested") + ) + ); + } + } From 8c0d4f5970144fab33f17d0feccc6e52c650b9a7 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Fri, 18 Oct 2024 16:36:22 +0530 Subject: [PATCH 09/12] Replacing null check in nodes/indices stats with requested metrics Signed-off-by: Swetha Guptha --- .../cluster/stats/ClusterStatsIndices.java | 24 +++++++++---------- .../cluster/stats/ClusterStatsNodes.java | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 9620c682e746f..9ebe36531c208 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -103,22 +103,22 @@ public ClusterStatsIndices( } } - if (docs != null) { + if (indicesMetrics.contains(IndexMetric.DOCS)) { docs.add(r.getAggregatedNodeLevelStats().commonStats.docs); } - if (store != null) { + if (indicesMetrics.contains(IndexMetric.STORE)) { store.add(r.getAggregatedNodeLevelStats().commonStats.store); } - if (fieldData != null) { + if (indicesMetrics.contains(IndexMetric.FIELDDATA)) { fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData); } - if (queryCache != null) { + if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) { queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache); } - if (completion != null) { + if (indicesMetrics.contains(IndexMetric.COMPLETION)) { completion.add(r.getAggregatedNodeLevelStats().commonStats.completion); } - if (segments != null) { + if (indicesMetrics.contains(IndexMetric.SEGMENTS)) { segments.add(r.getAggregatedNodeLevelStats().commonStats.segments); } } else { @@ -136,23 +136,23 @@ public ClusterStatsIndices( if (shardStats.getShardRouting().primary()) { indexShardStats.primaries++; - if (docs != null) { + if (indicesMetrics.contains(IndexMetric.DOCS)) { docs.add(shardCommonStats.docs); } } - if (store != null) { + if (indicesMetrics.contains(IndexMetric.STORE)) { store.add(shardCommonStats.store); } - if (fieldData != null) { + if (indicesMetrics.contains(IndexMetric.FIELDDATA)) { fieldData.add(shardCommonStats.fieldData); } - if (queryCache != null) { + if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) { queryCache.add(shardCommonStats.queryCache); } - if (completion != null) { + if (indicesMetrics.contains(IndexMetric.COMPLETION)) { completion.add(shardCommonStats.completion); } - if (segments != null) { + if (indicesMetrics.contains(IndexMetric.SEGMENTS)) { segments.add(shardCommonStats.segments); } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 657e3b425db21..4c76a29a0fd8b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -121,7 +121,7 @@ public class ClusterStatsNodes implements ToXContentFragment { nodeInfos.add(nodeResponse.nodeInfo()); nodeStats.add(nodeResponse.nodeStats()); this.versions.add(nodeResponse.nodeInfo().getVersion()); - if (this.plugins != null) { + if (requestedMetrics.contains(ClusterStatsRequest.Metric.PLUGINS)) { this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos()); } @@ -131,7 +131,7 @@ public class ClusterStatsNodes implements ToXContentFragment { if (!seenAddresses.add(inetAddress)) { continue; } - if (this.fs != null && nodeResponse.nodeStats().getFs() != null) { + if (requestedMetrics.contains(ClusterStatsRequest.Metric.FS) != null && nodeResponse.nodeStats().getFs() != null) { this.fs.add(nodeResponse.nodeStats().getFs().getTotal()); } } From c5621f1648069c115c894a3237edab9f720a7900 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Fri, 18 Oct 2024 17:45:53 +0530 Subject: [PATCH 10/12] Build fix. Signed-off-by: Swetha Guptha --- .../action/admin/cluster/stats/ClusterStatsNodes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 4c76a29a0fd8b..bf8218a66fc17 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -131,7 +131,7 @@ public class ClusterStatsNodes implements ToXContentFragment { if (!seenAddresses.add(inetAddress)) { continue; } - if (requestedMetrics.contains(ClusterStatsRequest.Metric.FS) != null && nodeResponse.nodeStats().getFs() != null) { + if (requestedMetrics.contains(ClusterStatsRequest.Metric.FS) && nodeResponse.nodeStats().getFs() != null) { this.fs.add(nodeResponse.nodeStats().getFs().getTotal()); } } From 302e7ec18c5910bb4926f0fcccbe69b6ed46919e Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Mon, 21 Oct 2024 23:32:59 +0530 Subject: [PATCH 11/12] Add cluster stats rolling version upgrade IT. Signed-off-by: Swetha Guptha --- .../opensearch/upgrades/ClusterStatsIT.java | 67 ++++++++++++ .../stats/ClusterStatsRequestTests.java | 102 ------------------ 2 files changed, 67 insertions(+), 102 deletions(-) create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java delete mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java new file mode 100644 index 0000000000000..1c5f35db8ec46 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.upgrades; + +import org.opensearch.Version; +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ClusterStatsIT extends AbstractRollingTestCase { + + private final List nodeStatsMetrics = List.of("os", "process", "jvm", "fs", "plugins", "ingest", "network_types", "discovery_types", "packaging_types"); + + private final List indicesStatsMetrics = List.of("shards", "docs", "store", "fielddata", "query_cache", "completion", "segments", "analysis", "mappings"); + + public void testClusterStats() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cluster/stats")); + validateClusterStatsWithFilterResponse(response, nodeStatsMetrics, indicesStatsMetrics); + if (AbstractRollingTestCase.UPGRADE_FROM_VERSION.onOrAfter(Version.V_3_0_0) || ( + CLUSTER_TYPE == ClusterType.UPGRADED && Version.CURRENT.onOrAfter(Version.V_3_0_0))) { + response = client().performRequest(new Request("GET", "/_cluster/stats/os/nodes/_all")); + validateClusterStatsWithFilterResponse(response, List.of("os"), Collections.emptyList()); + response = client().performRequest(new Request("GET", "/_cluster/stats/indices/mappings/nodes/_all")); + validateClusterStatsWithFilterResponse(response, Collections.emptyList(), List.of("mappings")); + response = client().performRequest(new Request("GET", "/_cluster/stats/os,indices/mappings/nodes/_all")); + validateClusterStatsWithFilterResponse(response, List.of("os"), List.of("mappings")); + } + } + + private void validateClusterStatsWithFilterResponse(Response response, List requestedNodesStatsMetrics, List requestedIndicesStatsMetrics) throws IOException { + assertEquals(200, response.getStatusLine().getStatusCode()); + Map entity = entityAsMap(response); + if (requestedNodesStatsMetrics != null && !requestedNodesStatsMetrics.isEmpty()) { + assertTrue(entity.containsKey("nodes")); + Map nodesStats = (Map) entity.get("nodes"); + for (String metric : nodeStatsMetrics) { + if (requestedNodesStatsMetrics.contains(metric)) { + assertTrue(nodesStats.containsKey(metric)); + } else { + assertFalse(nodesStats.containsKey(metric)); + } + } + } + + if (requestedIndicesStatsMetrics != null && !requestedIndicesStatsMetrics.isEmpty()) { + assertTrue(entity.containsKey("indices")); + Map indicesStats = (Map) entity.get("indices"); + for (String metric : indicesStatsMetrics) { + if (requestedIndicesStatsMetrics.contains(metric)) { + assertTrue(indicesStats.containsKey(metric)); + } else { + assertFalse(indicesStats.containsKey(metric)); + } + } + } + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java deleted file mode 100644 index 7d2c109bad83a..0000000000000 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.action.admin.cluster.stats; - -import org.opensearch.Version; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.test.OpenSearchTestCase; - -import java.util.Set; - -public class ClusterStatsRequestTests extends OpenSearchTestCase { - - public void testSerializationWithVersion3x() throws Exception { - ClusterStatsRequest clusterStatsRequest = getClusterStatsRequest(); - - Version testVersion = Version.V_3_0_0; - - BytesStreamOutput output = new BytesStreamOutput(); - output.setVersion(testVersion); - clusterStatsRequest.writeTo(output); - - StreamInput streamInput = output.bytes().streamInput(); - streamInput.setVersion(testVersion); - ClusterStatsRequest deserializedClusterStatsRequest = new ClusterStatsRequest(streamInput); - - validateClusterStatsRequest( - Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.PLUGINS, ClusterStatsRequest.Metric.INDICES), - Set.of( - ClusterStatsRequest.IndexMetric.SHARDS, - ClusterStatsRequest.IndexMetric.QUERY_CACHE, - ClusterStatsRequest.IndexMetric.MAPPINGS - ), - Version.V_3_0_0, - deserializedClusterStatsRequest - ); - assertEquals(-1, streamInput.read()); - } - - public void testSerializationOnVersionBelow3x() throws Exception { - ClusterStatsRequest clusterStatsRequest = getClusterStatsRequest(); - - Version testVersion = Version.V_2_17_0; - - BytesStreamOutput output = new BytesStreamOutput(); - output.setVersion(testVersion); - clusterStatsRequest.writeTo(output); - - StreamInput streamInput = output.bytes().streamInput(); - streamInput.setVersion(testVersion); - ClusterStatsRequest deserializedClusterStatsRequest = new ClusterStatsRequest(streamInput); - - validateClusterStatsRequest( - Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.PLUGINS, ClusterStatsRequest.Metric.INDICES), - Set.of( - ClusterStatsRequest.IndexMetric.SHARDS, - ClusterStatsRequest.IndexMetric.QUERY_CACHE, - ClusterStatsRequest.IndexMetric.MAPPINGS - ), - Version.V_2_17_0, - deserializedClusterStatsRequest - ); - assertEquals(-1, streamInput.read()); - } - - private ClusterStatsRequest getClusterStatsRequest() { - ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(); - clusterStatsRequest.computeAllMetrics(true); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.OS); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.PLUGINS); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.INDICES); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.SHARDS); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.QUERY_CACHE); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.MAPPINGS); - clusterStatsRequest.useAggregatedNodeLevelResponses(true); - return clusterStatsRequest; - } - - private void validateClusterStatsRequest( - Set metrics, - Set indexMetrics, - Version version, - ClusterStatsRequest deserializedClusterStatsRequest - ) { - if (version.before(Version.V_3_0_0)) { - assertEquals(true, deserializedClusterStatsRequest.computeAllMetrics()); - assertTrue(deserializedClusterStatsRequest.requestedMetrics().isEmpty()); - assertTrue(deserializedClusterStatsRequest.indicesMetrics().isEmpty()); - } else { - assertEquals(true, deserializedClusterStatsRequest.computeAllMetrics()); - assertEquals(metrics, deserializedClusterStatsRequest.requestedMetrics()); - assertEquals(indexMetrics, deserializedClusterStatsRequest.indicesMetrics()); - } - } - -} From 6b87984bf6cf0dcf82806e2e556b7839c305c393 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Tue, 22 Oct 2024 18:00:07 +0530 Subject: [PATCH 12/12] Add change log. Signed-off-by: Swetha Guptha --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3e26935ea6e..c781fbbcabdf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) - Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) -- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387) +- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387)) +- URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))