From dacd105fd3b152cfad053110c12ea7f1d368e934 Mon Sep 17 00:00:00 2001 From: Swetha Guptha Date: Fri, 4 Oct 2024 16:49:43 +0530 Subject: [PATCH] Test case changes and refactoring. Signed-off-by: Swetha Guptha --- .../admin/cluster/stats/ClusterStatsIT.java | 273 +++++++++++------- .../cluster/stats/ClusterStatsRequest.java | 10 +- .../stats/TransportClusterStatsAction.java | 2 +- .../admin/cluster/RestClusterStatsAction.java | 144 ++++----- .../stats/ClusterStatsRequestTests.java | 89 ++++-- .../stats/ClusterStatsResponseTests.java | 61 +++- .../cluster/RestClusterStatsActionTests.java | 18 +- 7 files changed, 385 insertions(+), 212 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index aa4f799db4083..9116fe8f2b300 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -321,7 +321,8 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent()); } - public void testValuesSmokeScreenWithMetricFilter() throws IOException, ExecutionException, InterruptedException { + public void testValuesSmokeScreenWithNodeStatsAndIndicesStatsMetricsFilter() throws IOException, ExecutionException, + InterruptedException { internalCluster().startNodes(randomIntBetween(1, 3)); index("test1", "type", "1", "f", "f"); @@ -401,6 +402,40 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception { } public void testFieldTypes() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + client().admin() + .indices() + .prepareCreate("test2") + .setMapping( + "{\"properties\":{\"foo\":{\"type\": \"keyword\"},\"bar\":{\"properties\":{\"baz\":{\"type\":\"keyword\"}," + + "\"eggplant\":{\"type\":\"integer\"}}}}}" + ) + .get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(randomBoolean()).get(); + assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3)); + Set stats = response.getIndicesStats().getMappings().getFieldTypeStats(); + for (IndexFeatureStats stat : stats) { + if (stat.getName().equals("integer")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } else if (stat.getName().equals("keyword")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(3)); + } else if (stat.getName().equals("object")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } + } + } + + public void testFieldTypesWithMappingsFilter() { internalCluster().startNode(); ensureGreen(); ClusterStatsResponse response = client().admin() @@ -558,7 +593,7 @@ public void testNodeRolesWithDataNodeLegacySettings() throws ExecutionException, assertEquals(expectedNodesRoles, Set.of(getNodeRoles(client, 0), getNodeRoles(client, 1))); } - public void testClusterStatsApplyMetricFilterDisabled() throws IOException { + public void testClusterStatsWithNodeMetricsFilter() { internalCluster().startNode(); ensureGreen(); @@ -571,54 +606,34 @@ public void testClusterStatsApplyMetricFilterDisabled() throws IOException { assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); assertNotNull(response.getNodesStats()); assertNotNull(response.getIndicesStats()); - ClusterStatsResponse statsResponseWithMetricFilterDisabled = client().admin() + ClusterStatsResponse statsResponseWithAllNodeStatsMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .computeAllMetrics(true) - .requestMetrics(Set.of(Metric.FS, Metric.JVM, Metric.PLUGINS, Metric.OS)) + .requestMetrics(ClusterStatsNodes.NODE_STATS_METRICS) + .computeAllMetrics(false) .get(); - - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats()); + assertNotNull(statsResponseWithAllNodeStatsMetrics); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats()); + assertNull(statsResponseWithAllNodeStatsMetrics.getIndicesStats()); + validateNodeStatsOutput(ClusterStatsNodes.NODE_STATS_METRICS, statsResponseWithAllNodeStatsMetrics); assertEquals( response.getNodesStats().getCounts().getTotal(), - statsResponseWithMetricFilterDisabled.getNodesStats().getCounts().getTotal() + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getTotal() ); assertEquals( response.getNodesStats().getCounts().getRoles(), - statsResponseWithMetricFilterDisabled.getNodesStats().getCounts().getRoles() - ); - assertEquals(response.getNodesStats().getVersions(), statsResponseWithMetricFilterDisabled.getNodesStats().getVersions()); - assertEquals(response.getNodesStats().getPlugins(), statsResponseWithMetricFilterDisabled.getNodesStats().getPlugins()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getOs()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getJvm()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getProcess()); - assertNotNull(statsResponseWithMetricFilterDisabled.getNodesStats().getFs()); - - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats()); - assertEquals(response.getIndicesStats().getFieldData(), statsResponseWithMetricFilterDisabled.getIndicesStats().getFieldData()); - assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithMetricFilterDisabled.getIndicesStats().getAnalysis()); - assertEquals(response.getIndicesStats().getMappings(), statsResponseWithMetricFilterDisabled.getIndicesStats().getMappings()); - assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithMetricFilterDisabled.getIndicesStats().getIndexCount()); - assertEquals( - response.getIndicesStats().getShards().getTotal(), - statsResponseWithMetricFilterDisabled.getIndicesStats().getShards().getTotal() - ); - assertEquals( - response.getIndicesStats().getShards().getPrimaries(), - statsResponseWithMetricFilterDisabled.getIndicesStats().getShards().getPrimaries() + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getRoles() ); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getSegments()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getDocs()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getQueryCache()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getCompletion()); - assertNotNull(statsResponseWithMetricFilterDisabled.getIndicesStats().getStore()); + assertEquals(response.getNodesStats().getVersions(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getVersions()); + assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); } - public void testClusterStatsWithMetricsFilter() { + public void testClusterStatsWithIndicesOnlyMetricsFilter() { internalCluster().startNode(); ensureGreen(); @@ -635,55 +650,26 @@ public void testClusterStatsWithMetricsFilter() { assertNotNull(response.getNodesStats()); assertNotNull(response.getIndicesStats()); - ClusterStatsResponse statsResponseWithAllNodeStatsMetrics = client().admin() - .cluster() - .prepareClusterStats() - .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(ClusterStatsNodes.NODE_STATS_METRICS) - .computeAllMetrics(false) - .get(); - assertNotNull(statsResponseWithAllNodeStatsMetrics); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats()); - assertNull(statsResponseWithAllNodeStatsMetrics.getIndicesStats()); - assertEquals( - response.getNodesStats().getCounts().getTotal(), - statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getTotal() - ); - assertEquals( - response.getNodesStats().getCounts().getRoles(), - statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getRoles() - ); - assertEquals(response.getNodesStats().getVersions(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getVersions()); - assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getOs()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getJvm()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getProcess()); - assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats().getFs()); - ClusterStatsResponse statsResponseWithIndicesRequestMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.PLUGINS, Metric.INDICES)) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.values())) .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithIndicesRequestMetrics); - assertNotNull(statsResponseWithIndicesRequestMetrics.getNodesStats()); - assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getOs()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getFs()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getProcess()); - assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats().getJvm()); + assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats()); assertNotNull(statsResponseWithIndicesRequestMetrics.getIndicesStats()); - assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithIndicesRequestMetrics.getIndicesStats().getIndexCount()); + validateIndicesStatsOutput(Set.of(IndexMetric.values()), statsResponseWithIndicesRequestMetrics); } - public void testClusterStatsWithIndexMetricFilter() { + public void testClusterStatsWithSelectiveNodeMetricAndIndexMetricsFilter() { internalCluster().startNode(); ensureGreen(); client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); - IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source("{\"test_type\" : \"metrics_filter\"}"); + IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source(Map.of("test_type", "metrics_filter")); client().index(indexRequest); ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() @@ -697,35 +683,23 @@ public void testClusterStatsWithIndexMetricFilter() { assertNotNull(response.getNodesStats()); assertNotNull(response.getIndicesStats()); - ClusterStatsResponse statsResponseWithSpecificIndicesMetrics = client().admin() - .cluster() - .prepareClusterStats() - .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.INDICES)) - .indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)) - .computeAllMetrics(false) - .get(); - assertNotNull(statsResponseWithSpecificIndicesMetrics); - assertNull(statsResponseWithSpecificIndicesMetrics.getNodesStats()); - assertNotNull(statsResponseWithSpecificIndicesMetrics.getIndicesStats()); - assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getIndexCount()); - assertEquals(response.getIndicesStats().getMappings(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getMappings()); - assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getAnalysis()); - ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() .cluster() .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) - .requestMetrics(Set.of(Metric.INDICES)) - .indexMetrics(Set.of(IndexMetric.values())) + .requestMetrics(Set.of(Metric.OS, Metric.FS, Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE)) .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithAllIndicesMetrics); - assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getNodesStats()); assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + validateNodeStatsOutput(Set.of(Metric.FS, Metric.OS), statsResponseWithAllIndicesMetrics); + validateIndicesStatsOutput( + Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE), + statsResponseWithAllIndicesMetrics + ); assertEquals(response.getIndicesStats().getFieldData(), statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData()); - assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithAllIndicesMetrics.getIndicesStats().getAnalysis()); - assertEquals(response.getIndicesStats().getMappings(), statsResponseWithAllIndicesMetrics.getIndicesStats().getMappings()); assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithAllIndicesMetrics.getIndicesStats().getIndexCount()); assertEquals( response.getIndicesStats().getShards().getTotal(), @@ -735,11 +709,42 @@ public void testClusterStatsWithIndexMetricFilter() { response.getIndicesStats().getShards().getPrimaries(), statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getPrimaries() ); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getQueryCache()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getCompletion()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getStore()); + } + + public void testClusterStatsWithMappingsAndAnalysisStatsIndexMetricsFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source(Map.of("test_type", "metrics_filter")); + client().index(indexRequest); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats()); + + ClusterStatsResponse statsResponseWithSpecificIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithSpecificIndicesMetrics); + assertNull(statsResponseWithSpecificIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithSpecificIndicesMetrics.getIndicesStats()); + validateIndicesStatsOutput(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS), statsResponseWithSpecificIndicesMetrics); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getIndexCount()); + assertEquals(response.getIndicesStats().getMappings(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getMappings()); + assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getAnalysis()); } public void testClusterStatsWithIndexMetricWithDocsFilter() throws IOException { @@ -756,18 +761,90 @@ public void testClusterStatsWithIndexMetricWithDocsFilter() throws IOException { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.INDICES)) .indexMetrics(Set.of(IndexMetric.DOCS)) - .computeAllMetrics(true) + .computeAllMetrics(false) .get(); assertNotNull(statsResponseWithAllIndicesMetrics); assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); - assertNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getShards()); - assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs()); + validateIndicesStatsOutput(Set.of(IndexMetric.DOCS), statsResponseWithAllIndicesMetrics); assertEquals(2, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getCount()); assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getDeleted()); assertTrue(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getAverageSizeInBytes() > 0); } + private void validateNodeStatsOutput(Set expectedMetrics, ClusterStatsResponse clusterStatsResponse) { + // Ingest, network types, discovery types and packaging types stats are not included here as they don't have a get method exposed. + Set NodeMetrics = Set.of(Metric.OS, Metric.JVM, Metric.FS, Metric.PROCESS, Metric.PLUGINS); + for (Metric metric : NodeMetrics) { + Object object = null; + switch (metric) { + case OS: + object = clusterStatsResponse.getNodesStats().getOs(); + break; + case JVM: + object = clusterStatsResponse.getNodesStats().getJvm(); + break; + case FS: + object = clusterStatsResponse.getNodesStats().getFs(); + break; + case PROCESS: + object = clusterStatsResponse.getNodesStats().getProcess(); + break; + case PLUGINS: + object = clusterStatsResponse.getNodesStats().getPlugins(); + break; + } + if (expectedMetrics.contains(metric)) { + assertNotNull(object); + } else { + assertNull(object); + } + } + } + + private void validateIndicesStatsOutput( + Set expectedMetrics, + ClusterStatsResponse clusterStatsResponse + ) { + for (IndexMetric indexMetric : IndexMetric.values()) { + Object object = null; + switch (indexMetric) { + case SHARDS: + object = clusterStatsResponse.getIndicesStats().getShards(); + break; + case DOCS: + object = clusterStatsResponse.getIndicesStats().getDocs(); + break; + case STORE: + object = clusterStatsResponse.getIndicesStats().getStore(); + break; + case FIELDDATA: + object = clusterStatsResponse.getIndicesStats().getFieldData(); + break; + case QUERY_CACHE: + object = clusterStatsResponse.getIndicesStats().getQueryCache(); + break; + case COMPLETION: + object = clusterStatsResponse.getIndicesStats().getCompletion(); + break; + case SEGMENTS: + object = clusterStatsResponse.getIndicesStats().getSegments(); + break; + case ANALYSIS: + object = clusterStatsResponse.getIndicesStats().getAnalysis(); + break; + case MAPPINGS: + object = clusterStatsResponse.getIndicesStats().getMappings(); + break; + } + if (expectedMetrics.contains(indexMetric)) { + assertNotNull(object); + } else { + assertNull(object); + } + } + } + private Map getExpectedCounts( int dataRoleCount, int masterRoleCount, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 4b0a32517f48e..1bfacd7cc6451 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -52,7 +52,7 @@ public class ClusterStatsRequest extends BaseNodesRequest { private final Set requestedMetrics = new HashSet<>(); private final Set indexMetricsRequested = new HashSet<>(); - private Boolean computeAllMetric = true; + private Boolean computeAllMetrics = true; public ClusterStatsRequest(StreamInput in) throws IOException { super(in); @@ -60,7 +60,7 @@ public ClusterStatsRequest(StreamInput in) throws IOException { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } if (in.getVersion().onOrAfter(Version.V_3_0_0)) { - computeAllMetric = in.readOptionalBoolean(); + computeAllMetrics = in.readOptionalBoolean(); final long longMetricsFlags = in.readLong(); for (Metric metric : Metric.values()) { if ((longMetricsFlags & (1 << metric.getIndex())) != 0) { @@ -95,11 +95,11 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon } public boolean computeAllMetrics() { - return computeAllMetric; + return computeAllMetrics; } public void computeAllMetrics(boolean computeAllMetrics) { - this.computeAllMetric = computeAllMetrics; + this.computeAllMetrics = computeAllMetrics; } /** @@ -136,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } if (out.getVersion().onOrAfter(Version.V_3_0_0)) { - out.writeOptionalBoolean(computeAllMetric); + out.writeOptionalBoolean(computeAllMetrics); long longMetricFlags = 0; for (Metric metric : requestedMetrics) { longMetricFlags |= (1 << metric.getIndex()); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index b7a645240f407..342a949480be8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -266,7 +266,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq } /** - * A metric is required when: all cluster stats are required (OR) if the metric was requested + * A metric is required when: all cluster stats are required (OR) if the metric is requested * @param metric * @param clusterStatsRequest * @return diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index daa4a75285823..f10ac9e4a1f9b 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -33,6 +33,8 @@ package org.opensearch.rest.action.admin.cluster; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; @@ -79,7 +81,7 @@ public List routes() { static { Map> metricRequestConsumerMap = new HashMap<>(); - for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { + for (Metric metric : Metric.values()) { metricRequestConsumerMap.put(metric.metricName(), request -> request.addMetric(metric)); } METRIC_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricRequestConsumerMap); @@ -87,7 +89,7 @@ public List routes() { static { Map> metricMap = new HashMap<>(); - for (ClusterStatsRequest.IndexMetric indexMetric : ClusterStatsRequest.IndexMetric.values()) { + for (IndexMetric indexMetric : IndexMetric.values()) { metricMap.put(indexMetric.metricName(), request -> request.addIndexMetric(indexMetric)); } INDEX_METRIC_TO_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricMap); @@ -106,6 +108,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC public static ClusterStatsRequest fromRequest(final RestRequest request) { Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + String indicesMetricsDefaultValue = metrics.contains(Metric.INDICES.metricName()) || metrics.contains("_all") ? "_all" : null; + Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", indicesMetricsDefaultValue)); String[] nodeIds = request.paramAsStringArray("nodeId", null); ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); @@ -113,82 +117,25 @@ public static ClusterStatsRequest fromRequest(final RestRequest request) { clusterStatsRequest.useAggregatedNodeLevelResponses(true); if (!metrics.isEmpty()) { - if (metrics.size() > 1 && metrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ); - } - - final Set metricsRequested = new HashSet<>(); - if (metrics.contains("_all")) { - metricsRequested.addAll(METRIC_REQUEST_CONSUMER_MAP.keySet()); - } else { - metricsRequested.addAll(metrics); - } - - Set indexMetrics = Strings.tokenizeByCommaToSet( - request.param("index_metric", metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName()) ? "_all" : null) - ); - - if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual index metrics [%s]", - request.path(), - request.param("index_metric") - ) - ); - } - - final Set invalidMetrics = new TreeSet<>(); - for (String metric : metricsRequested) { - Consumer clusterStatsRequestConsumer = METRIC_REQUEST_CONSUMER_MAP.get(metric); - if (clusterStatsRequestConsumer != null) { - clusterStatsRequestConsumer.accept(clusterStatsRequest); - } else { - invalidMetrics.add(metric); - } - } + paramValidations(metrics, indexMetrics, request); + final Set metricsRequested = metrics.contains("_all") + ? new HashSet<>(METRIC_REQUEST_CONSUMER_MAP.keySet()) + : new HashSet<>(metrics); + Set invalidMetrics = validateAndSetRequestedMetrics(metricsRequested, METRIC_REQUEST_CONSUMER_MAP, clusterStatsRequest); if (!invalidMetrics.isEmpty()) { throw new IllegalArgumentException( unrecognizedStrings(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric") ); } - - if (!metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName()) && !indexMetrics.isEmpty()) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains index metrics [%s] but indices stats not requested", - request.path(), - request.param("index_metric") - ) + if (metricsRequested.contains(Metric.INDICES.metricName())) { + final Set indexMetricsRequested = indexMetrics.contains("_all") + ? INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet() + : new HashSet<>(indexMetrics); + Set invalidIndexMetrics = validateAndSetRequestedMetrics( + indexMetricsRequested, + INDEX_METRIC_TO_REQUEST_CONSUMER_MAP, + clusterStatsRequest ); - } - - if (metricsRequested.contains(ClusterStatsRequest.Metric.INDICES.metricName())) { - final Set indexMetricsRequested = new HashSet<>(); - if (indexMetrics.contains("_all")) { - indexMetricsRequested.addAll(INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()); - } else { - indexMetricsRequested.addAll(indexMetrics); - } - final Set invalidIndexMetrics = new TreeSet<>(); - for (String indexMetric : indexMetricsRequested) { - Consumer clusterStatsRequestConsumer = INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.get(indexMetric); - if (clusterStatsRequestConsumer != null) { - clusterStatsRequestConsumer.accept(clusterStatsRequest); - } else { - invalidIndexMetrics.add(indexMetric); - } - } - if (!invalidIndexMetrics.isEmpty()) { throw new IllegalArgumentException( unrecognizedStrings(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") @@ -197,9 +144,62 @@ public static ClusterStatsRequest fromRequest(final RestRequest request) { } clusterStatsRequest.computeAllMetrics(false); } + return clusterStatsRequest; } + private static void paramValidations(Set metrics, Set indexMetrics, RestRequest request) { + if (metrics.size() > 1 && metrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ); + } + + if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual index metrics [%s]", + request.path(), + request.param("index_metric") + ) + ); + } + + if (!metrics.contains(Metric.INDICES.metricName()) && !metrics.contains("_all") && !indexMetrics.isEmpty()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains index metrics [%s] but indices stats not requested", + request.path(), + request.param("index_metric") + ) + ); + } + } + + private static Set validateAndSetRequestedMetrics( + Set metrics, + Map> metricConsumerMap, + ClusterStatsRequest clusterStatsRequest + ) { + final Set invalidMetrics = new TreeSet<>(); + for (String metric : metrics) { + Consumer clusterStatsRequestConsumer = metricConsumerMap.get(metric); + if (clusterStatsRequestConsumer != null) { + clusterStatsRequestConsumer.accept(clusterStatsRequest); + } else { + invalidMetrics.add(metric); + } + } + return invalidMetrics; + } + @Override public boolean canTripCircuitBreaker() { return false; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java index cd452693779f2..7d2c109bad83a 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestTests.java @@ -12,22 +12,41 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.test.VersionUtils; + +import java.util.Set; public class ClusterStatsRequestTests extends OpenSearchTestCase { - public void testSerialization() throws Exception { - ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(); - clusterStatsRequest.computeAllMetrics(randomBoolean()); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.OS); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.PLUGINS); - clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.INDICES); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.SHARDS); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.QUERY_CACHE); - clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.MAPPINGS); - clusterStatsRequest.useAggregatedNodeLevelResponses(randomBoolean()); + public void testSerializationWithVersion3x() throws Exception { + ClusterStatsRequest clusterStatsRequest = getClusterStatsRequest(); + + Version testVersion = Version.V_3_0_0; + + BytesStreamOutput output = new BytesStreamOutput(); + output.setVersion(testVersion); + clusterStatsRequest.writeTo(output); + + StreamInput streamInput = output.bytes().streamInput(); + streamInput.setVersion(testVersion); + ClusterStatsRequest deserializedClusterStatsRequest = new ClusterStatsRequest(streamInput); + + validateClusterStatsRequest( + Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.PLUGINS, ClusterStatsRequest.Metric.INDICES), + Set.of( + ClusterStatsRequest.IndexMetric.SHARDS, + ClusterStatsRequest.IndexMetric.QUERY_CACHE, + ClusterStatsRequest.IndexMetric.MAPPINGS + ), + Version.V_3_0_0, + deserializedClusterStatsRequest + ); + assertEquals(-1, streamInput.read()); + } - Version testVersion = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT); + public void testSerializationOnVersionBelow3x() throws Exception { + ClusterStatsRequest clusterStatsRequest = getClusterStatsRequest(); + + Version testVersion = Version.V_2_17_0; BytesStreamOutput output = new BytesStreamOutput(); output.setVersion(testVersion); @@ -37,13 +56,47 @@ public void testSerialization() throws Exception { streamInput.setVersion(testVersion); ClusterStatsRequest deserializedClusterStatsRequest = new ClusterStatsRequest(streamInput); - assertEquals(clusterStatsRequest.computeAllMetrics(), deserializedClusterStatsRequest.computeAllMetrics()); - assertEquals(clusterStatsRequest.requestedMetrics(), deserializedClusterStatsRequest.requestedMetrics()); - assertEquals(clusterStatsRequest.indicesMetrics(), deserializedClusterStatsRequest.indicesMetrics()); - assertEquals( - clusterStatsRequest.useAggregatedNodeLevelResponses(), - deserializedClusterStatsRequest.useAggregatedNodeLevelResponses() + validateClusterStatsRequest( + Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.PLUGINS, ClusterStatsRequest.Metric.INDICES), + Set.of( + ClusterStatsRequest.IndexMetric.SHARDS, + ClusterStatsRequest.IndexMetric.QUERY_CACHE, + ClusterStatsRequest.IndexMetric.MAPPINGS + ), + Version.V_2_17_0, + deserializedClusterStatsRequest ); + assertEquals(-1, streamInput.read()); + } + + private ClusterStatsRequest getClusterStatsRequest() { + ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(); + clusterStatsRequest.computeAllMetrics(true); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.OS); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.PLUGINS); + clusterStatsRequest.addMetric(ClusterStatsRequest.Metric.INDICES); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.SHARDS); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.QUERY_CACHE); + clusterStatsRequest.addIndexMetric(ClusterStatsRequest.IndexMetric.MAPPINGS); + clusterStatsRequest.useAggregatedNodeLevelResponses(true); + return clusterStatsRequest; + } + + private void validateClusterStatsRequest( + Set metrics, + Set indexMetrics, + Version version, + ClusterStatsRequest deserializedClusterStatsRequest + ) { + if (version.before(Version.V_3_0_0)) { + assertEquals(true, deserializedClusterStatsRequest.computeAllMetrics()); + assertTrue(deserializedClusterStatsRequest.requestedMetrics().isEmpty()); + assertTrue(deserializedClusterStatsRequest.indicesMetrics().isEmpty()); + } else { + assertEquals(true, deserializedClusterStatsRequest.computeAllMetrics()); + assertEquals(metrics, deserializedClusterStatsRequest.requestedMetrics()); + assertEquals(indexMetrics, deserializedClusterStatsRequest.indicesMetrics()); + } } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java index 850460d2f86d7..abd607d5787dc 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java @@ -13,6 +13,7 @@ import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.ShardStats; @@ -37,7 +38,9 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.StoreStats; import org.opensearch.monitor.jvm.JvmInfo; +import org.opensearch.monitor.jvm.JvmStats; import org.opensearch.monitor.os.OsInfo; +import org.opensearch.monitor.process.ProcessStats; import org.opensearch.search.suggest.completion.CompletionStats; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportInfo; @@ -72,7 +75,7 @@ public void testSerializationWithIndicesMappingAndAnalysisStats() throws Excepti List.of(), ClusterState.EMPTY_STATE, Set.of(ClusterStatsRequest.Metric.INDICES), - Set.of(ClusterStatsRequest.IndexMetric.MAPPINGS, ClusterStatsRequest.IndexMetric.ANALYSIS) + Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS) ); BytesStreamOutput output = new BytesStreamOutput(); clusterStatsResponse.writeTo(output); @@ -107,8 +110,15 @@ public void testSerializationWithoutIndicesMappingAndAnalysisStats() throws Exce defaultClusterStatsNodeResponses, List.of(), ClusterState.EMPTY_STATE, - Set.of(), - Set.of() + Set.of(ClusterStatsRequest.Metric.INDICES, ClusterStatsRequest.Metric.PROCESS, ClusterStatsRequest.Metric.JVM), + Set.of( + IndexMetric.DOCS, + IndexMetric.STORE, + IndexMetric.SEGMENTS, + IndexMetric.QUERY_CACHE, + IndexMetric.FIELDDATA, + IndexMetric.COMPLETION + ) ); BytesStreamOutput output = new BytesStreamOutput(); clusterStatsResponse.writeTo(output); @@ -118,10 +128,28 @@ public void testSerializationWithoutIndicesMappingAndAnalysisStats() throws Exce assertEquals(clusterStatsResponse.timestamp, deserializedClusterStatsResponse.timestamp); assertEquals(clusterStatsResponse.status, deserializedClusterStatsResponse.status); assertEquals(clusterStatsResponse.clusterUUID, deserializedClusterStatsResponse.clusterUUID); - assertNull(clusterStatsResponse.indicesStats); + assertNotNull(deserializedClusterStatsResponse.nodesStats); + assertNotNull(deserializedClusterStatsResponse.nodesStats.getProcess()); + assertNotNull(deserializedClusterStatsResponse.nodesStats.getJvm()); + assertNotNull(deserializedClusterStatsResponse.indicesStats); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getDocs()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getStore()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getSegments()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getQueryCache()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getFieldData()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getCompletion()); + assertNull(deserializedClusterStatsResponse.indicesStats.getMappings()); + assertNull(deserializedClusterStatsResponse.indicesStats.getAnalysis()); } private ClusterStatsNodeResponse createClusterStatsNodeResponse(DiscoveryNode node, ShardStats[] shardStats) throws IOException { + JvmStats.GarbageCollector[] garbageCollectorsArray = new JvmStats.GarbageCollector[1]; + garbageCollectorsArray[0] = new JvmStats.GarbageCollector( + randomAlphaOfLengthBetween(3, 10), + randomNonNegativeLong(), + randomNonNegativeLong() + ); + JvmStats.GarbageCollectors garbageCollectors = new JvmStats.GarbageCollectors(garbageCollectorsArray); NodeInfo nodeInfo = new NodeInfo( Version.CURRENT, Build.CURRENT, @@ -148,8 +176,29 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse(DiscoveryNode no randomNonNegativeLong(), null, null, - null, - null, + new ProcessStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + new ProcessStats.Cpu(randomShort(), randomNonNegativeLong()), + new ProcessStats.Mem(randomNonNegativeLong()) + ), + new JvmStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + new JvmStats.Mem( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + Collections.emptyList() + ), + new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), + garbageCollectors, + Collections.emptyList(), + new JvmStats.Classes(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) + ), null, null, null, diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java index e3eb525696272..8c8284fa36211 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java @@ -38,7 +38,7 @@ public void testFromRequestBasePath() { ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); assertNotNull(clusterStatsRequest); assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); - assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.computeAllMetrics()); assertNotNull(clusterStatsRequest.requestedMetrics()); assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { @@ -60,12 +60,10 @@ public void testFromRequestWithNodeStatsMetricsFilter() { ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); assertNotNull(clusterStatsRequest); assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); - assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.computeAllMetrics()); assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); assertEquals(2, clusterStatsRequest.requestedMetrics().size()); - for (ClusterStatsRequest.Metric metric : metricsRequested) { - assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); - } + assertEquals(metricsRequested, clusterStatsRequest.requestedMetrics()); assertTrue(clusterStatsRequest.indicesMetrics().isEmpty()); } @@ -90,17 +88,13 @@ public void testFromRequestWithIndicesStatsMetricsFilter() { ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); assertNotNull(clusterStatsRequest); assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); - assertTrue(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.computeAllMetrics()); assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); assertEquals(3, clusterStatsRequest.requestedMetrics().size()); - for (ClusterStatsRequest.Metric metric : metricsRequested) { - assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); - } + assertEquals(metricsRequested, clusterStatsRequest.requestedMetrics()); assertFalse(clusterStatsRequest.indicesMetrics().isEmpty()); assertEquals(2, clusterStatsRequest.indicesMetrics().size()); - for (ClusterStatsRequest.IndexMetric indexMetric : indicesMetricsRequested) { - assertTrue(clusterStatsRequest.indicesMetrics().contains(indexMetric)); - } + assertEquals(indicesMetricsRequested, clusterStatsRequest.indicesMetrics()); } public void testUnrecognizedMetric() {