From d581be88c682ba92ca8bbf1bfd9165772f8929c2 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Wed, 16 Oct 2024 17:30:27 +0530 Subject: [PATCH] All test cases working --- pinot-common/pom.xml | 2 +- .../metrics/BrokerJMXToPromMetricsTest.java | 25 +++++++++- .../ControllerJMXToPromMetricsTest.java | 45 +++++++++++++++++ .../metrics/MinionJMXToPromMetricsTest.java | 6 +++ .../metrics/PinotJMXToPromMetricsTest.java | 47 +++++++++++++++-- .../metrics/ServerJMXToPromMetricsTest.java | 50 ++++++++++++++++++- 6 files changed, 167 insertions(+), 8 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index a5c1c84e261..799308ce3b6 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -52,7 +52,7 @@ false - -javaagent:${project.build.directory}/agent/jmx_prometheus_javaagent-0.19.0.jar=9021:${project.basedir}/../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml + diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java index c6100b73bf4..1a8f855a016 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java @@ -21,10 +21,14 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import io.prometheus.jmx.JavaAgent; +import java.net.URI; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -43,8 +47,18 @@ public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private BrokerMetrics _brokerMetrics; + private int _exporterPort; + @BeforeClass - public void setup() { + public void setup() + throws Exception { + + _exporterPort = 9000 + new Random().nextInt(1000); + String agentArgs = String.format("%s:%s", _exporterPort, + "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml"); + + JavaAgent.agentmain(agentArgs, null); + PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); @@ -141,4 +155,13 @@ public void brokerMeterTest() { } }); } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 130fe376a26..9e64cf3f4d8 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -21,14 +21,27 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import io.prometheus.jmx.BuildInfoCollector; import io.prometheus.jmx.JavaAgent; +import io.prometheus.jmx.JmxCollector; +import io.prometheus.jmx.common.http.ConfigurationException; +import io.prometheus.jmx.common.http.HTTPServerFactory; +import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; +import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; +import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.helix.task.TaskState; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -45,10 +58,33 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String TASK_TYPE = "ClusterHealthCheck"; private ControllerMetrics _controllerMetrics; + private int _exporterPort; + + private HTTPServer _httpServer; + @BeforeClass public void setup() throws Exception { + _exporterPort = 9000 + new Random().nextInt(1000); + String agentArgs = String.format("%s:%s", _exporterPort, + "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml"); + + String host = "0.0.0.0"; + + try { + JMXExporterConfig config = parseConfig(agentArgs, host); + CollectorRegistry registry = new CollectorRegistry(); + (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); + DefaultExports.register(registry); + _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); + } catch (ConfigurationException var4) { + System.err.println("Configuration Exception : " + var4.getMessage()); + System.exit(1); + } catch (IllegalArgumentException var5) { + System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage()); + System.exit(1); + } PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, @@ -219,4 +255,13 @@ private boolean isRemaining(ControllerGauge gauge, List gaugesA private static String getStrippedMetricName(ControllerGauge controllerGauge) { return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java index 34abd0f39b8..f0a416bfad2 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java @@ -23,6 +23,7 @@ import com.yammer.metrics.reporting.JmxReporter; import java.util.List; import java.util.stream.Stream; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -102,4 +103,9 @@ public void meterTest() { @Test public void gaugeTest() { } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + return null; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 0a0c483f4d2..5566f6bd39c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -20,15 +20,19 @@ package org.apache.pinot.common.metrics; import com.google.common.base.Objects; +import io.prometheus.jmx.JavaAgent; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; @@ -38,7 +42,7 @@ import org.testng.Assert; -public class PinotJMXToPromMetricsTest { +public abstract class PinotJMXToPromMetricsTest { protected HttpClient _httpClient; @@ -184,10 +188,7 @@ protected List parseExportedPromMetrics(String response) return exportedPromMetrics; } - protected SimpleHttpResponse getExportedPromMetrics() - throws IOException, URISyntaxException { - return _httpClient.sendGetRequest(new URI("http://localhost:9021/metrics")); - } + protected abstract SimpleHttpResponse getExportedPromMetrics(); public static class PromMetric { private final String _metricName; @@ -278,4 +279,40 @@ public String toString() { return sb.toString(); } } + + public static JMXExporterConfig parseConfig(String args, String ifc) { + Pattern pattern = Pattern.compile("^(?:((?:[\\w.-]+)|(?:\\[.+])):)?(\\d{1,5}):(.+)"); + Matcher matcher = pattern.matcher(args); + if (!matcher.matches()) { + throw new IllegalArgumentException("Malformed arguments - " + args); + } else { + String givenHost = matcher.group(1); + String givenPort = matcher.group(2); + String givenConfigFile = matcher.group(3); + int port = Integer.parseInt(givenPort); + InetSocketAddress socket; + if (givenHost != null && !givenHost.isEmpty()) { + socket = new InetSocketAddress(givenHost, port); + } else { + socket = new InetSocketAddress(ifc, port); + givenHost = ifc; + } + + return new JMXExporterConfig(givenHost, port, givenConfigFile, socket); + } + } + + public static class JMXExporterConfig { + String host; + int port; + String file; + InetSocketAddress socket; + + JMXExporterConfig(String host, int port, String file, InetSocketAddress socket) { + this.host = host; + this.port = port; + this.file = file; + this.socket = socket; + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index cd047125703..40d366327af 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -21,10 +21,22 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import io.prometheus.jmx.BuildInfoCollector; +import io.prometheus.jmx.JavaAgent; +import io.prometheus.jmx.JmxCollector; +import io.prometheus.jmx.common.http.ConfigurationException; +import io.prometheus.jmx.common.http.HTTPServerFactory; +import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; +import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; +import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports; +import java.io.File; +import java.net.URI; import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -40,8 +52,35 @@ public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; private ServerMetrics _serverMetrics; + private int _exporterPort; + + private HTTPServer _httpServer; + @BeforeClass - public void setup() { + public void setup() + throws Exception { + + _exporterPort = 9000 + new Random().nextInt(1000); + String agentArgs = String.format("%s:%s", _exporterPort, + "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml"); + + String host = "0.0.0.0"; + + try { + JMXExporterConfig config = parseConfig(agentArgs, host); + CollectorRegistry registry = new CollectorRegistry(); + (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); + DefaultExports.register(registry); + _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); + } catch (ConfigurationException var4) { + System.err.println("Configuration Exception : " + var4.getMessage()); + System.exit(1); + } catch (IllegalArgumentException var5) { + System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage()); + System.exit(1); + } + + PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); @@ -225,4 +264,13 @@ private void assertMeterExportedCorrectly(String exportedMeterName) { private void assertMeterExportedCorrectly(String exportedMeterName, List labels) { assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX); } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } }