diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
index a5c1c84e261..799308ce3b6 100644
--- a/pinot-common/pom.xml
+++ b/pinot-common/pom.xml
@@ -52,7 +52,7 @@
false
- -javaagent:${project.build.directory}/agent/jmx_prometheus_javaagent-0.19.0.jar=9021:${project.basedir}/../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
+
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java
index c6100b73bf4..1a8f855a016 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java
@@ -21,10 +21,14 @@
import com.yammer.metrics.core.MetricsRegistry;
import com.yammer.metrics.reporting.JmxReporter;
+import io.prometheus.jmx.JavaAgent;
+import java.net.URI;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
import org.apache.pinot.common.utils.http.HttpClient;
import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -43,8 +47,18 @@ public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest {
private BrokerMetrics _brokerMetrics;
+ private int _exporterPort;
+
@BeforeClass
- public void setup() {
+ public void setup()
+ throws Exception {
+
+ _exporterPort = 9000 + new Random().nextInt(1000);
+ String agentArgs = String.format("%s:%s", _exporterPort,
+ "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml");
+
+ JavaAgent.agentmain(agentArgs, null);
+
PinotConfiguration pinotConfiguration = new PinotConfiguration();
pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME,
"org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory");
@@ -141,4 +155,13 @@ public void brokerMeterTest() {
}
});
}
+
+ @Override
+ protected SimpleHttpResponse getExportedPromMetrics() {
+ try {
+ return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics"));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java
index 130fe376a26..9e64cf3f4d8 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java
@@ -21,14 +21,27 @@
import com.yammer.metrics.core.MetricsRegistry;
import com.yammer.metrics.reporting.JmxReporter;
+import io.prometheus.jmx.BuildInfoCollector;
import io.prometheus.jmx.JavaAgent;
+import io.prometheus.jmx.JmxCollector;
+import io.prometheus.jmx.common.http.ConfigurationException;
+import io.prometheus.jmx.common.http.HTTPServerFactory;
+import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry;
+import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.helix.task.TaskState;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
import org.apache.pinot.common.utils.http.HttpClient;
import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -45,10 +58,33 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest {
private static final String TASK_TYPE = "ClusterHealthCheck";
private ControllerMetrics _controllerMetrics;
+ private int _exporterPort;
+
+ private HTTPServer _httpServer;
+
@BeforeClass
public void setup()
throws Exception {
+ _exporterPort = 9000 + new Random().nextInt(1000);
+ String agentArgs = String.format("%s:%s", _exporterPort,
+ "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml");
+
+ String host = "0.0.0.0";
+
+ try {
+ JMXExporterConfig config = parseConfig(agentArgs, host);
+ CollectorRegistry registry = new CollectorRegistry();
+ (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry);
+ DefaultExports.register(registry);
+ _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file));
+ } catch (ConfigurationException var4) {
+ System.err.println("Configuration Exception : " + var4.getMessage());
+ System.exit(1);
+ } catch (IllegalArgumentException var5) {
+ System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage());
+ System.exit(1);
+ }
PinotConfiguration pinotConfiguration = new PinotConfiguration();
pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME,
@@ -219,4 +255,13 @@ private boolean isRemaining(ControllerGauge gauge, List gaugesA
private static String getStrippedMetricName(ControllerGauge controllerGauge) {
return StringUtils.remove(controllerGauge.getGaugeName(), "controller");
}
+
+ @Override
+ protected SimpleHttpResponse getExportedPromMetrics() {
+ try {
+ return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics"));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java
index 34abd0f39b8..f0a416bfad2 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java
@@ -23,6 +23,7 @@
import com.yammer.metrics.reporting.JmxReporter;
import java.util.List;
import java.util.stream.Stream;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
import org.apache.pinot.common.utils.http.HttpClient;
import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -102,4 +103,9 @@ public void meterTest() {
@Test
public void gaugeTest() {
}
+
+ @Override
+ protected SimpleHttpResponse getExportedPromMetrics() {
+ return null;
+ }
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java
index 0a0c483f4d2..5566f6bd39c 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java
@@ -20,15 +20,19 @@
package org.apache.pinot.common.metrics;
import com.google.common.base.Objects;
+import io.prometheus.jmx.JavaAgent;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.utils.SimpleHttpResponse;
@@ -38,7 +42,7 @@
import org.testng.Assert;
-public class PinotJMXToPromMetricsTest {
+public abstract class PinotJMXToPromMetricsTest {
protected HttpClient _httpClient;
@@ -184,10 +188,7 @@ protected List parseExportedPromMetrics(String response)
return exportedPromMetrics;
}
- protected SimpleHttpResponse getExportedPromMetrics()
- throws IOException, URISyntaxException {
- return _httpClient.sendGetRequest(new URI("http://localhost:9021/metrics"));
- }
+ protected abstract SimpleHttpResponse getExportedPromMetrics();
public static class PromMetric {
private final String _metricName;
@@ -278,4 +279,40 @@ public String toString() {
return sb.toString();
}
}
+
+ public static JMXExporterConfig parseConfig(String args, String ifc) {
+ Pattern pattern = Pattern.compile("^(?:((?:[\\w.-]+)|(?:\\[.+])):)?(\\d{1,5}):(.+)");
+ Matcher matcher = pattern.matcher(args);
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Malformed arguments - " + args);
+ } else {
+ String givenHost = matcher.group(1);
+ String givenPort = matcher.group(2);
+ String givenConfigFile = matcher.group(3);
+ int port = Integer.parseInt(givenPort);
+ InetSocketAddress socket;
+ if (givenHost != null && !givenHost.isEmpty()) {
+ socket = new InetSocketAddress(givenHost, port);
+ } else {
+ socket = new InetSocketAddress(ifc, port);
+ givenHost = ifc;
+ }
+
+ return new JMXExporterConfig(givenHost, port, givenConfigFile, socket);
+ }
+ }
+
+ public static class JMXExporterConfig {
+ String host;
+ int port;
+ String file;
+ InetSocketAddress socket;
+
+ JMXExporterConfig(String host, int port, String file, InetSocketAddress socket) {
+ this.host = host;
+ this.port = port;
+ this.file = file;
+ this.socket = socket;
+ }
+ }
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java
index cd047125703..40d366327af 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java
@@ -21,10 +21,22 @@
import com.yammer.metrics.core.MetricsRegistry;
import com.yammer.metrics.reporting.JmxReporter;
+import io.prometheus.jmx.BuildInfoCollector;
+import io.prometheus.jmx.JavaAgent;
+import io.prometheus.jmx.JmxCollector;
+import io.prometheus.jmx.common.http.ConfigurationException;
+import io.prometheus.jmx.common.http.HTTPServerFactory;
+import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry;
+import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports;
+import java.io.File;
+import java.net.URI;
import java.util.Arrays;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
import org.apache.pinot.common.utils.http.HttpClient;
import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -40,8 +52,35 @@ public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest {
private static final String EXPORTED_METRIC_PREFIX = "pinot_server_";
private ServerMetrics _serverMetrics;
+ private int _exporterPort;
+
+ private HTTPServer _httpServer;
+
@BeforeClass
- public void setup() {
+ public void setup()
+ throws Exception {
+
+ _exporterPort = 9000 + new Random().nextInt(1000);
+ String agentArgs = String.format("%s:%s", _exporterPort,
+ "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml");
+
+ String host = "0.0.0.0";
+
+ try {
+ JMXExporterConfig config = parseConfig(agentArgs, host);
+ CollectorRegistry registry = new CollectorRegistry();
+ (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry);
+ DefaultExports.register(registry);
+ _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file));
+ } catch (ConfigurationException var4) {
+ System.err.println("Configuration Exception : " + var4.getMessage());
+ System.exit(1);
+ } catch (IllegalArgumentException var5) {
+ System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage());
+ System.exit(1);
+ }
+
+
PinotConfiguration pinotConfiguration = new PinotConfiguration();
pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME,
"org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory");
@@ -225,4 +264,13 @@ private void assertMeterExportedCorrectly(String exportedMeterName) {
private void assertMeterExportedCorrectly(String exportedMeterName, List labels) {
assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX);
}
+
+ @Override
+ protected SimpleHttpResponse getExportedPromMetrics() {
+ try {
+ return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics"));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}