All test cases working

apache · Oct 16, 2024 · d581be8 · d581be8
1 parent d8f11b9
commit d581be8
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 8 deletions.
diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
@@ -52,7 +52,7 @@
               <value>false</value>
             </property>
           </properties>
-          <argLine>-javaagent:${project.build.directory}/agent/jmx_prometheus_javaagent-0.19.0.jar=9021:${project.basedir}/../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml</argLine>
+<!--          <argLine>-javaagent:${project.build.directory}/agent/jmx_prometheus_javaagent-0.19.0.jar=9021:${project.basedir}/../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml</argLine>-->
         </configuration>
       </plugin>
 

diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java
@@ -21,10 +21,14 @@
 
 import com.yammer.metrics.core.MetricsRegistry;
 import com.yammer.metrics.reporting.JmxReporter;
+import io.prometheus.jmx.JavaAgent;
+import java.net.URI;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
 import org.apache.pinot.common.utils.http.HttpClient;
 import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
 import org.apache.pinot.spi.env.PinotConfiguration;
@@ -43,8 +47,18 @@ public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest {
 
   private BrokerMetrics _brokerMetrics;
 
+  private int _exporterPort;
+
   @BeforeClass
-  public void setup() {
+  public void setup()
+      throws Exception {
+
+    _exporterPort = 9000 + new Random().nextInt(1000);
+    String agentArgs = String.format("%s:%s", _exporterPort,
+        "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml");
+
+    JavaAgent.agentmain(agentArgs, null);
+
     PinotConfiguration pinotConfiguration = new PinotConfiguration();
     pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME,
         "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory");
@@ -141,4 +155,13 @@ public void brokerMeterTest() {
       }
     });
   }
+
+  @Override
+  protected SimpleHttpResponse getExportedPromMetrics() {
+    try {
+      return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics"));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
 }
diff --git a/...-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/...-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java
@@ -21,14 +21,27 @@
 
 import com.yammer.metrics.core.MetricsRegistry;
 import com.yammer.metrics.reporting.JmxReporter;
+import io.prometheus.jmx.BuildInfoCollector;
 import io.prometheus.jmx.JavaAgent;
+import io.prometheus.jmx.JmxCollector;
+import io.prometheus.jmx.common.http.ConfigurationException;
+import io.prometheus.jmx.common.http.HTTPServerFactory;
+import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry;
+import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.helix.task.TaskState;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
 import org.apache.pinot.common.utils.http.HttpClient;
 import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
 import org.apache.pinot.spi.env.PinotConfiguration;
@@ -45,10 +58,33 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest {
   private static final String TASK_TYPE = "ClusterHealthCheck";
   private ControllerMetrics _controllerMetrics;
 
+  private int _exporterPort;
+
+  private HTTPServer _httpServer;
+
   @BeforeClass
   public void setup()
       throws Exception {
 
+    _exporterPort = 9000 + new Random().nextInt(1000);
+    String agentArgs = String.format("%s:%s", _exporterPort,
+        "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml");
+
+    String host = "0.0.0.0";
+
+    try {
+      JMXExporterConfig config = parseConfig(agentArgs, host);
+      CollectorRegistry registry = new CollectorRegistry();
+      (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry);
+      DefaultExports.register(registry);
+      _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file));
+    } catch (ConfigurationException var4) {
+      System.err.println("Configuration Exception : " + var4.getMessage());
+      System.exit(1);
+    } catch (IllegalArgumentException var5) {
+      System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]<port>:<yaml configuration file> " + var5.getMessage());
+      System.exit(1);
+    }
 
     PinotConfiguration pinotConfiguration = new PinotConfiguration();
     pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME,
@@ -219,4 +255,13 @@ private boolean isRemaining(ControllerGauge gauge, List<ControllerGauge> gaugesA
   private static String getStrippedMetricName(ControllerGauge controllerGauge) {
     return StringUtils.remove(controllerGauge.getGaugeName(), "controller");
   }
+
+  @Override
+  protected SimpleHttpResponse getExportedPromMetrics() {
+    try {
+      return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics"));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
 }
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java
@@ -23,6 +23,7 @@
 import com.yammer.metrics.reporting.JmxReporter;
 import java.util.List;
 import java.util.stream.Stream;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
 import org.apache.pinot.common.utils.http.HttpClient;
 import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
 import org.apache.pinot.spi.env.PinotConfiguration;
@@ -102,4 +103,9 @@ public void meterTest() {
   @Test
   public void gaugeTest() {
   }
+
+  @Override
+  protected SimpleHttpResponse getExportedPromMetrics() {
+    return null;
+  }
 }
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java
@@ -20,15 +20,19 @@
 package org.apache.pinot.common.metrics;
 
 import com.google.common.base.Objects;
+import io.prometheus.jmx.JavaAgent;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.StringReader;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pinot.common.utils.SimpleHttpResponse;
@@ -38,7 +42,7 @@
 import org.testng.Assert;
 
 
-public class PinotJMXToPromMetricsTest {
+public abstract class PinotJMXToPromMetricsTest {
 
   protected HttpClient _httpClient;
 
@@ -184,10 +188,7 @@ protected List<PromMetric> parseExportedPromMetrics(String response)
     return exportedPromMetrics;
   }
 
-  protected SimpleHttpResponse getExportedPromMetrics()
-      throws IOException, URISyntaxException {
-    return _httpClient.sendGetRequest(new URI("http://localhost:9021/metrics"));
-  }
+  protected abstract SimpleHttpResponse getExportedPromMetrics();
 
   public static class PromMetric {
     private final String _metricName;
@@ -278,4 +279,40 @@ public String toString() {
       return sb.toString();
     }
   }
+
+  public static JMXExporterConfig parseConfig(String args, String ifc) {
+    Pattern pattern = Pattern.compile("^(?:((?:[\\w.-]+)|(?:\\[.+])):)?(\\d{1,5}):(.+)");
+    Matcher matcher = pattern.matcher(args);
+    if (!matcher.matches()) {
+      throw new IllegalArgumentException("Malformed arguments - " + args);
+    } else {
+      String givenHost = matcher.group(1);
+      String givenPort = matcher.group(2);
+      String givenConfigFile = matcher.group(3);
+      int port = Integer.parseInt(givenPort);
+      InetSocketAddress socket;
+      if (givenHost != null && !givenHost.isEmpty()) {
+        socket = new InetSocketAddress(givenHost, port);
+      } else {
+        socket = new InetSocketAddress(ifc, port);
+        givenHost = ifc;
+      }
+
+      return new JMXExporterConfig(givenHost, port, givenConfigFile, socket);
+    }
+  }
+
+  public static class JMXExporterConfig {
+    String host;
+    int port;
+    String file;
+    InetSocketAddress socket;
+
+    JMXExporterConfig(String host, int port, String file, InetSocketAddress socket) {
+      this.host = host;
+      this.port = port;
+      this.file = file;
+      this.socket = socket;
+    }
+  }
 }
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java
@@ -21,10 +21,22 @@
 
 import com.yammer.metrics.core.MetricsRegistry;
 import com.yammer.metrics.reporting.JmxReporter;
+import io.prometheus.jmx.BuildInfoCollector;
+import io.prometheus.jmx.JavaAgent;
+import io.prometheus.jmx.JmxCollector;
+import io.prometheus.jmx.common.http.ConfigurationException;
+import io.prometheus.jmx.common.http.HTTPServerFactory;
+import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry;
+import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports;
+import java.io.File;
+import java.net.URI;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
 import org.apache.pinot.common.utils.http.HttpClient;
 import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry;
 import org.apache.pinot.spi.env.PinotConfiguration;
@@ -40,8 +52,35 @@ public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest {
   private static final String EXPORTED_METRIC_PREFIX = "pinot_server_";
   private ServerMetrics _serverMetrics;
 
+  private int _exporterPort;
+
+  private HTTPServer _httpServer;
+
   @BeforeClass
-  public void setup() {
+  public void setup()
+      throws Exception {
+
+    _exporterPort = 9000 + new Random().nextInt(1000);
+    String agentArgs = String.format("%s:%s", _exporterPort,
+        "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml");
+
+    String host = "0.0.0.0";
+
+    try {
+      JMXExporterConfig config = parseConfig(agentArgs, host);
+      CollectorRegistry registry = new CollectorRegistry();
+      (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry);
+      DefaultExports.register(registry);
+      _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file));
+    } catch (ConfigurationException var4) {
+      System.err.println("Configuration Exception : " + var4.getMessage());
+      System.exit(1);
+    } catch (IllegalArgumentException var5) {
+      System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]<port>:<yaml configuration file> " + var5.getMessage());
+      System.exit(1);
+    }
+
+
     PinotConfiguration pinotConfiguration = new PinotConfiguration();
     pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME,
         "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory");
@@ -225,4 +264,13 @@ private void assertMeterExportedCorrectly(String exportedMeterName) {
   private void assertMeterExportedCorrectly(String exportedMeterName, List<String> labels) {
     assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX);
   }
+
+  @Override
+  protected SimpleHttpResponse getExportedPromMetrics() {
+    try {
+      return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics"));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
 }