This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 09e88120c4 Test Cases for JMX -> Prom Exporter Regexps (#14155)
09e88120c4 is described below
commit 09e88120c42d70346636d534749ef28ccd20d39d
Author: Prashant Pandey <84911643+suddend...@users.noreply.github.com>
AuthorDate: Wed Oct 23 22:01:06 2024 +0530
Test Cases for JMX -> Prom Exporter Regexps (#14155)
---
.../jmx_prometheus_javaagent/configs/server.yml | 14 +-
pinot-common/pom.xml | 16 +
.../prometheus/BrokerPrometheusMetricsTest.java | 130 ++++++
.../ControllerPrometheusMetricsTest.java | 197 +++++++++
.../prometheus/MinionPrometheusMetricsTest.java | 128 ++++++
.../prometheus/PinotPrometheusMetricsTest.java | 469 +++++++++++++++++++++
.../prometheus/ServerPrometheusMetricsTest.java | 196 +++++++++
.../DropwizardBrokerPrometheusMetricsTest.java | 43 ++
.../DropwizardControllerPrometheusMetricsTest.java | 44 ++
.../DropwizardMinionPrometheusMetricsTest.java | 44 ++
.../DropwizardServerPrometheusMetricsTest.java | 45 ++
.../yammer/YammerBrokerPrometheusMetricsTest.java | 38 ++
.../YammerControllerPrometheusMetricsTest.java | 38 ++
.../yammer/YammerMinionPrometheusMetricsTest.java | 38 ++
.../yammer/YammerServerPrometheusMetricsTest.java | 38 ++
pom.xml | 8 +
16 files changed, 1485 insertions(+), 1 deletion(-)
diff --git
a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml
b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml
index 780c9fffc5..8751bfa517 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml
@@ -94,7 +94,7 @@ rules:
- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ServerMetrics\",
name=\"pinot\\.server\\.realtimeConsumptionExceptions\"><>(\\w+)"
name: "pinot_server_realtime_consumptionExceptions_$1"
cache: true
-- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ServerMetrics\",
name=\"pinot\\.server\\.(([^.]+)\\.)?([^.]*)_(OFFLINE|REALTIME)\\-(.+)\\-(\\w+)\\.(invalidRealtimeRowsDropped|incompleteRealtimeRowsConsumed|rowsWithErrors|realtimeRowsFiltered|realtimeRowsConsumed|realtimeRowsFetched|streamConsumerCreateExceptions)\"><>(\\w+)"
+- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ServerMetrics\",
name=\"pinot\\.server\\.(([^.]+)\\.)?([^.]*)_(OFFLINE|REALTIME)\\-(.+)\\-(\\w+)\\.(invalidRealtimeRowsDropped|incompleteRealtimeRowsConsumed|rowsWithErrors|realtimeRowsFiltered|realtimeRowsConsumed|realtimeRowsFetched|streamConsumerCreateExceptions|realtimeRowsSanitized)\"><>(\\w+)"
name: "pinot_server_$7_$8"
cache: true
labels:
@@ -132,6 +132,18 @@ rules:
labels:
database: "$2"
table: "$1$3"
+- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ServerMetrics\",
name=\"pinot\\.server\\.luceneIndexingDelayMs\\.(([^.]+)\\.)?([^.]*)\"><>(\\w+)"
+ name: "pinot_server_luceneIndexingDelayMs_$4"
+ cache: true
+ labels:
+ database: "$2"
+ table: "$1$3"
+- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ServerMetrics\",
name=\"pinot\\.server\\.luceneIndexingDelayDocs\\.(([^.]+)\\.)?([^.]*)\"><>(\\w+)"
+ name: "pinot_server_luceneIndexingDelayDocs_$4"
+ cache: true
+ labels:
+ database: "$2"
+ table: "$1$3"
- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ServerMetrics\",
name=\"pinot\\.server\\.numResizes\\.(([^.]+)\\.)?([^.]*)_(OFFLINE|REALTIME)\"><>(\\w+)"
name: "pinot_server_numResizes_$5"
cache: true
diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
index af34dd8173..af2001a9e1 100644
--- a/pinot-common/pom.xml
+++ b/pinot-common/pom.xml
@@ -52,6 +52,13 @@
<value>false</value> <!-- This will disable all default
listeners -->
</property>
</properties>
+ <!--Disabling tests as Pinot currently uses Yammer and these tests
fail for DropwizardMetricsFactory-->
+ <excludes>
+ <exclude>**/DropwizardBrokerPrometheusMetricsTest.java</exclude>
+ <exclude>**/DropwizardServerPrometheusMetricsTest.java</exclude>
+ <exclude>**/DropwizardMinionPrometheusMetricsTest.java</exclude>
+
<exclude>**/DropwizardControllerPrometheusMetricsTest.java</exclude>
+ </excludes>
</configuration>
</plugin>
@@ -107,6 +114,10 @@
</plugins>
</build>
<dependencies>
+ <dependency>
+ <groupId>io.prometheus.jmx</groupId>
+ <artifactId>jmx_prometheus_javaagent</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-segment-spi</artifactId>
@@ -269,6 +280,11 @@
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-dropwizard</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>net.sourceforge.fmpp</groupId>
<artifactId>fmpp</artifactId>
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..399e5b400b
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.metrics.prometheus;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.common.metrics.BrokerGauge;
+import org.apache.pinot.common.metrics.BrokerMeter;
+import org.apache.pinot.common.metrics.BrokerMetrics;
+import org.apache.pinot.common.metrics.BrokerTimer;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+public abstract class BrokerPrometheusMetricsTest extends
PinotPrometheusMetricsTest {
+
+ private static final String EXPORTED_METRIC_PREFIX = "pinot_broker_";
+
+ private static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions";
+
+ private static final List<BrokerMeter> GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX =
+ List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS,
BrokerMeter.UNCAUGHT_POST_EXCEPTIONS,
+ BrokerMeter.QUERY_REJECTED_EXCEPTIONS,
BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS,
+ BrokerMeter.RESOURCE_MISSING_EXCEPTIONS);
+
+ private static final List<BrokerMeter> METERS_ACCEPTING_RAW_TABLENAME =
+ List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS,
BrokerMeter.DOCUMENTS_SCANNED,
+ BrokerMeter.ENTRIES_SCANNED_IN_FILTER,
BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS,
+ BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED,
+ BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS,
+ BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED,
BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS,
+ BrokerMeter.ENTRIES_SCANNED_POST_FILTER,
BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE,
+ BrokerMeter.QUERY_QUOTA_EXCEEDED);
+
+ private BrokerMetrics _brokerMetrics;
+
+ @BeforeClass
+ public void setup()
+ throws Exception {
+ _brokerMetrics = new
BrokerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry());
+ }
+
+ @Test(dataProvider = "brokerTimers")
+ public void timerTest(BrokerTimer timer) {
+ if (timer.isGlobal()) {
+ _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS);
+ assertTimerExportedCorrectly(timer.getTimerName(),
EXPORTED_METRIC_PREFIX);
+ } else {
+
_brokerMetrics.addTimedTableValue(PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME,
timer, 30_000L,
+ TimeUnit.MILLISECONDS);
+ assertTimerExportedCorrectly(timer.getTimerName(),
PinotPrometheusMetricsTest.ExportedLabels.TABLENAME,
+ EXPORTED_METRIC_PREFIX);
+ }
+ }
+
+ @Test(dataProvider = "brokerGauges")
+ public void gaugeTest(BrokerGauge gauge) {
+ if (gauge.isGlobal()) {
+ _brokerMetrics.setOrUpdateGlobalGauge(gauge, () -> 5L);
+ assertGaugeExportedCorrectly(gauge.getGaugeName(),
EXPORTED_METRIC_PREFIX);
+ } else {
+ if (gauge == BrokerGauge.REQUEST_SIZE) {
+
_brokerMetrics.setOrUpdateTableGauge(PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME,
gauge, 5L);
+ assertGaugeExportedCorrectly(gauge.getGaugeName(),
PinotPrometheusMetricsTest.ExportedLabels.TABLENAME,
+ EXPORTED_METRIC_PREFIX);
+ } else {
+ _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L);
+ assertGaugeExportedCorrectly(gauge.getGaugeName(),
+ PinotPrometheusMetricsTest.ExportedLabels.TABLENAME_TABLETYPE,
EXPORTED_METRIC_PREFIX);
+ }
+ }
+ }
+
+ @Test(dataProvider = "brokerMeters")
+ public void meterTest(BrokerMeter meter) {
+ if (meter.isGlobal()) {
+ _brokerMetrics.addMeteredGlobalValue(meter, 5L);
+ if (GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX.contains(meter)) {
+ String exportedMeterPrefix = String.format("%s_%s",
EXPORTED_METRIC_PREFIX_EXCEPTIONS,
+ StringUtils.remove(meter.getMeterName(), "Exceptions"));
+ assertMeterExportedCorrectly(exportedMeterPrefix,
EXPORTED_METRIC_PREFIX);
+ } else {
+ assertMeterExportedCorrectly(meter.getMeterName(),
EXPORTED_METRIC_PREFIX);
+ }
+ } else {
+ if (METERS_ACCEPTING_RAW_TABLENAME.contains(meter)) {
+
_brokerMetrics.addMeteredTableValue(PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME,
meter, 5L);
+ assertMeterExportedCorrectly(meter.getMeterName(),
PinotPrometheusMetricsTest.ExportedLabels.TABLENAME,
+ EXPORTED_METRIC_PREFIX);
+ } else {
+ _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L);
+ assertMeterExportedCorrectly(meter.getMeterName(),
+ PinotPrometheusMetricsTest.ExportedLabels.TABLENAME_TABLETYPE,
EXPORTED_METRIC_PREFIX);
+ }
+ }
+ }
+
+ @DataProvider(name = "brokerTimers")
+ public Object[] brokerTimers() {
+ return BrokerTimer.values();
+ }
+
+ @DataProvider(name = "brokerMeters")
+ public Object[] brokerMeters() {
+ return BrokerMeter.values();
+ }
+
+ @DataProvider(name = "brokerGauges")
+ public Object[] brokerGauges() {
+ return BrokerGauge.values();
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..7fcb76eae1
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.metrics.prometheus;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.helix.task.TaskState;
+import org.apache.pinot.common.metrics.ControllerGauge;
+import org.apache.pinot.common.metrics.ControllerMeter;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.common.metrics.ControllerTimer;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+public abstract class ControllerPrometheusMetricsTest extends
PinotPrometheusMetricsTest {
+ //all exported controller metrics have this prefix
+ private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_";
+ private static final String LABEL_KEY_TASK_TYPE = "taskType";
+
+ //that accept global gauge with suffix
+ private static final List<ControllerGauge> GLOBAL_GAUGES_ACCEPTING_TASKTYPE =
+ List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS,
ControllerGauge.NUM_MINION_SUBTASKS_RUNNING,
+ ControllerGauge.NUM_MINION_SUBTASKS_WAITING,
ControllerGauge.NUM_MINION_SUBTASKS_ERROR,
+ ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE,
ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR);
+
+ //local gauges that accept partition
+ private static final List<ControllerGauge> GAUGES_ACCEPTING_PARTITION =
+ List.of(ControllerGauge.MAX_RECORDS_LAG,
ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS);
+
+ //these accept task type
+ private static final List<ControllerGauge> GAUGES_ACCEPTING_TASKTYPE =
+ List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE,
+ ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION,
+ ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR);
+
+ private static final List<ControllerGauge> GAUGES_ACCEPTING_RAW_TABLENAME =
+ List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE);
+
+ private ControllerMetrics _controllerMetrics;
+
+ @BeforeClass
+ public void setup()
+ throws Exception {
+ _controllerMetrics = new
ControllerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry());
+ }
+
+ @Test(dataProvider = "controllerTimers")
+ public void timerTest(ControllerTimer controllerTimer) {
+ if (controllerTimer.isGlobal()) {
+ _controllerMetrics.addTimedValue(controllerTimer, 30_000,
TimeUnit.MILLISECONDS);
+ assertTimerExportedCorrectly(controllerTimer.getTimerName(),
EXPORTED_METRIC_PREFIX);
+ } else {
+ _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE,
controllerTimer, 30_000L, TimeUnit.MILLISECONDS);
+ _controllerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME,
controllerTimer, 30_000L,
+ TimeUnit.MILLISECONDS);
+
+ assertTimerExportedCorrectly(controllerTimer.getTimerName(),
ExportedLabels.TABLENAME_TABLETYPE,
+ EXPORTED_METRIC_PREFIX);
+ assertTimerExportedCorrectly(controllerTimer.getTimerName(),
ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX);
+ }
+ }
+
+ @Test(dataProvider = "controllerMeters")
+ public void meterTest(ControllerMeter meter) {
+ if (meter.isGlobal()) {
+ _controllerMetrics.addMeteredGlobalValue(meter, 5L);
+ String meterName = meter.getMeterName();
+ //some meters contain a "controller" prefix. For example,
controllerInstancePostError. These meters are
+ // exported as 'pinot_controller_pinot_controller_InstancePostError'. So
we strip the 'controller' from
+ // 'controllerInstancePostError'
+ String strippedMeterName = StringUtils.remove(meterName, "controller");
+ assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX);
+ } else {
+
+ String meterName = meter.getMeterName();
+ String strippedMeterName = StringUtils.remove(meterName, "controller");
+
+ if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) {
+ addMeterWithLabels(meter,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC);
+ assertMeterExportedCorrectly(meterName,
+ List.of(ExportedLabelKeys.TABLE,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX);
+ } else if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) {
+ addMeterWithLabels(meter,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC);
+ assertMeterExportedCorrectly(
+ String.format("%s_%s", strippedMeterName,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC),
+ EXPORTED_METRIC_PREFIX);
+ } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) {
+ addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE + "." +
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC);
+ assertMeterExportedCorrectly(meterName,
ExportedLabels.CONTROLLER_TASKTYPE_TABLENAME_TABLETYPE,
+ EXPORTED_METRIC_PREFIX);
+ } else {
+ addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE);
+ addMeterWithLabels(meter, ExportedLabelValues.TABLENAME);
+ if (meter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) {
+ assertMeterExportedCorrectly(meterName,
ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX);
+ assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME,
EXPORTED_METRIC_PREFIX);
+ } else {
+ assertMeterExportedCorrectly(strippedMeterName,
ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX);
+ assertMeterExportedCorrectly(strippedMeterName,
ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX);
+ }
+ }
+ }
+ }
+
+ @Test(dataProvider = "controllerGauges")
+ public void gaugeTest(ControllerGauge controllerGauge) {
+ if (controllerGauge.isGlobal()) {
+ _controllerMetrics.setValueOfGlobalGauge(controllerGauge,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L);
+ //some global gauges also accept the taskType (which should not be).
todo: this should be fixed
+ if (GLOBAL_GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) {
+ _controllerMetrics.setValueOfGlobalGauge(controllerGauge,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L);
+ String strippedMetricName = getStrippedMetricName(controllerGauge);
+ assertGaugeExportedCorrectly(strippedMetricName,
+ List.of(LABEL_KEY_TASK_TYPE,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX);
+ } else {
+ _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L);
+ String strippedMetricName = getStrippedMetricName(controllerGauge);
+ assertGaugeExportedCorrectly(strippedMetricName,
EXPORTED_METRIC_PREFIX);
+ }
+ } else {
+ if (GAUGES_ACCEPTING_PARTITION.contains(controllerGauge)) {
+ _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3,
controllerGauge, 10L);
+ String strippedGaugeName = getStrippedMetricName(controllerGauge);
+ assertGaugeExportedCorrectly(strippedGaugeName,
ExportedLabels.PARTITION_TABLENAME_TABLETYPE,
+ EXPORTED_METRIC_PREFIX);
+ } else if (GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) {
+ _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC,
+ controllerGauge, () -> 50L);
+ assertGaugeExportedCorrectly(controllerGauge.getGaugeName(),
+ ExportedLabels.TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE,
EXPORTED_METRIC_PREFIX);
+ } else if (GAUGES_ACCEPTING_RAW_TABLENAME.contains(controllerGauge)) {
+ addGaugeWithLabels(controllerGauge, ExportedLabelValues.TABLENAME);
+ assertGaugeExportedCorrectly(controllerGauge.getGaugeName(),
ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX);
+ } else if (controllerGauge ==
ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED) {
+ addGaugeWithLabels(controllerGauge,
+ String.format("%s.%s", TABLE_NAME_WITH_TYPE,
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC));
+
assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(),
+ ExportedLabels.TABLENAMEWITHTYPE_CONTROLLER_TASKTYPE,
EXPORTED_METRIC_PREFIX);
+ } else if (controllerGauge == ControllerGauge.TASK_STATUS) {
+ addGaugeWithLabels(controllerGauge,
+ String.format("%s.%s",
ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, TaskState.IN_PROGRESS));
+
assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(),
+ ExportedLabels.JOBSTATUS_CONTROLLER_TASKTYPE,
EXPORTED_METRIC_PREFIX);
+ } else {
+ addGaugeWithLabels(controllerGauge, TABLE_NAME_WITH_TYPE);
+ assertGaugeExportedCorrectly(controllerGauge.getGaugeName(),
ExportedLabels.TABLENAME_TABLETYPE,
+ EXPORTED_METRIC_PREFIX);
+ }
+ }
+ }
+
+ private void addGaugeWithLabels(ControllerGauge gauge, String labels) {
+ _controllerMetrics.setValueOfTableGauge(labels, gauge, 5L);
+ }
+
+ private static String getStrippedMetricName(ControllerGauge controllerGauge)
{
+ return StringUtils.remove(controllerGauge.getGaugeName(), "controller");
+ }
+
+ private void addMeterWithLabels(ControllerMeter meter, String labels) {
+ _controllerMetrics.addMeteredTableValue(labels, meter, 1L);
+ }
+
+ @DataProvider(name = "controllerTimers")
+ public Object[] controllerTimers() {
+ return ControllerTimer.values();
+ }
+
+ @DataProvider(name = "controllerMeters")
+ public Object[] controllerMeters() {
+ return ControllerMeter.values();
+ }
+
+ @DataProvider(name = "controllerGauges")
+ public Object[] controllerGauges() {
+ return ControllerGauge.values();
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java
new file mode 100644
index 0000000000..35a766b0ca
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.metrics.prometheus;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.common.metrics.MinionGauge;
+import org.apache.pinot.common.metrics.MinionMeter;
+import org.apache.pinot.common.metrics.MinionMetrics;
+import org.apache.pinot.common.metrics.MinionTimer;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+public abstract class MinionPrometheusMetricsTest extends
PinotPrometheusMetricsTest {
+ //all exported minion metrics have this prefix
+ private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_";
+ private static final String METER_PREFIX_NO_TASKS = "numberTasks";
+
+ private MinionMetrics _minionMetrics;
+
+ @BeforeClass
+ public void setup() {
+ _minionMetrics = new
MinionMetrics(_pinotMetricsFactory.getPinotMetricsRegistry());
+ }
+
+ @Test(dataProvider = "minionTimers")
+ public void timerTest(MinionTimer timer) {
+
+
_minionMetrics.addTimedValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT,
timer, 30L, TimeUnit.MILLISECONDS);
+ assertTimerExportedCorrectly(timer.getTimerName(),
+ List.of(ExportedLabelKeys.ID,
ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX);
+
+ _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE,
ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, timer, 30L,
+ TimeUnit.MILLISECONDS);
+
+ if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) {
+ assertTimerExportedCorrectly(timer.getTimerName(),
+ List.of(ExportedLabelKeys.DATABASE,
ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME, ExportedLabelKeys.TABLE,
+ "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX);
+ } else {
+ assertTimerExportedCorrectly(timer.getTimerName(),
ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE,
+ EXPORTED_METRIC_PREFIX);
+ }
+ }
+
+ @Test(dataProvider = "minionMeters")
+ public void meterTest(MinionMeter meter) {
+ if (meter.isGlobal()) {
+ validateGlobalMeters(meter);
+ } else {
+ validateMetersWithLabels(meter);
+ }
+ }
+
+ private void validateGlobalMeters(MinionMeter meter) {
+ _minionMetrics.addMeteredGlobalValue(meter, 5L);
+ assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX);
+ }
+
+ private void validateMetersWithLabels(MinionMeter meter) {
+ if (meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)) {
+ _minionMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME,
meter, 1L);
+ assertMeterExportedCorrectly(meter.getMeterName(),
List.of(ExportedLabelKeys.ID, ExportedLabelValues.TABLENAME),
+ EXPORTED_METRIC_PREFIX);
+
+
_minionMetrics.addMeteredValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT,
meter, 1L);
+ assertMeterExportedCorrectly(meter.getMeterName(),
+ List.of(ExportedLabelKeys.ID,
ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX);
+ } else if (meter == MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT || meter ==
MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT) {
+
+ _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L);
+ assertMeterExportedCorrectly(meter.getMeterName(),
List.of(ExportedLabelKeys.ID, TABLE_NAME_WITH_TYPE),
+ EXPORTED_METRIC_PREFIX);
+ } else {
+ //all remaining meters are also being used as global meters, check their
usage
+ _minionMetrics.addMeteredGlobalValue(meter, 1L);
+ _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE,
ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter,
+ 1L);
+ assertMeterExportedCorrectly(meter.getMeterName(),
EXPORTED_METRIC_PREFIX);
+ assertMeterExportedCorrectly(meter.getMeterName(),
ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE,
+ EXPORTED_METRIC_PREFIX);
+ }
+ }
+
+ @Test(dataProvider = "minionGauges")
+ public void gaugeTest(MinionGauge gauge) {
+ if (gauge.isGlobal()) {
+ _minionMetrics.setValueOfGlobalGauge(gauge, 1L);
+ assertGaugeExportedCorrectly(gauge.getGaugeName(),
EXPORTED_METRIC_PREFIX);
+ } else {
+ _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L);
+ assertGaugeExportedCorrectly(gauge.getGaugeName(),
ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX);
+ }
+ }
+
+ @DataProvider(name = "minionTimers")
+ public Object[] minionTimers() {
+ return MinionTimer.values();
+ }
+
+ @DataProvider(name = "minionMeters")
+ public Object[] minionMeters() {
+ return MinionMeter.values();
+ }
+
+ @DataProvider(name = "minionGauges")
+ public Object[] minionGauges() {
+ return MinionGauge.values();
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java
new file mode 100644
index 0000000000..a3f21ad91d
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java
@@ -0,0 +1,469 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.metrics.prometheus;
+
+import com.google.common.base.Objects;
+import com.google.common.io.Resources;
+import io.prometheus.jmx.JmxCollector;
+import io.prometheus.jmx.common.http.HTTPServerFactory;
+import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry;
+import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
+import org.apache.pinot.common.utils.http.HttpClient;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.metrics.PinotMetricUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+
+import static
org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelKeys.*;
+import static
org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC;
+import static
org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.IN_PROGRESS;
+import static
org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME;
+import static
org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.TABLETYPE_REALTIME;
+import static
org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME;
+
+
+public abstract class PinotPrometheusMetricsTest {
+
+ protected static final String TABLE_NAME_WITH_TYPE =
+
TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(ExportedLabelValues.TABLENAME);
+ protected static final String KAFKA_TOPIC = "myTopic";
+ protected static final String PARTITION_GROUP_ID = "partitionGroupId";
+ protected static final String CLIENT_ID =
+ String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC,
PARTITION_GROUP_ID);
+
+ protected HttpClient _httpClient;
+
+ protected PinotMetricsFactory _pinotMetricsFactory;
+
+ //each meter defined in code is exported with these measurements
+ private static final List<String> METER_TYPES =
+ List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate",
"FifteenMinuteRate");
+
+ //each timer defined in code is exported with these measurements
+ private static final List<String> TIMER_TYPES =
+ List.of("Count", "FiveMinuteRate", "Max", "999thPercentile",
"95thPercentile", "75thPercentile", "98thPercentile",
+ "OneMinuteRate", "50thPercentile", "99thPercentile",
"FifteenMinuteRate", "Mean", "StdDev", "MeanRate",
+ "Min");
+
+ //each gauge defined in code is exported with these measurements
+ private static final List<String> GAUGE_TYPES = List.of("Value");
+
+ private HTTPServer _httpServer;
+
+ @BeforeClass
+ public void setupTest() {
+ PinotConfiguration pinotConfiguration = new PinotConfiguration();
+
+ _pinotMetricsFactory = getPinotMetricsFactory();
+ pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME,
+ _pinotMetricsFactory.getClass().getCanonicalName());
+ PinotMetricUtils.init(pinotConfiguration);
+
+
_pinotMetricsFactory.makePinotJmxReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start();
+ _httpClient = new HttpClient();
+ _httpServer = startExporter();
+ }
+
+ @AfterClass
+ public void cleanup() {
+ _httpServer.close();
+ }
+
+ /**
+ * Pinot currently uses the JMX->Prom exporter to export metrics to
Prometheus. Normally, this runs as an agent in the
+ * JVM. In this case however, we've got tests using four different config
files (server.yml, broker.yml,
+ * controller.yml and minion.yml). Loading the same agent in the same JVM
multiple times isn't allowed, we are copying
+ * the agent's code to some degree and starting up the HTTP servers
manually. For impl, see:
+ * <a
href="https://github.com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216";>...</a>
+ *
/jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48
+ *
+ * @return the corresponding HTTP server on a random unoccupied port
+ */
+ protected HTTPServer startExporter() {
+ String args = String.format("%s:%s", 0, getConfigFile());
+ try {
+ JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0");
+ CollectorRegistry registry = new CollectorRegistry();
+ JmxCollector jmxCollector = new JmxCollector(new File(config._file),
JmxCollector.Mode.AGENT);
+ jmxCollector.register(registry);
+ DefaultExports.register(registry);
+ return (new HTTPServerFactory()).createHTTPServer(config._socket,
registry, true, new File(config._file));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix,
String exportedMetricPrefix) {
+ List<PromMetric> promMetrics;
+ try {
+ promMetrics =
parseExportedPromMetrics(getExportedPromMetrics().getResponse());
+ for (String gaugeType : GAUGE_TYPES) {
+ PromMetric expectedMetric = PromMetric.withName(exportedMetricPrefix +
exportedGaugePrefix + "_" + gaugeType);
+ Assert.assertTrue(promMetrics.contains(expectedMetric),
+ "Cannot find gauge: " + expectedMetric + " in exported metrics");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix,
List<String> labels,
+ String exportedMetricPrefix) {
+ List<PromMetric> promMetrics;
+ try {
+ promMetrics =
parseExportedPromMetrics(getExportedPromMetrics().getResponse());
+ for (String gaugeType : GAUGE_TYPES) {
+ PromMetric expectedGauge =
+ PromMetric.withNameAndLabels(exportedMetricPrefix +
exportedGaugePrefix + "_" + gaugeType, labels);
+ Assert.assertTrue(promMetrics.contains(expectedGauge),
+ "Cannot find gauge: " + expectedGauge + " in exported metrics");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void assertTimerExportedCorrectly(String exportedTimerPrefix,
String exportedMetricPrefix) {
+ List<PromMetric> promMetrics;
+ try {
+ promMetrics =
parseExportedPromMetrics(getExportedPromMetrics().getResponse());
+ for (String meterType : TIMER_TYPES) {
+ PromMetric expectedTimer = PromMetric.withName(exportedMetricPrefix +
exportedTimerPrefix + "_" + meterType);
+ Assert.assertTrue(promMetrics.contains(expectedTimer),
+ "Cannot find timer: " + expectedTimer + " in exported metrics");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void assertTimerExportedCorrectly(String exportedTimerPrefix,
List<String> labels,
+ String exportedMetricPrefix) {
+ List<PromMetric> promMetrics;
+ try {
+ promMetrics =
parseExportedPromMetrics(getExportedPromMetrics().getResponse());
+ for (String meterType : METER_TYPES) {
+ Assert.assertTrue(promMetrics.contains(
+ PromMetric.withNameAndLabels(exportedMetricPrefix +
exportedTimerPrefix + "_" + meterType, labels)),
+ exportedTimerPrefix);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void assertMeterExportedCorrectly(String exportedMeterPrefix,
String exportedMetricPrefix) {
+ List<PromMetric> promMetrics;
+ try {
+ promMetrics =
parseExportedPromMetrics(getExportedPromMetrics().getResponse());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ for (String meterType : METER_TYPES) {
+ PromMetric expectedMetric = PromMetric.withName(exportedMetricPrefix +
exportedMeterPrefix + "_" + meterType);
+ Assert.assertTrue(promMetrics.contains(expectedMetric),
+ "Cannot find metric: " + expectedMetric + " in the exported
metrics");
+ }
+ }
+
+ protected void assertMeterExportedCorrectly(String exportedMeterPrefix,
List<String> labels,
+ String exportedMetricPrefix) {
+ List<PromMetric> promMetrics;
+ try {
+ promMetrics =
parseExportedPromMetrics(getExportedPromMetrics().getResponse());
+ for (String meterType : METER_TYPES) {
+ PromMetric expectedMetric =
+ PromMetric.withNameAndLabels(exportedMetricPrefix +
exportedMeterPrefix + "_" + meterType, labels);
+ Assert.assertTrue(promMetrics.contains(expectedMetric),
+ "Cannot find metric: " + expectedMetric + " in the exported
metrics");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected List<PromMetric> parseExportedPromMetrics(String response)
+ throws IOException {
+
+ List<PromMetric> exportedPromMetrics = new ArrayList<>();
+
+ try (BufferedReader reader = new BufferedReader(new
StringReader(response))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (line.startsWith("pinot_")) {
+ exportedPromMetrics.add(PromMetric.fromExportedMetric(line));
+ }
+ }
+ reader.close();
+ return exportedPromMetrics;
+ }
+ }
+
+ protected SimpleHttpResponse getExportedPromMetrics() {
+ try {
+ return _httpClient.sendGetRequest(new URI("http://localhost:"; +
_httpServer.getPort() + "/metrics"));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected abstract PinotMetricsFactory getPinotMetricsFactory();
+
+ protected abstract String getConfigFile();
+
+ /*
+ Implementation copied from: https://github
+
.com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216/jmx_prometheus_javaagent/src/main/java
+ /io/prometheus/jmx/JavaAgent.java#L88
+ */
+ private static JMXExporterConfig parseExporterConfig(String args, String
ifc) {
+ Pattern pattern =
Pattern.compile("^(?:((?:[\\w.-]+)|(?:\\[.+])):)?(\\d{1,5}):(.+)");
+ Matcher matcher = pattern.matcher(args);
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Malformed arguments - " + args);
+ } else {
+ String givenHost = matcher.group(1);
+ String givenPort = matcher.group(2);
+ String givenConfigFile = matcher.group(3);
+ int port = Integer.parseInt(givenPort);
+ InetSocketAddress socket;
+ if (givenHost != null && !givenHost.isEmpty()) {
+ socket = new InetSocketAddress(givenHost, port);
+ } else {
+ socket = new InetSocketAddress(ifc, port);
+ givenHost = ifc;
+ }
+
+ return new JMXExporterConfig(givenHost, port, givenConfigFile, socket);
+ }
+ }
+
+ private static class JMXExporterConfig {
+ String _host;
+ int _port;
+ String _file;
+ InetSocketAddress _socket;
+
+ JMXExporterConfig(String host, int port, String file, InetSocketAddress
socket) {
+ _host = host;
+ _port = port;
+ _file = file;
+ _socket = socket;
+ }
+ }
+
+ private String loadResourceAsString(String resourceFileName) {
+ URL url = Resources.getResource(resourceFileName);
+ try {
+ return Resources.toString(url, StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static class ExportedLabels {
+
+ public static final List<String> TABLENAME = List.of(TABLE,
ExportedLabelValues.TABLENAME);
+ public static final List<String> TABLENAME_TABLETYPE =
+ List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE,
TABLETYPE_REALTIME);
+ public static final List<String> PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC =
+ List.of(PARTITION, PARTITION_GROUP_ID, TABLE,
ExportedLabelValues.TABLENAME, TABLETYPE,
+ TableType.REALTIME.toString(), TOPIC, KAFKA_TOPIC);
+ public static final List<String> PARTITION_TABLENAME_TABLETYPE =
+ List.of(PARTITION, "3", TABLE, ExportedLabelValues.TABLENAME,
TABLETYPE, TableType.REALTIME.toString());
+
+ public static final List<String> TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE =
+ List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE,
TABLETYPE_REALTIME, TASKTYPE,
+ CONTROLLER_PERIODIC_TASK_CHC);
+
+ public static final List<String> TABLENAMEWITHTYPE_CONTROLLER_TASKTYPE =
+ List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE,
CONTROLLER_PERIODIC_TASK_CHC);
+
+ public static final List<String> JOBSTATUS_CONTROLLER_TASKTYPE =
+ List.of(STATUS, IN_PROGRESS, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC);
+
+ public static final List<String> CONTROLLER_TASKTYPE_TABLENAME_TABLETYPE =
+ List.of(PERIODIC_TASK, CONTROLLER_PERIODIC_TASK_CHC, TABLE,
ExportedLabelValues.TABLENAME, TABLETYPE,
+ TABLETYPE_REALTIME);
+
+ public static final List<String> TABLENAME_TABLETYPE_MINION_TASKTYPE =
+ List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.TABLENAME,
ExportedLabelKeys.TABLETYPE,
+ ExportedLabelValues.TABLETYPE_REALTIME, ExportedLabelKeys.TASKTYPE,
+ ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT);
+ }
+
+ public static class ExportedLabelKeys {
+ public static final String TABLE = "table";
+ public static final String ID = "id";
+ public static final String TABLETYPE = "tableType";
+ public static final String TOPIC = "topic";
+ public static final String PARTITION = "partition";
+ public static final String TASKTYPE = "taskType";
+ public static final String PERIODIC_TASK = "periodicTask";
+ public static final String STATUS = "status";
+ public static final String DATABASE = "database";
+ }
+
+ public static class ExportedLabelValues {
+ public static final String TABLENAME = "myTable";
+ public static final String TABLETYPE_REALTIME = "REALTIME";
+ public static final String TABLENAME_WITH_TYPE_REALTIME =
+
TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(TABLENAME);
+ public static final String CONTROLLER_PERIODIC_TASK_CHC =
"ClusterHealthCheck";
+ public static final String MINION_TASK_SEGMENT_IMPORT =
"SegmentImportTask";
+ public static final String IN_PROGRESS = "IN_PROGRESS";
+ }
+
+ /*
+ * Represents an exported Prometheus metric. A Prom metric looks like:
+ * pinot_server_realtimeRowsSanitized_Count{app="pinot",
cluster_name="pinot",
+ * component="pinot-server", component_name="server-default-tenant-1"}
+ */
+ private static class PromMetric {
+ private final String _metricName;
+ private final Map<String, String> _labels;
+
+ public String getMetricName() {
+ return _metricName;
+ }
+
+ public Map<String, String> getLabels() {
+ return _labels;
+ }
+
+ //make constructor private so that it can be instantiated only with the
factory methods
+ private PromMetric(String metricName, Map<String, String> labels) {
+ _metricName = metricName;
+ _labels = labels;
+ }
+
+ /**
+ * Create an instance of {@link PromMetric} from an exported Prometheus
metric
+ *
+ * @param exportedMetric the exported Prom metric (name + labels)
+ * @return the corresponding {@link PromMetric}
+ */
+ public static PromMetric fromExportedMetric(String exportedMetric) {
+ int spaceIndex = exportedMetric.indexOf(' ');
+ String metricWithoutVal = exportedMetric.substring(0, spaceIndex);
+ int braceIndex = metricWithoutVal.indexOf('{');
+
+ if (braceIndex != -1) {
+ String metricName = metricWithoutVal.substring(0, braceIndex);
+ String labelsString = metricWithoutVal.substring(braceIndex + 1,
metricWithoutVal.lastIndexOf('}'));
+ Map<String, String> labels = parseLabels(labelsString);
+ return new PromMetric(metricName, labels);
+ } else {
+ return new PromMetric(metricWithoutVal, new LinkedHashMap<>());
+ }
+ }
+
+ /**
+ * Creates an instance of {@link PromMetric} with a name and an empty
label list
+ *
+ * @param metricName the metric name
+ * @return the corresponding PromMetric
+ */
+ public static PromMetric withName(String metricName) {
+ return new PromMetric(metricName, new LinkedHashMap<>());
+ }
+
+ /**
+ * Creates an instance of {@link PromMetric} with a name and an label list
+ *
+ * @param metricName the metric name
+ * @param labels the labels
+ * @return the corresponding PromMetric
+ */
+ public static PromMetric withNameAndLabels(String metricName, List<String>
labels) {
+ Map<String, String> labelMap = new LinkedHashMap<>();
+ for (int i = 0; i < labels.size(); i += 2) {
+ labelMap.put(labels.get(i), labels.get(i + 1));
+ }
+ return new PromMetric(metricName, labelMap);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PromMetric that = (PromMetric) o;
+ return metricNamesAreSimilar(that) && Objects.equal(_labels,
that._labels);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(_metricName, _labels);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(_metricName);
+ if (!_labels.isEmpty()) {
+ sb.append('{');
+ sb.append(_labels.entrySet().stream().map(e -> e.getKey() + "=\"" +
e.getValue() + "\"")
+ .collect(Collectors.joining(",")));
+ sb.append('}');
+ }
+ return sb.toString();
+ }
+
+ private boolean metricNamesAreSimilar(PromMetric that) {
+ String processedMetricNameThis = StringUtils.remove(_metricName, "_");
+ String processedMetricNameThat = StringUtils.remove(that._metricName,
"_");
+ return StringUtils.equalsIgnoreCase(processedMetricNameThis,
processedMetricNameThat);
+ }
+
+ private static Map<String, String> parseLabels(String labelsString) {
+ return labelsString.isEmpty() ? new LinkedHashMap<>()
+ : java.util.Arrays.stream(labelsString.split(",")).map(kvPair ->
kvPair.split("="))
+ .collect(Collectors.toMap(kv -> kv[0], kv ->
removeQuotes(kv[1]), (v1, v2) -> v2, LinkedHashMap::new));
+ }
+
+ private static String removeQuotes(String value) {
+ return value.startsWith("\"") ? value.substring(1, value.length() - 1) :
value;
+ }
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..366d538ec1
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.metrics.prometheus;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.common.metrics.ServerGauge;
+import org.apache.pinot.common.metrics.ServerMeter;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.metrics.ServerTimer;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+public abstract class ServerPrometheusMetricsTest extends
PinotPrometheusMetricsTest {
+ //all exported server metrics have this prefix
+ private static final String EXPORTED_METRIC_PREFIX = "pinot_server_";
+
+ private static final List<ServerMeter> METERS_ACCEPTING_CLIENT_ID =
+ List.of(ServerMeter.REALTIME_ROWS_CONSUMED,
ServerMeter.REALTIME_ROWS_SANITIZED,
+ ServerMeter.REALTIME_ROWS_FETCHED,
ServerMeter.REALTIME_ROWS_FILTERED,
+ ServerMeter.INVALID_REALTIME_ROWS_DROPPED,
ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED,
+ ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS,
ServerMeter.ROWS_WITH_ERRORS);
+
+ private static final List<ServerMeter> METERS_ACCEPTING_RAW_TABLE_NAMES =
+ List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE,
ServerMeter.SEGMENT_UPLOAD_SUCCESS,
+ ServerMeter.SEGMENT_UPLOAD_TIMEOUT);
+
+ //gauges that accept clientId
+ private static final List<ServerGauge> GAUGES_ACCEPTING_CLIENT_ID =
+ List.of(ServerGauge.LLC_PARTITION_CONSUMING,
ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED,
+ ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS,
+ ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS,
+
ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS,
+ ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS,
+ ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS);
+
+ private static final List<ServerGauge> GAUGES_ACCEPTING_PARTITION =
+ List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT,
ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT,
+ ServerGauge.REALTIME_INGESTION_OFFSET_LAG,
ServerGauge.REALTIME_INGESTION_DELAY_MS,
+ ServerGauge.UPSERT_PRIMARY_KEYS_COUNT,
ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS,
+ ServerGauge.DEDUP_PRIMARY_KEYS_COUNT);
+
+ private static final List<ServerGauge> GAUGES_ACCEPTING_RAW_TABLE_NAME =
+ List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED,
ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS,
+ ServerGauge.LUCENE_INDEXING_DELAY_MS,
ServerGauge.LUCENE_INDEXING_DELAY_DOCS);
+
+ private ServerMetrics _serverMetrics;
+
+ @BeforeClass
+ public void setup()
+ throws Exception {
+ _serverMetrics = new
ServerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry());
+ }
+
+ @Test(dataProvider = "serverTimers")
+ public void timerTest(ServerTimer serverTimer) {
+ if (serverTimer.isGlobal()) {
+ _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS);
+ assertTimerExportedCorrectly(serverTimer.getTimerName(),
EXPORTED_METRIC_PREFIX);
+ } else {
+ _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer,
30_000L, TimeUnit.MILLISECONDS);
+ _serverMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME,
serverTimer, 30_000L, TimeUnit.MILLISECONDS);
+
+ assertTimerExportedCorrectly(serverTimer.getTimerName(),
ExportedLabels.TABLENAME_TABLETYPE,
+ EXPORTED_METRIC_PREFIX);
+ assertTimerExportedCorrectly(serverTimer.getTimerName(),
ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX);
+ }
+ }
+
+ @Test(dataProvider = "serverMeters")
+ public void meterTest(ServerMeter serverMeter) {
+ if (serverMeter.isGlobal()) {
+ _serverMetrics.addMeteredGlobalValue(serverMeter, 4L);
+ //we cannot use raw meter names for all meters as exported metrics don't
follow any convention currently.
+ // For example, meters that track realtime exceptions start with prefix
"realtime_exceptions"
+ if (meterTrackingRealtimeExceptions(serverMeter)) {
+
assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter));
+ } else {
+ assertMeterExportedCorrectly(serverMeter.getMeterName());
+ }
+ } else {
+ if (METERS_ACCEPTING_CLIENT_ID.contains(serverMeter)) {
+ addMeterWithLabels(serverMeter, CLIENT_ID);
+ assertMeterExportedCorrectly(serverMeter.getMeterName(),
+ ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC);
+ } else if (METERS_ACCEPTING_RAW_TABLE_NAMES.contains(serverMeter)) {
+ addMeterWithLabels(serverMeter, ExportedLabelValues.TABLENAME);
+ assertMeterExportedCorrectly(serverMeter.getMeterName(),
ExportedLabels.TABLENAME);
+ } else {
+ //we pass tableNameWithType to all remaining meters
+ addMeterWithLabels(serverMeter, TABLE_NAME_WITH_TYPE);
+ assertMeterExportedCorrectly(serverMeter.getMeterName(),
ExportedLabels.TABLENAME_TABLETYPE);
+ }
+ }
+ }
+
+ @Test(dataProvider = "serverGauges")
+ public void gaugeTest(ServerGauge serverGauge) {
+ if (serverGauge.isGlobal()) {
+ _serverMetrics.setValueOfGlobalGauge(serverGauge, 10L);
+ assertGaugeExportedCorrectly(serverGauge.getGaugeName(),
EXPORTED_METRIC_PREFIX);
+ } else {
+ if (serverGauge == ServerGauge.DEDUP_PRIMARY_KEYS_COUNT) {
+ //this gauge is currently exported as:
`pinot_server_${partitionId}_Value{database="dedupPrimaryKeysCount",
+ // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We
add an explicit test for it to maintain
+ // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT
should be moved to
+ // gaugesThatAcceptPartition. It should be exported as:
+ // `pinot_server_dedupPrimaryKeysCount_Value{partition="3",
table="myTable",tableType="REALTIME",}`
+ addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE);
+ assertGaugeExportedCorrectly(String.valueOf(3),
+ List.of(ExportedLabelKeys.DATABASE, serverGauge.getGaugeName(),
ExportedLabelKeys.TABLE,
+ "dedupPrimaryKeysCount.myTable", ExportedLabelKeys.TABLETYPE,
ExportedLabelValues.TABLETYPE_REALTIME),
+ EXPORTED_METRIC_PREFIX);
+ } else if (GAUGES_ACCEPTING_CLIENT_ID.contains(serverGauge)) {
+ addGaugeWithLabels(serverGauge, CLIENT_ID);
+ assertGaugeExportedCorrectly(serverGauge.getGaugeName(),
+ ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC,
EXPORTED_METRIC_PREFIX);
+ } else if (GAUGES_ACCEPTING_PARTITION.contains(serverGauge)) {
+ addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE);
+ assertGaugeExportedCorrectly(serverGauge.getGaugeName(),
ExportedLabels.PARTITION_TABLENAME_TABLETYPE,
+ EXPORTED_METRIC_PREFIX);
+ } else if (GAUGES_ACCEPTING_RAW_TABLE_NAME.contains(serverGauge)) {
+ addGaugeWithLabels(serverGauge, ExportedLabelValues.TABLENAME);
+ assertGaugeExportedCorrectly(serverGauge.getGaugeName(),
ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX);
+ } else {
+ addGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE);
+ assertGaugeExportedCorrectly(serverGauge.getGaugeName(),
ExportedLabels.TABLENAME_TABLETYPE,
+ EXPORTED_METRIC_PREFIX);
+ }
+ }
+ }
+
+ private void addGaugeWithLabels(ServerGauge serverGauge, String labels) {
+ _serverMetrics.setValueOfTableGauge(labels, serverGauge, 100L);
+ }
+
+ private void addPartitionGaugeWithLabels(ServerGauge serverGauge, String
labels) {
+ _serverMetrics.setValueOfPartitionGauge(labels, 3, serverGauge, 100L);
+ }
+
+ public void addMeterWithLabels(ServerMeter serverMeter, String labels) {
+ _serverMetrics.addMeteredTableValue(labels, serverMeter, 4L);
+ }
+
+ @DataProvider(name = "serverTimers")
+ public Object[] serverTimers() {
+ return ServerTimer.values(); // Provide all values of ServerTimer enum
+ }
+
+ @DataProvider(name = "serverMeters")
+ public Object[] serverMeter() {
+ return ServerMeter.values(); // Provide all values of ServerTimer enum
+ }
+
+ @DataProvider(name = "serverGauges")
+ public Object[] serverGauge() {
+ return ServerGauge.values(); // Provide all values of ServerTimer enum
+ }
+
+ private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) {
+ return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS
+ || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS
+ || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS ||
serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS;
+ }
+
+ private String getRealtimeExceptionMeterName(ServerMeter serverMeter) {
+ String meterName = serverMeter.getMeterName();
+ return "realtime_exceptions_" + meterName.substring(0,
meterName.lastIndexOf("Exceptions"));
+ }
+
+ private void assertMeterExportedCorrectly(String exportedMeterName) {
+ assertMeterExportedCorrectly(exportedMeterName, EXPORTED_METRIC_PREFIX);
+ }
+
+ private void assertMeterExportedCorrectly(String exportedMeterName,
List<String> labels) {
+ assertMeterExportedCorrectly(exportedMeterName, labels,
EXPORTED_METRIC_PREFIX);
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..8d1abd44b2
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.dropwizard;
+
+import org.apache.pinot.common.metrics.prometheus.BrokerPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+import org.testng.annotations.Test;
+
+
+/**
+ * Disabling tests as Pinot currently uses Yammer and these tests fail for for
{@link DropwizardMetricsFactory}
+ */
+@Test(enabled = false)
+public class DropwizardBrokerPrometheusMetricsTest extends
BrokerPrometheusMetricsTest {
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new DropwizardMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ //todo: return the correct dir once this test is enabled
+ return null;
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..005d83466b
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.dropwizard;
+
+import
org.apache.pinot.common.metrics.prometheus.ControllerPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+import org.testng.annotations.Test;
+
+
+/**
+ * Disabling tests as Pinot currently uses Yammer and these tests fail for for
{@link DropwizardMetricsFactory}
+ */
+@Test(enabled = false)
+public class DropwizardControllerPrometheusMetricsTest extends
ControllerPrometheusMetricsTest {
+
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new DropwizardMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ //todo: return the correct dir once this test is enabled
+ return null;
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java
new file mode 100644
index 0000000000..08183428a8
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.dropwizard;
+
+import org.apache.pinot.common.metrics.prometheus.MinionPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+import org.testng.annotations.Test;
+
+
+/**
+ * Disabling tests as Pinot currently uses Yammer and these tests fail for for
{@link DropwizardMetricsFactory}
+ */
+@Test(enabled = false)
+public class DropwizardMinionPrometheusMetricsTest extends
MinionPrometheusMetricsTest {
+
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new DropwizardMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ //todo: return the correct dir once this test is enabled
+ return null;
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..9fc5bf4b96
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.dropwizard;
+
+import org.apache.pinot.common.metrics.prometheus.ServerPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory;
+import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+import org.testng.annotations.Test;
+
+
+/**
+ * Disabling tests as Pinot currently uses Yammer and these tests fail for for
{@link DropwizardMetricsFactory}
+ */
+@Test(enabled = false)
+public class DropwizardServerPrometheusMetricsTest extends
ServerPrometheusMetricsTest {
+
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new YammerMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ //todo: return the correct dir once this test is enabled
+ return null;
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..d58a3fdd3c
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.yammer;
+
+import org.apache.pinot.common.metrics.prometheus.BrokerPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+
+
+public class YammerBrokerPrometheusMetricsTest extends
BrokerPrometheusMetricsTest {
+
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new YammerMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ return
"../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml";
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..e7f7f11dc8
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.yammer;
+
+import
org.apache.pinot.common.metrics.prometheus.ControllerPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+
+
+public class YammerControllerPrometheusMetricsTest extends
ControllerPrometheusMetricsTest {
+
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new YammerMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ return
"../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml";
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java
new file mode 100644
index 0000000000..520c809e7a
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.yammer;
+
+import org.apache.pinot.common.metrics.prometheus.MinionPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+
+
+public class YammerMinionPrometheusMetricsTest extends
MinionPrometheusMetricsTest {
+
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new YammerMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ return
"../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml";
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java
new file mode 100644
index 0000000000..e32228716d
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.common.metrics.prometheus.yammer;
+
+import org.apache.pinot.common.metrics.prometheus.ServerPrometheusMetricsTest;
+import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory;
+import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory;
+
+
+public class YammerServerPrometheusMetricsTest extends
ServerPrometheusMetricsTest {
+
+ @Override
+ protected PinotMetricsFactory getPinotMetricsFactory() {
+ return new YammerMetricsFactory();
+ }
+
+ @Override
+ protected String getConfigFile() {
+ return
"../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml";
+ }
+}
diff --git a/pom.xml b/pom.xml
index f5d381a119..e00cfae2f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -801,6 +801,14 @@
</exclusions>
</dependency>
+ <!-- JMX exporter-->
+ <dependency>
+ <groupId>io.prometheus.jmx</groupId>
+ <artifactId>jmx_prometheus_javaagent</artifactId>
+ <version>0.19.0</version>
+ <scope>test</scope>
+ </dependency>
+
<!-- netty BOM -->
<dependency>
<groupId>io.netty</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
