This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 709abb0 Adding Additional JMX Metrics to Prometheus config (#7079)
709abb0 is described below
commit 709abb01dabe8487856c1c2a2cffb01795f2a00c
Author: Tim Santos <t...@cortexdata.io>
AuthorDate: Tue Jun 22 15:24:45 2021 -0700
Adding Additional JMX Metrics to Prometheus config (#7079)
* Adding prometheus configs in progress
* Adding additional metrics to Prometheus config
* Adding README and removing redundant metrics in config.
---
.../pinot/etc/jmx_prometheus_javaagent/README.md | 22 +++++++
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 72 +++++++++++++++++++++-
2 files changed, 93 insertions(+), 1 deletion(-)
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/README.md
b/docker/images/pinot/etc/jmx_prometheus_javaagent/README.md
new file mode 100644
index 0000000..18708bd
--- /dev/null
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/README.md
@@ -0,0 +1,22 @@
+#### What is Prometheus?
+
+Prometheus is an open-source event monitoring system. It records real-time
metrics in a time series database and can be queried with its own query
language PromQL. Prometheus has 4 metric types (Gauge, Counter, Histogram,
Summary).
+
+Prometheus is not a full-fledged dashboarding solution and needs to be hooked
up with Grafana to generate dashboards.
+
+#### How do Pinot metrics end up in Prometheus?
+
+Currently, Pinot metrics are exposed as JMX mbeans through the
PinotJmxReporter. These JMX mbeans are consumed by Prometheus using the
[Prometheus JMX Exporter](https://github.com/prometheus/jmx_exporter). A fairly
comprehensive Prometheus JMX Exporter config can be found under
`docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml`.
+
+See the [Pinot
docs](https://docs.pinot.apache.org/operators/operating-pinot/monitoring) for
more info.
+
+#### How can I view and test metrics?
+
+First, you need to make sure the metrics are exposed though JMX. Note, that if
no metric has been published (e.g. no values recorded), the metric will not
show up in JMX or Prometheus server.
+With a local Pinot deployment, you can launch `jconsole`, select your local
deployment and view all the metrics exposed as jmx mbeans. To see if the
metrics are being consumed with the Prometheus JMX Exporter and your config
file, you can set the JAVA_OPTS env variable before running Pinot locally.
+
+`export
JAVA_OPTS="-javaagent:jmx_prometheus_javaagent-0.12.0.jar=8080:pinot.yml -Xms4G
-Xmx4G -XX:MaxDirectMemorySize=30g
-Dlog4j2.configurationFile=conf/pinot-admin-log4j2.xml
-Dplugins.dir=$BASEDIR/plugins"
+bin/pinot-admin.sh ....
+`
+
+This will expose a port at 8080 to dump metrics as Prometheus format for
Prometheus scraper to fetch.
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
index 9b38cc9..8850d67 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
@@ -77,7 +77,29 @@ rules:
labels:
taskType: "$1"
status: "$2"
-# Pinot Broker
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.pinotLeadControllerResourceEnabled\"><>(\\w+)"
+ name: "pinot_controller_pinotLeadControllerResourceEnabled_$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.offlineTableEstimatedSize.(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_offlineTableEstimatedSize_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.tableQuota.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_tableQuota_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.tableStorageQuotaUtilization.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_tableStorageQuotaUtilization_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.tableStorageEstMissingSegmentPercent.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_tableStorageEstMissingSegmentPercent_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+
+ # Pinot Broker
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+).authorization\"><>(\\w+)"
name: "pinot_broker_authorization_$2"
labels:
@@ -152,6 +174,36 @@ rules:
name: "pinot_broker_exceptions_$1_$2"
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.routingTableUpdateTime\"><>(\\w+)"
name: "pinot_broker_routingTableUpdateTime_$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+)\\.brokerResponsesWithPartialServersResponded\"><>(\\w+)"
+ name: "pinot_broker_brokerResponsesWithPartialServersResponded_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+)\\.brokerResponsesWithNumGroupsLimitReached\"><>(\\w+)"
+ name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+)\\.requestConnectionTimeouts\"><>(\\w+)"
+ name: "pinot_broker_requestConnectionTimeouts_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+)\\.queryQuotaExceeded\"><>(\\w+)"
+ name: "pinot_broker_queryQuotaExceeded_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+)_(\\w+).serverMissingForRouting\"><>(\\w+)"
+ name: "pinot_broker_serverMissingForRouting_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+)\\.deserialization\"><>(\\w+)"
+ name: "pinot_broker_deserialization_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+)\\.requestConnectionWait\"><>(\\w+)"
+ name: "pinot_broker_requestConnectionWait_$2"
+ labels:
+ table: "$1"
+
# Pinot Server
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\",
name=\"pinot.server.documentCount.(\\w+)_(\\w+)\"><>(\\w+)"
name: "pinot_server_documentCount_$3"
@@ -231,6 +283,24 @@ rules:
name: "pinot_server_netty_tcp_$2_$3"
labels:
id: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\",
name=\"pinot.server.nettyConnection(\\w+)\"><>(\\w+)"
+ name: "pinot_server_nettyConnection_$1_$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\",
name=\"pinot.server.realtimeSegmentNumPartitions.(\\w+)\"><>(\\w+)"
+ name: "pinot_server_realtimeSegmentNumPartitions_$2"
+ labels:
+ table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\",
name=\"pinot.server.resizeTimeMs.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_server_resizeTimeMs_$3"
+ labels:
+ table: "$1"
+ tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ServerMetrics\",
name=\"pinot.server.upsertPrimaryKeysCount.(\\w+)_(\\w+).(\\w+)\"><>(\\w+)"
+ name: "pinot_server_upsertPrimaryKeysCount_$4"
+ labels:
+ table: "$1"
+ tableType: "$2"
+ partition: "$3"
+
# Pinot Minions
- pattern: "\"org.apache.pinot.minion.metrics\"<type=\"MinionMetrics\",
name=\"pinot.minion.(\\w+)\"><>(\\w+)"
name: "pinot_minion_$1_$2"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
