This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a14e95e4eeb [feature](metrics) add catalog/db/table num metrics
(#47891)
a14e95e4eeb is described below
commit a14e95e4eeb3143bf40e002f54dedabae25b7f5f
Author: Mingyu Chen (Rayner) <morning...@163.com>
AuthorDate: Sat Feb 15 21:09:12 2025 +0800
[feature](metrics) add catalog/db/table num metrics (#47891)
### What problem does this PR solve?
Add FE metrics for
1. num of catalogs
2. num of internal databases
3. num of internal tables
```
# HELP doris_fe_catalog_num total catalog num
# TYPE doris_fe_catalog_num gauge
doris_fe_catalog_num 1
# HELP doris_fe_internal_database_num total internal database num
# TYPE doris_fe_internal_database_num gauge
doris_fe_internal_database_num 4
# HELP doris_fe_internal_table_num total internal table num
# TYPE doris_fe_internal_table_num gauge
doris_fe_internal_table_num 44
```
---
.../java/org/apache/doris/catalog/Database.java | 4 ++
.../org/apache/doris/datasource/CatalogMgr.java | 4 ++
.../apache/doris/datasource/InternalCatalog.java | 4 ++
.../java/org/apache/doris/metric/MetricRepo.java | 34 +++++++++++++++++
.../java/org/apache/doris/metric/MetricsTest.java | 43 ++++++++++++++++++++++
5 files changed, 89 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 4f205be0381..4a6bc249b4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -1016,4 +1016,8 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>,
public String toString() {
return toJson();
}
+
+ public int getTableNum() {
+ return idToTable.size();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index b3d561e518f..80c568b0b56 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -895,4 +895,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
public Set<CatalogIf> getCopyOfCatalog() {
return new HashSet<>(idToCatalog.values());
}
+
+ public int getCatalogNum() {
+ return idToCatalog.size();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index de8f4684572..60fbec9630b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -266,6 +266,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
return Lists.newArrayList(idToDb.keySet());
}
+ public int getDbNum() {
+ return idToDb.size();
+ }
+
@Nullable
@Override
public Database getDbNullable(String dbName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index 9705e6a0542..4f64588b7cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -19,6 +19,7 @@ package org.apache.doris.metric;
import org.apache.doris.alter.Alter;
import org.apache.doris.alter.AlterJobV2.JobType;
+import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.system.CloudSystemInfoService;
import org.apache.doris.common.Config;
@@ -151,6 +152,11 @@ public final class MetricRepo {
public static Histogram HISTO_COMMIT_AND_PUBLISH_LATENCY;
+ // Catlaog/Database/Table num
+ public static GaugeMetric<Integer> GAUGE_CATALOG_NUM;
+ public static GaugeMetric<Integer> GAUGE_INTERNAL_DATABASE_NUM;
+ public static GaugeMetric<Integer> GAUGE_INTERNAL_TABLE_NUM;
+
private static Map<Pair<EtlJobType, JobState>, Long> loadJobNum =
Maps.newHashMap();
private static ScheduledThreadPoolExecutor metricTimer =
ThreadPoolManager.newDaemonScheduledThreadPool(1,
@@ -566,6 +572,34 @@ public final class MetricRepo {
HISTO_COMMIT_AND_PUBLISH_LATENCY = METRIC_REGISTER.histogram(
MetricRegistry.name("txn_commit_and_publish", "latency",
"ms"));
+ GAUGE_CATALOG_NUM = new GaugeMetric<Integer>("catalog_num",
+ MetricUnit.NOUNIT, "total catalog num") {
+ @Override
+ public Integer getValue() {
+ return Env.getCurrentEnv().getCatalogMgr().getCatalogNum();
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_CATALOG_NUM);
+
+ GAUGE_INTERNAL_DATABASE_NUM = new
GaugeMetric<Integer>("internal_database_num",
+ MetricUnit.NOUNIT, "total internal database num") {
+ @Override
+ public Integer getValue() {
+ return
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog().getDbNum();
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_INTERNAL_DATABASE_NUM);
+
+ GAUGE_INTERNAL_TABLE_NUM = new
GaugeMetric<Integer>("internal_table_num",
+ MetricUnit.NOUNIT, "total internal table num") {
+ @Override
+ public Integer getValue() {
+ return
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog().getAllDbs().stream()
+ .map(d -> (Database)
d).map(Database::getTableNum).reduce(0, Integer::sum);
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_INTERNAL_TABLE_NUM);
+
// init system metrics
initSystemMetrics();
CloudMetrics.init();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
index 5b8c4bdd728..3ce3b475153 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.metric;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.JsonUtil;
+import org.apache.doris.metric.Metric.MetricUnit;
import org.apache.doris.monitor.jvm.JvmService;
import org.apache.doris.monitor.jvm.JvmStats;
@@ -122,4 +123,46 @@ public class MetricsTest {
Assert.assertTrue(size.get() <
JsonUtil.parseArray(finalMetricJson).size());
}
+
+ @Test
+ public void testCatalogAndDatabaseMetrics() {
+ List<Metric> catalogMetrics =
MetricRepo.getMetricsByName("catalog_num");
+ Assert.assertEquals(1, catalogMetrics.size());
+ GaugeMetric<Integer> catalogMetric = (GaugeMetric<Integer>)
catalogMetrics.get(0);
+ Assert.assertEquals("catalog_num", catalogMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, catalogMetric.getUnit());
+ Assert.assertEquals("total catalog num",
catalogMetric.getDescription());
+
+ List<Metric> dbMetrics =
MetricRepo.getMetricsByName("internal_database_num");
+ Assert.assertEquals(1, dbMetrics.size());
+ GaugeMetric<Integer> dbMetric = (GaugeMetric<Integer>)
dbMetrics.get(0);
+ Assert.assertEquals("internal_database_num", dbMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, dbMetric.getUnit());
+ Assert.assertEquals("total internal database num",
dbMetric.getDescription());
+
+ List<Metric> tableMetrics =
MetricRepo.getMetricsByName("internal_table_num");
+ Assert.assertEquals(1, tableMetrics.size());
+ GaugeMetric<Integer> tableMetric = (GaugeMetric<Integer>)
tableMetrics.get(0);
+ Assert.assertEquals("internal_table_num", tableMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, tableMetric.getUnit());
+ Assert.assertEquals("total internal table num",
tableMetric.getDescription());
+
+ // Test metrics in Prometheus format
+ MetricVisitor visitor = new PrometheusMetricVisitor();
+ MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
+ String metricResult = visitor.finish();
+
+ Assert.assertTrue(metricResult.contains("# TYPE doris_fe_catalog_num
gauge"));
+ Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_internal_database_num gauge"));
+ Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_internal_table_num gauge"));
+
+ // Test metrics in JSON format
+ JsonMetricVisitor jsonVisitor = new JsonMetricVisitor();
+ MetricRepo.DORIS_METRIC_REGISTER.accept(jsonVisitor);
+ String jsonResult = jsonVisitor.finish();
+
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_catalog_num\""));
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_internal_database_num\""));
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_internal_table_num\""));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
