This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new f541d6fbe [AMORO-405]: Added overview page in AMS dashboard (#3031)
f541d6fbe is described below
commit f541d6fbed9d4e4b42bd190db4f4287a638a2bf4
Author: Marig_Weizhi <[email protected]>
AuthorDate: Thu Sep 12 19:30:07 2024 +0800
[AMORO-405]: Added overview page in AMS dashboard (#3031)
* add overview page
* Update the style of SingleDataCard MultipleDataCard
* Implementing cards basic styles
* add OverviewController
* get sum of tables sizes
* add pieChartCard、TableHealthCard、UserHistoryCard
* update the visual design of the overview page
* format code
* add mock data for Optimizing Status, Summary Cards, Table Format
* get overview summary
* add overview optimizing status and overview format api
* fix memory unit
* fix imports
* add listTableMetas method to TableService
* front-end of the UnhealthTablesCard and OperationsCard
* Formatting Code，i18
* add unhealthTables and latestOperations API
* update unhealth tables api and format code
* restore pnpm-lock.yaml
* Optimize and clean up code
* implements OverviewMetricsReporter
* move OverviewMetricsReporter to ams-server
* fix unit and i18 bug
* fix unit bug
* fix overview getTableFormat bug
* add resource api
* added idle&committing for OptimizerGroupMetrics
* update design for overview page
* update resource usage api
* implement overviewSummary/dataSize api
* fixed unit bug
* move overview page
* add overview cache unit test
* update format and i18n
* show health score on overview page
* Remove introduce page,and format code
---------
Co-authored-by: ZhouJinsong <[email protected]>
---
.../amoro/server/dashboard/DashboardServer.java | 14 +
.../amoro/server/dashboard/OverviewCache.java | 302 +++++++++++++++++++++
.../server/dashboard/OverviewMetricsReporter.java | 78 ++++++
.../dashboard/controller/OverviewController.java | 130 +++++++++
.../dashboard/model/OverviewDataSizeItem.java | 52 ++++
.../dashboard/model/OverviewResourceUsageItem.java | 66 +++++
.../server/dashboard/model/OverviewSummary.java | 92 +++++++
.../dashboard/model/OverviewTopTableItem.java | 86 ++++++
.../server/optimizing/OptimizerGroupMetrics.java | 30 ++
.../amoro/server/table/TableSummaryMetrics.java | 2 +-
.../org.apache.amoro.metrics.MetricReporter | 19 ++
.../amoro/server/dashboard/TestOverviewCache.java | 154 +++++++++++
.../server/optimizing/TestOptimizingQueue.java | 18 ++
.../server/table/TestTableSummaryMetrics.java | 2 +-
amoro-web/mock/modules/overview.js | 103 ++++---
amoro-web/src/components/Sidebar.vue | 13 +-
amoro-web/src/components/echarts/index.ts | 2 +
amoro-web/src/language/en.ts | 10 +
amoro-web/src/language/zh.ts | 10 +
amoro-web/src/router/index.ts | 10 +-
amoro-web/src/services/overview.service.ts | 58 ++++
amoro-web/src/types/common.type.ts | 35 ++-
amoro-web/src/views/introduce/index-bakl.vue | 230 ----------------
amoro-web/src/views/introduce/index.vue | 185 -------------
.../src/views/overview/components/DataSizeCard.vue | 121 +++++++++
.../views/overview/components/MultipleDataCard.vue | 53 ++++
.../src/views/overview/components/PieChartCard.vue | 81 ++++++
.../overview/components/ResourceUsageCard.vue | 119 ++++++++
.../views/overview/components/SingleDataCard.vue | 45 +++
.../views/overview/components/Top10TablesCard.vue | 181 ++++++++++++
amoro-web/src/views/overview/index.vue | 82 ++++++
.../amoro-bin/conf/plugins/metric-reporters.yaml | 15 +-
docs/user-guides/metrics.md | 22 +-
33 files changed, 1935 insertions(+), 485 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
index edf91aece..2e865974f 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
@@ -39,6 +39,7 @@ import
org.apache.amoro.server.dashboard.controller.CatalogController;
import org.apache.amoro.server.dashboard.controller.HealthCheckController;
import org.apache.amoro.server.dashboard.controller.LoginController;
import org.apache.amoro.server.dashboard.controller.OptimizerController;
+import org.apache.amoro.server.dashboard.controller.OverviewController;
import org.apache.amoro.server.dashboard.controller.PlatformFileInfoController;
import org.apache.amoro.server.dashboard.controller.SettingController;
import org.apache.amoro.server.dashboard.controller.TableController;
@@ -82,6 +83,7 @@ public class DashboardServer {
private final TableController tableController;
private final TerminalController terminalController;
private final VersionController versionController;
+ private final OverviewController overviewController;
private final String authType;
private final String basicAuthUser;
@@ -103,6 +105,7 @@ public class DashboardServer {
this.tableController = new TableController(tableService, tableDescriptor,
serviceConfig);
this.terminalController = new TerminalController(terminalManager);
this.versionController = new VersionController();
+ this.overviewController = new OverviewController();
this.authType =
serviceConfig.get(AmoroManagementConf.HTTP_SERVER_REST_AUTH_TYPE);
this.basicAuthUser = serviceConfig.get(AmoroManagementConf.ADMIN_USERNAME);
@@ -327,6 +330,17 @@ public class DashboardServer {
// version api
get("/versionInfo", versionController::getVersionInfo);
+
+ // overview apis
+ path(
+ "/overview",
+ () -> {
+ get("/summary", overviewController::getSummary);
+ get("/resource", overviewController::getResourceUsageHistory);
+ get("/optimizing", overviewController::getOptimizingStatus);
+ get("/dataSize", overviewController::getDataSizeHistory);
+ get("/top", overviewController::getTopTables);
+ });
};
}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/OverviewCache.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/OverviewCache.java
new file mode 100644
index 000000000..20b5b6222
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/OverviewCache.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard;
+
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_COMMITTING_TABLES;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_EXECUTING_TABLES;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_IDLE_TABLES;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_MEMORY_BYTES_ALLOCATED;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_PENDING_TABLES;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_PLANING_TABLES;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_THREADS;
+import static
org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_HEALTH_SCORE;
+import static
org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_TOTAL_FILES;
+import static
org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_TOTAL_FILES_SIZE;
+
+import org.apache.amoro.metrics.Counter;
+import org.apache.amoro.metrics.Gauge;
+import org.apache.amoro.metrics.Metric;
+import org.apache.amoro.metrics.MetricDefine;
+import org.apache.amoro.metrics.MetricKey;
+import org.apache.amoro.metrics.MetricSet;
+import org.apache.amoro.server.dashboard.model.OverviewDataSizeItem;
+import org.apache.amoro.server.dashboard.model.OverviewResourceUsageItem;
+import org.apache.amoro.server.dashboard.model.OverviewTopTableItem;
+import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class OverviewCache {
+
+ public static final String STATUS_PENDING = "Pending";
+ public static final String STATUS_PLANING = "Planing";
+ public static final String STATUS_EXECUTING = "Executing";
+ public static final String STATUS_IDLE = "Idle";
+ public static final String STATUS_COMMITTING = "Committing";
+
+ private static final Logger LOG =
LoggerFactory.getLogger(OverviewCache.class);
+
+ private volatile List<OverviewTopTableItem> allTopTableItem = new
ArrayList<>();
+ private final Map<String, Long> optimizingStatusCountMap = new
ConcurrentHashMap<>();
+ private final ConcurrentLinkedDeque<OverviewResourceUsageItem>
resourceUsageHistory =
+ new ConcurrentLinkedDeque<>();
+ private final ConcurrentLinkedDeque<OverviewDataSizeItem> dataSizeHistory =
+ new ConcurrentLinkedDeque<>();
+ private final AtomicInteger totalCatalog = new AtomicInteger();
+ private final AtomicLong totalDataSize = new AtomicLong();
+ private final AtomicInteger totalTableCount = new AtomicInteger();
+ private final AtomicInteger totalCpu = new AtomicInteger();
+ private final AtomicLong totalMemory = new AtomicLong();
+
+ private MetricSet metricSet;
+ private int maxRecordCount;
+ private static volatile OverviewCache INSTANCE;
+
+ private OverviewCache() {}
+
+ /** @return Get the singleton object. */
+ public static OverviewCache getInstance() {
+ if (INSTANCE == null) {
+ synchronized (OverviewCache.class) {
+ if (INSTANCE == null) {
+ INSTANCE = new OverviewCache();
+ }
+ }
+ }
+ return INSTANCE;
+ }
+
+ public void initialize(int maxRecordCount, MetricSet globalMetricSet) {
+ this.maxRecordCount = maxRecordCount;
+ this.metricSet = globalMetricSet;
+ }
+
+ public List<OverviewTopTableItem> getAllTopTableItem() {
+ return ImmutableList.copyOf(allTopTableItem);
+ }
+
+ public int getTotalCatalog() {
+ return totalCatalog.get();
+ }
+
+ public int getTotalTableCount() {
+ return totalTableCount.get();
+ }
+
+ public long getTotalDataSize() {
+ return totalDataSize.get();
+ }
+
+ public int getTotalCpu() {
+ return totalCpu.get();
+ }
+
+ public long getTotalMemory() {
+ return totalMemory.get();
+ }
+
+ public List<OverviewResourceUsageItem> getResourceUsageHistory(long
startTime) {
+ return resourceUsageHistory.stream()
+ .filter(item -> item.getTs() >= startTime)
+ .collect(Collectors.toList());
+ }
+
+ public List<OverviewDataSizeItem> getDataSizeHistory(long startTime) {
+ return dataSizeHistory.stream()
+ .filter(item -> item.getTs() >= startTime)
+ .collect(Collectors.toList());
+ }
+
+ public Map<String, Long> getOptimizingStatus() {
+ return optimizingStatusCountMap;
+ }
+
+ public void refresh() {
+ long start = System.currentTimeMillis();
+ LOG.info("Updating overview cache");
+ try {
+ // Already registered metrics may change,
+ // so the metricDefineMap needs to be recalculated at each refresh
+ Map<MetricDefine, List<MetricKey>> metricDefineMap =
+ metricSet.getMetrics().keySet().stream()
+ .collect(
+ Collectors.groupingBy(
+ MetricKey::getDefine,
+ Collectors.mapping(Function.identity(),
Collectors.toList())));
+
+ updateResourceUsage(start, metricDefineMap);
+ updateTableDetail(start, metricDefineMap);
+ updateOptimizingStatus(metricDefineMap);
+
+ } catch (Exception e) {
+ LOG.error("OverviewRefresher error", e);
+ } finally {
+ long end = System.currentTimeMillis();
+ LOG.info("Refresher overview cache took {} ms.", end - start);
+ }
+ }
+
+ private void updateResourceUsage(long ts, Map<MetricDefine, List<MetricKey>>
metricDefineMap) {
+ int optimizerGroupThreadCount =
+ (int) sumMetricValuesByDefine(metricDefineMap,
OPTIMIZER_GROUP_THREADS);
+ long optimizerGroupMemoryBytes =
+ sumMetricValuesByDefine(metricDefineMap,
OPTIMIZER_GROUP_MEMORY_BYTES_ALLOCATED);
+
+ this.totalCpu.set(optimizerGroupThreadCount);
+ this.totalMemory.set(optimizerGroupMemoryBytes);
+ addAndCheck(
+ new OverviewResourceUsageItem(ts, optimizerGroupThreadCount,
optimizerGroupMemoryBytes));
+ }
+
+ private void updateTableDetail(long ts, Map<MetricDefine, List<MetricKey>>
metricDefineMap) {
+ Map<String, OverviewTopTableItem> topTableItemMap = Maps.newHashMap();
+ Set<String> allCatalogs = Sets.newHashSet();
+ Map<MetricKey, Metric> registeredMetrics = metricSet.getMetrics();
+ long totalTableSize = 0L;
+
+ // table size
+ List<MetricKey> metricKeys =
metricDefineMap.get(TABLE_SUMMARY_TOTAL_FILES_SIZE);
+ for (MetricKey metricKey : metricKeys) {
+ String tableName = fullTableName(metricKey);
+ allCatalogs.add(catalog(metricKey));
+ OverviewTopTableItem tableItem =
+ topTableItemMap.computeIfAbsent(tableName, ignore -> new
OverviewTopTableItem(tableName));
+ long tableSize = covertValue(registeredMetrics.get(metricKey));
+ tableItem.setTableSize(tableSize);
+ totalTableSize += tableSize;
+ }
+
+ // file count
+ metricKeys = metricDefineMap.get(TABLE_SUMMARY_TOTAL_FILES);
+ for (MetricKey metricKey : metricKeys) {
+ String tableName = fullTableName(metricKey);
+ allCatalogs.add(catalog(metricKey));
+ OverviewTopTableItem tableItem =
+ topTableItemMap.computeIfAbsent(tableName, ignore -> new
OverviewTopTableItem(tableName));
+ int fileCount = (int) covertValue(registeredMetrics.get(metricKey));
+ tableItem.setFileCount(fileCount);
+ tableItem.setAverageFileSize(fileCount == 0 ? 0 :
tableItem.getTableSize() / fileCount);
+ }
+
+ // health score
+ metricKeys = metricDefineMap.get(TABLE_SUMMARY_HEALTH_SCORE);
+ for (MetricKey metricKey : metricKeys) {
+ String tableName = fullTableName(metricKey);
+ allCatalogs.add(catalog(metricKey));
+ OverviewTopTableItem tableItem =
+ topTableItemMap.computeIfAbsent(tableName, ignore -> new
OverviewTopTableItem(tableName));
+ int healthScore = (int) covertValue(registeredMetrics.get(metricKey));
+ tableItem.setHealthScore(healthScore);
+ }
+
+ this.totalDataSize.set(totalTableSize);
+ this.totalCatalog.set(allCatalogs.size());
+ this.totalTableCount.set(metricKeys.size());
+ this.allTopTableItem = new ArrayList<>(topTableItemMap.values());
+ addAndCheck(new OverviewDataSizeItem(ts, totalTableSize));
+ }
+
+ private void updateOptimizingStatus(Map<MetricDefine, List<MetricKey>>
metricDefineMap) {
+ optimizingStatusCountMap.put(
+ STATUS_PENDING, sumMetricValuesByDefine(metricDefineMap,
OPTIMIZER_GROUP_PENDING_TABLES));
+ optimizingStatusCountMap.put(
+ STATUS_PLANING, sumMetricValuesByDefine(metricDefineMap,
OPTIMIZER_GROUP_PLANING_TABLES));
+ optimizingStatusCountMap.put(
+ STATUS_EXECUTING,
+ sumMetricValuesByDefine(metricDefineMap,
OPTIMIZER_GROUP_EXECUTING_TABLES));
+ optimizingStatusCountMap.put(
+ STATUS_IDLE, sumMetricValuesByDefine(metricDefineMap,
OPTIMIZER_GROUP_IDLE_TABLES));
+ optimizingStatusCountMap.put(
+ STATUS_COMMITTING,
+ sumMetricValuesByDefine(metricDefineMap,
OPTIMIZER_GROUP_COMMITTING_TABLES));
+ }
+
+ private void addAndCheck(OverviewDataSizeItem dataSizeItem) {
+ dataSizeHistory.add(dataSizeItem);
+ checkSize(dataSizeHistory);
+ }
+
+ private void addAndCheck(OverviewResourceUsageItem resourceUsageItem) {
+ resourceUsageHistory.add(resourceUsageItem);
+ checkSize(resourceUsageHistory);
+ }
+
+ private <T> void checkSize(Deque<T> deque) {
+ if (deque.size() > maxRecordCount) {
+ deque.poll();
+ }
+ }
+
+ private String fullTableName(MetricKey metricKey) {
+ return catalog(metricKey)
+ .concat(".")
+ .concat(database(metricKey))
+ .concat(".")
+ .concat(table(metricKey));
+ }
+
+ private String catalog(MetricKey metricKey) {
+ return metricKey.valueOfTag("catalog");
+ }
+
+ private String database(MetricKey metricKey) {
+ return metricKey.valueOfTag("database");
+ }
+
+ private String table(MetricKey metricKey) {
+ return metricKey.valueOfTag("table");
+ }
+
+ private long sumMetricValuesByDefine(
+ Map<MetricDefine, List<MetricKey>> metricDefineMap, MetricDefine
metricDefine) {
+ List<MetricKey> metricKeys = metricDefineMap.get(metricDefine);
+ if ((metricKeys == null)) {
+ return 0;
+ }
+ return metricKeys.stream()
+ .map(metricKey -> covertValue(metricSet.getMetrics().get(metricKey)))
+ .mapToLong(Long::longValue)
+ .sum();
+ }
+
+ private long covertValue(Metric metric) {
+ if (metric instanceof Counter) {
+ return ((Counter) metric).getCount();
+ } else if (metric instanceof Gauge) {
+ return ((Gauge<?>) metric).getValue().longValue();
+ } else {
+ throw new IllegalStateException(
+ "unknown metric implement class:" + metric.getClass().getName());
+ }
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/OverviewMetricsReporter.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/OverviewMetricsReporter.java
new file mode 100644
index 000000000..42a9facdc
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/OverviewMetricsReporter.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard;
+
+import org.apache.amoro.metrics.MetricReporter;
+import org.apache.amoro.metrics.MetricSet;
+import
org.apache.amoro.shade.guava32.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/** Overview exporter */
+public class OverviewMetricsReporter implements MetricReporter {
+
+ public static final String REFRESH_INTERVAL = "refresh-interval";
+ public static final String MAX_HISTORY_RECORDS = "max-history-records";
+
+ private final ScheduledExecutorService overviewUpdaterScheduler =
+ Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder()
+ .setNameFormat("overview-updater-scheduler-%d")
+ .setDaemon(true)
+ .build());
+
+ private long overviewRefreshingInterval;
+ private int maxRecordCount;
+
+ @Override
+ public void open(Map<String, String> properties) {
+ overviewRefreshingInterval =
+ Optional.ofNullable(properties.get(REFRESH_INTERVAL))
+ .map(Long::valueOf)
+ .orElseThrow(
+ () -> new IllegalArgumentException("Lack required property: "
+ REFRESH_INTERVAL));
+
+ maxRecordCount =
+ Optional.ofNullable(properties.get(MAX_HISTORY_RECORDS))
+ .map(Integer::valueOf)
+ .orElseThrow(
+ () ->
+ new IllegalArgumentException("Lack required property: " +
MAX_HISTORY_RECORDS));
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String name() {
+ return "overview-exporter";
+ }
+
+ @Override
+ public void setGlobalMetricSet(MetricSet globalMetricSet) {
+ OverviewCache overviewCache = OverviewCache.getInstance();
+ overviewCache.initialize(maxRecordCount, globalMetricSet);
+ overviewUpdaterScheduler.scheduleAtFixedRate(
+ overviewCache::refresh, 1000L, overviewRefreshingInterval,
TimeUnit.MILLISECONDS);
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/OverviewController.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/OverviewController.java
new file mode 100644
index 000000000..484cf42bb
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/OverviewController.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard.controller;
+
+import io.javalin.http.Context;
+import org.apache.amoro.server.dashboard.OverviewCache;
+import org.apache.amoro.server.dashboard.model.OverviewDataSizeItem;
+import org.apache.amoro.server.dashboard.model.OverviewResourceUsageItem;
+import org.apache.amoro.server.dashboard.model.OverviewSummary;
+import org.apache.amoro.server.dashboard.model.OverviewTopTableItem;
+import org.apache.amoro.server.dashboard.response.OkResponse;
+import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
+import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/** The controller that handles overview page requests. */
+public class OverviewController {
+
+ private OverviewCache overviewCache;
+
+ public OverviewController() {
+ this.overviewCache = OverviewCache.getInstance();
+ }
+
+ public void getResourceUsageHistory(Context ctx) {
+ String startTime = ctx.queryParam("startTime");
+ Preconditions.checkArgument(StringUtils.isNumeric(startTime), "invalid
startTime!");
+ List<OverviewResourceUsageItem> resourceUsageHistory =
+ overviewCache.getResourceUsageHistory(Long.parseLong(startTime));
+ ctx.json(OkResponse.of(resourceUsageHistory));
+ }
+
+ public void getDataSizeHistory(Context ctx) {
+ String startTime = ctx.queryParam("startTime");
+ Preconditions.checkArgument(StringUtils.isNumeric(startTime), "invalid
startTime!");
+ List<OverviewDataSizeItem> dataSizeHistory =
+ overviewCache.getDataSizeHistory(Long.parseLong(startTime));
+ ctx.json(OkResponse.of(dataSizeHistory));
+ }
+
+ public void getTopTables(Context ctx) {
+ String order = ctx.queryParam("order");
+ String orderBy = ctx.queryParam("orderBy");
+ Integer limit = ctx.queryParamAsClass("limit",
Integer.class).getOrDefault(10);
+ Preconditions.checkArgument(StringUtils.isNotBlank(order), "order can not
be empty");
+ Preconditions.checkArgument(StringUtils.isNotBlank(orderBy), "orderBy can
not be empty");
+ List<OverviewTopTableItem> top10Tables;
+ boolean isAsc = "asc".equals(order);
+ switch (orderBy) {
+ case "tableSize":
+ Comparator<OverviewTopTableItem> tableSizeComparator =
+ Comparator.comparingLong(OverviewTopTableItem::getTableSize);
+ top10Tables = getTopTables(isAsc, tableSizeComparator, limit);
+ break;
+ case "fileCount":
+ Comparator<OverviewTopTableItem> fileCountComparator =
+ Comparator.comparingLong(OverviewTopTableItem::getFileCount);
+ top10Tables = getTopTables(isAsc, fileCountComparator, limit);
+ break;
+ case "healthScore":
+ // If the table health score of the table has not been calculated,
+ // it should be sorted at the end of the list
+ Comparator<OverviewTopTableItem> healthScoreComparator =
+ Comparator.comparingLong(
+ item ->
+ item.getHealthScore() < 0
+ ? (isAsc ? Long.MAX_VALUE : Long.MIN_VALUE)
+ : item.getTableSize());
+ top10Tables = getTopTables(isAsc, healthScoreComparator, limit);
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid orderBy: " + orderBy);
+ }
+ ctx.json(OkResponse.of(top10Tables));
+ }
+
+ private List<OverviewTopTableItem> getTopTables(
+ boolean asc, Comparator<OverviewTopTableItem> comparator, int limit) {
+ return overviewCache.getAllTopTableItem().stream()
+ .sorted(
+ asc
+ ? comparator.thenComparing(OverviewTopTableItem::getTableName)
+ :
comparator.reversed().thenComparing(OverviewTopTableItem::getTableName))
+ .limit(limit)
+ .collect(Collectors.toList());
+ }
+
+ public void getSummary(Context ctx) {
+ int totalCatalog = overviewCache.getTotalCatalog();
+ int totalTableCount = overviewCache.getTotalTableCount();
+ long totalDataSize = overviewCache.getTotalDataSize();
+ int totalCpu = overviewCache.getTotalCpu();
+ long totalMemory = overviewCache.getTotalMemory();
+
+ OverviewSummary overviewSummary =
+ new OverviewSummary(totalCatalog, totalTableCount, totalDataSize,
totalCpu, totalMemory);
+ ctx.json(OkResponse.of(overviewSummary));
+ }
+
+ public void getOptimizingStatus(Context ctx) {
+ Map<String, Long> optimizingStatus = overviewCache.getOptimizingStatus();
+ List<ImmutableMap<String, ? extends Serializable>> optimizingStatusList =
+ optimizingStatus.entrySet().stream()
+ .map(status -> ImmutableMap.of("name", status.getKey(), "value",
status.getValue()))
+ .collect(Collectors.toList());
+ ctx.json(OkResponse.of(optimizingStatusList));
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewDataSizeItem.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewDataSizeItem.java
new file mode 100644
index 000000000..39a11a256
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewDataSizeItem.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard.model;
+
+import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects;
+
+public class OverviewDataSizeItem {
+ private long ts;
+ private long dataSize;
+
+ public OverviewDataSizeItem(long ts, long dataSize) {
+ this.ts = ts;
+ this.dataSize = dataSize;
+ }
+
+ public long getTs() {
+ return ts;
+ }
+
+ public void setTs(long ts) {
+ this.ts = ts;
+ }
+
+ public long getDataSize() {
+ return dataSize;
+ }
+
+ public void setDataSize(long dataSize) {
+ this.dataSize = dataSize;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("ts", ts).add("dataSize",
dataSize).toString();
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewResourceUsageItem.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewResourceUsageItem.java
new file mode 100644
index 000000000..79bf88d9e
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewResourceUsageItem.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard.model;
+
+import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects;
+
+public class OverviewResourceUsageItem {
+ private long ts;
+ private int totalCpu;
+ private long totalMemory;
+
+ public OverviewResourceUsageItem(long ts, int totalCpu, long totalMemory) {
+ this.ts = ts;
+ this.totalCpu = totalCpu;
+ this.totalMemory = totalMemory;
+ }
+
+ public long getTs() {
+ return ts;
+ }
+
+ public void setTs(long ts) {
+ this.ts = ts;
+ }
+
+ public int getTotalCpu() {
+ return totalCpu;
+ }
+
+ public void setTotalCpu(int totalCpu) {
+ this.totalCpu = totalCpu;
+ }
+
+ public long getTotalMemory() {
+ return totalMemory;
+ }
+
+ public void setTotalMemory(long totalMemory) {
+ this.totalMemory = totalMemory;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("ts", ts)
+ .add("totalCpu", totalCpu)
+ .add("totalMemory", totalMemory)
+ .toString();
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewSummary.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewSummary.java
new file mode 100644
index 000000000..bf028c8e0
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewSummary.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard.model;
+
+import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects;
+
+public class OverviewSummary {
+
+ private int catalogCnt;
+ private int tableCnt;
+ private long tableTotalSize;
+ private int totalCpu;
+ private long totalMemory;
+
+ public OverviewSummary() {}
+
+ public OverviewSummary(
+ int catalogCnt, int tableCnt, long tableTotalSize, int totalCpu, long
totalMemory) {
+ this.catalogCnt = catalogCnt;
+ this.tableCnt = tableCnt;
+ this.tableTotalSize = tableTotalSize;
+ this.totalCpu = totalCpu;
+ this.totalMemory = totalMemory;
+ }
+
+ public int getCatalogCnt() {
+ return catalogCnt;
+ }
+
+ public void setCatalogCnt(int catalogCnt) {
+ this.catalogCnt = catalogCnt;
+ }
+
+ public int getTableCnt() {
+ return tableCnt;
+ }
+
+ public void setTableCnt(int tableCnt) {
+ this.tableCnt = tableCnt;
+ }
+
+ public long getTableTotalSize() {
+ return tableTotalSize;
+ }
+
+ public void setTableTotalSize(long tableTotalSize) {
+ this.tableTotalSize = tableTotalSize;
+ }
+
+ public int getTotalCpu() {
+ return totalCpu;
+ }
+
+ public void setTotalCpu(int totalCpu) {
+ this.totalCpu = totalCpu;
+ }
+
+ public long getTotalMemory() {
+ return totalMemory;
+ }
+
+ public void setTotalMemory(long totalMemory) {
+ this.totalMemory = totalMemory;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("catalogCnt", catalogCnt)
+ .add("tableCnt", tableCnt)
+ .add("tableTotalSize", tableTotalSize)
+ .add("totalCpu", totalCpu)
+ .add("totalMemory", totalMemory)
+ .toString();
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewTopTableItem.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewTopTableItem.java
new file mode 100644
index 000000000..2fffc9b8e
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/model/OverviewTopTableItem.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard.model;
+
+import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects;
+
+public class OverviewTopTableItem {
+ private String tableName;
+ private long tableSize;
+ private int fileCount;
+ private long averageFileSize;
+ private int healthScore;
+
+ public OverviewTopTableItem() {}
+
+ public OverviewTopTableItem(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public long getTableSize() {
+ return tableSize;
+ }
+
+ public void setTableSize(long tableSize) {
+ this.tableSize = tableSize;
+ }
+
+ public int getFileCount() {
+ return fileCount;
+ }
+
+ public void setFileCount(int fileCount) {
+ this.fileCount = fileCount;
+ }
+
+ public long getAverageFileSize() {
+ return averageFileSize;
+ }
+
+ public void setAverageFileSize(long averageFileSize) {
+ this.averageFileSize = averageFileSize;
+ }
+
+ public int getHealthScore() {
+ return healthScore;
+ }
+
+ public void setHealthScore(int healthScore) {
+ this.healthScore = healthScore;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("tableName", tableName)
+ .add("tableSize", tableSize)
+ .add("fileCount", fileCount)
+ .add("averageFileSize", averageFileSize)
+ .add("healthScore", healthScore)
+ .toString();
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizerGroupMetrics.java
b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizerGroupMetrics.java
index c3abfb244..dfa30a5b1 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizerGroupMetrics.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizerGroupMetrics.java
@@ -19,6 +19,8 @@
package org.apache.amoro.server.optimizing;
import static org.apache.amoro.metrics.MetricDefine.defineGauge;
+import static org.apache.amoro.server.optimizing.OptimizingStatus.COMMITTING;
+import static org.apache.amoro.server.optimizing.OptimizingStatus.IDLE;
import static org.apache.amoro.server.optimizing.OptimizingStatus.PENDING;
import static org.apache.amoro.server.optimizing.OptimizingStatus.PLANNING;
import static org.apache.amoro.server.optimizing.TaskRuntime.Status.ACKED;
@@ -72,6 +74,18 @@ public class OptimizerGroupMetrics {
.withTags(GROUP_TAG)
.build();
+ public static final MetricDefine OPTIMIZER_GROUP_IDLE_TABLES =
+ defineGauge("optimizer_group_idle_tables")
+ .withDescription("Number of idle tables in optimizer group")
+ .withTags(GROUP_TAG)
+ .build();
+
+ public static final MetricDefine OPTIMIZER_GROUP_COMMITTING_TABLES =
+ defineGauge("optimizer_group_committing_tables")
+ .withDescription("Number of committing tables in optimizer group")
+ .withTags(GROUP_TAG)
+ .build();
+
public static final MetricDefine OPTIMIZER_GROUP_OPTIMIZER_INSTANCES =
defineGauge("optimizer_group_optimizer_instances")
.withDescription("Number of optimizer instances in optimizer group")
@@ -149,6 +163,22 @@ public class OptimizerGroupMetrics {
optimizingQueue.getSchedulingPolicy().getTableRuntimeMap().values().stream()
.filter(t -> t.getOptimizingStatus().isProcessing())
.count());
+ registerMetric(
+ registry,
+ OPTIMIZER_GROUP_IDLE_TABLES,
+ (Gauge<Long>)
+ () ->
+
optimizingQueue.getSchedulingPolicy().getTableRuntimeMap().values().stream()
+ .filter(t -> t.getOptimizingStatus().equals(IDLE))
+ .count());
+ registerMetric(
+ registry,
+ OPTIMIZER_GROUP_COMMITTING_TABLES,
+ (Gauge<Long>)
+ () ->
+
optimizingQueue.getSchedulingPolicy().getTableRuntimeMap().values().stream()
+ .filter(t -> t.getOptimizingStatus().equals(COMMITTING))
+ .count());
registerMetric(
registry, OPTIMIZER_GROUP_OPTIMIZER_INSTANCES, (Gauge<Integer>)
optimizerInstances::size);
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java
b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java
index 7d4dfd860..748f400ee 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java
@@ -143,7 +143,7 @@ public class TableSummaryMetrics {
private long dataFilesRecords = 0L;
private long equalityDeleteFilesRecords = 0L;
private long snapshots = 0L;
- private long healthScore = 0L;
+ private long healthScore = -1L; // -1 means not calculated
public TableSummaryMetrics(ServerTableIdentifier identifier) {
this.identifier = identifier;
diff --git
a/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.metrics.MetricReporter
b/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.metrics.MetricReporter
new file mode 100644
index 000000000..57053ec93
--- /dev/null
+++
b/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.metrics.MetricReporter
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.amoro.server.dashboard.OverviewMetricsReporter
\ No newline at end of file
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewCache.java
b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewCache.java
new file mode 100644
index 000000000..96a8d789c
--- /dev/null
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewCache.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard;
+
+import static
org.apache.amoro.server.dashboard.OverviewCache.STATUS_COMMITTING;
+import static org.apache.amoro.server.dashboard.OverviewCache.STATUS_EXECUTING;
+import static org.apache.amoro.server.dashboard.OverviewCache.STATUS_IDLE;
+import static org.apache.amoro.server.dashboard.OverviewCache.STATUS_PENDING;
+import static org.apache.amoro.server.dashboard.OverviewCache.STATUS_PLANING;
+
+import org.apache.amoro.BasicTableTestHelper;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.TableTestHelper;
+import org.apache.amoro.catalog.BasicCatalogTestHelper;
+import org.apache.amoro.catalog.CatalogTestHelper;
+import org.apache.amoro.io.MixedDataTestHelpers;
+import org.apache.amoro.server.dashboard.model.OverviewTopTableItem;
+import org.apache.amoro.server.manager.MetricManager;
+import org.apache.amoro.server.table.AMSTableTestBase;
+import org.apache.amoro.server.table.TableRuntime;
+import org.apache.amoro.server.table.executor.TableRuntimeRefreshExecutor;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
+import org.apache.amoro.table.MixedTable;
+import org.apache.amoro.table.UnkeyedTable;
+import org.apache.iceberg.AppendFiles;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.data.Record;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class TestOverviewCache extends AMSTableTestBase {
+
+ private OverviewCache overviewCache;
+
+ @Parameterized.Parameters(name = "{0}, {1}")
+ public static Object[] parameters() {
+ return new Object[][] {
+ {new BasicCatalogTestHelper(TableFormat.ICEBERG), new
BasicTableTestHelper(true, false)}
+ };
+ }
+
+ public TestOverviewCache(CatalogTestHelper catalogTestHelper,
TableTestHelper tableTestHelper) {
+ super(catalogTestHelper, tableTestHelper, false);
+ }
+
+ @Before
+ public void prepare() {
+ createDatabase();
+ createTable();
+ this.overviewCache = OverviewCache.getInstance();
+ this.overviewCache.initialize(10,
MetricManager.getInstance().getGlobalRegistry());
+ this.overviewCache.refresh();
+ }
+
+ @After
+ public void clear() {
+ try {
+ dropTable();
+ dropDatabase();
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+
+ private void initTableWithFiles() {
+ UnkeyedTable table =
+ ((MixedTable)
tableService().loadTable(serverTableIdentifier()).originalTable())
+ .asUnkeyedTable();
+ appendData(table, 1);
+ appendData(table, 2);
+ TableRuntime runtime = tableService().getRuntime(serverTableIdentifier());
+ runtime.refresh(tableService().loadTable(serverTableIdentifier()));
+ }
+
+ private void appendData(UnkeyedTable table, int id) {
+ ArrayList<Record> newRecords =
+ Lists.newArrayList(
+ MixedDataTestHelpers.createRecord(
+ table.schema(), id, "111", 0L, "2022-01-01T12:00:00"));
+ List<DataFile> dataFiles = MixedDataTestHelpers.writeBaseStore(table, 0L,
newRecords, false);
+ AppendFiles appendFiles = table.newAppend();
+ dataFiles.forEach(appendFiles::appendFile);
+ appendFiles.commit();
+ }
+
+ void refreshPending() {
+ TableRuntimeRefreshExecutor refresher =
+ new TableRuntimeRefreshExecutor(tableService(), 1, Integer.MAX_VALUE);
+ refresher.execute(tableService().getRuntime(serverTableIdentifier()));
+ refresher.dispose();
+ }
+
+ @Test
+ public void testOverviewCache() {
+ // empty table
+ Assertions.assertEquals(1, overviewCache.getTotalCatalog());
+ Assertions.assertEquals(1, overviewCache.getTotalTableCount());
+ Assertions.assertEquals(0, overviewCache.getTotalDataSize());
+ Assertions.assertEquals(0, overviewCache.getTotalCpu());
+ Assertions.assertEquals(0, overviewCache.getTotalMemory());
+
+ Assertions.assertEquals(0,
overviewCache.getOptimizingStatus().get(STATUS_PENDING));
+ Assertions.assertEquals(0,
overviewCache.getOptimizingStatus().get(STATUS_COMMITTING));
+ Assertions.assertEquals(0,
overviewCache.getOptimizingStatus().get(STATUS_EXECUTING));
+ Assertions.assertEquals(0,
overviewCache.getOptimizingStatus().get(STATUS_PLANING));
+ Assertions.assertEquals(1,
overviewCache.getOptimizingStatus().get(STATUS_IDLE));
+
+ Assertions.assertEquals(1, overviewCache.getDataSizeHistory(0).size());
+ Assertions.assertEquals(1,
overviewCache.getResourceUsageHistory(0).size());
+
+ List<OverviewTopTableItem> allTopTableItem =
overviewCache.getAllTopTableItem();
+ Assertions.assertEquals(1, allTopTableItem.size());
+ Assertions.assertEquals(-1, allTopTableItem.get(0).getHealthScore());
+
+ // insert data
+ initTableWithFiles();
+ refreshPending();
+ overviewCache.refresh();
+
+ Assertions.assertTrue(overviewCache.getTotalDataSize() > 0);
+
+ Assertions.assertEquals(1,
overviewCache.getOptimizingStatus().get(STATUS_PENDING));
+ Assertions.assertEquals(0,
overviewCache.getOptimizingStatus().get(STATUS_IDLE));
+
+ Assertions.assertEquals(2, overviewCache.getDataSizeHistory(0).size());
+ Assertions.assertEquals(2,
overviewCache.getResourceUsageHistory(0).size());
+ allTopTableItem = overviewCache.getAllTopTableItem();
+ Assertions.assertEquals(100, allTopTableItem.get(0).getHealthScore());
+ }
+}
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java
b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java
index 122f08e90..0032af9bc 100644
---
a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java
@@ -19,8 +19,10 @@
package org.apache.amoro.server.optimizing;
import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.GROUP_TAG;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_COMMITTING_TABLES;
import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_EXECUTING_TABLES;
import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_EXECUTING_TASKS;
+import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_IDLE_TABLES;
import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_MEMORY_BYTES_ALLOCATED;
import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_OPTIMIZER_INSTANCES;
import static
org.apache.amoro.server.optimizing.OptimizerGroupMetrics.OPTIMIZER_GROUP_PENDING_TABLES;
@@ -267,12 +269,20 @@ public class TestOptimizingQueue extends AMSTableTestBase
{
Gauge<Long> executingTablesGauge =
(Gauge<Long>)
registry.getMetrics().get(new
MetricKey(OPTIMIZER_GROUP_EXECUTING_TABLES, tagValues));
+ Gauge<Long> idleTablesGauge =
+ (Gauge<Long>)
+ registry.getMetrics().get(new
MetricKey(OPTIMIZER_GROUP_IDLE_TABLES, tagValues));
+ Gauge<Long> committingTablesGauge =
+ (Gauge<Long>)
+ registry.getMetrics().get(new
MetricKey(OPTIMIZER_GROUP_COMMITTING_TABLES, tagValues));
Assert.assertEquals(0, queueTasksGauge.getValue().longValue());
Assert.assertEquals(0, executingTasksGauge.getValue().longValue());
Assert.assertEquals(0, planingTablesGauge.getValue().longValue());
Assert.assertEquals(1, pendingTablesGauge.getValue().longValue());
Assert.assertEquals(0, executingTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, idleTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, committingTablesGauge.getValue().longValue());
TaskRuntime task = queue.pollTask(MAX_POLLING_TIME);
Assert.assertNotNull(task);
@@ -282,6 +292,8 @@ public class TestOptimizingQueue extends AMSTableTestBase {
Assert.assertEquals(0, planingTablesGauge.getValue().longValue());
Assert.assertEquals(0, pendingTablesGauge.getValue().longValue());
Assert.assertEquals(1, executingTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, idleTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, committingTablesGauge.getValue().longValue());
task.ack(optimizerThread);
Assert.assertEquals(0, queueTasksGauge.getValue().longValue());
@@ -289,6 +301,8 @@ public class TestOptimizingQueue extends AMSTableTestBase {
Assert.assertEquals(0, planingTablesGauge.getValue().longValue());
Assert.assertEquals(0, pendingTablesGauge.getValue().longValue());
Assert.assertEquals(1, executingTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, idleTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, committingTablesGauge.getValue().longValue());
task.complete(
optimizerThread,
@@ -298,6 +312,8 @@ public class TestOptimizingQueue extends AMSTableTestBase {
Assert.assertEquals(0, planingTablesGauge.getValue().longValue());
Assert.assertEquals(0, pendingTablesGauge.getValue().longValue());
Assert.assertEquals(1, executingTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, idleTablesGauge.getValue().longValue());
+ Assert.assertEquals(1, committingTablesGauge.getValue().longValue());
OptimizingProcess optimizingProcess =
tableRuntimeMeta.getTableRuntime().getOptimizingProcess();
optimizingProcess.commit();
@@ -306,6 +322,8 @@ public class TestOptimizingQueue extends AMSTableTestBase {
Assert.assertEquals(0, planingTablesGauge.getValue().longValue());
Assert.assertEquals(0, pendingTablesGauge.getValue().longValue());
Assert.assertEquals(0, executingTablesGauge.getValue().longValue());
+ Assert.assertEquals(1, idleTablesGauge.getValue().longValue());
+ Assert.assertEquals(0, committingTablesGauge.getValue().longValue());
queue.dispose();
}
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java
b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java
index e0b992e78..9bc914ef7 100644
---
a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java
@@ -187,7 +187,7 @@ public class TestTableSummaryMetrics extends
AMSTableTestBase {
Assertions.assertEquals(0, dataRecords.getValue());
Assertions.assertEquals(0, posDelRecords.getValue());
Assertions.assertEquals(0, eqDelRecords.getValue());
- Assertions.assertEquals(0, healthScore.getValue());
+ Assertions.assertEquals(-1, healthScore.getValue());
// refresh metrics
initTableWithFiles();
diff --git a/amoro-web/mock/modules/overview.js
b/amoro-web/mock/modules/overview.js
index 9c19594de..1e5ac4475 100644
--- a/amoro-web/mock/modules/overview.js
+++ b/amoro-web/mock/modules/overview.js
@@ -28,59 +28,90 @@ export default [
tableCnt: 37944,
tableTotalSize: 10585900,
totalCpu: '6',
- totalMemory: 62464
+ totalMemory: 62464000000
}
}),
},
{
- url: '/mock/ams/v1/overview/top/tables',
+ url: '/mock/ams/v1/overview/optimizing',
method: 'get',
response: () => ({
code: 200,
msg: 'success',
result: [
- {
- tableName: 'trino_online_env_hive.spark_test.ctpri',
- size: 12938982,
- fileCnt: 57889
- },
- {
- tableName: 'trino_online_env_hive.spark_test.ctpp_col',
- size: 329043290,
- fileCnt: 79910
- }
+ { value: 40, name: 'Executing' },
+ { value: 10, name: 'Committing' },
+ { value: 2, name: 'Planning' },
+ { value: 3, name: 'Pending' },
+ { value: 50, name: 'Idle' },
]
}),
},
{
- url: '/mock/ams/v1/overview/metric/optimize/resource',
+ url: '/mock/ams/v1/overview/top',
method: 'get',
response: () => ({
code: 200,
msg: 'success',
- result: {
- timeLine: [
- '10-09 14:48'
- ],
- usedCpu: [
- '83.24'
- ],
- usedCpuDivision: [
- '1828C/2196C'
- ],
- usedCpuPercent: [
- '83.24%'
- ],
- usedMem: [
- '83.24'
- ],
- usedMemDivision: [
- '1828C/2196C10364G'
- ],
- usedMemPercent: [
- '83.24%'
- ]
- }
+ result: [
+ { tableName: 'test_catalog.db.user', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '47',},
+ { tableName: 'test_catalog.db.user1', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '52',},
+ { tableName: 'test_catalog.db.user2', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '66',},
+ { tableName: 'test_catalog.db.user3', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '77',},
+ { tableName: 'test_catalog.db.user4', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '78',},
+ { tableName: 'test_catalog.db.user5', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '81',},
+ { tableName: 'test_catalog.db.user6', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '85',},
+ { tableName: 'test_catalog.db.user7', tableSize: '1774', fileCount:
'2', averageFileSize: '887', healthScore: '88',},
+ { tableName: 'test_catalog.db.user8', tableSize: '0', fileCount: '0',
averageFileSize: '0', healthScore: '100',},
+ { tableName: 'test_catalog.db.user9', tableSize: '0', fileCount: '0',
averageFileSize: '0', healthScore: '100',},
+ ]
+ }),
+ },
+ {
+ url: '/mock/ams/v1/overview/resource',
+ method: 'get',
+ response: () => ({
+ code: 200,
+ msg: 'success',
+ result: [{
+ ts: 1724119207500,
+ totalCpu: 2,
+ totalMemory: 4048000000
+ },{
+ ts: 1724122892000,
+ totalCpu: 5,
+ totalMemory: 8096000000
+ },{
+ ts: 1724126441000,
+ totalCpu: 12,
+ totalMemory: 32492000000
+ },{
+ ts: 1724130154000,
+ totalCpu: 7,
+ totalMemory: 14462000000
+ },{
+ ts: 1724133605600,
+ totalCpu: 26,
+ totalMemory: 50176000000
+ }]
+ }),
+ },
+ {
+ url: '/mock/ams/v1/overview/dataSize',
+ method: 'get',
+ response: () => ({
+ code: 200,
+ msg: 'success',
+ result: [{
+ ts: 1724119200000,
+ dataSize: 1024000
+ },{
+ ts: 1724122800000,
+ dataSize: 2048000
+ },{
+ ts: 1724126400000,
+ dataSize: 4096000
+ }]
}),
},
]
diff --git a/amoro-web/src/components/Sidebar.vue
b/amoro-web/src/components/Sidebar.vue
index 2136f6e7a..6bbccf878 100644
--- a/amoro-web/src/components/Sidebar.vue
+++ b/amoro-web/src/components/Sidebar.vue
@@ -58,6 +58,11 @@ export default defineComponent({
},
]
const allMenu: MenuItem[] = [
+ {
+ key: 'overview',
+ title: t('overview'),
+ icon: 'overview',
+ },
{
key: 'tables',
title: t('tables'),
@@ -142,9 +147,9 @@ export default defineComponent({
}, time)
}
- const viewIntroduce = () => {
+ const viewOverview = () => {
router.push({
- path: '/introduce',
+ path: '/overview',
})
}
@@ -158,7 +163,7 @@ export default defineComponent({
store,
toggleTablesMenu,
goCreatePage,
- viewIntroduce,
+ viewOverview,
}
},
})
@@ -166,7 +171,7 @@ export default defineComponent({
<template>
<div :class="{ 'side-bar-collapsed': collapsed }" class="side-bar">
- <div :class="{ 'logo-collapsed': collapsed }" class="logo g-flex-ae"
@mouseenter="toggleTablesMenu(false)" @click="viewIntroduce">
+ <div :class="{ 'logo-collapsed': collapsed }" class="logo g-flex-ae"
@mouseenter="toggleTablesMenu(false)" @click="viewOverview">
<img src="../assets/images/logo1.svg" class="logo-img" alt="">
<img v-show="!collapsed" src="../assets/images/arctic-dashboard1.svg"
class="arctic-name" alt="">
</div>
diff --git a/amoro-web/src/components/echarts/index.ts
b/amoro-web/src/components/echarts/index.ts
index 10a50b680..e411ad3a2 100644
--- a/amoro-web/src/components/echarts/index.ts
+++ b/amoro-web/src/components/echarts/index.ts
@@ -26,6 +26,7 @@ import type {
import {
BarChart,
LineChart,
+ PieChart,
} from 'echarts/charts'
import type {
DatasetComponentOption,
@@ -67,6 +68,7 @@ echarts.use([
UniversalTransition,
CanvasRenderer,
LegendComponent,
+ PieChart,
])
export default echarts
diff --git a/amoro-web/src/language/en.ts b/amoro-web/src/language/en.ts
index 06ba5513e..2c08f18de 100644
--- a/amoro-web/src/language/en.ts
+++ b/amoro-web/src/language/en.ts
@@ -17,8 +17,14 @@
*/
export default {
+ last1h: 'Last 1 h',
+ last12h: 'Last 12 h',
+ last24h: 'Last 24 h',
+ last7day: 'Last 7 day',
healthScore: 'Health Score',
overview: 'Overview',
+ top10Tables: 'Top 10 Tables',
+ latestOperations: 'Latest Operations',
catalogs: 'Catalogs',
catalog: 'Catalog',
tables: 'Tables',
@@ -37,6 +43,7 @@ export default {
token: 'Token',
cluster: 'Cluster',
database: 'Database | Databases',
+ tableSize: 'Table Size',
table: 'Table | Tables',
size: 'Size',
file: 'File',
@@ -153,11 +160,14 @@ export default {
duplicateKey: 'duplicate key',
lastUpgradingFailed: 'Last Upgrading Failed',
data: 'Data',
+ dataSize: 'Data Size',
+ resource: 'Resource',
resourceCpu: 'Resource Cpu',
resourceMemory: 'Resource Memory',
resourceUsage: 'Resource Usage',
top10: 'Top 10',
numIndex: 'No.',
+ cpuCores: 'CPU Cores',
cpu: 'CPU',
memory: 'Memory',
lastTime: 'Last {time}',
diff --git a/amoro-web/src/language/zh.ts b/amoro-web/src/language/zh.ts
index 0936135cc..c2678c631 100644
--- a/amoro-web/src/language/zh.ts
+++ b/amoro-web/src/language/zh.ts
@@ -17,8 +17,14 @@
*/
export default {
+ last1h: '最近 1小时',
+ last12h: '最近 12小时',
+ last24h: '最近 24小时',
+ last7day: '最近 7天',
healthScore: '健康度',
overview: '总览',
+ top10Tables: '表 TOP 10',
+ latestOperations: '最近操作',
catalogs: '目录',
catalog: '目录',
tables: '表',
@@ -37,6 +43,7 @@ export default {
token: 'Token',
cluster: '集群',
database: '库',
+ tableSize: '表大小',
table: '表',
size: '大小',
file: '文件',
@@ -153,11 +160,14 @@ export default {
duplicateKey: '重复键',
lastUpgradingFailed: '上次升级失败',
data: '数据',
+ dataSize: '数据规模',
+ resource: '资源',
resourceCpu: 'CPU 使用情况',
resourceMemory: '内存使用情况',
resourceUsage: '资源使用情况',
top10: '前 10',
numIndex: '编号',
+ cpuCores: 'CPU 核心数',
cpu: 'CPU',
memory: '内存',
lastTime: '上次 {time}',
diff --git a/amoro-web/src/router/index.ts b/amoro-web/src/router/index.ts
index 21c3cf04c..13537c88c 100644
--- a/amoro-web/src/router/index.ts
+++ b/amoro-web/src/router/index.ts
@@ -30,14 +30,14 @@ const CreateTable = () =>
import('@/views/tables/create.vue')
const Settings = () => import('@/views/settings/index.vue')
const Terminal = () => import('@/views/terminal/index.vue')
const Login = () => import('@/views/login/index.vue')
-const Introduce = () => import('@/views/introduce/index.vue')
const Resource = () => import('@/views/resource/index.vue')
+const Overview = () => import('@/views/overview/index.vue')
const routes: Array<RouteRecordRaw> = [
{
path: '/',
name: 'Home',
- redirect: 'introduce', // overview
+ redirect: 'overview', // overview
component: Home,
children: [
{
@@ -85,9 +85,9 @@ const routes: Array<RouteRecordRaw> = [
component: Terminal,
},
{
- path: 'introduce',
- name: 'Introduce',
- component: Introduce,
+ path: 'overview',
+ name: 'Overview',
+ component: Overview,
},
// {
// path: 'resource',
diff --git a/amoro-web/src/services/overview.service.ts
b/amoro-web/src/services/overview.service.ts
new file mode 100644
index 000000000..f6d1e22be
--- /dev/null
+++ b/amoro-web/src/services/overview.service.ts
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import request from '@/utils/request'
+
+export function getOverviewSummary() {
+ return request.get('ams/v1/overview/summary')
+}
+
+export function getOverviewFormat() {
+ return request.get('ams/v1/overview/format')
+}
+
+export function getOverviewOptimizingStatus() {
+ return request.get('ams/v1/overview/optimizing')
+}
+
+export function getOverviewOperations() {
+ return request.get('ams/v1/overview/operations')
+}
+
+export function getTop10TableList(params: {
+ order?: string
+ orderBy?: string
+ count?: number
+}) {
+ const { order, orderBy, count } = params
+ return request.get(`ams/v1/overview/top`, {
+ params: {
+ order: order || 'asc',
+ orderBy: orderBy || 'healthScore',
+ count: count || 10,
+ },
+ })
+}
+
+export function getResourceUsageList(startTime: number) {
+ return request.get(`ams/v1/overview/resource`, { params: { startTime } })
+}
+
+export function getDataSizeList(startTime: number) {
+ return request.get(`ams/v1/overview/dataSize`, { params: { startTime } })
+}
diff --git a/amoro-web/src/types/common.type.ts
b/amoro-web/src/types/common.type.ts
index 4af04c14d..a2da8d9df 100644
--- a/amoro-web/src/types/common.type.ts
+++ b/amoro-web/src/types/common.type.ts
@@ -72,6 +72,25 @@ export interface IBaseDetailInfo {
healthScore: number
}
+export interface ResourceUsageItem {
+ ts: number
+ totalCpu: number
+ totalMemory: number
+}
+
+export interface DataSizeItem {
+ ts: string
+ dataSize: number
+}
+
+export interface ITopTableItem {
+ tableName: string
+ tableSize: number
+ fileCount: number
+ averageFileSize: number
+ healthScore: number
+}
+
export interface DetailColumnItem {
checked?: unknown
field: string
@@ -189,6 +208,13 @@ export interface SnapshotItem {
recordsSummaryForChart: Record<string, number>
}
+export interface OverviwOperationItem {
+ tableName: string
+ tableIdentifier: ITableIdentifier
+ operation: string
+ ts: number | string
+}
+
export interface OperationItem {
ts: number | string
operation: string
@@ -302,15 +328,6 @@ export interface IChartLineData {
data2: number[] | string[]
}
-export interface ITopTableItem {
- tableName: string
- size: number
- fileCnt: number
- displaySize: string
- tableNameOnly: string
- index: number
-}
-
export interface IOptimizeGroup {
container: string
name: string
diff --git a/amoro-web/src/views/introduce/index-bakl.vue
b/amoro-web/src/views/introduce/index-bakl.vue
deleted file mode 100644
index f3cc5826c..000000000
--- a/amoro-web/src/views/introduce/index-bakl.vue
+++ /dev/null
@@ -1,230 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-/ -->
-
-<script lang="ts">
-import { defineComponent, onMounted } from 'vue'
-
-export default defineComponent({
- name: 'Introduce',
- components: {},
- setup() {
- onMounted(() => {})
- return {}
- },
-})
-</script>
-
-<template>
- <div class="introduce-wrap">
- <h1 class="title">
- Introduce
- </h1>
- <p class="desc">
- Arctic is a LakeHouse management system under open architecture, which
on top of data lake open formats provides more optimizations for streaming and
upsert scenarios, as well as a set of pluggable self-optimizing mechanisms and
management services. Using Arctic could help various data platforms, tools and
products build out-of-the-box, streaming and batch unified LakeHouses quickly.
- </p>
- <h2 class="sub-title">
- What is Arctic
- </h2>
- <p class="desc">
- Currently, Arctic is a LakeHouse management system on top of iceberg
format. Benefit from the thriving ecology of Apache Iceberg, Arctic could be
used on kinds of data lakes on premise or clouds with varities of engines.
Several concepts should be known before your deeper steps:
- </p>
- <img
- src="@/assets/images/introduce_arctic.png"
- alt=""
- class="arctic-introduce"
- >
- <div class="desc">
- <ol>
- <li>AMS and optimizers - Arctic Management Service provides management
features including self-optimizing mechanisms running on optimizers, which
could be scaled as demand and scheduled on different platforms.</li>
- <li>
- Mutiple formats — Arctic use formats analogous to MySQL or
ClickHouse using storage engines to meet different scenarios. Two formats were
available since Arctic v0.4.
- <div>
- <p>
- <span class="dot" />Iceberg format — learn more about iceberg
format details and usage with different engines:
- <a
- href="https://iceberg.apache.org/docs/latest/";
- target="_blank"
- rel="noopener noreferrer"
- >
- Iceberg Docs
- </a>
- </p>
- <p>
- <span class="dot" />Mixed streaming format - if you are
interested in advanced features like auto-bucket, logstore, hive compatible,
strict PK constraints etc. learn Arctic
- <a
-
href="https://arctic.netease.com/ch/concepts/table-formats/#mixed-iceberg-format";
- target="_blank"
- rel="noopener noreferrer"
- >
- Mixed Iceberg format
- </a> and
- <a
-
href="https://arctic.netease.com/ch/concepts/table-formats/#mixed-hive-format";
- target="_blank"
- rel="noopener noreferrer"
- >
- Mixed Hive format
- </a>
- </p>
- </div>
- </li>
- </ol>
- </div>
- <h2 class="sub-title">
- Arctic features
- </h2>
- <ol>
- <li>Defining keys - supports defining primary key with strict
constraints, and more types of keys in future</li>
- <li>Self-optimizing - user-insensitive asynchronous self-optimization
mechanisms could keep lakehouse fresh and healthy</li>
- <li>
- Management features - dashboard UI to support catalog/table
management, SQL terminal and all kinds of metrics
- </li>
- <li>Formats compatible - Hive/Iceberg format compatible means writing
and reading through native Hive/Iceberg connector</li>
- <li>Better data pipeline SLA - using LogStore like kafka to accelarate
streaming data pipeline to ms/s latency</li>
- <li>Better OLAP performace - provides auto-bucket feature for better
compaction and merge-on-read performance</li>
- <li>Concurrent conflicts resovling - Flink or Spark could concurrent
write data without worring about conflicts</li>
- </ol>
- <h2 class="sub-title">
- Engines supported
- </h2>
- <p class="desc">
- Arctic support multiple processing engines as below:
- </p>
- <table class="table">
- <thead class="ant-table-thead">
- <tr>
- <th>Processing Engine</th>
- <th>Version</th>
- </tr>
- </thead>
- <tbody class="ant-table-tbody">
- <tr>
- <td>
- <span class="td-val">Flink</span>
- </td>
- <td>
- <span class="td-val">1.12.x,1.14.x and 1.15.x</span>
- </td>
- </tr>
- <tr>
- <td>
- <span class="td-val">Spark</span>
- </td>
- <td>
- <span class="td-val">2.3, 3.1</span>
- </td>
- </tr>
- <tr>
- <td>
- <span class="td-val">Trino</span>
- </td>
- <td>
- <span class="td-val">380</span>
- </td>
- </tr>
- </tbody>
- </table>
- <h2 class="sub-title">
- Quickstart
- </h2>
- <p class="desc">
- Visit
- <a
- href="https://arctic.netease.com/ch/quickstart/quick-demo/";
- target="_blank"
- rel="noopener noreferrer"
- >
- https://arctic.netease.com/ch/quickstart/quick-demo/
- </a>
- to quickly explore what arctic can do.
- </p>
- <h2 class="sub-title">
- Join Community
- </h2>
- <p class="desc">
- If you are interested in Lakehouse, Data Lake Format, welcome to join
our community, we welcome any organizations, teams and individuals to grow
together, and sincerely hope to help users better use Data Lake Format through
open source.
- </p>
- <p class="desc">
- Join the Arctic WeChat Group: Add " <code>kllnn999</code> " as a friend
on WeChat and specify "Arctic lover".
- </p>
- </div>
-</template>
-
-<style lang="less" scoped>
-.introduce-wrap {
- font-size: 16px;
- box-sizing: border-box;
- padding: 24px 48px;
- max-width: 1200px;
- .title {
- font-size: 30px;
- font-weight: bold;
- padding-bottom: 7px;
- margin-bottom: 16px;
- border-bottom: 1px solid #e5e5e5;
- }
- .arctic-introduce {
- width: 80%;
- margin: 0 auto 16px;
- display: block;
- }
- .desc {
- margin-bottom: 16px;
- }
- .dot {
- display: inline-block;
- width: 6px;
- height: 6px;
- border-radius: 6px;
- border: 1px solid #000;
- margin-right: 12px;
- vertical-align: middle;
- }
- .sub-title {
- margin: 24px 0 16px;
- font-size: 24px;
- font-weight: bold;
- border-bottom: 1px solid #e5e5e5;
- padding-bottom: 7px;
- }
- code {
- padding: 2px 4px;
- background-color: rgba(175, 184, 193, 0.2);
- border-radius: 4px;
- display: inline-block;
- line-height: 14px;
- font-size: 14px;
- }
- ol li {
- line-height: 24px;
- margin-top: 4px;
- padding-left: 32px;
- position: relative;
- &::after {
- content: "";
- position: absolute;
- width: 6px;
- height: 6px;
- border-radius: 100%;
- background-color: #000;
- left: 14px;
- top: 10px;
- }
- }
-}
-</style>
diff --git a/amoro-web/src/views/introduce/index.vue
b/amoro-web/src/views/introduce/index.vue
deleted file mode 100644
index 097c7e999..000000000
--- a/amoro-web/src/views/introduce/index.vue
+++ /dev/null
@@ -1,185 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-/ -->
-
-<script lang="ts">
-import { defineComponent, onMounted } from 'vue'
-import SelfManaged from '@/assets/images/Self-managed.png'
-import MultipleFormats from '@/assets/images/Multiple-Formats.png'
-import CatalogService from '@/assets/images/Catalog-Service.png'
-import RichPlugins from '@/assets/images/Rich-Plugins.png'
-import ManagementTools from '@/assets/images/Management-tools.png'
-import InfrastructureIdependent from
'@/assets/images/Infrastructure-Idependent.png'
-
-export default defineComponent({
- name: 'Introduce',
- components: {},
- setup() {
- onMounted(() => {})
- const features = [
- {
- title: 'Self-optimizing',
- desc: 'Continuously optimizing tables, including compacting small
files, change files, regularly delete expired files to keep high query
performance and reducing storage costs.',
- icon: SelfManaged,
- },
- {
- title: 'Multiple Formats',
- desc: 'Support different table formats such as Iceberg, Mixed-Iceberg
and Mixed-Hive to meet different scenario requirements and provide them with
unified management capabilities.',
- icon: MultipleFormats,
- },
- {
- title: 'Catalog Service',
- desc: 'Provide an unified catalog service for all compute engines,
which can also used with existing metadata store service such as Hive Metastore
and AWS Glue.',
- icon: CatalogService,
- },
- {
- title: 'Rich Plugins',
- desc: 'Provide various plugins to integrate with other systems, like
continuously optimizing with Flink and data analysis with Spark and Kyuubi.',
- icon: RichPlugins,
- },
- {
- title: 'Management Tools',
- desc: 'Provide a variety of management tools, including WEB UI and
standard SQL command line, to help you get started faster and integrate with
other systems more easily.',
- icon: ManagementTools,
- },
- {
- title: 'Infrastructure Independent',
- desc: 'Can be easily deployed and used in private environments, cloud
environments, hybrid cloud environments, and multi-cloud environments.',
- icon: InfrastructureIdependent,
- },
- ]
- return {
- features,
- }
- },
-})
-</script>
-
-<template>
- <div class="introduce-content">
- <section class="home-section">
- <div class="content">
- <div class="title">
- Amoro builds lake-native data warehouse and architecture for users,
platforms and products
- </div>
- <img class="img" src="@/assets/images/home-content.png">
- </div>
- </section>
- <div class="home-feature">
- <div class="content">
- <div class="title">
- Key Features
- </div>
- <div class="features">
- <div v-for="item in features" :key="item.title" class="feature-item">
- <div class="item-title">
- <img :src="item.icon">
- {{ item.title }}
- </div>
- <div class="item-desc">
- {{ item.desc }}
- </div>
- </div>
- </div>
- </div>
- </div>
- </div>
-</template>
-
-<style lang="less" scoped>
-.home-section {
- margin: 0 auto;
- display: flex;
- justify-content: center;
- background-color: #fff;
- .content {
- width: 100%;
- max-width: 1182px;
- padding: 64px 30px;
- .img {
- max-width: 100%;
- }
- }
-}
-
-.home-section .content .title,
-.home-feature .content .title {
- font-size: 20px;
- font-weight: bold;
- line-height: 24px;
- text-align: center;
- margin-bottom: 40px;
-}
-.home-feature {
- background: #f5f6fa;
- margin: 0 auto;
- display: flex;
- justify-content: center;
- .content {
- width: 100%;
- max-width: 1182px;
- padding: 64px 41px 80px;
- .title {
- font-size: 32px;
- }
- .features {
- display: grid;
- grid-row-gap: 24px;
- grid-column-gap: 24px;
- max-width: 1100px;
- grid-template-columns: auto auto;
- .feature-item {
- background-color: #fff;
- padding: 32px;
- border-radius: 5px;
- position: relative;
- .fix-icon {
- position: absolute;
- top: 0px;
- right: 20px;
- }
- }
- }
- }
-}
-
-@media screen and (max-width: 800px) {
- .home-feature .features {
- grid-template-columns: auto;
- }
-}
-
-.home-feature .features .item-title {
- font-size: 24px;
- font-weight: bold;
- line-height: 28px;
- display: flex;
- align-items: center;
-}
-
-.home-feature .features .item-title img {
- margin-right: 13px;
-}
-
-.home-feature .features .item-desc {
- margin-top: 24px;
- font-size: 14px;
- font-weight: normal;
- line-height: 22px;
- color: #53576a;
-}
-</style>
diff --git a/amoro-web/src/views/overview/components/DataSizeCard.vue
b/amoro-web/src/views/overview/components/DataSizeCard.vue
new file mode 100644
index 000000000..e7c0ec72b
--- /dev/null
+++ b/amoro-web/src/views/overview/components/DataSizeCard.vue
@@ -0,0 +1,121 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/ -->
+
+<script lang="ts" setup>
+import { useI18n } from 'vue-i18n'
+import { onMounted, ref } from 'vue'
+import { getDataSizeList } from '@/services/overview.service'
+import { bytesToSize, dateFormat } from '@/utils'
+import type { DataSizeItem } from '@/types/common.type'
+import Chart from '@/components/echarts/Chart.vue'
+
+const { t } = useI18n()
+const timeRange = ref('24')
+const loading = ref<boolean>(false)
+
+function dataSizeFormatter(params: any[]): string {
+ const dataParam = params[0]
+ const dataSize = bytesToSize(dataParam.value)
+ let str = `<span style="font-size: 12px">${params[0].axisValue}</span><br/>`
+ str += `<span style="display:
inline-block;background-color:${dataParam.color}; margin-right: 6px; width:
6px;height: 6px;"></span>${t('dataSize')}: ${dataSize}<br/>`
+ return str
+}
+
+const dataSizeChartOption = ref({
+ tooltip: {
+ trigger: 'axis',
+ formatter: dataSizeFormatter,
+ },
+ legend: { data: [t('dataSize')] },
+ xAxis: {
+ type: 'category',
+ data: [''],
+ },
+ yAxis: {
+ type: 'value',
+ axisLabel: { formatter: (value: number) => { return
`${bytesToSize(value)}` } },
+ },
+ series: [
+ {
+ name: t('dataSize'),
+ type: 'line',
+ data: [-1],
+ },
+ ],
+})
+
+async function updateData() {
+ try {
+ loading.value = true
+ const times: string[] = []
+ const dataSizeList: number[] = []
+
+ const startTime = new Date().getTime() -
Number.parseFloat(timeRange.value) * 60 * 60 * 1000
+ const result: DataSizeItem[] = await getDataSizeList(startTime)
+ result.forEach((item) => {
+ times.push(dateFormat(item.ts))
+ dataSizeList.push(item.dataSize)
+ })
+ dataSizeChartOption.value.xAxis.data = times
+ dataSizeChartOption.value.series[0].data = dataSizeList
+ }
+ catch (error) {
+ }
+ finally {
+ loading.value = false
+ }
+}
+
+onMounted(() => {
+ updateData()
+})
+</script>
+
+<template>
+ <a-card>
+ <template #title>
+ <a-row justify="space-between">
+ <span class="card-title" v-text="t('dataSize')" />
+ <div style="display: flex; justify-content: space-between;
align-items: center;">
+ <a-select v-model:value="timeRange" style="width: 120px"
@change="updateData">
+ <a-select-option value="1">
+ {{ t('last1h') }}
+ </a-select-option>
+ <a-select-option value="12">
+ {{ t('last12h') }}
+ </a-select-option>
+ <a-select-option value="24">
+ {{ t('last24h') }}
+ </a-select-option>
+ <a-select-option value="168">
+ {{ t('last7day') }}
+ </a-select-option>
+ </a-select>
+ </div>
+ </a-row>
+ </template>
+
+ <Chart :loading="loading" :options="dataSizeChartOption" />
+ </a-card>
+</template>
+
+<style scoped>
+.card-title {
+ font-size: 18px;
+}
+</style>
diff --git a/amoro-web/src/views/overview/components/MultipleDataCard.vue
b/amoro-web/src/views/overview/components/MultipleDataCard.vue
new file mode 100644
index 000000000..9d181e761
--- /dev/null
+++ b/amoro-web/src/views/overview/components/MultipleDataCard.vue
@@ -0,0 +1,53 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/ -->
+
+<script setup lang="ts">
+import { useI18n } from 'vue-i18n'
+import { defineProps } from 'vue'
+import type { IKeyAndValue } from '@/types/common.type'
+
+const props = defineProps<{ title: string, data: IKeyAndValue[] }>()
+const { t } = useI18n()
+</script>
+
+<template>
+ <a-card class="multiple-data" :body-style="{ paddingTop: '0px' }">
+ <template #title>
+ <span class="card-title">{{ props.title }}</span>
+ </template>
+ <a-row>
+ <a-col v-for="item in props.data" :key="item.key" :span="12">
+ <a-statistic
+ :title="t(item.key)" :value="item.value"
+ :value-style="{ color: '#07A7F0', fontSize: '28px' }"
style="margin-right: 20px;"
+ />
+ </a-col>
+ </a-row>
+ </a-card>
+</template>
+
+<style scoped>
+.card-title {
+ font-size: 28px;
+}
+
+.multiple-data {
+ padding-top: 0px;
+ height: 150px;
+}
+</style>
diff --git a/amoro-web/src/views/overview/components/PieChartCard.vue
b/amoro-web/src/views/overview/components/PieChartCard.vue
new file mode 100644
index 000000000..7f0b83b89
--- /dev/null
+++ b/amoro-web/src/views/overview/components/PieChartCard.vue
@@ -0,0 +1,81 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/ -->
+
+<script lang="ts" setup>
+import { onMounted, ref, watch } from 'vue'
+import Chart from '@/components/echarts/Chart.vue'
+
+const props = defineProps<{
+ title: string
+ data: {
+ value: number
+ name: string
+ }[]
+}>()
+
+const pieChartOption = ref({})
+
+function renderChart() {
+ pieChartOption.value = {
+ tooltip: {
+ trigger: 'item',
+ },
+ legend: {
+ orient: 'vertical',
+ left: 'left',
+ },
+ series: [
+ {
+ name: props.title,
+ type: 'pie',
+ radius: '50%',
+ data: props.data,
+ emphasis: {
+ itemStyle: {
+ shadowBlur: 10,
+ shadowOffsetX: 0,
+ shadowColor: 'rgba(0, 0, 0, 0.5)',
+ },
+ },
+ },
+ ],
+ }
+}
+
+onMounted(renderChart)
+watch(() => props.data, renderChart)
+</script>
+
+<template>
+ <a-card class="pie-chart-card">
+ <template #title>
+ <span class="card-title" v-text="title" />
+ </template>
+ <Chart :options="pieChartOption" />
+ </a-card>
+</template>
+
+<style scoped>
+.pie-chart-card {
+ height: 500px;
+}
+
+.card-title {
+ font-size: 18px;
+}
+</style>
diff --git a/amoro-web/src/views/overview/components/ResourceUsageCard.vue
b/amoro-web/src/views/overview/components/ResourceUsageCard.vue
new file mode 100644
index 000000000..16c959c9b
--- /dev/null
+++ b/amoro-web/src/views/overview/components/ResourceUsageCard.vue
@@ -0,0 +1,119 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/ -->
+
+<script lang="ts" setup>
+import { useI18n } from 'vue-i18n'
+import { onMounted, ref } from 'vue'
+import Chart from '@/components/echarts/Chart.vue'
+import { bytesToSize, dateFormat } from '@/utils'
+import { getResourceUsageList } from '@/services/overview.service'
+import type { ResourceUsageItem } from '@/types/common.type'
+
+const { t } = useI18n()
+const timeRange = ref('24')
+const loading = ref<boolean>(false)
+
+function resourceFormatter(params: any[]): string {
+ const cpuParam = params[0]
+ const memoryParam = params[1]
+ const memorySize = bytesToSize(memoryParam.value)
+ let str = `<span style="font-size: 12px">${params[0].axisValue}</span><br/>`
+ str += `<span style="display:
inline-block;background-color:${cpuParam.color}; margin-right: 6px; width:
6px;height: 6px;"></span>${t('cpu')}: ${cpuParam.value} Core<br/>`
+ str += `<span style="display:
inline-block;background-color:${memoryParam.color}; margin-right: 6px; width:
6px;height: 6px;"></span>${t('memory')}: ${memorySize}<br/>`
+ return str
+}
+
+const option = ref({
+ tooltip: {
+ trigger: 'axis',
+ formatter: resourceFormatter,
+ },
+ legend: { data: [t('cpu'), t('memory')] },
+ xAxis: { type: 'category', data: [''] },
+ yAxis: [
+ { type: 'value', name: t('cpuCores'), axisLabel: { formatter: '{value}' }
},
+ { type: 'value', name: t('memory'), axisLabel: { formatter: (value:
number) => { return `${bytesToSize(value)}` } } },
+ ],
+ series: [
+ { name: t('cpu'), type: 'line', yAxisIndex: 0, data: [-1] },
+ { name: t('memory'), type: 'line', yAxisIndex: 1, data: [-1] },
+ ],
+})
+
+async function updateData() {
+ try {
+ loading.value = true
+ const times: string[] = []
+ const cpu: number[] = []
+ const memory: number[] = []
+
+ const startTime = new Date().getTime() -
Number.parseFloat(timeRange.value) * 60 * 60 * 1000
+ const result: ResourceUsageItem[] = await getResourceUsageList(startTime)
+ result.forEach((item) => {
+ times.push(dateFormat(item.ts))
+ cpu.push(item.totalCpu)
+ memory.push(item.totalMemory)
+ })
+ option.value.xAxis.data = times
+ option.value.series[0].data = cpu
+ option.value.series[1].data = memory
+ }
+ catch (error) {
+ }
+ finally {
+ loading.value = false
+ }
+}
+
+onMounted(() => {
+ updateData()
+})
+</script>
+
+<template>
+ <a-card>
+ <template #title>
+ <a-row justify="space-between">
+ <span class="card-title" v-text="t('resourceUsage')" />
+ <div style="display: flex; justify-content: space-between;
align-items: center;">
+ <a-select v-model:value="timeRange" style="width: 120px"
@change="updateData">
+ <a-select-option value="1">
+ {{ t('last1h') }}
+ </a-select-option>
+ <a-select-option value="12">
+ {{ t('last12h') }}
+ </a-select-option>
+ <a-select-option value="24">
+ {{ t('last24h') }}
+ </a-select-option>
+ <a-select-option value="168">
+ {{ t('last7day') }}
+ </a-select-option>
+ </a-select>
+ </div>
+ </a-row>
+ </template>
+ <Chart :loading="loading" :options="option" />
+ </a-card>
+</template>
+
+<style scoped>
+.card-title {
+ font-size: 18px;
+}
+</style>
diff --git a/amoro-web/src/views/overview/components/SingleDataCard.vue
b/amoro-web/src/views/overview/components/SingleDataCard.vue
new file mode 100644
index 000000000..ce8fe40e2
--- /dev/null
+++ b/amoro-web/src/views/overview/components/SingleDataCard.vue
@@ -0,0 +1,45 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/ -->
+
+<script setup lang="ts">
+import { defineProps } from 'vue'
+
+const props = defineProps<{ title: string, value: string }>()
+</script>
+
+<template>
+ <a-card class="single-data">
+ <template #title>
+ <span class="card-title">{{ props.title }}</span>
+ </template>
+ <a-statistic
+ :value="props.value" :value-style="{ color: '#07A7F0', fontSize: '28px'
}"
+ style="margin-right: 50px"
+ />
+ </a-card>
+</template>
+
+<style scoped>
+.card-title {
+ font-size: 28px;
+}
+
+.single-data {
+ height: 150px;
+}
+</style>
diff --git a/amoro-web/src/views/overview/components/Top10TablesCard.vue
b/amoro-web/src/views/overview/components/Top10TablesCard.vue
new file mode 100644
index 000000000..db24af469
--- /dev/null
+++ b/amoro-web/src/views/overview/components/Top10TablesCard.vue
@@ -0,0 +1,181 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/ -->
+
+<script lang="ts" setup>
+import { computed, onMounted, reactive, ref } from 'vue'
+import { useI18n } from 'vue-i18n'
+import { useRouter } from 'vue-router'
+import type { ITopTableItem } from '@/types/common.type'
+import { getTop10TableList } from '@/services/overview.service'
+import { bytesToSize } from '@/utils'
+
+const { t } = useI18n()
+const router = useRouter()
+const orderBy = ref('healthScore')
+const loading = ref<boolean>(false)
+const dataSource = reactive<ITopTableItem[]>([])
+
+const columns = computed(() => [
+ {
+ title: t('tableName'),
+ dataIndex: 'tableName',
+ width: 300,
+ ellipsis: true,
+ },
+ {
+ title: t('tableSize'),
+ dataIndex: 'tableSize',
+ width: 100,
+ ellipsis: true,
+ },
+ {
+ title: t('fileCount'),
+ dataIndex: 'fileCount',
+ width: 100,
+ ellipsis: true,
+ },
+ {
+ title: t('averageFileSize'),
+ dataIndex: 'averageFileSize',
+ width: 140,
+ ellipsis: true,
+ },
+ {
+ title: t('healthScore'),
+ dataIndex: 'healthScore',
+ width: 110,
+ ellipsis: true,
+ },
+])
+
+function goTableDetail(record: ITopTableItem) {
+ try {
+ const table = (record.tableName || '').split('.')
+ router.push({
+ path: '/tables',
+ query: {
+ catalog: table[0],
+ db: table[1],
+ table: table[2],
+ },
+ })
+ }
+ catch (error) {
+ }
+}
+
+async function getTop10Tables() {
+ try {
+ loading.value = true
+ dataSource.length = 0
+ const params = {
+ order: orderBy.value === 'healthScore' ? 'asc' : 'desc',
+ orderBy: orderBy.value,
+ }
+ const result = await getTop10TableList(params)
+ result.forEach((ele: ITopTableItem) => {
+ dataSource.push(ele)
+ })
+ }
+ catch (error) {
+ }
+ finally {
+ loading.value = false
+ }
+}
+
+onMounted(() => {
+ getTop10Tables()
+})
+</script>
+
+<template>
+ <a-card class="top-tables-card">
+ <template #title>
+ <a-row justify="space-between">
+ <span class="card-title" v-text="t('top10Tables')" />
+
+ <div style="display: flex; justify-content: space-between;
align-items: center;">
+ <a-select v-model:value="orderBy" style="width: 150px"
@change="getTop10Tables">
+ <a-select-option value="tableSize">
+ {{ t('tableSize') }}
+ </a-select-option>
+ <a-select-option value="fileCount">
+ {{ t('fileCount') }}
+ </a-select-option>
+ <a-select-option value="healthScore">
+ {{ t('healthScore') }}
+ </a-select-option>
+ </a-select>
+ </div>
+ </a-row>
+ </template>
+
+ <div class="list-wrap">
+ <a-table
+ class="ant-table-common" :columns="columns" :data-source="dataSource"
:scroll="{ x: '100%', y: 350 }" :loading="loading"
+ :pagination="false"
+ >
+ <template #bodyCell="{ column, record }">
+ <template v-if="column.dataIndex === 'tableName'">
+ <span :title="record.tableName" class="primary-link"
@click="goTableDetail(record)">
+ {{ record.tableName }}
+ </span>
+ </template>
+ <template v-if="column.dataIndex === 'tableSize'">
+ {{ bytesToSize(record.tableSize) }}
+ </template>
+ <template v-if="column.dataIndex === 'averageFileSize'">
+ {{ bytesToSize(record.averageFileSize) }}
+ </template>
+ <template v-if="column.dataIndex === 'healthScore'">
+ {{ record.healthScore == null || record.healthScore < 0 ? 'N/A' :
record.healthScore }}
+ </template>
+ </template>
+ </a-table>
+ </div>
+ </a-card>
+</template>
+
+<style lang="less" scoped>
+.card-title {
+ font-size: 18px;
+}
+
+.top-tables-card {
+ height: 500px;
+}
+
+.list-wrap {
+ .primary-link {
+ color: @primary-color;
+
+ &:hover {
+ cursor: pointer;
+ }
+
+ &.disabled {
+ color: #999;
+
+ &:hover {
+ cursor: not-allowed;
+ }
+ }
+ }
+}
+</style>
diff --git a/amoro-web/src/views/overview/index.vue
b/amoro-web/src/views/overview/index.vue
new file mode 100644
index 000000000..fe0089338
--- /dev/null
+++ b/amoro-web/src/views/overview/index.vue
@@ -0,0 +1,82 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/ -->
+
+<script lang="ts" setup>
+import { onMounted, ref } from 'vue'
+import { useI18n } from 'vue-i18n'
+import SingleDataCard from './components/SingleDataCard.vue'
+import MultipleDataCard from './components/MultipleDataCard.vue'
+import Top10TablesCard from './components/Top10TablesCard.vue'
+import ResourceUsageCard from './components/ResourceUsageCard.vue'
+import DataSizeCard from './components/DataSizeCard.vue'
+import PieChartCard from './components/PieChartCard.vue'
+import { getOverviewOptimizingStatus, getOverviewSummary } from
'@/services/overview.service'
+import { bytesToSize } from '@/utils'
+import type { IKeyAndValue } from '@/types/common.type'
+
+const { t } = useI18n()
+
+const singleData = ref<IKeyAndValue[]>([])
+const multipleData = ref<IKeyAndValue[]>([])
+const optimizingStatusData = ref<{ value: number, name: string }[]>([])
+
+async function getCurOverviewData() {
+ const summaryResult = await getOverviewSummary()
+ const tableSize = bytesToSize(summaryResult.tableTotalSize)
+ const memorySize = bytesToSize(summaryResult.totalMemory)
+ singleData.value.push({ key: 'catalog', value: summaryResult.catalogCnt })
+ singleData.value.push({ key: 'table', value: summaryResult.tableCnt })
+ singleData.value.push({ key: 'data', value: tableSize })
+ multipleData.value.push({ key: 'cpu', value: summaryResult.totalCpu })
+ multipleData.value.push({ key: 'memory', value: memorySize })
+
+ optimizingStatusData.value = await getOverviewOptimizingStatus()
+}
+
+onMounted(() => {
+ getCurOverviewData()
+})
+</script>
+
+<template>
+ <div :style="{ background: '#F8F7F8', padding: '24px', minHeight: '900px' }"
class="overview-content">
+ <a-row :gutter="[16, 8]">
+ <a-col v-for="(card, index) in singleData" :key="index" :span="6">
+ <SingleDataCard :title="t(card.key)" :value="card.value" />
+ </a-col>
+ <a-col :span="6">
+ <MultipleDataCard :title="t('resource')" :data="multipleData" />
+ </a-col>
+ <a-col :span="12">
+ <ResourceUsageCard />
+ </a-col>
+ <a-col :span="12">
+ <DataSizeCard />
+ </a-col>
+ <a-col :span="12">
+ <PieChartCard :title="t('optimizingStatus')"
:data="optimizingStatusData" />
+ </a-col>
+ <a-col :span="12">
+ <Top10TablesCard />
+ </a-col>
+ </a-row>
+ </div>
+</template>
+
+<style lang="less" scoped>
+</style>
diff --git a/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml
b/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml
index 1a9639f6a..52c8924f3 100644
--- a/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml
+++ b/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml
@@ -10,7 +10,7 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.max-history-records
# See the License for the specific language governing permissions and
# limitations under the License.
#
@@ -18,8 +18,15 @@
# This file is config file for plugin metric reporter
# configurations of metric reporters
-#metric-reporters:
+metric-reporters:
+ - name: overview-exporter # configs for overview exporter
+ enabled: true # if false, overview page will not
fetch data
+ properties:
+ refresh-interval: 180000 # 3 min
+ max-history-records: 3360 # Keep 7 days history by default,
7 * 24 * 60 / 3 = 3360
+
# - name: prometheus-exporter # configs for prometheus exporter
-# enabled: true
+# enabled: false
# properties:
-# port: 7001
\ No newline at end of file
+# port: 7001
+
diff --git a/docs/user-guides/metrics.md b/docs/user-guides/metrics.md
index d65ce4c9e..435413af7 100644
--- a/docs/user-guides/metrics.md
+++ b/docs/user-guides/metrics.md
@@ -65,16 +65,18 @@ Amoro has supported built-in metrics to measure status of
table self-optimizing
## Optimizer Group metrics
-| Metric Name | Type | Tags | Description
|
-|----------------------------------------|--------|-------|--------------------------------------------------|
-| optimizer_group_pending_tasks | Gauge | group | Number of pending
tasks in optimizer group |
-| optimizer_group_executing_tasks | Gauge | group | Number of
executing tasks in optimizer group |
-| optimizer_group_planing_tables | Gauge | group | Number of planing
tables in optimizer group |
-| optimizer_group_pending_tables | Gauge | group | Number of pending
tables in optimizer group |
-| optimizer_group_executing_tables | Gauge | group | Number of
executing tables in optimizer group |
-| optimizer_group_optimizer_instances | Gauge | group | Number of
optimizer instances in optimizer group |
-| optimizer_group_memory_bytes_allocated | Gauge | group | Memory bytes
allocated in optimizer group |
-| optimizer_group_threads | Gauge | group | Number of total
threads in optimizer group |
+| Metric Name | Type | Tags | Description
|
+|-----------------------------------------|--------|-------|--------------------------------------------------|
+| optimizer_group_pending_tasks | Gauge | group | Number of pending
tasks in optimizer group |
+| optimizer_group_executing_tasks | Gauge | group | Number of
executing tasks in optimizer group |
+| optimizer_group_planing_tables | Gauge | group | Number of planing
tables in optimizer group |
+| optimizer_group_pending_tables | Gauge | group | Number of pending
tables in optimizer group |
+| optimizer_group_executing_tables | Gauge | group | Number of
executing tables in optimizer group |
+| optimizer_group_idle_tables | Gauge | group | Number of idle
tables in optimizer group |
+| optimizer_group_committing_tables | Gauge | group | Number of
committing tables in optimizer group |
+| optimizer_group_optimizer_instances | Gauge | group | Number of
optimizer instances in optimizer group |
+| optimizer_group_memory_bytes_allocated | Gauge | group | Memory bytes
allocated in optimizer group |
+| optimizer_group_threads | Gauge | group | Number of total
threads in optimizer group |
## Orphan Files Cleaning metrics
