This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1a82ba6fc6 Add CPU metrics for minion purge task (#12337)
1a82ba6fc6 is described below
commit 1a82ba6fc6e9ac90f8149bf9432b8cc94eeab8b0
Author: Florence Zhang <flore...@gmail.com>
AuthorDate: Mon Jan 29 21:01:02 2024 -0800
Add CPU metrics for minion purge task (#12337)
---
.../org/apache/pinot/common/metrics/MinionTimer.java | 2 +-
.../plugin/minion/tasks/purge/PurgeTaskExecutor.java | 19 +++++++++++++++++--
.../minion/tasks/purge/PurgeTaskExecutorTest.java | 3 ++-
3 files changed, 20 insertions(+), 4 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java
index ff29a2c367..67d2830c81 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java
@@ -22,7 +22,7 @@ import org.apache.pinot.common.Utils;
public enum MinionTimer implements AbstractMetrics.Timer {
- TASK_EXECUTION(false), TASK_QUEUEING(false);
+ TASK_EXECUTION(false), TASK_QUEUEING(false), TASK_THREAD_CPU_TIME_NS(false);
private final String _timerName;
private final boolean _global;
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutor.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutor.java
index bbc9af3b54..f6b0ca8e39 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutor.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutor.java
@@ -19,9 +19,14 @@
package org.apache.pinot.plugin.minion.tasks.purge;
import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
import java.util.Collections;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import
org.apache.pinot.common.metadata.segment.SegmentZKMetadataCustomMapModifier;
+import org.apache.pinot.common.metrics.MinionMetrics;
+import org.apache.pinot.common.metrics.MinionTimer;
import org.apache.pinot.core.common.MinionConstants;
import org.apache.pinot.core.minion.PinotTaskConfig;
import org.apache.pinot.core.minion.SegmentPurger;
@@ -30,13 +35,18 @@ import
org.apache.pinot.plugin.minion.tasks.SegmentConversionResult;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PurgeTaskExecutor extends BaseSingleSegmentConversionExecutor {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(PurgeTaskExecutor.class);
+ protected final MinionMetrics _minionMetrics = MinionMetrics.get();
public static final String RECORD_PURGER_KEY = "recordPurger";
public static final String RECORD_MODIFIER_KEY = "recordModifier";
public static final String NUM_RECORDS_PURGED_KEY = "numRecordsPurged";
public static final String NUM_RECORDS_MODIFIED_KEY = "numRecordsModified";
+ private static final ThreadMXBean MX_BEAN =
ManagementFactory.getThreadMXBean();
@Override
protected SegmentConversionResult convert(PinotTaskConfig pinotTaskConfig,
File indexDir, File workingDir)
@@ -44,6 +54,7 @@ public class PurgeTaskExecutor extends
BaseSingleSegmentConversionExecutor {
Map<String, String> configs = pinotTaskConfig.getConfigs();
String tableNameWithType = configs.get(MinionConstants.TABLE_NAME_KEY);
String rawTableName =
TableNameBuilder.extractRawTableName(tableNameWithType);
+ String taskType = pinotTaskConfig.getTaskType();
SegmentPurger.RecordPurgerFactory recordPurgerFactory =
MINION_CONTEXT.getRecordPurgerFactory();
TableConfig tableConfig = getTableConfig(tableNameWithType);
@@ -57,7 +68,11 @@ public class PurgeTaskExecutor extends
BaseSingleSegmentConversionExecutor {
_eventObserver.notifyProgress(pinotTaskConfig, "Purging segment: " +
indexDir);
SegmentPurger segmentPurger =
new SegmentPurger(indexDir, workingDir, tableConfig, schema,
recordPurger, recordModifier);
+ long purgeTaskStartTimeNs = MX_BEAN.getCurrentThreadCpuTime();
File purgedSegmentFile = segmentPurger.purgeSegment();
+ long purgeTaskEndTimeNs = MX_BEAN.getCurrentThreadCpuTime();
+ _minionMetrics.addTimedTableValue(tableNameWithType, taskType,
MinionTimer.TASK_THREAD_CPU_TIME_NS,
+ purgeTaskEndTimeNs - purgeTaskStartTimeNs, TimeUnit.NANOSECONDS);
if (purgedSegmentFile == null) {
purgedSegmentFile = indexDir;
}
@@ -73,8 +88,8 @@ public class PurgeTaskExecutor extends
BaseSingleSegmentConversionExecutor {
@Override
protected SegmentZKMetadataCustomMapModifier
getSegmentZKMetadataCustomMapModifier(PinotTaskConfig pinotTaskConfig,
SegmentConversionResult segmentConversionResult) {
- return new
SegmentZKMetadataCustomMapModifier(SegmentZKMetadataCustomMapModifier.ModifyMode.UPDATE,
Collections
- .singletonMap(MinionConstants.PurgeTask.TASK_TYPE +
MinionConstants.TASK_TIME_SUFFIX,
+ return new
SegmentZKMetadataCustomMapModifier(SegmentZKMetadataCustomMapModifier.ModifyMode.UPDATE,
+ Collections.singletonMap(MinionConstants.PurgeTask.TASK_TYPE +
MinionConstants.TASK_TIME_SUFFIX,
String.valueOf(System.currentTimeMillis())));
}
}
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutorTest.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutorTest.java
index 29a2cdd25a..8cf0cb1414 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutorTest.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/purge/PurgeTaskExecutorTest.java
@@ -26,6 +26,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.helix.AccessOption;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.pinot.common.metrics.MinionMetrics;
import org.apache.pinot.common.utils.SchemaUtils;
import org.apache.pinot.common.utils.config.TableConfigUtils;
import org.apache.pinot.core.common.MinionConstants;
@@ -70,7 +71,7 @@ public class PurgeTaskExecutorTest {
public void setUp()
throws Exception {
FileUtils.deleteDirectory(TEMP_DIR);
-
+ MinionMetrics.register(Mockito.mock(MinionMetrics.class));
TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
Schema schema = new Schema.SchemaBuilder().addSingleValueDimension(D1,
FieldSpec.DataType.INT).build();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
