This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 04df7fedfe1 Enable table based filtering for minion subtask details
(#16977)
04df7fedfe1 is described below
commit 04df7fedfe14091880087f92a062c1e6a2a1e1c2
Author: Shounak kulkarni <[email protected]>
AuthorDate: Thu Oct 9 03:02:34 2025 +0530
Enable table based filtering for minion subtask details (#16977)
---
.../api/resources/PinotTaskRestletResource.java | 10 ++++--
.../core/minion/PinotHelixTaskResourceManager.java | 37 +++++++++++++++-------
.../src/main/resources/app/pages/SubTaskDetail.tsx | 4 +--
.../src/main/resources/app/pages/TaskDetail.tsx | 2 +-
.../src/main/resources/app/requests/index.ts | 4 +--
.../main/resources/app/utils/PinotMethodUtils.ts | 6 ++--
.../minion/PinotHelixTaskResourceManagerTest.java | 23 ++++++++------
7 files changed, 55 insertions(+), 31 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
index 90e6732dc96..56c5aed10e8 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
@@ -384,8 +384,14 @@ public class PinotTaskRestletResource {
@ApiParam(value = "verbosity (Prints information for the given task
name."
+ "By default, only prints subtask details for running and error
tasks. "
+ "Value of > 0 prints subtask details for all tasks)")
- @DefaultValue("0") @QueryParam("verbosity") int verbosity) {
- return _pinotHelixTaskResourceManager.getTaskDebugInfo(taskName,
verbosity);
+ @DefaultValue("0") @QueryParam("verbosity") int verbosity,
+ @ApiParam(value = "Table name with type (e.g., 'myTable_OFFLINE') to
filter subtasks by table. "
+ + "Only subtasks for this table will be returned.")
+ @QueryParam("tableName") @Nullable String tableNameWithType, @Context
HttpHeaders httpHeaders) {
+ if (tableNameWithType != null) {
+ tableNameWithType = DatabaseUtils.translateTableName(tableNameWithType,
httpHeaders);
+ }
+ return _pinotHelixTaskResourceManager.getTaskDebugInfo(taskName,
tableNameWithType, verbosity);
}
@GET
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
index 8415b5c2b71..3b44228c543 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
@@ -965,13 +965,27 @@ public class PinotHelixTaskResourceManager {
* @return TaskDebugInfo contains details for subtasks in this task batch.
*/
public synchronized TaskDebugInfo getTaskDebugInfo(String taskName, int
verbosity) {
+ return getTaskDebugInfo(taskName, null, verbosity);
+ }
+
+ /**
+ * Given a taskName and table name collects status of the (sub)tasks in the
taskName for the table.
+ *
+ * @param taskName Pinot taskName
+ * @param tableNameWithType table name for which subtask status to fetch
+ * @param verbosity By default, does not show details for completed
tasks.
+ * If verbosity > 0, shows details for all tasks.
+ * @return TaskDebugInfo contains details for subtasks in this task batch.
+ */
+ public synchronized TaskDebugInfo getTaskDebugInfo(String taskName,
@Nullable String tableNameWithType,
+ int verbosity) {
String taskType = getTaskType(taskName);
WorkflowContext workflowContext =
_taskDriver.getWorkflowContext(getHelixJobQueueName(taskType));
if (workflowContext == null) {
return null;
}
String helixJobName = getHelixJobName(taskName);
- return getTaskDebugInfo(workflowContext, helixJobName, null, verbosity);
+ return getTaskDebugInfo(workflowContext, helixJobName, tableNameWithType,
verbosity);
}
private synchronized TaskDebugInfo getTaskDebugInfo(WorkflowContext
workflowContext, String helixJobName,
@@ -1003,14 +1017,23 @@ public class PinotHelixTaskResourceManager {
TaskCount subtaskCount = new TaskCount();
for (int partition : partitionSet) {
// First get the partition's state and update the subtaskCount
+ String taskIdForPartition =
jobContext.getTaskIdForPartition(partition);
TaskPartitionState partitionState =
jobContext.getPartitionState(partition);
+ TaskConfig helixTaskConfig =
jobConfig.getTaskConfig(taskIdForPartition);
+ PinotTaskConfig pinotTaskConfig = null;
+ if (helixTaskConfig != null) {
+ pinotTaskConfig =
PinotTaskConfig.fromHelixTaskConfig(helixTaskConfig);
+ if ((tableNameWithType != null) &&
(!tableNameWithType.equals(pinotTaskConfig.getTableName()))) {
+ // Filter task configs that match this table name
+ continue;
+ }
+ }
subtaskCount.addTaskState(partitionState);
// Skip details for COMPLETED tasks
if (!showCompleted && partitionState == TaskPartitionState.COMPLETED) {
continue;
}
SubtaskDebugInfo subtaskDebugInfo = new SubtaskDebugInfo();
- String taskIdForPartition =
jobContext.getTaskIdForPartition(partition);
subtaskDebugInfo.setTaskId(taskIdForPartition);
subtaskDebugInfo.setState(partitionState);
subtaskDebugInfo.setTriggeredBy(triggeredBy);
@@ -1024,15 +1047,7 @@ public class PinotHelixTaskResourceManager {
}
subtaskDebugInfo.setParticipant(jobContext.getAssignedParticipant(partition));
subtaskDebugInfo.setInfo(jobContext.getPartitionInfo(partition));
- TaskConfig helixTaskConfig =
jobConfig.getTaskConfig(taskIdForPartition);
- if (helixTaskConfig != null) {
- PinotTaskConfig pinotTaskConfig =
PinotTaskConfig.fromHelixTaskConfig(helixTaskConfig);
- if ((tableNameWithType != null) &&
(!tableNameWithType.equals(pinotTaskConfig.getTableName()))) {
- // Filter task configs that match this table name
- continue;
- }
- subtaskDebugInfo.setTaskConfig(pinotTaskConfig);
- }
+ subtaskDebugInfo.setTaskConfig(pinotTaskConfig);
taskDebugInfo.addSubtaskInfo(subtaskDebugInfo);
}
taskDebugInfo.setSubtaskCount(subtaskCount);
diff --git a/pinot-controller/src/main/resources/app/pages/SubTaskDetail.tsx
b/pinot-controller/src/main/resources/app/pages/SubTaskDetail.tsx
index 282e14ba1cd..39f94b24aed 100644
--- a/pinot-controller/src/main/resources/app/pages/SubTaskDetail.tsx
+++ b/pinot-controller/src/main/resources/app/pages/SubTaskDetail.tsx
@@ -80,12 +80,12 @@ const useStyles = makeStyles(() => ({
const TaskDetail = (props) => {
const classes = useStyles();
const { currentTimezone } = useTimezone();
- const { subTaskID, taskID } = props.match.params;
+ const { subTaskID, taskID, queueTableName } = props.match.params;
const [taskDebugData, setTaskDebugData] = useState({});
const [taskProgressData, setTaskProgressData] =
useState<TaskProgressStatus[] | string>("");
const fetchTaskDebugData = async () => {
- const debugRes = await PinotMethodUtils.getTaskDebugData(taskID);
+ const debugRes = await PinotMethodUtils.getTaskDebugData(taskID,
queueTableName);
const subTaskData = find(debugRes.data.subtaskInfos, (subTask) =>
get(subTask, 'taskId', '') === subTaskID);
setTaskDebugData(subTaskData);
};
diff --git a/pinot-controller/src/main/resources/app/pages/TaskDetail.tsx
b/pinot-controller/src/main/resources/app/pages/TaskDetail.tsx
index a70182dac47..a35005ab2c7 100644
--- a/pinot-controller/src/main/resources/app/pages/TaskDetail.tsx
+++ b/pinot-controller/src/main/resources/app/pages/TaskDetail.tsx
@@ -83,7 +83,7 @@ const TaskDetail = (props) => {
const fetchData = useCallback(async () => {
setFetching(true);
const [debugRes, runtimeConfig] = await Promise.all([
- PinotMethodUtils.getTaskDebugData(taskID),
+ PinotMethodUtils.getTaskDebugData(taskID, queueTableName),
PinotMethodUtils.getTaskRuntimeConfigData(taskID)
]);
const subtaskTableRecords = [];
diff --git a/pinot-controller/src/main/resources/app/requests/index.ts
b/pinot-controller/src/main/resources/app/requests/index.ts
index 16101d2ad27..eedcca327a6 100644
--- a/pinot-controller/src/main/resources/app/requests/index.ts
+++ b/pinot-controller/src/main/resources/app/requests/index.ts
@@ -209,8 +209,8 @@ export const getTasks = (tableName: string, taskType:
string): Promise<AxiosResp
export const getTaskRuntimeConfig = (taskName: string):
Promise<AxiosResponse<TaskRuntimeConfig>> =>
baseApi.get(`/tasks/task/${taskName}/runtime/config`, { headers: {
...headers, Accept: 'application/json' }});
-export const getTaskDebug = (taskName: string):
Promise<AxiosResponse<OperationResponse>> =>
- baseApi.get(`/tasks/task/${taskName}/debug?verbosity=1`, { headers: {
...headers, Accept: 'application/json' } });
+export const getTaskDebug = (taskName: string, tableName: string):
Promise<AxiosResponse<OperationResponse>> =>
+
baseApi.get(`/tasks/task/${taskName}/debug?verbosity=1&tableName=${tableName}`,
{ headers: { ...headers, Accept: 'application/json' } });
export const getTaskProgress = (taskName: string, subTaskName: string):
Promise<AxiosResponse<TaskProgressResponse>> =>
baseApi.get(`/tasks/subtask/${taskName}/progress`, { headers: { ...headers,
Accept: 'application/json' }, params: {subtaskNames: subTaskName} });
diff --git a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
index f184207a736..c74568a5c00 100644
--- a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
+++ b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
@@ -943,7 +943,7 @@ const getTasksList = async (tableName, taskType) => {
getTasks(tableName, taskType).then(async (response)=>{
const promiseArr = [];
const fetchInfo = async (taskID, status) => {
- const debugData = await getTaskDebugData(taskID);
+ const debugData = await getTaskDebugData(taskID, tableName);
const subtaskCount = get(debugData, 'data.subtaskCount', {});
const total = get(subtaskCount, 'total', 0);
const completed = get(subtaskCount, 'completed', 0);
@@ -980,8 +980,8 @@ const getTaskRuntimeConfigData = async (taskName: string)
=> {
return response.data;
}
-const getTaskDebugData = async (taskName) => {
- const debugRes = await getTaskDebug(taskName);
+const getTaskDebugData = async (taskName, tableName) => {
+ const debugRes = await getTaskDebug(taskName, tableName);
return debugRes;
};
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManagerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManagerTest.java
index cee3c246f46..c920b66433d 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManagerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManagerTest.java
@@ -414,20 +414,23 @@ public class PinotHelixTaskResourceManagerTest {
// Test filtering by table
Map<String, PinotHelixTaskResourceManager.TaskDebugInfo> result =
- mgr.getTasksDebugInfoByTable(taskType, tableNameWithType, 0);
-
- // Verify that only tasks for the target table are returned
+ mgr.getTasksDebugInfoByTable(taskType, tableNameWithType, 1);
assertEquals(result.size(), 1);
assertTrue(result.containsKey(taskName));
+ verifyResultFilteredByTable(result.get(taskName), "task1",
TaskPartitionState.RUNNING, tableNameWithType);
- PinotHelixTaskResourceManager.TaskDebugInfo taskDebugInfo =
result.get(taskName);
- assertEquals(taskDebugInfo.getTaskState(), TaskState.IN_PROGRESS);
+ PinotHelixTaskResourceManager.TaskDebugInfo result2 =
+ mgr.getTaskDebugInfo(taskName, otherTableNameWithType, 1);
+ verifyResultFilteredByTable(result2, "task2",
TaskPartitionState.COMPLETED, otherTableNameWithType);
+ }
+ private static void
verifyResultFilteredByTable(PinotHelixTaskResourceManager.TaskDebugInfo
taskDebugInfo,
+ String taskId, TaskPartitionState state, String tableNameWithType) {
// Verify that only subtasks for the target table are included
List<PinotHelixTaskResourceManager.SubtaskDebugInfo> subtaskInfos =
taskDebugInfo.getSubtaskInfos();
assertEquals(subtaskInfos.size(), 1);
- assertEquals(subtaskInfos.get(0).getTaskId(), "task1");
- assertEquals(subtaskInfos.get(0).getState(), TaskPartitionState.RUNNING);
+ assertEquals(subtaskInfos.get(0).getTaskId(), taskId);
+ assertEquals(subtaskInfos.get(0).getState(), state);
// Verify the task config belongs to the target table
PinotTaskConfig taskConfig = subtaskInfos.get(0).getTaskConfig();
@@ -555,12 +558,12 @@ public class PinotHelixTaskResourceManagerTest {
PinotHelixTaskResourceManager mgr = new
PinotHelixTaskResourceManager(helixResourceManager, taskDriver);
// Test getTaskDebugInfo with table filtering (should not filter when
tableNameWithType is null)
- PinotHelixTaskResourceManager.TaskDebugInfo result =
mgr.getTaskDebugInfo(taskName, 0);
+ PinotHelixTaskResourceManager.TaskDebugInfo result =
mgr.getTaskDebugInfo(taskName, null, 1);
// Verify that all subtasks are included when no table filtering is applied
assertEquals(result.getTaskState(), TaskState.IN_PROGRESS);
List<PinotHelixTaskResourceManager.SubtaskDebugInfo> subtaskInfos =
result.getSubtaskInfos();
- assertEquals(subtaskInfos.size(), 1); // Only running tasks are shown with
verbosity = 0
+ assertEquals(subtaskInfos.size(), 2);
// Verify the subtask is included
assertEquals(subtaskInfos.get(0).getTaskId(), "task1");
@@ -589,7 +592,7 @@ public class PinotHelixTaskResourceManagerTest {
PinotHelixTaskResourceManager mgr = new
PinotHelixTaskResourceManager(helixResourceManager, taskDriver);
// Test getTaskDebugInfo with null job context
- PinotHelixTaskResourceManager.TaskDebugInfo result =
mgr.getTaskDebugInfo(taskName, 0);
+ PinotHelixTaskResourceManager.TaskDebugInfo result =
mgr.getTaskDebugInfo(taskName, null, 0);
// Verify that basic task info is still returned even with null job context
assertEquals(result.getTaskState(), TaskState.IN_PROGRESS);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
