Ravi Nori has uploaded a new change for review. Change subject: core : Handle incomplete tasks on server restart ......................................................................
core : Handle incomplete tasks on server restart This patch chnages the way asyc task place holders are cleared on server restart for commands that are partially submitted to vdsm. If a task has 4 place holders and only 2 have been submitted to vdsm, the database has 2 tasks with vdsm id and 2 with empty vdsm id on server restarted. When the server is restarted the tasks are grouped by root command id and the commands that have partially submitted taska ra handled differently. The task place holder with empty vdsm id are removed from the database. The tasks which have vdsm id are added to the partiallyCompletedCommandTasks list. On storage up event when AddStoragePoolExistingTasks is called if the task is still running on vdsm we attempt to stop it and clear the task. If the task has already completed we fail the task and remove it from database. Change-Id: I3a979d816d9ec8cf25119a33742c8e6af4ff42a8 Signed-off-by: Ravi Nori <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AsyncTaskManager.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CommandBase.java 2 files changed, 120 insertions(+), 35 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/30/15630/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AsyncTaskManager.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AsyncTaskManager.java index 75a90e7..2f3cec4 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AsyncTaskManager.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AsyncTaskManager.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -67,7 +68,7 @@ /**Map of tasks in DB per storage pool that exist after restart **/ private ConcurrentMap<Guid, List<AsyncTasks>> tasksInDbAfterRestart = null; - + private List<AsyncTasks> partiallyCompletedCommandTasks = new ArrayList<>(); private static final AsyncTaskManager _taskManager = new AsyncTaskManager(); @@ -88,17 +89,18 @@ new Object[] {}, Config.<Integer> GetValue(ConfigValues.AsyncTaskStatusCacheRefreshRateInSeconds), Config.<Integer> GetValue(ConfigValues.AsyncTaskStatusCacheRefreshRateInSeconds), TimeUnit.SECONDS); _cacheTimeInMinutes = Config.<Integer> GetValue(ConfigValues.AsyncTaskStatusCachingTimeInMinutes); - tasksInDbAfterRestart = new ConcurrentHashMap<Guid, List<AsyncTasks>>(); - for (AsyncTasks task: DbFacade.getInstance().getAsyncTaskDao().getAll()) { - if (NGuid.Empty.equals(task.getVdsmTaskId())) { - log.infoFormat("Failing task {0} as it does not have a vdsm id.", task.getTaskId()); - failTaskWithoutVdsmId(task); + tasksInDbAfterRestart = new ConcurrentHashMap(); + Map<Guid, List<AsyncTasks>> rootCommandIdToTasksMap = groupTasksByRootCommandId(DbFacade.getInstance().getAsyncTaskDao().getAll()); + for (Guid rootCommandId : rootCommandIdToTasksMap.keySet()) { + if (hasTasksWithoutVdsmId(rootCommandIdToTasksMap.get(rootCommandId))) { + log.infoFormat("Root Command {0} has tasks without vdsm id.", rootCommandId); + handleTasksWithoutVdsmId(rootCommandIdToTasksMap.get(rootCommandId)); continue; } - tasksInDbAfterRestart.putIfAbsent(task.getStoragePoolId(), new ArrayList<AsyncTasks>()); - List<AsyncTasks> tasksPerStoragePool = tasksInDbAfterRestart.get(task.getStoragePoolId()); - tasksInDbAfterRestart.put(task.getStoragePoolId(), tasksPerStoragePool); - tasksPerStoragePool.add(task); + for (AsyncTasks task : rootCommandIdToTasksMap.get(rootCommandId)) { + tasksInDbAfterRestart.putIfAbsent(task.getStoragePoolId(), new ArrayList<AsyncTasks>()); + tasksInDbAfterRestart.get(task.getStoragePoolId()).add(task); + } } } @@ -181,7 +183,7 @@ return false; } - public static void failTaskWithoutVdsmId(final AsyncTasks task) { + public void handleTasksWithoutVdsmId(final List<AsyncTasks> tasks) { ThreadPoolUtil.execute(new Runnable() { @SuppressWarnings("synthetic-access") @Override @@ -189,7 +191,9 @@ TransactionSupport.executeInNewTransaction(new TransactionMethod<Object>() { @Override public Object runInTransaction() { - logAndFailTaskWithoutVdsmId(task); + for (AsyncTasks task : tasks) { + handleTaskWithoutVdsmId(task); + } return null; } }); @@ -197,31 +201,73 @@ }); } - private static void logAndFailTaskWithoutVdsmId(final AsyncTasks task) { - log.infoFormat( - "Failing task with out vdsm id and AsyncTaskType {0} : Task '{1}' Parent Command {2}", - task.getTaskType(), - task.getTaskId(), - (task.getaction_type())); - task.getTaskParameters().setTaskGroupSuccess(false); - ExecutionHandler.endTaskStep(task.getStepId(), JobExecutionStatus.FAILED); - removeTaskFromDbByTaskId(task.getTaskId()); - if (task.getTaskType() == AsyncTaskType.unknown) { - log.infoFormat( - "Not calling endAction for task with out vdsm id and AsyncTaskType {0} : Task '{1}' Parent Command {2}", - task.getTaskType(), - task.getTaskId(), - (task.getaction_type())); + private Map<Guid, List<AsyncTasks>> groupTasksByRootCommandId(List<AsyncTasks> tasksInDB) { + Map<Guid, List<AsyncTasks>> rootCommandIdToCommandsMap = new HashMap<>(); + for (AsyncTasks task : tasksInDB) { + Guid rootCommandId = task.getRootCommandId(); + if (!rootCommandIdToCommandsMap.containsKey(rootCommandId)) { + rootCommandIdToCommandsMap.put(rootCommandId, new ArrayList<AsyncTasks>()); + } + rootCommandIdToCommandsMap.get(rootCommandId).add(task); + } + return rootCommandIdToCommandsMap; + } + + private boolean hasTasksWithoutVdsmId(List<AsyncTasks> tasks) { + for (AsyncTasks task : tasks) { + if (hasEmptyVdsmId(task)) { + return true; + } + } + return false; + } + + private static boolean hasEmptyVdsmId(AsyncTasks task) { + return Guid.Empty.equals(task.getVdsmTaskId()); + } + + private void handleTaskWithoutVdsmId(final AsyncTasks task) { + if (hasEmptyVdsmId(task)) { + removeTaskFromDbByTaskId(task.getTaskId()); return; } - NGuid stepId = task.getStepId(); - ExecutionContext context = null; - if (stepId != null) { - context = ExecutionHandler.createFinalizingContext(stepId.getValue()); + partiallyCompletedCommandTasks.add(task); + } + + private boolean isPartiallyCompletedCommandTask(SPMAsyncTask spmTask) { + for (AsyncTasks task : partiallyCompletedCommandTasks) { + if (spmTask.getVdsmTaskId().equals(task.getVdsmTaskId())) { + return true; + } } - Backend.getInstance().endAction(task.getaction_type(), - task.getActionParameters(), - new CommandContext(context)); + return false; + } + + private void removePartiallyCompletedTaskFromList(Guid vdsmId) { + Iterator<AsyncTasks> iter = partiallyCompletedCommandTasks.iterator(); + while (iter.hasNext()) { + if (vdsmId.equals(iter.next().getVdsmTaskId())) { + iter.remove(); + return; + } + } + } + + private void handlePartiallyCompletedTask(final SPMAsyncTask task) { + ThreadPoolUtil.execute(new Runnable() { + @SuppressWarnings("synthetic-access") + @Override + public void run() { + TransactionSupport.executeInNewTransaction(new TransactionMethod<Object>() { + @Override + public Object runInTransaction() { + task.stopTask(true); + task.clearAsyncTask(true); + return null; + } + }); + } + }); } protected static void removeTaskFromDbByTaskId(Guid taskId) { @@ -570,6 +616,11 @@ if (!_tasks.containsKey(creationInfo.getVdsmTaskId())) { try { SPMAsyncTask task = AsyncTaskFactory.Construct(creationInfo); + if (isPartiallyCompletedCommandTask(task)) { + removePartiallyCompletedTaskFromList(task.getVdsmTaskId()); + handlePartiallyCompletedTask(task); + continue; + } addTaskToManager(task); newlyAddedTasks.add(task); } catch (Exception e) { @@ -617,12 +668,46 @@ } } } + // any partially complted tasks in DB that have not been handled and + // are still in the partiallyCompletedCommandTasks list can be + // removed as they have completed successfully before server restart + for (AsyncTasks task : partiallyCompletedCommandTasks) { + logAndFailPartiallyCompletedCommandTask(task); + } + //Either the tasks were only in DB - so they were removed from db, or they are polled - //in any case no need to hold them in the map that represents the tasksInDbAfterRestart tasksInDbAfterRestart.remove(sp.getId()); } + private static void logAndFailPartiallyCompletedCommandTask(final AsyncTasks task) { + log.infoFormat( + "Failing partially completed task AsyncTaskType {0} : Task '{1}' Parent Command {2}", + task.getTaskType(), + task.getTaskId(), + (task.getaction_type())); + task.getTaskParameters().setTaskGroupSuccess(false); + ExecutionHandler.endTaskStep(task.getStepId(), JobExecutionStatus.FAILED); + removeTaskFromDbByTaskId(task.getTaskId()); + if (task.getTaskType() == AsyncTaskType.unknown) { + log.infoFormat( + "Not calling endAction for task with out vdsm id and AsyncTaskType {0} : Task '{1}' Parent Command {2}", + task.getTaskType(), + task.getTaskId(), + (task.getaction_type())); + return; + } + NGuid stepId = task.getStepId(); + ExecutionContext context = null; + if (stepId != null) { + context = ExecutionHandler.createFinalizingContext(stepId.getValue()); + } + Backend.getInstance().endAction(task.getaction_type(), + task.getActionParameters(), + new CommandContext(context)); + } + /** * Retrieves all tasks from the specified storage pool and stops them. * diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CommandBase.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CommandBase.java index f129e4e..be465ce 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CommandBase.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CommandBase.java @@ -337,7 +337,7 @@ for (Guid asyncTaskId : getReturnValue().getTaskPlaceHolderIdList()) { AsyncTasks task = getAsyncTaskDao().get(asyncTaskId); if (NGuid.Empty.equals(task.getVdsmTaskId())) { - AsyncTaskManager.failTaskWithoutVdsmId(task); + AsyncTaskManager.removeTaskFromDbByTaskId(task.getTaskId()); } } } -- To view, visit http://gerrit.ovirt.org/15630 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3a979d816d9ec8cf25119a33742c8e6af4ff42a8 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Ravi Nori <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
