Sahina Bose has uploaded a new change for review. Change subject: engine: Wait before clearing gluster tasks ......................................................................
engine: Wait before clearing gluster tasks Introduced a wait time before clearing any orphan gluster tasks to take care of possible race condition in returning task list from gluster CLI Change-Id: I3a8a4878abb7ef46adedfdb099404884419f89de Bug-Url: https://bugzilla.redhat.com/1065227 Signed-off-by: Sahina Bose <sab...@redhat.com> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJob.java M backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJobTest.java 2 files changed, 24 insertions(+), 10 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/12/24712/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJob.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJob.java index 5042472..13766a1 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJob.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJob.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.ovirt.engine.core.bll.Backend; @@ -353,6 +354,10 @@ values.put(GlusterConstants.JOB_INFO, " "); for (Step step: steps) { + if (TimeUnit.MILLISECONDS.toMinutes(System.currentTimeMillis() - step.getStartTime().getTime()) < 10) { + //This task has been recently created. We will give it 10 mins before clearing it. + continue; + } step.markStepEnded(JobExecutionStatus.UNKNOWN); step.setStatus(JobExecutionStatus.UNKNOWN); step.setDescription(ExecutionMessageDirector.resolveStepMessage(step.getStepType(), values)); diff --git a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJobTest.java b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJobTest.java index 5bfdbe3..b6510b0 100755 --- a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJobTest.java +++ b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/gluster/GlusterTasksSyncJobTest.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -57,7 +58,8 @@ private static final Guid[] TASK_GUIDS = {new Guid("EE111111-1111-1111-1111-111111111111"), new Guid("EE222222-2222-2222-2222-222222222222"), - new Guid("EE333333-3333-3333-3333-333333333333")}; + new Guid("EE333333-3333-3333-3333-333333333333"), + new Guid("EE444444-4444-4444-4444-444444444444")}; private static final Guid[] VOL_GUIDS = {new Guid("AA111111-1111-1111-1111-111111111111"), new Guid("AA222222-2222-2222-2222-222222222222"), @@ -144,11 +146,12 @@ @Test public void cleanOrphanTasks() { doReturn(getTasks()).when(provider).getTaskListForCluster(CLUSTER_GUIDS[1]); - doReturn(Arrays.asList(TASK_GUIDS[2])).when(provider).getMonitoredTaskIDsInDB(); + doReturn(Arrays.asList(TASK_GUIDS[2],TASK_GUIDS[3])).when(provider).getMonitoredTaskIDsInDB(); prepareMocks(); tasksSyncJob.updateGlusterAsyncTasks(); Mockito.verify(jobRepository, times(1)).updateStep(any(Step.class)); + //endStepJob will not be called for TASK_GUIDS[3], so times is = 2 Mockito.verify(taskUtils, times(2)).endStepJob(any(Step.class)); } @@ -157,7 +160,7 @@ doReturn(new HashMap<>()).when(provider).getTaskListForCluster(CLUSTER_GUIDS[1]); doReturn(Arrays.asList(TASK_GUIDS[2])).when(provider).getMonitoredTaskIDsInDB(); doReturn(null).when(volumeDao).getVolumeByGlusterTask(TASK_GUIDS[2]); - doReturn(getSteps()).when(stepDao).getStepsByExternalId(TASK_GUIDS[2]); + doReturn(getSteps(TASK_GUIDS[2])).when(stepDao).getStepsByExternalId(TASK_GUIDS[2]); tasksSyncJob.updateGlusterAsyncTasks(); Mockito.verify(taskUtils, times(1)).endStepJob(any(Step.class)); @@ -212,7 +215,7 @@ doReturn(getTasks(JobExecutionStatus.ABORTED)).when(provider).getTaskListForCluster(CLUSTER_GUIDS[1]); prepareMocks(); tasksSyncJob.updateGlusterAsyncTasks(); - Mockito.verify(jobRepository, times(0)).updateStep(any(Step.class)); + Mockito.verify(jobRepository, times(0)).updateStep(any(Step.class)); Mockito.verify(taskUtils, times(2)).endStepJob(any(Step.class)); } @@ -220,9 +223,9 @@ doReturn(getVolume(0)).when(volumeDao).getVolumeByGlusterTask(TASK_GUIDS[0]); doReturn(getVolume(1)).when(volumeDao).getVolumeByGlusterTask(TASK_GUIDS[1]); doReturn(getVolume(1)).when(volumeDao).getVolumeByGlusterTask(TASK_GUIDS[2]); - doReturn(getSteps()).when(stepDao).getStepsByExternalId(TASK_GUIDS[0]); - doReturn(getSteps()).when(stepDao).getStepsByExternalId(TASK_GUIDS[1]); - doReturn(getSteps()).when(stepDao).getStepsByExternalId(TASK_GUIDS[2]); + doReturn(getSteps(TASK_GUIDS[0])).when(stepDao).getStepsByExternalId(TASK_GUIDS[0]); + doReturn(getSteps(TASK_GUIDS[1])).when(stepDao).getStepsByExternalId(TASK_GUIDS[1]); + doReturn(getSteps(TASK_GUIDS[2])).when(stepDao).getStepsByExternalId(TASK_GUIDS[2]); } private void prepareMocksForTasksFromCLI() { @@ -232,15 +235,21 @@ doReturn(new ArrayList<Step>()).when(stepDao).getStepsByExternalId(TASK_GUIDS[1]); } - private List<Step> getSteps() { + private List<Step> getSteps(Guid taskGuid) { List<Step> steps = new ArrayList<>(); - steps.add(createStep()); + steps.add(createStep(taskGuid)); return steps; } - private Step createStep() { + private Step createStep(Guid taskGuid) { Step step = new Step(); step.setStepType(StepEnum.REBALANCING_VOLUME); + Calendar stepTime = Calendar.getInstance(); + if (taskGuid.equals(TASK_GUIDS[2])) { + //only create TASK_GUIDS[2] as older job + stepTime.set(Calendar.HOUR, stepTime.get(Calendar.HOUR) -1); + } + step.setStartTime(stepTime.getTime()); return step; } -- To view, visit http://gerrit.ovirt.org/24712 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3a8a4878abb7ef46adedfdb099404884419f89de Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Sahina Bose <sab...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches
