Martin Peřina has uploaded a new change for review. Change subject: core: Move VDS to Maintenance only if StopSPM is successful ......................................................................
core: Move VDS to Maintenance only if StopSPM is successful Fixes MaintenanceNumberOfVdssCommand so only VDSs for which StopSPM was executed successfully, will continues with Move to Maintenance process. Change-Id: I5c58f9a9629d2e7a496f02c4dececeb842d44543 Bug-Url: https://bugzilla.redhat.com/1023145 Signed-off-by: Martin Perina <mper...@redhat.com> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/MaintenanceNumberOfVdssCommand.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/vdscommands/SetVdsStatusVDSCommandParameters.java M backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java M backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/SetVdsStatusVDSCommand.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/SpmStopVDSCommand.java 7 files changed, 96 insertions(+), 42 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/05/23005/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/MaintenanceNumberOfVdssCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/MaintenanceNumberOfVdssCommand.java index 1df7a74..ae02ad6 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/MaintenanceNumberOfVdssCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/MaintenanceNumberOfVdssCommand.java @@ -4,6 +4,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -37,7 +38,6 @@ @NonTransactiveCommandAttribute public class MaintenanceNumberOfVdssCommand<T extends MaintenanceNumberOfVdssParameters> extends CommandBase<T> { private final HashMap<Guid, VDS> vdssToMaintenance = new HashMap<Guid, VDS>(); - private ArrayList<Guid> _vdsGroupIds; private final List<PermissionSubject> inspectedEntitiesMap; private Map<String, Pair<String, String>> sharedLockMap; @@ -54,29 +54,40 @@ private void MoveVdssToGoingToMaintenanceMode() { List<VDS> spms = new ArrayList<VDS>(); - for (VDS vds : vdssToMaintenance.values()) { + Iterator<VDS> it = vdssToMaintenance.values().iterator(); + while (it.hasNext()) { + VDS vds = it.next(); + // SPMs will move to Prepare For Maintenance later after standard hosts if (vds.getSpmStatus() != VdsSpmStatus.SPM) { - setVdsStatusToPrepareForMaintaice(vds); + if (!setVdsStatusToPrepareForMaintenance(vds)) { + it.remove(); + } } else { spms.add(vds); } } for (VDS vds : spms) { - setVdsStatusToPrepareForMaintaice(vds); + if (!setVdsStatusToPrepareForMaintenance(vds)) { + vdssToMaintenance.remove(vds.getId()); + } } freeLock(); } - private void setVdsStatusToPrepareForMaintaice(VDS vds) { + private boolean setVdsStatusToPrepareForMaintenance(VDS vds) { + boolean result = true; if (vds.getStatus() != VDSStatus.PreparingForMaintenance && vds.getStatus() != VDSStatus.NonResponsive && vds.getStatus() != VDSStatus.Down) { - runVdsCommand(VDSCommandType.SetVdsStatus, - new SetVdsStatusVDSCommandParameters(vds.getId(), VDSStatus.PreparingForMaintenance)); + SetVdsStatusVDSCommandParameters params = + new SetVdsStatusVDSCommandParameters(vds.getId(), VDSStatus.PreparingForMaintenance); + params.setStopSpmFailureLogged(true); + result = runVdsCommand(VDSCommandType.SetVdsStatus, params).getSucceeded(); } + return result; } private void MigrateAllVdss() { - for (Guid vdsId : getParameters().getVdsIdList()) { + for (Guid vdsId : vdssToMaintenance.keySet()) { // ParametersCurrentUser = CurrentUser MaintenanceVdsParameters tempVar = new MaintenanceVdsParameters(vdsId, getParameters().getIsInternal()); tempVar.setSessionId(getParameters().getSessionId()); @@ -102,11 +113,17 @@ protected void executeCommand() { MoveVdssToGoingToMaintenanceMode(); MigrateAllVdss(); - // set network to operational / non-operational - for (Guid id : _vdsGroupIds) { - List<Network> networks = DbFacade.getInstance().getNetworkDao().getAllForCluster(id); - for (Network net : networks) { - NetworkClusterHelper.setStatus(id, net); + + // find clusters for hosts that should move to maintenance + Set<Guid> clusters = new HashSet<>(); + for (VDS vds : vdssToMaintenance.values()) { + if (!clusters.contains(vds.getVdsGroupId())) { + clusters.add(vds.getVdsGroupId()); + // set network to operational / non-operational + List<Network> networks = DbFacade.getInstance().getNetworkDao().getAllForCluster(vds.getVdsGroupId()); + for (Network net : networks) { + NetworkClusterHelper.setStatus(vds.getVdsGroupId(), net); + } } } setSucceeded(true); @@ -115,7 +132,7 @@ @Override protected boolean canDoAction() { boolean result = true; - _vdsGroupIds = new ArrayList<Guid>(); + Set<Guid> clustersAsSet = new HashSet<Guid>(); Set<Guid> vdsWithRunningVMs = new HashSet<Guid>(); List<String> hostNotRespondingList = new ArrayList<String>(); List<String> hostsWithNonMigratableVms = new ArrayList<String>(); @@ -157,7 +174,7 @@ if (vms.size() > 0) { vdsWithRunningVMs.add(vdsId); } - _vdsGroupIds.add(vds.getVdsGroupId()); + clustersAsSet.add(vds.getVdsGroupId()); List<String> nonMigratableVmDescriptionsToFrontEnd = new ArrayList<String>(); for (VM vm : vms) { @@ -224,8 +241,6 @@ // In the end - if the clusters list is not empty - this is an // error, use the "problematic clusters list" to format an error to // the client - Set<Guid> clustersAsSet = new HashSet<Guid>(); - clustersAsSet.addAll(_vdsGroupIds); List<String> problematicClusters = new ArrayList<String>(); List<String> allHostsWithRunningVms = new ArrayList<String>(); for (Guid clusterID : clustersAsSet) { diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java index 66da936..714c63b 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java @@ -18,6 +18,7 @@ VDS_MAINTENANCE_FAILED(17, AuditLogTimeInterval.MINUTE.getValue()), // When VDS is transferred to maintenance mode VDS_ACTIVATE_FAILED(18, AuditLogTimeInterval.MINUTE.getValue()), // When VDS is reactivated VDS_RECOVER_FAILED(19, AuditLogTimeInterval.MINUTE.getValue()), // When VDS changes status down->up + VDS_STATUS_CHANGE_FAILED_DUE_TO_STOP_SPM_FAILURE(20, AuditLogTimeInterval.MINUTE.getValue()), VDS_SLOW_STORAGE_RESPONSE_TIME(123, AuditLogTimeInterval.MINUTE.getValue() * 5), // ? VDS_ALREADY_IN_REQUESTED_STATUS(493), VDS_MANUAL_FENCE_STATUS(494), diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/vdscommands/SetVdsStatusVDSCommandParameters.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/vdscommands/SetVdsStatusVDSCommandParameters.java index 641fbb5..de28694 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/vdscommands/SetVdsStatusVDSCommandParameters.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/vdscommands/SetVdsStatusVDSCommandParameters.java @@ -8,10 +8,16 @@ private VDSStatus _status; private NonOperationalReason nonOperationalReason; + /** + * Flag to display SPM stop command failure in audit log + */ + private boolean stopSpmFailureLogged; + public SetVdsStatusVDSCommandParameters(Guid vdsId, VDSStatus status) { super(vdsId); _status = status; nonOperationalReason = NonOperationalReason.NONE; + stopSpmFailureLogged = false; } public SetVdsStatusVDSCommandParameters(Guid vdsId, VDSStatus status, NonOperationalReason nonOperationalReason) { @@ -26,6 +32,7 @@ public SetVdsStatusVDSCommandParameters() { _status = VDSStatus.Unassigned; nonOperationalReason = NonOperationalReason.NONE; + stopSpmFailureLogged = false; } public NonOperationalReason getNonOperationalReason() { @@ -36,11 +43,20 @@ this.nonOperationalReason = (nonOperationalReason == null ? NonOperationalReason.NONE : nonOperationalReason); } + public boolean isStopSpmFailureLogged() { + return stopSpmFailureLogged; + } + + public void setStopSpmFailureLogged(boolean stopSpmFailureLogged) { + this.stopSpmFailureLogged = stopSpmFailureLogged; + } + @Override public String toString() { - return String.format("%s, status=%s, nonOperationalReason=%s", + return String.format("%s, status=%s, nonOperationalReason=%s, stopSpmFailureLogged=%s", super.toString(), getStatus(), - getNonOperationalReason()); + getNonOperationalReason(), + isStopSpmFailureLogged()); } } diff --git a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java index e77f5e4..1b90c81 100644 --- a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java +++ b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java @@ -292,6 +292,7 @@ severities.put(AuditLogType.VDS_RECOVER_FAILED_VMS_UNKNOWN, AuditLogSeverity.ERROR); severities.put(AuditLogType.VDS_MAINTENANCE, AuditLogSeverity.WARNING); severities.put(AuditLogType.VDS_MAINTENANCE_FAILED, AuditLogSeverity.ERROR); + severities.put(AuditLogType.VDS_STATUS_CHANGE_FAILED_DUE_TO_STOP_SPM_FAILURE, AuditLogSeverity.WARNING); severities.put(AuditLogType.USER_VDS_MAINTENANCE_MIGRATION_FAILED, AuditLogSeverity.WARNING); severities.put(AuditLogType.SYSTEM_VDS_RESTART, AuditLogSeverity.NORMAL); severities.put(AuditLogType.SYSTEM_FAILED_VDS_RESTART, AuditLogSeverity.ERROR); diff --git a/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties b/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties index fc89536..f87cb4d 100644 --- a/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties +++ b/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties @@ -714,3 +714,4 @@ DISK_ALIGNMENT_SCAN_FAILURE=Alignment scan of disk '${DiskAlias}' failed. DISK_ALIGNMENT_SCAN_SUCCESS=Alignment scan of disk '${DiskAlias}' is complete. VM_MEMORY_NOT_IN_RECOMMENDED_RANGE=VM ${VmName} was configured with ${VmMemInMb}mb of memory while the recommended value range is ${VmMinMemInMb}mb - ${VmMaxMemInMb}mb +VDS_STATUS_CHANGE_FAILED_DUE_TO_STOP_SPM_FAILURE=Failed to change status of host '${VdsName}' due to a failure to stop the spm. diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/SetVdsStatusVDSCommand.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/SetVdsStatusVDSCommand.java index b4b9ec2..8fd3972 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/SetVdsStatusVDSCommand.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/SetVdsStatusVDSCommand.java @@ -1,11 +1,14 @@ package org.ovirt.engine.core.vdsbroker; +import org.ovirt.engine.core.common.AuditLogType; import org.ovirt.engine.core.common.businessentities.VDS; import org.ovirt.engine.core.common.businessentities.VDSStatus; import org.ovirt.engine.core.common.businessentities.VdsSpmStatus; import org.ovirt.engine.core.common.vdscommands.ResetIrsVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.SetVdsStatusVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.VDSCommandType; +import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector; +import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase; import org.ovirt.engine.core.utils.log.Log; import org.ovirt.engine.core.utils.log.LogFactory; import org.ovirt.engine.core.utils.transaction.TransactionMethod; @@ -23,6 +26,31 @@ if (_vdsManager != null) { final VDS vds = getVds(); + if (vds.getSpmStatus() != VdsSpmStatus.None && parameters.getStatus() != VDSStatus.Up) { + log.infoFormat("VDS {0} is spm and moved from up calling resetIrs.", vds.getName()); + // check if this host was spm and reset if do. + getVDSReturnValue().setSucceeded( + ResourceManager + .getInstance() + .runVdsCommand( + VDSCommandType.ResetIrs, + new ResetIrsVDSCommandParameters(vds.getStoragePoolId(), vds.getId())) + .getSucceeded()); + + if (!getVDSReturnValue().getSucceeded()) { + if (getParameters().isStopSpmFailureLogged()) { + AuditLogableBase base = new AuditLogableBase(); + base.setVds(vds); + AuditLogDirector.log(base, AuditLogType.VDS_STATUS_CHANGE_FAILED_DUE_TO_STOP_SPM_FAILURE); + } + + if (parameters.getStatus() == VDSStatus.PreparingForMaintenance) { + // ResetIrs command failed, SPM host status cannot be moved to Preparing For Maintenance + return; + } + } + } + updateVdsFromParameters(parameters, vds); TransactionSupport.executeInNewTransaction(new TransactionMethod<Void>() { @@ -34,18 +62,6 @@ return null; } }); - - if (vds.getSpmStatus() != VdsSpmStatus.None && parameters.getStatus() != VDSStatus.Up) { - log.infoFormat("VDS {0} is spm and moved from up calling ResetIrs.", vds.getName()); - // check if this host was spm and reset if do. - getVDSReturnValue().setSucceeded( - ResourceManager - .getInstance() - .runVdsCommand( - VDSCommandType.ResetIrs, - new ResetIrsVDSCommandParameters(vds.getStoragePoolId(), vds.getId())) - .getSucceeded()); - } } else { getVDSReturnValue().setSucceeded(false); } diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/SpmStopVDSCommand.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/SpmStopVDSCommand.java index f844f58..cb4661e 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/SpmStopVDSCommand.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/SpmStopVDSCommand.java @@ -41,24 +41,28 @@ performSpmStop = ((HashMap<Guid, AsyncTaskStatus>) vdsReturnValue.getReturnValue()).isEmpty(); } } catch (Exception e) { - log.infoFormat("SpmStopVDSCommand::Could not get tasks on vds {0} stopping SPM", - getVds().getName()); + performSpmStop = false; + log.infoFormat("SpmStopVDSCommand::Could not get tasks on vds {0}, reason: {1}", + getVds().getName(), + e.getMessage()); } if (performSpmStop) { log.infoFormat("SpmStopVDSCommand::Stopping SPM on vds {0}, pool id {1}", getVds().getName(), getParameters().getStoragePoolId()); status = getBroker().spmStop(getParameters().getStoragePoolId().toString()); proceedProxyReturnValue(); - } else if (getVDSReturnValue().getVdsError() == null) { + } else { getVDSReturnValue().setSucceeded(false); - VDSError error = new VDSError(); - error.setCode(VdcBllErrors.TaskInProgress); - getVDSReturnValue().setVdsError(error); - } else if (getVDSReturnValue().getVdsError().getCode() == VdcBllErrors.VDS_NETWORK_ERROR) { - log.infoFormat( - "SpmStopVDSCommand::Could not get tasks on vds {0} - network exception, not stopping spm! pool id {1}", - getVds().getName(), - getParameters().getStoragePoolId()); + if (getVDSReturnValue().getVdsError() == null) { + VDSError error = new VDSError(); + error.setCode(VdcBllErrors.TaskInProgress); + getVDSReturnValue().setVdsError(error); + } else if (getVDSReturnValue().getVdsError().getCode() == VdcBllErrors.VDS_NETWORK_ERROR) { + log.infoFormat( + "SpmStopVDSCommand::Could not get tasks on vds {0} - network exception, not stopping spm! pool id {1}", + getVds().getName(), + getParameters().getStoragePoolId()); + } } } else { log.infoFormat("SpmStopVDSCommand:: vds {0} is in {1} status - not performing spm stop, pool id {2}", -- To view, visit http://gerrit.ovirt.org/23005 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5c58f9a9629d2e7a496f02c4dececeb842d44543 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: ovirt-engine-3.3 Gerrit-Owner: Martin Peřina <mper...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches
