Martin Peřina has uploaded a new change for review. Change subject: core: Delay fencing until host storage lease renewal interval passed ......................................................................
core: Delay fencing until host storage lease renewal interval passed Delays fencing execution until host storage lease renewal interval passed in the case that fencing policy contains option to skip fencing when host has live lease to storage. Change-Id: I45da28ae8a3007ed124d145a303f6201474a0f1c Bug-Url: https://bugzilla.redhat.com/1090799 Signed-off-by: Martin Perina <mper...@redhat.com> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsNotRespondingTreatmentCommand.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/FenceVdsActionParameters.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java M packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql 7 files changed, 47 insertions(+), 4 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/87/31087/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java index 107ac94..e66562f 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java @@ -206,7 +206,7 @@ } @Override - public void vdsNotResponding(final VDS vds, final boolean executeSshSoftFencing) { + public void vdsNotResponding(final VDS vds, final boolean executeSshSoftFencing, final long lastUpdate) { ExecutionHandler.updateSpecificActionJobCompleted(vds.getId(), VdcActionType.MaintenanceVds, false); ThreadPoolUtil.execute(new Runnable() { @Override @@ -229,8 +229,12 @@ if (MonitoringStrategyFactory.getMonitoringStrategyForVds(vds).isPowerManagementSupported() && shouldExecRealFencing) { + FenceVdsActionParameters params = new FenceVdsActionParameters( + vds.getId(), + FenceActionType.Restart); + params.setLastUpdate(lastUpdate); Backend.getInstance().runInternalAction(VdcActionType.VdsNotRespondingTreatment, - new FenceVdsActionParameters(vds.getId(), FenceActionType.Restart), + params, ExecutionHandler.createInternalJobContext()); } moveBricksToUnknown(vds); diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsNotRespondingTreatmentCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsNotRespondingTreatmentCommand.java index f8b7d3e..ab77e3e 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsNotRespondingTreatmentCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsNotRespondingTreatmentCommand.java @@ -17,6 +17,9 @@ import org.ovirt.engine.core.common.businessentities.FencingPolicy; import org.ovirt.engine.core.common.businessentities.StoragePoolStatus; import org.ovirt.engine.core.common.businessentities.VdsSpmStatus; +import org.ovirt.engine.core.common.config.Config; +import org.ovirt.engine.core.common.config.ConfigValues; +import org.ovirt.engine.core.utils.ThreadUtils; /** * @see RestartVdsCommand on why this command is requiring a lock @@ -88,6 +91,18 @@ ).getFencingPolicy(); getParameters().setFencingPolicy(fencingPolicy); + if (fencingPolicy.isSkipFencingIfSDActive()) { + // host storage lease should be renewed each ConfigValues.HostStorageLeaseAliveInterval + // so we need to be sure not to execute fencing before host is non responsive for longer time + long interval = Config.<Integer>getValue(ConfigValues.HostStorageLeaseAliveCheckingInterval) * 1000; + long difference = System.currentTimeMillis() - getParameters().getLastUpdate(); + if (difference < interval) { + int sleepMs = (int)(interval - difference); + log.debug(String.format("Sleeping %s ms before proceeding with fence execution", sleepMs)); + ThreadUtils.sleep(sleepMs); + } + } + // Make sure that the StopVdsCommand that runs by the RestartVds // don't write over our job, and disrupt marking the job status correctly ExecutionContext ec = (ExecutionContext) ObjectUtils.clone(this.getExecutionContext()); diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/FenceVdsActionParameters.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/FenceVdsActionParameters.java index e0a5e2e..9aa24fb 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/FenceVdsActionParameters.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/FenceVdsActionParameters.java @@ -20,6 +20,11 @@ private FencingPolicy fencingPolicy; + /** + * Number milliseconds since last successful connection to host in monitoring + */ + private long lastUpdate; + public FenceVdsActionParameters() { this(null, FenceActionType.Restart); } @@ -61,4 +66,13 @@ public void setFencingPolicy(FencingPolicy fencingPolicy) { this.fencingPolicy = fencingPolicy; } + + public long getLastUpdate() { + return lastUpdate; + } + + public void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } + } diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java index 3c33ec8..55b5273 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java @@ -11,7 +11,7 @@ import org.ovirt.engine.core.compat.TransactionScopeOption; public interface IVdsEventListener { - void vdsNotResponding(VDS vds, boolean executeSshSoftFencing); // BLL + void vdsNotResponding(VDS vds, boolean executeSshSoftFencing, long lastUpdate); // BLL void vdsNonOperational(Guid vdsId, NonOperationalReason type, boolean logCommand, Guid domainId); // BLL diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java index beec639..2989e15 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java @@ -1875,5 +1875,13 @@ @DefaultValueAttribute("true") SkipFencingIfSDActiveSupported, + /** + * Interval in seconds after which is safe to check host storage lease status when host stopped responding + * to monitoring + */ + @TypeConverterAttribute(Integer.class) + @DefaultValueAttribute("90") + HostStorageLeaseAliveCheckingInterval, + Invalid } diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java index dae28ce..2a2413a 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java @@ -710,7 +710,7 @@ if (!sshSoftFencingExecuted.getAndSet(true)) { executeSshSoftFencing = true; } - ResourceManager.getInstance().getEventListener().vdsNotResponding(vds, executeSshSoftFencing); + ResourceManager.getInstance().getEventListener().vdsNotResponding(vds, executeSshSoftFencing, lastUpdate); } return true; } diff --git a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql index 04fd79d..9a8c356 100644 --- a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql +++ b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql @@ -796,6 +796,8 @@ select fn_db_add_config_value('SkipFencingIfSDActiveSupported', 'false', '3.3'); select fn_db_add_config_value('SkipFencingIfSDActiveSupported', 'false', '3.4'); +select fn_db_add_config_value('HostStorageLeaseAliveCheckingInterval', '90', 'general'); + ------------------------------------------------------------------------------------ -- Update with override section ------------------------------------------------------------------------------------ -- To view, visit http://gerrit.ovirt.org/31087 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I45da28ae8a3007ed124d145a303f6201474a0f1c Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Martin Peřina <mper...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches
