Eli Mesika has uploaded a new change for review. Change subject: core: handle fence agent power wait param on stop ......................................................................
core: handle fence agent power wait param on stop When a host restart is dome manually or as a result of a non-responsive host treatment and in the case that power wait parameter is used the host may stay in 'off' state and even release its lock on HA VMs before the host is really down This is the scenario: 1) A restart command is issued and actually performed as stop -> wait for 'off' status -> start -> wait for 'on' status 2) power wait parameter is added to the command implicitly or explicitly giving a delay of X seconds before the operation is actually performed 3) The fence agent script returns immediately with 'off' status 4) A 'on' command is sent to the fence agent by the start operation 5) X seconds passed and the host is actually shutdown This patch handles this by adding a new configuration value that maps fence agents to the name of the parameter for power wait and it also adds a custom configuration value that may be set for custom fence agents. Upon stop operation, we will wait a fixed delay (5 seconds hard-coded) before starting to sample the host status, if power wait parameter is used, the value is extracted and we will wait for 5 + X seconds. Change-Id: I310e076ecf84988cacd0b179954d2460d7988b91 Bug-Url: https://bugzilla.redhat.com/show_bug.cgi?id=1093742 Signed-off-by: Eli Mesika <emes...@redhat.com> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/FenceVdsBaseCommand.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java M backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/FenceConfigHelper.java M backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/VdsFenceOptions.java M packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql M packaging/etc/engine-config/engine-config.properties 6 files changed, 85 insertions(+), 1 deletion(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/83/29383/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/FenceVdsBaseCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/FenceVdsBaseCommand.java index d185b57..033e7dd 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/FenceVdsBaseCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/FenceVdsBaseCommand.java @@ -3,12 +3,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.ovirt.engine.core.bll.context.CommandContext; @@ -20,7 +22,9 @@ import org.ovirt.engine.core.common.businessentities.FenceStatusReturnValue; import org.ovirt.engine.core.common.businessentities.VDSStatus; import org.ovirt.engine.core.common.businessentities.VM; +import org.ovirt.engine.core.common.businessentities.VdsStatic; import org.ovirt.engine.core.common.config.Config; +import org.ovirt.engine.core.common.config.ConfigCommon; import org.ovirt.engine.core.common.config.ConfigValues; import org.ovirt.engine.core.common.errors.VdcBLLException; import org.ovirt.engine.core.common.errors.VdcBllErrors; @@ -35,6 +39,8 @@ import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector; import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase; import org.ovirt.engine.core.utils.ThreadUtils; +import org.ovirt.engine.core.utils.pm.FenceConfigHelper; +import org.ovirt.engine.core.utils.pm.VdsFenceOptions; import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil; public abstract class FenceVdsBaseCommand<T extends FenceVdsActionParameters> extends VdsCommand<T> { @@ -472,7 +478,7 @@ // This is done because if we will attempt to get host status immediately // in most cases it will not turn from on/off to off/on and we will need // to wait a full cycle for it. - ThreadUtils.sleep(SLEEP_BEFORE_FIRST_ATTEMPT); + ThreadUtils.sleep(getSleep(actionType, order)); while (!statusReached && i <= getRerties()) { log.infoFormat("Attempt {0} to get vds {1} status", i, vdsName); if (executor.findProxyHost()) { @@ -521,6 +527,36 @@ return statusReached; } + private int getSleep(FenceActionType actionType, FenceAgentOrder order) { + if (actionType != FenceActionType.Stop) { + return SLEEP_BEFORE_FIRST_ATTEMPT; + } + // We have to find out if power off delay was used and add this to the wait time + // since otherwise the command will return immediately with 'off' status and + // subsequent 'on' command issued during this delay will be overridden by the actual shutdown + String agent = (order == FenceAgentOrder.Primary) ? getVds().getPmType() : getVds().getPmSecondaryType(); + String options = (order == FenceAgentOrder.Primary) ? getVds().getPmOptions() : getVds().getPmSecondaryOptions(); + options = VdsFenceOptions.getDefaultAgentOptions(agent, options); + HashMap<String, String> optionsMap = VdsStatic.pmOptionsStringToMap(options); + String powerWaitParamSettings = FenceConfigHelper.getFenceConfigurationValue(ConfigValues.FencePowerWaitParam.name(), ConfigCommon.defaultConfigurationVersion); + String powerWaitParam = VdsFenceOptions.getAgentPowerWaitParam(agent, powerWaitParamSettings); + if (powerWaitParam == null) { + // no power wait for this agent + return SLEEP_BEFORE_FIRST_ATTEMPT; + } + if (optionsMap.containsKey(powerWaitParam)) { + try { + Integer powerWaitValueInSec = Integer.parseInt(optionsMap.get(powerWaitParam)); + return SLEEP_BEFORE_FIRST_ATTEMPT + (int)TimeUnit.SECONDS.toMillis(powerWaitValueInSec); + } + catch(NumberFormatException nfe) { + // illegal value + return SLEEP_BEFORE_FIRST_ATTEMPT; + } + } + return SLEEP_BEFORE_FIRST_ATTEMPT; + } + protected void setStatus(VDSStatus status) { if (getVds().getStatus() != status) { runVdsCommand(VDSCommandType.SetVdsStatus, diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java index 588e797..a0e02da 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java @@ -1840,5 +1840,19 @@ @DefaultValueAttribute("300") AlertOnNumberOfLVs, + /** + * Defines the parameter name used by the agent script to delay host on/off + */ + @TypeConverterAttribute(String.class) + @DefaultValueAttribute("apc=power_wait,apc_snmp=power_wait,bladecenter=power_wait,cisco_ucs=power_wait,drac5=power_wait,drac7=power_wait,eps=delay,hpblade=power_wait,ilo=power_wait,ilo2=power_wait,ilo3=power_wait,ilo4=power_wait,ipmilan=power_wait,rsa=power_wait,rsb=power_wait,wti=power_wait") + FencePowerWaitParam, + + /** + * Defines the parameter name used by the custom agent script to delay host on/off + */ + @TypeConverterAttribute(String.class) + @DefaultValueAttribute("") + CustomFencePowerWaitParam, + Invalid } diff --git a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/FenceConfigHelper.java b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/FenceConfigHelper.java index 5c9f40a..1b81d3c 100644 --- a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/FenceConfigHelper.java +++ b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/FenceConfigHelper.java @@ -19,6 +19,7 @@ private static final String FenceAgentDefaultParamsExpr = "([\\w]+([=][\\w]+){0,1}[,]{0,1})+"; private static final String VdsFenceOptionMappingExpr = "([\\w]+[:]([\\w]*[=][\\w]*[,]{0,1}[;]{0,1}){0,3}[;]{0,1})+"; private static final String CustomVdsFenceTypeExpr = "((\\w)+[,]{0,1})+"; + private static final String FencePowerWaitParamExpr = "((\\w)+[=](\\w)+[,]{0,1})+"; private static final String COMMA = ","; private static final String SEMICOLON = ";"; @@ -30,21 +31,25 @@ keyValidatorMap.put("FenceAgentDefaultParams", FenceAgentDefaultParamsExpr); keyValidatorMap.put("VdsFenceOptionMapping", VdsFenceOptionMappingExpr); keyValidatorMap.put("VdsFenceType", CustomVdsFenceTypeExpr); + keyValidatorMap.put("FencePowerWaitParam", FencePowerWaitParamExpr); keyValidatorMap.put("CustomFenceAgentMapping", FenceAgentMappingExpr); keyValidatorMap.put("CustomFenceAgentDefaultParams", FenceAgentDefaultParamsExpr); keyValidatorMap.put("CustomVdsFenceOptionMapping", VdsFenceOptionMappingExpr); keyValidatorMap.put("CustomVdsFenceType", CustomVdsFenceTypeExpr); + keyValidatorMap.put("CustomFencePowerWaitParam", FencePowerWaitParamExpr); keyValidatorExampleMap = new HashMap<String, String>(); keyValidatorExampleMap.put("CustomFenceAgentMapping", "agent1=agent2,agent3=agent4"); keyValidatorExampleMap.put("CustomFenceAgentDefaultParams", "agent1=key1=val1,flag;key2=val2"); keyValidatorExampleMap.put("CustomVdsFenceOptionMapping", "agent1:secure=secure;agent2:port=ipport,slot=slot"); keyValidatorExampleMap.put("CustomVdsFenceType", "agent1,agent2"); + keyValidatorExampleMap.put("CustomFencePowerWaitParam", "agent1=power_wait,agent2=delay"); keySeparatorMap = new HashMap<String, String>(); keySeparatorMap.put("FenceAgentMapping", COMMA); keySeparatorMap.put("FenceAgentDefaultParams", SEMICOLON); keySeparatorMap.put("VdsFenceOptionMapping", SEMICOLON); keySeparatorMap.put("VdsFenceType", COMMA); + keySeparatorMap.put("FencePowerWaitParam", COMMA); initialized = true; } } diff --git a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/VdsFenceOptions.java b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/VdsFenceOptions.java index 2f4bffb..9f120f2 100644 --- a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/VdsFenceOptions.java +++ b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/pm/VdsFenceOptions.java @@ -291,6 +291,31 @@ } /** + * handles agent power wait parameter mapping + * @param agent + * @param powerWait + * @return + */ + public static String getAgentPowerWaitParam(String agent, String powerWait) { + String param = null; + // result has the format [<agent>=<real agent>[,]]* + String[] settings = powerWait.split(Pattern.quote(COMMA), -1); + if (settings.length > 0) { + for (String setting : settings) { + // get the <agent>=<real agent> pair + String[] pair = setting.split(Pattern.quote(EQUAL), -1); + if (pair.length == 2) { + if (agent.equalsIgnoreCase(pair[0])) { + param = pair[1]; + break; + } + } + } + } + return param; + } + + /** * handles agent default options * * @param agent diff --git a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql index 9ea49e2..1b2c583 100644 --- a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql +++ b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql @@ -127,12 +127,14 @@ select fn_db_add_config_value('CustomFenceAgentDefaultParams','','general'); select fn_db_add_config_value('FenceAgentMapping','drac7=ipmilan,ilo2=ilo,ilo3=ipmilan,ilo4=ipmilan','general'); select fn_db_add_config_value('CustomFenceAgentMapping','','general'); +select fn_db_add_config_value('CustomFencePowerWaitParam','','general'); select fn_db_add_config_value('FenceProxyDefaultPreferences','cluster,dc','general'); select fn_db_add_config_value('FenceQuietTimeBetweenOperationsInSec','180','general'); select fn_db_add_config_value('FenceStartStatusDelayBetweenRetriesInSec','60','general'); select fn_db_add_config_value('FenceStartStatusRetries','3','general'); select fn_db_add_config_value('FenceStopStatusDelayBetweenRetriesInSec','60','general'); select fn_db_add_config_value('FenceStopStatusRetries','3','general'); +select fn_db_add_config_value('FencePowerWaitParam','apc=power_wait,apc_snmp=power_wait,bladecenter=power_wait,cisco_ucs=power_wait,drac5=power_wait,drac7=power_wait,eps=delay,hpblade=power_wait,ilo=power_wait,ilo2=power_wait,ilo3=power_wait,ilo4=power_wait,ipmilan=power_wait,rsa=power_wait,rsb=power_wait,wti=power_wait','general'); select fn_db_add_config_value('FilteringLUNsEnabled','true','3.0'); select fn_db_add_config_value('FindFenceProxyDelayBetweenRetriesInSec','30','general'); select fn_db_add_config_value('FindFenceProxyRetries','3','general'); diff --git a/packaging/etc/engine-config/engine-config.properties b/packaging/etc/engine-config/engine-config.properties index d12a26c..a70de6f 100644 --- a/packaging/etc/engine-config/engine-config.properties +++ b/packaging/etc/engine-config/engine-config.properties @@ -418,6 +418,8 @@ CustomVdsFenceOptionMapping.type=FenceConfig CustomVdsFenceType.description="Fence agents types. Format ((\\w)+[,]{0,1})+. Example: agent1,agent2" CustomVdsFenceType.type=FenceConfig +CustomFencePowerWaitParam.type=FenceConfig +CustomFencePowerWaitParam.description="Maps a fencing agent to the param for delay on/off actions. Format : ((\\w)+[=](\\w)+[,]{0,1})+. Example: agent1=power_wait,agent2=delay" # Random Number Generator device VirtIoRngDeviceSupported.description="Defines whether VirtIO Random Number Generator devices is supported for given cluster level." VirtIoRngDeviceSupported.type=Boolean -- To view, visit http://gerrit.ovirt.org/29383 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I310e076ecf84988cacd0b179954d2460d7988b91 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Eli Mesika <emes...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches
