Martin Peřina has uploaded a new change for review. Change subject: core: Introduce SingleAgentPmActionExecutor ......................................................................
core: Introduce SingleAgentPmActionExecutor Introduces SignleAgentPmActionExecutor which is responsible to execute power management action on the host using signle fence agent. It contains logic to get status of the host or to start/stop the host and to verify requested status using FenceAgentExecutor. Change-Id: I95958b4821fa8f55ee8913e926f9528ae56e20e8 Bug-Url: https://bugzilla.redhat.com/1182510 Signed-off-by: Martin Perina <mper...@redhat.com> --- A backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutor.java A backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutorTest.java 2 files changed, 493 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/58/38358/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutor.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutor.java new file mode 100644 index 0000000..347cd78 --- /dev/null +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutor.java @@ -0,0 +1,232 @@ +package org.ovirt.engine.core.bll.pm; + +import java.util.concurrent.TimeUnit; + +import org.ovirt.engine.core.common.businessentities.FenceAgent; +import org.ovirt.engine.core.common.businessentities.FencingPolicy; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.pm.FenceActionType; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult.Status; +import org.ovirt.engine.core.common.businessentities.pm.HostPowerStatus; +import org.ovirt.engine.core.common.config.Config; +import org.ovirt.engine.core.common.config.ConfigValues; +import org.ovirt.engine.core.utils.ThreadUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages fence execution for the host using specified fence agent + */ +public class SingleAgentPmActionExecutor { + private static final Logger log = LoggerFactory.getLogger(SingleAgentPmActionExecutor.class); + + /** + * Number of ms to wait after host was fenced to fetch host power status + */ + private static final int SLEEP_BEFORE_FIRST_ATTEMPT = 5000; + + /** + * Number of allowed {@code HostPowerStatus.UNKNOWN} status results to determine if fence host operation was + * successful + */ + private static final int UNKNOWN_RESULT_LIMIT = 3; + + private final VDS fencedHost; + private FenceAgent fenceAgent; + private FencingPolicy fencingPolicy; + private FenceActionType actionType; + private int allowedFenceActionRetries; + private HostPowerStatus requestedPowerStatus; + private int allowedWaitForStatusRetries; + private long delayBetweenRetries; + + public SingleAgentPmActionExecutor(VDS fencedHost, FenceAgent fenceAgent, FencingPolicy fencingPolicy) { + this.fencedHost = fencedHost; + this.fenceAgent = fenceAgent; + this.fencingPolicy = fencingPolicy; + } + + /** + * Execute fence action using specified agent + * + * @param actionType + * fence action type + * @throws java.lang.IllegalArgumentException + * if actionType is not {@code FenceActionType.START} or {@code FenceActionType.STOP} + * @return result of fence operation + */ + public FenceOperationResult fence(FenceActionType actionType) { + setupParams(actionType); + + if (actionType == FenceActionType.STATUS) { + return getStatus(); + } else { + return changeStatus(); + } + } + + /** + * Setup parameters for specified fence action + */ + protected void setupParams(FenceActionType actionType) { + this.actionType = actionType; + switch (actionType) { + case START: + requestedPowerStatus = HostPowerStatus.ON; + allowedFenceActionRetries = 1; + allowedWaitForStatusRetries = Config.<Integer>getValue(ConfigValues.FenceStartStatusRetries); + delayBetweenRetries = TimeUnit.SECONDS.toMillis( + Config.<Integer>getValue(ConfigValues.FenceStartStatusDelayBetweenRetriesInSec)); + break; + + case STOP: + requestedPowerStatus = HostPowerStatus.OFF; + allowedFenceActionRetries = 0; + allowedWaitForStatusRetries = Config.<Integer>getValue(ConfigValues.FenceStopStatusRetries); + delayBetweenRetries = TimeUnit.SECONDS.toMillis( + Config.<Integer>getValue(ConfigValues.FenceStopStatusDelayBetweenRetriesInSec)); + break; + + case STATUS: + break; + } + } + + /** + * Returns new instance of {@link FenceAgentExecutor} + */ + protected FenceAgentExecutor createAgentExecutor() { + return new FenceAgentExecutor( + fencedHost, + fencingPolicy); + } + + /** + * Fetches power status of the host using specified agent + */ + protected FenceOperationResult getStatus() { + return createAgentExecutor().fence(FenceActionType.STATUS, fenceAgent); + } + + /** + * Executes start or stop fence operation using specified agent + */ + protected FenceOperationResult changeStatus() { + FenceAgentExecutor agentExecutor = createAgentExecutor(); + FenceOperationResult statusResult = null; + // start at -1 because 1st fence attempt is regular and not a retry + int fenceRetries = -1; + + do { + FenceOperationResult result = agentExecutor.fence(actionType, fenceAgent); + + if (result.getStatus() == Status.SKIPPED_ALREADY_IN_STATUS + || result.getStatus() == Status.SKIPPED_DUE_TO_POLICY) { + // end processing, skipping is handled in caller + return result; + } + + if (result.getStatus() == Status.SUCCESS) { + // fence operation was successful, verify if host power status changed + statusResult = waitForStatus(); + if (isRequestedStatusAchieved(statusResult)) { + // requested host power status reached, end with success + return statusResult; + } + } + fenceRetries++; + } while (fenceRetries < allowedFenceActionRetries); + + return new FenceOperationResult( + Status.ERROR, + // fail safe, at least one fence attempt should always be executed, so statusResult shouldn't be null + statusResult == null + ? HostPowerStatus.UNKNOWN + : statusResult.getPowerStatus(), + "Allowed retries to verify host power status exceeded"); + } + + /** + * Executes status operation until requested host power status is reached or allowed number of retries exceeded + * to determine of start/stop fence operation was successful + */ + protected FenceOperationResult waitForStatus() { + FenceOperationResult statusResult = null; + // start at -1, because the 1st iteration is regular and not a retry + int statusRetries = -1; + int unknownStatusReceived = 0; + + log.info( + "Waiting for host '{}' to reach status '{}'", + fencedHost.getHostName(), + requestedPowerStatus); + + // Waiting before first attempt to check the host status. + // This is done because if we will attempt to get host status immediately + // in most cases it will not turn from on/off to off/on and we will need + // to wait a full cycle for it. + ThreadUtils.sleep(getSleepBeforeFirstAttempt()); + + while (statusRetries < allowedWaitForStatusRetries) { + log.info("Attempt {} to get host '{}' status", statusRetries, fencedHost.getHostName()); + statusResult = getStatus(); + if (statusResult.getStatus() == Status.SUCCESS) { + if (statusResult.getPowerStatus() == HostPowerStatus.UNKNOWN) { + if (unknownStatusReceived < getUnknownResultLimit() + && statusRetries < allowedWaitForStatusRetries) { + // unknown power status received, wait a while and retry + ThreadUtils.sleep((int) delayBetweenRetries); + statusRetries++; + unknownStatusReceived++; + } else { + // No need to retry, agent definitions are corrupted + log.error( + "Host '{}' PM Agent definitions are corrupted, aborting fence operation.", + fencedHost.getHostName()); + return new FenceOperationResult( + Status.ERROR, + HostPowerStatus.UNKNOWN, + statusResult.getMessage()); + } + } else if (statusResult.getPowerStatus() == requestedPowerStatus) { + log.info("Host '{}' status is '{}'", fencedHost.getHostName(), requestedPowerStatus); + return new FenceOperationResult( + Status.SUCCESS, + requestedPowerStatus, + ""); + } else { + // host is still not in requested power status + statusRetries++; + if (statusRetries < allowedWaitForStatusRetries) { + ThreadUtils.sleep((int) delayBetweenRetries); + } + } + } else { + log.error("Failed to get host '{}' status.", fencedHost.getHostName()); + return statusResult; + } + } + + log.error( + "Allowed retries to wait until requested status of host '{}' is reached was exceeded.", + fencedHost.getHostName()); + return new FenceOperationResult( + Status.ERROR, + statusResult == null ? HostPowerStatus.UNKNOWN : statusResult.getPowerStatus(), + statusResult == null ? "" : statusResult.getMessage()); + } + + protected boolean isRequestedStatusAchieved(FenceOperationResult result) { + return result.getStatus() == Status.SUCCESS + && result.getPowerStatus() == requestedPowerStatus; + } + + protected int getSleepBeforeFirstAttempt() { + return SLEEP_BEFORE_FIRST_ATTEMPT; + } + + protected int getUnknownResultLimit() { + return UNKNOWN_RESULT_LIMIT; + } +} diff --git a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutorTest.java b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutorTest.java new file mode 100644 index 0000000..cea20fa --- /dev/null +++ b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/SingleAgentPmActionExecutorTest.java @@ -0,0 +1,261 @@ +package org.ovirt.engine.core.bll.pm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.OngoingStubbing; +import org.ovirt.engine.core.common.businessentities.FenceAgent; +import org.ovirt.engine.core.common.businessentities.FencingPolicy; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.pm.FenceActionType; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult.Status; +import org.ovirt.engine.core.common.businessentities.pm.HostPowerStatus; +import org.ovirt.engine.core.common.config.ConfigValues; +import org.ovirt.engine.core.utils.MockConfigRule; + +@RunWith(MockitoJUnitRunner.class) +public class SingleAgentPmActionExecutorTest { + @ClassRule + public static MockConfigRule configRule = + new MockConfigRule( + MockConfigRule.mockConfig(ConfigValues.FenceStartStatusRetries, 1), + MockConfigRule.mockConfig(ConfigValues.FenceStopStatusRetries, 1), + MockConfigRule.mockConfig(ConfigValues.FenceStartStatusDelayBetweenRetriesInSec, 0), + MockConfigRule.mockConfig(ConfigValues.FenceStopStatusDelayBetweenRetriesInSec, 0)); + + @Mock + FenceAgentExecutor fenceAgentExecutor; + + @Mock + FenceAgent fenceAgent; + + @Mock + VDS fencedHost; + private SingleAgentPmActionExecutor executor; + + @Before + public void setup() { + executor = spy(new SingleAgentPmActionExecutor(fencedHost, fenceAgent, new FencingPolicy())); + doReturn(fenceAgentExecutor).when(executor).createAgentExecutor(); + doReturn(0).when(executor).getSleepBeforeFirstAttempt(); + doReturn(0).when(executor).getUnknownResultLimit(); + doReturn("host1").when(fencedHost).getHostName(); + } + + /** + * Test successful status action + */ + @Test + public void successfulGetStatus() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON, ""); + FenceOperationResult[] expectedResults = { + expectedResult + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test failed status action + */ + @Test + public void failedGetStatus() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN, ""); + FenceOperationResult[] expectedResults = { + expectedResult + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test successful start action + */ + @Test + public void successfulStart() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON, ""); + FenceOperationResult[] expectedResults = { + // result of start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + expectedResult + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test successful start action with 1 status retry + */ + @Test + public void successfulStartWithStatusRetry() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON, ""); + FenceOperationResult[] expectedResults = { + // result of start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON, ""), + expectedResult + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test start action with status retries exceeded + */ + @Test + public void failedStartStatusRetriesExceeded() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.OFF, ""); + FenceOperationResult[] expectedResults = { + // result of 1st start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF, ""), + // result of 2nd status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF, ""), + // result of 2nd start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF, ""), + // result of 2nd status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF, ""), + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test start action with UNKNOWN power status limit exceeded + */ + @Test + public void failedStartUnknownStatusLimitExceeded() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN, ""); + FenceOperationResult[] expectedResults = { + // result of 1st start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 2nd start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test start action with UNKNOWN power status limit exceeded on 1st attempt, but 2nd attempt is successful + */ + @Test + public void successfulStartUnknownStatusLimitExceededOn1stAttempt() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""); + FenceOperationResult[] expectedResults = { + // result of 1st start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 2nd start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of 1st status action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON, ""), + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test successful start action, when the 1st start attempt failed, but the 2nd one was successful + */ + @Test + public void successfulStartWithStartRetry() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON, ""); + FenceOperationResult[] expectedResults = { + // result of the 1st start action + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN, ""), + // result of the 2nd start action + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.UNKNOWN, ""), + // result of status action + expectedResult + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test failed stop action, when the 1st start attempt failed and retrying fence for stop is not allowed + */ + @Test + public void failedStopWithStopRetry() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN, ""); + FenceOperationResult[] expectedResults = { + // result of the 1st stop action + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN, ""), + }; + mockFenceActionResults(expectedResults); + + FenceOperationResult result = executor.fence(FenceActionType.STOP); + + validateResult(expectedResult, result); + } + + protected void mockFenceActionResults(FenceOperationResult[] results) { + OngoingStubbing<FenceOperationResult> fenceMethodResult = + when(fenceAgentExecutor.fence(any(FenceActionType.class), any(FenceAgent.class))); + + for (FenceOperationResult result : results) { + fenceMethodResult = fenceMethodResult.thenReturn(result); + } + } + + protected void validateResult(FenceOperationResult expected, FenceOperationResult actual) { + assertNotNull(actual); + assertEquals(expected.getStatus(), actual.getStatus()); + assertEquals(expected.getPowerStatus(), actual.getPowerStatus()); + } +} -- To view, visit https://gerrit.ovirt.org/38358 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I95958b4821fa8f55ee8913e926f9528ae56e20e8 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Martin Peřina <mper...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches
