Hello ofri masad, I'd like you to do a code review. Please visit
http://gerrit.ovirt.org/18520 to review the following change. Change subject: core: Add event logs for balloon issues ...................................................................... core: Add event logs for balloon issues Added AuditLog for two cases: - balloon is requested but the balloon driver on the VM is not responding. - balloon is inflated but MOM lost control over the balloon device (caused by guest agent failure). Conflicts: frontend/webadmin/modules/gwt-common/src/main/resources/org/ovirt/engine/core/Common.gwt.xml packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql Change-Id: Ie83440251779d49b83747a51111a89edccc88f9c Signed-off-by: Ofri Masad <oma...@redhat.com> --- M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java A backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmBalloonInfo.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmStatistics.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java M backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java M backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsProperties.java M frontend/webadmin/modules/gwt-common/src/main/resources/org/ovirt/engine/core/Common.gwt.xml 10 files changed, 199 insertions(+), 20 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/20/18520/1 diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java index c62f32c..42cd84d 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java @@ -381,6 +381,8 @@ VM_POWER_DOWN_FAILED(147), VM_MEMORY_UNDER_GUARANTEED_VALUE(148, AuditLogTimeInterval.MINUTE.getValue() * 15), + VM_BALLOON_DRIVER_ERROR(149, AuditLogTimeInterval.MINUTE.getValue() * 15), + VM_BALLOON_DRIVER_UNCONTROLLED(150, AuditLogTimeInterval.MINUTE.getValue() * 15), USER_ADD_VM_POOL(300), USER_ADD_VM_POOL_FAILED(301), diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmBalloonInfo.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmBalloonInfo.java new file mode 100644 index 0000000..6c84ac9 --- /dev/null +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmBalloonInfo.java @@ -0,0 +1,54 @@ +package org.ovirt.engine.core.common.businessentities; + + +import java.io.Serializable; + +public class VmBalloonInfo implements Serializable { + + private static final long serialVersionUID = -6215874051244541874L; + private Long currentMemory; + private Long balloonMaxMemory; + private Long balloonTargetMemory; + private Long balloonMinMemory; + private boolean balloonDeviceEnabled; + + public Long getCurrentMemory() { + return currentMemory; + } + + public void setCurrentMemory(Long currentMemory) { + this.currentMemory = currentMemory; + } + + public Long getBalloonMaxMemory() { + return balloonMaxMemory; + } + + public void setBalloonMaxMemory(Long balloonMaxMemory) { + this.balloonMaxMemory = balloonMaxMemory; + } + + public Long getBalloonTargetMemory() { + return balloonTargetMemory; + } + + public void setBalloonTargetMemory(Long balloonTargetMemory) { + this.balloonTargetMemory = balloonTargetMemory; + } + + public Long getBalloonMinMemory() { + return balloonMinMemory; + } + + public void setBalloonMinMemory(Long balloonMinMemory) { + this.balloonMinMemory = balloonMinMemory; + } + + public boolean isBalloonDeviceEnabled() { + return balloonDeviceEnabled; + } + + public void setBalloonDeviceEnabled(boolean balloonDeviceEnabled) { + this.balloonDeviceEnabled = balloonDeviceEnabled; + } +} diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmStatistics.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmStatistics.java index 29fb3ad..f605e7c 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmStatistics.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/VmStatistics.java @@ -12,7 +12,7 @@ private Double cpu_sysField = 0.0; // NOT PERSISTED - private Long currentMemory; + private VmBalloonInfo vmBalloonInfo; @Override public int hashCode() { @@ -132,14 +132,6 @@ this.usage_mem_percentField = value; } - public Long getCurrentMemory() { - return currentMemory; - } - - public void setCurrentMemory(Long value) { - currentMemory = value; - } - private String disksUsage; /** @@ -189,4 +181,12 @@ public int compareTo(VmStatistics o) { return BusinessEntityGuidComparator.<VmStatistics>newInstance().compare(this,o); } + + public VmBalloonInfo getVmBalloonInfo() { + return vmBalloonInfo; + } + + public void setVmBalloonInfo(VmBalloonInfo vmBalloonInfo) { + this.vmBalloonInfo = vmBalloonInfo; + } } diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java index 8581df8..52143a0 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java @@ -1454,6 +1454,10 @@ MomPoliciesOnHostSupported(530), + @TypeConverterAttribute(Integer.class) + @DefaultValueAttribute("3") + IterationsWithBalloonProblem(525), + Invalid(65535); private int intValue; diff --git a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java index 30b65cc..46224dc 100644 --- a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java +++ b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java @@ -514,6 +514,8 @@ severities.put(AuditLogType.VM_PAUSED_EPERM, AuditLogSeverity.ERROR); severities.put(AuditLogType.VM_POWER_DOWN_FAILED, AuditLogSeverity.WARNING); severities.put(AuditLogType.VM_MEMORY_UNDER_GUARANTEED_VALUE, AuditLogSeverity.ERROR); + severities.put(AuditLogType.VM_BALLOON_DRIVER_ERROR, AuditLogSeverity.ERROR); + severities.put(AuditLogType.VM_BALLOON_DRIVER_UNCONTROLLED, AuditLogSeverity.ERROR); severities.put(AuditLogType.USER_RUN_VM, AuditLogSeverity.NORMAL); severities.put(AuditLogType.USER_RUN_VM_AS_STATELESS, AuditLogSeverity.NORMAL); severities.put(AuditLogType.USER_FAILED_RUN_VM, AuditLogSeverity.ERROR); diff --git a/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties b/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties index 393b2b5..a70c587 100644 --- a/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties +++ b/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties @@ -316,6 +316,8 @@ VM_PAUSED_EPERM=VM ${VmName} has paused due to storage permissions problem. VM_POWER_DOWN_FAILED=Shutdown of VM ${VmName} failed. VM_MEMORY_UNDER_GUARANTEED_VALUE=VM ${VmName} on host ${VdsName} was guaranteed ${MemGuaranteed} MB but currently has ${MemActual} MB +VM_BALLOON_DRIVER_ERROR=The Balloon driver on VM ${VmName} on host ${VdsName} is requested but unavailable. +VM_BALLOON_DRIVER_UNCONTROLLED=The Balloon device on VM ${VmName} on host ${VdsName} is inflated but the device cannot be controlled (guest agent is down). VDS_INSTALL=Host ${VdsName} installed VDS_INSTALL_FAILED=Host ${VdsName} installation failed. ${FailedInstallMessage}. VDS_INITIALIZING=Host ${VdsName} is initializing. Message: ${ErrorMessage} diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java index f700a11..5af3ae9 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java @@ -25,11 +25,13 @@ import org.ovirt.engine.core.common.businessentities.DiskImageDynamic; import org.ovirt.engine.core.common.businessentities.Entities; import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.VDSGroup; import org.ovirt.engine.core.common.businessentities.VDSStatus; import org.ovirt.engine.core.common.businessentities.VM; import org.ovirt.engine.core.common.businessentities.VMStatus; import org.ovirt.engine.core.common.businessentities.VdsDynamic; import org.ovirt.engine.core.common.businessentities.VdsStatistics; +import org.ovirt.engine.core.common.businessentities.VmBalloonInfo; import org.ovirt.engine.core.common.businessentities.VmDevice; import org.ovirt.engine.core.common.businessentities.VmDeviceGeneralType; import org.ovirt.engine.core.common.businessentities.VmDeviceId; @@ -104,6 +106,8 @@ private final List<Guid> _vmsMovedToDown = new ArrayList<Guid>(); private final List<Guid> _vmsToRemoveFromAsync = new ArrayList<Guid>(); private final List<Guid> _succededToRunVms = new ArrayList<Guid>(); + private static final Map<Guid,Integer> vmsWithBalloonDriverProblem = new HashMap<>(); + private static final Map<Guid,Integer> vmsWithUncontrolledBalloon = new HashMap<>(); private boolean _saveVdsDynamic; private VDSStatus _firstStatus = VDSStatus.forValue(0); private boolean _saveVdsStatistics; @@ -963,6 +967,8 @@ proceedWatchdogEvents(); + proceedBalloonCheck(); + proceedDownVms(); proceedGuaranteedMemoryCheck(); @@ -1313,21 +1319,118 @@ continue; } VmStatistics vmStatistics = vmInternalData.getVmStatistics(); - if (vmStatistics != null && vmStatistics.getCurrentMemory() != null && - vmStatistics.getCurrentMemory() > 0 && - savedVm.getMinAllocatedMem() > vmStatistics.getCurrentMemory() / TO_MEGA_BYTES) { + if (vmStatistics != null && vmStatistics.getVmBalloonInfo().getCurrentMemory() != null && + vmStatistics.getVmBalloonInfo().getCurrentMemory() > 0 && + savedVm.getMinAllocatedMem() > vmStatistics.getVmBalloonInfo().getCurrentMemory() / TO_MEGA_BYTES) { AuditLogableBase auditLogable = new AuditLogableBase(); auditLogable.addCustomValue("VmName", savedVm.getName()); auditLogable.addCustomValue("VdsName", this._vds.getName()); auditLogable.addCustomValue("MemGuaranteed", String.valueOf(savedVm.getMinAllocatedMem())); auditLogable.addCustomValue("MemActual", - Long.toString((vmStatistics.getCurrentMemory() / TO_MEGA_BYTES))); + Long.toString((vmStatistics.getVmBalloonInfo().getCurrentMemory() / TO_MEGA_BYTES))); auditLog(auditLogable, AuditLogType.VM_MEMORY_UNDER_GUARANTEED_VALUE); } } } + private void proceedBalloonCheck() { + if (isBalloonActiveOnHost()) { + for (VmInternalData vmInternalData : _runningVms.values()) { + VmBalloonInfo balloonInfo = vmInternalData.getVmStatistics().getVmBalloonInfo(); + Guid vmId = vmInternalData.getVmDynamic().getId(); + if (_vmDict.get(vmId) == null) { + continue; // if vm is unknown - continue + } + + if (isBalloonDeviceActiveOnVm(vmInternalData) + && (balloonInfo.getCurrentMemory().intValue() == balloonInfo.getBalloonMaxMemory().intValue() + || balloonInfo.getCurrentMemory().intValue() != balloonInfo.getBalloonTargetMemory().intValue())) { + vmBalloonDriverIsRequestedAndUnavailable(vmId); + } else { + vmBalloonDriverIsNotRequestedOrAvailable(vmId); + } + + if (vmInternalData.getVmStatistics().getusage_mem_percent() != null + && vmInternalData.getVmStatistics().getusage_mem_percent() == 0 // guest agent is down + && balloonInfo.getCurrentMemory().intValue() != balloonInfo.getBalloonMaxMemory().intValue()) { + guestAgentIsDownAndBalloonInfalted(vmId); + } else { + guestAgentIsUpOrBalloonDeflated(vmId); + } + + } + } + } + + // remove the vm from the list of vms with uncontrolled inflated balloon + private void guestAgentIsUpOrBalloonDeflated(Guid vmId) { + vmsWithUncontrolledBalloon.remove(vmId); + } + + // add the vm to the list of vms with uncontrolled inflated balloon or increment its counter + // if it is already in the list + private void guestAgentIsDownAndBalloonInfalted(Guid vmId) { + Integer currentVal = vmsWithUncontrolledBalloon.get(vmId); + if (currentVal == null) { + vmsWithUncontrolledBalloon.put(vmId, 1); + } else { + vmsWithUncontrolledBalloon.put(vmId, currentVal + 1); + if (currentVal >= Config.<Integer> GetValue(ConfigValues.IterationsWithBalloonProblem)) { + AuditLogableBase auditLogable = new AuditLogableBase(); + auditLogable.setVmId(vmId); + AuditLogDirector.log(auditLogable, AuditLogType.VM_BALLOON_DRIVER_UNCONTROLLED); + vmsWithUncontrolledBalloon.put(vmId, 0); + } + } + } + + // remove the vm from the list of vms with balloon driver problem + private void vmBalloonDriverIsNotRequestedOrAvailable(Guid vmId) { + vmsWithBalloonDriverProblem.remove(vmId); + } + + // add the vm to the list of vms with balloon driver problem or increment its counter + // if it is already in the list + private void vmBalloonDriverIsRequestedAndUnavailable(Guid vmId) { + Integer currentVal = vmsWithBalloonDriverProblem.get(vmId); + if (currentVal == null) { + vmsWithBalloonDriverProblem.put(vmId, 1); + } else { + vmsWithBalloonDriverProblem.put(vmId, currentVal + 1); + if (currentVal >= Config.<Integer> GetValue(ConfigValues.IterationsWithBalloonProblem)) { + AuditLogableBase auditLogable = new AuditLogableBase(); + auditLogable.setVmId(vmId); + AuditLogDirector.log(auditLogable, AuditLogType.VM_BALLOON_DRIVER_ERROR); + vmsWithBalloonDriverProblem.put(vmId, 0); + } + } + } + + private boolean isBalloonActiveOnHost() { + VDSGroup cluster = getDbFacade().getVdsGroupDao().get(_vds.getVdsGroupId()); + return cluster != null && cluster.isEnableBallooning(); + } + + private boolean isBalloonDeviceActiveOnVm(VmInternalData vmInternalData) { + VM savedVm = _vmDict.get(vmInternalData.getVmDynamic().getId()); + + if (savedVm != null) { + VmBalloonInfo balloonInfo = vmInternalData.getVmStatistics().getVmBalloonInfo(); + return savedVm.getMinAllocatedMem() < savedVm.getMemSizeMb() // minimum allocated mem of VM == total mem, ballooning is impossible + && balloonInfo.isBalloonDeviceEnabled() + && balloonInfo.getBalloonTargetMemory().intValue() != balloonInfo.getBalloonMaxMemory().intValue(); // ballooning was not requested/enabled on this VM + } + return false; + } + + private Long toMegaByte(Long kilobytes) { + if (kilobytes != null && kilobytes > 0) { + return kilobytes / TO_MEGA_BYTES; + } + return 0L; + } + protected static boolean isNewWatchdogEvent(VmDynamic vmDynamic, VM vmTo) { Long lastWatchdogEvent = vmDynamic.getLastWatchdogEvent(); return vmTo != null && lastWatchdogEvent != null diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java index 729efc8..96e4a5b 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java @@ -26,6 +26,7 @@ import org.ovirt.engine.core.common.businessentities.VDSDomainsData; import org.ovirt.engine.core.common.businessentities.VMStatus; import org.ovirt.engine.core.common.businessentities.VdsTransparentHugePagesState; +import org.ovirt.engine.core.common.businessentities.VmBalloonInfo; import org.ovirt.engine.core.common.businessentities.VmDynamic; import org.ovirt.engine.core.common.businessentities.VmExitStatus; import org.ovirt.engine.core.common.businessentities.VmGuestAgentInterface; @@ -306,16 +307,24 @@ // ------------- vm memory statistics ----------------------- vm.setusage_mem_percent(AssignIntValue(xmlRpcStruct, VdsProperties.vm_usage_mem_percent)); - vm.setCurrentMemory(getCurrMemory(xmlRpcStruct)); - + vm.setVmBalloonInfo(getBalloonInfo(xmlRpcStruct)); } - private static Long getCurrMemory(Map<String, Object> xmlRpcStruct) { + private static VmBalloonInfo getBalloonInfo(Map<String, Object> xmlRpcStruct) { Map<String, Object> balloonInfo = (Map<String, Object>) xmlRpcStruct.get(VdsProperties.vm_balloonInfo); - if (balloonInfo != null) { - return AssignLongValue(balloonInfo, VdsProperties.vm_balloon_cur); + VmBalloonInfo vmBalloonInfo = new VmBalloonInfo(); + if (balloonInfo != null && balloonInfo.size() > 0) { + vmBalloonInfo.setCurrentMemory(AssignLongValue(balloonInfo, VdsProperties.vm_balloon_cur)); + vmBalloonInfo.setBalloonMaxMemory(AssignLongValue(balloonInfo, VdsProperties.vm_balloon_max)); + vmBalloonInfo.setBalloonTargetMemory(AssignLongValue(balloonInfo, VdsProperties.vm_balloon_target)); + vmBalloonInfo.setBalloonMinMemory(AssignLongValue(balloonInfo, VdsProperties.vm_balloon_min)); + if (balloonInfo.size() >= 4) { // only if all 4 properties are found the balloon is considered enabled (available from 3.3) + vmBalloonInfo.setBalloonDeviceEnabled(true); + } + } else { + vmBalloonInfo.setBalloonDeviceEnabled(false); } - return null; + return vmBalloonInfo; } public static void updateVDSDynamicData(VDS vds, Map<String, Object> xmlRpcStruct) { diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsProperties.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsProperties.java index 62019bf..b39fe5c 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsProperties.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsProperties.java @@ -194,6 +194,9 @@ public static final String vm_balloonInfo = "balloonInfo"; public static final String vm_balloon_cur = "balloon_cur"; + public static final String vm_balloon_max = "balloon_max"; + public static final String vm_balloon_min = "balloon_min"; + public static final String vm_balloon_target = "balloon_target"; public static final String DriveC = "hda"; // drive C: public static final String DriveE = "hdb"; // drive E: (D: is the CD-ROM) diff --git a/frontend/webadmin/modules/gwt-common/src/main/resources/org/ovirt/engine/core/Common.gwt.xml b/frontend/webadmin/modules/gwt-common/src/main/resources/org/ovirt/engine/core/Common.gwt.xml index a4126a3..dfbc941 100644 --- a/frontend/webadmin/modules/gwt-common/src/main/resources/org/ovirt/engine/core/Common.gwt.xml +++ b/frontend/webadmin/modules/gwt-common/src/main/resources/org/ovirt/engine/core/Common.gwt.xml @@ -44,7 +44,7 @@ <include name="common/businessentities/OpenstackNetworkProviderProperties.java" /> <include name="common/businessentities/OpenstackNetworkPluginType.java" /> <include name="common/businessentities/DiskAlignment.java" /> - <include name="common/businessentities/OpenstackImageProviderProperties.java" /> + <include name="common/businessentities/VmBalloonInfo.java" /> <!-- Network business entities --> <include name="common/businessentities/network/VdsNetworkInterface.java" /> -- To view, visit http://gerrit.ovirt.org/18520 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ie83440251779d49b83747a51111a89edccc88f9c Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: ovirt-engine-3.3 Gerrit-Owner: Martin Sivák <msi...@redhat.com> Gerrit-Reviewer: ofri masad <oma...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches