Frank Kobzik has uploaded a new change for review. Change subject: core: Implement optimized vm data retrieval ......................................................................
core: Implement optimized vm data retrieval This draft makes use of optimized data retrieval mechanism implemented in VDSM (http://gerrit.ovirt.org/14541). Change-Id: Ifff9d828c67e51a5e80388fdc77355d4b6415f34 Signed-off-by: Frantisek Kobzik <fkob...@redhat.com> --- M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java A backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/AllVmRuntimeStats.java A backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/GetAllVmRuntimeStatsVDSCommand.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/IVdsServer.java A backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/InnerVmRuntimeStats.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerConnector.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerWrapper.java 8 files changed, 311 insertions(+), 7 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/26/19126/1 diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java index f700a11..f1de3b9 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java @@ -13,7 +13,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; @@ -71,6 +70,7 @@ import org.ovirt.engine.core.vdsbroker.irsbroker.IRSErrorException; import org.ovirt.engine.core.vdsbroker.vdsbroker.DestroyVDSCommand; import org.ovirt.engine.core.vdsbroker.vdsbroker.FullListVdsCommand; +import org.ovirt.engine.core.vdsbroker.vdsbroker.GetAllVmRuntimeStatsVDSCommand; import org.ovirt.engine.core.vdsbroker.vdsbroker.GetAllVmStatsVDSCommand; import org.ovirt.engine.core.vdsbroker.vdsbroker.GetStatsVDSCommand; import org.ovirt.engine.core.vdsbroker.vdsbroker.GetVmStatsVDSCommand; @@ -949,14 +949,23 @@ } VdsBrokerCommand<VdsIdAndVdsVDSCommandParametersBase> command; - if (!_vdsManager.getRefreshStatistics()) { - command = new ListVDSCommand<VdsIdAndVdsVDSCommandParametersBase>( + + if (_vdsManager.getRefreshStatistics()) { + command = new GetAllVmStatsVDSCommand<>( new VdsIdAndVdsVDSCommandParametersBase(_vds)); + _runningVms = (Map<Guid, VmInternalData>) command.executeWithReturnValue(); } else { - command = new GetAllVmStatsVDSCommand<VdsIdAndVdsVDSCommandParametersBase>( + command = new GetAllVmRuntimeStatsVDSCommand<>( new VdsIdAndVdsVDSCommandParametersBase(_vds)); + _runningVms = (Map<Guid, VmInternalData>) command.executeWithReturnValue(); + + if (!command.getVDSReturnValue().getSucceeded()) { // fallback to list command + log.info(String.format("Data optimization not enabled for host %s. Falling back to list command.", _vds.getName())); + command = new ListVDSCommand<>( + new VdsIdAndVdsVDSCommandParametersBase(_vds)); + _runningVms = (Map<Guid, VmInternalData>) command.executeWithReturnValue(); + } } - _runningVms = (Map<Guid, VmInternalData>) command.executeWithReturnValue(); if (command.getVDSReturnValue().getSucceeded()) { List<VM> running = checkVmsStatusChanged(); diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/AllVmRuntimeStats.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/AllVmRuntimeStats.java new file mode 100644 index 0000000..ff5be47 --- /dev/null +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/AllVmRuntimeStats.java @@ -0,0 +1,36 @@ +package org.ovirt.engine.core.vdsbroker.vdsbroker; + +import java.io.Serializable; +import java.util.Map; + +public class AllVmRuntimeStats implements Serializable { + private static final String STATUS = "status"; + private static final String HASHES = "hashes"; + private static final String RUNTIME_STATS = "runtimeStats"; + + private StatusForXmlRpc status; + private Map<String, Map<String, Object>> infoMap; // guid -> propKey -> propVal + + public AllVmRuntimeStats(Map<String, Object> innerMap) { + status = new StatusForXmlRpc((Map<String, Object>) innerMap.get(STATUS)); + infoMap = (Map<String, Map<String, Object>>) innerMap.get(RUNTIME_STATS); + } + + public Map<String, Map<String, Object>> getAllRuntimeStats() { + return infoMap; + } + + public Map<String, Object> getAllRuntimeStatsForVm(String vmId) { + return infoMap.get(vmId); + } + + public Map<GetAllVmRuntimeStatsVDSCommand.Hash, String> getHashesForVm(String vmId) { + return (Map<GetAllVmRuntimeStatsVDSCommand.Hash, String>) infoMap.get(vmId).get(HASHES); + } + + public StatusForXmlRpc getStatus() { + return status; + } + +} + diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/GetAllVmRuntimeStatsVDSCommand.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/GetAllVmRuntimeStatsVDSCommand.java new file mode 100644 index 0000000..5d17ff0 --- /dev/null +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/GetAllVmRuntimeStatsVDSCommand.java @@ -0,0 +1,170 @@ +package org.ovirt.engine.core.vdsbroker.vdsbroker; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.ovirt.engine.core.common.businessentities.VmDynamic; +import org.ovirt.engine.core.common.businessentities.VmStatistics; +import org.ovirt.engine.core.common.vdscommands.VdsIdAndVdsVDSCommandParametersBase; +import org.ovirt.engine.core.compat.Guid; +import org.ovirt.engine.core.vdsbroker.vdsbroker.entities.VmInternalData; + +/** + * Command for retrieving vm stats from VDSM that uses data optimization. + * + * TODO: + * - device stats every ~5 mins, not in each iteration + * - write test to check if all retrieved data is correctly set + */ +public class GetAllVmRuntimeStatsVDSCommand<P extends VdsIdAndVdsVDSCommandParametersBase> extends VdsBrokerCommand<P> { + + public enum Hash { + VM_STATUS("vmStatus"), + VM_CONF_INFO("vmConfInfo"), + DEVICE_STATS("deviceStats"), + VM_GUEST_DETAILS("vmGuestDetails"); + + private final String text; + + private Hash(final String val) { + text = val; + } + + public String toString() { + return text; + } + + // needed because the hashes names is allvmstats struct doesn't correspond to vdsm verbs + public String toStatsHashName() { + switch (this) { + case VM_STATUS: + return "status"; + case VM_CONF_INFO: + return "info"; + case VM_GUEST_DETAILS: + return "guestDetails"; + default: + return null; + } + } + } + + // todo static field should be moved into a holder class + private static AllVmRuntimeStats allVmRuntimeStats; + // map that contains all properties for given vm. vmId -> prop -> name + private static Map<String, Map<String, String>> allVmStats = new HashMap<>(); + // map of vmIds to update for given hash. hash -> [vmIds] + private Map<Hash, List<String>> hashesToUpdate = new HashMap<>(); + // old hash values for given vm to determine which hashes to update. vmId -> hash -> hashValue + private static Map<String, Map<Hash, String>> oldHashValues = new HashMap<>(); + + public GetAllVmRuntimeStatsVDSCommand(P parameters) { + super(parameters, parameters.getVds()); + } + + @Override + protected void ExecuteVdsBrokerCommand() { + allVmRuntimeStats = getBroker().getAllVmRuntimeStats(); + + ProceedProxyReturnValue(); + + initHashes(); + updateMainStatsAndSetHashes(); + updateSubStats(); + setReturnValue(buildVmData()); + } + + private void initHashes() { + for (Hash hash : Hash.values()) { + hashesToUpdate.put(hash, new ArrayList<String>()); + } + } + + /** + * This method goes through the values got via getAllVmRuntimeStats and puts them into allVmStats for all vms. + * For all vms it also compares retrieved hashes and if they differ from their last value, it updates the + * hashesToUpdate collection. + */ + private void updateMainStatsAndSetHashes() { + for (String vmId : allVmRuntimeStats.getAllRuntimeStats().keySet()) { + allVmStats.put(vmId, (Map) allVmRuntimeStats.getAllRuntimeStatsForVm(vmId)); + allVmStats.remove("status"); + + if (oldHashValues.get(vmId) == null) { + oldHashValues.put(vmId, new HashMap<Hash, String>()); + } + + for (Hash hash : Hash.values()) { + if (isHashDifferentForVm(hash, allVmRuntimeStats.getHashesForVm(vmId).get(hash.toStatsHashName()), vmId)) { + logUpgrade(vmId, hash.toString()); + hashesToUpdate.get(hash).add(vmId); + oldHashValues.get(vmId).put(hash, allVmRuntimeStats.getHashesForVm(vmId).get(hash.toStatsHashName())); + } + } + } + } + + /** + * This method updates all "substats" (that correspond to hashes). + */ + private void updateSubStats() { + for (Hash hash : hashesToUpdate.keySet()) { + InnerVmRuntimeStats innerVmRuntimeStats = getBroker().getInnerVmRuntimeStats(hash, hashesToUpdate.get(hash)); + if (innerVmRuntimeStats.getInnerVmStats() == null) { + continue; + } + for (String vmId : innerVmRuntimeStats.getInnerVmStats().keySet()) { + allVmStats.get(vmId).putAll(innerVmRuntimeStats.getInnerStatsForVm(vmId)); + } + } + } + + /** + * This method builds VmInternalData objects for each vm based on allVmStats collection. + */ + private Map<Guid, VmInternalData> buildVmData() { + Map<Guid, VmInternalData> returnVMs = new HashMap<>(); + + for (String vmId : allVmStats.keySet()) { + VmDynamic vmDynamic = new ExtendedVmDynamic(getVds()); + vmDynamic.setId(new Guid(vmId)); + VdsBrokerObjectsBuilder.updateVMDynamicData(vmDynamic, (Map) allVmStats.get(vmId)); + + VmStatistics vmStatistics = new VmStatistics(); + vmStatistics.setId(new Guid(vmId)); + VdsBrokerObjectsBuilder.updateVMStatisticsData(vmStatistics, (Map) allVmStats.get(vmId)); + + VmInternalData vmInternalData = new VmInternalData(vmDynamic, vmStatistics, + VdsBrokerObjectsBuilder.buildVmGuestAgentInterfacesData(vmDynamic.getId(), (Map) allVmStats.get(vmId))); + + returnVMs.put(new Guid(vmId), vmInternalData); + } + + return returnVMs; + } + + private void logUpgrade(String vmId, String hashName) { + log.info(new StringBuilder().append("Must upgrade hash ").append(hashName).append(" for vm ").append(vmId).toString()); + } + + private boolean isHashDifferentForVm(Hash hash, String hashValue, String vmId) { + return hashValue == null || oldHashValues == null || oldHashValues.get(vmId) == null || hashValue.equals(oldHashValues.get(vmId).get(hash)); + } + + @Override + protected boolean getIsPrintReturnValue() { + return false; + } + + @Override + protected StatusForXmlRpc getReturnStatus() { + return allVmRuntimeStats.getStatus(); + } + + @Override + protected Object getReturnValueFromBroker() { + return allVmRuntimeStats.getAllRuntimeStats(); + } + +} diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/IVdsServer.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/IVdsServer.java index 873e82b..731c2ba 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/IVdsServer.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/IVdsServer.java @@ -1,5 +1,6 @@ package org.ovirt.engine.core.vdsbroker.vdsbroker; +import java.util.List; import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; @@ -50,6 +51,10 @@ VMInfoListReturnForXmlRpc getAllVmStats(); + AllVmRuntimeStats getAllVmRuntimeStats(); + + InnerVmRuntimeStats getInnerVmRuntimeStats(GetAllVmRuntimeStatsVDSCommand.Hash statHash, List<String> vmIds); + StatusOnlyReturnForXmlRpc migrate(Map<String, String> migrationInfo); StatusOnlyReturnForXmlRpc migrateStatus(String vmId); diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/InnerVmRuntimeStats.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/InnerVmRuntimeStats.java new file mode 100644 index 0000000..f024085 --- /dev/null +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/InnerVmRuntimeStats.java @@ -0,0 +1,28 @@ +package org.ovirt.engine.core.vdsbroker.vdsbroker; + +import java.io.Serializable; +import java.util.Map; + +public class InnerVmRuntimeStats implements Serializable{ + private static final String STATUS = "status"; + + private StatusForXmlRpc status; + private Map<String, Map<String, String>> infoMap; // id -> prop -> val + + public InnerVmRuntimeStats(Map<String, Object> innerMap, String statName) { + status = new StatusForXmlRpc((Map<String, Object>) innerMap.get(STATUS)); + infoMap = (Map<String, Map<String, String>>) innerMap.get(statName); + } + + public StatusForXmlRpc getStatus() { + return status; + } + + public Map<String, Map<String, String>> getInnerVmStats() { + return infoMap; + } + + public Map<String, String> getInnerStatsForVm(String vmId) { + return infoMap.get(vmId); + } +} diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java index fac9e68..ed47d71 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsBrokerObjectsBuilder.java @@ -889,7 +889,7 @@ } } - private static VMStatus convertToVmStatus(String statusName) { + public static VMStatus convertToVmStatus(String statusName) { VMStatus status = VMStatus.Unassigned; if ("Running".equals(statusName) || "Unknown".equals(statusName)) { status = VMStatus.Up; diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerConnector.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerConnector.java index 9c18928..1f8c058 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerConnector.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerConnector.java @@ -36,6 +36,12 @@ public Map<String, Object> getAllVmStats(); + public Map<String, Object> getAllVmRuntimeStats(); + public Map<String, Object> getVmStatus(String[] vmIds); + public Map<String, Object> getVmConfInfo(String[] vmIds); + public Map<String, Object> getAllVmDeviceStats(); + public Map<String, Object> getVmGuestDetails(String[] vmIds); + public Map<String, Object> migrate(Map<String, String> migrationInfo); public Map<String, Object> migrateStatus(String vmId); diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerWrapper.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerWrapper.java index 024005d..96787c9 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerWrapper.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/vdsbroker/VdsServerWrapper.java @@ -1,10 +1,10 @@ package org.ovirt.engine.core.vdsbroker.vdsbroker; import java.lang.reflect.UndeclaredThrowableException; +import java.util.List; import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; - import org.apache.commons.httpclient.HttpClient; import org.ovirt.engine.core.compat.Guid; import org.ovirt.engine.core.vdsbroker.gluster.GlusterHookContentInfoReturnForXmlRpc; @@ -18,6 +18,9 @@ import org.ovirt.engine.core.vdsbroker.irsbroker.IsoListReturnForXmlRpc; import org.ovirt.engine.core.vdsbroker.irsbroker.OneUuidReturnForXmlRpc; import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcRunTimeException; + + +import static org.ovirt.engine.core.vdsbroker.vdsbroker.GetAllVmRuntimeStatsVDSCommand.Hash; @SuppressWarnings({"rawtypes", "unchecked"}) public class VdsServerWrapper implements IVdsServer { @@ -213,6 +216,53 @@ } @Override + public AllVmRuntimeStats getAllVmRuntimeStats() { + try { + Map<String, Object> xmlRpcReturnValue = vdsServer.getAllVmRuntimeStats(); + AllVmRuntimeStats wrapper = new AllVmRuntimeStats(xmlRpcReturnValue); + return wrapper; + } catch (UndeclaredThrowableException ute) { + throw new XmlRpcRunTimeException(ute); + } + } + + @Override + public InnerVmRuntimeStats getInnerVmRuntimeStats(Hash statHash, List<String> vmIds) { + try { + Map<String, Object> xmlRpcReturnValue = null; + + switch (statHash) { + case VM_STATUS: + xmlRpcReturnValue = vdsServer.getVmStatus(toArray(vmIds)); + break; + case VM_CONF_INFO: + xmlRpcReturnValue = vdsServer.getVmConfInfo(toArray(vmIds)); + break; + case DEVICE_STATS: + xmlRpcReturnValue = vdsServer.getAllVmDeviceStats(); + break; + case VM_GUEST_DETAILS: + xmlRpcReturnValue = vdsServer.getVmGuestDetails(toArray(vmIds)); + break; + } + InnerVmRuntimeStats wrapper = new InnerVmRuntimeStats(xmlRpcReturnValue, statHash.toString()); + return wrapper; + } catch (UndeclaredThrowableException ute) { + throw new XmlRpcRunTimeException(ute); + } + } + + private String[] toArray(List<String> vmIds) { + String[] array = new String[vmIds.size()]; + + for (int i = 0; i < array.length; i++) { + array[i] = vmIds.get(i); + } + + return array; + } + + @Override public StatusOnlyReturnForXmlRpc migrate(Map<String, String> migrationInfo) { try { Map<String, Object> xmlRpcReturnValue = vdsServer.migrate(migrationInfo); -- To view, visit http://gerrit.ovirt.org/19126 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ifff9d828c67e51a5e80388fdc77355d4b6415f34 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Frank Kobzik <fkob...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches
