Noam Slomianko has uploaded a new change for review. Change subject: External scheduler - move to class based plugins ......................................................................
External scheduler - move to class based plugins - modules now contain classes and do not direcly hold the functions - Move to the final function names - Added examples and a README file on plugins Signed-off-by: Noam Slomianko <[email protected]> Change-Id: I94c7de6d036bd279f34c0ba246a129d88caa130a --- A plugins/README M plugins/dummy.py A plugins/examples/even_vm_distribution.py A plugins/examples/max_vm_filter.py A plugins/examples/vm_balance.py A plugins/pylolz M src/ovirtscheduler/loader.py M src/ovirtscheduler/request_handler.py M src/ovirtscheduler/runner.py M src/ovirtscheduler/utils.py 10 files changed, 257 insertions(+), 132 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-scheduler-proxy refs/changes/39/18739/1 diff --git a/plugins/README b/plugins/README new file mode 100644 index 0000000..03dc510 --- /dev/null +++ b/plugins/README @@ -0,0 +1,32 @@ +Plugins are loaded from *.py files in the plugin folder. +Each file can contain many classes, each class that has a methods named do_filter/do_score/do_balance will be registered in the engine. +Classes should be of the form: + +Class Sample(): + properties_validation = '{key1 = regex1, key2 = regex2}' + def __init__(self): + ... + def do_filter(self, hosts, vm, args): + ''' + My filter description + ''' + ... + def do_score(self, hosts, vm, args): + ''' + My score description + ''' + ... + def do_balance(self, hosts, args): + ''' + My balance description + ''' + ... + +(a class can have all of the methods or just some of them) + +If present, the method comment will be taken and presented to the engine user selecting this filter. + +Additionally the class can contain a member called "properties_validation" that will be used to direct the engine what values to pass to the plugin. +This member is a string representing a map between the names of the expected args that will be passed to the filter, and the regex of possible values. +Example: properties_validation = '{ cpu = [0-9] }' will mean that you expect "args" to contain a key named "cpu" with values between 0 and 9 + diff --git a/plugins/dummy.py b/plugins/dummy.py index 1c6a108..19a685e 100755 --- a/plugins/dummy.py +++ b/plugins/dummy.py @@ -33,96 +33,48 @@ # do not remove this import, the ovirtsdk is not going to work without it -from ovirtsdk.xml import params -from ovirtsdk import api -import ovirtsdk.infrastructure.brokers +#from ovirtsdk.xml import params +#from ovirtsdk import api +#import ovirtsdk.infrastructure.brokers import sys -class SampleFilter(): +class dummy(): + # Notice: plugin filters are going to run in process that will be created and destroyed + # per request, you cannot save state in memory + def do_filter(self, hosts, vm, args): + '''This is a simple filter that returns all given host ID''' + try: + #use hosts IDs and VM ID to call the Rest API and make a decision - def __init__(self): - pass + acceptedHostsIDs = [] + for hostID in hosts: + #Do work + acceptedHostsIDs.append(hostID) + #as this will run as a process, communications will be through stdout + #use log and not print if you want to have debug information + print acceptedHostsIDs + except Exception as ex: + print >> sys.stderr, ex - def filter(self, hosts, vm, args): + #Files can hold all three supported functions (filterFucntion,scoreFunction,balanceFunction) + def do_score(self, hosts, vm, args): + '''This is a simple score function that returns all given host ID with score 50''' + try: + hostScores = [] + #use hosts IDs and VM ID to call the Rest API and make a decision + for hostID in hosts: + #Do work + hostScores.append((hostID, 50)) + print hostScores + except Exception as ex: + print >> sys.stderr, ex - #use hosts IDs and VM ID to call the Rest API and make a decision - - acceptedHostsIDs = [] - for hostID in hosts: - #Do work - acceptedHostsIDs.append(hostID) - - return acceptedHostsIDs - - -# Notice: plugin filters are going to run in process that will be created and destroyed -# per request, you cannot save state in memory -def filterFunction(hosts, vm, args): - '''This is a simple filter that returns all given host ID''' - try: - filterClassInstance = SampleFilter() - #as this will run as a process, communications will be through stdout - #use log and not print if you want to have debug information - print filterClassInstance.filter(hosts, vm, args) - except Exception as ex: - print >> sys.stderr, ex - -regex_filter = "" - - -#Files can hold all three supported functions (filterFucntion,scoreFunction,balanceFunction) -class SampleScore(): - def __init__(self): - pass - - def score(self, hosts, vm, args): - - hostScores = [] - #use hosts IDs and VM ID to call the Rest API and make a decision - ''' - Sample code: - #take from configuration - api = API(url='http://host:port', username='user@domain', password='password') - for id in hosts: - hostObject = api.hosts.get(query='id=' + id) - - ... etc - ''' - for hostID in hosts: - #Do work - hostScores.append((hostID, 50)) - - return hostScores - - -def scoreFunction(hosts, vm, args): - '''This is a simple score function that returns all given host ID with score 50''' - try: - scoreClassInstance = SampleScore() - print scoreClassInstance.score(hosts, vm, args) - except Exception as ex: - print >> sys.stderr, ex - -regex_score = "" - - -class SampleBalance(): - def __init__(self): - pass - - def balance(self, hosts, args): - #use hosts IDs to call the Rest API and make a decision - #return the wanted vm and a list of underutilised hosts - return '33333333-3333-3333-3333-333333333333', ['11111111-1111-1111-1111-111111111111'] - - -def balanceFunction(hosts, args): - '''This is a fake balance function that always return the guid 33333333-3333-3333-3333-333333333333''' - try: - balanceInstance = SampleBalance() - print balanceInstance.balance(hosts, args) - except Exception as ex: - print >> sys.stderr, ex - -regex_balance = "" \ No newline at end of file + def do_balance(self, hosts, args): + '''This is a fake balance function that always return the guid 33333333-3333-3333-3333-333333333333''' + try: + #use hosts IDs to call the Rest API and make a decision + #return the wanted vm and a list of underutilised hosts + print ('33333333-3333-3333-3333-333333333333', ['11111111-1111-1111-1111-111111111111']) + except Exception as ex: + print >> sys.stderr, ex \ No newline at end of file diff --git a/plugins/examples/even_vm_distribution.py b/plugins/examples/even_vm_distribution.py new file mode 100644 index 0000000..f7090af --- /dev/null +++ b/plugins/examples/even_vm_distribution.py @@ -0,0 +1,32 @@ +from ovirtsdk.xml import params +from ovirtsdk.api import API +import sys + + +class even_vm_distribution(): + '''rank hosts by the number of running vms on them, with the least first''' + + properties_validation = '' + + def do_score(self, hosts_ids, vm_id, args_map): + #open a connection to the rest api + try: + connection = API(url='http://host:port', + username='user@domain', password='') + except BaseException as ex: + #letting the external proxy know there was an error + print >> sys.stderr, ex + return + + #get all the hosts with the given ids + engine_hosts = \ + connection.hosts.list( + query=" or ".join(["id=%s" % u for u in hosts_ids])) + + #iterate over them and score them based on the number of vms running + host_scores = [] + for engine_host in engine_hosts: + if(engine_host and + engine_host.summary): + host_scores.append((engine_host.id, engine_host.summary.active)) + print host_scores diff --git a/plugins/examples/max_vm_filter.py b/plugins/examples/max_vm_filter.py new file mode 100644 index 0000000..f12c0db --- /dev/null +++ b/plugins/examples/max_vm_filter.py @@ -0,0 +1,37 @@ +from ovirtsdk.xml import params +from ovirtsdk.api import API +import sys + + +class max_vms(): + '''returns only hosts with less running vms then the maximum''' + + #What are the values this module will accept, used to present + #the user with options + properties_validation = 'maximum_vm_count=[0-9]*' + + def do_filter(self, hosts_ids, vm_id, args_map): + #open a connection to the rest api + try: + connection = API(url='http://host:port', + username='user@domain', password='') + except BaseException as ex: + #letting the external proxy know there was an error + print >> sys.stderr, ex + return + + #get our parameters from the map + maximum_vm_count = int(args_map.get('maximum_vm_count', 100)) + + #get all the hosts with the given ids + engine_hosts = \ + connection.hosts.list( + query=" or ".join(["id=%s" % u for u in hosts_ids])) + + #iterate over them and decide which to accept + accepted_host_ids = [] + for engine_host in engine_hosts: + if(engine_host and + engine_host.summary.active < maximum_vm_count): + accepted_host_ids.append(engine_host.id) + print accepted_host_ids diff --git a/plugins/examples/vm_balance.py b/plugins/examples/vm_balance.py new file mode 100644 index 0000000..577cace --- /dev/null +++ b/plugins/examples/vm_balance.py @@ -0,0 +1,56 @@ +from ovirtsdk.xml import params +from ovirtsdk.api import API +import sys +from time import strftime + + +class vm_balance(): + '''moves a vm from a host with to many''' + + #What are the values this module will accept, used to present + #the user with options + properties_validation = 'maximum_vm_count=[0-9]*' + + def do_balance(self, hosts_ids, args_map): + #open a connection to the rest api + try: + connection = API(url='http://host:port', + username='user@domain', password='') + except BaseException as ex: + #letting the external proxy know there was an error + print >> sys.stderr, ex + return + + #get our parameters from the map + maximum_vm_count = int(args_map.get('maximum_vm_count', 100)) + + #get all the hosts with the given ids + engine_hosts = \ + connection.hosts.list( + query=" or ".join(["id=%s" % u for u in hosts_ids])) + + #iterate over them and decide which to balance from + over_loaded_host = None + white_listed_hosts = [] + for engine_host in engine_hosts: + if(engine_host): + if (engine_host.summary.active < maximum_vm_count): + white_listed_hosts.append(engine_host.id) + continue + if(not over_loaded_host or + over_loaded_host.summary.active + < engine_host.summary.active): + over_loaded_host = engine_host + + if(not over_loaded_host): + return + + selected_vm = None + #just pick the first we find + host_vms = connection.vms.list('host='+over_loaded_host.name) + if host_vms: + selected_vm = host_vms[0].id + else: + return + + print (selected_vm, white_listed_hosts) diff --git a/plugins/pylolz b/plugins/pylolz new file mode 120000 index 0000000..acd4152 --- /dev/null +++ b/plugins/pylolz @@ -0,0 +1 @@ +/usr/bin/python \ No newline at end of file diff --git a/src/ovirtscheduler/loader.py b/src/ovirtscheduler/loader.py index 56259e1..ebc2f3b 100644 --- a/src/ovirtscheduler/loader.py +++ b/src/ovirtscheduler/loader.py @@ -1,39 +1,48 @@ import os import utils import sys +import inspect -def analyze(path, name): - retValue = (name,) - try: - os.chdir(path) - mod = __import__(name) +class loader(): + ''' + Loads a module and checks if it has certain functions + Will run as a process for safety, so it prints the result to stdout + ''' - retValue += \ - getAttributes(mod, - utils.FILTER, - utils.FILTER_REGEX) - retValue += \ - getAttributes(mod, - utils.SCORE, - utils.SCORE_REGEX) - retValue += \ - getAttributes(mod, - utils.BALANCE, - utils.BALANCE_REGEX) - except Exception as ex: - print >> sys.stderr, ex + def analyze(self, path, name): + retValue = (name,) + try: + os.chdir(path) + mod = __import__(name) - print retValue + for name, obj in inspect.getmembers(mod, inspect.isclass): + retValue += \ + self.getAttributes(obj, + name, + utils.FILTER) + retValue += \ + self.getAttributes(obj, + name, + utils.SCORE) + retValue += \ + self.getAttributes(obj, + name, + utils.BALANCE) + except Exception as ex: + print >> sys.stderr, ex + print retValue -def getAttributes(mod, function_name, regex_name): - description = "" - regex_map = "" - if hasattr(mod, function_name): - description = getattr(mod, function_name).__doc__ - if hasattr(mod, regex_name): - regex_map = getattr(mod, regex_name) - return ((function_name, description, regex_map),) - else: - return () + def getAttributes(self, cls, cls_name, function_name): + description = "" + regex_map = "" + if hasattr(cls, function_name): + func = getattr(cls, function_name) + if func.__doc__: + description = func.__doc__ + if hasattr(cls, utils.REGEX): + regex_map = getattr(cls, utils.REGEX) + return ((cls_name, function_name, description, regex_map),) + else: + return () diff --git a/src/ovirtscheduler/request_handler.py b/src/ovirtscheduler/request_handler.py index 49dd933..013fa1a 100644 --- a/src/ovirtscheduler/request_handler.py +++ b/src/ovirtscheduler/request_handler.py @@ -46,6 +46,7 @@ utils.BALANCE: self._balancers, utils.SCORE: self._scores } + self._class_to_module_map = {} self.loadModules() def loadModules(self): @@ -61,6 +62,7 @@ continue runner = PythonMethodRunner( self._analyzerDir, + utils.LOADER_MODULE, utils.LOADER_MODULE, utils.LOADER_FUNC, (self._pluginDir, module)) @@ -92,10 +94,11 @@ availableFunctions = runner.getResults() moduleName = availableFunctions[0] - for functionName, description, custom_properties_map \ + for className, functionName, description, custom_properties_map \ in availableFunctions[1:]: self._director[functionName][moduleName] = \ (description, custom_properties_map) + self._class_to_module_map[className] = moduleName self._logger.info("loadModules::registering::loaded- " "filters:" + str(self._filters) + @@ -139,6 +142,7 @@ filterRunners = [ PythonMethodRunner( self._pluginDir, + self._class_to_module_map[f], f, utils.FILTER, (hostIDs, vmID, properties_map)) @@ -191,6 +195,7 @@ scoreRunners = [ (PythonMethodRunner( self._pluginDir, + self._class_to_module_map[name], name, utils.SCORE, (hostIDs, vmID, properties_map)), weight) @@ -215,6 +220,7 @@ return runner = PythonMethodRunner(self._pluginDir, + self._class_to_module_map[balance], balance, utils.BALANCE, (hostIDs, properties_map)) diff --git a/src/ovirtscheduler/runner.py b/src/ovirtscheduler/runner.py index 465d2fb..83431a1 100755 --- a/src/ovirtscheduler/runner.py +++ b/src/ovirtscheduler/runner.py @@ -25,14 +25,14 @@ class PythonMethodRunner(Thread): - def __init__(self, path, module, method, args): + def __init__(self, path, module, cls, method, args): super(PythonMethodRunner, self).__init__(group=None) self._logger = logging.getLogger() self._path = path self._result = None self._error = None self._process = None - self._script = self.createScript(module, method, args) + self._script = self.createScript(module, cls, method, args) def run(self): try: @@ -46,9 +46,13 @@ self._error = "PythonMethodRunner::" \ "Unable to parse result: %s" \ " got error : %s " % (result, ex) - self._error = error + if error: + self._error = error except Exception as ex: self._error = ex + + if(self._error): + self._logger.error("PythonMethodRunner: script %s got error %s", self._script, self._error) def getResults(self): return self._result @@ -59,10 +63,11 @@ def stop(self): return utils.killProcess(self._process) - def createScript(self, module, method, args): - commandTemplate = "import %(module)s; %(module)s.%(method)s%(args)s" + def createScript(self, module, cls, method, args): + commandTemplate = "import %(module)s; %(module)s.%(class)s().%(method)s%(args)s" commandString = commandTemplate % { "module": module, + "class": cls, "method": method, "args": repr(utils.createFunctionArgs(args)) } diff --git a/src/ovirtscheduler/utils.py b/src/ovirtscheduler/utils.py index 48ae5c2..c60a44d 100644 --- a/src/ovirtscheduler/utils.py +++ b/src/ovirtscheduler/utils.py @@ -20,15 +20,10 @@ import subprocess from time import time -FILTER = 'filterFunction' -FILTER_DESCRIPTION = 'desc_filter' -FILTER_REGEX = 'regex_filter' -SCORE = 'scoreFunction' -SCORE_DESCRIPTION = 'desc_score' -SCORE_REGEX = 'regex_score' -BALANCE = 'balanceFunction' -BALANCE_DESCRIPTION = 'desc_balance' -BALANCE_REGEX = 'regex_balance' +FILTER = 'do_filter' +SCORE = 'do_score' +BALANCE = 'do_balance' +REGEX = 'properties_validation' LOADER_MODULE = 'loader' LOADER_FUNC = 'analyze' -- To view, visit http://gerrit.ovirt.org/18739 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I94c7de6d036bd279f34c0ba246a129d88caa130a Gerrit-PatchSet: 1 Gerrit-Project: ovirt-scheduler-proxy Gerrit-Branch: master Gerrit-Owner: Noam Slomianko <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
