Alon Bar-Lev has uploaded a new change for review.
Change subject: packaging: use yum API
......................................................................
packaging: use yum API
PREVIOUS IMPLEMENTATION
Use hybrid of yum API, yum cli, rpm cli
NEW IMPLEMENTATION
Use the yum python API, use single transaction, only top-level
components, proper logging.
The minyum module is shared between this module and bootstrap.
Change-Id: I01c0c10c3bca42770bf05222c8b2b82b01fee0a6
Signed-off-by: Alon Bar-Lev <alo...@redhat.com>
---
M Makefile
M packaging/fedora/setup/engine-upgrade.py
A packaging/fedora/setup/miniyum.py
M packaging/fedora/spec/ovirt-engine.spec.in
4 files changed, 997 insertions(+), 265 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/21/8221/1
diff --git a/Makefile b/Makefile
index c0ff1ad..c80067a 100644
--- a/Makefile
+++ b/Makefile
@@ -258,6 +258,7 @@
install -m 644 packaging/fedora/setup/setup_sequences.py
$(DESTDIR)$(DATA_DIR)/scripts
install -m 644 packaging/fedora/setup/setup_controller.py
$(DESTDIR)$(DATA_DIR)/scripts
install -m 644 packaging/fedora/setup/common_utils.py
$(DESTDIR)$(DATA_DIR)/scripts
+ install -m 644 packaging/fedora/setup/miniyum.py
$(DESTDIR)$(DATA_DIR)/scripts
install -m 644 packaging/fedora/setup/output_messages.py
$(DESTDIR)$(DATA_DIR)/scripts
install -m 644 packaging/fedora/setup/post_upgrade.py
$(DESTDIR)$(DATA_DIR)/scripts
diff --git a/packaging/fedora/setup/engine-upgrade.py
b/packaging/fedora/setup/engine-upgrade.py
index c2d43ed..f03aca4 100755
--- a/packaging/fedora/setup/engine-upgrade.py
+++ b/packaging/fedora/setup/engine-upgrade.py
@@ -5,13 +5,14 @@
import os
import shutil
import logging
+import signal
import traceback
import types
import pwd
from optparse import OptionParser
-import yum
import common_utils as utils
import basedefs
+from miniyum import MiniYum
# Consts
#TODO: Work with a real list here
@@ -117,6 +118,7 @@
MSG_INFO_UPGRADE_OK = "%s upgrade completed successfully!" % basedefs.APP_NAME
MSG_INFO_STOP_INSTALL_EXIT="Upgrade stopped, Goodbye."
MSG_INFO_UPDATE_ENGINE_PROFILE="Updating ovirt-engine Profile"
+MSG_INSTALL_GPG = "\nApprove installing GPG key userid=%s hexkeyid=%s"
MSG_ALERT_STOP_ENGINE="\nDuring the upgrade process, %s will not be
accessible.\n\
All existing running virtual machines will continue but you will not be able
to\n\
@@ -215,44 +217,80 @@
print MSG_ERROR_USER_NOT_ROOT%(username)
sys.exit(1)
-class MYum():
+class _miniyumsink(object):
+
+ def _currentfds(self):
+ return (
+ os.dup(sys.stdin.fileno()),
+ os.dup(sys.stdout.fileno()),
+ os.dup(sys.stderr.fileno())
+ )
+
def __init__(self):
- self.updated = False
- self.yumbase = None
- self.upackages = []
- self.ipackages = []
- self.__initbase()
- self.tid = None
+ self._fds = self._currentfds()
- def __initbase(self):
- self.yumbase = yum.YumBase()
- self.yumbase.preconf.disabled_plugins = ['versionlock']
- self.yumbase.conf.cache = False # Do not relay on existing cache
- self.yumbase.cleanMetadata()
- self.yumbase.cleanSqlite()
+ def __del__(self):
+ os.close(self._in)
+ os.close(self._out)
- def _validateRpmLockList(self):
- rpmLockList = []
- for rpmName in basedefs.RPM_LOCK_LIST.split():
- cmd = [
- basedefs.EXEC_RPM, "-q", rpmName,
- ]
- output, rc = utils.execCmd(cmdList=cmd)
- if rc == 0:
- rpmLockList.append(rpmName)
+ def verbose(self, msg):
+ logging.debug("YUM: VERB: %s" % msg)
- return rpmLockList
+ def info(self, msg):
+ logging.info("YUM: OK: %s" % msg)
+
+ def error(self, msg):
+ logging.error("YUM: FAIL: %s" % msg)
+
+ def keepAlive(self, msg):
+ pass
+
+ def askForGPGKeyImport(self, userid, hexkeyid):
+ logging.warning("YUM: APPROVE-GPG: userid=%s, hexkeyid=%s" % (
+ userid,
+ hexkeyid
+ ))
+ save = self._currentfds()
+ for i in range(3):
+ os.dup2(self._fds[i], i)
+ print MSG_INSTALL_GPG % (userid, hexkeyid)
+ ret = utils.askYesNo(INFO_Q_PROCEED)
+ for i in range(3):
+ os.dup2(save[i], i)
+ return ret
+
+class MYum():
+
+ class _transaction(object):
+ def __init__(self, parent, transaction):
+ self._parent = parent
+ self._transaction = transaction
+
+ def __enter__(self):
+ self._parent._unlock()
+ self._transaction.__enter__()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self._transaction.__exit__(exc_type, exc_value, traceback)
+ self._parent._lock()
+
+ def __init__(self, miniyum):
+ self._miniyum = miniyum
def _lock(self):
logging.debug("Yum lock started")
+ pkgs = [
+ p['display_name'] for p in
+ self._miniyum.queryPackages(
+ patterns=basedefs.RPM_LOCK_LIST.split()
+ )
+ if p['operation'] == 'installed'
+ ]
+
# Create RPM lock list
- cmd = [
- basedefs.EXEC_RPM, "-q",
- ] + self._validateRpmLockList()
- output, rc = utils.execCmd(cmdList=cmd, failOnError=True,
msg=MSG_ERROR_YUM_LOCK)
with open(basedefs.FILE_YUM_VERSION_LOCK, 'a') as yumlock:
- yumlock.write(output)
+ yumlock.write("\n".join(pkgs))
logging.debug("Yum lock completed successfully")
def _unlock(self):
@@ -270,163 +308,49 @@
fd.close()
logging.debug("Yum unlock completed successfully")
+ def clean(self):
+ with self._miniyum.transaction():
+ self._miniyum.clean()
+
+ def transaction(self):
+ return MYum._transaction(self, self._miniyum.transaction())
+
+ def begin(self):
+ self._miniyum.update(['ovirt-engine'])
+ self.emptyTransaction = not self._miniyum.buildTransaction()
+
def update(self):
- self.tid = self.getLatestTid(False)
- self._unlock()
- try:
- # yum update ovirt-engine
- # TODO: Run test transaction
- logging.debug("Yum update started")
- cmd = [
- basedefs.EXEC_YUM, "update", "-q", "-y",
- ] + RPM_LIST.split()
- output, rc = utils.execCmd(cmdList=cmd, failOnError=True,
msg=MSG_ERROR_YUM_UPDATE)
- logging.debug("Yum update completed successfully")
- finally:
- self._lock()
+ self._miniyum.processTransaction()
- def updateAvailable(self):
- logging.debug("Yum list updates started")
+ def getPackages(self):
+ return self._miniyum.queryTransaction()
- # Get packages info from yum
- rpms = RPM_LIST.split()
- logging.debug("Getting list of packages to upgrade")
- pkgs = self.yumbase.doPackageLists(patterns=rpms)
- upkgs = self.yumbase.doPackageLists(pkgnarrow="updates", patterns=rpms)
-
- # Save update candidates
- if upkgs.updates:
- self.upackages = [str(i) for i in sorted(upkgs.updates)] # list of
rpm names to update
- logging.debug("%s Packages marked for
update:"%(len(self.upackages)))
- logging.debug(self.upackages)
- else:
- logging.debug("No packages marked for update")
-
- # Save installed packages
- self.ipackages = [str(i) for i in sorted(pkgs.installed)] # list of
rpm names already installed
- logging.debug("Installed packages:")
- logging.debug(self.ipackages)
-
- logging.debug("Yum list updated completed successfully")
-
-
- # Return
- if upkgs.updates:
- return True
- else:
- return False
-
- def packageAvailable(self, pkg):
- pkglist = self.yumbase.doPackageLists(patterns=[pkg]).available
- return len(pkglist) > 0
-
- def packageInstalled(self, pkg):
- pkglist = self.yumbase.doPackageLists(patterns=[pkg]).installed
- return len(pkglist) > 0
-
- def depListForRemoval(self, pkgs):
-
- deplist = []
-
- # Create list of all packages to remove
- pkgs = self.yumbase.doPackageLists(patterns=pkgs).installed
- for pkg in pkgs:
- self.yumbase.remove(name=pkg.name)
-
- # Resolve dependencies for removing packages
- self.yumbase.resolveDeps()
-
- # Create a list of deps packages
- for pkg in self.yumbase.tsInfo.getMembers():
- if pkg.isDep:
- deplist.append(pkg.name)
-
- # Clear transactions from the 'self' object
- self.yumbase.closeRpmDB()
-
- # Return the list of dependencies
- return deplist
-
- def rollbackAvailable(self):
+ def rollbackAvailable(self, packages):
logging.debug("Yum rollback-avail started")
# Get All available packages in yum
rpms = RPM_LIST.split()
- pkgs = self.yumbase.pkgSack.returnPackages(patterns=rpms)
- available = [str(i) for i in sorted(pkgs)] # list of available rpm
names
- logging.debug("%s Packages available in yum:"%(len(available)))
- logging.debug(available)
+ pkgs = [
+ x['display_name'] for x in
+ self._miniyum.queryLocalCachePackages(patterns=rpms)
+ ]
+ logging.debug("%s Packages available in yum:"%(len(pkgs)))
+ logging.debug(pkgs)
# Verify all installed packages available in yum
# self.ipackages is populated in updateAvailable
- for installed in self.ipackages:
- if installed not in available:
+ name_packages = [p['name'] for p in packages]
+ for installed in [
+ x['display_name'] for x in
+ self._miniyum.queryPackages(patterns=rpms)
+ if x['operation'] == 'installed'
+ ]:
+ if installed in name_packages and not installed not in pkgs:
logging.debug("%s not available in yum"%(installed))
return False
logging.debug("Yum rollback-avail completed successfully")
return True
-
- def rollback(self):
- upgradeTid = self.getLatestTid(True)
- if int(upgradeTid) <= int(self.tid):
- logging.error("Mismatch in yum TID, target TID (%s) is not higher
than %s" %(upgradeTid, self.tid))
- raise Exception(MSG_ERROR_YUM_TID)
-
- if self.updated:
- self._unlock()
- try:
- # yum history undo 17
- # Do rollback only if update went well
- logging.debug("Yum rollback started")
- cmd = [
- basedefs.EXEC_YUM, "history", "-y", "undo", upgradeTid,
- ]
- output, rc = utils.execCmd(cmdList=cmd, failOnError=True,
msg=MSG_ERROR_YUM_HISTORY_UNDO)
- logging.debug("Yum rollback completed successfully")
- finally:
- self._lock()
- else:
- logging.debug("No rollback needed")
-
- def getLatestTid(self, updateOnly=False):
- logging.debug("Yum getLatestTid started")
- tid = None
-
- # Get the list
- cmd = [
- basedefs.EXEC_YUM, "history", "list", basedefs.ENGINE_RPM_NAME,
- ]
- output, rc = utils.execCmd(cmdList=cmd, failOnError=True,
msg=MSG_ERROR_YUM_HISTORY_LIST)
-
- # Parse last tid
- for line in output.splitlines():
- lsplit = line.split("|")
- if len(lsplit) > 3:
- if updateOnly:
- if 'Update' in lsplit[3].split() or "U" in
lsplit[3].split():
- tid = lsplit[0].strip()
- break
- else:
- if "Action" not in lsplit[3]: # Don't get header of output
- tid = lsplit[0].strip()
- break
- if tid is None:
- raise ValueError(MSG_ERROR_YUM_HISTORY_GETLAST)
-
- logging.debug("Found TID: %s" %(tid))
- logging.debug("Yum getLatestTid completed successfully")
- return tid
-
- def isCandidateForUpdate(self, rpm):
- candidate = False
- for package in self.upackages:
- if rpm in package:
- candidate = True
- return candidate
-
- def getUpdateCandidates(self):
- return self.upackages
class DB():
def __init__(self):
@@ -577,7 +501,7 @@
print ("[ " + utils.getColoredText(MSG_INFO_ERROR, basedefs.RED) + "
]").rjust(spaceLen+3)
raise
-def isUpdateRelatedToDb(yumo):
+def isUpdateRelatedToDb(packages):
"""
Verifies current update needs DB manipulation (backup/update/rollback)
"""
@@ -586,7 +510,7 @@
related = False
for rpm in RPM_BACKEND, RPM_DBSCRIPTS:
- if yumo.isCandidateForUpdate(rpm):
+ if rpm in [p['name'] for p in packages]:
related = True
logging.debug("isUpdateRelatedToDb value is %s"%(str(related)))
@@ -681,7 +605,18 @@
return False
def main(options):
- rhyum = MYum()
+ # BEGIN: PROCESS-INITIALIZATION
+ miniyumsink = _miniyumsink()
+ MiniYum.setup_log_hook(sink=miniyumsink)
+ extraLog = open(LOG_FILE, "a")
+ miniyum = MiniYum(sink=miniyumsink, extraLog=extraLog)
+ miniyum.selinux_role()
+ # END: PROCESS-INITIALIZATION
+
+ # we do not wish to be interrupted
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+ rhyum = MYum(miniyum)
db = DB()
DB_NAME_TEMP = "%s_%s" % (basedefs.DB_NAME, utils.getCurrentDateTime())
@@ -710,116 +645,106 @@
print MSG_ERROR_INCOMPATIBLE_UPGRADE
raise Exception(MSG_ERROR_INCOMPATIBLE_UPGRADE)
- # Check for upgrade, else exit
- print MSG_INFO_CHECK_UPDATE
- if not rhyum.updateAvailable():
- logging.debug(MSG_INFO_NO_UPGRADE_AVAIL)
- print MSG_INFO_NO_UPGRADE_AVAIL
- sys.exit(0)
- else:
- updates = rhyum.getUpdateCandidates()
- print MSG_INFO_UPGRADE_AVAIL % (len(updates))
- for package in updates:
- print " * %s" % package
+ rhyum.clean()
+
+ with rhyum.transaction():
+ # Check for upgrade, else exit
+ runFunc([rhyum.begin], MSG_INFO_CHECK_UPDATE)
+ if rhyum.emptyTransaction:
+ logging.debug(MSG_INFO_NO_UPGRADE_AVAIL)
+ print MSG_INFO_NO_UPGRADE_AVAIL
+ sys.exit(0)
+
+ packages = rhyum.getPackages()
+
+ print MSG_INFO_UPGRADE_AVAIL % (len(packages))
+ for p in packages:
+ print " * %s" % p['display_name']
if options.check_update:
sys.exit(100)
- # Check for setup package
- if rhyum.isCandidateForUpdate(RPM_SETUP) and not
options.force_current_setup_rpm:
- logging.debug(MSG_ERROR_NEW_SETUP_AVAIL)
- print MSG_ERROR_NEW_SETUP_AVAIL
- sys.exit(3)
+ name_packages = [p['name'] for p in packages]
+ if RPM_SETUP in name_packages and not options.force_current_setup_rpm:
+ logging.debug(MSG_ERROR_NEW_SETUP_AVAIL)
+ print MSG_ERROR_NEW_SETUP_AVAIL
+ sys.exit(3)
- # Make sure we will be able to rollback
- if not rhyum.rollbackAvailable() and options.yum_rollback:
- logging.debug(MSG_ERROR_NO_ROLLBACK_AVAIL)
- print MSG_ERROR_NO_ROLLBACK_AVAIL
- print MSG_ERROR_CHECK_LOG%(LOG_FILE)
- sys.exit(2)
+ # Make sure we will be able to rollback
+ if not rhyum.rollbackAvailable(packages) and options.yum_rollback:
+ logging.debug(MSG_ERROR_NO_ROLLBACK_AVAIL)
+ print MSG_ERROR_NO_ROLLBACK_AVAIL
+ print MSG_ERROR_CHECK_LOG%(LOG_FILE)
+ sys.exit(2)
- # No rollback in this case
- try:
- # We ask the user before stoping ovirt-engine or take command line
option
- if options.unattended_upgrade or checkEngine(engineService):
- # Stopping engine
- runFunc(stopEngineService, MSG_INFO_STOP_ENGINE)
- else:
- # This means that user chose not to stop ovirt-engine
- logging.debug("exiting gracefully")
- print MSG_INFO_STOP_INSTALL_EXIT
- sys.exit(0)
+ # No rollback in this case
+ try:
+ # We ask the user before stoping ovirt-engine or take command line
option
+ if options.unattended_upgrade or checkEngine(engineService):
+ # Stopping engine
+ runFunc(stopEngineService, MSG_INFO_STOP_ENGINE)
+ else:
+ # This means that user chose not to stop ovirt-engine
+ logging.debug("exiting gracefully")
+ print MSG_INFO_STOP_INSTALL_EXIT
+ sys.exit(0)
- # Backup DB
- if isUpdateRelatedToDb(rhyum):
- runFunc([db.backup], MSG_INFO_BACKUP_DB)
- runFunc([[db.rename, DB_NAME_TEMP]], MSG_INFO_RENAME_DB)
+ # Backup DB
+ if isUpdateRelatedToDb(packages):
+ runFunc([db.backup], MSG_INFO_BACKUP_DB)
+ runFunc([[db.rename, DB_NAME_TEMP]], MSG_INFO_RENAME_DB)
- except Exception as e:
- print e
- raise
+ except Exception as e:
+ print e
+ raise
- # In case of failure, do rollback
- try:
- # yum update
- runFunc(upgradeFunc, MSG_INFO_YUM_UPDATE)
+ # In case of failure, do rollback
+ try:
+ # yum update
+ runFunc(upgradeFunc, MSG_INFO_YUM_UPDATE)
- # If we're here, update/upgrade went fine, so
- rhyum.updated = True
+ # define db connections services
+ etlService = utils.Service("ovirt-engine-etl")
+ notificationService = utils.Service("ovirt-engine-notifierd")
- # define db connections services
- etlService = utils.Service("ovirt-engine-etl")
- notificationService = utils.Service("ovirt-engine-notifierd")
+ # check if update is relevant to db update
+ if isUpdateRelatedToDb(packages):
+ stopDbRelatedServices(etlService, notificationService)
- # check if update is relevant to db update
- if isUpdateRelatedToDb(rhyum):
- stopDbRelatedServices(etlService, notificationService)
+ # Update the db and restore its name back
+ runFunc([db.update], MSG_INFO_DB_UPDATE)
+ runFunc([[db.rename, basedefs.DB_NAME]], MSG_INFO_RESTORE_DB)
- # Update the db and restore its name back
- runFunc([db.update], MSG_INFO_DB_UPDATE)
- runFunc([[db.rename, basedefs.DB_NAME]], MSG_INFO_RESTORE_DB)
+ # Bring up any services we shut down before db upgrade
+ startDbRelatedServices(etlService, notificationService)
- # Bring up any services we shut down before db upgrade
- startDbRelatedServices(etlService, notificationService)
+ # post install conf
+ runFunc(postFunc, MSG_INFO_RUN_POST)
- # post install conf
- runFunc(postFunc, MSG_INFO_RUN_POST)
+ except:
+ logging.error(traceback.format_exc())
+ logging.error("Rolling back update")
- except:
- logging.error(traceback.format_exc())
- logging.error("Rolling back update")
+ print MSG_ERROR_UPGRADE
+ print MSG_INFO_REASON%(sys.exc_info()[1])
- print MSG_ERROR_UPGRADE
- print MSG_INFO_REASON%(sys.exc_info()[1])
+ # allow db restore
+ if isUpdateRelatedToDb(packages):
+ try:
+ runFunc([db.restore], MSG_INFO_DB_RESTORE)
+ except:
+ # This Exception have already been logged, so just pass
along
+ pass
- # allow db restore
- if isUpdateRelatedToDb(rhyum):
- try:
- runFunc([db.restore], MSG_INFO_DB_RESTORE)
- except:
- # This Exception have already been logged, so just pass along
- pass
+ raise
- # allow yum rollback even if db restore failed
- if options.yum_rollback:
- try:
- runFunc([rhyum.rollback], MSG_INFO_YUM_ROLLBACK)
- except:
- # This Exception have already been logged, so just pass along
- pass
- else:
- print MSG_INFO_NO_YUM_ROLLBACK
- logging.debug("Skipping yum rollback")
+ finally:
+ # start engine
+ runFunc([startEngine], MSG_INFO_START_ENGINE)
- raise
-
- finally:
- # start engine
- runFunc([startEngine], MSG_INFO_START_ENGINE)
-
- # Print log location on success
- addAdditionalMessages(etlService.isServiceAvailable())
- print "\n%s\n" % MSG_INFO_UPGRADE_OK
- printMessages()
+ # Print log location on success
+ addAdditionalMessages(etlService.isServiceAvailable())
+ print "\n%s\n" % MSG_INFO_UPGRADE_OK
+ printMessages()
if __name__ == '__main__':
try:
diff --git a/packaging/fedora/setup/miniyum.py
b/packaging/fedora/setup/miniyum.py
new file mode 100755
index 0000000..31cf35b
--- /dev/null
+++ b/packaging/fedora/setup/miniyum.py
@@ -0,0 +1,805 @@
+#!/usr/bin/python
+#
+# Copyright 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+import os
+import sys
+import logging
+import time
+import traceback
+
+import yum
+
+
+from yum.rpmtrans import RPMBaseCallback
+from yum.callbacks import PT_MESSAGES, PT_DOWNLOAD_PKGS
+from yum.Errors import YumBaseError
+from yum.callbacks import DownloadBaseCallback
+
+
+class MiniYum(object):
+
+ class _loghandler(logging.Handler):
+ """Required for extracting yum log output."""
+
+ def __init__(self, sink):
+ logging.Handler.__init__(self)
+ self._sink = sink
+
+ def emit(self, record):
+ if self._sink is not None:
+ self._sink.verbose(record.getMessage())
+
+ class _yumlogger(logging.Logger):
+ """Required for hacking yum log."""
+
+ _sink = None
+
+ def __init__(self, name, level=logging.NOTSET):
+ logging.Logger.__init__(self, name, level)
+
+ def addHandler(self, hdlr):
+ if self.name.startswith('yum') or self.name.startswith('rhsm'):
+ self.handlers = []
+ logging.Logger.addHandler(
+ self,
+ MiniYum._loghandler(
+ MiniYum._yumlogger._sink
+ )
+ )
+ else:
+ logging.Logger.addHandler(self, hdlr)
+
+ class _yumlistener(object):
+ """Required for extracting yum events."""
+
+ def __init__(self, sink):
+ self._sink = sink
+
+ def event(self, event, *args, **kwargs):
+ msg = "Status: "
+ if event in PT_MESSAGES:
+ msg += "%s" % PT_MESSAGES[event]
+ else:
+ msg += "Unknown(%d)" % event
+
+ if event == PT_DOWNLOAD_PKGS:
+ msg += " packages:"
+ for po in args[0]:
+ msg += " " + MiniYum._get_package_name(po)
+
+ self._sink.verbose(msg)
+ else:
+ self._sink.info(msg)
+
+ class _rpmcallback(RPMBaseCallback):
+ """Required for extracting rpm events."""
+
+ def __init__(self, sink):
+ RPMBaseCallback.__init__(self)
+ self._sink = sink
+ self._lastaction = None
+ self._lastpackage = None
+
+ def event(
+ self, package, action, te_current, te_total,
+ ts_current, ts_total
+ ):
+ if self._lastaction != action or package != self._lastpackage:
+ self._lastaction = action
+ self._lastpackage = package
+ self._sink.info("%s: %u/%u: %s" % (
+ self.action[action], ts_current,
+ ts_total, package))
+
+ def scriptout(self, package, msgs):
+ if msgs:
+ self._sink.verbose("Script sink: " + msgs)
+
+ self._sink.verbose("Done: %s" % (package))
+
+ def errorlog(self, msg):
+ self._sink.error(msg)
+
+ def filelog(self, package, action):
+ RPMBaseCallback.filelog(self, package, action)
+
+ def verify_txmbr(self, base, txmbr, count):
+ self._sink.info(
+ "Verify: %u/%u: %s" % (
+ count,
+ len(base.tsInfo),
+ txmbr
+ )
+ )
+
+ class _downloadcallback(DownloadBaseCallback):
+ """Required for extracting progress messages."""
+
+ def __init__(self, sink):
+ DownloadBaseCallback.__init__(self)
+ self._sink = sink
+
+ def updateProgress(self, name, frac, fread, ftime):
+ msg = "Downloading: %s %s(%d%%)" % (
+ name,
+ fread,
+ int(float(frac) * 100)
+ )
+ self._sink.verbose(msg)
+ self._sink.keepAlive(msg)
+ DownloadBaseCallback.updateProgress(self, name, frac, fread, ftime)
+
+ class _voidsink(object):
+ def verbose(self, msg):
+ """verbose log.
+
+ Keyword arguments:
+ msg -- message to print
+
+ """
+ pass
+
+ def info(self, msg):
+ """info log.
+
+ Keyword arguments:
+ msg -- message to print
+
+ """
+ pass
+
+ def error(self, msg):
+ """error log.
+
+ Keyword arguments:
+ msg -- message to print
+
+ """
+ pass
+
+ def keepAlive(self, msg):
+ """keepAlive log.
+
+ Keyword arguments:
+ msg -- message to print
+
+ """
+ pass
+
+ def askForGPGKeyImport(self, userid, hexkeyid):
+ """Ask for GPG Key import.
+
+ Keyword arguments:
+ userid -- user
+ hexkeyid - key id
+
+ return True to accept.
+
+ """
+ return False
+
+ class _disable_stdhandles(object):
+ """Disable stdin/stdout/stderr
+
+ Even after handling all logs, there are
+ some tools that writes to stderr/stdout!!!
+ these are not important messages, so we just
+ ignore for now
+
+ """
+
+ def __init__(self, rfile=None):
+ self._refcount = 0
+ self._rstdin = os.open(os.devnull, os.O_RDONLY)
+ if rfile is None:
+ self._rstdout = os.open(os.devnull, os.O_WRONLY)
+ self._should_close_rstdout = True
+ else:
+ self._rstdout = rfile.fileno()
+ self._should_close_rstdout = False
+
+ def __del__(self):
+ os.close(self._rstdin)
+ if self._should_close_rstdout:
+ os.close(self._rstdout)
+
+ def __enter__(self):
+ self._refcount += 1
+ if self._refcount == 1:
+ self._oldfds = []
+
+ for i in range(3):
+ self._oldfds.append(os.dup(i))
+ if i == 0:
+ os.dup2(self._rstdin, i)
+ else:
+ os.dup2(self._rstdout, i)
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self._refcount -= 1
+ if self._refcount == 0:
+ for i in range(len(self._oldfds)):
+ os.dup2(self._oldfds[i], i)
+ os.close(self._oldfds[i])
+
+ class _YumBase(yum.YumBase):
+ """Require to overrde base functions."""
+
+ def __init__(self, sink):
+ yum.YumBase.__init__(self)
+
+ self._sink = sink
+ self._lastpkg = None
+
+ def _askForGPGKeyImport(self, po, userid, hexkeyid):
+ return self._sink.askForGPGKeyImport(userid, hexkeyid)
+
+ def verifyPkg(self, fo, po, raiseError):
+ if self._lastpkg != po:
+ self._lastpkg = po
+ self._sink.info(
+ "Download/Verify: %s" % MiniYum._get_package_name(po)
+ )
+ yum.YumBase.verifyPkg(self, fo, po, raiseError)
+
+ class _MiniYumTransaction(object):
+ def __init__(self, managed):
+ self._managed = managed
+
+ def __enter__(self):
+ self._managed._beginTransaction()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self._managed._endTransaction(exc_type is not None)
+
+ @staticmethod
+ def _get_package_name(po):
+ return "%s-%s%s-%s.%s" % (
+ po.name,
+ "%s:" % po.epoch if po.epoch == 0 else "",
+ po.version,
+ po.release,
+ po.arch
+ )
+
+ @staticmethod
+ def _get_package_info(po):
+ info = {}
+ info['display_name'] = MiniYum._get_package_name(po)
+ for f in (
+ 'name',
+ 'version',
+ 'release',
+ 'epoch',
+ 'arch'
+ ):
+ info[f] = getattr(po, f)
+ return info
+
+ @staticmethod
+ def setup_log_hook(sink=None):
+ """logging hack for yum.
+
+ Keyword arguments:
+ sink -- callback sink (default None)
+
+ Yum packages uses logging package
+ intensively, but we have no clue which
+ log is used.
+ What we have done in constructor should have
+ redirect all output to us.
+ However, its lazy initialization of the
+ log handlers, diverse some output to its own
+ handlers.
+ So we set our own class to remove the hostile
+ handlers for the yum loggers.
+
+ Maybe someday this code can be removed.
+
+ Tested: rhel-6.3
+
+ """
+ MiniYum._yumlogger._sink = sink
+ logging.setLoggerClass(MiniYum._yumlogger)
+
+ def _beginTransaction(self):
+ """Lock (begin of transaction)
+
+ Need to disbale output as:
+ Freeing read locks for locker 0x84: 1316/139849637029632
+ Freeing read locks for locker 0x86: 1316/139849637029632
+
+ """
+ with self._disableOutput:
+ self._transactionBase = self._yb.history.last()
+ self._yb.doLock()
+
+ def _endTransaction(self, rollback=False):
+ """Unlock (end of transaction)."""
+ with self._disableOutput:
+ try:
+ if rollback:
+ self._sink.verbose("Performing rollback")
+ transactionCurrent = self._yb.history.last(
+ complete_transactions_only=False
+ )
+ if (
+ transactionCurrent is not None and
+ self._transactionBase.tid < transactionCurrent.tid
+ ):
+ if (
+ transactionCurrent.altered_lt_rpmdb or
+ transactionCurrent.altered_gt_rpmdb
+ ):
+ # safe guard?
+ pass
+ else:
+ try:
+ self._yb.repos.populateSack(
+ mdtype='all',
+ cacheonly=1
+ )
+ del self._yb.tsInfo
+ if self._yb.history_undo(transactionCurrent):
+ if self.buildTransaction():
+ self.processTransaction()
+ finally:
+ self._yb.repos.populateSack(
+ mdtype='all',
+ cacheonly=0
+ )
+
+ except Exception:
+ self._sink.error('Transaction end failed: %s' % (
+ traceback.format_exc()
+ ))
+ finally:
+ self._transactionBase = None
+
+ # forget current transaction
+ del self._yb.tsInfo
+ self._yb.doUnlock()
+
+ def _queue(self, action, call, packages, ignoreErrors=False):
+ ret = True
+
+ with self._disableOutput:
+ for package in packages:
+ try:
+ self._sink.verbose(
+ "queue package %s for %s" % (package, action)
+ )
+ call(name=package)
+ self._sink.verbose("package %s queued" % package)
+ except YumBaseError as e:
+ ret = False
+ msg = ""
+ if type(e.value) is list:
+ for s in e.value:
+ msg += str(s) + "\n"
+ else:
+ msg = str(e.value)
+
+ self._sink.error(
+ "cannot queue package %s: %s" % (package, msg)
+ )
+
+ if not ignoreErrors:
+ raise
+
+ except Exception as e:
+ self._sink.error(
+ "cannot queue package %s: %s" % (package, e)
+ )
+ raise
+
+ return ret
+
+ def __init__(self, sink=None, extraLog=None):
+ """Constructor.
+
+ Keyword arguments:
+ sink -- sink to use for interaction.
+ extraLog -- a File object for stdout/stderr redirection.
+
+ Notes:
+ extraLog is required in order to collect noise output
+ of yum going into stdout/stderr directly.
+
+ """
+ try:
+ if sink is None:
+ self._sink = MiniYum._voidsink()
+ else:
+ self._sink = sink
+
+ self._disableOutput = MiniYum._disable_stdhandles(rfile=extraLog)
+
+ self._yb = MiniYum._YumBase(self._sink)
+
+ for l in ('yum', 'rhsm'):
+ log = logging.getLogger(l)
+ log.propagate = False
+ log.handlers = []
+ log.addHandler(
+ MiniYum._loghandler(self._sink)
+ )
+
+ self._yb.repos.setProgressBar(
+ MiniYum._downloadcallback(self._sink)
+ )
+
+ except YumBaseError as e:
+ self._sink.error(str(e.value))
+ except Exception as e:
+ self._sink.error(str(e))
+
+ def selinux_role(self):
+ """Setup proper selinux role.
+
+ this must be called at beginning of process
+ to adjust proper roles for selinux.
+ it will re-execute the process with same arguments.
+
+ This has similar effect of:
+ # chcon -t rpm_exec_t miniyum.py
+
+ We must do this dynamic as this class is to be
+ used at bootstrap stage, so we cannot put any
+ persistent selinux policy changes.
+
+ """
+
+ try:
+ selinux = __import__('selinux', globals(), locals(), [], -1)
+ except ImportError:
+ with self.transaction():
+ self.install(['libselinux-python'])
+ if self.buildTransaction():
+ self.processTransaction()
+
+ if not 'selinux' in globals():
+ selinux = __import__('selinux', globals(), locals(), [], -1)
+ if selinux.is_selinux_enabled() and "MINIYUM_2ND" not in os.environ:
+ env = os.environ.copy()
+ env["MINIYUM_2ND"] = "1"
+ rc, ctx = selinux.getcon()
+ if rc != 0:
+ raise Exception("Cannot get selinux context")
+ ctx1 = selinux.context_new(ctx)
+ if not ctx1:
+ raise Exception("Cannot create selinux context")
+ if selinux.context_type_set(ctx1, 'rpm_t') != 0:
+ raise Exception("Cannot set type within selinux context")
+ if selinux.context_role_set(ctx1, 'system_r') != 0:
+ raise Exception("Cannot set role within selinux context")
+ if selinux.context_user_set(ctx1, 'unconfined_u') != 0:
+ raise Exception("Cannot set user within selinux context")
+ if selinux.setexeccon(selinux.context_str(ctx1)) != 0:
+ raise Exception("Cannot set selinux exec context")
+ os.execve(sys.executable, [sys.executable] + sys.argv, env)
+ os._exit(1)
+
+ def transaction(self):
+ """Manage transaction.
+
+ Usage:
+ with miniyum.transaction():
+ do anything
+ """
+ return MiniYum._MiniYumTransaction(self)
+
+ def clean(self):
+ """Clean yum data."""
+ with self._disableOutput:
+ self._yb.cleanMetadata()
+ self._yb.cleanPackages()
+ self._yb.cleanSqlite()
+
+ def install(self, packages, **kwargs):
+ """Install packages.
+
+ Keyword arguments:
+ packages -- packages to install.
+ ignoreErrors - to ignore errors, will return False
+
+ """
+ return self._queue("install", self._yb.install, packages, **kwargs)
+
+ def update(self, packages, **kwargs):
+ """Update packages.
+
+ Keyword arguments:
+ packages -- packages to install.
+ ignoreErrors - to ignore errors, will return False
+
+ """
+ return self._queue("update", self._yb.update, packages, **kwargs)
+
+ def installUpdate(self, packages, **kwargs):
+ """Install or update packages.
+
+ Keyword arguments:
+ packages -- packages to install.
+ ignoreErrors - to ignore errors, will return False
+
+ """
+ return (
+ self.install(packages, **kwargs) or
+ self.update(packages, **kwargs)
+ )
+
+ def remove(self, packages, **kwargs):
+ """Remove packages.
+
+ Keyword arguments:
+ packages -- packages to install.
+ ignoreErrors - to ignore errors, will return False
+
+ """
+ return self._queue("remove", self._yb.remove, packages, **kwargs)
+
+ def buildTransaction(self):
+ """Build transaction.
+
+ returns False if empty.
+
+ """
+ try:
+ with self._disableOutput:
+ ret = False
+ self._sink.verbose("Building transaction")
+ rc, msg = self._yb.buildTransaction()
+ if rc == 0:
+ self._sink.verbose("Empty transaction")
+ elif rc == 2:
+ ret = True
+ self._sink.verbose("Transaction built")
+ else:
+ raise YumBaseError(msg)
+
+ return ret
+
+ except YumBaseError as e:
+ msg = ""
+ if type(e.value) is list:
+ for s in e.value:
+ msg += str(s) + "\n"
+ else:
+ msg = str(e.value)
+ self._sink.error(msg)
+ raise
+
+ except Exception as e:
+ self._sink.error(str(e))
+ raise
+
+ def queryTransaction(self):
+ try:
+ with self._disableOutput:
+ ret = []
+ self._yb.tsInfo.makelists()
+ for op, l in (
+ ('install', self._yb.tsInfo.installed),
+ ('update', self._yb.tsInfo.updated),
+ ('install', self._yb.tsInfo.depinstalled),
+ ('update', self._yb.tsInfo.depupdated),
+ ):
+ for p in l:
+ info = MiniYum._get_package_info(p)
+ info['operation'] = op
+ ret.append(info)
+ return ret
+
+ except YumBaseError as e:
+ msg = ""
+ if type(e.value) is list:
+ for s in e.value:
+ msg += str(s) + "\n"
+ else:
+ msg = str(e.value)
+ self._sink.error(msg)
+ raise
+
+ except Exception as e:
+ self._sink.error(str(e))
+ raise
+
+ def queryPackages(self, pkgnarrow='all', patterns=None):
+ try:
+ with self._disableOutput:
+ ret = []
+ holder = self._yb.doPackageLists(
+ pkgnarrow=pkgnarrow,
+ patterns=patterns
+ )
+ for op, l in (
+ ('available', holder.available),
+ ('installed', holder.installed),
+ ('updates', holder.updates),
+ ('extras', holder.extras),
+ ('obsoletes', holder.obsoletes),
+ ('recent', holder.recent)
+ ):
+ for entry in l:
+ if isinstance(entry, tuple):
+ info = MiniYum._get_package_info(entry[0])
+ info['operation'] = op
+ ret.append(info)
+ info = MiniYum._get_package_info(entry[1])
+ info['operation'] = 'installed'
+ ret.append(info)
+ else:
+ info = MiniYum._get_package_info(entry)
+ info['operation'] = op
+ ret.append(info)
+
+ return ret
+
+ except YumBaseError as e:
+ msg = ""
+ if type(e.value) is list:
+ for s in e.value:
+ msg += str(s) + "\n"
+ else:
+ msg = str(e.value)
+ self._sink.error(msg)
+ raise
+
+ except Exception as e:
+ self._sink.error(str(e))
+ raise
+
+ def queryLocalCachePackages(self, patterns=None):
+ try:
+ with self._disableOutput:
+ return [
+ MiniYum._get_package_info(p)
+ for p in self._yb.pkgSack.returnPackages(patterns=patterns)
+ ]
+ except YumBaseError as e:
+ msg = ""
+ if type(e.value) is list:
+ for s in e.value:
+ msg += str(s) + "\n"
+ else:
+ msg = str(e.value)
+ self._sink.error(msg)
+ raise
+
+ except Exception as e:
+ self._sink.error(str(e))
+ raise
+
+ def processTransaction(self):
+ """Process built transaction."""
+
+ try:
+ with self._disableOutput:
+ self._sink.verbose("Processing transaction")
+ self._yb.processTransaction(
+ callback=MiniYum._yumlistener(sink=self._sink),
+ rpmTestDisplay=MiniYum._rpmcallback(sink=self._sink),
+ rpmDisplay=MiniYum._rpmcallback(sink=self._sink)
+ )
+ self._sink.verbose("Transaction processed")
+
+ except YumBaseError as e:
+ msg = ""
+ if type(e.value) is list:
+ for s in e.value:
+ msg += str(s) + "\n"
+ else:
+ msg = str(e.value)
+ self._sink.error(msg)
+ raise
+
+ except Exception as e:
+ self._sink.error(str(e))
+ raise
+
+
+class example(object):
+ class myminiyumsink(object):
+
+ KEEPALIVE_INTERVAL = 60
+
+ def __init__(self):
+ """dup the stdout as during yum operation so we redirect it."""
+ self._stream = os.dup(sys.stdout.fileno())
+ self._touch()
+
+ def __del__(self):
+ os.close(self._stream)
+
+ def _touch(self):
+ self._last = time.time()
+
+ def verbose(self, msg):
+ os.write(self._stream, ("VERB: -->%s<--\n" % msg).encode('utf-8'))
+
+ def info(self, msg):
+ self._touch()
+ os.write(self._stream, ("OK: -->%s<--\n" % msg).encode('utf-8'))
+
+ def error(self, msg):
+ self._touch()
+ os.write(self._stream, ("FAIL: -->%s<--\n" % msg).encode('utf-8'))
+
+ def keepAlive(self, msg):
+ if time.time() - self._last >= \
+ example.myminiyumsink.KEEPALIVE_INTERVAL:
+ self.info(msg)
+
+ def askForGPGKeyImport(self, userid, hexkeyid):
+ os.write(
+ self._stream,
+ (
+ "APPROVE-GPG: -->%s-%s<--\n" % (userid, hexkeyid)
+ ).encode('utf-8')
+ )
+ return True
+
+ @staticmethod
+ def main():
+ # BEGIN: PROCESS-INITIALIZATION
+ miniyumsink = example.myminiyumsink()
+ MiniYum.setup_log_hook(sink=miniyumsink)
+ extraLog = open("/tmp/miniyum.log", "a")
+ miniyum = MiniYum(sink=miniyumsink, extraLog=extraLog)
+ miniyum.selinux_role()
+ # END: PROCESS-INITIALIZATION
+
+ with miniyum.transaction():
+ miniyum.clean()
+
+ miniyumsink.info("Search Summary:")
+ for p in miniyum.queryPackages(patterns=['vdsm']):
+ miniyumsink.info(" %s - %s" % (
+ p['operation'],
+ p['display_name']
+ ))
+
+ with miniyum.transaction():
+ miniyum.remove(('cman',), ignoreErrors=True)
+ miniyum.install(('qemu-kvm-tools',))
+ miniyum.installUpdate(('vdsm', 'vdsm-cli'))
+ if miniyum.buildTransaction():
+ miniyumsink.info("Transaction Summary:")
+ for p in miniyum.queryTransaction():
+ miniyumsink.info(" %s - %s" % (
+ p['operation'],
+ p['display_name']
+ ))
+ miniyum.processTransaction()
+
+ try:
+ with miniyum.transaction():
+ miniyum.install(('pcsc-lite',))
+ if miniyum.buildTransaction():
+ miniyum.processTransaction()
+ raise Exception("Fail please")
+ except Exception as e:
+ if str(e) != "Fail please":
+ raise
+
+if __name__ == "__main__":
+ example.main()
diff --git a/packaging/fedora/spec/ovirt-engine.spec.in
b/packaging/fedora/spec/ovirt-engine.spec.in
index 22f4baa..6f8bb77 100644
--- a/packaging/fedora/spec/ovirt-engine.spec.in
+++ b/packaging/fedora/spec/ovirt-engine.spec.in
@@ -746,6 +746,7 @@
%{engine_data}/scripts/setup_sequences.py*
%{engine_data}/scripts/setup_controller.py*
%{engine_data}/scripts/common_utils.py*
+%{engine_data}/scripts/miniyum.py*
%{engine_data}/scripts/output_messages.py*
%{engine_data}/scripts/nfsutils.py*
%{engine_data}/scripts/engine-setup.py*
--
To view, visit http://gerrit.ovirt.org/8221
To unsubscribe, visit http://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I01c0c10c3bca42770bf05222c8b2b82b01fee0a6
Gerrit-PatchSet: 1
Gerrit-Project: ovirt-engine
Gerrit-Branch: master
Gerrit-Owner: Alon Bar-Lev <alo...@redhat.com>
_______________________________________________
Engine-patches mailing list
Engine-patches@ovirt.org
http://lists.ovirt.org/mailman/listinfo/engine-patches
