Hello guys, I took last few days to rewrite our eshowkw script from bash to python and enhance its functionality.
Not all of you are aware of this usefull script from gentoolkit-dev package, so let me introduce it: It is script that shows keywords for package and all its versions in nice table for easy review. If you take look on the output_gcc.log attachment it shows what it prints out for our gcc package. It checks out all versions, identify installed and masked packages, separate it by slots and detect redundant versions that could be in theory removed. So since I would like to have this script replace current eshowkw one I want to ask you for kind help on that script and review it and suggest improvements. If the script lack some feature you really want to use also let me know, maybe it wont be too hard to implement. Cheers Tomas
# # # #
Keywords for sys-devel/gcc:
| s
|
| p
|
| s a
|
| p r
|
| a p a c x x x
x x x x |
| a m i p s r 6 6 6 6 x
8 8 x 8 8 |
| m d a h i a m m p c p c 4 4 x 4 4 8
6 6 x x 8 6 6 x |
| d 6 r p a 6 6 i p - a - - - 6 - - 6
x - - 8 8 6 - - 8 |
| 6 4 m p 6 4 8 p p c o r s s f 4 o s -
8 f i 6 6 - o s 6 |
| 4 - - a 4 - k s p - p c o o r - p o c
6 r n - - n p o - | u
| a a p s - l l - - l - - c m e - l l e m e l y
- e t l m e e l w | n
| l m h i m m p s p f i i h h i m i - a n f a a e a n a g
f e e i a t n a i | u s
| p d a p a 6 i p c 3 a x b n n p p n i r a c b b r r b c b r w
b b r n c b b r n | s l
| h 6 r p 6 8 p p 6 9 s r 8 s u u u u u n i i o s s i i s o s i i
s s i u o s s i n | e o
| a 4 m a 4 k s c 4 0 h c 6 d x x x x x t x x s d d s s d s d s n
d d x x s d d s t | d t
--------------+-----------------------------------------------------------------------------------+-------
2.95.3-r9 | + + + +
| 2.95
2.95.3-r10 | ~ ~ ~ ~
|
--------------+-----------------------------------------------------------------------------------+-------
3.1.1-r2 | - + +
| 3.1
--------------+-----------------------------------------------------------------------------------+-------
3.2.2 |
| # 3.2
3.2.3-r4 | + - + + + + + +
|
--------------+-----------------------------------------------------------------------------------+-------
3.3.6-r1 | ~ ~
| 3.3
--------------+-----------------------------------------------------------------------------------+-------
3.4.6-r2 | + + + * ~ * ~ + + ~ + + + * * * * * * * * * * * * * * * * * * *
~ * * * * * * * * | 3.4
--------------+-----------------------------------------------------------------------------------+-------
4.0.4 | * * * * ~ * * * * * * * * * * * * * * * * * * * * * * * * * * *
* * * * * * * * * | 4.0
--------------+-----------------------------------------------------------------------------------+-------
4.1.2 | + + + + + * + + + + + + + * * * * * * * * * * * ~ * * * * * * *
~ * * * * * * * * | 4.1
--------------+-----------------------------------------------------------------------------------+-------
4.2.4-r1 | ~ ~ ~ + ~ ~ ~ ~ ~ ~
~ | 4.2
--------------+-----------------------------------------------------------------------------------+-------
4.3.3-r2 | ~ ~ ~ - ~ ~ ~ ~ ~ ~ ~ ~
~ | # 4.3
4.3.4 | + + + - + ~ + + + + + +
~ |
4.3.5 | ~ ~ ~ - ~ ~ ~ ~ ~ ~ ~ ~
~ |
--------------+-----------------------------------------------------------------------------------+-------
4.4.2 | ~ ~ ~ - ~ ~ ~ ~ - ~
~ | # 4.4
4.4.3-r2 | + + + - + ~ + + ~ + + +
~ |
4.4.3-r3 | ~ ~ + - + ~ ~ ~ ~ + ~ ~
~ |
[I]4.4.4-r1 | ~ ~ ~ + ~ ~ ~ ~ ~ ~ ~ ~
~ | #
4.4.4-r2 | ~ + ~ + ~ ~ ~ ~ ~ ~ ~ +
~ |
4.4.5 | ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
~ |
--------------+-----------------------------------------------------------------------------------+-------
[M]4.5.1 | ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
~ | 4.5
#!/usr/bin/python
# vim:fileencoding=utf-8
# License: WTFPL-2
# Author: Tomáš Chvátal <scarab...@gentoo.org>
MY_PN='eshowkw'
MY_PV='0.3.0'
import argparse
import sys
import portage
from functools import reduce
from portage.output import colorize
class KeywordFile:
__IMPARCHS = [ 'arm', 'amd64', 'x86' ]
def __readKeywords(self):
"""Read all available keywords from portage."""
return [x for x in portage.settings.archlist()
if not x.startswith('~')]
def __sortKeywords(self, keywords):
normal = [k for k in keywords
if len(k.split('-')) == 1]
longer = [k for k in keywords
if len(k.split('-')) != 1]
normal.sort()
longer.sort()
normal.extend(longer)
return normal
def __prepareAdditionalFields(self):
"""Prepare list of aditional fileds displayed by eshowkw"""
return [ 'unused', 'slot' ]
def __prepareChar(self, arch, position, count, maxlen, order = 'bottom', imp_arch = '', bold = False):
"""Return specified character for the list position."""
# first figure out what character we want to work with
# based on order and position in the string
char = ' '
if order == 'bottom' and (position < maxlen) and (position >= maxlen-len(arch)):
char = list(arch)[position-maxlen+len(arch)]
elif order == 'top' and position < len(arch):
char = list(arch)[position]
# figure out what color we want to use
if arch in imp_arch:
color = 'blue'
if char == ' ':
color = 'bg_lightgray'
elif arch in self.__IMPARCHS:
color = 'darkyellow'
# colors
if arch in imp_arch or arch in self.__IMPARCHS:
char = colorize(color, char)
# bolding
if bold != False and count%2 == 0:
char = colorize('bold', char)
return char
def __rotateContent(self, string, length, order = 'bottom', imp_arch = '', bold = False):
"""
Rotate string over 90 degrees:
alpha -> a
l
p
h
a
"""
# join used to have list of lines rather than list of chars
return [' '.join([self.__prepareChar(line, i, count, length, order, imp_arch, bold)
for count, line in enumerate(string)])
for i in range(length)]
def getFormatedKeywordsList(self):
"""Return the formated output as one string for printout."""
return ''.join([x for x in self.__keywords_list])
def __getVersionSpacer(self, length):
"""Generate spaces to have version string nicely aligned."""
return ''.join([' ' for i in range(length)])
def __prepareFormatting(self, rotated_keywords, rotated_additional, length):
"""Format result for printout as list per line of content."""
return ['%s| %s | %s\n' % (self.__getVersionSpacer(length), x, y)
for x, y in zip(rotated_keywords, rotated_additional)]
def __init__(self, imp_arch, version_length, order, bold):
"""
Initialize Keywords header content.
Accessable:
class.getFormatedKeywordsList()
class.keywords
class.keywords_count
class.additional_count
"""
self.keywords = self.__readKeywords()
self.keywords = self.__sortKeywords(self.keywords)
additional = self.__prepareAdditionalFields()
keywords_maxlen = reduce(lambda x, y: (x, y)[x < y],
[len(str) for str in self.keywords])
rotated_keywords = self.__rotateContent(self.keywords, keywords_maxlen, order, imp_arch, bold)
rotated_additional = self.__rotateContent(additional, keywords_maxlen, order, imp_arch, bold)
self.__keywords_list = self.__prepareFormatting(rotated_keywords, rotated_additional, version_length)
self.keywords_count = len(self.keywords)
self.additional_count = len(additional)
class PortagePackage:
class RedundancyChecker:
def getRedundant(self):
"""Return required redundant list"""
return self.__redundant
def __listRedundant(self, keywords, ignore_slots, slots):
"""List all redundant packages."""
if ignore_slots:
return self.__getRedundantAll(keywords)
else:
return self.__getRedundantSlots(keywords, slots)
def __getRedundantSlots(self, keywords, slots):
"""Search for redundant packages walking per keywords for specified slot."""
result = [self.__compareSelected([k for k, s in zip(keywords, slots)
if s == slot])
for slot in self.__uniq(slots)]
# this is required because the list itself is not just one level depth
print ''.join(result)
return list(''.join(result))
def __uniq(self, seq):
"""Remove all duplicate elements from list."""
seen = {}
result = []
for item in seq:
if item in seen:
continue
seen[item] = 1
result.append(item)
return result
def __getRedundantAll(self, keywords):
"""Search for redundant packages using all versions ignoring its slotting."""
tmp = keywords[:]
# this is required because the list itself is not just one level depth
return list(''.join(keywords))
def __compareSelected(self, kws):
"""
Rotate over list of keywords and compare each element with others.
Selectively remove each already compared list from the remaining keywords.
"""
result = []
kws.reverse()
for i in range(len(kws)):
kw = kws.pop()
if self.__compareKeywordWithRest(kw, kws):
result.append('#')
else:
result.append(' ')
if len(result) == 0:
result.append(' ')
return ''.join(result)
def __compareKeywordWithRest(self, keyword, keywords):
"""Compare keywords with list of keywords."""
for key in keywords:
if self.__checkShadow(keyword, key):
return True
return False
def __checkShadow(self, old, new):
"""Check if package version is overshadowed by other package version."""
# remove -* and -arch since they are useless for us
newclean = ["%s" % x for x in new.split()
if x != '-*' and not x.startswith('-')]
oldclean = ["%s" % x for x in old.split()
if x != '-*' and not x.startswith('-')]
tmp = set(newclean)
tmp.update("~%s" % x for x in newclean
if not x.startswith("~"))
if not set(oldclean).difference(tmp):
return True
else:
return False
def __init__(self, keywords, ignore_slots, slots):
"""
Query all relevant data for redundancy package checking
Accessable:
class.getRedundant
"""
self.__redundant = self.__listRedundant(keywords, ignore_slots, slots)
class VersionChecker:
def getVersionsLength(self):
"""Get version string length."""
# + 1 to add trailing space
return reduce(lambda x, y: (x, y)[x < y],
[len(str) for str in self.__versions]) + 1
def getVersions(self):
"""Get list of availible versions."""
return [self.__versionColorize(x)
for x in self.__versions]
def __getVersionsNoColor(self, packages):
"""Obtain properly aligned version strings without colors."""
return self.__stripStartingSpaces([self.__separateVersion(x) for x in packages])
def __stripStartingSpaces(self, pvs):
"""Strip starting whitespace if there is no real reason for it."""
if not self.__require_prepend:
return [x[len('[M]'):] for x in pvs]
else:
return pvs
def __fillSpaces(self, string, length):
"""Fill spaces to match maximal version length."""
spacer = ''.join([' ' for i in range(length-len(string))])
return '%s%s' % (string, spacer)
def __separateVersion(self, cpv):
"""Get version string for specfied cpv"""
#pv = portage.versions.cpv_getversion(cpv)
pv = self.cpv_getversion(cpv)
return self.__prependVersionInfo(cpv, pv)
# remove me when portage 2.1.9 is stable
def cpv_getversion(self, mycpv):
"""Returns the v (including revision) from an cpv."""
cp = portage.versions.cpv_getkey(mycpv)
if cp is None:
return None
return mycpv[len(cp+"-"):]
def __prependVersionInfo(self, cpv, pv):
"""Prefix version with string based on whether version is installed or masked."""
mask = self.__getMaskStatus(cpv)
install = self.__getInstallStatus(cpv)
if mask and install:
pv = '[M][I]%s' % pv
self.__require_longprepend = True
elif mask:
pv = '[M]%s' % pv
self.__require_prepend = True
elif install:
pv = '[I]%s' % pv
self.__require_prepend = True
else:
if self.__require_longprepend:
pv = ' %s' % pv
else:
pv = ' %s' % pv
return pv
def __getMaskStatus(self, cpv):
"""
Figure out if package is pmasked.
This also uses user settings in /etc/ so local changes are important.
"""
pmask = False
try:
if ''.join(portage.getmaskingstatus(cpv)) == 'package.mask':
pmask = True
except:
# occurs when package is not known by portdb
# so we consider it unmasked
pass
return pmask
def __getInstallStatus(self, cpv):
"""Check if package version we test is installed."""
vardb = portage.db[portage.settings['ROOT']]['vartree'].dbapi
return vardb.cpv_exists(cpv)
def __versionColorize(self, pv):
"""
Format version string. Append trailing whitespace and
prepend information if it is masked or installed version.
"""
pv = self.__fillSpaces(pv, self.getVersionsLength())
# allign the versions nicely (before colors mess lining up)
# set fontcolor on the package version if it is masked od installed
# we figure it out of the prepended string on the start of the pv
if pv.startswith('[M][I]'):
pv = colorize('darkyellow', pv)
elif pv.startswith('[M]'):
pv = colorize('darkred', pv)
elif pv.startswith('[I]'):
pv = colorize('bold', pv)
return pv
def __init__(self, packages):
"""
Query all relevant data for version data formatting
Accessable:
class.getVersions
class.getVersionsLength
"""
self.__require_longprepend = False
self.__require_prepend = False
self.__versions = self.__getVersionsNoColor(packages)
def __checkExist(self, portdb, package):
"""Check if specified package even exists."""
try:
matches = portdb.xmatch('match-all', package)
except portage.exception.AmbiguousPackageName as Arg:
msg_err = 'Ambiguous package name "%s".\n' % package
found = 'Possibilities: %s' % Arg
raise SystemExit('%s%s', (msg_err, found))
except portage.exception.InvalidAtom:
msg_err = 'No such package "%s"' % package
raise SystemExit(msg_err)
if len(matches) <= 0:
msg_err = 'No such package "%s"' % package
raise SystemExit(msg_err)
return matches
def __getMetadata(self, portdb, packages):
"""Obtain all KEYWORDS and SLOT from metadata"""
try:
metadata = [portdb.aux_get(i, ['KEYWORDS', 'SLOT'])
for i in packages]
except KeyError:
# portage prints out the error for us
raise SystemExit('')
return list(zip(*metadata))
def __init__(self, package, dbapi, ignore_slots = False):
"""
Query all relevant data from portage databases.
Accessable:
class.version_length
class.versions
class.slots
class.keywords
class.redundant
class.cpv
"""
packages = self.__checkExist(dbapi, package)
versions = self.VersionChecker(packages)
self.keywords, self.slots = self.__getMetadata(dbapi, packages)
redundant = self.RedundancyChecker(self.keywords, ignore_slots, self.slots)
self.version_length = versions.getVersionsLength()
self.versions = versions.getVersions()
self.redundant = redundant.getRedundant()
self.cpv = portage.cpv_getkey(packages[0])
class KeywordsFormat:
def __prepareKeywordChar(self, arch, kwlist, column, imp_arch = [], bold = False):
"""
Convert specified keywords for package into their visual replacements.
# possibilities:
# ~arch -> orange ~
# -arch -> red -
# arch -> darkgreen +
# -* -> red *
"""
keys = [ '~%s' % arch, '-%s' % arch, '%s' % arch, '-*' ]
values = [
colorize('darkyellow', '~'),
colorize('darkred', '-'),
colorize('darkgreen', '+'),
colorize('darkred', '*')
]
# convert kwlist from string to list for easier conditions
kwlist = kwlist.split(' ')
# check what keyword we have
c = ' '
for k, v in zip(keys, values):
if k in kwlist:
c = v
break
# append bgcolor if we highlight the arch
if arch in imp_arch:
c = colorize('bg_lightgray', c)
# append bold if enabled
if bold != False and column%2 == 0:
c = colorize('bold', c)
return c
def __convertKeywords(self, keywords_list, keywords, imp_arch, bold):
"""Convert each package keywords into its visual replacement."""
return [' '.join([self.__prepareKeywordChar(x, k, i, imp_arch, bold)
for i, x in enumerate(keywords_list)])
for k in keywords]
def getKeywords(self):
"""Convert keywords list into one nice printable string."""
return ''.join(self.__formated_keywords)
def __formatKeywords(self, versions, keywords, redundant, slots):
"""
Convert all relevant data into per line list.
Puts together versions keywords and additional data.
"""
# first we want to draw separator so we must use weird slot
former_slot = 'none-could-set-slot-like-this'
lister = []
for a, b, c, d in zip(versions, keywords, redundant, slots):
if d != former_slot:
former_slot = d
lister.append(self.__horizontal_separator)
else:
# do not add slot more than once
d = ' '
c = colorize('purple', c)
d = colorize('bold', d)
lister.append('%s| %s | %s %s\n' % (a, b, c, d))
return lister
def __getHorizontalSeparator(self, slots_length, pv_length, keywords_length, additional_length):
"""Create nice horizontal separator."""
ver_sep = ''.join(['-' for i in range(pv_length)])
# +1 required for additional trailing space
arch_sep = ''.join(['-' for i in range(keywords_length*2+1)])
# -1 because it is already covered by additional_len
add_sep = ''.join(['-' for i in range(additional_length*2+slots_length-1)])
return '%s+%s+%s\n' % (ver_sep, arch_sep, add_sep)
def __init__(self, pkg_versions, pkg_keywords, pkg_slots, pkg_redundant, keywords_list, keywords_count, additional_count, pv_length, imp_arch, bold):
"""
Convert all portage queried data into something visually acceptable
Accessable:
class.getKeywords()
"""
slots_length = reduce(lambda x, y: (x, y)[x < y],
[len(str) for str in pkg_slots])
keywords = self.__convertKeywords(keywords_list, pkg_keywords, imp_arch, bold)
self.__horizontal_separator = self.__getHorizontalSeparator(slots_length, pv_length, keywords_count, additional_count)
self.__formated_keywords = self.__formatKeywords(pkg_versions, keywords, pkg_redundant, pkg_slots)
def portage_parse(package, ignore_slots, use_overlays):
"""Create portdb instance pointing to portage porttree, optionaly also to overlays."""
dbapi = portage.db[portage.settings['ROOT']]['porttree'].dbapi
if not use_overlays:
dbapi.porttrees = [dbapi.porttree_root] # exclude overlays
portpkg = PortagePackage(package, dbapi, ignore_slots)
return portpkg.cpv, portpkg.versions, portpkg.slots, portpkg.redundant, portpkg.keywords, portpkg.version_length
def out_of_tree_parse(ignore_slots):
"""Check the pwd for ebuilds and prepare special instance of portdb pointing to this path."""
import fnmatch
import os
currdir = os.getcwd()
# check if there are actualy some ebuilds
ebuilds = ['%s' % file for file in os.listdir(currdir)
if fnmatch.fnmatch(file, '*.ebuild')]
if len(ebuilds) <= 0:
msg_err = 'No ebuilds at "%s"' % currdir
raise SystemExit(msg_err)
package=os.path.basename(currdir)
PORTDIR = os.path.abspath('../../')
mysettings = portage.config(env={'PORTDIR': PORTDIR})
dbapi = portage.portdbapi(mysettings['PORTDIR'], mysettings=mysettings)
dbapi.porttrees = [dbapi.porttree_root] # use only our dir
portpkg = PortagePackage(package, dbapi, ignore_slots)
return portpkg.cpv, portpkg.versions, portpkg.slots, portpkg.redundant, portpkg.keywords, portpkg.version_length
def display(cp, pv, slot, red, kw, pvl, imp_arch, order, bold, use_overlays = False):
"""Put all the data together and print them out."""
output = []
if use_overlays:
output.append(colorize('darkyellow', 'Including overlays'))
output.append('...\n')
output.append('Keywords for %s:\n' % colorize('blue', cp))
# header
keywords_list = KeywordFile(imp_arch, pvl, order, bold)
output.append(keywords_list.getFormatedKeywordsList())
# data
keywords_package = KeywordsFormat(pv, kw, slot, red, keywords_list.keywords, keywords_list.keywords_count, keywords_list.additional_count, pvl, imp_arch, bold)
output.append(keywords_package.getKeywords())
print(''.join(output))
def process_args(argv):
parser = argparse.ArgumentParser(prog=MY_PN,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='Display keywords for specified package or for package that is in pwd.')
parser.add_argument('-v', '--version', action='version', version=MY_PV, help='show package version and exit')
parser.add_argument('package', nargs='*', default=None, help='Packages to check.')
parser.add_argument('-a', '--arch', nargs='+', default=[], help='Highlight specified arch(s)')
parser.add_argument('-A', '--align', nargs='?', default='bottom', choices=['top', 'bottom'],
help='Specify alignment for keywords description.')
parser.add_argument('-B', '--bold', action='store_true', default=False,
help='Print out each other column in bold for easier visual separation.')
parser.add_argument('-C', '--color', action='store_true', default=False,
help='Force colored output')
parser.add_argument('-O', '--overlays', action='store_true', default=False,
help='Search also overlays (in effect only if some package is specified)')
parser.add_argument('-S', '--ignore-slot', action='store_true', default=False,
help='Ignore slots during detection of ignored pacakges.')
return parser.parse_args(args=argv[1:])
def main(argv):
#opts parsing
opts = process_args(argv)
ignore_slots = opts.ignore_slot
use_overlays = opts.overlays
# user can do both --arch=a,b,c or --arch a b c
if len(opts.arch) > 1:
opts.arch = ','.join(opts.arch)
highlight_arch = ''.join(opts.arch).split(',')
bold = opts.bold
ordering = opts.align
color = opts.color
package = opts.package
# disable colors when redirected and they are not forced on
if not color and not sys.stdout.isatty():
# disable colors
portage.output.nocolor()
# if user passes us some arg we use portage to find such package and
# display its info otherwise we just detect what we have in pwd and work
# with it
if len(package) > 0:
for i in package:
cp, pv, slot, red, kw, pvl = portage_parse(i, ignore_slots, use_overlays)
display(cp, pv, slot, red, kw, pvl, highlight_arch, ordering, bold, use_overlays)
else:
cp, pv, slot, red, kw, pvl = out_of_tree_parse(ignore_slots)
display(cp, pv, slot, red, kw, pvl, highlight_arch, ordering, bold, False)
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv))
signature.asc
Description: OpenPGP digital signature
