commit: 963ff0806c5ba897bd7ec5179989ab0576b6d9a1
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 24 01:34:37 2016 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Apr 24 01:34:37 2016 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=963ff080
More new data structure change fixes
Fix arches not being cleared every ebuild.
pym/repoman/modules/scan/arches/arches.py | 1 +
pym/repoman/modules/scan/depend/depend.py | 1 +
pym/repoman/modules/scan/ebuild/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 2 +-
pym/repoman/modules/scan/ebuild/isebuild.py | 11 +++--
pym/repoman/modules/scan/eclasses/live.py | 5 ++-
pym/repoman/modules/scan/keywords/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/modules/scan/use/use_flags.py | 13 +++---
pym/repoman/scanner.py | 62 ++++++++++++++++++---------
11 files changed, 67 insertions(+), 38 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..4df25a8 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
+ #dyn_arches.clear()
dyn_arches.update(arches)
return False
diff --git a/pym/repoman/modules/scan/depend/depend.py
b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
+ dyn_unknown.clear()
dyn_unknown.update(unknown_pkgs)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py
b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
'func_kwargs': {'checkdirlist': 'list', 'checkdir':
None, 'xpkg': None,
- 'validity_fuse': None, 'can_force': None,
'pkgs': 'dict',
+ 'validity_future': 'Future', 'can_force': None,
'pkgs': 'dict',
},
},
'ebuild-module': {
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
self.qatracker.add_error(
"ebuild.notadded", self.xpkg + "/" +
self.y_ebuild + ".ebuild")
# update the dynamic data
- self.set_result_raise([('ebuild', self)])
+ self.set_result_raise([(kwargs.get('ebuild'), self)])
return False
def set_pkg_data(self, **kwargs):
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py
b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
from repoman.qa_data import no_exec, allvars
from repoman.modules.scan.scanbase import ScanBase
+
class IsEbuild(ScanBase):
'''Performs basic tests to confirm it is an ebuild'''
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
self.continue_ = True
self.set_result_pass([(can_force, False)])
# set our updated data
- self.set_result_raise([
- (kwargs.get('pkgs'), pkgs),
- ])
+ dyn_pkgs = kwargs.get('pkgs')
+ # clear() sets it to None,
+ # we don't want to kill the pointer reference
+ # just set it back to an empty dict()
+ for key in list(dyn_pkgs):
+ dyn_pkgs.pop(key)
+ dyn_pkgs.update(pkgs)
return self.continue_
@property
diff --git a/pym/repoman/modules/scan/eclasses/live.py
b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class LiveEclassChecks(ScanBase):
@returns: dictionary, including {live_ebuild}
'''
+ ebuild = kwargs.get('ebuild').result()
# update the dynamic data
dyn_live = kwargs.get('live_ebuild')
- dyn_live.update(LIVE_ECLASSES.intersection(
- kwargs.get('ebuild').inherited))
+ #dyn_live.clear()
+ dyn_live.update(LIVE_ECLASSES.intersection(ebuild.inherited))
return False
def check(self, **kwargs):
diff --git a/pym/repoman/modules/scan/keywords/__init__.py
b/pym/repoman/modules/scan/keywords/__init__.py
index 2d92dba..2b3f6ca 100644
--- a/pym/repoman/modules/scan/keywords/__init__.py
+++ b/pym/repoman/modules/scan/keywords/__init__.py
@@ -20,8 +20,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'options', 'repo_metadata',
'profiles',
],
- 'func_kwargs': {'pkg': None, 'xpkg': None, 'ebuild':
None,
- 'y_ebuild': None, 'changed': None,
'live_ebuild': None,
+ 'func_kwargs': {'pkg': 'Future', 'xpkg': None,
'ebuild': 'Future',
+ 'y_ebuild': None, 'changed': None,
'live_ebuild': 'set',
},
},
}
diff --git a/pym/repoman/modules/scan/metadata/__init__.py
b/pym/repoman/modules/scan/metadata/__init__.py
index af9771b..9ccf377 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -48,7 +48,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {'ebuild': None, 'pkg': None,
+ 'func_kwargs': {'ebuild': None, 'pkg': 'Future',
},
},
'license-metadata': {
diff --git a/pym/repoman/modules/scan/metadata/unused.py
b/pym/repoman/modules/scan/metadata/unused.py
index 0184d78..fdca418 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -23,7 +23,7 @@ class UnusedCheck(ScanBase):
@param validity_future: Future instance
'''
xpkg = kwargs.get('xpkg')
- muselist = kwargs.get('muselist').result()
+ muselist = self.get_result(kwargs.get('muselist'), set())
used_useflags = kwargs.get('used_useflags')
valid_state = self.get_result(kwargs['validity_future'], True)
# check if there are unused local USE-descriptions in
metadata.xml
diff --git a/pym/repoman/modules/scan/use/use_flags.py
b/pym/repoman/modules/scan/use/use_flags.py
index 70f7583..b2b1cc0 100644
--- a/pym/repoman/modules/scan/use/use_flags.py
+++ b/pym/repoman/modules/scan/use/use_flags.py
@@ -42,20 +42,21 @@ class USEFlagChecks(ScanBase):
package = kwargs.get('xpkg')
ebuild = kwargs.get('ebuild').result()
y_ebuild = kwargs.get('y_ebuild')
- localUseFlags = kwargs.get('muselist').result()
+ localUseFlags = self.get_result(kwargs.get('muselist'), set())
+ dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
+ dyn_used = kwargs.get('used_useflags')
# reset state variables for the run
self.useFlags = []
self.defaultUseFlags = []
self.usedUseFlags = set()
+ dyn_ebuild_used.clear()
+ # perform the checks
self._checkGlobal(pkg)
self._checkMetadata(package, ebuild, y_ebuild, localUseFlags)
self._checkRequiredUSE(pkg, ebuild)
# update the dynamic data
- used_useflags =
kwargs.get('used_useflags').union(self.usedUseFlags)
- dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
- dyn_ebuild_used.update(self.usedUseFlags)
- dyn_used = kwargs.get('used_useflags')
- dyn_used.update(used_useflags)
+ dyn_ebuild_used.union(self.usedUseFlags)
+ dyn_used.update(self.usedUseFlags)
return False
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 3d2c69c..299d118 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -203,11 +203,6 @@ class Scanner(object):
}
# initialize the plugin checks here
self.modules = {}
- for mod in ['manifests', 'isebuild', 'keywords', 'files',
'vcsstatus',
- 'fetches', 'pkgmetadata']:
- mod_class = MODULE_CONTROLLER.get_class(mod)
- logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod_class.__name__] =
mod_class(**self.set_kwargs(mod))
def set_kwargs(self, mod):
'''Creates a limited set of kwargs to pass to the module's
__init__()
@@ -229,19 +224,32 @@ class Scanner(object):
@param dynamic_data: dictionary structure
@returns: dictionary
'''
+ #print("****** mod:", mod)
func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs']
+ #print(mod, "**** func_kwargs", func_kwargs)
# determine new keys
- required = set(func_kwargs.viewkeys())
- exist = set(dynamic_data.viewkeys())
- new = exist.difference(required)
+ required = set(list(func_kwargs))
+ #print(" required", required)
+ exist = set(list(dynamic_data))
+ #print(" exist", exist)
+ new = required.difference(exist)
+ #print(" new", new)
# update dynamic_data with initialized entries
for key in new:
- dynamic_data[key] = DATA_TYPES[func_kwargs['key']]()
+ #print(" adding", key, func_kwargs[key],
DATA_TYPES[func_kwargs[key]])
+ dynamic_data[key] = DATA_TYPES[func_kwargs[key]]()
kwargs = {}
for key in required:
+ #print(" kwargs[key]", key, dynamic_data[key])
kwargs[key] = dynamic_data[key]
return kwargs
+ def reset_futures(self, dynamic_data):
+ for key in list(dynamic_data):
+ #if key in ['ebuild', 'pkg']: # and
isinstance(dynamic_data[key], Future):
+ if isinstance(dynamic_data[key], Future) and key not in
['muselist']:
+ dynamic_data[key] = Future()
+
def scan_pkgs(self, can_force):
for xpkg in self.effective_scanlist:
xpkg_continue = False
@@ -270,16 +278,26 @@ class Scanner(object):
'repolevel': self.repolevel,
'catdir': catdir,
'pkgdir': pkgdir,
- 'validity_future': Future()
+ 'validity_future': Future(),
+ 'y_ebuild': None,
+ # this needs to be reset at the pkg level only,
+ # easiest is to just initialize it here
+ 'muselist': Future(),
}
# need to set it up for ==> self.modules or some other
ordered list
- for mod in ['Manifests', 'IsEbuild', 'KeywordChecks',
'FileChecks',
- 'VCSStatus', 'FetchChecks',
'PkgMetadata']:
- logging.debug("scan_pkgs; module: %s", mod)
- do_it, functions = self.modules[mod].runInPkgs
+ for mod in [('manifests', 'Manifests'), ('isebuild',
'IsEbuild'),
+ ('keywords', 'KeywordChecks'),
('files', 'FileChecks'),
+ ('vcsstatus', 'VCSStatus'),
('fetches', 'FetchChecks'),
+ ('pkgmetadata', 'PkgMetadata'),
+ ]:
+ mod_class = MODULE_CONTROLLER.get_class(mod[0])
+ logging.debug("Initializing class name: %s",
mod_class.__name__)
+ self.modules[mod_class.__name__] =
mod_class(**self.set_kwargs(mod[0]))
+ logging.debug("scan_pkgs; module: %s", mod[1])
+ do_it, functions =
self.modules[mod[1]].runInPkgs
if do_it:
for func in functions:
- _continue =
func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue =
func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
# If we can't access
all the metadata then it's totally unsafe to
# commit since there's
no way to generate a correct Manifest.
@@ -312,6 +330,7 @@ class Scanner(object):
dynamic_data['used_useflags'] = set()
for y_ebuild in ebuildlist:
+ self.reset_futures(dynamic_data)
dynamic_data['y_ebuild'] = y_ebuild
y_ebuild_continue = False
@@ -320,7 +339,8 @@ class Scanner(object):
for mod in [('ebuild', 'Ebuild'), ('live',
'LiveEclassChecks'),
('eapi', 'EAPIChecks'), ('ebuild_metadata',
'EbuildMetadata'),
('thirdpartymirrors', 'ThirdPartyMirrors'),
- ('description', 'DescriptionChecks'), (None,
'KeywordChecks'),
+ ('description', 'DescriptionChecks'),
+ ('keywords', 'KeywordChecks'),
('arches', 'ArchChecks'), ('depend',
'DependChecks'),
('use_flags', 'USEFlagChecks'), ('ruby',
'RubyEclassChecks'),
('license', 'LicenseChecks'), ('restrict',
'RestrictChecks'),
@@ -329,17 +349,17 @@ class Scanner(object):
('options', 'Options'), ('profile',
'ProfileDependsChecks'),
('unknown', 'DependUnknown'),
]:
- if mod[0]:
+ if mod[0] and mod[1] not in self.modules:
mod_class =
MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name:
%s", mod_class.__name__)
- self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod))
+ self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod[0]))
logging.debug("scan_ebuilds: module: %s",
mod[1])
do_it, functions =
self.modules[mod[1]].runInEbuilds
logging.debug("do_it: %s, functions: %s",
do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning
function: %s", func)
- _continue =
func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue =
func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
# If we can't access
all the metadata then it's totally unsafe to
# commit since there's
no way to generate a correct Manifest.
@@ -363,14 +383,14 @@ class Scanner(object):
if mod[0]:
mod_class = MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod))
+ self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod[0]))
logging.debug("scan_ebuilds final checks: module: %s",
mod[1])
do_it, functions = self.modules[mod[1]].runInFinal
logging.debug("do_it: %s, functions: %s", do_it,
[x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s",
func)
- _continue =
func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue =
func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
xpkg_complete = True
# logging.debug("\t>>>
Continuing")
