"bitbake -e" executions from get_bb_var calls are slow and slow down oe-selftest startup. Rationalise the code to avoid them and minimise the number of "parsing" locations we use by caching key variables and passing them around more.
This was particularly problematic with oe-selftest -j usage since it would have multiple bitbake -e executions per process making parallel usage particularly slow. Signed-off-by: Richard Purdie <[email protected]> --- meta/lib/oeqa/core/utils/concurrencytest.py | 5 +-- meta/lib/oeqa/selftest/context.py | 35 ++++++++++++--------- meta/lib/oeqa/utils/commands.py | 6 ++-- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py index 4f77589b005..5e20b0e1266 100644 --- a/meta/lib/oeqa/core/utils/concurrencytest.py +++ b/meta/lib/oeqa/core/utils/concurrencytest.py @@ -193,11 +193,12 @@ class dummybuf(object): # class ConcurrentTestSuite(unittest.TestSuite): - def __init__(self, suite, processes, setupfunc, removefunc): + def __init__(self, suite, processes, setupfunc, removefunc, bb_vars): super(ConcurrentTestSuite, self).__init__([suite]) self.processes = processes self.setupfunc = setupfunc self.removefunc = removefunc + self.bb_vars = bb_vars def run(self, result): testservers, totaltests = fork_for_tests(self.processes, self) @@ -243,7 +244,7 @@ class ConcurrentTestSuite(unittest.TestSuite): def fork_for_tests(concurrency_num, suite): testservers = [] if 'BUILDDIR' in os.environ: - selftestdir = get_test_layer() + selftestdir = get_test_layer(suite.bb_vars['BBLAYERS']) test_blocks = partition_tests(suite, concurrency_num) # Clear the tests from the original suite so it doesn't keep them alive diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py index 16486e7eb99..5a09aeedffe 100644 --- a/meta/lib/oeqa/selftest/context.py +++ b/meta/lib/oeqa/selftest/context.py @@ -35,12 +35,13 @@ def get_oeselftest_metadata(args): return result class NonConcurrentTestSuite(unittest.TestSuite): - def __init__(self, suite, processes, setupfunc, removefunc): + def __init__(self, suite, processes, setupfunc, removefunc, bb_vars): super().__init__([suite]) self.processes = processes self.suite = suite self.setupfunc = setupfunc self.removefunc = removefunc + self.bb_vars = bb_vars def run(self, result): (builddir, newbuilddir) = self.setupfunc("-st", None, self.suite) @@ -79,16 +80,15 @@ class OESelftestTestContext(OETestContext): else: self.removebuilddir = removebuilddir + def set_variables(self, vars): + self.bb_vars = vars + def setup_builddir(self, suffix, selftestdir, suite): - # Get SSTATE_DIR from the parent build dir - with bb.tinfoil.Tinfoil(tracking=True) as tinfoil: - tinfoil.prepare(quiet=2, config_only=True) - d = tinfoil.config_data - sstatedir = str(d.getVar('SSTATE_DIR')) + sstatedir = self.bb_vars['SSTATE_DIR'] builddir = os.environ['BUILDDIR'] if not selftestdir: - selftestdir = get_test_layer() + selftestdir = get_test_layer(self.bb_vars['BBLAYERS']) if self.newbuilddir: newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix) else: @@ -155,9 +155,9 @@ class OESelftestTestContext(OETestContext): if processes: from oeqa.core.utils.concurrencytest import ConcurrentTestSuite - return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir) + return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) else: - return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir) + return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) def runTests(self, processes=None, machine=None, skips=[]): if machine: @@ -270,7 +270,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor): builddir = os.environ.get("BUILDDIR") self.tc_kwargs['init']['config_paths'] = {} - self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer() + self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer(bbvars["BBLAYERS"]) self.tc_kwargs['init']['config_paths']['builddir'] = builddir self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf") self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf") @@ -310,10 +310,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor): meta_selftestdir = os.path.join( self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest') if os.path.isdir(meta_selftestdir): - runCmd("bitbake-layers add-layer %s" %meta_selftestdir) + runCmd("bitbake-layers add-layer %s" % meta_selftestdir) # reload data is needed because a meta-selftest layer was add self.tc.td = get_bb_vars() - self.tc.config_paths['testlayer_path'] = get_test_layer() + self.tc.config_paths['testlayer_path'] = get_test_layer(self.tc.td["BBLAYERS"]) else: self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir) raise OEQAPreRun @@ -351,8 +351,15 @@ class OESelftestTestContextExecutor(OETestContextExecutor): _add_layer_libs() - self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable") - runCmd("bitbake -e") + self.tc.logger.info("Checking base configuration is valid/parsable") + + with bb.tinfoil.Tinfoil(tracking=True) as tinfoil: + tinfoil.prepare(quiet=2, config_only=True) + d = tinfoil.config_data + vars = {} + vars['SSTATE_DIR'] = str(d.getVar('SSTATE_DIR')) + vars['BBLAYERS'] = str(d.getVar('BBLAYERS')) + self.tc.set_variables(vars) def get_json_result_dir(self, args): json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa') diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index c1f533802e2..575e3800175 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -285,8 +285,10 @@ def get_bb_vars(variables=None, target=None, postconfig=None): def get_bb_var(var, target=None, postconfig=None): return get_bb_vars([var], target, postconfig)[var] -def get_test_layer(): - layers = get_bb_var("BBLAYERS").split() +def get_test_layer(bblayers=None): + if bblayers is None: + bblayers = get_bb_var("BBLAYERS") + layers = bblayers.split() testlayer = None for l in layers: if '~' in l: -- 2.39.2
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#188031): https://lists.openembedded.org/g/openembedded-core/message/188031 Mute This Topic: https://lists.openembedded.org/mt/101510987/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
