commit: 2dc7f260e99203e183d352601c6a479ae482697b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 20 18:39:19 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 20 18:39:19 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2dc7f260
Add RunEmerge and CheckEmergeLogs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 21 +---
buildbot_gentoo_ci/db/model.py | 11 ++
buildbot_gentoo_ci/db/projects.py | 23 +++++
buildbot_gentoo_ci/steps/builders.py | 155 ++++++++++++++++++++++++++++-
buildbot_gentoo_ci/steps/update_db.py | 2 +-
5 files changed, 192 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 2916847..3c77762 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -115,31 +115,20 @@ def run_build_request():
# run --regen if needed on repo
# update packages before any tests
# run pretend on packages update on worker
- shell_commad_list = [
- 'emerge',
- '-uDNv',
- '--changed-deps',
- '--changed-use',
- '--pretend',
- '@world'
- ]
- f.addStep(buildbot_steps.SetPropertyFromCommandNewStyle(
- command=shell_commad_list,
- strip=True,
- extract_fn=builders.PersOutputOfEmerge,
- workdir='/'
- ))
+ f.addStep(builders.RunEmerge(step='pre-update'))
# look at the log to see if we need to do stuff
# run update package on worker
+ f.addStep(builders.RunEmerge(step='update'))
# check log
# run pretend @preserved-rebuild if needed
- # look at the log to see if we need to do stuff
- # run @preserved-rebuild
+ f.addStep(builders.RunEmerge(step='preserved-libs'))
# check log
# run depclean if set
# depclean pretend
+ f.addStep(builders.RunEmerge(step='pre-depclean'))
# look at the log to see if we need to do stuff
# depclean
+ f.addStep(builders.RunEmerge(step='depclean'))
# check log
# setup make.conf if build id has changes make.conf as dict from
SetMakeConf
# setup package.* env if build id has changes
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 4c5e04c..0a2c145 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -161,6 +161,17 @@ class Model(base.DBConnectorComponent):
sa.Column('value', sa.String(255), nullable=False),
)
+ projects_emerge_options = sautils.Table(
+ "projects_emerge_options", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('oneshot', sa.Boolean, default=True),
+ sa.Column('depclean', sa.Boolean, default=True),
+ sa.Column('preserved_libs', sa.Boolean, default=True),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 1f19a00..eb8f727 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -130,6 +130,20 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectEmergeOptionsByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_emerge_options
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict_projects_emerge_options(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -170,3 +184,12 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
value=row.value,
build_id=0
)
+
+ def _row2dict_projects_emerge_options(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ oneshot=row.oneshot,
+ depclean=row.depclean,
+ preserved_libs=row.preserved_libs
+ )
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 862f0d4..c0567c1 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -15,13 +15,15 @@ from buildbot.plugins import steps
def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output = {}
emerge_output['rc'] = rc
+ emerge_output['preserved_libs'] = False
+ emerge_output['depclean'] = False
package_dict = {}
print(stderr)
+ # split the lines
for line in stdout.split('\n'):
# package list
subdict = {}
- if line.startswith('[ebuild'):
- # split the line
+ if line.startswith('[ebuild') or line.startswith('[binary'):
# if binaries
if line.startswith('[ebuild'):
subdict['binary'] = False
@@ -57,7 +59,18 @@ def PersOutputOfEmerge(rc, stdout, stderr):
subdict['python_targets'] = None
# CPU_FLAGS_X86 list
package_dict[cpv] = subdict
- #FIXME: Handling of stderr output
+ if line.startswith('>>>'):
+ #FIXME: Handling of >>> output
+ pass
+ if line.startswith('!!!'):
+ #FIXME: Handling of !!! output
+ if line.startswith('!!! existing preserved libs'):
+ pass
+ #FIXME: Handling of depclean output dict of packages that get removed
or saved
+ # split the lines
+ #FIXME: Handling of stderr output
+ for line in stderr.split('\n'):
+ pass
return {
'emerge_output' : emerge_output
}
@@ -154,6 +167,8 @@ class SetupPropertys(BuildStep):
print(projectrepository_data)
project_data = yield
self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
self.setProperty('project_data', project_data, 'project_data')
+ self.setProperty('preserved_libs', False, 'preserved-libs')
+ self.setProperty('depclean', False, 'depclean')
return SUCCESS
class SetMakeProfile(BuildStep):
@@ -311,6 +326,7 @@ class SetMakeConf(BuildStep):
makeconf_variable_list.append('--rebuilt-binaries=y')
makeconf_variable_list.append('--usepkg=y')
makeconf_variable_list.append('--binpkg-respect-use=y')
+ makeconf_variable_list.append('--binpkg-changed-deps=y')
makeconf_variable_list.append('--nospinner')
makeconf_variable_list.append('--color=n')
makeconf_variable_list.append('--ask=n')
@@ -366,3 +382,136 @@ class SetMakeConf(BuildStep):
workdir='/etc/portage/')
])
return SUCCESS
+
+class RunEmerge(BuildStep):
+
+ name = 'RunEmerge'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, step=None,**kwargs):
+ self.step = step
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ projects_emerge_options = yield
self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+ shell_commad_list = [
+ 'emerge',
+ '-v'
+ ]
+ aftersteps_list = []
+ if self.step == 'pre-update':
+ shell_commad_list.append('-uDN')
+ shell_commad_list.append('--changed-deps')
+ shell_commad_list.append('--changed-use')
+ shell_commad_list.append('--pretend')
+ shell_commad_list.append('@world')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/'
+ ))
+ aftersteps_list.append(CheckEmergeLogs('pre-update'))
+
+ if self.step == 'update':
+ shell_commad_list.append('-uDNq')
+ shell_commad_list.append('--changed-deps')
+ shell_commad_list.append('--changed-use')
+ shell_commad_list.append('@world')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=None
+ ))
+ aftersteps_list.append(CheckEmergeLogs('update'))
+
+ if self.step == 'preserved-libs' and
self.getProperty('preserved_libs'):
+ shell_commad_list.append('-q')
+ shell_commad_list.append('@preserved-rebuild')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=None
+ ))
+ aftersteps_list.append(CheckEmergeLogs('preserved-libs'))
+ self.setProperty('preserved_libs', False, 'preserved-libs')
+
+ if self.step == 'pre-depclean' and projects_emerge_options['depclean']:
+ shell_commad_list.append('--pretend')
+ shell_commad_list.append('--depclean')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/'
+ ))
+ aftersteps_list.append(CheckEmergeLogs('depclean'))
+ self.setProperty('depclean', False, 'depclean')
+
+ if self.step == 'depclean' and self.getProperty('depclean') and
projects_emerge_options['depclean']:
+ shell_commad_list.append('-q')
+ shell_commad_list.append('--depclean')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/'
+ ))
+ aftersteps_list.append(CheckEmergeLogs('depclean'))
+ if not self.step is None:
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+
+class CheckEmergeLogs(BuildStep):
+
+ name = 'CheckLogs'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, step=None,**kwargs):
+ self.step = step
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ projects_emerge_options = yield
self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+ emerge_output = self.getProperty('emerge_output')
+ shell_commad_list = [
+ 'emerge',
+ '-v'
+ ]
+ aftersteps_list = []
+
+ #FIXME: Prosees the logs and do stuff
+ # preserved-libs
+ if emerge_output['preserved_libs'] and
projects_emerge_options['preserved_libs']:
+ self.setProperty('preserved_libs', True, 'preserved-libs')
+ # depclean
+ # FIXME: check if don't remove needed stuff.
+ if emerge_output['depclean'] and projects_emerge_options['depclean']:
+ self.setProperty('depclean', True, 'depclean')
+
+ if not self.step is None:
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py
b/buildbot_gentoo_ci/steps/update_db.py
index ee2ba88..3f20cb9 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -124,7 +124,7 @@ class CheckCPVGentooCiProject(BuildStep):
def run(self):
#self.cpv_changes = self.getProperty("cpv_changes")
self.cpv_changes = []
- self.cpv_changes.append('dev-python/django-3.1.5')
+ self.cpv_changes.append('dev-python/django-3.1.7')
self.cpv_changes.append('dev-python/scrypt-0.8.16')
print(self.cpv_changes)
print(self.getProperty("repository_data"))
