commit: c667f7b39ec20d30678e758dc74d669f6634797a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 31 17:43:03 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 31 17:43:03 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c667f7b3
Add SetMakeConf
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 3 +
buildbot_gentoo_ci/db/connector.py | 2 +
buildbot_gentoo_ci/db/model.py | 18 ++++++
buildbot_gentoo_ci/db/portages.py | 43 ++++++++++++++
buildbot_gentoo_ci/db/projects.py | 21 +++++++
buildbot_gentoo_ci/steps/builders.py | 95 ++++++++++++++++++++++++++++++
6 files changed, 182 insertions(+)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 3fad219..861a4e1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,6 +88,7 @@ def build_request_check():
def run_build_request():
f = util.BuildFactory()
# FIXME: 5
+ # Move the etc/portage stuff to is own file
# set needed Propertys
f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
@@ -105,4 +106,6 @@ def run_build_request():
f.addStep(builders.SetReposConf())
# update the repositorys listed in project_repository
f.addStep(builders.UpdateRepos())
+ # setup make.conf
+ f.addStep(builders.SetMakeConf())
return f
diff --git a/buildbot_gentoo_ci/db/connector.py
b/buildbot_gentoo_ci/db/connector.py
index 35febe6..7d218ab 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -36,6 +36,7 @@ from buildbot_gentoo_ci.db import categorys
from buildbot_gentoo_ci.db import packages
from buildbot_gentoo_ci.db import versions
from buildbot_gentoo_ci.db import keywords
+from buildbot_gentoo_ci.db import portages
upgrade_message = textwrap.dedent("""\
@@ -81,6 +82,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.packages = packages.PackagesConnectorComponent(self)
self.versions = versions.VersionsConnectorComponent(self)
self.keywords = keywords.KeywordsConnectorComponent(self)
+ self.portages = portages.PortagesConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 596d04e..bd111be 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -143,6 +143,24 @@ class Model(base.DBConnectorComponent):
sa.Column('value', sa.String(255), nullable=False),
)
+ portages_makeconf = sautils.Table(
+ "portages_makeconf", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('variable', sa.String(255), nullable=False),
+ )
+
+ projects_portages_makeconf = sautils.Table(
+ "projects_portages_makeconf", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('makeconf_id', sa.String(255),
+ sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('value', sa.String(255), nullable=False),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/db/portages.py
b/buildbot_gentoo_ci/db/portages.py
new file mode 100644
index 0000000..428fb84
--- /dev/null
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -0,0 +1,43 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class PortagesConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getVariables(self):
+ def thd(conn):
+ tbl = self.db.model.portages_makeconf
+ q = tbl.select()
+ return [self._row2dict(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ id=row.id,
+ variable=row.variable
+ )
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 00e1569..1f19a00 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -118,6 +118,18 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectMakeConfById(self, uuid, id):
+ def thd(conn):
+ tbl = self.db.model.projects_portages_makeconf
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.makeconf_id == id)
+ return [self._row2dict_projects_portages_makeconf(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -149,3 +161,12 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
directorys=row.directorys,
value=row.value
)
+
+ def _row2dict_projects_portages_makeconf(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ makeconf_id=row.makeconf_id,
+ value=row.value,
+ build_id=0
+ )
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index d3b3607..085d096 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -219,3 +219,98 @@ class UpdateRepos(BuildStep):
workdir=os.path.join(repository_path, ''))
])
return SUCCESS
+
+class SetMakeConf(BuildStep):
+
+ name = 'SetMakeConf'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ #FIXME: Make a dict before we pass it to the make.conf
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ makeconf_variables_data = yield
self.gentooci.db.portages.getVariables()
+ separator1 = '\n'
+ separator2 = ' '
+ makeconf_list = []
+ for k in makeconf_variables_data:
+ makeconf_variables_values_data = yield
self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
+ makeconf_variable_list = []
+ # we add some default values
+ #FIXME:
+ # we could set them in a config variables
+ # FEATURES
+ if k['variable'] == 'FEATURES':
+ makeconf_variable_list.append('xattr')
+ makeconf_variable_list.append('cgroup')
+ makeconf_variable_list.append('-news')
+ makeconf_variable_list.append('-collision-protect')
+ # EMERGE_DEFAULT_OPTS
+ if k['variable'] == 'EMERGE_DEFAULT_OPTS':
+ makeconf_variable_list.append('--buildpkg=y')
+ makeconf_variable_list.append('--rebuild-if-new-rev=y')
+ makeconf_variable_list.append('--rebuilt-binaries=y')
+ makeconf_variable_list.append('--usepkg=y')
+ makeconf_variable_list.append('--nospinner')
+ makeconf_variable_list.append('--color=n')
+ makeconf_variable_list.append('--ask=n')
+ # CFLAGS
+ if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
+ makeconf_variable_list.append('-O2')
+ makeconf_variable_list.append('-pipe')
+ makeconf_variable_list.append('-march=native')
+ makeconf_variable_list.append('-fno-diagnostics-color')
+ #FIXME:
+ # Depend on worker we may have to add a diffrent march
+ if k['variable'] == 'CXXFLAGS':
+ makeconf_variable_list.append('${CFLAGS}')
+ if k['variable'] == 'FFLAGS':
+ makeconf_variable_list.append('${FCFLAGS}')
+ if k['variable'] == 'ACCEPT_PROPERTIES':
+ makeconf_variable_list.append('-interactive')
+ if k['variable'] == 'ACCEPT_RESTRICT':
+ makeconf_variable_list.append('-fetch')
+ for v in makeconf_variables_values_data:
+ if v['build_id'] is 0:
+ makeconf_variable_list.append(v['value'])
+ if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list !=
[]:
+ makeconf_variable_list.append('ACCEPT_LICENSE="*"')
+ if makeconf_variable_list != []:
+ makeconf_variable_string = k['variable'] + '="' +
separator2.join(makeconf_variable_list) + '"'
+ makeconf_list.append(makeconf_variable_string)
+ # add hardcoded variables and values
+ #FIXME:
+ # we could set them in a config variables
+ makeconf_list.append('LC_MESSAGES=C')
+ makeconf_list.append('NOCOLOR="true"')
+ makeconf_list.append('GCC_COLORS=""')
+ makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
+ makeconf_list.append('CLEAN_DELAY=0')
+ makeconf_list.append('NOCOLOR=true')
+ makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
+ makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
+ makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
+ makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
+ # add ACCEPT_KEYWORDS from the project_data info
+ keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+ if project_data['status'] == 'unstable':
+ makeconf_keyword = '~' + keyword_data['name']
+ else:
+ makeconf_keyword = keyword_data['name']
+ makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
+ makeconf_string = separator1.join(makeconf_list)
+ print(makeconf_string)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(makeconf_string + separator1,
+ workerdest="make.conf",
+ workdir='/etc/portage/')
+ ])
+ return SUCCESS
