commit: 500c23afb393cc5330b82ba9d86a966e0abe6289
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 17 21:20:23 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 17 21:20:23 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=500c23af
Add BuildFactory build_request_check and start of run_build_request
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 8 +++
buildbot_gentoo_ci/config/buildfactorys.py | 12 ++++
buildbot_gentoo_ci/config/schedulers.py | 3 +
buildbot_gentoo_ci/db/model.py | 6 +-
buildbot_gentoo_ci/db/projects.py | 37 ++++++++++-
buildbot_gentoo_ci/steps/builders.py | 98 ++++++++++++++++++++++++++++++
buildbot_gentoo_ci/steps/version.py | 4 +-
7 files changed, 163 insertions(+), 5 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index 9fef46a..69dd840 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -38,4 +38,12 @@ def gentoo_builders(b=[]):
factory=buildfactorys.build_request_check()
)
)
+ # FIXME: get workers from db
+ # Use multiplay workers
+ b.append(util.BuilderConfig(
+ name='run_build_request',
+ workername='updatedb_1',
+ factory=buildfactorys.run_build_request()
+ )
+ )
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index ba96e5a..e0a6634 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -8,6 +8,7 @@ from buildbot_gentoo_ci.steps import update_db
from buildbot_gentoo_ci.steps import category
from buildbot_gentoo_ci.steps import package
from buildbot_gentoo_ci.steps import version
+from buildbot_gentoo_ci.steps import builders
def update_db_check():
f = util.BuildFactory()
@@ -78,4 +79,15 @@ def update_db_v():
def build_request_check():
f = util.BuildFactory()
# FIXME: 4
+ # get project_data
+ # check what tests to do
+ # triggger build request
+ f.addStep(builders.GetProjectRepositoryData())
+ return f
+
+def run_build_request():
+ f = util.BuildFactory()
+ # FIXME: 5
+ # Check if all is setup on the worker
+ f.addStep(builders.SetupBuildWorker())
return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 687b52f..7cfb122 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -56,10 +56,13 @@ def gentoo_schedulers():
builderNames=["update_v_data"])
build_request_data = schedulers.Triggerable(name="build_request_data",
builderNames=["build_request_data"])
+ run_build_request = schedulers.Triggerable(name="run_build_request",
+ builderNames=["run_build_request"])
s = []
s.append(test_updatedb)
#s.append(scheduler_update_db)
s.append(update_cpv_data)
s.append(update_v_data)
s.append(build_request_data)
+ s.append(run_build_request)
return s
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 9ad1331..b051b23 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -109,7 +109,7 @@ class Model(base.DBConnectorComponent):
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE'),
nullable=False),
- sa.Column('unstable', sa.Boolean, default=False),
+ sa.Column('status', sa.Enum('stable','unstable','all'),
nullable=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
sa.Column('created_by', sa.Integer,
@@ -121,12 +121,14 @@ class Model(base.DBConnectorComponent):
projects_repositorys = sautils.Table(
"projects_repositorys", metadata,
sa.Column('id', sa.Integer, primary_key=True),
- sa.Column('projects_uuid', sa.String(36),
+ sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('repository_uuid', sa.String(36),
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
+ sa.Column('auto', sa.Boolean, default=False),
+ sa.Column('pkgcheck', sa.Boolean, default=False),
)
keywords = sautils.Table(
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 4a14bac..065c9a0 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -53,6 +53,32 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects
+ q = tbl.select()
+ q = q.where(tbl.c.uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getProjectRepositorysByUuid(self, uuid, auto=True):
+ def thd(conn):
+ tbl = self.db.model.projects_repositorys
+ q = tbl.select()
+ q = q.where(tbl.c.repository_uuid == uuid)
+ q = q.where(tbl.c.auto == auto)
+ return [self._row2dict_projects_repositorys(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -62,8 +88,17 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
profile=row.profile,
profile_repository_uuid=row.profile_repository_uuid,
keyword_id=row.keyword_id,
- unstable=row.unstable,
+ status=row.status,
auto=row.auto,
enabled=row.enabled,
created_by=row.created_by
)
+
+ def _row2dict_projects_repositorys(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ repository_uuid=row.repository_uuid,
+ auto=row.auto,
+ pkgcheck=row.pkgcheck
+ )
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
new file mode 100644
index 0000000..29389a1
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -0,0 +1,98 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.plugins import steps
+
+class TriggerRunBuildRequest(BuildStep):
+
+ name = 'TriggerRunBuildRequest'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Trigger(
+ schedulerNames=['run_build_request'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'version_data' : self.getProperty("version_data"),
+ 'projectrepository_data' :
self.getProperty('projectrepository_data'),
+ 'repository_data' :
self.getProperty("repository_data"),
+ 'use_data' : self.getProperty("use_data"),
+ 'fullcheck' : self.getProperty("fullcheck"),
+ }
+ )])
+ return SUCCESS
+
+class GetProjectRepositoryData(BuildStep):
+
+ name = 'GetProjectRepositoryData'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ self.projectrepositorys_data = yield
self.gentooci.db.projects.getProjectRepositorysByUuid(self.getProperty("repository_data")['uuid'])
+ if self.projectrepositorys_data is None:
+ print('No Projects have this %s repository for testing' %
self.getProperty("repository_data")['name'])
+ return SUCCESS
+ # for loop to get all the projects that have the repository
+ for projectrepository_data in self.projectrepositorys_data:
+ # get project data
+ project_data = yield
self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
+ # check if auto, enabled and not in config.project['project']
+ if project_data['auto'] is True and project_data['enabled'] is
True and project_data['name'] != self.gentooci.config.project['project']:
+ # set Property projectrepository_data so we can use it in the
trigger
+ self.setProperty('projectrepository_data',
projectrepository_data, 'projectrepository_data')
+ self.setProperty('use_data', None, 'use_data')
+ # get name o project keyword
+ project_keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+ # if not * (all keywords)
+ if project_keyword_data['name'] != '*' or
project_data['status'] == 'all':
+ self.setProperty('fullcheck', False, 'fullcheck')
+ # get status of the keyword on cpv
+ version_keywords_data =
self.getProperty("version_keyword_dict")[project_keyword_data['name']]
+ # if unstable trigger BuildRequest on cpv
+ if project_data['status'] ==
version_keywords_data['status']:
+ yield
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+ return SUCCESS
+
+class SetupBuildWorker(BuildStep):
+
+ name = 'SetupBuildWorker'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ print('build this %s' % self.getProperty("cpv"))
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index e274e65..5430e6e 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -194,7 +194,7 @@ class TriggerBuildCheck(BuildStep):
@defer.inlineCallbacks
def run(self):
- yield steps.Trigger(
+ yield self.build.addStepsAfterCurrentStep([steps.Trigger(
schedulerNames=['build_request_data'],
waitForFinish=False,
updateSourceStamp=False,
@@ -204,7 +204,7 @@ class TriggerBuildCheck(BuildStep):
'version_keyword_dict' :
self.getProperty('version_keyword_dict'),
'repository_data' :
self.getProperty("repository_data"),
}
- )
+ )])
return SUCCESS
class DeleteOldVersion(BuildStep):
