commit: 7c12da2ec9470fcf82b37943e2a1cc523351aefe
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 28 01:09:38 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jan 28 01:09:38 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c12da2e
Add SetReposConf and UpdateRepos
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 13 ++--
buildbot_gentoo_ci/db/model.py | 3 +-
buildbot_gentoo_ci/db/projects.py | 29 ++++++++-
buildbot_gentoo_ci/steps/builders.py | 98 ++++++++++++++++++++++++++----
4 files changed, 123 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index a6bafb5..3fad219 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,8 +88,8 @@ def build_request_check():
def run_build_request():
f = util.BuildFactory()
# FIXME: 5
- # update repo for the profile
- f.addStep(builders.UpdateProfileRepo())
+ # set needed Propertys
+ f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
workdir='/etc/'))
@@ -99,9 +99,10 @@ def run_build_request():
f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
workdir='/etc/portage/'))
f.addStep(builders.SetMakeProfile())
- # setup repo.conf dir
- #f.addStep(buildbot_steps.MakeDirectory(dir="repo.conf",
+ # setup repos.conf dir
+ f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
workdir='/etc/portage/'))
- # check if we have all repository's in repos.conf listed in
project_repository's
- # update all repos listed in project_repository's
+ f.addStep(builders.SetReposConf())
+ # update the repositorys listed in project_repository
+ f.addStep(builders.UpdateRepos())
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0defb0c..596d04e 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -138,7 +138,8 @@ class Model(base.DBConnectorComponent):
sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
- sa.Column('directorys', sa.Enum('make.profile'), nullable=False),
+ # FIXME: directorys should be moved to own table
+ sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'),
nullable=False),
sa.Column('value', sa.String(255), nullable=False),
)
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 5c3406a..00e1569 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -80,7 +80,19 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
return res
@defer.inlineCallbacks
- def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+ def getRepositorysByProjectUuid(self, uuid, auto=True):
+ def thd(conn):
+ tbl = self.db.model.projects_repositorys
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.auto == auto)
+ return [self._row2dict_projects_repositorys(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getAllProjectPortageByUuidAndDirectory(self, uuid, directory):
def thd(conn):
tbl = self.db.model.projects_portage
q = tbl.select()
@@ -91,6 +103,21 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+ def thd(conn):
+ tbl = self.db.model.projects_portage
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.directorys == directory)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict_projects_portage(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 1c8cbb0..d3b3607 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -80,9 +80,9 @@ class GetProjectRepositoryData(BuildStep):
yield
self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return SUCCESS
-class UpdateProfileRepo(BuildStep):
+class SetupPropertys(BuildStep):
- name = 'UpdateProfileRepo'
+ name = 'SetupPropertys'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
@@ -101,17 +101,8 @@ class UpdateProfileRepo(BuildStep):
self.setProperty('portage_repos_path', self.portage_repos_path,
'portage_repos_path')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
- repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
project_data = yield
self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
self.setProperty('project_data', project_data, 'project_data')
- self.profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
- profile_repository_path = yield os.path.join(self.portage_repos_path,
self.profile_repository_data['name'])
- yield self.build.addStepsAfterCurrentStep([
- steps.Git(repourl=self.profile_repository_data['mirror_url'],
- mode='incremental',
- submodules=True,
- workdir=os.path.join(profile_repository_path, ''))
- ])
return SUCCESS
class SetMakeProfile(BuildStep):
@@ -133,7 +124,7 @@ class SetMakeProfile(BuildStep):
project_data = self.getProperty('project_data')
profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
makeprofiles_paths = []
- makeprofiles_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
+ makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
for makeprofile in makeprofiles_data:
makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', makeprofile['value'], '')
makeprofiles_paths.append('../../..' + makeprofile_path)
@@ -145,3 +136,86 @@ class SetMakeProfile(BuildStep):
workdir='/etc/portage/')
])
return SUCCESS
+
+class SetReposConf(BuildStep):
+
+ name = 'SetReposConf'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ # setup the default.conf
+ repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'],
'repos.conf')
+ if repos_conf_data is None:
+ print('Default repo is not set in repos.conf')
+ return FAILURE
+ # check if repos_conf_data['value'] is vaild repo name
+ separator = '\n'
+ default_conf = []
+ default_conf.append('[DEFAULT]')
+ default_conf.append('main-repo = ' + repos_conf_data['value'])
+ default_conf.append('auto-sync = no')
+ default_conf_string = separator.join(default_conf)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(default_conf_string + separator,
+ workerdest="repos.conf/default.conf",
+ workdir='/etc/portage/')
+ ])
+ # add all repos that project have in projects_repositorys to
repos.conf/reponame.conf
+ projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+ for project_repository_data in projects_repositorys_data:
+ repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+ repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
+ repository_conf = []
+ repository_conf.append('[' + repository_data['name'] + ']')
+ repository_conf.append('location = ' + repository_path)
+ repository_conf.append('sync-uri = ' +
repository_data['mirror_url'])
+ repository_conf.append('sync-type = git')
+ repository_conf.append('auto-sync = no')
+ repository_conf_string = separator.join(repository_conf)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(repository_conf_string + separator,
+ workerdest='repos.conf/' +
repository_data['name'] + '.conf',
+ workdir='/etc/portage/')
+ ])
+ return SUCCESS
+
+class UpdateRepos(BuildStep):
+
+ name = 'UpdateRepos'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ # update/add all repos that in project_repository for the project
+ projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+ for project_repository_data in projects_repositorys_data:
+ repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+ repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Git(repourl=repository_data['mirror_url'],
+ mode='incremental',
+ submodules=True,
+ workdir=os.path.join(repository_path, ''))
+ ])
+ return SUCCESS
