commit: e940caa7779d5d0f0e3864670723e0ea3a0b3e1d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 13 18:39:23 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Mar 13 18:39:23 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e940caa7
Add support to use db for portage config insted of repo on local worker
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 3 +
buildbot_gentoo_ci/config/buildfactorys.py | 27 +++---
buildbot_gentoo_ci/steps/portage.py | 106 ++++++++++++++++++++++
buildbot_gentoo_ci/steps/update_db.py | 140 +++++++++++++++++++----------
4 files changed, 218 insertions(+), 58 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index cba9fcc..c7b4469 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -9,6 +9,7 @@ def gentoo_builders(b=[]):
b.append(util.BuilderConfig(
name='update_db_check',
workername='updatedb_1',
+ workerbuilddir='builds',
factory=buildfactorys.update_db_check()
)
)
@@ -19,6 +20,7 @@ def gentoo_builders(b=[]):
b.append(util.BuilderConfig(
name='update_cpv_data',
workername='updatedb_1',
+ workerbuilddir='builds',
factory=buildfactorys.update_db_cp()
)
)
@@ -27,6 +29,7 @@ def gentoo_builders(b=[]):
b.append(util.BuilderConfig(
name='update_v_data',
workername='updatedb_1',
+ workerbuilddir='builds',
factory=buildfactorys.update_db_v()
)
)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 31543f9..f5a3b35 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -15,20 +15,21 @@ def update_db_check():
f = util.BuildFactory()
# FIXME: 1
# Get base project data from db
- # return project_repository, profile_repository,
- # project
+ # return profile_repository, project
f.addStep(update_db.GetDataGentooCiProject())
- # Check if base project repo is cloned
- # Check if profile repo is cloned
- # check if the profile link is there
- # check if the repository is cloned
- f.addStep(update_db.CheckPathGentooCiProject())
- # check if etc/portage has no error
- # return config_root
- f.addStep(update_db.CheckProjectGentooCiProject())
- # Make a for loop and trigger new builders with cpv from cpv_changes
- # return cpv, repository, project_data, config_root
- f.addStep(update_db.CheckCPVGentooCiProject())
+ # Check if needed path is there
+ f.addStep(update_db.CheckPath())
+ # update the repos
+ f.addStep(update_db.UpdateRepos())
+ # setup the profile
+ f.addStep(portage.SetMakeProfileLocal())
+ # setup repos.conf dir
+ f.addStep(portage.SetReposConfLocal())
+ # setup make.conf
+ f.addStep(portage.SetMakeConfLocal())
+ # Make a for loop and trigger new builders with cpv from git_changes
+ # return cpv, repository, project_data
+ f.addStep(update_db.TriggerCheckForCPV())
return f
def update_db_cp():
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 30361e6..4fbe141 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -11,6 +11,15 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
[email protected]
+def WriteTextToFile(path, text_list):
+ separator = '\n'
+ text_string = separator.join(text_list)
+ with open(path, "a") as f:
+ yield f.write(text_string)
+ yield f.write(separator)
+ yield f.close
+
class SetMakeProfile(BuildStep):
name = 'SetMakeProfile'
@@ -292,3 +301,100 @@ class SetEnvDefault(BuildStep):
])
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+
+class SetMakeProfileLocal(BuildStep):
+
+ name = 'SetMakeProfileLocal'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ parent_path = yield os.path.join('portage', 'make.profile', 'parent')
+ if os.path.isfile(parent_path):
+ return SUCCESS
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ self.repository_basedir =
self.gentooci.config.project['repository_basedir']
+ makeprofiles_paths = []
+ makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'],
'make.profile')
+ for makeprofile in makeprofiles_data:
+ makeprofile_path = yield os.path.join(self.repository_basedir,
self.getProperty("profile_repository_data")['name'], 'profiles',
makeprofile['value'], '')
+ makeprofiles_paths.append('../../' + makeprofile_path)
+ yield WriteTextToFile(parent_path, makeprofiles_paths)
+ return SUCCESS
+
+class SetReposConfLocal(BuildStep):
+
+ name = 'SetReposConfLocal'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ repos_conf_path = yield os.path.join('portage', 'repos.conf')
+ repos_conf_default_path = yield os.path.join(repos_conf_path,
'default.conf')
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ self.repository_basedir =
self.gentooci.config.project['repository_basedir']
+ if not os.path.isfile(repos_conf_default_path):
+ # setup the default.conf
+ repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'],
'repos.conf')
+ if repos_conf_data is None:
+ print('Default repo is not set in repos.conf')
+ return FAILURE
+ default_conf = []
+ default_conf.append('[DEFAULT]')
+ default_conf.append('main-repo = ' + repos_conf_data['value'])
+ default_conf.append('auto-sync = no')
+ yield WriteTextToFile(repos_conf_default_path, default_conf)
+ repos_conf_repository_path = yield os.path.join(repos_conf_path,
self.getProperty("repository_data")['name'] + '.conf')
+ if not os.path.isfile(repos_conf_repository_path):
+ repository_path = yield os.path.join(self.repository_basedir,
self.getProperty("repository_data")['name'])
+ repository_conf = []
+ repository_conf.append('[' +
self.getProperty("repository_data")['name'] + ']')
+ repository_conf.append('location = ' + repository_path)
+ repository_conf.append('sync-uri = ' +
self.getProperty("repository_data")['mirror_url'])
+ repository_conf.append('sync-type = git')
+ repository_conf.append('auto-sync = no')
+ yield WriteTextToFile(repos_conf_repository_path, repository_conf)
+ return SUCCESS
+
+class SetMakeConfLocal(BuildStep):
+
+ name = 'SetMakeConfLocal'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ make_conf_path = yield os.path.join('portage', 'make.conf')
+ if os.path.isfile(make_conf_path):
+ return SUCCESS
+ makeconf_list = []
+ makeconf_list.append('CFLAGS=""')
+ makeconf_list.append('CXXFLAGS=""')
+ makeconf_list.append('ACCEPT_LICENSE="*"')
+ makeconf_list.append('USE=""')
+ makeconf_list.append('ACCEPT_KEYWORDS="~amd64 amd64"')
+ makeconf_list.append('EMERGE_DEFAULT_OPTS=""')
+ makeconf_list.append('ABI_X86="32 64"')
+ makeconf_list.append('FEATURES=""')
+ yield WriteTextToFile(make_conf_path, makeconf_list)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py
b/buildbot_gentoo_ci/steps/update_db.py
index 94d0b4f..bfcd2ef 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import os
+import pygit2
from portage import config as portage_config
from portage.versions import catpkgsplit
@@ -15,8 +16,6 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
-#from buildbot_gentoo_ci.steps.updatedb_functions import category
-
class GetDataGentooCiProject(BuildStep):
def __init__(self, **kwargs):
@@ -29,16 +28,11 @@ class GetDataGentooCiProject(BuildStep):
if self.project_data is None:
log.err('No data for project in the database')
return FAILURE
- self.project_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.project_data['project_repository_uuid'])
- if self.project_repository_data is None:
- log.err('No data for repository in the database')
- return FAILURE
self.profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.project_data['profile_repository_uuid'])
if self.profile_repository_data is None:
log.err('No data for repository in the database')
return FAILURE
print(self.project_data)
- print(self.project_repository_data)
print(self.profile_repository_data)
print(self.getProperty("git_changes"))
print(self.getProperty("repository"))
@@ -51,12 +45,18 @@ class GetDataGentooCiProject(BuildStep):
if repository:
self.repository_data = yield
self.gentooci.db.repositorys.getRepositoryByName(repository)
self.setProperty("project_data", self.project_data, 'project_data')
- self.setProperty("project_repository_data",
self.project_repository_data, 'project_repository_data')
self.setProperty("profile_repository_data",
self.profile_repository_data, 'profile_repository_data')
self.setProperty("repository_data", self.repository_data,
'repository_data')
return SUCCESS
-class CheckPathGentooCiProject(BuildStep):
+class CheckPath(BuildStep):
+
+ name = 'CheckPath'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
def __init__(self, **kwargs):
super().__init__(**kwargs)
@@ -65,59 +65,110 @@ class CheckPathGentooCiProject(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir =
self.gentooci.config.project['repository_basedir']
- self.profile_repository_data =
self.getProperty("profile_repository_data")
- self.project_repository_data =
self.getProperty("project_repository_data")
- self.repository_data = self.getProperty("repository_data")
- self.project_data = self.getProperty("project_data")
- self.project_path = yield os.path.join(self.repository_basedir,
self.project_repository_data['name'] + '.git')
- self.repository_path = yield os.path.join(self.repository_basedir,
self.repository_data['name'] + '.git')
- self.portage_path = yield os.path.join(self.project_path,
self.project_data['name'], 'etc/portage')
+ self.portage_path = 'portage'
+ self.profile_path = yield os.path.join(self.portage_path,
'make.profile')
+ self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
+ print(os.getcwd())
+ print(self.getProperty("builddir"))
+ yield os.chdir(self.getProperty("builddir"))
success = True
+ print(os.getcwd())
for x in [
- os.path.join(self.repository_basedir,
self.profile_repository_data['name'] + '.git'),
- self.project_path,
- self.portage_path,
- os.path.join(self.portage_path, 'make.profile'),
- self.repository_path
- # check the path of make.profile is project_data['profile']
+ self.profile_path,
+ self.repos_path,
+ self.repository_basedir
]:
- is_dir = True
if not os.path.isdir(x):
- is_dir = False
- success = False
- print("isdir(%s): %s" %(x, is_dir))
- print(self.getProperty("builddir"))
- if not success:
- return FAILURE
+ os.makedirs(x)
return SUCCESS
-class CheckProjectGentooCiProject(BuildStep):
+class UpdateRepos(BuildStep):
+
+ name = 'UpdateRepos'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
def __init__(self, **kwargs):
super().__init__(**kwargs)
+ # Origin:
https://github.com/MichaelBoselowitz/pygit2-examples/blob/master/examples.py#L54
+ # Modifyed by Gentoo Authors.
+ @defer.inlineCallbacks
+ def gitPull(self, repo, remote_name='origin', branch='master'):
+ for remote in repo.remotes:
+ if remote.name == remote_name:
+ yield remote.fetch()
+ remote_master_id = yield
repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+ merge_result, _ = yield repo.merge_analysis(remote_master_id)
+ # Up to date, do nothing
+ if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+ return
+ # We can just fastforward
+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
+ yield repo.checkout_tree(repo.get(remote_master_id))
+ try:
+ master_ref = yield
repo.lookup_reference('refs/heads/%s' % (branch))
+ yield master_ref.set_target(remote_master_id)
+ except KeyError:
+ yield repo.create_branch(branch,
repo.get(remote_master_id))
+ yield repo.head.set_target(remote_master_id)
+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
+ yield repo.merge(remote_master_id)
+
+ if repo.index.conflicts is not None:
+ for conflict in repo.index.conflicts:
+ print('Conflicts found in:', conflict[0].path)
+ raise AssertionError('Conflicts, ahhhhh!!')
+
+ user = yield repo.default_signature
+ tree = yield repo.index.write_tree()
+ commit = yield repo.create_commit('HEAD',
+ user,
+ user,
+ 'Merge!',
+ tree,
+ [repo.head.target,
remote_master_id])
+ # We need to do this or git CLI will think we are still
merging.
+ yield repo.state_cleanup()
+ else:
+ raise AssertionError('Unknown merge analysis result')
+
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir =
self.gentooci.config.project['repository_basedir']
- self.project_repository_data =
self.getProperty("project_repository_data")
- self.project_data = self.getProperty("project_data")
- self.project_path = yield os.path.join(self.repository_basedir,
self.project_repository_data['name'] + '.git')
- self.config_root = yield os.path.join(self.project_path,
self.project_data['name'], '')
- self.make_conf_file = yield os.path.join(self.config_root,
'etc/portage', '') + 'make.conf'
- try:
- getconfig(self.make_conf_file, tolerant=0, allow_sourcing=True,
expand=True)
- mysettings = portage_config(config_root = self.config_root)
- mysettings.validate()
- except ParseError as e:
- print("project portage conf has error %s" %(str(e)))
- return FAILURE
- self.setProperty("config_root", self.config_root, 'config_root')
+ self.profile_repository_path = yield
os.path.join(self.repository_basedir,
self.getProperty("profile_repository_data")['name'])
+ repo_path = yield
pygit2.discover_repository(self.profile_repository_path)
+ print(repo_path)
+ if repo_path is None:
+ yield
pygit2.clone_repository(self.getProperty("profile_repository_data")['mirror_url'],
self.profile_repository_path)
+ else:
+ repo = yield pygit2.Repository(repo_path)
+ yield self.gitPull(repo)
+ if self.getProperty("profile_repository_data")['name'] !=
self.getProperty("repository_data")['name']:
+ self.repository_path = yield os.path.join(self.repository_basedir,
self.getProperty("repository_data")['name'])
+ repo_path = yield pygit2.discover_repository(self.repository_path)
+ if repo_path is None:
+ yield
pygit2.clone_repository(self.getProperty("profile_repository_data")['mirror_url'],
self.repository_path)
+ else:
+ repo = yield pygit2.Repository(repo_path)
+ yield self.gitPull(repo)
return SUCCESS
-class CheckCPVGentooCiProject(BuildStep):
+class TriggerCheckForCPV(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+ name = 'TriggerCheckForCPV'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
@defer.inlineCallbacks
def run(self):
self.git_changes = self.getProperty("git_changes")
@@ -152,7 +203,6 @@ class CheckCPVGentooCiProject(BuildStep):
updateSourceStamp=False,
set_properties={
'cpv' : cpv,
- 'config_root' :
self.getProperty("config_root"),
'project_data' :
self.getProperty("project_data"),
'repository_data' :
self.getProperty("repository_data"),
'revision_data' : revision_data,
