commit: 6eeb75c0327219634379666ebe534a71810132a9
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue May 12 18:37:17 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue May 12 18:37:17 2015 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6eeb75c0
Use git to sync instead of emerge --sync
tbc/pym/jobs.py | 6 +--
tbc/pym/sync.py | 142 +++++++++++++++++++++++++++++++---------------------
tbc/pym/updatedb.py | 84 ++++++++++++++++++-------------
3 files changed, 137 insertions(+), 95 deletions(-)
diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py
index b75e344..a805a22 100644
--- a/tbc/pym/jobs.py
+++ b/tbc/pym/jobs.py
@@ -2,7 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
-from tbc.sync import sync_tree
+from tbc.sync import git_repo_sync_main
#from tbc.buildquerydb import add_buildquery_main, del_buildquery_main
from tbc.updatedb import update_db_main
from tbc.sqlquerys import get_config_id, add_tbc_logs, get_jobs,
update_job_list
@@ -47,7 +47,7 @@ def jobs_main(session, config_id):
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
- if sync_tree(session) and update_db_main(session,
config_id):
+ if update_db_main(session, git_repo_sync_main(session),
config_id):
update_job_list(session, "Done", job_id)
log_msg = "Job %s is done.." % (job_id,)
add_tbc_logs(session, log_msg, "info",
config_id)
@@ -59,7 +59,7 @@ def jobs_main(session, config_id):
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
- if update_db_main(session, config_id):
+ if update_db_main(session, None, config_id):
update_job_list(session, "Done", job_id)
log_msg = "Job %s is done.." % (job_id,)
add_tbc_logs(session, log_msg, "info",
config_id)
diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
index f1bc56a..2622b25 100644
--- a/tbc/pym/sync.py
+++ b/tbc/pym/sync.py
@@ -10,12 +10,49 @@ import time
from pygit2 import Repository, GIT_MERGE_ANALYSIS_FASTFORWARD,
GIT_MERGE_ANALYSIS_NORMAL, \
GIT_MERGE_ANALYSIS_UP_TO_DATE
-from _emerge.main import emerge_main
from tbc.sqlquerys import get_config_id, add_tbc_logs, get_config_all_info,
get_configmetadata_info
-from tbc.updatedb import update_db_main
from tbc.readconf import read_config_settings
-def sync_tree(session):
+def git_repos_list(session, myportdb):
+ repo_trees_list = myportdb.porttrees
+ for repo_dir in repo_trees_list:
+ repo_dir_list = []
+ repo_dir_list.append(repo_dir)
+ return repo_dir_list
+
+def git_fetch(session, git_repo, config_id):
+ repo = Repository(git_repo)
+ remote = repo.remotes["origin"]
+ remote.fetch()
+ return repo
+
+def git_merge(session, repo, config_id):
+ remote_master_id =
repo.lookup_reference('refs/remotes/origin/master').target
+ merge_result, _ = repo.merge_analysis(remote_master_id)
+ if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
+ log_msg = "Repo is up to date"
+ add_zobcs_logs(session, log_msg, "info", config_id)
+ elif merge_result & GIT_MERGE_ANALYSIS_FASTFORWARD:
+ repo.checkout_tree(repo.get(remote_master_id))
+ master_ref = repo.lookup_reference('refs/heads/master')
+ master_ref.set_target(remote_master_id)
+ repo.head.set_target(remote_master_id)
+ elif merge_result & GIT_MERGE_ANALYSIS_NORMAL:
+ repo.merge(remote_master_id)
+ assert repo.index.conflicts is None, 'Conflicts, ahhhh!'
+ user = repo.default_signature
+ tree = repo.index.write_tree()
+ commit = repo.create_commit('HEAD',
+ user,
+ user,
+ 'Merge!',
+ tree,
+ [repo.head.target, remote_master_id])
+ repo.state_cleanup()
+ else:
+ raise AssertionError('Unknown merge analysis result')
+
+def git_sync_main(session):
tbc_settings_dict = read_config_settings()
_hostname = tbc_settings_dict['hostname']
_config = tbc_settings_dict['tbc_config']
@@ -23,6 +60,7 @@ def sync_tree(session):
host_config = _hostname +"/" + _config
default_config_root = tbc_settings_dict['tbc_gitrepopath'] + "/" +
host_config + "/"
mysettings = portage.config(config_root = default_config_root)
+ myportdb = portage.portdbapi(mysettings=mysettings)
GuestBusy = True
log_msg = "Waiting for Guest to be idel"
add_tbc_logs(session, log_msg, "info", config_id)
@@ -37,67 +75,59 @@ def sync_tree(session):
Status_list.append(ConfigMetadata.Status)
if not 'Runing' in Status_list:
GuestBusy = False
- time.sleep(30)
+ else:
+ time.sleep(30)
try:
os.remove(mysettings['PORTDIR'] + "/profiles/config/parent")
os.rmdir(mysettings['PORTDIR'] + "/profiles/config")
except:
pass
- tmpcmdline = []
- tmpcmdline.append("--sync")
- tmpcmdline.append("--quiet")
- tmpcmdline.append("--config-root=" + default_config_root)
- log_msg = "Emerge --sync"
+
+ repo_cp_dict = {}
+ for repo_dir in git_repos_list(session, myportdb):
+ attr = {}
+ repo = git_fetch(session, repo_dir, config_id)
+ remote_master_id =
repo.lookup_reference('refs/remotes/origin/master').target
+ merge_result, _ = repo.merge_analysis(remote_master_id)
+ if not merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
+ git_merge(session, repo, config_id)
+ out = repo.diff('HEAD', 'HEAD^')
+ repo_diff = out.patch
+ cp_list = []
+ reponame = myportdb.getRepositoryName(repo_dir)
+ for diff_line in repo_diff.splitlines():
+ if re.search("Manifest", diff_line) and
re.search("^diff --git", diff_line):
+ diff_line2 = re.split(' ',
re.sub('[a-b]/', '', re.sub('diff --git ', '', diff_line)))
+ if diff_line2[0] == diff_line2[1] or
"Manifest" in diff_line2[0]:
+ cp = re.sub('/Manifest', '',
diff_line2[0])
+ cp_list.append(cp)
+ else:
+ cp = re.sub('/Manifest', '',
diff_line2[1])
+ cp_list.append(cp)
+ attr['cp_list'] = cp_list
+ repo_cp_dict[reponame] = attr
+ else:
+ log_msg = "Repo is up to date"
+ add_tbc_logs(session, log_msg, "info", config_id)
+
+ # Need to add a config dir so we can use profiles/base for reading the
tree.
+ # We may allready have the dir on local repo when we sync.
+ try:
+ os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
+ with open(mysettings['PORTDIR'] + "/profiles/config/parent",
"w") as f:
+ f.write("../base\n")
+ f.close()
+ except:
+ pass
+ log_msg = "Repo sync ... Done."
add_tbc_logs(session, log_msg, "info", config_id)
- fail_sync = emerge_main(args=tmpcmdline)
- if fail_sync:
- log_msg = "Emerge --sync fail!"
- add_tbc_logs(session, log_msg, "error", config_id)
- return False
- else:
- # Need to add a config dir so we can use profiles/base for
reading the tree.
- # We may allready have the dir on local repo when we sync.
- try:
- os.mkdir(mysettings['PORTDIR'] + "/profiles/config",
0o777)
- with open(mysettings['PORTDIR'] +
"/profiles/config/parent", "w") as f:
- f.write("../base\n")
- f.close()
- except:
- pass
- log_msg = "Emerge --sync ... Done."
- add_tbc_logs(session, log_msg, "info", config_id)
- return True
+ return repo_cp_dict
def git_pull(session, git_repo, config_id):
log_msg = "Git pull"
- add_zobcs_logs(session, log_msg, "info", config_id)
- repo = Repository(git_repo + ".git")
- remote = repo.remotes["origin"]
- remote.fetch()
- remote_master_id =
repo.lookup_reference('refs/remotes/origin/master').target
- merge_result, _ = repo.merge_analysis(remote_master_id)
- if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
- log_msg = "Repo is up to date"
- add_zobcs_logs(session, log_msg, "info", config_id)
- elif merge_result & GIT_MERGE_ANALYSIS_FASTFORWARD:
- repo.checkout_tree(repo.get(remote_master_id))
- master_ref = repo.lookup_reference('refs/heads/master')
- master_ref.set_target(remote_master_id)
- repo.head.set_target(remote_master_id)
- elif merge_result & GIT_MERGE_ANALYSIS_NORMAL:
- repo.merge(remote_master_id)
- assert repo.index.conflicts is None, 'Conflicts, ahhhh!'
- user = repo.default_signature
- tree = repo.index.write_tree()
- commit = repo.create_commit('HEAD',
- user,
- user,
- 'Merge!',
- tree,
- [repo.head.target, remote_master_id])
- repo.state_cleanup()
- else:
- raise AssertionError('Unknown merge analysis result')
+ add_tbc_logs(session, log_msg, "info", config_id)
+ reop = git_fetch(session, git_repo, config_id)
+ git_merge(session, repo, config_id)
log_msg = "Git pull ... Done"
- add_zobcs_logs(session, log_msg, "info", config_id)
+ add_tbc_logs(session, log_msg, "info", config_id)
return True
diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py
index 90a9e0a..3cbe495 100644
--- a/tbc/pym/updatedb.py
+++ b/tbc/pym/updatedb.py
@@ -15,7 +15,7 @@ from tbc.sqlquerys import add_tbc_logs, get_package_info,
update_repo_db, \
from tbc.check_setup import check_make_conf
from tbc.package import tbc_package
# Get the options from the config file set in tbc.readconf
-from tbc.readconf import get_conf_settings
+from tbc.readconf import read_config_settings
def init_portage_settings(session, config_id, tbc_settings_dict):
# check config setup
@@ -74,44 +74,56 @@ def update_cpv_db(session, config_id, tbc_settings_dict):
break
time.sleep(30)
- mysettings = init_portage_settings(session, config_id,
tbc_settings_dict)
log_msg = "Checking categories, package, ebuilds"
add_tbc_logs(session, log_msg, "info", config_id)
new_build_jobs_list = []
- # Setup portdb, package
+ # Setup settings, portdb and pool
+ mysettings = init_portage_settings(session, config_id,
tbc_settings_dict)
myportdb = portage.portdbapi(mysettings=mysettings)
- repo_list = ()
- repos_trees_list = []
-
+
# Use all cores when multiprocessing
- pool_cores= multiprocessing.cpu_count()
- pool = multiprocessing.Pool(processes=pool_cores)
-
- # Will run some update checks and update package if needed
-
- # Get the repos and update the repos db
- repo_list = myportdb.getRepositories()
- update_repo_db(session, repo_list)
-
- # Close the db for the multiprocessing pool will make new ones
- # and we don't need this one for some time.
-
- # Get the rootdirs for the repos
- repo_trees_list = myportdb.porttrees
- for repo_dir in repo_trees_list:
- repo = myportdb.getRepositoryName(repo_dir)
- repo_dir_list = []
- repo_dir_list.append(repo_dir)
+ pool_cores = multiprocessing.cpu_count()
+ pool = multiprocessing.Pool(processes = pool_cores)
+
+ # Get packages and repo
+ if repo_cp_dict is None:
+ repo_list = []
+ repos_trees_list = []
+
+ # Get the repos and update the repos db
+ repo_list = myportdb.getRepositories()
+ update_repo_db(session, repo_list)
+
+ # Get the rootdirs for the repos
+ repo_trees_list = myportdb.porttrees
+ for repo_dir in repo_trees_list:
+ repo = myportdb.getRepositoryName(repo_dir)
+ repo_dir_list = []
+ repo_dir_list.append(repo_dir)
+
+ # Get the package list from the repo
+ package_list_tree = myportdb.cp_all(trees=repo_dir_list)
+
+ # Run the update package for all package in the list
and in a multiprocessing pool
+ for cp in sorted(package_list_tree):
+ pool.apply_async(update_cpv_db_pool,
(mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id,))
+ # use this when debuging
+ #update_cpv_db_pool(mysettings, myportdb, cp,
repo, zobcs_settings_dict, config_id)
+ else:
+ # Update needed repos and packages in the dict
+ for repo, v in repo_cp_dict.items():
+ # Get the repos and update the repos db
+ repo_list = []
+ repo_list.append(repo)
+ update_repo_db(session, repo_list)
- # Get the package list from the repo
- package_list_tree = myportdb.cp_all(trees=repo_dir_list)
+ # Run the update package for all package in the list
and in a multiprocessing pool
+ for cp in v['cp_list']:
+ pool.apply_async(update_cpv_db_pool,
(mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id,))
+ # use this when debuging
+ #update_cpv_db_pool(mysettings, myportdb, cp,
repo, zobcs_settings_dict, config_id)
- # Run the update package for all package in the list and in a
multiprocessing pool
- for cp in sorted(package_list_tree):
- pool.apply_async(update_cpv_db_pool, (mysettings,
myportdb, cp, repo, tbc_settings_dict, config_id,))
- # use this when debuging
- #update_cpv_db_pool(mysettings, myportdb, cp, repo,
tbc_settings_dict, config_id)
#close and join the multiprocessing pools
pool.close()
@@ -119,17 +131,17 @@ def update_cpv_db(session, config_id, tbc_settings_dict):
log_msg = "Checking categories, package and ebuilds ... done"
add_tbc_logs(session, log_msg, "info", config_id)
-def update_db_main(session, config_id):
+def update_db_main(session, repo_cp_dict, config_id):
# Main
-
+ if repo_cp_dict == {}:
+ return True
# Logging
- reader = get_conf_settings()
- tbc_settings_dict=reader.read_tbc_settings_all()
+ tbc_settings_dict = reader. read_config_settings()
log_msg = "Update db started."
add_tbc_logs(session, log_msg, "info", config_id)
# Update the cpv db
- update_cpv_db(session, config_id, tbc_settings_dict)
+ update_cpv_db(session, repo_cp_dict, config_id, tbc_settings_dict)
log_msg = "Update db ... Done."
add_tbc_logs(session, log_msg, "info", config_id)
return True
