commit: 85d70df173a81e8257172d39be1ae971be54422b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 26 20:30:38 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 26 20:30:38 2016 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=85d70df1
check all ebuilds in package instead of only check Manifest
pym/tbc/build_log.py | 5 +-
pym/tbc/db_mapping.py | 1 -
pym/tbc/package.py | 151 +++++++++++++++++++++-----------------------------
pym/tbc/sqlquerys.py | 5 --
pym/tbc/sync.py | 20 +++++--
5 files changed, 80 insertions(+), 102 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index a16cdc6..ec4aaa7 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -29,7 +29,7 @@ from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \
get_package_info, get_build_job_id, get_use_id, get_config_info,
get_hilight_info, get_error_info_list, \
add_e_info, get_fail_times, add_fail_times, update_fail_times,
del_old_build_jobs, add_old_ebuild, \
- update_buildjobs_status, update_mtime_sql, add_repoman_qa,
get_config_id_fqdn, get_setup_info, \
+ update_buildjobs_status, add_repoman_qa, get_config_id_fqdn,
get_setup_info, \
add_repoman_log
from tbc.log import write_log
@@ -112,8 +112,7 @@ def get_build_dict_db(session, config_id, settings,
tbc_settings_dict, pkg):
if status:
if ebuild_id_list is None:
log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv,
repo,)
- add_logs(session, log_msg, "info", config_id)
- update_mtime_sql(session, build_dict['package_id'], 0)
+ add_logs(session, log_msg, "error", config_id)
else:
old_ebuild_id_list = []
for ebuild_id in ebuild_id_list:
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 2636bb6..d9fcf25 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -84,7 +84,6 @@ class Packages(Base):
CategoryId = Column('category_id', Integer,
ForeignKey('categories.category_id'))
Package = Column('package',String(150))
RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id'))
- Mtime = Column('mtime', DateTime, default=0)
Active = Column('active', Boolean, default=False)
TimeStamp = Column('time_stamp', DateTime, nullable=False,
default=datetime.datetime.utcnow)
__tablename__ = 'packages'
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 65a0f40..68574cf 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -17,7 +17,7 @@ from tbc.build_log import check_repoman_full
from tbc.log import write_log
from tbc.sqlquerys import get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list,
add_old_ebuild, \
- get_package_metadata_sql, update_package_metadata, update_mtime_sql, \
+ get_package_metadata_sql, update_package_metadata, \
get_package_info_from_package_id, get_config_all_info,
add_new_package_sql, \
get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info,
get_setup_info, \
get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
@@ -210,7 +210,7 @@ class tbc_package(object):
package_metadataDict[package_id] = attDict
return package_metadataDict
- def add_package(self, packageDict, package_metadataDict, package_id,
new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree):
+ def add_package(self, packageDict, package_metadataDict, package_id,
new_ebuild_id_list, old_ebuild_id_list):
# Use packageDict to update the db
ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
@@ -227,9 +227,6 @@ class tbc_package(object):
# update package metadata
update_package_metadata(self._session, package_metadataDict)
- # update the cp manifest checksum
- update_mtime_sql(self._session, package_id, manifest_mtime_tree)
-
# Get the best cpv for the configs and add it to
config_cpv_listDict
PackageInfo, CategoryInfo, RepoInfo =
get_package_info_from_package_id(self._session, package_id)
cp = CategoryInfo.Category + '/' + PackageInfo.Package
@@ -244,18 +241,6 @@ class tbc_package(object):
# Add the ebuild to the build jobs table if needed
self.add_new_build_job_db(ebuild_id_list, packageDict,
config_cpv_listDict)
- def get_manifest_mtime_tree(self, pkgdir, cp, repo, mytree):
- # Get the cp manifest file mtime.
- try:
- mtime = os.path.getmtime(pkgdir + "/Manifest")
- except:
- log_msg = "QA: Can't checksum the Manifest file.
:%s:%s" % (cp, repo,)
- write_log(self._session, log_msg, "warning",
self._config_id, 'packages.get_manifest_mtime_tree')
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- write_log(self._session, log_msg, "warning",
self._config_id, 'packages.get_manifest_mtime_tree')
- return False
- return
datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
-
def add_new_package_db(self, cp, repo):
# Add new categories package ebuild to tables package and
ebuilds
# C = Checking
@@ -268,9 +253,6 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
- manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp,
repo, mytree)
- if not manifest_mtime_tree:
- return None
package_id = add_new_package_sql(self._session, cp, repo)
@@ -332,83 +314,78 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
- manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp,
repo, mytree)
- if not manifest_mtime_tree:
+
+ # Get the ebuild list for cp
+ old_ebuild_id_list = []
+ ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1,
mytree=mytree)
+ if ebuild_list_tree == []:
+ log_msg = "QA: Can't get the ebuilds list. %s:%s" %
(cp, repo,)
+ write_log(self._session, log_msg, "error",
self._config_id, 'packages.update_package_db')
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ write_log(self._session, log_msg, "warning",
self._config_id, 'packages.update_package_db')
return None
- # if we NOT have the same mtime in the db update the package
- if PackageInfo.Mtime is None or manifest_mtime_tree >
PackageInfo.Mtime:
+ package_metadataDict = self.get_package_metadataDict(pkgdir,
repodir, package_id, cp)
+ packageDict ={}
+ new_ebuild_id_list = []
+ package_updated = False
+ for cpv in sorted(ebuild_list_tree):
- # U = Update
- log_msg = "U %s:%s" % (cp, repo)
- write_log(self._session, log_msg, "info",
self._config_id, 'packages.update_package_db')
+ # split out ebuild version
+ ebuild_version_tree =
portage.versions.cpv_getversion(cpv)
- # Get the ebuild list for cp
- old_ebuild_id_list = []
- ebuild_list_tree = self._myportdb.cp_list(cp,
use_cache=1, mytree=mytree)
- if ebuild_list_tree == []:
- log_msg = "QA: Can't get the ebuilds list.
%s:%s" % (cp, repo,)
- write_log(self._session, log_msg, "error",
self._config_id, 'packages.update_package_db')
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- write_log(self._session, log_msg, "warning",
self._config_id, 'packages.update_package_db')
- return None
+ # Get packageDict for cpv
+ packageDict[cpv] = self.get_packageDict(pkgdir, cpv,
repo)
+
+ # take package descriptions from the ebuilds
+ if
package_metadataDict[package_id]['metadata_xml_descriptions'] !=
packageDict[cpv]['ebuild_version_descriptions_tree']:
+
package_metadataDict[package_id]['metadata_xml_descriptions'] =
packageDict[cpv]['ebuild_version_descriptions_tree']
+
+ # Get the checksum of the ebuild in tree and db
+ ebuild_version_checksum_tree =
packageDict[cpv]['checksum']
+ checksums_db, fail =
get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
+
+ # check if we have dupes of the checksum from db
+ if checksums_db is None:
+ ebuild_version_manifest_checksum_db = None
+ elif fail:
+ dupe_ebuild_id_list = []
+ for checksum in checksums_db:
+ ebuilds_id , status =
get_ebuild_id_db(self._session, checksum, package_id)
+ for ebuild_id in ebuilds_id:
+ log_msg = "U %s:%s:%s Dups of
checksums" % (cpv, repo, ebuild_id,)
+ write_log(self._session,
log_msg, "warning", self._config_id, 'packages.update_package_db')
+
dupe_ebuild_id_list.append(ebuild_id)
+ add_old_ebuild(self._session,
dupe_ebuild_id_list)
+ ebuild_version_manifest_checksum_db = None
+ else:
+ ebuild_version_manifest_checksum_db =
checksums_db
+
+ # Check if the checksum have change
+ if ebuild_version_manifest_checksum_db is None:
+ # N = New ebuild
+ log_msg = "N %s:%s" % (cpv, repo,)
+ write_log(self._session, log_msg, "info",
self._config_id, 'packages.update_package_db')
+ packageDict[cpv]['new'] = True
+ package_updated = True
+ elif ebuild_version_checksum_tree !=
ebuild_version_manifest_checksum_db:
+ # U = Updated ebuild
+ log_msg = "U %s:%s" % (cpv, repo,)
+ write_log(self._session, log_msg, "info",
self._config_id, 'packages.update_package_db')
+ package_updated = True
+ else:
+ # Remove cpv from packageDict and add ebuild to
new ebuils list
+ del packageDict[cpv]
+ ebuild_id , status =
get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
+ new_ebuild_id_list.append(ebuild_id)
+ self.add_package(packageDict, package_metadataDict, package_id,
new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
+ if package_updated:
# Check cp with repoman full
status = check_repoman_full(self._session, pkgdir,
package_id, self._config_id)
if status:
log_msg = "Repoman %s::%s ... Fail." % (cp,
repo)
write_log(self._session, log_msg, "warning",
self._config_id, 'packages.update_package_db')
- package_metadataDict =
self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
- packageDict ={}
- new_ebuild_id_list = []
- for cpv in sorted(ebuild_list_tree):
-
- # split out ebuild version
- ebuild_version_tree =
portage.versions.cpv_getversion(cpv)
-
- # Get packageDict for cpv
- packageDict[cpv] = self.get_packageDict(pkgdir,
cpv, repo)
-
- # take package descriptions from the ebuilds
- if
package_metadataDict[package_id]['metadata_xml_descriptions'] !=
packageDict[cpv]['ebuild_version_descriptions_tree']:
-
package_metadataDict[package_id]['metadata_xml_descriptions'] =
packageDict[cpv]['ebuild_version_descriptions_tree']
-
- # Get the checksum of the ebuild in tree and db
- ebuild_version_checksum_tree =
packageDict[cpv]['checksum']
- checksums_db, fail =
get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
-
- # check if we have dupes of the checksum from db
- if checksums_db is None:
- ebuild_version_manifest_checksum_db =
None
- elif fail:
- dupe_ebuild_id_list = []
- for checksum in checksums_db:
- ebuilds_id , status =
get_ebuild_id_db(self._session, checksum, package_id)
- for ebuild_id in ebuilds_id:
- log_msg = "U %s:%s:%s
Dups of checksums" % (cpv, repo, ebuild_id,)
-
write_log(self._session, log_msg, "warning", self._config_id,
'packages.update_package_db')
-
dupe_ebuild_id_list.append(ebuild_id)
- add_old_ebuild(self._session,
dupe_ebuild_id_list)
- ebuild_version_manifest_checksum_db =
None
- else:
- ebuild_version_manifest_checksum_db =
checksums_db
-
- # Check if the checksum have change
- if ebuild_version_manifest_checksum_db is None:
- # N = New ebuild
- log_msg = "N %s:%s" % (cpv, repo,)
- write_log(self._session, log_msg,
"info", self._config_id, 'packages.update_package_db')
- packageDict[cpv]['new'] = True
- elif ebuild_version_checksum_tree !=
ebuild_version_manifest_checksum_db:
- # U = Updated ebuild
- log_msg = "U %s:%s" % (cpv, repo,)
- write_log(self._session, log_msg,
"info", self._config_id, 'packages.update_package_db')
- else:
- # Remove cpv from packageDict and add
ebuild to new ebuils list
- del packageDict[cpv]
- ebuild_id , status =
get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
- new_ebuild_id_list.append(ebuild_id)
- self.add_package(packageDict, package_metadataDict,
package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
write_log(self._session, log_msg, "info", self._config_id,
'packages.update_package_db')
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index e43b8fc..4161461 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -515,11 +515,6 @@ def update_package_metadata(session, package_metadataDict):
EmailInfo = update_email_info(session, email)
PackagesEmailInfo =
update_package_email_info(session, EmailInfo.EmailId, k)
-def update_mtime_sql(session, package_id, manifest_mtime_tree):
- PackagesInfo = session.query(Packages).filter_by(PackageId =
package_id).one()
- PackagesInfo.Mtime = manifest_mtime_tree
- session.commit()
-
def get_package_info_from_package_id(session, package_id):
PackageInfo = session.query(Packages).filter_by(PackageId =
package_id).one()
CategoryInfo = session.query(Categories).filter_by(CategoryId =
PackageInfo.CategoryId).one()
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 1732f1f..2d22e74 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -70,9 +70,10 @@ def git_sync_main(session):
except:
pass
- # check git diffs witch Manifests get updated and pass that to a dict
+ # check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
+ search_list = [ '^metadata', '^eclass', '^licenses', '^profiles',
'^scripts',]
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
@@ -82,14 +83,21 @@ def git_sync_main(session):
if not repouptodate:
cp_list = []
attr = {}
- # We check for Manifest changes and add the package to
a list
+ # We check for dir changes and add the package to a list
repo_diff = repo.git.diff('origin', '--name-only')
- write_log(session, 'Git diff: %s' % (repo_diff,),
"debug", config_id, 'sync.git_sync_main')
+ write_log(session, 'Git dir diff:\n%s' % (repo_diff,),
"debug", config_id, 'sync.git_sync_main')
for diff_line in repo_diff.splitlines():
- if re.search("Manifest$", diff_line):
- cp = re.sub('/Manifest', '',
diff_line)
- cp_list.append(cp)
+ find_search = True
+ for search_line in search_list:
+ if re.search(search_line, diff_line):
+ find_search = False
+ if find_search:
+ splited_diff_line = re.split('/',
diff_line)
+ cp = splited_diff_line[0] + '/' +
splited_diff_line[1]
+ if not cp in cp_list:
+ cp_list.append(cp)
attr['cp_list'] = cp_list
+ write_log(session, 'Git CP Diff: %s' % (cp_list,),
"debug", config_id, 'sync.git_sync_main')
repo_cp_dict[reponame] = attr
git_merge(repo, info_list[0])
else:
