commit: 2bd236b7fefd255aee66d70848906c66f5da056a Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Tue Jul 21 15:48:22 2015 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Tue Jul 21 15:48:22 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2bd236b7
add support for changlog sql/structure_dump.sql | 32 +++++++++++++++++++++++++++++--- tbc/pym/db_mapping.py | 2 ++ tbc/pym/package.py | 34 +++++++++++++++++++++++++++------- tbc/pym/sqlquerys.py | 4 +++- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/sql/structure_dump.sql b/sql/structure_dump.sql index 459f6bf..cb1b30f 100644 --- a/sql/structure_dump.sql +++ b/sql/structure_dump.sql @@ -3,7 +3,7 @@ -- http://www.phpmyadmin.net -- -- Host: localhost --- Generation Time: Jul 17, 2015 at 08:35 PM +-- Generation Time: Jul 21, 2015 at 03:29 PM -- Server version: 10.0.15-MariaDB-log -- PHP Version: 5.6.10-pl0-gentoo @@ -213,6 +213,19 @@ CREATE TABLE IF NOT EXISTS `categories` ( -- -------------------------------------------------------- -- +-- Table structure for table `categories_metadata` +-- + +CREATE TABLE IF NOT EXISTS `categories_metadata` ( +`id` int(11) NOT NULL, + `category_id` int(11) NOT NULL, + `checksum` varchar(100) NOT NULL, + `descriptions` text NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Categories main table (C)'; + +-- -------------------------------------------------------- + +-- -- Table structure for table `configs` -- @@ -474,7 +487,9 @@ CREATE TABLE IF NOT EXISTS `packages_emails` ( CREATE TABLE IF NOT EXISTS `packages_metadata` ( `id` int(11) NOT NULL, `package_id` int(11) NOT NULL, - `checksum` varchar(100) NOT NULL + `checksum` varchar(100) NOT NULL, + `checksum_chlog` varchar(100) NOT NULL, + `changlog` text NOT NULL ) ENGINE=MyISAM DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -599,6 +614,12 @@ ALTER TABLE `categories` ADD PRIMARY KEY (`category_id`); -- +-- Indexes for table `categories_metadata` +-- +ALTER TABLE `categories_metadata` + ADD PRIMARY KEY (`id`); + +-- -- Indexes for table `configs` -- ALTER TABLE `configs` @@ -807,6 +828,11 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; ALTER TABLE `categories` MODIFY `category_id` int(11) NOT NULL AUTO_INCREMENT; -- +-- AUTO_INCREMENT for table `categories_metadata` +-- +ALTER TABLE `categories_metadata` +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; +-- -- AUTO_INCREMENT for table `configs` -- ALTER TABLE `configs` @@ -930,7 +956,7 @@ DELIMITER $$ -- -- Events -- -CREATE DEFINER=`tbc`@`localhost` EVENT `add_esync_jobs` ON SCHEDULE EVERY 1 HOUR STARTS '2012-12-23 17:15:13' ON COMPLETION NOT PRESERVE ENABLE DO BEGIN +CREATE DEFINER=`tbc`@`localhost` EVENT `add_esync_jobs` ON SCHEDULE EVERY 30 MINUTE STARTS '2012-12-23 17:15:13' ON COMPLETION NOT PRESERVE ENABLE DO BEGIN CALL add_jobs_esync(); END$$ diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py index cf4a054..37eaec8 100644 --- a/tbc/pym/db_mapping.py +++ b/tbc/pym/db_mapping.py @@ -104,6 +104,8 @@ class PackagesMetadata(Base): Id = Column('id', Integer, primary_key=True) PackageId = Column('package_id', Integer, ForeignKey('packages.package_id')) Checksum = Column('checksum', String(100)) + ChecksumChanglog = Column('checksum_chlog', String(100)) + Changlog = Column('changlog', Text) __tablename__ = 'packages_metadata' class Ebuilds(Base): diff --git a/tbc/pym/package.py b/tbc/pym/package.py index 8d156d8..c8a70db 100644 --- a/tbc/pym/package.py +++ b/tbc/pym/package.py @@ -2,10 +2,11 @@ # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +import re import portage from portage.xml.metadata import MetaDataXML from tbc.flags import tbc_use_flags -from tbc.text import get_ebuild_cvs_revision +from tbc.text import get_ebuild_cvs_revision, get_log_text_dict from tbc.flags import tbc_use_flags from tbc.qachecks import digestcheck, check_repoman from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \ @@ -156,19 +157,37 @@ class tbc_package(object): log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,) add_logs(self._session, log_msg, "info", self._config_id) i = i +1 + def get_changelog_text(self,pkgdir): + changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir + "/ChangeLog") + spec = 3 + spec_tmp = 1 + changelog_text_tree = '' + for index, text_line inchangelog_text_dict.items(): + if re.search('^#', text_line) + changelog_text_tree = changelog_text_tree + text_line + elif re.search('^\n', text_line) and re.search('^\*', changelog_text_dict[index + 1]) + changelog_text_tree = changelog_text_tree + text_line + spec_tmp = spec_tmp + 1 + spec = spec + 1 + elif re.search('^\n', text_line) and not re.search('^\*', changelog_text_dict[index + 1]) + if spec_tmp == spec: + break + else + spec_tmp = spec_tmp + 1 + changelog_text_tree = changelog_text_tree + text_line + else: + changelog_text_tree = changelog_text_tree + text_line + return changelog_text_tree def get_package_metadataDict(self, pkgdir, package_id): # Make package_metadataDict attDict = {} package_metadataDict = {} md_email_list = [] - # changelog_checksum_tree = portage.checksum.sha256hash(pkgdir + "/ChangeLog") - # changelog_text_tree = get_file_text(pkgdir + "/ChangeLog") herd = None pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd) - #metadata_xml_text_tree = get_file_text(pkgdir + "/metadata.xml") - # attDict['changelog_checksum'] = changelog_checksum_tree[0] - # attDict['changelog_text'] = changelog_text_tree + attDict['changelog_checksum'] = portage.checksum.sha256hash(pkgdir + "/ChangeLog")[0] + attDict['changelog_text'] = self.get_changelog_text(pkgdir) tmp_herds = pkg_md.herds() if tmp_herds != (): attDict['metadata_xml_herds'] = tmp_herds[0] @@ -181,7 +200,8 @@ class tbc_package(object): log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml") add_logs(self._session, log_msg, "qa", self._config_id) attDict['metadata_xml_email'] = False - attDict['metadata_xml_descriptions'] = pkg_md.descriptions() + metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0]) + attDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree) attDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0] #attDict['metadata_xml_text'] = metadata_xml_text_tree package_metadataDict[package_id] = attDict diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py index a521dbc..7a64db7 100644 --- a/tbc/pym/sqlquerys.py +++ b/tbc/pym/sqlquerys.py @@ -490,10 +490,12 @@ def update_package_metadata(session, package_metadataDict): try: PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one() except NoResultFound as e: - session.add(PackagesMetadata(PackageId = k, Checksum = v['metadata_xml_checksum'])) + session.add(PackagesMetadata(PackageId = k, Checksum = v['metadata_xml_checksum'], ChecksumChanglog = v['changelog_checksum'], Changlog = v['changelog_text'])) session.commit() else: PackagesMetadataInfo.Checksum = v['metadata_xml_checksum'] + PackagesMetadataInfo.ChecksumChanglog = v['changelog_checksum'] + PackagesMetadataInfo.Changlog = v['changelog_text'] session.commit() if v['metadata_xml_email']: for email in v['metadata_xml_email']:
