commit: c2cffc0b424f3afcebc973031519fc3fea4629d6 Author: Michał Górny <mgorny <AT> gentoo <DOT> org> AuthorDate: Tue Jul 17 15:39:01 2018 +0000 Commit: Michał Górny <mgorny <AT> gentoo <DOT> org> CommitDate: Tue Jul 17 15:40:51 2018 +0000 URL: https://gitweb.gentoo.org/proj/qa-scripts.git/commit/?id=c2cffc0b
Convert pkgcheck2html to submodule .gitmodules | 3 + pkgcheck2html | 1 + pkgcheck2html/jinja2htmlcompress.py | 150 ---------------------------------- pkgcheck2html/output.html.jinja | 79 ------------------ pkgcheck2html/pkgcheck2html.conf.json | 97 ---------------------- pkgcheck2html/pkgcheck2html.py | 146 --------------------------------- 6 files changed, 4 insertions(+), 472 deletions(-) diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..870148a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pkgcheck2html"] + path = pkgcheck2html + url = https://github.com/mgorny/pkgcheck-result-parser diff --git a/pkgcheck2html b/pkgcheck2html new file mode 160000 index 0000000..1bfec8e --- /dev/null +++ b/pkgcheck2html @@ -0,0 +1 @@ +Subproject commit 1bfec8e37a2a770e47ed0971c4c3684292073ace diff --git a/pkgcheck2html/jinja2htmlcompress.py b/pkgcheck2html/jinja2htmlcompress.py deleted file mode 100644 index 5dfb211..0000000 --- a/pkgcheck2html/jinja2htmlcompress.py +++ /dev/null @@ -1,150 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2htmlcompress - ~~~~~~~~~~~~~~~~~~ - - A Jinja2 extension that eliminates useless whitespace at template - compilation time without extra overhead. - - :copyright: (c) 2011 by Armin Ronacher. - :license: BSD, see LICENSE for more details. -""" -import re -from jinja2.ext import Extension -from jinja2.lexer import Token, describe_token -from jinja2 import TemplateSyntaxError - - -_tag_re = re.compile(r'(?:<(/?)([a-zA-Z0-9_-]+)\s*|(>\s*))(?s)') -_ws_normalize_re = re.compile(r'[ \t\r\n]+') - - -class StreamProcessContext(object): - - def __init__(self, stream): - self.stream = stream - self.token = None - self.stack = [] - - def fail(self, message): - raise TemplateSyntaxError(message, self.token.lineno, - self.stream.name, self.stream.filename) - - -def _make_dict_from_listing(listing): - rv = {} - for keys, value in listing: - for key in keys: - rv[key] = value - return rv - - -class HTMLCompress(Extension): - isolated_elements = set(['script', 'style', 'noscript', 'textarea']) - void_elements = set(['br', 'img', 'area', 'hr', 'param', 'input', - 'embed', 'col']) - block_elements = set(['div', 'p', 'form', 'ul', 'ol', 'li', 'table', 'tr', - 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th', 'dl', - 'dt', 'dd', 'blockquote', 'h1', 'h2', 'h3', 'h4', - 'h5', 'h6', 'pre']) - breaking_rules = _make_dict_from_listing([ - (['p'], set(['#block'])), - (['li'], set(['li'])), - (['td', 'th'], set(['td', 'th', 'tr', 'tbody', 'thead', 'tfoot'])), - (['tr'], set(['tr', 'tbody', 'thead', 'tfoot'])), - (['thead', 'tbody', 'tfoot'], set(['thead', 'tbody', 'tfoot'])), - (['dd', 'dt'], set(['dl', 'dt', 'dd'])) - ]) - - def is_isolated(self, stack): - for tag in reversed(stack): - if tag in self.isolated_elements: - return True - return False - - def is_breaking(self, tag, other_tag): - breaking = self.breaking_rules.get(other_tag) - return breaking and (tag in breaking or - ('#block' in breaking and tag in self.block_elements)) - - def enter_tag(self, tag, ctx): - while ctx.stack and self.is_breaking(tag, ctx.stack[-1]): - self.leave_tag(ctx.stack[-1], ctx) - if tag not in self.void_elements: - ctx.stack.append(tag) - - def leave_tag(self, tag, ctx): - if not ctx.stack: - ctx.fail('Tried to leave "%s" but something closed ' - 'it already' % tag) - if tag == ctx.stack[-1]: - ctx.stack.pop() - return - for idx, other_tag in enumerate(reversed(ctx.stack)): - if other_tag == tag: - for num in xrange(idx + 1): - ctx.stack.pop() - elif not self.breaking_rules.get(other_tag): - break - - def normalize(self, ctx): - pos = 0 - buffer = [] - def write_data(value): - if not self.is_isolated(ctx.stack): - value = _ws_normalize_re.sub(' ', value.strip()) - buffer.append(value) - - for match in _tag_re.finditer(ctx.token.value): - closes, tag, sole = match.groups() - preamble = ctx.token.value[pos:match.start()] - write_data(preamble) - if sole: - write_data(sole) - else: - buffer.append(match.group()) - (closes and self.leave_tag or self.enter_tag)(tag, ctx) - pos = match.end() - - write_data(ctx.token.value[pos:]) - return u''.join(buffer) - - def filter_stream(self, stream): - ctx = StreamProcessContext(stream) - for token in stream: - if token.type != 'data': - yield token - continue - ctx.token = token - value = self.normalize(ctx) - yield Token(token.lineno, 'data', value) - - -class SelectiveHTMLCompress(HTMLCompress): - - def filter_stream(self, stream): - ctx = StreamProcessContext(stream) - strip_depth = 0 - while 1: - if stream.current.type == 'block_begin': - if stream.look().test('name:strip') or \ - stream.look().test('name:endstrip'): - stream.skip() - if stream.current.value == 'strip': - strip_depth += 1 - else: - strip_depth -= 1 - if strip_depth < 0: - ctx.fail('Unexpected tag endstrip') - stream.skip() - if stream.current.type != 'block_end': - ctx.fail('expected end of block, got %s' % - describe_token(stream.current)) - stream.skip() - if strip_depth > 0 and stream.current.type == 'data': - ctx.token = stream.current - value = self.normalize(ctx) - yield Token(stream.current.lineno, 'data', value) - else: - yield stream.current - stream.next() diff --git a/pkgcheck2html/output.html.jinja b/pkgcheck2html/output.html.jinja deleted file mode 100644 index a18408c..0000000 --- a/pkgcheck2html/output.html.jinja +++ /dev/null @@ -1,79 +0,0 @@ -<!DOCTYPE html> -<html> - <head> - <meta charset="utf-8"/> - <title>Gentoo CI - QA check results</title> - <link rel="stylesheet" type="text/css" href="/pkgcheck2html.css" /> - </head> - - <body> - <h1>QA check results</h1> - - {% if errors or warnings or staging %} - <div class="nav"> - <h2>issues</h2> - - <ul> - {% for g, pkgs in errors %} - <li class="err heading">{{ g }}</li> - {% for pkg in pkgs %} - <li class="err"><a href="#{{ pkg|join('/') }}">{{ pkg|join('/') }}</a></li> - {% endfor %} - {% endfor %} - {% for g, pkgs in warnings %} - <li class="warn heading">{{ g }}</li> - {% for pkg in pkgs %} - <li class="warn"><a href="#{{ pkg|join('/') }}">{{ pkg|join('/') }}</a></li> - {% endfor %} - {% endfor %} - {% for g, pkgs in staging %} - <li class="staging heading">{{ g }}</li> - {% for pkg in pkgs %} - <li class="staging"><a href="#{{ pkg|join('/') }}">{{ pkg|join('/') }}</a></li> - {% endfor %} - {% endfor %} - </ul> - </div> - {% endif %} - - <div class="content"> - <table> - {% for g, r in results %} - {% set h2_id = g[0] if g else "global" %} - <tr><th colspan="3" class="h2" id="{{ h2_id }}"> - {{ g[0] if g else "Global-scope results" }} - <small><a href="#{{ h2_id }}">¶</a></small> - </th></tr> - - {% for g, r in r %} - {% if g[0] %} - {% set h3_id = g[0] + "/" + g[1] if g[1] else "_cat" %} - <tr><th colspan="3" class="h3" id="{{ h3_id }}"> - {{ g[1] if g[1] else "Category results" }} - <small><a href="#{{ h3_id }}">¶</a></small> - </th></tr> - {% endif %} - - {% for g, r in r %} - {% for rx in r %} - {% set class_str = "" %} - {% if rx.css_class %} - {% set class_str = ' class="' + rx.css_class + '"' %} - {% endif %} - <tr{{ class_str }}> - <td>{{ g[2] if loop.index == 1 else "" }}</td> - <td>{{ rx.class }}</td> - <td>{{ rx.msg|escape }}</td> - </tr> - {% endfor %} - {% endfor %} - {% endfor %} - {% endfor %} - </table> - </div> - - <address>Generated based on results from: {{ ts.strftime("%F %T UTC") }}</address> - </body> -</html> - -<!-- vim:se ft=jinja : --> diff --git a/pkgcheck2html/pkgcheck2html.conf.json b/pkgcheck2html/pkgcheck2html.conf.json deleted file mode 100644 index 99b54bb..0000000 --- a/pkgcheck2html/pkgcheck2html.conf.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "AbsoluteSymlink": "", - "ArchesWithoutProfiles": "", - "BadFilename": "warn", - "BadInsIntoDir": "", - "BadPackageUpdate": "err", - "BadProfileEntry": "", - "BadProto": "err", - "BadRestricts": "", - "CatBadlyFormedXml": "warn", - "CatInvalidXml": "warn", - "CatMetadataXmlEmptyElement": "warn", - "CatMetadataXmlIndentation": "", - "CatMetadataXmlInvalidCatRef": "warn", - "CatMetadataXmlInvalidPkgRef": "warn", - "CatMissingMetadataXml": "warn", - "ConflictingChksums": "err", - "CrappyDescription": "warn", - "DeprecatedChksum": "staging", - "DeprecatedEAPI": "", - "DeprecatedEclass": "", - "DirectorySizeViolation": "", - "DoubleEmptyLine": "warn", - "DroppedKeywords": "", - "DuplicateFiles": "", - "EmptyFile": "warn", - "ExecutableFile": "", - "GLEP73BackAlteration": "", - "GLEP73Conflict": "", - "GLEP73Immutability": "warn", - "GLEP73SelfConflicting": "warn", - "GLEP73Syntax": "warn", - "Glep31Violation": "warn", - "HttpsAvailable": "", - "InvalidKeywords": "err", - "InvalidPN": "err", - "InvalidUtf8": "err", - "LaggingStable": "", - "MetadataError": "err", - "MismatchedPN": "warn", - "MissingChksum": "staging", - "MissingLicense": "err", - "MissingManifest": "err", - "MissingSlotDep": "", - "MissingUri": "", - "MovedPackageUpdate": "err", - "MultiMovePackageUpdate": "warn", - "NoFinalNewline": "warn", - "NonExistentDeps": "", - "NonexistentProfilePath": "err", - "NonsolvableDeps": "err", - "NonsolvableDepsInDev": "staging", - "NonsolvableDepsInExp": "staging", - "NonsolvableDepsInStable": "err", - "OldMultiMovePackageUpdate": "", - "OldPackageUpdate": "warn", - "PkgBadlyFormedXml": "warn", - "PkgInvalidXml": "warn", - "PkgMetadataXmlEmptyElement": "warn", - "PkgMetadataXmlIndentation": "", - "PkgMetadataXmlInvalidCatRef": "warn", - "PkgMetadataXmlInvalidPkgRef": "warn", - "PkgMetadataXmlInvalidProject": "warn", - "PkgMissingMetadataXml": "warn", - "PortageInternals": "", - "RedundantVersion": "", - "RequiredUseDefaults": "", - "SizeViolation": "", - "StaleUnstable": "", - "StupidKeywords": "warn", - "TrailingEmptyLine": "warn", - "UnknownCategories": "err", - "UnknownLicenses": "err", - "UnknownManifest": "err", - "UnknownProfileArches": "", - "UnknownProfilePackageUse": "", - "UnknownProfilePackages": "", - "UnknownProfileStatus": "", - "UnknownProfileUse": "", - "UnnecessaryManifest": "warn", - "UnstableOnly": "", - "UnstatedIUSE": "err", - "UnusedEclasses": "", - "UnusedGlobalFlags": "", - "UnusedInMastersEclasses": "", - "UnusedInMastersGlobalFlags": "", - "UnusedInMastersLicenses": "", - "UnusedInMastersMirrors": "", - "UnusedLicenses": "", - "UnusedLocalFlags": "warn", - "UnusedMirrors": "", - "UnusedProfileDirs": "", - "VisibleVcsPkg": "err", - "VulnerablePackage": "", - "WhitespaceFound": "", - "WrongIndentFound": "warn" -} diff --git a/pkgcheck2html/pkgcheck2html.py b/pkgcheck2html/pkgcheck2html.py deleted file mode 100755 index ffdf76a..0000000 --- a/pkgcheck2html/pkgcheck2html.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# vim:se fileencoding=utf8 : -# (c) 2015-2017 Michał Górny -# 2-clause BSD license - -import argparse -import collections -import datetime -import io -import json -import os -import os.path -import sys -import xml.etree.ElementTree - -import jinja2 - - -class Result(object): - def __init__(self, el, class_mapping): - self._el = el - self._class_mapping = class_mapping - - def __getattr__(self, key): - return self._el.findtext(key) or '' - - @property - def css_class(self): - return self._class_mapping.get(getattr(self, 'class'), '') - - -def result_sort_key(r): - return (r.category, r.package, r.version, getattr(r, 'class'), r.msg) - - -def get_results(input_paths, class_mapping): - for input_path in input_paths: - if input_path == '-': - input_path = sys.stdin - checks = xml.etree.ElementTree.parse(input_path).getroot() - for r in checks: - yield Result(r, class_mapping) - - -def split_result_group(it): - for r in it: - if not r.category: - yield ((), r) - elif not r.package: - yield ((r.category,), r) - elif not r.version: - yield ((r.category, r.package), r) - else: - yield ((r.category, r.package, r.version), r) - - -def group_results(it, level = 3): - prev_group = () - prev_l = [] - - for g, r in split_result_group(it): - if g[:level] != prev_group: - if prev_l: - yield (prev_group, prev_l) - prev_group = g[:level] - prev_l = [] - prev_l.append(r) - yield (prev_group, prev_l) - - -def deep_group(it, level = 1): - for g, r in group_results(it, level): - if level > 3: - for x in r: - yield x - else: - yield (g, deep_group(r, level+1)) - - -def find_of_class(it, cls, level = 2): - out = collections.defaultdict(set) - - for g, r in group_results(it, level): - for x in r: - if x.css_class == cls: - out[getattr(x, 'class')].add(g) - - return [(k, sorted(v)) for k, v in sorted(out.items())] - - -def get_result_timestamp(paths): - for p in paths: - st = os.stat(p) - return datetime.datetime.utcfromtimestamp(st.st_mtime) - - -def main(*args): - p = argparse.ArgumentParser() - p.add_argument('-o', '--output', default='-', - help='Output HTML file ("-" for stdout)') - p.add_argument('-t', '--timestamp', default=None, - help='Timestamp for results (git ISO8601-like UTC)') - p.add_argument('files', nargs='+', - help='Input XML files') - args = p.parse_args(args) - - conf_path = os.path.join(os.path.dirname(__file__), 'pkgcheck2html.conf.json') - with io.open(conf_path, 'r', encoding='utf8') as f: - class_mapping = json.load(f) - - jenv = jinja2.Environment( - loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), - extensions=['jinja2htmlcompress.HTMLCompress']) - t = jenv.get_template('output.html.jinja') - - results = sorted(get_results(args.files, class_mapping), key=result_sort_key) - - types = {} - for r in results: - cl = getattr(r, 'class') - if cl not in types: - types[cl] = 0 - types[cl] += 1 - - if args.timestamp is not None: - ts = datetime.datetime.strptime(args.timestamp, '%Y-%m-%d %H:%M:%S') - else: - ts = get_result_timestamp(args.files) - - out = t.render( - results = deep_group(results), - warnings = find_of_class(results, 'warn'), - staging = find_of_class(results, 'staging'), - errors = find_of_class(results, 'err'), - ts = ts, - ) - - if args.output == '-': - sys.stdout.write(out) - else: - with io.open(args.output, 'w', encoding='utf8') as f: - f.write(out) - - -if __name__ == '__main__': - sys.exit(main(*sys.argv[1:]))
