Hi!
I created a preliminary patch to solve this issue.
(See the attachment)
Cheers,
Matthias
--
I welcome VSRE emails. See http://vsre.info/
From b89dc9c6980b60be139bf82da36c98ce29e9d3f1 Mon Sep 17 00:00:00 2001
From: Matthias Klumpp <matth...@tenstral.net>
Date: Fri, 6 May 2016 21:31:26 +0200
Subject: [PATCH] Aggregate and display AppStream issue hint stats
This is WIP.
---
.../debian/migrations/0002_appstreamstats.py | 24 +
distro_tracker/vendor/debian/models.py | 45 ++
.../templates/debian/appstream-action-item.html | 14 +
.../debian/templates/debian/appstream-link.html | 10 +
distro_tracker/vendor/debian/tests.py | 674 +++++++++++++++++++++
distro_tracker/vendor/debian/tracker_panels.py | 30 +
distro_tracker/vendor/debian/tracker_tasks.py | 174 ++++++
7 files changed, 971 insertions(+)
create mode 100644 distro_tracker/vendor/debian/migrations/0002_appstreamstats.py
create mode 100644 distro_tracker/vendor/debian/templates/debian/appstream-action-item.html
create mode 100644 distro_tracker/vendor/debian/templates/debian/appstream-link.html
diff --git a/distro_tracker/vendor/debian/migrations/0002_appstreamstats.py b/distro_tracker/vendor/debian/migrations/0002_appstreamstats.py
new file mode 100644
index 0000000..341c520
--- /dev/null
+++ b/distro_tracker/vendor/debian/migrations/0002_appstreamstats.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0007_keywords_descriptions'),
+ ('debian', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='AppStreamStats',
+ fields=[
+ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+ ('stats', jsonfield.fields.JSONField(default=dict)),
+ ('package', models.OneToOneField(related_name='appstream_stats', to='core.PackageName')),
+ ],
+ ),
+ ]
diff --git a/distro_tracker/vendor/debian/models.py b/distro_tracker/vendor/debian/models.py
index 9287a41..3634026 100644
--- a/distro_tracker/vendor/debian/models.py
+++ b/distro_tracker/vendor/debian/models.py
@@ -89,6 +89,51 @@ def get_lintian_url(self, full=False):
@python_2_unicode_compatible
+class AppStreamStats(models.Model):
+ """
+ Model for AppStream hint stats of packages.
+ """
+ package = models.OneToOneField(PackageName, related_name='appstream_stats')
+ stats = JSONField()
+
+ def __str__(self):
+ return 'AppStream hints for package {package}'.format(
+ package=self.package)
+
+ def get_appstream_url(self):
+ """
+ Returns the AppStream URL for the package matching the
+ :class:`AppStreamStats
+ <distro_tracker.vendor.debian.models.AppStreamHints>`.
+ """
+ package = get_or_none(SourcePackageName, pk=self.package.pk)
+ if not package:
+ return ''
+ maintainer_email = ''
+ if package.main_version:
+ maintainer = package.main_version.maintainer
+ if maintainer:
+ maintainer_email = maintainer.email
+ # Adapt the maintainer URL to the form expected by appstream.d.o
+ pkg_maintainer_email = re.sub(
+ r"""[àáèéëêòöøîìùñ~/\(\)" ']""",
+ '_',
+ maintainer_email)
+
+ if not package.main_version:
+ return ''
+
+ # TODO: What is the proper way to get (guess?) the archive-component vai the source-pkg here?
+ section = "main"
+
+ return (
+ 'https://appstream.debian.org/sid/{section}/issues/index.html#{maintainer}'.format(
+ section=section,
+ maintainer=pkg_maintainer_email)
+ )
+
+
+@python_2_unicode_compatible
class PackageTransition(models.Model):
package = models.ForeignKey(PackageName, related_name='package_transitions')
transition_name = models.CharField(max_length=50)
diff --git a/distro_tracker/vendor/debian/templates/debian/appstream-action-item.html b/distro_tracker/vendor/debian/templates/debian/appstream-action-item.html
new file mode 100644
index 0000000..d9298ad
--- /dev/null
+++ b/distro_tracker/vendor/debian/templates/debian/appstream-action-item.html
@@ -0,0 +1,14 @@
+{% with warnings=item.extra_data.warnings %}
+{% with errors=item.extra_data.errors %}
+<a href="https://wiki.debian.org/AppStream";>AppStream</a> found
+<a href="{{ item.extra_data.appstream_url }}">
+{% if errors %}
+<span>{{ errors }} error{% if errors > 1 %}s{% endif %}</span>
+{% if warnings %}and{% endif %}
+{% endif %}
+{% if warnings %}
+<span>{{ warnings }} warning{% if warnings > 1 %}s{% endif %}</span>
+{% endif %}
+</a>
+for this package. You should get rid of them to provide more metadata about this software.
+{% endwith %}{% endwith %}
diff --git a/distro_tracker/vendor/debian/templates/debian/appstream-link.html b/distro_tracker/vendor/debian/templates/debian/appstream-link.html
new file mode 100644
index 0000000..5649bc8
--- /dev/null
+++ b/distro_tracker/vendor/debian/templates/debian/appstream-link.html
@@ -0,0 +1,10 @@
+{% with appstream_hints=item.context.appstream_hints %}
+<a href="{{ item.context.appstream_url }}" title="report about metadata issues spotted by AppStream">appstream</a>
+{% if appstream_hints.errors or appstream_hints.warnings %}
+{% with warnings=appstream_hints.warnings|default:"0" %}
+{% with errors=appstream_hints.errors|default:"0" %}
+<small>(<span title="errors">{{ errors }}</span>, <span title="warnings">{{ warnings }}</span>)</small>
+{% endwith %}
+{% endwith %}
+{% endif %}
+{% endwith %}
diff --git a/distro_tracker/vendor/debian/tests.py b/distro_tracker/vendor/debian/tests.py
index 538debf..d8f826f 100644
--- a/distro_tracker/vendor/debian/tests.py
+++ b/distro_tracker/vendor/debian/tests.py
@@ -41,6 +41,7 @@
from distro_tracker.core.models import SourcePackage
from distro_tracker.core.models import PseudoPackageName
from distro_tracker.core.models import SourcePackageName
+from distro_tracker.core.models import BinaryPackageName
from distro_tracker.core.models import Repository
from distro_tracker.core.tasks import run_task
from distro_tracker.core.retrieve_data import UpdateRepositoriesTask
@@ -76,6 +77,8 @@
from distro_tracker.vendor.debian.models import UbuntuPackage
from distro_tracker.vendor.debian.tracker_tasks import UpdateLintianStatsTask
from distro_tracker.vendor.debian.models import LintianStats
+from distro_tracker.vendor.debian.tracker_tasks import UpdateAppStreamStatsTask
+from distro_tracker.vendor.debian.models import AppStreamStats
from distro_tracker.vendor.debian.management.commands\
.tracker_import_old_subscriber_dump \
import Command as ImportOldSubscribersCommand
@@ -93,6 +96,7 @@
import yaml
import json
import logging
+import zlib
logging.disable(logging.CRITICAL)
@@ -1404,6 +1408,676 @@ def test_update_does_not_affect_other_item_types(self, mock_requests):
self.assertEqual(2, self.package_name.action_items.count())
+class UpdateAppStreamStatsTaskTest(TestCase):
+
+ """
+ Tests for the
+ :class:`distro_tracker.vendor.debian.tracker_tasks.UpdateAppStreamStatsTask`
+ task.
+ """
+
+ def setUp(self):
+ self.package_name = SourcePackageName.objects.create(
+ name='dummy-package')
+ self.package = SourcePackage(
+ source_package_name=self.package_name, version='1.0.0')
+
+ self._tagdef_url = u'https://appstream.debian.org/hints/asgen-hints.json'
+ self._hints_url_template = u'https://appstream.debian.org/hints/sid/{section}/Hints-{arch}.json.gz'
+ self._tag_definitions = """{
+ "tag-mock-error": {
+ "text": "Mocking an error tag.",
+ "severity": "error"
+ },
+ "tag-mock-warning": {
+ "text": "Mocking a warning tag.",
+ "severity": "warning"
+ },
+ "tag-mock-info": {
+ "text": "Mocking an info tag.",
+ "severity": "info"
+ }
+ }"""
+
+ def run_task(self):
+ """
+ Runs the AppStream hints update task.
+ """
+ task = UpdateAppStreamStatsTask()
+ task.execute()
+
+ def _set_mock_response(self, mock_requests, text="", status_code=200):
+ """
+ Helper method which sets a mock response to the given mock requests
+ module.
+ """
+
+ mock_response = mock_requests.models.Response()
+ mock_response.status_code = status_code
+ mock_response.ok = status_code < 400
+
+ def compress_text(s):
+ """
+ Helper to GZip-compress a string.
+ """
+ compressor = zlib.compressobj(9,
+ zlib.DEFLATED,
+ 16 + zlib.MAX_WBITS,
+ zlib.DEF_MEM_LEVEL,
+ 0)
+ data = compressor.compress(s)
+ data += compressor.flush()
+ return data
+
+ def build_response(*args, **kwargs):
+ if args[0] == self._tagdef_url:
+ # the tag definitions are requested
+ mock_response.content = self._tag_definitions.encode('utf-8')
+ mock_response.json.return_value = json.loads(self._tag_definitions)
+ elif args[0] == self._hints_url_template.format(section='main', arch='amd64'):
+ # hint data was requested
+ data = compress_text(text)
+ mock_response.text = data
+ mock_response.content = data
+ else:
+ # return a compressed, but empty hints document as default
+ data = compress_text('[]')
+ mock_response.text = data
+ mock_response.content = data
+
+ return mock_response
+
+ mock_requests.get.side_effect = build_response
+
+ def get_action_item_type(self):
+ return ActionItemType.objects.get_or_create(
+ type_name=UpdateAppStreamStatsTask.ACTION_ITEM_TYPE_NAME)[0]
+
+ def assert_correct_severity_stats(self, hints, expected_hints):
+ """
+ Helper method which asserts that the given hint stats match the expected
+ stats.
+ """
+ for severity in ['errors', 'warnings', 'infos']:
+ count = hints[severity] if severity in hints else 0
+ expected_count = expected_hints[severity] if severity in expected_hints else 0
+ self.assertEqual(count, expected_count)
+
+ def assert_action_item_error_and_warning_count(self, item, errors=0, warnings=0):
+ """
+ Helper method which checks if an instance of
+ :class:`distro_tracker.core.ActionItem` contains the given error and
+ warning count in its extra_data.
+ """
+ self.assertEqual(item.extra_data['errors'], errors)
+ self.assertEqual(item.extra_data['warnings'], warnings)
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_hint_stats_created(self, mock_requests):
+ """
+ Tests that stats are created for a package that previously did not have
+ any AppStream stats.
+ """
+
+ testdata = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test1.desktop": [
+ {
+ "vars": {
+ "icon_fname": "dummy.xpm"
+ },
+ "tag": "tag-mock-error"
+ }
+ ],
+ "org.example.test2.desktop": [
+ {
+ "vars": {
+ "icon_fname": "dummy.xpm"
+ },
+ "tag": "tag-mock-error"
+ },
+ {
+ "vars": { },
+ "tag": "tag-mock-warning"
+ }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=testdata)
+
+ self.run_task()
+
+ # The stats have been created
+ self.assertEqual(1, AppStreamStats.objects.count())
+ # They are associated with the correct package.
+ stats = AppStreamStats.objects.all()[0]
+ self.assertEqual(stats.package.name, 'dummy-package')
+ # The category counts themselves are correct
+ self.assert_correct_severity_stats(stats.stats, {'errors': 2, 'warnings': 1, 'infos': 0})
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_hint_stats_updated(self, mock_requests):
+ """
+ Tests that when a package already had associated AppStream stats, they are
+ correctly updated after running the task.
+ """
+
+ # Create the pre-existing stats for the package
+ AppStreamStats.objects.create(
+ package=self.package_name, stats={'errors': 1, 'warnings': 3})
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ },
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+
+ self.run_task()
+
+ # Still only one AppStream stats object
+ self.assertEqual(1, AppStreamStats.objects.count())
+ # The package is still correct
+ stats = AppStreamStats.objects.all()[0]
+ self.assertEqual(stats.package.name, 'dummy-package')
+ # The stats have been updated
+ self.assert_correct_severity_stats(stats.stats, {'errors': 2, 'warnings': 0, 'infos': 0})
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_stats_created_multiple_packages(self, mock_requests):
+ """
+ Tests that stats are correctly creatd when there are stats for
+ multiple packages in the response.
+ """
+ # Create a second package.
+ SourcePackageName.objects.create(name='other-package')
+
+ as_hints_data = """[
+ {
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test1.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ },
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ }
+ ]
+ }
+ },
+ {
+ "package": "other-package\/1.2\/amd64",
+ "hints": {
+ "org.example.test2.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ },
+ {
+ "vars": {},
+ "tag": "tag-mock-warning"
+ }
+ ]
+ }
+ }
+ ]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # Stats created for both packages
+ self.assertEqual(2, AppStreamStats.objects.count())
+ all_names = [stats.package.name
+ for stats in AppStreamStats.objects.all()]
+ self.assertIn('dummy-package', all_names)
+ self.assertIn('other-package', all_names)
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_stats_associated_with_source(self, mock_requests):
+ """
+ Tests that we correctly map the binary packages to source packages,
+ and the stats are accurate.
+ """
+
+ # Create source packages and connected binary packages
+ bin1 = BinaryPackageName.objects.create(name="alpha-package-bin")
+ bin2 = BinaryPackageName.objects.create(name="alpha-package-data")
+
+ src_name1 = SourcePackageName.objects.create(name='alpha-package')
+ src_pkg1, _ = SourcePackage.objects.get_or_create(
+ source_package_name=src_name1, version='1.0.0')
+ src_pkg1.binary_packages = [bin1, bin2]
+ src_pkg1.save()
+
+ bin3 = BinaryPackageName.objects.create(name="beta-common")
+ src_name2 = SourcePackageName.objects.create(name='beta-package')
+ src_pkg2, _ = SourcePackage.objects.get_or_create(
+ source_package_name=src_name2, version='1.2.0')
+ src_pkg2.binary_packages = [bin3]
+ src_pkg2.save()
+
+ # Set mock data
+ as_hints_data = """[
+ {
+ "package": "alpha-package-bin\/1.0\/amd64",
+ "hints": {
+ "org.example.AlphaTest1.desktop": [
+ { "tag": "tag-mock-error", "vars": {} },
+ { "tag": "tag-mock-warning", "vars": {} }
+ ]
+ }
+ },
+ {
+ "package": "alpha-package-data\/1.0\/amd64",
+ "hints": {
+ "org.example.AlphaTest2.desktop": [
+ { "tag": "tag-mock-warning", "vars": {} }
+ ]
+ }
+ },
+ {
+ "package": "beta-common\/1.2\/amd64",
+ "hints": {
+ "org.example.BetaTest1.desktop": [
+ { "tag": "tag-mock-error", "vars": {} },
+ { "tag": "tag-mock-error", "vars": {} }
+ ]
+ }
+ }
+ ]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # Stats created for two source packages
+ self.assertEqual(2, AppStreamStats.objects.count())
+ all_names = [stats.package.name
+ for stats in AppStreamStats.objects.all()]
+
+ # source packages should be in the result
+ self.assertIn('alpha-package', all_names)
+ self.assertIn('beta-package', all_names)
+
+ # binary packages should not be there
+ self.assertNotIn('alpha-package-bin', all_names)
+ self.assertNotIn('alpha-package-data', all_names)
+ self.assertNotIn('beta-common', all_names)
+
+ # check if the stats are correct
+ stats = AppStreamStats.objects.get(package__name='alpha-package')
+ self.assert_correct_severity_stats(stats.stats, {'errors': 1, 'warnings': 2, 'infos': 0})
+
+ stats = AppStreamStats.objects.get(package__name='beta-package')
+ self.assert_correct_severity_stats(stats.stats, {'errors': 2, 'warnings': 0, 'infos': 0})
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_unknown_package(self, mock_requests):
+ """
+ Tests that when an unknown package is encountered, no stats are created.
+ """
+
+ as_hints_data = """[{
+ "package": "nonexistant\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # There are no stats
+ self.assertEqual(0, AppStreamStats.objects.count())
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_updated(self, mock_requests):
+ """
+ Tests that an existing action item is updated with new data.
+ """
+ # Create an existing action item
+ old_item = ActionItem.objects.create(
+ package=self.package_name,
+ item_type=self.get_action_item_type(),
+ short_description="Short description...",
+ extra_data={'errors': 1, 'warnings': 2})
+ old_timestamp = old_item.last_updated_timestamp
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ },
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+
+ self.run_task()
+
+ # An action item is created.
+ self.assertEqual(1, ActionItem.objects.count())
+ # Extra data updated?
+ item = ActionItem.objects.all()[0]
+ self.assert_action_item_error_and_warning_count(item, 2, 0)
+
+ # The timestamp is updated
+ self.assertNotEqual(old_timestamp, item.last_updated_timestamp)
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_not_updated(self, mock_requests):
+ """
+ Tests that an existing action item is left unchanged when the update
+ shows unchanged stats.
+ """
+ errors, warnings = 2, 0
+ # Create an existing action item
+ old_item = ActionItem.objects.create(
+ package=self.package_name,
+ item_type=self.get_action_item_type(),
+ short_description="Short description...",
+ extra_data={'appstream_url': u'https://appstream.debian.org/sid/main/issues/index.html#',
+ 'errors': errors,
+ 'warnings': warnings})
+ old_timestamp = old_item.last_updated_timestamp
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ },
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # An action item is created.
+ self.assertEqual(1, ActionItem.objects.count())
+ # Item unchanged?
+ item = ActionItem.objects.all()[0]
+ self.assertEqual(old_timestamp, item.last_updated_timestamp)
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_created(self, mock_requests):
+ """
+ Tests that an action item is created when the package has errors and
+ warnings.
+ """
+
+ # Sanity check: there were no action items in the beginning
+ self.assertEqual(0, ActionItem.objects.count())
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-error"
+ },
+ {
+ "vars": {},
+ "tag": "tag-mock-warning"
+ }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # An action item is created.
+ self.assertEqual(1, ActionItem.objects.count())
+ # The item is linked to the correct package
+ item = ActionItem.objects.all()[0]
+ self.assertEqual(item.package.name, self.package_name.name)
+ # The correct number of errors and warnings is stored in the item
+ self.assert_action_item_error_and_warning_count(item, errors=1, warnings=1)
+ # It is a high severity issue
+ self.assertEqual('high', item.get_severity_display())
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_not_created(self, mock_requests):
+ """
+ Tests that no action item is created when the package has no errors or
+ warnings.
+ """
+
+ # Sanity check: there were no action items in the beginning
+ self.assertEqual(0, ActionItem.objects.count())
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ {
+ "vars": {},
+ "tag": "tag-mock-info"
+ }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # Still no action items.
+ self.assertEqual(0, ActionItem.objects.count())
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_created_errors(self, mock_requests):
+ """
+ Tests that an action item is created when the package has errors.
+ """
+
+ # Sanity check: there were no action items in the beginning
+ self.assertEqual(0, ActionItem.objects.count())
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ { "tag": "tag-mock-error", "vars": {} },
+ { "tag": "tag-mock-error", "vars": {} }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # An action item is created.
+ self.assertEqual(1, ActionItem.objects.count())
+ # The correct number of errors and warnings is stored in the item
+ item = ActionItem.objects.all()[0]
+ self.assert_action_item_error_and_warning_count(item, errors=2, warnings=0)
+ # It has the correct type
+ self.assertEqual(
+ item.item_type.type_name,
+ UpdateAppStreamStatsTask.ACTION_ITEM_TYPE_NAME)
+ # It is a high severity issue
+ self.assertEqual('high', item.get_severity_display())
+ # Correct full description template
+ self.assertEqual(
+ item.full_description_template,
+ UpdateAppStreamStatsTask.ITEM_FULL_DESCRIPTION_TEMPLATE)
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_created_errors(self, mock_requests):
+ """
+ Tests that an action item is created when the package has warnings.
+ """
+
+ # Sanity check: there were no action items in the beginning
+ self.assertEqual(0, ActionItem.objects.count())
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ { "tag": "tag-mock-warning", "vars": {} },
+ { "tag": "tag-mock-warning", "vars": {} }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # An action item is created.
+ self.assertEqual(1, ActionItem.objects.count())
+ # The correct number of errors and warnings is stored in the item
+ item = ActionItem.objects.all()[0]
+ self.assert_action_item_error_and_warning_count(item, errors=0, warnings=2)
+ # It should be a normal severity issue
+ self.assertEqual('normal', item.get_severity_display())
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_removed(self, mock_requests):
+ """
+ Tests that a previously existing action item is removed if the updated
+ hints no longer contain errors or warnings.
+ """
+ # Make sure an item exists for the package
+ ActionItem.objects.create(
+ package=self.package_name,
+ item_type=self.get_action_item_type(),
+ short_description="Short description...",
+ extra_data={'errors': 1, 'warnings': 2})
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ { "tag": "tag-mock-info", "vars": {} }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # There are no action items any longer.
+ self.assertEqual(0, self.package_name.action_items.count())
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_removed_no_data(self, mock_requests):
+ """
+ Tests that a previously existing action item is removed when the
+ updated hints no longer contain any information for the package.
+ """
+ ActionItem.objects.create(
+ package=self.package_name,
+ item_type=self.get_action_item_type(),
+ short_description="Short description...",
+ extra_data={'errors': 1, 'warnings': 2})
+
+ as_hints_data = """[{
+ "package": "some-unrelated-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test.desktop": [
+ { "tag": "tag-mock-error", "vars": {} }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # There are no action items any longer.
+ self.assertEqual(0, self.package_name.action_items.count())
+
+ @mock.patch('distro_tracker.core.utils.http.requests')
+ def test_action_item_created_multiple_packages(self, mock_requests):
+ """
+ Tests that action items are created correctly when there are stats
+ for multiple different packages in the response.
+ """
+
+ other_package = PackageName.objects.create(
+ name='other-package',
+ source=True)
+ # Sanity check: there were no action items in the beginning
+ self.assertEqual(0, ActionItem.objects.count())
+
+ as_hints_data = """[{
+ "package": "dummy-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test1.desktop": [
+ { "tag": "tag-mock-error", "vars": {} },
+ { "tag": "tag-mock-error", "vars": {} }
+ ]
+ }
+ },
+ {
+ "package": "other-package\/1.4\/amd64",
+ "hints": {
+ "org.example.test2.desktop": [
+ { "tag": "tag-mock-warning", "vars": {} },
+ { "tag": "tag-mock-warning", "vars": {} }
+ ]
+ }
+ },
+ {
+ "package": "some-package\/1.0\/amd64",
+ "hints": {
+ "org.example.test3.desktop": [
+ { "tag": "tag-mock-error", "vars": {} }
+ ]
+ }
+ }]"""
+
+ self._set_mock_response(mock_requests, text=as_hints_data)
+ self.run_task()
+
+ # Action items are created for two packages.
+ self.assertEqual(1, self.package_name.action_items.count())
+ self.assertEqual(1, other_package.action_items.count())
+ # The items contain correct data.
+ item = self.package_name.action_items.all()[0]
+ self.assert_action_item_error_and_warning_count(item, errors=2, warnings=0)
+
+ item = other_package.action_items.all()[0]
+ self.assert_action_item_error_and_warning_count(item, errors=0, warnings=2)
+
+
class DebianBugActionItemsTests(TestCase):
"""
diff --git a/distro_tracker/vendor/debian/tracker_panels.py b/distro_tracker/vendor/debian/tracker_panels.py
index c54c283..c835052 100644
--- a/distro_tracker/vendor/debian/tracker_panels.py
+++ b/distro_tracker/vendor/debian/tracker_panels.py
@@ -28,6 +28,7 @@
from distro_tracker.vendor.debian.models import LintianStats
from distro_tracker.vendor.debian.models import PackageExcuses
from distro_tracker.vendor.debian.models import UbuntuPackage
+from distro_tracker.vendor.debian.models import AppStreamHints
class LintianLink(LinksPanel.ItemProvider):
@@ -59,6 +60,35 @@ def get_panel_items(self):
return []
+class AppStreamLink(LinksPanel.ItemProvider):
+ """
+ If there are any known AppStream hints for the package, provides a link to
+ the AppStream hints page.
+ """
+ def get_panel_items(self):
+ try:
+ appstream_hints = self.package.appstream_hints
+ except AppStreamHints.DoesNotExist:
+ return []
+
+ if sum(appstream_hints.hints.values()):
+ warnings, errors = (
+ appstream_hints.hints.get('warnings', 0),
+ appstream_hints.hints.get('errors', 0))
+ has_errors_or_warnings = warnings or errors
+ # Get the full URL only if the package does not have any errors or
+ # warnings
+ url = appstream_hints.get_appstream_url()
+ return [
+ TemplatePanelItem('debian/appstream-link.html', {
+ 'appstream_hints': appstream_hints.hints,
+ 'appstream_url': url,
+ })
+ ]
+
+ return []
+
+
class BuildLogCheckLinks(LinksPanel.ItemProvider):
def get_panel_items(self):
if not isinstance(self.package, SourcePackageName):
diff --git a/distro_tracker/vendor/debian/tracker_tasks.py b/distro_tracker/vendor/debian/tracker_tasks.py
index b523890..e09a4f8 100644
--- a/distro_tracker/vendor/debian/tracker_tasks.py
+++ b/distro_tracker/vendor/debian/tracker_tasks.py
@@ -35,6 +35,7 @@
from distro_tracker.vendor.debian.models import PackageTransition
from distro_tracker.vendor.debian.models import PackageExcuses
from distro_tracker.vendor.debian.models import UbuntuPackage
+from distro_tracker.vendor.debian.models import AppStreamStats
from distro_tracker.core.utils.http import HttpCache
from distro_tracker.core.utils.http import get_resource_content
from distro_tracker.core.utils.packages import package_hashdir
@@ -45,6 +46,7 @@
import os
import re
import json
+import zlib
import hashlib
import itertools
@@ -698,6 +700,178 @@ def execute(self):
LintianStats.objects.bulk_create(stats)
+class UpdateAppStreamStatsTask(BaseTask):
+ """
+ Updates packages' AppStream issue hints data.
+ """
+ ACTION_ITEM_TYPE_NAME = 'appstream-issue-hints'
+ ITEM_DESCRIPTION = 'AppStream hints: <a href="{url}">{report}</a>'
+ ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/appstream-action-item.html'
+
+ def __init__(self, force_update=False, *args, **kwargs):
+ super(UpdateAppStreamStatsTask, self).__init__(*args, **kwargs)
+ self.force_update = force_update
+ self.appstream_action_item_type = ActionItemType.objects.create_or_update(
+ type_name=self.ACTION_ITEM_TYPE_NAME,
+ full_description_template=self.ITEM_FULL_DESCRIPTION_TEMPLATE)
+ self._tag_severities = {}
+
+ def set_parameters(self, parameters):
+ if 'force_update' in parameters:
+ self.force_update = parameters['force_update']
+
+ def _load_tag_severities(self):
+ url = 'https://appstream.debian.org/hints/asgen-hints.json'
+ cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY)
+ response, updated = cache.update(url, force=True)
+ response.raise_for_status()
+
+ data = response.json()
+ for tag, info in data.items():
+ self._tag_severities[tag] = info['severity']
+
+ def _load_appstream_hint_stats(self, section, arch, all_stats={}):
+ url = 'https://appstream.debian.org/hints/sid/{section}/Hints-{arch}.json.gz'.format(section=section, arch=arch)
+ cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY)
+ response, _ = cache.update(url, force=self.force_update)
+ response.raise_for_status()
+
+ jdata = zlib.decompress(response.content, 16 + zlib.MAX_WBITS)
+ hints = json.loads(jdata)
+ for hint in hints:
+ pkid = hint['package']
+ parts = pkid.split('/')
+ package_name = parts[0]
+
+ # get the source package for this binary package name
+ src_pkgname = None
+ if SourcePackageName.objects.exists_with_name(package_name):
+ package = SourcePackageName.objects.get(name=package_name)
+ src_pkgname = package.name
+ elif BinaryPackageName.objects.exists_with_name(package_name):
+ bin_package = BinaryPackageName.objects.get(name=package_name)
+ package = bin_package.main_source_package_name
+ src_pkgname = package.name
+ else:
+ src_pkgname = package_name
+
+ if not src_pkgname in all_stats:
+ all_stats[src_pkgname] = {}
+ for cid, h in hint['hints'].items():
+ for e in h:
+ severity = self._tag_severities[e['tag']]
+ sevkey = "errors"
+ if severity == "warning":
+ sevkey = "warnings"
+ elif severity == "info":
+ sevkey = "infos"
+ if not sevkey in all_stats[src_pkgname]:
+ all_stats[src_pkgname][sevkey] = 1
+ else:
+ all_stats[src_pkgname][sevkey] += 1
+
+ return all_stats
+
+ def update_action_item(self, package, as_stats):
+ """
+ Updates the :class:`ActionItem` for the given package based on the
+ :class:`AppStreamStats <distro_tracker.vendor.debian.models.AppStreamStats`
+ given in ``as_stats``. If the package has errors or warnings an
+ :class:`ActionItem` is created.
+ """
+ package_stats = as_stats.stats
+ stats_warnings = package_stats.get('warnings')
+ stats_errors = package_stats.get('errors', 0)
+ warnings, errors = (stats_warnings if stats_warnings else 0,
+ stats_errors if stats_errors else 0)
+
+ # Get the old action item for this warning, if it exists.
+ appstream_action_item = package.get_action_item_for_type(
+ self.appstream_action_item_type.type_name)
+ if not warnings and not errors:
+ if appstream_action_item:
+ # If the item previously existed, delete it now since there
+ # are no longer any warnings/errors.
+ appstream_action_item.delete()
+ return
+
+ # The item didn't previously have an action item: create it now
+ if appstream_action_item is None:
+ appstream_action_item = ActionItem(
+ package=package,
+ item_type=self.appstream_action_item_type)
+
+ appstream_url = as_stats.get_appstream_url()
+ new_extra_data = {
+ 'warnings': warnings,
+ 'errors': errors,
+ 'appstream_url': appstream_url,
+ }
+ if appstream_action_item.extra_data:
+ old_extra_data = appstream_action_item.extra_data
+ if (old_extra_data['warnings'] == warnings and
+ old_extra_data['errors'] == errors):
+ # No need to update
+ return
+
+ appstream_action_item.extra_data = new_extra_data
+
+ if errors and warnings:
+ report = '{} error{} and {} warning{}'.format(
+ errors,
+ 's' if errors > 1 else '',
+ warnings,
+ 's' if warnings > 1 else '')
+ elif errors:
+ report = '{} error{}'.format(
+ errors,
+ 's' if errors > 1 else '')
+ elif warnings:
+ report = '{} warning{}'.format(
+ warnings,
+ 's' if warnings > 1 else '')
+
+ appstream_action_item.short_description = self.ITEM_DESCRIPTION.format(
+ url=appstream_url,
+ report=report)
+
+ # If there are errors make the item a high severity issue
+ if errors:
+ appstream_action_item.severity = ActionItem.SEVERITY_HIGH
+
+ appstream_action_item.save()
+
+ def execute(self):
+ self._load_tag_severities()
+ all_stats = {}
+ self._load_appstream_hint_stats("non-free", "amd64", all_stats)
+ self._load_appstream_hint_stats("contrib", "amd64", all_stats)
+ self._load_appstream_hint_stats("main", "amd64", all_stats)
+ if not all_stats:
+ return
+
+ # Discard all old hints
+ AppStreamStats.objects.all().delete()
+
+ packages = PackageName.objects.filter(name__in=all_stats.keys())
+ packages.prefetch_related('action_items')
+ # Remove action items for packages which no longer have associated
+ # AppStream hints.
+ ActionItem.objects.delete_obsolete_items(
+ [self.appstream_action_item_type], all_stats.keys())
+
+ stats = []
+ for package in packages:
+ package_stats = all_stats[package.name]
+ # Save the raw AppStream hints
+ as_stats = AppStreamStats(package=package, stats=package_stats)
+ stats.append(as_stats)
+ # Create an ActionItem if there are errors or warnings
+ self.update_action_item(package, as_stats)
+
+ AppStreamStats.objects.bulk_create(stats)
+
+
class UpdateTransitionsTask(BaseTask):
REJECT_LIST_URL = 'https://ftp-master.debian.org/transitions.yaml'
PACKAGE_TRANSITION_LIST_URL = (
