On Thu, 2014-11-27 at 10:49 +0800, Paul Wise wrote: > * tests
Implemented in the attached patch. > * extended description for the action item Still to be done. -- bye, pabs https://wiki.debian.org/PaulWise
From 2976e4e45b7dced9900801d9cf6d85034297470a Mon Sep 17 00:00:00 2001
From: Paul Wise <p...@debian.org>
Date: Wed, 26 Nov 2014 18:55:50 +0800
Subject: [PATCH] WIP: add reproducible builds (Closes: #739497)
---
.../debian/templates/debian/logcheck-links.html | 4 +
distro_tracker/vendor/debian/tests.py | 107 +++++++++++++++++++++
distro_tracker/vendor/debian/tracker_panels.py | 8 ++
distro_tracker/vendor/debian/tracker_tasks.py | 83 ++++++++++++++++
4 files changed, 202 insertions(+)
diff --git a/distro_tracker/vendor/debian/templates/debian/logcheck-links.html b/distro_tracker/vendor/debian/templates/debian/logcheck-links.html
index ffaf2d4..f3e1b2a 100644
--- a/distro_tracker/vendor/debian/templates/debian/logcheck-links.html
+++ b/distro_tracker/vendor/debian/templates/debian/logcheck-links.html
@@ -13,4 +13,8 @@
{% endif %}
<span>,</span>
<span><a title="clang compiler build log" href="http://buildd-clang.debian.net/package.php?{{ query_string }}">clang</a></span>
+{% if item.context.has_reproducibility %}
+<span>,</span>
+<span><a title="report about build reproducibility" href="{{ item.context.reproducibility_url }}">reproducibility</a></span>
+{% endif %}
{% endwith %}
diff --git a/distro_tracker/vendor/debian/tests.py b/distro_tracker/vendor/debian/tests.py
index c330770..009c029 100644
--- a/distro_tracker/vendor/debian/tests.py
+++ b/distro_tracker/vendor/debian/tests.py
@@ -69,6 +69,7 @@ from distro_tracker.vendor.debian.tracker_tasks \
import UpdateAutoRemovalsStatsTask
from distro_tracker.vendor.debian.tracker_tasks \
import UpdatePackageScreenshotsTask
+from distro_tracker.vendor.debian.tracker_tasks import UpdateBuildReproducibilityTask
from distro_tracker.vendor.debian.models import DebianContributor
from distro_tracker.vendor.debian.models import UbuntuPackage
from distro_tracker.vendor.debian.tracker_tasks import UpdateLintianStatsTask
@@ -5188,3 +5189,109 @@ class UpdatePackageScreenshotsTaskTest(TestCase):
info = self.dummy_package.packageextractedinfo_set.get(key='general')
self.assertEqual(info.value['name'], 'dummy')
+
+
+@mock.patch('distro_tracker.core.utils.http.requests')
+class UpdateBuildReproducibilityTaskTest(TestCase):
+ """
+ Tests for the:class:`distro_tracker.vendor.debian.tracker_tasks.
+ UpdateBuildReproducibilityTask` task.
+ """
+ def setUp(self):
+ self.json_data = """
+ [{
+ "package": "dummy",
+ "version": "1.2-3",
+ "status": "unreproducible",
+ "suite": "sid"
+ }]
+ """
+ self.other_json_data = """
+ [{
+ "package": "other",
+ "version": "1.2-3",
+ "status": "unreproducible",
+ "suite": "sid"
+ }]
+ """
+ self.dummy_package = SourcePackageName.objects.create(name='dummy')
+ PackageExtractedInfo.objects.create(
+ package=self.dummy_package,
+ key='general',
+ value={
+ 'name': 'dummy',
+ 'maintainer': {
+ 'email': 'j...@example.com',
+ }
+ }
+ )
+
+ def run_task(self):
+ """
+ Runs the build reproducibility status update task.
+ """
+ task = UpdateBuildReproducibilityTask()
+ task.execute()
+
+ def test_extractedinfo_item_for_without_reproducibility(self, mock_requests):
+ """
+ Tests that packages without reproducibility info don't claim to have them.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+ other_package = SourcePackageName.objects.create(name='other-package')
+
+ self.run_task()
+
+ with self.assertRaises(PackageExtractedInfo.DoesNotExist):
+ other_package.packageextractedinfo_set.get(key='reproducibility')
+
+ def test_no_extractedinfo_for_unknown_package(self, mock_requests):
+ """
+ Tests that BuildReproducibilityTask doesn't fail with an unknown package.
+ """
+ set_mock_response(mock_requests, text=self.other_json_data)
+
+ self.run_task()
+
+ count = PackageExtractedInfo.objects.filter(key='reproducibility').count()
+ self.assertEqual(0, count)
+
+ def test_extractedinfo_for_package_with_reproducibility(self, mock_requests):
+ """
+ Tests that PackageExtractedInfo for a package with reproducibility info is correct.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+
+ self.run_task()
+
+ info = self.dummy_package.packageextractedinfo_set.get(key='reproducibility')
+
+ self.assertEqual(info.value['reproducibility'], 'unreproducible')
+
+ def test_extractedinfo_is_dropped_when_no_more_reproducibility(self, mock_requests):
+ """
+ Tests that PackageExtractedInfo is dropped if reproducibility info goes away.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+ self.run_task()
+
+ set_mock_response(mock_requests, text=self.other_json_data)
+ self.run_task()
+
+ with self.assertRaises(PackageExtractedInfo.DoesNotExist):
+ self.dummy_package.packageextractedinfo_set.get(key='reproducibility')
+
+ def test_other_extractedinfo_keys_not_dropped(self, mock_requests):
+ """
+ Ensure that other PackageExtractedInfo keys are not dropped when
+ deleting the reproducibility key.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+ self.run_task()
+
+ set_mock_response(mock_requests, text=self.other_json_data)
+ self.run_task()
+
+ info = self.dummy_package.packageextractedinfo_set.get(key='general')
+
+ self.assertEqual(info.value['name'], 'dummy')
diff --git a/distro_tracker/vendor/debian/tracker_panels.py b/distro_tracker/vendor/debian/tracker_panels.py
index d7cd4bc..0b9d762 100644
--- a/distro_tracker/vendor/debian/tracker_panels.py
+++ b/distro_tracker/vendor/debian/tracker_panels.py
@@ -77,12 +77,20 @@ class BuildLogCheckLinks(LinksPanel.ItemProvider):
logcheck_url = \
"https://qa.debian.org/bls/packages/{hash}/{pkg}.html".format(
hash=self.package.name[0], pkg=self.package.name)
+ try:
+ infos = self.package.packageextractedinfo_set.get(key='reproducibility')
+ has_reproducibility = True
+ except PackageExtractedInfo.DoesNotExist:
+ has_reproducibility = False
+ reproducibility_url = "https://reproducible.debian.net/rb-pkg/{pkg}.html".format(pkg=self.package.name)
return [
TemplatePanelItem('debian/logcheck-links.html', {
'package_query_string': query_string,
'has_checks': has_checks,
'logcheck_url': logcheck_url,
+ 'has_reproducibility': has_reproducibility,
+ 'reproducibility_url': reproducibility_url,
'has_experimental': has_experimental,
})
]
diff --git a/distro_tracker/vendor/debian/tracker_tasks.py b/distro_tracker/vendor/debian/tracker_tasks.py
index 8d797b7..3a08a45 100644
--- a/distro_tracker/vendor/debian/tracker_tasks.py
+++ b/distro_tracker/vendor/debian/tracker_tasks.py
@@ -2262,3 +2262,86 @@ class UpdatePackageScreenshotsTask(BaseTask):
extracted_info.append(screenshot_info)
PackageExtractedInfo.objects.bulk_create(extracted_info)
+
+
+class UpdateBuildReproducibilityTask(BaseTask):
+ ACTION_ITEM_TYPE_NAME = 'debian-build-reproducibility'
+ ACTION_ITEM_TEMPLATE = 'debian/build-reproducibility-action-item.html'
+ ITEM_DESCRIPTION = {
+ 'blacklisted': '<a href="{url}">Blacklisted</a> from build reproducibility testing',
+ 'FTBFS': '<a href="{url}">Fails to build</a> during reproducibility testing',
+ 'reproducible': None,
+ 'unreproducible': '<a href="{url}">Does not build reproducibly</a> during testing and no .buildinfo file is created',
+ 'unreproducible-with-buildinfo': '<a href="{url}">Does not build reproducibly</a> during testing',
+ '404': None,
+ 'not for us': None,
+ }
+
+ def __init__(self, force_update=False, *args, **kwargs):
+ super(UpdateBuildReproducibilityTask, self).__init__(*args, **kwargs)
+ self.force_update = force_update
+ self.action_item_type = ActionItemType.objects.create_or_update(
+ type_name=self.ACTION_ITEM_TYPE_NAME,
+ full_description_template=self.ACTION_ITEM_TEMPLATE)
+
+ def set_parameters(self, parameters):
+ if 'force_update' in parameters:
+ self.force_update = parameters['force_update']
+
+ def get_build_reproducibility_content(self):
+ url = 'https://reproducible.debian.net/reproducible.json'
+ return get_resource_content(url)
+
+ def get_build_reproducibility(self):
+ reproducibilities = self.get_build_reproducibility_content()
+ reproducibilities = json.loads(reproducibilities)
+ reproducibilities = dict([(item['package'], item['status']) for item in reproducibilities])
+ return reproducibilities
+
+ def update_action_item(self, package, status):
+ action_item = package.get_action_item_for_type(self.action_item_type.type_name)
+ if action_item is None:
+ action_item = ActionItem(
+ package=package,
+ item_type=self.action_item_type,
+ severity=ActionItem.SEVERITY_NORMAL)
+
+ if not (status and status in self.ITEM_DESCRIPTION and self.ITEM_DESCRIPTION[status]):
+ action_item.delete()
+ return
+
+ url = "https://reproducible.debian.net/rb-pkg/{pkg}.html".format(pkg=package.name)
+ action_item.short_description = self.ITEM_DESCRIPTION[status].format(url=url)
+ action_item.save()
+
+ def execute(self):
+ reproducibilities = self.get_build_reproducibility()
+ if reproducibilities is None:
+ return
+
+ with transaction.atomic():
+ PackageExtractedInfo.objects.filter(key='reproducibility').delete()
+
+ packages = []
+ extracted_info = []
+
+ for name, status in reproducibilities.items():
+ try:
+ package = SourcePackageName.objects.get(name=name)
+ packages.append(package)
+ self.update_action_item(package, status)
+ except SourcePackageName.DoesNotExist:
+ continue
+
+ try:
+ reproducibility_info = package.packageextractedinfo_set.get(key='reproducibility')
+ reproducibility_info.value['reproducibility'] = status
+ except PackageExtractedInfo.DoesNotExist:
+ reproducibility_info = PackageExtractedInfo(
+ key='reproducibility',
+ package=package,
+ value={'reproducibility': status})
+ extracted_info.append(reproducibility_info)
+
+ ActionItem.objects.delete_obsolete_items([self.action_item_type], packages)
+ PackageExtractedInfo.objects.bulk_create(extracted_info)
--
2.1.4
signature.asc
Description: This is a digitally signed message part
