Hi all,
I attach an updated patch to add support for parsing ftp-master removals.
Comments welcome!
cheers
Stuart
--
Stuart Prescott http://www.nanonanonano.net/ stu...@nanonanonano.net
Debian Developer http://www.debian.org/ stu...@debian.org
GPG fingerprint 90E2 D2C1 AD14 6A1B 7EBB 891D BBC1 7EBB 1396 F2F7
>From 2262bc01ab488daaeb6a8e588d2218588a63f3b2 Mon Sep 17 00:00:00 2001
From: Stuart Prescott <stu...@debian.org>
Date: Sat, 16 Aug 2014 16:05:50 +1000
Subject: [PATCH] Add support for parsing ftp-master removals
---
lib/debian/deb822.py | 121 ++++++++++++++++++++++++++++++++++++++++++++
tests/test_deb822.py | 25 +++++++++
tests/test_removals.822 | 132 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 278 insertions(+)
create mode 100644 tests/test_removals.822
diff --git a/lib/debian/deb822.py b/lib/debian/deb822.py
index 8de03dc..f3a7334 100644
--- a/lib/debian/deb822.py
+++ b/lib/debian/deb822.py
@@ -37,6 +37,8 @@ except (ImportError, AttributeError):
import chardet
import collections
+import datetime
+import email.utils
import re
import subprocess
import sys
@@ -1598,6 +1600,125 @@ class RestrictedWrapper(object):
return self.__data.dump(*args, **kwargs)
+class Removals(Deb822):
+ """Represent an ftp-master removals.822 file
+
+ Removal of packages from the archive are recorded by ftp-masters.
+ See https://ftp-master.debian.org/#removed
+ """
+ __sources_line_re = re.compile(r'\s*'
+ r'(?P<package>.+?)'
+ r'_'
+ r'(?P<version>[^\s]+)'
+ r'\s*'
+ )
+ __binaries_line_re = re.compile(r'\s*'
+ r'(?P<package>.+?)'
+ r'_'
+ r'(?P<version>[^\s]+)'
+ r'\s+'
+ r'\[(?P<archs>.+)\]'
+ )
+
+ @property
+ def date(self):
+ """ a datetime object for the removal action """
+ ts = email.utils.mktime_tz(email.utils.parsedate_tz(self['date']))
+ return datetime.datetime.fromtimestamp(ts)
+
+ @property
+ def bug(self):
+ """ list of bug numbers that had requested the package removal
+
+ The bug numbers are returned as integers.
+
+ Note: there is normally only one entry in this list but there may be
+ more than one.
+ """
+ if 'bug' not in self:
+ return []
+ return [int(b) for b in self['bug'].split(",")]
+
+ @property
+ def also_wnpp(self):
+ """ list of WNPP bug numbers closed by the removal
+
+ The bug numbers are returned as integers.
+ """
+ if 'also-wnpp' not in self:
+ return []
+ return [int(b) for b in self['also-wnpp'].split(" ")]
+
+ @property
+ def also_bugs(self):
+ """ list of bug numbers in the package closed by the removal
+
+ The bug numbers are returned as integers.
+
+ Removal of a package implicitly also closes all bugs associated with
+ the package.
+ """
+ if 'also-bugs' not in self:
+ return []
+ return [int(b) for b in self['also-bugs'].split(" ")]
+
+ @property
+ def sources(self):
+ """ list of source packages that were removed
+
+ A list of dicts is returned, each dict has the form:
+ {
+ 'source': 'some-package-name',
+ 'version': '1.2.3-1'
+ }
+
+ Note: There may be no source packages removed at all if the removal is
+ only of a binary package. An empty list is returned in that case.
+ """
+ if hasattr(self, '_sources'):
+ return self._sources
+
+ s = []
+ if 'sources' in self:
+ for line in self['sources'].splitlines():
+ matches = self.__sources_line_re.match(line)
+ if matches:
+ s.append({
+ 'source': matches.group('package'),
+ 'version': matches.group('version'),
+ })
+ self._sources = s
+ return s
+
+ @property
+ def binaries(self):
+ """ list of binary packages that were removed
+
+ A list of dicts is returned, each dict has the form:
+ {
+ 'package': 'some-package-name',
+ 'version': '1.2.3-1',
+ 'architectures': set(['i386', 'amd64'])
+ }
+ """
+ if hasattr(self, '_binaries'):
+ return self._binaries
+
+ b = []
+ if 'binaries' in self:
+ for line in self['binaries'].splitlines():
+ matches = self.__binaries_line_re.match(line)
+ if matches:
+ b.append({
+ 'package': matches.group('package'),
+ 'version': matches.group('version'),
+ 'architectures':
+ set(matches.group('archs').split(', ')),
+ })
+ self._binaries = b
+ return b
+
+
class _CaseInsensitiveString(str):
"""Case insensitive string.
"""
diff --git a/tests/test_deb822.py b/tests/test_deb822.py
index 2bf17af..41c7755 100755
--- a/tests/test_deb822.py
+++ b/tests/test_deb822.py
@@ -19,6 +19,7 @@
from __future__ import absolute_import
+import email.utils
import io
import os
import re
@@ -951,6 +952,30 @@ Description: python modules to work with Debian-related data formats
changes = deb822.Changes(f)
self.assertEqual('python-debian', changes['Source'])
+ def test_removals(self):
+ with open('test_removals.822') as f:
+ removals = deb822.Removals.iter_paragraphs(f)
+ r = next(removals)
+ self.assertEqual(r['suite'], 'unstable')
+ self.assertEqual(r['date'], u'Wed, 01 Jan 2014 17:03:54 +0000')
+ self.assertEqual(r.date.strftime('%s'), '1388595834')
+ self.assertEqual(len(r.binaries), 1)
+ self.assertEqual(r.binaries[0]['package'], 'libzoom-ruby')
+ self.assertEqual(r.binaries[0]['version'], '0.4.1-5')
+ self.assertEqual(r.binaries[0]['architectures'], set(['all']))
+ r = next(removals)
+ self.assertEqual(len(r.binaries), 3)
+ r = next(removals)
+ self.assertEqual(r['bug'], '753912')
+ self.assertEqual(r.bug, [753912])
+ self.assertEqual(r.also_wnpp, [123456])
+ r = next(removals)
+ self.assertEqual(r.binaries[0]['architectures'],
+ set(['amd64', 'armel', 'armhf', 'hurd-i386',
+ 'i386', 'kfreebsd-amd64', 'kfreebsd-i386',
+ 'mips', 'mipsel', 'powerpc', 's390x',
+ 'sparc']))
+
class TestPkgRelations(unittest.TestCase):
# TODO(jsw): Stop overriding this for Python versions that actually include
diff --git a/tests/test_removals.822 b/tests/test_removals.822
new file mode 100644
index 0000000..967453f
--- /dev/null
+++ b/tests/test_removals.822
@@ -0,0 +1,132 @@
+Date: Wed, 01 Jan 2014 17:03:54 +0000
+Ftpmaster: Ansgar Burchardt
+Suite: unstable
+Binaries:
+ libzoom-ruby_0.4.1-5 [all]
+Reason: [auto-cruft] no longer built from source
+
+Date: Mon, 07 Jul 2014 11:52:03 +0000
+Ftpmaster: Scott Kitterman
+Suite: experimental
+Binaries:
+ gnustep-back0.22-art_0.22.0-1 [amd64]
+ gnustep-back0.22-cairo_0.22.0-1 [amd64]
+ gnustep-gpbs_0.22.0-1 [amd64]
+Reason: [auto-cruft] NBS (no longer built by gnustep-back)
+
+Date: Mon, 07 Jul 2014 11:55:28 +0000
+Ftpmaster: Scott Kitterman
+Suite: unstable
+Sources:
+ redmine-plugin-markdown_2.0.1+git20130424-1
+Binaries:
+ redmine-plugin-markdown_2.0.1+git20130424-1 [all]
+Reason: ROM; obsolete
+Bug: 753912
+Also-WNPP: 123456
+
+Date: Thu, 31 Jul 2014 12:17:40 +0000
+Ftpmaster: Scott Kitterman
+Suite: unstable
+Sources:
+ yafaray_0.1.5-3
+Binaries:
+ yafaray_0.1.5-3+b2 [amd64, armel, armhf, hurd-i386, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+Reason: ROM; Dead upstream, unmaintained
+Bug: 756408
+Also-WNPP:
+
+Date: Fri, 01 Aug 2014 10:45:41 +0000
+Ftpmaster: Scott Kitterman
+Suite: experimental
+Sources:
+ baloo-widgets_4:4.13.2-1
+ brian_1.4.1-1
+ gwenview_4:4.13.1-1
+ kdepim_4:4.13.1-1
+ kdepim-runtime_4:4.13.1-1
+ nepomuk-core_4:4.13.1-1
+ pykde4_4:4.13.1-1
+Binaries:
+ akonadiconsole_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ akregator_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ blogilo_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ gwenview_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ gwenview-dbg_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kaddressbook_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kaddressbook-mobile_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kalarm_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kdepim_4:4.13.1-1 [all]
+ kdepim-dbg_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kdepim-kresources_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kdepim-mobile_4:4.13.1-1 [all]
+ kdepim-mobileui-data_4:4.13.1-1 [all]
+ kdepim-runtime_4:4.13.1-1 [amd64, armel, armhf, hurd-i386, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kdepim-runtime-dbg_4:4.13.1-1 [amd64, armel, armhf, hurd-i386, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kjots_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kleopatra_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kmail_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kmail-mobile_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ knode_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ knotes_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ konsolekalendar_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ kontact_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ korganizer_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ korganizer-mobile_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ ktimetracker_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libbaloowidgets-dev_4:4.13.2-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libbaloowidgets4_4:4.13.2-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libbaloowidgets4-dbg_4:4.13.2-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libcalendarsupport4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libcomposereditorng4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libeventviews4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libincidenceeditorsng4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libkdepim4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libkdepimdbusinterfaces4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libkdepimmobileui4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libkdgantt2-0_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libkleo4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libkmanagesieve4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libkpgp4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libksieve4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libksieveui4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libmailcommon4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libmailimporter4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libmessagecomposer4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libmessagecore4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libmessagelist4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libmessageviewer4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libnepomukcore4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libnoteshared4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libpimcommon4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libsendlater4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ libtemplateparser4_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ nepomuk-core-data_4:4.13.1-1 [all]
+ nepomuk-core-dbg_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ nepomuk-core-dev_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ nepomuk-core-runtime_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ notes-mobile_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ python-brian_1.4.1-1 [all]
+ python-brian-doc_1.4.1-1 [all]
+ python-brian-lib_1.4.1-1 [amd64, armel, armhf, hurd-i386, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ python-kde4_4:4.13.1-1 [amd64, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mipsel, powerpc, s390x]
+ python-kde4-dbg_4:4.13.1-1 [amd64, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mipsel, powerpc, s390x]
+ python-kde4-dev_4:4.13.1-1 [all]
+ python-kde4-doc_4:4.13.1-1 [all]
+ python3-pykde4_4:4.13.1-1 [amd64, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mipsel, powerpc, s390x]
+ python3-pykde4-dbg_4:4.13.1-1 [amd64, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mipsel, powerpc, s390x]
+ storageservicemanager_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+ tasks-mobile_4:4.13.1-1 [amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390x, sparc]
+Reason: [auto-cruft] NVIU
+
+Date: Fri, 01 Aug 2014 10:47:02 +0000
+Ftpmaster: Scott Kitterman
+Suite: unstable
+Sources:
+ mingw32-runtime_3.13-1
+Binaries:
+ mingw32-runtime_3.13-1 [all]
+Reason: ROM; obsolete, superceded by mingw-w64
+Bug: 756732
+Also-Bugs: 429313 498529 521758 607215 607218 609869 648306
+Also-WNPP:
--
2.1.4
