On Sun, Dec 19, 2010 at 02:02:14AM +0100, Goswin von Brederlow wrote:
> Hi,
>
> it's been some month since this was reported. Is anyone working on this?
Nobody that I know of. Since I'm not using any package with multiple
tarballs myself I'm lacking a clear picture about the workflow.
Marco sent in patches for git-buildpackage (to recreate the tarballs) a
while ago which I'll attach here so they don't get lost. I'd welcome
feedback if this is how people would use multiple tarballs. E.g. I'm not
sure if we want to push all tarballs deltas onto the same pristine-tar
branch.
Cheers,
-- Guido
>From 2a9504a509e502e05e065d2e6f7132e4ac5567c1 Mon Sep 17 00:00:00 2001
From: Marco Nenciarini <mnen...@debian.org>
Date: Tue, 18 May 2010 18:06:24 +0200
Subject: [PATCH 1/2] Initial implementation of multi orig support
---
gbp/deb.py | 39 +++++++++++++++++++++++++++++++++++++++
gbp/git.py | 10 ++++++++++
git-buildpackage | 51 ++++++++++++++++++++++++++++++++++++++++++---------
3 files changed, 91 insertions(+), 9 deletions(-)
diff --git a/gbp/deb.py b/gbp/deb.py
index b50abbd..df3e25e 100644
--- a/gbp/deb.py
+++ b/gbp/deb.py
@@ -161,6 +161,30 @@ def orig_file(cp, compression):
return "%s_%s.orig.tar.%s" % (cp['Source'], cp['Upstream-Version'], ext)
+def orig_components(main, filelist):
+ """Given an orig tarball name and a list of files, this method
+ finds component tarballs in the list that belong to the first one.
+ @return dict(component_name=file_name, ...)
+ """
+ (path, name) = os.path.split(main)
+ if not name:
+ return False
+
+ m = re.search(r'^(?P<source>.+).orig.tar.(gz|bz2)', name)
+ if not m:
+ return False
+
+ list = {}
+ pattern = re.compile(r'%s.orig-(?P<component>[\w-]+).tar.(gz|bz2)' % (m.group('source')))
+ for f in filelist:
+ (path, name) = os.path.split(f)
+ m = pattern.search(name)
+ if m:
+ list[m.group('component')]=f
+
+ return list
+
+
def is_native(cp):
"Is this a debian native package"
return [ True, False ]['-' in cp['Version']]
@@ -204,6 +228,21 @@ def symlink_orig(cp, compression, orig_dir, output_dir, force=False):
os.symlink(src, dst)
except OSError:
return False
+
+ if not os.access(orig_dir, os.F_OK):
+ return False
+ for f in orig_components(src, os.listdir(orig_dir)).values():
+ src = os.path.join(orig_dir, f)
+ dst = os.path.join(output_dir, f)
+ if not os.access(src, os.F_OK):
+ return False
+ try:
+ if os.access(dst, os.F_OK) and force:
+ os.unlink(dst)
+ os.symlink(src, dst)
+ except OSError:
+ return False
+
return True
def do_uscan():
diff --git a/gbp/git.py b/gbp/git.py
index 7743bdd..30a2bb0 100644
--- a/gbp/git.py
+++ b/gbp/git.py
@@ -136,6 +136,16 @@ class GitRepository(object):
else:
return []
+ def branch_files(self, branch):
+ """List files in a branch"""
+ out, ret = self.__git_getoutput('ls-tree', ['-z','--name-only', '-r', branch])
+ if ret:
+ raise GitRepositoryError, "Error listing files %d" % ret
+ if out:
+ return [ file for file in out[0].split('\0') if file ]
+ else:
+ return []
+
def commits(self, since=None, until=None, paths=None, options=None):
"""get commits from start to end touching pathds"""
diff --git a/git-buildpackage b/git-buildpackage
index 7f2b2d4..75362a6 100755
--- a/git-buildpackage
+++ b/git-buildpackage
@@ -23,6 +23,7 @@ import errno
import os, os.path
import pipes
import sys
+import re
import time
import gbp.deb as du
from gbp.git import (GitRepositoryError, GitRepository, build_tag)
@@ -82,12 +83,13 @@ def dump_tree(export_dir, treeish):
def move_old_export(target):
- """move a build tree away if it exists"""
+ """move a build tree away if it exists. This function also create the target directory"""
try:
os.mkdir(target)
except OSError, (e, msg):
if e == errno.EEXIST:
os.rename(target, "%s.obsolete.%s" % (target, time.time()))
+ os.mkdir(target) # this time it can't fail
def prepare_output_dir(dir):
@@ -113,6 +115,14 @@ def pristine_tar_build_orig(repo, cp, output_dir, options):
if not repo.has_branch(pt.branch):
print >>sys.stderr, 'Pristine-tar branch "%s" not found' % pt.branch
pt.checkout(os.path.join(output_dir, du.orig_file(cp, options.comp_type)))
+
+ # find all files in pristine-tar branch
+ filelist = [ os.path.splitext(f) for f in repo.branch_files(pt.branch) ]
+ filelist = [ f[0] for f in filelist if f[1] == '.id' ]
+
+ # now build the required extra files
+ for f in du.orig_components(du.orig_file(cp, options.comp_type), filelist).values():
+ pt.checkout(os.path.join(output_dir, f))
return True
else:
return False
@@ -155,13 +165,8 @@ def drop_index(repo):
if os.path.exists(wc_index):
os.unlink(wc_index)
-def extract_orig(orig_tarball, dest_dir):
- """extract orig tarball to export dir before exporting from git"""
- print "Extracting %s to '%s'" % (os.path.basename(orig_tarball), dest_dir)
-
- move_old_export(dest_dir)
- du.unpack_orig(orig_tarball, dest_dir, '')
-
+def remove_toplevel(dest_dir):
+ """remove leading directory of a tar archive"""
# Check if tarball extracts into a single folder or not:
tar_topdir = du.tar_toplevel(dest_dir)
if tar_topdir != dest_dir:
@@ -174,6 +179,33 @@ def extract_orig(orig_tarball, dest_dir):
# Remove that single folder:
os.rmdir(tar_topdir)
+def extract_orig(orig_tarball, dest_dir, filter):
+ """extract orig tarball to export dir before exporting from git"""
+ print "Extracting %s to '%s'" % (os.path.basename(orig_tarball), dest_dir)
+
+ # move_old_export(dest_dir) was pushed up to the caller because we need to
+ # be able to extract an orig_tarball without side effects
+
+ # Make sure the destination dir exists and is empty
+ RemoveTree(dest_dir)()
+ os.mkdir(dest_dir)
+
+ # unpack the main tarball
+ du.unpack_orig(orig_tarball, dest_dir, filter)
+
+ # make sure to remove the top level folder if there is only one
+ remove_toplevel(dest_dir)
+
+ # eventually unpack the additional tarballs
+ src_dir = os.path.dirname(orig_tarball)
+ for c, f in du.orig_components(orig_tarball, os.listdir(src_dir)).iteritems():
+ component_dir = os.path.join(dest_dir, c)
+ print "Extracting %s to '%s'" % (f, component_dir)
+ RemoveTree(component_dir)()
+ os.mkdir(component_dir)
+ du.unpack_orig(os.path.join(src_dir, f), component_dir, filter)
+ remove_toplevel(component_dir)
+
def main(argv):
changelog = 'debian/changelog'
retval = 0
@@ -333,7 +365,8 @@ def main(argv):
if options.overlay:
if du.is_native(cp):
raise GbpError, "Cannot overlay Debian native package"
- extract_orig(os.path.join(output_dir, du.orig_file(cp, options.comp_type)), tmp_dir)
+ move_old_export(tmp_dir)
+ extract_orig(os.path.join(output_dir, du.orig_file(cp, options.comp_type)), tmp_dir, '')
print "Exporting '%s' to '%s'" % (options.export, tmp_dir)
dump_tree(tmp_dir, tree)
--
1.7.1
>From 8d592fcd4ad9f757defa901eb64e24725de91b5e Mon Sep 17 00:00:00 2001
From: Marco Nenciarini <mnen...@debian.org>
Date: Tue, 18 May 2010 19:12:59 +0200
Subject: [PATCH 2/2] Move extract_orig to gbp/deb.py to facilitate its use in git-import-*
---
gbp/deb.py | 41 +++++++++++++++++++++++++++++++++++++++++
git-buildpackage | 43 +------------------------------------------
2 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/gbp/deb.py b/gbp/deb.py
index df3e25e..b682b93 100644
--- a/gbp/deb.py
+++ b/gbp/deb.py
@@ -272,6 +272,47 @@ def do_uscan():
return (True, None)
return (True, tarball)
+def remove_toplevel(dest_dir):
+ """remove leading directory of a tar archive"""
+ # Check if tarball extracts into a single folder or not:
+ tar_topdir = tar_toplevel(dest_dir)
+ if tar_topdir != dest_dir:
+ # If it extracts a single folder, move all of its contents to dest_dir:
+ r = glob.glob("%s/*" % tar_topdir)
+ r.extend(glob.glob("%s/.*" % tar_topdir)) # include hidden files and folders
+ for f in r:
+ os.rename(f, os.path.join(dest_dir, os.path.basename(f)))
+
+ # Remove that single folder:
+ os.rmdir(tar_topdir)
+
+def extract_orig(orig_tarball, dest_dir, filter):
+ """extract orig tarball to export dir before exporting from git"""
+ print "Extracting %s to '%s'" % (os.path.basename(orig_tarball), dest_dir)
+
+ # move_old_export(dest_dir) was pushed up to the caller because we need to
+ # be able to extract an orig_tarball without side effects
+
+ # Make sure the destination dir exists and is empty
+ gbpc.RemoveTree(dest_dir)()
+ os.mkdir(dest_dir)
+
+ # unpack the main tarball
+ unpack_orig(orig_tarball, dest_dir, filter)
+
+ # make sure to remove the top level folder if there is only one
+ remove_toplevel(dest_dir)
+
+ # eventually unpack the additional tarballs
+ src_dir = os.path.dirname(orig_tarball)
+ for c, f in orig_components(orig_tarball, os.listdir(src_dir)).iteritems():
+ component_dir = os.path.join(dest_dir, c)
+ print "Extracting %s to '%s'" % (f, component_dir)
+ gbpc.RemoveTree(component_dir)()
+ os.mkdir(component_dir)
+ unpack_orig(os.path.join(src_dir, f), component_dir, filter)
+ remove_toplevel(component_dir)
+
def unpack_orig(archive, tmpdir, filters):
"""
unpack a .orig.tar.gz to tmpdir, leave the cleanup to the caller in case of
diff --git a/git-buildpackage b/git-buildpackage
index 75362a6..234a588 100755
--- a/git-buildpackage
+++ b/git-buildpackage
@@ -165,47 +165,6 @@ def drop_index(repo):
if os.path.exists(wc_index):
os.unlink(wc_index)
-def remove_toplevel(dest_dir):
- """remove leading directory of a tar archive"""
- # Check if tarball extracts into a single folder or not:
- tar_topdir = du.tar_toplevel(dest_dir)
- if tar_topdir != dest_dir:
- # If it extracts a single folder, move all of its contents to dest_dir:
- r = glob("%s/*" % tar_topdir)
- r.extend(glob("%s/.*" % tar_topdir)) # include hidden files and folders
- for f in r:
- os.rename(f, os.path.join(dest_dir, os.path.basename(f)))
-
- # Remove that single folder:
- os.rmdir(tar_topdir)
-
-def extract_orig(orig_tarball, dest_dir, filter):
- """extract orig tarball to export dir before exporting from git"""
- print "Extracting %s to '%s'" % (os.path.basename(orig_tarball), dest_dir)
-
- # move_old_export(dest_dir) was pushed up to the caller because we need to
- # be able to extract an orig_tarball without side effects
-
- # Make sure the destination dir exists and is empty
- RemoveTree(dest_dir)()
- os.mkdir(dest_dir)
-
- # unpack the main tarball
- du.unpack_orig(orig_tarball, dest_dir, filter)
-
- # make sure to remove the top level folder if there is only one
- remove_toplevel(dest_dir)
-
- # eventually unpack the additional tarballs
- src_dir = os.path.dirname(orig_tarball)
- for c, f in du.orig_components(orig_tarball, os.listdir(src_dir)).iteritems():
- component_dir = os.path.join(dest_dir, c)
- print "Extracting %s to '%s'" % (f, component_dir)
- RemoveTree(component_dir)()
- os.mkdir(component_dir)
- du.unpack_orig(os.path.join(src_dir, f), component_dir, filter)
- remove_toplevel(component_dir)
-
def main(argv):
changelog = 'debian/changelog'
retval = 0
@@ -366,7 +325,7 @@ def main(argv):
if du.is_native(cp):
raise GbpError, "Cannot overlay Debian native package"
move_old_export(tmp_dir)
- extract_orig(os.path.join(output_dir, du.orig_file(cp, options.comp_type)), tmp_dir, '')
+ du.extract_orig(os.path.join(output_dir, du.orig_file(cp, options.comp_type)), tmp_dir, '')
print "Exporting '%s' to '%s'" % (options.export, tmp_dir)
dump_tree(tmp_dir, tree)
--
1.7.1
