bin/lo-pack-sources | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)
New commits: commit cb9883b1e9fb620f280e950de2bb50f9ec3a4c5e Author: Sebastian Andrzej Siewior <[email protected]> AuthorDate: Tue Jan 24 23:00:44 2023 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Sat Feb 11 22:45:18 2023 +0000 lo-pack-sources: Use threaded compression for xz. The xz utility can compress in parallel by using more CPU cores. The more CPUs are available the quicker it gets. The option -T0 automatically detects the number of available CPUs and uses all of them. Additional benefit is that xz starting with v5.4 can decompress the archive using multiple CPUs as well. The parallel compression works by splitting the input in multiple blocks of equal size which are compressed in parallel. Since the state is not preserved / reuse across blocks, the compression gets a little worse: -6 255M (current default) -T0 261M -eT0 259M -7T0 251M -7eT0 249M -7 uses larger blocks and dictionary which requires more memory on the compressing and decompressing side vs the current default -6. The -e option spends some extra cycles and improves the compression a bit. Use parallel compression and spend some extra cycles. Signed-off-by: Sebastian Andrzej Siewior <[email protected]> Change-Id: Ib69c4bc1996ddd52153b8ac27b653c21e3998a68 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/146793 Tested-by: Jenkins Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/lo-pack-sources b/bin/lo-pack-sources index b4150ab87ad7..289a595b06e0 100755 --- a/bin/lo-pack-sources +++ b/bin/lo-pack-sources @@ -283,10 +283,22 @@ sub pack_module_sources($$$$) sub generate_module_tarball($$$$$$$$) { my ($source_dir, $release_version, $module, $md5, $bzip2, $xz, $lo_topdir_name, $module_tarball_name) = @_; + my $PARALLELISM = $ENV{'PARALLELISM'}; + my $CPUS_XZ = 0; + + # Set CPUS_ to the number of CPUs that should be used. If PARALLELISM + # is set then this is used otherwise autodetect is used. + if (defined $PARALLELISM) { + if ($PARALLELISM > 0) { + $CPUS_XZ = $PARALLELISM; + } else { + $CPUS_XZ = 1; + } + } my $temp_dir = prepare_module_sources($source_dir, $release_version, $module, $lo_topdir_name); pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.bz2", "bzip2 -z --best > ") if (defined $bzip2); - pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "xz -z > ") if (defined $xz); + pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "xz -z -T$CPUS_XZ -e > ") if (defined $xz); remove_tempdir($temp_dir); } commit 30f49b56ba6712fb8fb3dd8febb870a1157e1516 Author: Sebastian Andrzej Siewior <[email protected]> AuthorDate: Fri Feb 10 21:49:25 2023 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Sat Feb 11 22:45:10 2023 +0000 lo-pack-sources: Split archiver invocation away from tar. Instead of passing --bzip2 or --xz invoke the compression program via a pipe. This make the replacement of compressor as well its options easier. Signed-off-by: Sebastian Andrzej Siewior <[email protected]> Change-Id: Ib69c4bc1996ddd52153b8ac27b653c21e3998a67 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/146105 Tested-by: Jenkins Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/lo-pack-sources b/bin/lo-pack-sources index 8c795dc170c5..b4150ab87ad7 100755 --- a/bin/lo-pack-sources +++ b/bin/lo-pack-sources @@ -155,11 +155,11 @@ sub generate_sources_version_file($$) sub generate_tarball($$$) { - my ($dir, $tarball, $tar_compress_option) = @_; + my ($dir, $tarball, $compressor_option) = @_; print "Creating $tarball..."; # generate the tarball in the current directory; avoid "./" prefix in the stored paths; show progress - system ("tar -c $tar_compress_option -f $tarball -C $dir $lo_topdir_name") && + system ("tar -c -C $dir $lo_topdir_name | $compressor_option $tarball") && die "Error: releasing failed: $!\n"; print "\n"; } @@ -274,9 +274,9 @@ sub prepare_module_sources($$$$) sub pack_module_sources($$$$) { - my ($temp_dir, $md5, $tarball, $tar_compress_option) = @_; + my ($temp_dir, $md5, $tarball, $compressor_option) = @_; - generate_tarball($temp_dir, $tarball, $tar_compress_option); + generate_tarball($temp_dir, $tarball, $compressor_option); generate_md5($tarball) if (defined $md5); } @@ -285,8 +285,8 @@ sub generate_module_tarball($$$$$$$$) my ($source_dir, $release_version, $module, $md5, $bzip2, $xz, $lo_topdir_name, $module_tarball_name) = @_; my $temp_dir = prepare_module_sources($source_dir, $release_version, $module, $lo_topdir_name); - pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.bz2", "--bzip2") if (defined $bzip2); - pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "--xz") if (defined $xz); + pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.bz2", "bzip2 -z --best > ") if (defined $bzip2); + pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "xz -z > ") if (defined $xz); remove_tempdir($temp_dir); }
