Package: squashfs-tools
Version: 1:4.3-9
Severity: wishlist
Tags: patch
User: reproducible-bui...@lists.alioth.debian.org
Usertags: toolchain randomness
X-Debbugs-Cc: reproducible-b...@lists.alioth.debian.org, Alexander Couzens
<lyn...@fe80.eu>
Hi Laszlo,
Re. <https://bugs.debian.org/918480#32> and similar, please find
attached the following patch that I have rebased against
squashfs-tools 4.3-9:
From: Alexander Couzens <lyn...@fe80.eu>
Date: Tue, 8 Jan 2019 10:57:00 +0100
Subject: remove frag_deflator_thread
frag_deflator_thread compress fragments.
Replace the deflator_thread with a function and
use the function instead of the to_frag queue.
---
squashfs-tools/info.c | 5 ---
squashfs-tools/mksquashfs.c | 76
++++++++++++++++-----------------------------
squashfs-tools/mksquashfs.h | 2 +-
squashfs-tools/restore.c | 15 ++-------
4 files changed, 30 insertions(+), 68 deletions(-)
This, I believe, is the final patch required for reproducible squashfs
images at least for Tails [0] and would mean we could drop our custom
package in that derivative distribution.
[0] https://tails.boum.org/
Best wishes,
--
,''`.
: :' : Chris Lamb
`. `'` la...@debian.org / chris-lamb.co.uk
`-
From: Alexander Couzens <lyn...@fe80.eu>
Date: Tue, 8 Jan 2019 10:57:00 +0100
Subject: remove frag_deflator_thread
frag_deflator_thread compress fragments.
Replace the deflator_thread with a function and
use the function instead of the to_frag queue.
---
squashfs-tools/info.c | 5 ---
squashfs-tools/mksquashfs.c | 76 ++++++++++++++++-----------------------------
squashfs-tools/mksquashfs.h | 2 +-
squashfs-tools/restore.c | 15 ++-------
4 files changed, 30 insertions(+), 68 deletions(-)
diff --git a/squashfs-tools/info.c b/squashfs-tools/info.c
index 7968c77..028d578 100644
--- a/squashfs-tools/info.c
+++ b/squashfs-tools/info.c
@@ -96,11 +96,6 @@ void dump_state()
printf("compressed block queue (deflate thread(s) -> main thread)\n");
dump_seq_queue(to_main, 0);
- printf("uncompressed packed fragment queue (main thread -> fragment"
- " deflate thread(s))\n");
- dump_queue(to_frag);
-
-
printf("locked frag queue (compressed frags waiting while multi-block"
" file is written)\n");
dump_queue(locked_fragment);
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
index 62888ec..465f9fe 100644
--- a/squashfs-tools/mksquashfs.c
+++ b/squashfs-tools/mksquashfs.c
@@ -260,10 +260,10 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
struct cache *bwriter_buffer, *fwriter_buffer;
struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
- *to_frag, *locked_fragment, *to_process_frag;
+ *locked_fragment, *to_process_frag;
struct seq_queue *to_main;
pthread_t reader_thread, writer_thread, main_thread;
-pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
+pthread_t *deflator_thread, *frag_thread;
pthread_t *restore_thread = NULL;
pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -309,7 +309,7 @@ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
unsigned short get_checksum_mem(char *buff, int bytes);
void check_usable_phys_mem(int total_mem);
-
+void frag_deflator(struct file_buffer *file_buffer);
void prep_exit()
{
@@ -1551,7 +1551,7 @@ void write_fragment(struct file_buffer *fragment)
pthread_mutex_lock(&fragment_mutex);
fragment_table[fragment->block].unused = 0;
fragments_outstanding ++;
- queue_put(to_frag, fragment);
+ frag_deflator(fragment);
pthread_cleanup_pop(1);
}
@@ -2423,51 +2423,34 @@ void *deflator(void *arg)
}
-void *frag_deflator(void *arg)
+void frag_deflator(struct file_buffer *file_buffer)
{
- void *stream = NULL;
- int res;
-
- res = compressor_init(comp, &stream, block_size, 1);
- if(res)
- BAD_ERROR("frag_deflator:: compressor_init failed\n");
- pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
-
- while(1) {
- int c_byte, compressed_size;
- struct file_buffer *file_buffer = queue_get(to_frag);
- struct file_buffer *write_buffer =
+ int c_byte, compressed_size;
+ struct file_buffer *write_buffer =
cache_get(fwriter_buffer, file_buffer->block);
- c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
- file_buffer->size, block_size, noF, 1);
- compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
- write_buffer->size = compressed_size;
- pthread_mutex_lock(&fragment_mutex);
- if(fragments_locked == FALSE) {
- fragment_table[file_buffer->block].size = c_byte;
- fragment_table[file_buffer->block].start_block = bytes;
- write_buffer->block = bytes;
- bytes += compressed_size;
- fragments_outstanding --;
- queue_put(to_writer, write_buffer);
- pthread_mutex_unlock(&fragment_mutex);
- TRACE("Writing fragment %lld, uncompressed size %d, "
- "compressed size %d\n", file_buffer->block,
- file_buffer->size, compressed_size);
- } else {
- add_pending_fragment(write_buffer, c_byte,
- file_buffer->block);
- pthread_mutex_unlock(&fragment_mutex);
- }
- cache_block_put(file_buffer);
+ c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
+ file_buffer->size, block_size, noF, 1);
+ compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
+ write_buffer->size = compressed_size;
+ if(fragments_locked == FALSE) {
+ fragment_table[file_buffer->block].size = c_byte;
+ fragment_table[file_buffer->block].start_block = bytes;
+ write_buffer->block = bytes;
+ bytes += compressed_size;
+ fragments_outstanding --;
+ queue_put(to_writer, write_buffer);
+ TRACE("Writing fragment %lld, uncompressed size %d, "
+ "compressed size %d\n", file_buffer->block,
+ file_buffer->size, compressed_size);
+ } else {
+ add_pending_fragment(write_buffer, c_byte,
+ file_buffer->block);
}
-
- pthread_cleanup_pop(0);
+ cache_block_put(file_buffer);
}
-
struct file_buffer *get_file_buffer()
{
struct file_buffer *file_buffer = seq_queue_get(to_main);
@@ -4137,19 +4120,17 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
multiply_overflow(processors * 3, sizeof(pthread_t)))
BAD_ERROR("Processors too large\n");
- deflator_thread = malloc(processors * 3 * sizeof(pthread_t));
+ deflator_thread = malloc(processors * 2 * sizeof(pthread_t));
if(deflator_thread == NULL)
MEM_ERROR();
- frag_deflator_thread = &deflator_thread[processors];
- frag_thread = &frag_deflator_thread[processors];
+ frag_thread = &deflator_thread[processors];
to_reader = queue_init(1);
to_deflate = queue_init(reader_size);
to_process_frag = queue_init(reader_size);
to_writer = queue_init(bwriter_size + fwriter_size);
from_writer = queue_init(1);
- to_frag = queue_init(fragment_size);
locked_fragment = queue_init(fragment_size);
to_main = seq_queue_init();
reader_buffer = cache_init(block_size, reader_size, 0, 0);
@@ -4165,9 +4146,6 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
for(i = 0; i < processors; i++) {
if(pthread_create(&deflator_thread[i], NULL, deflator, NULL))
BAD_ERROR("Failed to create thread\n");
- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
- NULL) != 0)
- BAD_ERROR("Failed to create thread\n");
if(pthread_create(&frag_thread[i], NULL, frag_thrd,
(void *) destination_file) != 0)
BAD_ERROR("Failed to create thread\n");
diff --git a/squashfs-tools/mksquashfs.h b/squashfs-tools/mksquashfs.h
index 397e17c..47e089d 100644
--- a/squashfs-tools/mksquashfs.h
+++ b/squashfs-tools/mksquashfs.h
@@ -134,7 +134,7 @@ struct append_file {
extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
struct cache *bwriter_buffer, *fwriter_buffer;
extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
- *to_frag, *locked_fragment, *to_process_frag;
+ *locked_fragment, *to_process_frag;
extern struct append_file **file_mapping;
extern struct seq_queue *to_main;
extern pthread_mutex_t fragment_mutex, dup_mutex;
diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c
index 5e336b3..a7aaf2e 100644
--- a/squashfs-tools/restore.c
+++ b/squashfs-tools/restore.c
@@ -47,8 +47,8 @@
#define TRUE 1
extern pthread_t reader_thread, writer_thread, main_thread;
-extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
-extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag;
+extern pthread_t *deflator_thread, *frag_thread;
+extern struct queue *to_deflate, *to_writer, *to_process_frag;
extern struct seq_queue *to_main;
extern void restorefs();
extern int processors;
@@ -120,17 +120,6 @@ void *restore_thrd(void *arg)
pthread_cancel(main_thread);
pthread_join(main_thread, NULL);
- /* then flush the main thread to fragment deflator thread(s)
- * queue. The fragment deflator thread(s) will idle
- */
- queue_flush(to_frag);
-
- /* now kill the fragment deflator thread(s) */
- for(i = 0; i < processors; i++)
- pthread_cancel(frag_deflator_thread[i]);
- for(i = 0; i < processors; i++)
- pthread_join(frag_deflator_thread[i], NULL);
-
/*
* then flush the main thread/fragment deflator thread(s)
* to writer thread queue. The writer thread will idle
