patches/squashfs-tools/0016-remove-frag_deflator_thread.patch

   1 From: Alexander Couzens <lynxis@fe80.eu>
   2 Date: Tue, 8 Jan 2019 10:57:00 +0100
   3 Subject: remove frag_deflator_thread
   4
   5 frag_deflator_thread compress fragments.
   6 Replace the deflator_thread with a function and
   7 use the function instead of the to_frag queue.
   8
   9 Updated Fri, 18 Jan 2019 19:22:58 +0000 by Chris Lamb <lamby@debian.org> to
  10 fix issue with lack of compressor initialisation affecting (at least) LZO
  11 compression.
  12 ---
  13  squashfs-tools/info.c       |  5 ----
  14  squashfs-tools/mksquashfs.c | 71 +++++++++++++++++----------------------------
  15  squashfs-tools/mksquashfs.h |  2 +-
  16  squashfs-tools/restore.c    | 15 ++--------
  17  4 files changed, 30 insertions(+), 63 deletions(-)
  18
  19 diff --git a/squashfs-tools/info.c b/squashfs-tools/info.c
  20 index 7968c77..028d578 100644
  21 --- a/squashfs-tools/info.c
  22 +++ b/squashfs-tools/info.c
  23 @@ -96,11 +96,6 @@ void dump_state()
  24         printf("compressed block queue (deflate thread(s) -> main thread)\n");
  25         dump_seq_queue(to_main, 0);
  26
  27 -       printf("uncompressed packed fragment queue (main thread -> fragment"
  28 -                                               " deflate thread(s))\n");
  29 -       dump_queue(to_frag);
  30 -
  31 -
  32         printf("locked frag queue (compressed frags waiting while multi-block"
  33                                                 " file is written)\n");
  34         dump_queue(locked_fragment);
  35 diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
  36 index 62888ec..09fb68a 100644
  37 --- a/squashfs-tools/mksquashfs.c
  38 +++ b/squashfs-tools/mksquashfs.c
  39 @@ -260,10 +260,10 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
  40  struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
  41  struct cache *bwriter_buffer, *fwriter_buffer;
  42  struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
  43 -       *to_frag, *locked_fragment, *to_process_frag;
  44 +       *locked_fragment, *to_process_frag;
  45  struct seq_queue *to_main;
  46  pthread_t reader_thread, writer_thread, main_thread;
  47 -pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
  48 +pthread_t *deflator_thread, *frag_thread;
  49  pthread_t *restore_thread = NULL;
  50  pthread_mutex_t        fragment_mutex = PTHREAD_MUTEX_INITIALIZER;
  51  pthread_mutex_t        pos_mutex = PTHREAD_MUTEX_INITIALIZER;
  52 @@ -309,7 +309,7 @@ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
  53  void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
  54  unsigned short get_checksum_mem(char *buff, int bytes);
  55  void check_usable_phys_mem(int total_mem);
  56 -
  57 +void frag_deflator(struct file_buffer *file_buffer);
  58
  59  void prep_exit()
  60  {
  61 @@ -1551,7 +1551,7 @@ void write_fragment(struct file_buffer *fragment)
  62         pthread_mutex_lock(&fragment_mutex);
  63         fragment_table[fragment->block].unused = 0;
  64         fragments_outstanding ++;
  65 -       queue_put(to_frag, fragment);
  66 +       frag_deflator(fragment);
  67         pthread_cleanup_pop(1);
  68  }
  69
  70 @@ -2423,51 +2423,39 @@ void *deflator(void *arg)
  71  }
  72
  73
  74 -void *frag_deflator(void *arg)
  75 +void frag_deflator(struct file_buffer *file_buffer)
  76  {
  77 -       void *stream = NULL;
  78         int res;
  79
  80         res = compressor_init(comp, &stream, block_size, 1);
  81         if(res)
  82                 BAD_ERROR("frag_deflator:: compressor_init failed\n");
  83
  84 -       pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  85 -
  86 -       while(1) {
  87 -               int c_byte, compressed_size;
  88 -               struct file_buffer *file_buffer = queue_get(to_frag);
  89 -               struct file_buffer *write_buffer =
  90 +       int c_byte, compressed_size;
  91 +       struct file_buffer *write_buffer =
  92                         cache_get(fwriter_buffer, file_buffer->block);
  93
  94 -               c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
  95 -                       file_buffer->size, block_size, noF, 1);
  96 -               compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
  97 -               write_buffer->size = compressed_size;
  98 -               pthread_mutex_lock(&fragment_mutex);
  99 -               if(fragments_locked == FALSE) {
 100 -                       fragment_table[file_buffer->block].size = c_byte;
 101 -                       fragment_table[file_buffer->block].start_block = bytes;
 102 -                       write_buffer->block = bytes;
 103 -                       bytes += compressed_size;
 104 -                       fragments_outstanding --;
 105 -                       queue_put(to_writer, write_buffer);
 106 -                       pthread_mutex_unlock(&fragment_mutex);
 107 -                       TRACE("Writing fragment %lld, uncompressed size %d, "
 108 -                               "compressed size %d\n", file_buffer->block,
 109 -                               file_buffer->size, compressed_size);
 110 -               } else {
 111 -                               add_pending_fragment(write_buffer, c_byte,
 112 -                                       file_buffer->block);
 113 -                               pthread_mutex_unlock(&fragment_mutex);
 114 -               }
 115 -               cache_block_put(file_buffer);
 116 +       c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
 117 +                        file_buffer->size, block_size, noF, 1);
 118 +       compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
 119 +       write_buffer->size = compressed_size;
 120 +       if(fragments_locked == FALSE) {
 121 +               fragment_table[file_buffer->block].size = c_byte;
 122 +               fragment_table[file_buffer->block].start_block = bytes;
 123 +               write_buffer->block = bytes;
 124 +               bytes += compressed_size;
 125 +               fragments_outstanding --;
 126 +               queue_put(to_writer, write_buffer);
 127 +               TRACE("Writing fragment %lld, uncompressed size %d, "
 128 +                     "compressed size %d\n", file_buffer->block,
 129 +                     file_buffer->size, compressed_size);
 130 +       } else {
 131 +               add_pending_fragment(write_buffer, c_byte,
 132 +                                    file_buffer->block);
 133         }
 134 -
 135 -       pthread_cleanup_pop(0);
 136 +       cache_block_put(file_buffer);
 137  }
 138
 139 -
 140  struct file_buffer *get_file_buffer()
 141  {
 142         struct file_buffer *file_buffer = seq_queue_get(to_main);
 143 @@ -4137,19 +4125,17 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
 144                         multiply_overflow(processors * 3, sizeof(pthread_t)))
 145                 BAD_ERROR("Processors too large\n");
 146
 147 -       deflator_thread = malloc(processors * 3 * sizeof(pthread_t));
 148 +       deflator_thread = malloc(processors * 2 * sizeof(pthread_t));
 149         if(deflator_thread == NULL)
 150                 MEM_ERROR();
 151
 152 -       frag_deflator_thread = &deflator_thread[processors];
 153 -       frag_thread = &frag_deflator_thread[processors];
 154 +       frag_thread = &deflator_thread[processors];
 155
 156         to_reader = queue_init(1);
 157         to_deflate = queue_init(reader_size);
 158         to_process_frag = queue_init(reader_size);
 159         to_writer = queue_init(bwriter_size + fwriter_size);
 160         from_writer = queue_init(1);
 161 -       to_frag = queue_init(fragment_size);
 162         locked_fragment = queue_init(fragment_size);
 163         to_main = seq_queue_init();
 164         reader_buffer = cache_init(block_size, reader_size, 0, 0);
 165 @@ -4165,9 +4151,6 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
 166         for(i = 0; i < processors; i++) {
 167                 if(pthread_create(&deflator_thread[i], NULL, deflator, NULL))
 168                         BAD_ERROR("Failed to create thread\n");
 169 -               if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
 170 -                               NULL) != 0)
 171 -                       BAD_ERROR("Failed to create thread\n");
 172                 if(pthread_create(&frag_thread[i], NULL, frag_thrd,
 173                                 (void *) destination_file) != 0)
 174                         BAD_ERROR("Failed to create thread\n");
 175 diff --git a/squashfs-tools/mksquashfs.h b/squashfs-tools/mksquashfs.h
 176 index 397e17c..47e089d 100644
 177 --- a/squashfs-tools/mksquashfs.h
 178 +++ b/squashfs-tools/mksquashfs.h
 179 @@ -134,7 +134,7 @@ struct append_file {
 180  extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
 181  struct cache *bwriter_buffer, *fwriter_buffer;
 182  extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
 183 -       *to_frag, *locked_fragment, *to_process_frag;
 184 +       *locked_fragment, *to_process_frag;
 185  extern struct append_file **file_mapping;
 186  extern struct seq_queue *to_main;
 187  extern pthread_mutex_t fragment_mutex, dup_mutex;
 188 diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c
 189 index 5e336b3..a7aaf2e 100644
 190 --- a/squashfs-tools/restore.c
 191 +++ b/squashfs-tools/restore.c
 192 @@ -47,8 +47,8 @@
 193  #define TRUE 1
 194
 195  extern pthread_t reader_thread, writer_thread, main_thread;
 196 -extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
 197 -extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag;
 198 +extern pthread_t *deflator_thread, *frag_thread;
 199 +extern struct queue *to_deflate, *to_writer, *to_process_frag;
 200  extern struct seq_queue *to_main;
 201  extern void restorefs();
 202  extern int processors;
 203 @@ -120,17 +120,6 @@ void *restore_thrd(void *arg)
 204                 pthread_cancel(main_thread);
 205                 pthread_join(main_thread, NULL);
 206
 207 -               /* then flush the main thread to fragment deflator thread(s)
 208 -                * queue.  The fragment deflator thread(s) will idle
 209 -                */
 210 -               queue_flush(to_frag);
 211 -
 212 -               /* now kill the fragment deflator thread(s) */
 213 -               for(i = 0; i < processors; i++)
 214 -                       pthread_cancel(frag_deflator_thread[i]);
 215 -               for(i = 0; i < processors; i++)
 216 -                       pthread_join(frag_deflator_thread[i], NULL);
 217 -
 218                 /*
 219                  * then flush the main thread/fragment deflator thread(s)
 220                  * to writer thread queue.  The writer thread will idle