doc/examples/04_compress_easy_mt.c

   1 // SPDX-License-Identifier: 0BSD
   2
   3 ///////////////////////////////////////////////////////////////////////////////
   4 //
   5 /// \file       04_compress_easy_mt.c
   6 /// \brief      Compress in multi-call mode using LZMA2 in multi-threaded mode
   7 ///
   8 /// Usage:      ./04_compress_easy_mt < INFILE > OUTFILE
   9 ///
  10 /// Example:    ./04_compress_easy_mt < foo > foo.xz
  11 //
  12 //  Author:     Lasse Collin
  13 //
  14 ///////////////////////////////////////////////////////////////////////////////
  15
  16 #include <stdbool.h>
  17 #include <stdlib.h>
  18 #include <stdio.h>
  19 #include <string.h>
  20 #include <errno.h>
  21 #include <lzma.h>
  22
  23
  24 static bool
  25 init_encoder(lzma_stream *strm)
  26 {
  27         // The threaded encoder takes the options as pointer to
  28         // a lzma_mt structure.
  29         lzma_mt mt = {
  30                 // No flags are needed.
  31                 .flags = 0,
  32
  33                 // Let liblzma determine a sane block size.
  34                 .block_size = 0,
  35
  36                 // Use no timeout for lzma_code() calls by setting timeout
  37                 // to zero. That is, sometimes lzma_code() might block for
  38                 // a long time (from several seconds to even minutes).
  39                 // If this is not OK, for example due to progress indicator
  40                 // needing updates, specify a timeout in milliseconds here.
  41                 // See the documentation of lzma_mt in lzma/container.h for
  42                 // information how to choose a reasonable timeout.
  43                 .timeout = 0,
  44
  45                 // Use the default preset (6) for LZMA2.
  46                 // To use a preset, filters must be set to NULL.
  47                 .preset = LZMA_PRESET_DEFAULT,
  48                 .filters = NULL,
  49
  50                 // Use CRC64 for integrity checking. See also
  51                 // 01_compress_easy.c about choosing the integrity check.
  52                 .check = LZMA_CHECK_CRC64,
  53         };
  54
  55         // Detect how many threads the CPU supports.
  56         mt.threads = lzma_cputhreads();
  57
  58         // If the number of CPU cores/threads cannot be detected,
  59         // use one thread. Note that this isn't the same as the normal
  60         // single-threaded mode as this will still split the data into
  61         // blocks and use more RAM than the normal single-threaded mode.
  62         // You may want to consider using lzma_easy_encoder() or
  63         // lzma_stream_encoder() instead of lzma_stream_encoder_mt() if
  64         // lzma_cputhreads() returns 0 or 1.
  65         if (mt.threads == 0)
  66                 mt.threads = 1;
  67
  68         // If the number of CPU cores/threads exceeds threads_max,
  69         // limit the number of threads to keep memory usage lower.
  70         // The number 8 is arbitrarily chosen and may be too low or
  71         // high depending on the compression preset and the computer
  72         // being used.
  73         //
  74         // FIXME: A better way could be to check the amount of RAM
  75         // (or available RAM) and use lzma_stream_encoder_mt_memusage()
  76         // to determine if the number of threads should be reduced.
  77         const uint32_t threads_max = 8;
  78         if (mt.threads > threads_max)
  79                 mt.threads = threads_max;
  80
  81         // Initialize the threaded encoder.
  82         lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);
  83
  84         if (ret == LZMA_OK)
  85                 return true;
  86
  87         const char *msg;
  88         switch (ret) {
  89         case LZMA_MEM_ERROR:
  90                 msg = "Memory allocation failed";
  91                 break;
  92
  93         case LZMA_OPTIONS_ERROR:
  94                 // We are no longer using a plain preset so this error
  95                 // message has been edited accordingly compared to
  96                 // 01_compress_easy.c.
  97                 msg = "Specified filter chain is not supported";
  98                 break;
  99
 100         case LZMA_UNSUPPORTED_CHECK:
 101                 msg = "Specified integrity check is not supported";
 102                 break;
 103
 104         default:
 105                 msg = "Unknown error, possibly a bug";
 106                 break;
 107         }
 108
 109         fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n",
 110                         msg, ret);
 111         return false;
 112 }
 113
 114
 115 // This function is identical to the one in 01_compress_easy.c.
 116 static bool
 117 compress(lzma_stream *strm, FILE *infile, FILE *outfile)
 118 {
 119         lzma_action action = LZMA_RUN;
 120
 121         uint8_t inbuf[BUFSIZ];
 122         uint8_t outbuf[BUFSIZ];
 123
 124         strm->next_in = NULL;
 125         strm->avail_in = 0;
 126         strm->next_out = outbuf;
 127         strm->avail_out = sizeof(outbuf);
 128
 129         while (true) {
 130                 if (strm->avail_in == 0 && !feof(infile)) {
 131                         strm->next_in = inbuf;
 132                         strm->avail_in = fread(inbuf, 1, sizeof(inbuf),
 133                                         infile);
 134
 135                         if (ferror(infile)) {
 136                                 fprintf(stderr, "Read error: %s\n",
 137                                                 strerror(errno));
 138                                 return false;
 139                         }
 140
 141                         if (feof(infile))
 142                                 action = LZMA_FINISH;
 143                 }
 144
 145                 lzma_ret ret = lzma_code(strm, action);
 146
 147                 if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
 148                         size_t write_size = sizeof(outbuf) - strm->avail_out;
 149
 150                         if (fwrite(outbuf, 1, write_size, outfile)
 151                                         != write_size) {
 152                                 fprintf(stderr, "Write error: %s\n",
 153                                                 strerror(errno));
 154                                 return false;
 155                         }
 156
 157                         strm->next_out = outbuf;
 158                         strm->avail_out = sizeof(outbuf);
 159                 }
 160
 161                 if (ret != LZMA_OK) {
 162                         if (ret == LZMA_STREAM_END)
 163                                 return true;
 164
 165                         const char *msg;
 166                         switch (ret) {
 167                         case LZMA_MEM_ERROR:
 168                                 msg = "Memory allocation failed";
 169                                 break;
 170
 171                         case LZMA_DATA_ERROR:
 172                                 msg = "File size limits exceeded";
 173                                 break;
 174
 175                         default:
 176                                 msg = "Unknown error, possibly a bug";
 177                                 break;
 178                         }
 179
 180                         fprintf(stderr, "Encoder error: %s (error code %u)\n",
 181                                         msg, ret);
 182                         return false;
 183                 }
 184         }
 185 }
 186
 187
 188 extern int
 189 main(void)
 190 {
 191         lzma_stream strm = LZMA_STREAM_INIT;
 192
 193         bool success = init_encoder(&strm);
 194         if (success)
 195                 success = compress(&strm, stdin, stdout);
 196
 197         lzma_end(&strm);
 198
 199         if (fclose(stdout)) {
 200                 fprintf(stderr, "Write error: %s\n", strerror(errno));
 201                 success = false;
 202         }
 203
 204         return success ? EXIT_SUCCESS : EXIT_FAILURE;
 205 }