zlib/examples/gzjoin.c

   1 /* gzjoin -- command to join gzip files into one gzip file
   2
   3   Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved
   4   version 1.2, 14 Aug 2012
   5
   6   This software is provided 'as-is', without any express or implied
   7   warranty.  In no event will the author be held liable for any damages
   8   arising from the use of this software.
   9
  10   Permission is granted to anyone to use this software for any purpose,
  11   including commercial applications, and to alter it and redistribute it
  12   freely, subject to the following restrictions:
  13
  14   1. The origin of this software must not be misrepresented; you must not
  15      claim that you wrote the original software. If you use this software
  16      in a product, an acknowledgment in the product documentation would be
  17      appreciated but is not required.
  18   2. Altered source versions must be plainly marked as such, and must not be
  19      misrepresented as being the original software.
  20   3. This notice may not be removed or altered from any source distribution.
  21
  22   Mark Adler    madler@alumni.caltech.edu
  23  */
  24
  25 /*
  26  * Change history:
  27  *
  28  * 1.0  11 Dec 2004     - First version
  29  * 1.1  12 Jun 2005     - Changed ssize_t to long for portability
  30  * 1.2  14 Aug 2012     - Clean up for z_const usage
  31  */
  32
  33 /*
  34    gzjoin takes one or more gzip files on the command line and writes out a
  35    single gzip file that will uncompress to the concatenation of the
  36    uncompressed data from the individual gzip files.  gzjoin does this without
  37    having to recompress any of the data and without having to calculate a new
  38    crc32 for the concatenated uncompressed data.  gzjoin does however have to
  39    decompress all of the input data in order to find the bits in the compressed
  40    data that need to be modified to concatenate the streams.
  41
  42    gzjoin does not do an integrity check on the input gzip files other than
  43    checking the gzip header and decompressing the compressed data.  They are
  44    otherwise assumed to be complete and correct.
  45
  46    Each joint between gzip files removes at least 18 bytes of previous trailer
  47    and subsequent header, and inserts an average of about three bytes to the
  48    compressed data in order to connect the streams.  The output gzip file
  49    has a minimal ten-byte gzip header with no file name or modification time.
  50
  51    This program was written to illustrate the use of the Z_BLOCK option of
  52    inflate() and the crc32_combine() function.  gzjoin will not compile with
  53    versions of zlib earlier than 1.2.3.
  54  */
  55
  56 #include <stdio.h>      /* fputs(), fprintf(), fwrite(), putc() */
  57 #include <stdlib.h>     /* exit(), malloc(), free() */
  58 #include <fcntl.h>      /* open() */
  59 #include <unistd.h>     /* close(), read(), lseek() */
  60 #include "zlib.h"
  61     /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */
  62
  63 #define local static
  64
  65 /* exit with an error (return a value to allow use in an expression) */
  66 local int bail(char *why1, char *why2)
  67 {
  68     fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2);
  69     exit(1);
  70     return 0;
  71 }
  72
  73 /* -- simple buffered file input with access to the buffer -- */
  74
  75 #define CHUNK 32768         /* must be a power of two and fit in unsigned */
  76
  77 /* bin buffered input file type */
  78 typedef struct {
  79     char *name;             /* name of file for error messages */
  80     int fd;                 /* file descriptor */
  81     unsigned left;          /* bytes remaining at next */
  82     unsigned char *next;    /* next byte to read */
  83     unsigned char *buf;     /* allocated buffer of length CHUNK */
  84 } bin;
  85
  86 /* close a buffered file and free allocated memory */
  87 local void bclose(bin *in)
  88 {
  89     if (in != NULL) {
  90         if (in->fd != -1)
  91             close(in->fd);
  92         if (in->buf != NULL)
  93             free(in->buf);
  94         free(in);
  95     }
  96 }
  97
  98 /* open a buffered file for input, return a pointer to type bin, or NULL on
  99    failure */
 100 local bin *bopen(char *name)
 101 {
 102     bin *in;
 103
 104     in = malloc(sizeof(bin));
 105     if (in == NULL)
 106         return NULL;
 107     in->buf = malloc(CHUNK);
 108     in->fd = open(name, O_RDONLY, 0);
 109     if (in->buf == NULL || in->fd == -1) {
 110         bclose(in);
 111         return NULL;
 112     }
 113     in->left = 0;
 114     in->next = in->buf;
 115     in->name = name;
 116     return in;
 117 }
 118
 119 /* load buffer from file, return -1 on read error, 0 or 1 on success, with
 120    1 indicating that end-of-file was reached */
 121 local int bload(bin *in)
 122 {
 123     long len;
 124
 125     if (in == NULL)
 126         return -1;
 127     if (in->left != 0)
 128         return 0;
 129     in->next = in->buf;
 130     do {
 131         len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left);
 132         if (len < 0)
 133             return -1;
 134         in->left += (unsigned)len;
 135     } while (len != 0 && in->left < CHUNK);
 136     return len == 0 ? 1 : 0;
 137 }
 138
 139 /* get a byte from the file, bail if end of file */
 140 #define bget(in) (in->left ? 0 : bload(in), \
 141                   in->left ? (in->left--, *(in->next)++) : \
 142                     bail("unexpected end of file on ", in->name))
 143
 144 /* get a four-byte little-endian unsigned integer from file */
 145 local unsigned long bget4(bin *in)
 146 {
 147     unsigned long val;
 148
 149     val = bget(in);
 150     val += (unsigned long)(bget(in)) << 8;
 151     val += (unsigned long)(bget(in)) << 16;
 152     val += (unsigned long)(bget(in)) << 24;
 153     return val;
 154 }
 155
 156 /* skip bytes in file */
 157 local void bskip(bin *in, unsigned skip)
 158 {
 159     /* check pointer */
 160     if (in == NULL)
 161         return;
 162
 163     /* easy case -- skip bytes in buffer */
 164     if (skip <= in->left) {
 165         in->left -= skip;
 166         in->next += skip;
 167         return;
 168     }
 169
 170     /* skip what's in buffer, discard buffer contents */
 171     skip -= in->left;
 172     in->left = 0;
 173
 174     /* seek past multiples of CHUNK bytes */
 175     if (skip > CHUNK) {
 176         unsigned left;
 177
 178         left = skip & (CHUNK - 1);
 179         if (left == 0) {
 180             /* exact number of chunks: seek all the way minus one byte to check
 181                for end-of-file with a read */
 182             lseek(in->fd, skip - 1, SEEK_CUR);
 183             if (read(in->fd, in->buf, 1) != 1)
 184                 bail("unexpected end of file on ", in->name);
 185             return;
 186         }
 187
 188         /* skip the integral chunks, update skip with remainder */
 189         lseek(in->fd, skip - left, SEEK_CUR);
 190         skip = left;
 191     }
 192
 193     /* read more input and skip remainder */
 194     bload(in);
 195     if (skip > in->left)
 196         bail("unexpected end of file on ", in->name);
 197     in->left -= skip;
 198     in->next += skip;
 199 }
 200
 201 /* -- end of buffered input functions -- */
 202
 203 /* skip the gzip header from file in */
 204 local void gzhead(bin *in)
 205 {
 206     int flags;
 207
 208     /* verify gzip magic header and compression method */
 209     if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8)
 210         bail(in->name, " is not a valid gzip file");
 211
 212     /* get and verify flags */
 213     flags = bget(in);
 214     if ((flags & 0xe0) != 0)
 215         bail("unknown reserved bits set in ", in->name);
 216
 217     /* skip modification time, extra flags, and os */
 218     bskip(in, 6);
 219
 220     /* skip extra field if present */
 221     if (flags & 4) {
 222         unsigned len;
 223
 224         len = bget(in);
 225         len += (unsigned)(bget(in)) << 8;
 226         bskip(in, len);
 227     }
 228
 229     /* skip file name if present */
 230     if (flags & 8)
 231         while (bget(in) != 0)
 232             ;
 233
 234     /* skip comment if present */
 235     if (flags & 16)
 236         while (bget(in) != 0)
 237             ;
 238
 239     /* skip header crc if present */
 240     if (flags & 2)
 241         bskip(in, 2);
 242 }
 243
 244 /* write a four-byte little-endian unsigned integer to out */
 245 local void put4(unsigned long val, FILE *out)
 246 {
 247     putc(val & 0xff, out);
 248     putc((val >> 8) & 0xff, out);
 249     putc((val >> 16) & 0xff, out);
 250     putc((val >> 24) & 0xff, out);
 251 }
 252
 253 /* Load up zlib stream from buffered input, bail if end of file */
 254 local void zpull(z_streamp strm, bin *in)
 255 {
 256     if (in->left == 0)
 257         bload(in);
 258     if (in->left == 0)
 259         bail("unexpected end of file on ", in->name);
 260     strm->avail_in = in->left;
 261     strm->next_in = in->next;
 262 }
 263
 264 /* Write header for gzip file to out and initialize trailer. */
 265 local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out)
 266 {
 267     fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
 268     *crc = crc32(0L, Z_NULL, 0);
 269     *tot = 0;
 270 }
 271
 272 /* Copy the compressed data from name, zeroing the last block bit of the last
 273    block if clr is true, and adding empty blocks as needed to get to a byte
 274    boundary.  If clr is false, then the last block becomes the last block of
 275    the output, and the gzip trailer is written.  crc and tot maintains the
 276    crc and length (modulo 2^32) of the output for the trailer.  The resulting
 277    gzip file is written to out.  gzinit() must be called before the first call
 278    of gzcopy() to write the gzip header and to initialize crc and tot. */
 279 local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot,
 280                   FILE *out)
 281 {
 282     int ret;                /* return value from zlib functions */
 283     int pos;                /* where the "last block" bit is in byte */
 284     int last;               /* true if processing the last block */
 285     bin *in;                /* buffered input file */
 286     unsigned char *start;   /* start of compressed data in buffer */
 287     unsigned char *junk;    /* buffer for uncompressed data -- discarded */
 288     z_off_t len;            /* length of uncompressed data (support > 4 GB) */
 289     z_stream strm;          /* zlib inflate stream */
 290
 291     /* open gzip file and skip header */
 292     in = bopen(name);
 293     if (in == NULL)
 294         bail("could not open ", name);
 295     gzhead(in);
 296
 297     /* allocate buffer for uncompressed data and initialize raw inflate
 298        stream */
 299     junk = malloc(CHUNK);
 300     strm.zalloc = Z_NULL;
 301     strm.zfree = Z_NULL;
 302     strm.opaque = Z_NULL;
 303     strm.avail_in = 0;
 304     strm.next_in = Z_NULL;
 305     ret = inflateInit2(&strm, -15);
 306     if (junk == NULL || ret != Z_OK)
 307         bail("out of memory", "");
 308
 309     /* inflate and copy compressed data, clear last-block bit if requested */
 310     len = 0;
 311     zpull(&strm, in);
 312     start = in->next;
 313     last = start[0] & 1;
 314     if (last && clr)
 315         start[0] &= ~1;
 316     strm.avail_out = 0;
 317     for (;;) {
 318         /* if input used and output done, write used input and get more */
 319         if (strm.avail_in == 0 && strm.avail_out != 0) {
 320             fwrite(start, 1, strm.next_in - start, out);
 321             start = in->buf;
 322             in->left = 0;
 323             zpull(&strm, in);
 324         }
 325
 326         /* decompress -- return early when end-of-block reached */
 327         strm.avail_out = CHUNK;
 328         strm.next_out = junk;
 329         ret = inflate(&strm, Z_BLOCK);
 330         switch (ret) {
 331         case Z_MEM_ERROR:
 332             bail("out of memory", "");
 333         case Z_DATA_ERROR:
 334             bail("invalid compressed data in ", in->name);
 335         }
 336
 337         /* update length of uncompressed data */
 338         len += CHUNK - strm.avail_out;
 339
 340         /* check for block boundary (only get this when block copied out) */
 341         if (strm.data_type & 128) {
 342             /* if that was the last block, then done */
 343             if (last)
 344                 break;
 345
 346             /* number of unused bits in last byte */
 347             pos = strm.data_type & 7;
 348
 349             /* find the next last-block bit */
 350             if (pos != 0) {
 351                 /* next last-block bit is in last used byte */
 352                 pos = 0x100 >> pos;
 353                 last = strm.next_in[-1] & pos;
 354                 if (last && clr)
 355                     in->buf[strm.next_in - in->buf - 1] &= ~pos;
 356             }
 357             else {
 358                 /* next last-block bit is in next unused byte */
 359                 if (strm.avail_in == 0) {
 360                     /* don't have that byte yet -- get it */
 361                     fwrite(start, 1, strm.next_in - start, out);
 362                     start = in->buf;
 363                     in->left = 0;
 364                     zpull(&strm, in);
 365                 }
 366                 last = strm.next_in[0] & 1;
 367                 if (last && clr)
 368                     in->buf[strm.next_in - in->buf] &= ~1;
 369             }
 370         }
 371     }
 372
 373     /* update buffer with unused input */
 374     in->left = strm.avail_in;
 375     in->next = in->buf + (strm.next_in - in->buf);
 376
 377     /* copy used input, write empty blocks to get to byte boundary */
 378     pos = strm.data_type & 7;
 379     fwrite(start, 1, in->next - start - 1, out);
 380     last = in->next[-1];
 381     if (pos == 0 || !clr)
 382         /* already at byte boundary, or last file: write last byte */
 383         putc(last, out);
 384     else {
 385         /* append empty blocks to last byte */
 386         last &= ((0x100 >> pos) - 1);       /* assure unused bits are zero */
 387         if (pos & 1) {
 388             /* odd -- append an empty stored block */
 389             putc(last, out);
 390             if (pos == 1)
 391                 putc(0, out);               /* two more bits in block header */
 392             fwrite("\0\0\xff\xff", 1, 4, out);
 393         }
 394         else {
 395             /* even -- append 1, 2, or 3 empty fixed blocks */
 396             switch (pos) {
 397             case 6:
 398                 putc(last | 8, out);
 399                 last = 0;
 400             case 4:
 401                 putc(last | 0x20, out);
 402                 last = 0;
 403             case 2:
 404                 putc(last | 0x80, out);
 405                 putc(0, out);
 406             }
 407         }
 408     }
 409
 410     /* update crc and tot */
 411     *crc = crc32_combine(*crc, bget4(in), len);
 412     *tot += (unsigned long)len;
 413
 414     /* clean up */
 415     inflateEnd(&strm);
 416     free(junk);
 417     bclose(in);
 418
 419     /* write trailer if this is the last gzip file */
 420     if (!clr) {
 421         put4(*crc, out);
 422         put4(*tot, out);
 423     }
 424 }
 425
 426 /* join the gzip files on the command line, write result to stdout */
 427 int main(int argc, char **argv)
 428 {
 429     unsigned long crc, tot;     /* running crc and total uncompressed length */
 430
 431     /* skip command name */
 432     argc--;
 433     argv++;
 434
 435     /* show usage if no arguments */
 436     if (argc == 0) {
 437         fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n",
 438               stderr);
 439         return 0;
 440     }
 441
 442     /* join gzip files on command line and write to stdout */
 443     gzinit(&crc, &tot, stdout);
 444     while (argc--)
 445         gzcopy(*argv++, argc, &crc, &tot, stdout);
 446
 447     /* done */
 448     return 0;
 449 }