zlib/examples/gzjoin.c

   1 /* gzjoin -- command to join gzip files into one gzip file
   2
   3   Copyright (C) 2004 Mark Adler, all rights reserved
   4   version 1.0, 11 Dec 2004
   5
   6   This software is provided 'as-is', without any express or implied
   7   warranty.  In no event will the author be held liable for any damages
   8   arising from the use of this software.
   9
  10   Permission is granted to anyone to use this software for any purpose,
  11   including commercial applications, and to alter it and redistribute it
  12   freely, subject to the following restrictions:
  13
  14   1. The origin of this software must not be misrepresented; you must not
  15      claim that you wrote the original software. If you use this software
  16      in a product, an acknowledgment in the product documentation would be
  17      appreciated but is not required.
  18   2. Altered source versions must be plainly marked as such, and must not be
  19      misrepresented as being the original software.
  20   3. This notice may not be removed or altered from any source distribution.
  21
  22   Mark Adler    madler@alumni.caltech.edu
  23  */
  24
  25 /*
  26  * Change history:
  27  *
  28  * 1.0  11 Dec 2004     - First version
  29  * 1.1  12 Jun 2005     - Changed ssize_t to long for portability
  30  */
  31
  32 /*
  33    gzjoin takes one or more gzip files on the command line and writes out a
  34    single gzip file that will uncompress to the concatenation of the
  35    uncompressed data from the individual gzip files.  gzjoin does this without
  36    having to recompress any of the data and without having to calculate a new
  37    crc32 for the concatenated uncompressed data.  gzjoin does however have to
  38    decompress all of the input data in order to find the bits in the compressed
  39    data that need to be modified to concatenate the streams.
  40
  41    gzjoin does not do an integrity check on the input gzip files other than
  42    checking the gzip header and decompressing the compressed data.  They are
  43    otherwise assumed to be complete and correct.
  44
  45    Each joint between gzip files removes at least 18 bytes of previous trailer
  46    and subsequent header, and inserts an average of about three bytes to the
  47    compressed data in order to connect the streams.  The output gzip file
  48    has a minimal ten-byte gzip header with no file name or modification time.
  49
  50    This program was written to illustrate the use of the Z_BLOCK option of
  51    inflate() and the crc32_combine() function.  gzjoin will not compile with
  52    versions of zlib earlier than 1.2.3.
  53  */
  54
  55 #include <stdio.h>      /* fputs(), fprintf(), fwrite(), putc() */
  56 #include <stdlib.h>     /* exit(), malloc(), free() */
  57 #include <fcntl.h>      /* open() */
  58 #include <unistd.h>     /* close(), read(), lseek() */
  59 #include "zlib.h"
  60     /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */
  61
  62 #define local static
  63
  64 /* exit with an error (return a value to allow use in an expression) */
  65 local int bail(char *why1, char *why2)
  66 {
  67     fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2);
  68     exit(1);
  69     return 0;
  70 }
  71
  72 /* -- simple buffered file input with access to the buffer -- */
  73
  74 #define CHUNK 32768         /* must be a power of two and fit in unsigned */
  75
  76 /* bin buffered input file type */
  77 typedef struct {
  78     char *name;             /* name of file for error messages */
  79     int fd;                 /* file descriptor */
  80     unsigned left;          /* bytes remaining at next */
  81     unsigned char *next;    /* next byte to read */
  82     unsigned char *buf;     /* allocated buffer of length CHUNK */
  83 } bin;
  84
  85 /* close a buffered file and free allocated memory */
  86 local void bclose(bin *in)
  87 {
  88     if (in != NULL) {
  89         if (in->fd != -1)
  90             close(in->fd);
  91         if (in->buf != NULL)
  92             free(in->buf);
  93         free(in);
  94     }
  95 }
  96
  97 /* open a buffered file for input, return a pointer to type bin, or NULL on
  98    failure */
  99 local bin *bopen(char *name)
 100 {
 101     bin *in;
 102
 103     in = malloc(sizeof(bin));
 104     if (in == NULL)
 105         return NULL;
 106     in->buf = malloc(CHUNK);
 107     in->fd = open(name, O_RDONLY, 0);
 108     if (in->buf == NULL || in->fd == -1) {
 109         bclose(in);
 110         return NULL;
 111     }
 112     in->left = 0;
 113     in->next = in->buf;
 114     in->name = name;
 115     return in;
 116 }
 117
 118 /* load buffer from file, return -1 on read error, 0 or 1 on success, with
 119    1 indicating that end-of-file was reached */
 120 local int bload(bin *in)
 121 {
 122     long len;
 123
 124     if (in == NULL)
 125         return -1;
 126     if (in->left != 0)
 127         return 0;
 128     in->next = in->buf;
 129     do {
 130         len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left);
 131         if (len < 0)
 132             return -1;
 133         in->left += (unsigned)len;
 134     } while (len != 0 && in->left < CHUNK);
 135     return len == 0 ? 1 : 0;
 136 }
 137
 138 /* get a byte from the file, bail if end of file */
 139 #define bget(in) (in->left ? 0 : bload(in), \
 140                   in->left ? (in->left--, *(in->next)++) : \
 141                     bail("unexpected end of file on ", in->name))
 142
 143 /* get a four-byte little-endian unsigned integer from file */
 144 local unsigned long bget4(bin *in)
 145 {
 146     unsigned long val;
 147
 148     val = bget(in);
 149     val += (unsigned long)(bget(in)) << 8;
 150     val += (unsigned long)(bget(in)) << 16;
 151     val += (unsigned long)(bget(in)) << 24;
 152     return val;
 153 }
 154
 155 /* skip bytes in file */
 156 local void bskip(bin *in, unsigned skip)
 157 {
 158     /* check pointer */
 159     if (in == NULL)
 160         return;
 161
 162     /* easy case -- skip bytes in buffer */
 163     if (skip <= in->left) {
 164         in->left -= skip;
 165         in->next += skip;
 166         return;
 167     }
 168
 169     /* skip what's in buffer, discard buffer contents */
 170     skip -= in->left;
 171     in->left = 0;
 172
 173     /* seek past multiples of CHUNK bytes */
 174     if (skip > CHUNK) {
 175         unsigned left;
 176
 177         left = skip & (CHUNK - 1);
 178         if (left == 0) {
 179             /* exact number of chunks: seek all the way minus one byte to check
 180                for end-of-file with a read */
 181             lseek(in->fd, skip - 1, SEEK_CUR);
 182             if (read(in->fd, in->buf, 1) != 1)
 183                 bail("unexpected end of file on ", in->name);
 184             return;
 185         }
 186
 187         /* skip the integral chunks, update skip with remainder */
 188         lseek(in->fd, skip - left, SEEK_CUR);
 189         skip = left;
 190     }
 191
 192     /* read more input and skip remainder */
 193     bload(in);
 194     if (skip > in->left)
 195         bail("unexpected end of file on ", in->name);
 196     in->left -= skip;
 197     in->next += skip;
 198 }
 199
 200 /* -- end of buffered input functions -- */
 201
 202 /* skip the gzip header from file in */
 203 local void gzhead(bin *in)
 204 {
 205     int flags;
 206
 207     /* verify gzip magic header and compression method */
 208     if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8)
 209         bail(in->name, " is not a valid gzip file");
 210
 211     /* get and verify flags */
 212     flags = bget(in);
 213     if ((flags & 0xe0) != 0)
 214         bail("unknown reserved bits set in ", in->name);
 215
 216     /* skip modification time, extra flags, and os */
 217     bskip(in, 6);
 218
 219     /* skip extra field if present */
 220     if (flags & 4) {
 221         unsigned len;
 222
 223         len = bget(in);
 224         len += (unsigned)(bget(in)) << 8;
 225         bskip(in, len);
 226     }
 227
 228     /* skip file name if present */
 229     if (flags & 8)
 230         while (bget(in) != 0)
 231             ;
 232
 233     /* skip comment if present */
 234     if (flags & 16)
 235         while (bget(in) != 0)
 236             ;
 237
 238     /* skip header crc if present */
 239     if (flags & 2)
 240         bskip(in, 2);
 241 }
 242
 243 /* write a four-byte little-endian unsigned integer to out */
 244 local void put4(unsigned long val, FILE *out)
 245 {
 246     putc(val & 0xff, out);
 247     putc((val >> 8) & 0xff, out);
 248     putc((val >> 16) & 0xff, out);
 249     putc((val >> 24) & 0xff, out);
 250 }
 251
 252 /* Load up zlib stream from buffered input, bail if end of file */
 253 local void zpull(z_streamp strm, bin *in)
 254 {
 255     if (in->left == 0)
 256         bload(in);
 257     if (in->left == 0)
 258         bail("unexpected end of file on ", in->name);
 259     strm->avail_in = in->left;
 260     strm->next_in = in->next;
 261 }
 262
 263 /* Write header for gzip file to out and initialize trailer. */
 264 local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out)
 265 {
 266     fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
 267     *crc = crc32(0L, Z_NULL, 0);
 268     *tot = 0;
 269 }
 270
 271 /* Copy the compressed data from name, zeroing the last block bit of the last
 272    block if clr is true, and adding empty blocks as needed to get to a byte
 273    boundary.  If clr is false, then the last block becomes the last block of
 274    the output, and the gzip trailer is written.  crc and tot maintains the
 275    crc and length (modulo 2^32) of the output for the trailer.  The resulting
 276    gzip file is written to out.  gzinit() must be called before the first call
 277    of gzcopy() to write the gzip header and to initialize crc and tot. */
 278 local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot,
 279                   FILE *out)
 280 {
 281     int ret;                /* return value from zlib functions */
 282     int pos;                /* where the "last block" bit is in byte */
 283     int last;               /* true if processing the last block */
 284     bin *in;                /* buffered input file */
 285     unsigned char *start;   /* start of compressed data in buffer */
 286     unsigned char *junk;    /* buffer for uncompressed data -- discarded */
 287     z_off_t len;            /* length of uncompressed data (support > 4 GB) */
 288     z_stream strm;          /* zlib inflate stream */
 289
 290     /* open gzip file and skip header */
 291     in = bopen(name);
 292     if (in == NULL)
 293         bail("could not open ", name);
 294     gzhead(in);
 295
 296     /* allocate buffer for uncompressed data and initialize raw inflate
 297        stream */
 298     junk = malloc(CHUNK);
 299     strm.zalloc = Z_NULL;
 300     strm.zfree = Z_NULL;
 301     strm.opaque = Z_NULL;
 302     strm.avail_in = 0;
 303     strm.next_in = Z_NULL;
 304     ret = inflateInit2(&strm, -15);
 305     if (junk == NULL || ret != Z_OK)
 306         bail("out of memory", "");
 307
 308     /* inflate and copy compressed data, clear last-block bit if requested */
 309     len = 0;
 310     zpull(&strm, in);
 311     start = strm.next_in;
 312     last = start[0] & 1;
 313     if (last && clr)
 314         start[0] &= ~1;
 315     strm.avail_out = 0;
 316     for (;;) {
 317         /* if input used and output done, write used input and get more */
 318         if (strm.avail_in == 0 && strm.avail_out != 0) {
 319             fwrite(start, 1, strm.next_in - start, out);
 320             start = in->buf;
 321             in->left = 0;
 322             zpull(&strm, in);
 323         }
 324
 325         /* decompress -- return early when end-of-block reached */
 326         strm.avail_out = CHUNK;
 327         strm.next_out = junk;
 328         ret = inflate(&strm, Z_BLOCK);
 329         switch (ret) {
 330         case Z_MEM_ERROR:
 331             bail("out of memory", "");
 332         case Z_DATA_ERROR:
 333             bail("invalid compressed data in ", in->name);
 334         }
 335
 336         /* update length of uncompressed data */
 337         len += CHUNK - strm.avail_out;
 338
 339         /* check for block boundary (only get this when block copied out) */
 340         if (strm.data_type & 128) {
 341             /* if that was the last block, then done */
 342             if (last)
 343                 break;
 344
 345             /* number of unused bits in last byte */
 346             pos = strm.data_type & 7;
 347
 348             /* find the next last-block bit */
 349             if (pos != 0) {
 350                 /* next last-block bit is in last used byte */
 351                 pos = 0x100 >> pos;
 352                 last = strm.next_in[-1] & pos;
 353                 if (last && clr)
 354                     strm.next_in[-1] &= ~pos;
 355             }
 356             else {
 357                 /* next last-block bit is in next unused byte */
 358                 if (strm.avail_in == 0) {
 359                     /* don't have that byte yet -- get it */
 360                     fwrite(start, 1, strm.next_in - start, out);
 361                     start = in->buf;
 362                     in->left = 0;
 363                     zpull(&strm, in);
 364                 }
 365                 last = strm.next_in[0] & 1;
 366                 if (last && clr)
 367                     strm.next_in[0] &= ~1;
 368             }
 369         }
 370     }
 371
 372     /* update buffer with unused input */
 373     in->left = strm.avail_in;
 374     in->next = strm.next_in;
 375
 376     /* copy used input, write empty blocks to get to byte boundary */
 377     pos = strm.data_type & 7;
 378     fwrite(start, 1, in->next - start - 1, out);
 379     last = in->next[-1];
 380     if (pos == 0 || !clr)
 381         /* already at byte boundary, or last file: write last byte */
 382         putc(last, out);
 383     else {
 384         /* append empty blocks to last byte */
 385         last &= ((0x100 >> pos) - 1);       /* assure unused bits are zero */
 386         if (pos & 1) {
 387             /* odd -- append an empty stored block */
 388             putc(last, out);
 389             if (pos == 1)
 390                 putc(0, out);               /* two more bits in block header */
 391             fwrite("\0\0\xff\xff", 1, 4, out);
 392         }
 393         else {
 394             /* even -- append 1, 2, or 3 empty fixed blocks */
 395             switch (pos) {
 396             case 6:
 397                 putc(last | 8, out);
 398                 last = 0;
 399             case 4:
 400                 putc(last | 0x20, out);
 401                 last = 0;
 402             case 2:
 403                 putc(last | 0x80, out);
 404                 putc(0, out);
 405             }
 406         }
 407     }
 408
 409     /* update crc and tot */
 410     *crc = crc32_combine(*crc, bget4(in), len);
 411     *tot += (unsigned long)len;
 412
 413     /* clean up */
 414     inflateEnd(&strm);
 415     free(junk);
 416     bclose(in);
 417
 418     /* write trailer if this is the last gzip file */
 419     if (!clr) {
 420         put4(*crc, out);
 421         put4(*tot, out);
 422     }
 423 }
 424
 425 /* join the gzip files on the command line, write result to stdout */
 426 int main(int argc, char **argv)
 427 {
 428     unsigned long crc, tot;     /* running crc and total uncompressed length */
 429
 430     /* skip command name */
 431     argc--;
 432     argv++;
 433
 434     /* show usage if no arguments */
 435     if (argc == 0) {
 436         fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n",
 437               stderr);
 438         return 0;
 439     }
 440
 441     /* join gzip files on command line and write to stdout */
 442     gzinit(&crc, &tot, stdout);
 443     while (argc--)
 444         gzcopy(*argv++, argc, &crc, &tot, stdout);
 445
 446     /* done */
 447     return 0;
 448 }