zlib/examples/gzlog.c

   1 /*
   2  * gzlog.c
   3  * Copyright (C) 2004 Mark Adler
   4  * For conditions of distribution and use, see copyright notice in gzlog.h
   5  * version 1.0, 26 Nov 2004
   6  *
   7  */
   8
   9 #include <string.h>             /* memcmp() */
  10 #include <stdlib.h>             /* malloc(), free(), NULL */
  11 #include <sys/types.h>          /* size_t, off_t */
  12 #include <unistd.h>             /* read(), close(), sleep(), ftruncate(), */
  13                                 /* lseek() */
  14 #include <fcntl.h>              /* open() */
  15 #include <sys/file.h>           /* flock() */
  16 #include "zlib.h"               /* deflateInit2(), deflate(), deflateEnd() */
  17
  18 #include "gzlog.h"              /* interface */
  19 #define local static
  20
  21 /* log object structure */
  22 typedef struct {
  23     int id;                 /* object identifier */
  24     int fd;                 /* log file descriptor */
  25     off_t extra;            /* offset of extra "ap" subfield */
  26     off_t mark_off;         /* offset of marked data */
  27     off_t last_off;         /* offset of last block */
  28     unsigned long crc;      /* uncompressed crc */
  29     unsigned long len;      /* uncompressed length (modulo 2^32) */
  30     unsigned stored;        /* length of current stored block */
  31 } gz_log;
  32
  33 #define GZLOGID 19334       /* gz_log object identifier */
  34
  35 #define LOCK_RETRY 1            /* retry lock once a second */
  36 #define LOCK_PATIENCE 1200      /* try about twenty minutes before forcing */
  37
  38 /* acquire a lock on a file */
  39 local int lock(int fd)
  40 {
  41     int patience;
  42
  43     /* try to lock every LOCK_RETRY seconds for LOCK_PATIENCE seconds */
  44     patience = LOCK_PATIENCE;
  45     do {
  46         if (flock(fd, LOCK_EX + LOCK_NB) == 0)
  47             return 0;
  48         (void)sleep(LOCK_RETRY);
  49         patience -= LOCK_RETRY;
  50     } while (patience > 0);
  51
  52     /* we've run out of patience -- give up */
  53     return -1;
  54 }
  55
  56 /* release lock */
  57 local void unlock(int fd)
  58 {
  59     (void)flock(fd, LOCK_UN);
  60 }
  61
  62 /* release a log object */
  63 local void log_clean(gz_log *log)
  64 {
  65     unlock(log->fd);
  66     (void)close(log->fd);
  67     free(log);
  68 }
  69
  70 /* read an unsigned long from a byte buffer little-endian */
  71 local unsigned long make_ulg(unsigned char *buf)
  72 {
  73     int n;
  74     unsigned long val;
  75
  76     val = (unsigned long)(*buf++);
  77     for (n = 8; n < 32; n += 8)
  78         val += (unsigned long)(*buf++) << n;
  79     return val;
  80 }
  81
  82 /* read an off_t from a byte buffer little-endian */
  83 local off_t make_off(unsigned char *buf)
  84 {
  85     int n;
  86     off_t val;
  87
  88     val = (off_t)(*buf++);
  89     for (n = 8; n < 64; n += 8)
  90         val += (off_t)(*buf++) << n;
  91     return val;
  92 }
  93
  94 /* write an unsigned long little-endian to byte buffer */
  95 local void dice_ulg(unsigned long val, unsigned char *buf)
  96 {
  97     int n;
  98
  99     for (n = 0; n < 4; n++) {
 100         *buf++ = val & 0xff;
 101         val >>= 8;
 102     }
 103 }
 104
 105 /* write an off_t little-endian to byte buffer */
 106 local void dice_off(off_t val, unsigned char *buf)
 107 {
 108     int n;
 109
 110     for (n = 0; n < 8; n++) {
 111         *buf++ = val & 0xff;
 112         val >>= 8;
 113     }
 114 }
 115
 116 /* initial, empty gzip file for appending */
 117 local char empty_gz[] = {
 118     0x1f, 0x8b,                 /* magic gzip id */
 119     8,                          /* compression method is deflate */
 120     4,                          /* there is an extra field */
 121     0, 0, 0, 0,                 /* no modification time provided */
 122     0, 0xff,                    /* no extra flags, no OS */
 123     20, 0, 'a', 'p', 16, 0,     /* extra field with "ap" subfield */
 124     32, 0, 0, 0, 0, 0, 0, 0,    /* offset of uncompressed data */
 125     32, 0, 0, 0, 0, 0, 0, 0,    /* offset of last block */
 126     1, 0, 0, 0xff, 0xff,        /* empty stored block (last) */
 127     0, 0, 0, 0,                 /* crc */
 128     0, 0, 0, 0                  /* uncompressed length */
 129 };
 130
 131 /* initialize a log object with locking */
 132 void *gzlog_open(char *path)
 133 {
 134     unsigned xlen;
 135     unsigned char temp[20];
 136     unsigned sub_len;
 137     int good;
 138     gz_log *log;
 139
 140     /* allocate log structure */
 141     log = malloc(sizeof(gz_log));
 142     if (log == NULL)
 143         return NULL;
 144     log->id = GZLOGID;
 145
 146     /* open file, creating it if necessary, and locking it */
 147     log->fd = open(path, O_RDWR | O_CREAT, 0600);
 148     if (log->fd < 0) {
 149         free(log);
 150         return NULL;
 151     }
 152     if (lock(log->fd)) {
 153         close(log->fd);
 154         free(log);
 155         return NULL;
 156     }
 157
 158     /* if file is empty, write new gzip stream */
 159     if (lseek(log->fd, 0, SEEK_END) == 0) {
 160         if (write(log->fd, empty_gz, sizeof(empty_gz)) != sizeof(empty_gz)) {
 161             log_clean(log);
 162             return NULL;
 163         }
 164     }
 165
 166     /* check gzip header */
 167     (void)lseek(log->fd, 0, SEEK_SET);
 168     if (read(log->fd, temp, 12) != 12 || temp[0] != 0x1f ||
 169         temp[1] != 0x8b || temp[2] != 8 || (temp[3] & 4) == 0) {
 170         log_clean(log);
 171         return NULL;
 172     }
 173
 174     /* process extra field to find "ap" sub-field */
 175     xlen = temp[10] + (temp[11] << 8);
 176     good = 0;
 177     while (xlen) {
 178         if (xlen < 4 || read(log->fd, temp, 4) != 4)
 179             break;
 180         sub_len = temp[2];
 181         sub_len += temp[3] << 8;
 182         xlen -= 4;
 183         if (memcmp(temp, "ap", 2) == 0 && sub_len == 16) {
 184             good = 1;
 185             break;
 186         }
 187         if (xlen < sub_len)
 188             break;
 189         (void)lseek(log->fd, sub_len, SEEK_CUR);
 190         xlen -= sub_len;
 191     }
 192     if (!good) {
 193         log_clean(log);
 194         return NULL;
 195     }
 196
 197     /* read in "ap" sub-field */
 198     log->extra = lseek(log->fd, 0, SEEK_CUR);
 199     if (read(log->fd, temp, 16) != 16) {
 200         log_clean(log);
 201         return NULL;
 202     }
 203     log->mark_off = make_off(temp);
 204     log->last_off = make_off(temp + 8);
 205
 206     /* get crc, length of gzip file */
 207     (void)lseek(log->fd, log->last_off, SEEK_SET);
 208     if (read(log->fd, temp, 13) != 13 ||
 209         memcmp(temp, "\001\000\000\377\377", 5) != 0) {
 210         log_clean(log);
 211         return NULL;
 212     }
 213     log->crc = make_ulg(temp + 5);
 214     log->len = make_ulg(temp + 9);
 215
 216     /* set up to write over empty last block */
 217     (void)lseek(log->fd, log->last_off + 5, SEEK_SET);
 218     log->stored = 0;
 219     return (void *)log;
 220 }
 221
 222 /* maximum amount to put in a stored block before starting a new one */
 223 #define MAX_BLOCK 16384
 224
 225 /* write a block to a log object */
 226 int gzlog_write(void *obj, char *data, size_t len)
 227 {
 228     size_t some;
 229     unsigned char temp[5];
 230     gz_log *log;
 231
 232     /* check object */
 233     log = (gz_log *)obj;
 234     if (log == NULL || log->id != GZLOGID)
 235         return 1;
 236
 237     /* write stored blocks until all of the input is written */
 238     do {
 239         some = MAX_BLOCK - log->stored;
 240         if (some > len)
 241             some = len;
 242         if (write(log->fd, data, some) != some)
 243             return 1;
 244         log->crc = crc32(log->crc, data, some);
 245         log->len += some;
 246         len -= some;
 247         data += some;
 248         log->stored += some;
 249
 250         /* if the stored block is full, end it and start another */
 251         if (log->stored == MAX_BLOCK) {
 252             (void)lseek(log->fd, log->last_off, SEEK_SET);
 253             temp[0] = 0;
 254             dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16),
 255                      temp + 1);
 256             if (write(log->fd, temp, 5) != 5)
 257                 return 1;
 258             log->last_off = lseek(log->fd, log->stored, SEEK_CUR);
 259             (void)lseek(log->fd, 5, SEEK_CUR);
 260             log->stored = 0;
 261         }
 262     } while (len);
 263     return 0;
 264 }
 265
 266 /* recompress the remaining stored deflate data in place */
 267 local int recomp(gz_log *log)
 268 {
 269     z_stream strm;
 270     size_t len, max;
 271     unsigned char *in;
 272     unsigned char *out;
 273     unsigned char temp[16];
 274
 275     /* allocate space and read it all in (it's around 1 MB) */
 276     len = log->last_off - log->mark_off;
 277     max = len + (len >> 12) + (len >> 14) + 11;
 278     out = malloc(max);
 279     if (out == NULL)
 280         return 1;
 281     in = malloc(len);
 282     if (in == NULL) {
 283         free(out);
 284         return 1;
 285     }
 286     (void)lseek(log->fd, log->mark_off, SEEK_SET);
 287     if (read(log->fd, in, len) != len) {
 288         free(in);
 289         free(out);
 290         return 1;
 291     }
 292
 293     /* recompress in memory, decoding stored data as we go */
 294     /* note: this assumes that unsigned is four bytes or more */
 295     /*       consider not making that assumption */
 296     strm.zalloc = Z_NULL;
 297     strm.zfree = Z_NULL;
 298     strm.opaque = Z_NULL;
 299     if (deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED, -15, 8,
 300         Z_DEFAULT_STRATEGY) != Z_OK) {
 301         free(in);
 302         free(out);
 303         return 1;
 304     }
 305     strm.next_in = in;
 306     strm.avail_out = max;
 307     strm.next_out = out;
 308     while (len >= 5) {
 309         if (strm.next_in[0] != 0)
 310             break;
 311         strm.avail_in = strm.next_in[1] + (strm.next_in[2] << 8);
 312         strm.next_in += 5;
 313         len -= 5;
 314         if (strm.avail_in != 0) {
 315             if (len < strm.avail_in)
 316                 break;
 317             len -= strm.avail_in;
 318             (void)deflate(&strm, Z_NO_FLUSH);
 319             if (strm.avail_in != 0 || strm.avail_out == 0)
 320                 break;
 321         }
 322     }
 323     (void)deflate(&strm, Z_SYNC_FLUSH);
 324     (void)deflateEnd(&strm);
 325     free(in);
 326     if (len != 0 || strm.avail_out == 0) {
 327         free(out);
 328         return 1;
 329     }
 330
 331     /* overwrite stored data with compressed data */
 332     (void)lseek(log->fd, log->mark_off, SEEK_SET);
 333     len = max - strm.avail_out;
 334     if (write(log->fd, out, len) != len) {
 335         free(out);
 336         return 1;
 337     }
 338     free(out);
 339
 340     /* write last empty block, crc, and length */
 341     log->mark_off = log->last_off = lseek(log->fd, 0, SEEK_CUR);
 342     temp[0] = 1;
 343     dice_ulg(0xffffL << 16, temp + 1);
 344     dice_ulg(log->crc, temp + 5);
 345     dice_ulg(log->len, temp + 9);
 346     if (write(log->fd, temp, 13) != 13)
 347         return 1;
 348
 349     /* truncate file to discard remaining stored data and old trailer */
 350     ftruncate(log->fd, lseek(log->fd, 0, SEEK_CUR));
 351
 352     /* update extra field to point to new last empty block */
 353     (void)lseek(log->fd, log->extra, SEEK_SET);
 354     dice_off(log->mark_off, temp);
 355     dice_off(log->last_off, temp + 8);
 356     if (write(log->fd, temp, 16) != 16)
 357         return 1;
 358     return 0;
 359 }
 360
 361 /* maximum accumulation of stored blocks before compressing */
 362 #define MAX_STORED 1048576
 363
 364 /* close log object */
 365 int gzlog_close(void *obj)
 366 {
 367     unsigned char temp[8];
 368     gz_log *log;
 369
 370     /* check object */
 371     log = (gz_log *)obj;
 372     if (log == NULL || log->id != GZLOGID)
 373         return 1;
 374
 375     /* go to start of most recent block being written */
 376     (void)lseek(log->fd, log->last_off, SEEK_SET);
 377
 378     /* if some stuff was put there, update block */
 379     if (log->stored) {
 380         temp[0] = 0;
 381         dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16),
 382                  temp + 1);
 383         if (write(log->fd, temp, 5) != 5)
 384             return 1;
 385         log->last_off = lseek(log->fd, log->stored, SEEK_CUR);
 386     }
 387
 388     /* write last block (empty) */
 389     if (write(log->fd, "\001\000\000\377\377", 5) != 5)
 390         return 1;
 391
 392     /* write updated crc and uncompressed length */
 393     dice_ulg(log->crc, temp);
 394     dice_ulg(log->len, temp + 4);
 395     if (write(log->fd, temp, 8) != 8)
 396         return 1;
 397
 398     /* put offset of that last block in gzip extra block */
 399     (void)lseek(log->fd, log->extra + 8, SEEK_SET);
 400     dice_off(log->last_off, temp);
 401     if (write(log->fd, temp, 8) != 8)
 402         return 1;
 403
 404     /* if more than 1 MB stored, then time to compress it */
 405     if (log->last_off - log->mark_off > MAX_STORED) {
 406         if (recomp(log))
 407             return 1;
 408     }
 409
 410     /* unlock and close file */
 411     log_clean(log);
 412     return 0;
 413 }