src/mod_compress.c

   1 #include "first.h"
   2
   3 #include "base.h"
   4 #include "log.h"
   5 #include "buffer.h"
   6 #include "response.h"
   7 #include "stat_cache.h"
   8
   9 #include "plugin.h"
  10
  11 #include "crc32.h"
  12 #include "etag.h"
  13
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16
  17 #include <assert.h>
  18 #include <fcntl.h>
  19 #include <unistd.h>
  20 #include <ctype.h>
  21 #include <stdlib.h>
  22 #include <string.h>
  23 #include <strings.h>
  24 #include <errno.h>
  25 #include <time.h>
  26
  27 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ
  28 # define USE_ZLIB
  29 # include <zlib.h>
  30 #endif
  31
  32 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
  33 # define USE_BZ2LIB
  34 /* we don't need stdio interface */
  35 # define BZ_NO_STDIO
  36 # include <bzlib.h>
  37 #endif
  38
  39 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
  40 #define USE_MMAP
  41
  42 #include "sys-mmap.h"
  43 #include <setjmp.h>
  44 #include <signal.h>
  45
  46 static volatile int sigbus_jmp_valid;
  47 static sigjmp_buf sigbus_jmp;
  48
  49 static void sigbus_handler(int sig) {
  50         UNUSED(sig);
  51         if (sigbus_jmp_valid) siglongjmp(sigbus_jmp, 1);
  52         log_failed_assert(__FILE__, __LINE__, "SIGBUS");
  53 }
  54 #endif
  55
  56 /* request: accept-encoding */
  57 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0)
  58 #define HTTP_ACCEPT_ENCODING_GZIP     BV(1)
  59 #define HTTP_ACCEPT_ENCODING_DEFLATE  BV(2)
  60 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3)
  61 #define HTTP_ACCEPT_ENCODING_BZIP2    BV(4)
  62 #define HTTP_ACCEPT_ENCODING_X_GZIP   BV(5)
  63 #define HTTP_ACCEPT_ENCODING_X_BZIP2  BV(6)
  64
  65 #ifdef __WIN32
  66 # define mkdir(x,y) mkdir(x)
  67 #endif
  68
  69 typedef struct {
  70         buffer *compress_cache_dir;
  71         array  *compress;
  72         off_t   compress_max_filesize; /** max filesize in kb */
  73         int     allowed_encodings;
  74         double  max_loadavg;
  75 } plugin_config;
  76
  77 typedef struct {
  78         PLUGIN_DATA;
  79         buffer *ofn;
  80         buffer *b;
  81
  82         plugin_config **config_storage;
  83         plugin_config conf;
  84 } plugin_data;
  85
  86 INIT_FUNC(mod_compress_init) {
  87         plugin_data *p;
  88
  89         p = calloc(1, sizeof(*p));
  90
  91         p->ofn = buffer_init();
  92         p->b = buffer_init();
  93
  94         return p;
  95 }
  96
  97 FREE_FUNC(mod_compress_free) {
  98         plugin_data *p = p_d;
  99
 100         UNUSED(srv);
 101
 102         if (!p) return HANDLER_GO_ON;
 103
 104         buffer_free(p->ofn);
 105         buffer_free(p->b);
 106
 107         if (p->config_storage) {
 108                 size_t i;
 109                 for (i = 0; i < srv->config_context->used; i++) {
 110                         plugin_config *s = p->config_storage[i];
 111
 112                         if (NULL == s) continue;
 113
 114                         array_free(s->compress);
 115                         buffer_free(s->compress_cache_dir);
 116
 117                         free(s);
 118                 }
 119                 free(p->config_storage);
 120         }
 121
 122
 123         free(p);
 124
 125         return HANDLER_GO_ON;
 126 }
 127
 128 /* 0 on success, -1 for error */
 129 static int mkdir_recursive(char *dir) {
 130         char *p = dir;
 131
 132         if (!dir || !dir[0])
 133                 return 0;
 134
 135         while ((p = strchr(p + 1, '/')) != NULL) {
 136
 137                 *p = '\0';
 138                 if ((mkdir(dir, 0700) != 0) && (errno != EEXIST)) {
 139                         *p = '/';
 140                         return -1;
 141                 }
 142
 143                 *p++ = '/';
 144                 if (!*p) return 0; /* Ignore trailing slash */
 145         }
 146
 147         return (mkdir(dir, 0700) != 0) && (errno != EEXIST) ? -1 : 0;
 148 }
 149
 150 /* 0 on success, -1 for error */
 151 static int mkdir_for_file(char *filename) {
 152         char *p = filename;
 153
 154         if (!filename || !filename[0])
 155                 return -1;
 156
 157         while ((p = strchr(p + 1, '/')) != NULL) {
 158
 159                 *p = '\0';
 160                 if ((mkdir(filename, 0700) != 0) && (errno != EEXIST)) {
 161                         *p = '/';
 162                         return -1;
 163                 }
 164
 165                 *p++ = '/';
 166                 if (!*p) return -1; /* Unexpected trailing slash in filename */
 167         }
 168
 169         return 0;
 170 }
 171
 172 SETDEFAULTS_FUNC(mod_compress_setdefaults) {
 173         plugin_data *p = p_d;
 174         size_t i = 0;
 175
 176         config_values_t cv[] = {
 177                 { "compress.cache-dir",             NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
 178                 { "compress.filetype",              NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
 179                 { "compress.max-filesize",          NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
 180                 { "compress.allowed-encodings",     NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
 181                 { "compress.max-loadavg",           NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
 182                 { NULL,                             NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET }
 183         };
 184
 185         p->config_storage = calloc(1, srv->config_context->used * sizeof(plugin_config *));
 186
 187         for (i = 0; i < srv->config_context->used; i++) {
 188                 data_config const* config = (data_config const*)srv->config_context->data[i];
 189                 plugin_config *s;
 190                 array  *encodings_arr = array_init();
 191
 192                 s = calloc(1, sizeof(plugin_config));
 193                 s->compress_cache_dir = buffer_init();
 194                 s->compress = array_init();
 195                 s->compress_max_filesize = 0;
 196                 s->allowed_encodings = 0;
 197                 s->max_loadavg = 0.0;
 198
 199                 cv[0].destination = s->compress_cache_dir;
 200                 cv[1].destination = s->compress;
 201                 cv[2].destination = &(s->compress_max_filesize);
 202                 cv[3].destination = encodings_arr; /* temp array for allowed encodings list */
 203                 cv[4].destination = srv->tmp_buf;
 204                 buffer_string_set_length(srv->tmp_buf, 0);
 205
 206                 p->config_storage[i] = s;
 207
 208                 if (0 != config_insert_values_global(srv, config->value, cv, i == 0 ? T_CONFIG_SCOPE_SERVER : T_CONFIG_SCOPE_CONNECTION)) {
 209                         return HANDLER_ERROR;
 210                 }
 211
 212                 if (!buffer_string_is_empty(srv->tmp_buf)) {
 213                         s->max_loadavg = strtod(srv->tmp_buf->ptr, NULL);
 214                 }
 215
 216                 if (encodings_arr->used) {
 217                         size_t j = 0;
 218                         for (j = 0; j < encodings_arr->used; j++) {
 219 #if defined(USE_ZLIB) || defined(USE_BZ2LIB)
 220                                 data_string *ds = (data_string *)encodings_arr->data[j];
 221 #endif
 222 #ifdef USE_ZLIB
 223                                 if (NULL != strstr(ds->value->ptr, "gzip"))
 224                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_GZIP | HTTP_ACCEPT_ENCODING_X_GZIP;
 225                                 if (NULL != strstr(ds->value->ptr, "x-gzip"))
 226                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_X_GZIP;
 227                                 if (NULL != strstr(ds->value->ptr, "deflate"))
 228                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_DEFLATE;
 229                                 /*
 230                                 if (NULL != strstr(ds->value->ptr, "compress"))
 231                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_COMPRESS;
 232                                 */
 233 #endif
 234 #ifdef USE_BZ2LIB
 235                                 if (NULL != strstr(ds->value->ptr, "bzip2"))
 236                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2 | HTTP_ACCEPT_ENCODING_X_BZIP2;
 237                                 if (NULL != strstr(ds->value->ptr, "x-bzip2"))
 238                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_X_BZIP2;
 239 #endif
 240                         }
 241                 } else {
 242                         /* default encodings */
 243                         s->allowed_encodings = 0
 244 #ifdef USE_ZLIB
 245                                 | HTTP_ACCEPT_ENCODING_GZIP | HTTP_ACCEPT_ENCODING_X_GZIP | HTTP_ACCEPT_ENCODING_DEFLATE
 246 #endif
 247 #ifdef USE_BZ2LIB
 248                                 | HTTP_ACCEPT_ENCODING_BZIP2 | HTTP_ACCEPT_ENCODING_X_BZIP2
 249 #endif
 250                                 ;
 251                 }
 252
 253                 array_free(encodings_arr);
 254
 255                 if (!buffer_string_is_empty(s->compress_cache_dir)) {
 256                         struct stat st;
 257                         mkdir_recursive(s->compress_cache_dir->ptr);
 258
 259                         if (0 != stat(s->compress_cache_dir->ptr, &st)) {
 260                                 log_error_write(srv, __FILE__, __LINE__, "sbs", "can't stat compress.cache-dir",
 261                                                 s->compress_cache_dir, strerror(errno));
 262
 263                                 return HANDLER_ERROR;
 264                         }
 265                 }
 266         }
 267
 268         return HANDLER_GO_ON;
 269
 270 }
 271
 272 #ifdef USE_ZLIB
 273 static int deflate_file_to_buffer_gzip(server *srv, connection *con, plugin_data *p, char *start, off_t st_size, time_t mtime) {
 274         unsigned char *c;
 275         unsigned long crc;
 276         z_stream z;
 277         size_t outlen;
 278
 279         UNUSED(srv);
 280         UNUSED(con);
 281
 282         z.zalloc = Z_NULL;
 283         z.zfree = Z_NULL;
 284         z.opaque = Z_NULL;
 285
 286         if (Z_OK != deflateInit2(&z,
 287                                  Z_DEFAULT_COMPRESSION,
 288                                  Z_DEFLATED,
 289                                  -MAX_WBITS,  /* supress zlib-header */
 290                                  8,
 291                                  Z_DEFAULT_STRATEGY)) {
 292                 return -1;
 293         }
 294
 295         z.next_in = (unsigned char *)start;
 296         z.avail_in = st_size;
 297         z.total_in = 0;
 298
 299
 300         buffer_string_prepare_copy(p->b, (z.avail_in * 1.1) + 12 + 18);
 301
 302         /* write gzip header */
 303
 304         c = (unsigned char *)p->b->ptr;
 305         c[0] = 0x1f;
 306         c[1] = 0x8b;
 307         c[2] = Z_DEFLATED;
 308         c[3] = 0; /* options */
 309         c[4] = (mtime >>  0) & 0xff;
 310         c[5] = (mtime >>  8) & 0xff;
 311         c[6] = (mtime >> 16) & 0xff;
 312         c[7] = (mtime >> 24) & 0xff;
 313         c[8] = 0x00; /* extra flags */
 314         c[9] = 0x03; /* UNIX */
 315
 316         outlen = 10;
 317         z.next_out = (unsigned char *)p->b->ptr + outlen;
 318         z.avail_out = p->b->size - outlen - 9;
 319         z.total_out = 0;
 320
 321         if (Z_STREAM_END != deflate(&z, Z_FINISH)) {
 322                 deflateEnd(&z);
 323                 return -1;
 324         }
 325
 326         /* trailer */
 327         outlen += z.total_out;
 328
 329         crc = generate_crc32c(start, st_size);
 330
 331         c = (unsigned char *)p->b->ptr + outlen;
 332
 333         c[0] = (crc >>  0) & 0xff;
 334         c[1] = (crc >>  8) & 0xff;
 335         c[2] = (crc >> 16) & 0xff;
 336         c[3] = (crc >> 24) & 0xff;
 337         c[4] = (z.total_in >>  0) & 0xff;
 338         c[5] = (z.total_in >>  8) & 0xff;
 339         c[6] = (z.total_in >> 16) & 0xff;
 340         c[7] = (z.total_in >> 24) & 0xff;
 341         outlen += 8;
 342         buffer_commit(p->b, outlen);
 343
 344         if (Z_OK != deflateEnd(&z)) {
 345                 return -1;
 346         }
 347
 348         return 0;
 349 }
 350
 351 static int deflate_file_to_buffer_deflate(server *srv, connection *con, plugin_data *p, unsigned char *start, off_t st_size) {
 352         z_stream z;
 353
 354         UNUSED(srv);
 355         UNUSED(con);
 356
 357         z.zalloc = Z_NULL;
 358         z.zfree = Z_NULL;
 359         z.opaque = Z_NULL;
 360
 361         if (Z_OK != deflateInit2(&z,
 362                                  Z_DEFAULT_COMPRESSION,
 363                                  Z_DEFLATED,
 364                                  -MAX_WBITS,  /* supress zlib-header */
 365                                  8,
 366                                  Z_DEFAULT_STRATEGY)) {
 367                 return -1;
 368         }
 369
 370         z.next_in = start;
 371         z.avail_in = st_size;
 372         z.total_in = 0;
 373
 374         buffer_string_prepare_copy(p->b, (z.avail_in * 1.1) + 12);
 375
 376         z.next_out = (unsigned char *)p->b->ptr;
 377         z.avail_out = p->b->size - 1;
 378         z.total_out = 0;
 379
 380         if (Z_STREAM_END != deflate(&z, Z_FINISH)) {
 381                 deflateEnd(&z);
 382                 return -1;
 383         }
 384
 385         if (Z_OK != deflateEnd(&z)) {
 386                 return -1;
 387         }
 388
 389         /* trailer */
 390         buffer_commit(p->b, z.total_out);
 391
 392         return 0;
 393 }
 394
 395 #endif
 396
 397 #ifdef USE_BZ2LIB
 398 static int deflate_file_to_buffer_bzip2(server *srv, connection *con, plugin_data *p, unsigned char *start, off_t st_size) {
 399         bz_stream bz;
 400
 401         UNUSED(srv);
 402         UNUSED(con);
 403
 404         bz.bzalloc = NULL;
 405         bz.bzfree = NULL;
 406         bz.opaque = NULL;
 407
 408         if (BZ_OK != BZ2_bzCompressInit(&bz,
 409                                         9, /* blocksize = 900k */
 410                                         0, /* no output */
 411                                         0)) { /* workFactor: default */
 412                 return -1;
 413         }
 414
 415         bz.next_in = (char *)start;
 416         bz.avail_in = st_size;
 417         bz.total_in_lo32 = 0;
 418         bz.total_in_hi32 = 0;
 419
 420         buffer_string_prepare_copy(p->b, (bz.avail_in * 1.1) + 12);
 421
 422         bz.next_out = p->b->ptr;
 423         bz.avail_out = p->b->size - 1;
 424         bz.total_out_lo32 = 0;
 425         bz.total_out_hi32 = 0;
 426
 427         if (BZ_STREAM_END != BZ2_bzCompress(&bz, BZ_FINISH)) {
 428                 BZ2_bzCompressEnd(&bz);
 429                 return -1;
 430         }
 431
 432         if (BZ_OK != BZ2_bzCompressEnd(&bz)) {
 433                 return -1;
 434         }
 435
 436         /* file is too large for now */
 437         if (bz.total_out_hi32) return -1;
 438
 439         /* trailer */
 440         buffer_commit(p->b, bz.total_out_lo32);
 441
 442         return 0;
 443 }
 444 #endif
 445
 446 static void mod_compress_note_ratio(server *srv, connection *con, off_t in, off_t out) {
 447     /* store compression ratio in con->environment
 448      * for possible logging by mod_accesslog
 449      * (late in response handling, so not seen by most other modules) */
 450     /*(should be called only at end of successful response compression)*/
 451     char ratio[LI_ITOSTRING_LENGTH];
 452     if (0 == in) return;
 453     li_itostrn(ratio, sizeof(ratio), out * 100 / in);
 454     array_set_key_value(con->environment,
 455                         CONST_STR_LEN("ratio"),
 456                         ratio, strlen(ratio));
 457     UNUSED(srv);
 458 }
 459
 460 static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) {
 461         int ifd, ofd;
 462         int ret;
 463 #ifdef USE_MMAP
 464         volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */
 465 #endif
 466         void *start;
 467         const char *filename = fn->ptr;
 468         stat_cache_entry *sce_ofn;
 469         ssize_t r;
 470
 471         /* overflow */
 472         if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1;
 473
 474         /* don't mmap files > 128Mb
 475          *
 476          * we could use a sliding window, but currently there is no need for it
 477          */
 478
 479         if (sce->st.st_size > 128 * 1024 * 1024) return -1;
 480
 481         buffer_reset(p->ofn);
 482         buffer_copy_buffer(p->ofn, p->conf.compress_cache_dir);
 483         buffer_append_slash(p->ofn);
 484
 485         if (0 == strncmp(con->physical.path->ptr, con->physical.doc_root->ptr, buffer_string_length(con->physical.doc_root))) {
 486                 buffer_append_string(p->ofn, con->physical.path->ptr + buffer_string_length(con->physical.doc_root));
 487         } else {
 488                 buffer_append_string_buffer(p->ofn, con->uri.path);
 489         }
 490
 491         switch(type) {
 492         case HTTP_ACCEPT_ENCODING_GZIP:
 493         case HTTP_ACCEPT_ENCODING_X_GZIP:
 494                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-gzip-"));
 495                 break;
 496         case HTTP_ACCEPT_ENCODING_DEFLATE:
 497                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-deflate-"));
 498                 break;
 499         case HTTP_ACCEPT_ENCODING_BZIP2:
 500         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 501                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-bzip2-"));
 502                 break;
 503         default:
 504                 log_error_write(srv, __FILE__, __LINE__, "sd", "unknown compression type", type);
 505                 return -1;
 506         }
 507
 508         buffer_append_string_buffer(p->ofn, sce->etag);
 509
 510         if (HANDLER_ERROR != stat_cache_get_entry(srv, con, p->ofn, &sce_ofn)) {
 511                 if (0 == sce->st.st_size) return -1; /* cache file being created */
 512                 /* cache-entry exists */
 513 #if 0
 514                 log_error_write(srv, __FILE__, __LINE__, "bs", p->ofn, "compress-cache hit");
 515 #endif
 516                 mod_compress_note_ratio(srv, con, sce->st.st_size, sce_ofn->st.st_size);
 517                 buffer_copy_buffer(con->physical.path, p->ofn);
 518                 return 0;
 519         }
 520
 521         if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) {
 522                 return -1;
 523         }
 524
 525         if (-1 == mkdir_for_file(p->ofn->ptr)) {
 526                 log_error_write(srv, __FILE__, __LINE__, "sb", "couldn't create directory for file", p->ofn);
 527                 return -1;
 528         }
 529
 530         if (-1 == (ofd = open(p->ofn->ptr, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0600))) {
 531                 if (errno == EEXIST) {
 532                         return -1; /* cache file being created */
 533                 }
 534
 535                 log_error_write(srv, __FILE__, __LINE__, "sbss", "creating cachefile", p->ofn, "failed", strerror(errno));
 536
 537                 return -1;
 538         }
 539 #if 0
 540         log_error_write(srv, __FILE__, __LINE__, "bs", p->ofn, "compress-cache miss");
 541 #endif
 542         if (-1 == (ifd = open(filename, O_RDONLY | O_BINARY))) {
 543                 log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno));
 544
 545                 close(ofd);
 546
 547                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 548                 if (-1 == unlink(p->ofn->ptr)) {
 549                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 550                 }
 551
 552                 return -1;
 553         }
 554
 555 #ifdef USE_MMAP
 556         if (MAP_FAILED != (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) {
 557                 mapped = 1;
 558                 signal(SIGBUS, sigbus_handler);
 559                 sigbus_jmp_valid = 1;
 560                 if (0 != sigsetjmp(sigbus_jmp, 1)) {
 561                         sigbus_jmp_valid = 0;
 562
 563                         log_error_write(srv, __FILE__, __LINE__, "sbd", "SIGBUS in mmap:",
 564                                 fn, ifd);
 565
 566                         munmap(start, sce->st.st_size);
 567                         close(ofd);
 568                         close(ifd);
 569
 570                         /* Remove the incomplete cache file, so that later hits aren't served from it */
 571                         if (-1 == unlink(p->ofn->ptr)) {
 572                                 log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 573                         }
 574
 575                         return -1;
 576                 }
 577         } else
 578 #endif  /* FIXME: might attempt to read very large file completely into memory; see compress.max-filesize config option */
 579         if (NULL == (start = malloc(sce->st.st_size)) || sce->st.st_size != read(ifd, start, sce->st.st_size)) {
 580                 log_error_write(srv, __FILE__, __LINE__, "sbss", "reading", fn, "failed", strerror(errno));
 581
 582                 close(ofd);
 583                 close(ifd);
 584                 free(start);
 585
 586                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 587                 if (-1 == unlink(p->ofn->ptr)) {
 588                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 589                 }
 590
 591                 return -1;
 592         }
 593
 594         ret = -1;
 595         switch(type) {
 596 #ifdef USE_ZLIB
 597         case HTTP_ACCEPT_ENCODING_GZIP:
 598         case HTTP_ACCEPT_ENCODING_X_GZIP:
 599                 ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime);
 600                 break;
 601         case HTTP_ACCEPT_ENCODING_DEFLATE:
 602                 ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size);
 603                 break;
 604 #endif
 605 #ifdef USE_BZ2LIB
 606         case HTTP_ACCEPT_ENCODING_BZIP2:
 607         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 608                 ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size);
 609                 break;
 610 #endif
 611         }
 612
 613         if (ret == 0) {
 614                 r = write(ofd, CONST_BUF_LEN(p->b));
 615                 if (-1 == r) {
 616                         log_error_write(srv, __FILE__, __LINE__, "sbss", "writing cachefile", p->ofn, "failed:", strerror(errno));
 617                         ret = -1;
 618                 } else if ((size_t)r != buffer_string_length(p->b)) {
 619                         log_error_write(srv, __FILE__, __LINE__, "sbs", "writing cachefile", p->ofn, "failed: not enough bytes written");
 620                         ret = -1;
 621                 }
 622         }
 623
 624 #ifdef USE_MMAP
 625         if (mapped) {
 626                 sigbus_jmp_valid = 0;
 627                 munmap(start, sce->st.st_size);
 628         } else
 629 #endif
 630                 free(start);
 631
 632         close(ifd);
 633
 634         if (0 != close(ofd) || ret != 0) {
 635                 if (0 == ret) {
 636                         log_error_write(srv, __FILE__, __LINE__, "sbss", "writing cachefile", p->ofn, "failed:", strerror(errno));
 637                 }
 638
 639                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 640                 if (-1 == unlink(p->ofn->ptr)) {
 641                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 642                 }
 643
 644                 return -1;
 645         }
 646
 647         buffer_copy_buffer(con->physical.path, p->ofn);
 648         mod_compress_note_ratio(srv, con, sce->st.st_size,
 649                                 (off_t)buffer_string_length(p->b));
 650
 651         return 0;
 652 }
 653
 654 static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) {
 655         int ifd;
 656         int ret = -1;
 657 #ifdef USE_MMAP
 658         volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */
 659 #endif
 660         void *start;
 661
 662         /* overflow */
 663         if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1;
 664
 665         /* don't mmap files > 128M
 666          *
 667          * we could use a sliding window, but currently there is no need for it
 668          */
 669
 670         if (sce->st.st_size > 128 * 1024 * 1024) return -1;
 671
 672         if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) {
 673                 return -1;
 674         }
 675
 676         if (-1 == (ifd = open(fn->ptr, O_RDONLY | O_BINARY))) {
 677                 log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno));
 678
 679                 return -1;
 680         }
 681
 682 #ifdef USE_MMAP
 683         if (MAP_FAILED != (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) {
 684                 mapped = 1;
 685                 signal(SIGBUS, sigbus_handler);
 686                 sigbus_jmp_valid = 1;
 687                 if (0 != sigsetjmp(sigbus_jmp, 1)) {
 688                         sigbus_jmp_valid = 0;
 689
 690                         log_error_write(srv, __FILE__, __LINE__, "sbd", "SIGBUS in mmap:",
 691                                 fn, ifd);
 692
 693                         munmap(start, sce->st.st_size);
 694                         close(ifd);
 695                         return -1;
 696                 }
 697         } else
 698 #endif  /* FIXME: might attempt to read very large file completely into memory; see compress.max-filesize config option */
 699         if (NULL == (start = malloc(sce->st.st_size)) || sce->st.st_size != read(ifd, start, sce->st.st_size)) {
 700                 log_error_write(srv, __FILE__, __LINE__, "sbss", "reading", fn, "failed", strerror(errno));
 701
 702                 close(ifd);
 703                 free(start);
 704                 return -1;
 705         }
 706
 707         switch(type) {
 708 #ifdef USE_ZLIB
 709         case HTTP_ACCEPT_ENCODING_GZIP:
 710         case HTTP_ACCEPT_ENCODING_X_GZIP:
 711                 ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime);
 712                 break;
 713         case HTTP_ACCEPT_ENCODING_DEFLATE:
 714                 ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size);
 715                 break;
 716 #endif
 717 #ifdef USE_BZ2LIB
 718         case HTTP_ACCEPT_ENCODING_BZIP2:
 719         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 720                 ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size);
 721                 break;
 722 #endif
 723         default:
 724                 ret = -1;
 725                 break;
 726         }
 727
 728 #ifdef USE_MMAP
 729         if (mapped) {
 730                 sigbus_jmp_valid = 0;
 731                 munmap(start, sce->st.st_size);
 732         } else
 733 #endif
 734                 free(start);
 735
 736         close(ifd);
 737
 738         if (ret != 0) return -1;
 739
 740         mod_compress_note_ratio(srv, con, sce->st.st_size,
 741                                 (off_t)buffer_string_length(p->b));
 742         chunkqueue_reset(con->write_queue);
 743         chunkqueue_append_buffer(con->write_queue, p->b);
 744
 745         buffer_reset(con->physical.path);
 746
 747         con->file_finished = 1;
 748         con->file_started  = 1;
 749
 750         return 0;
 751 }
 752
 753
 754 #define PATCH(x) \
 755         p->conf.x = s->x;
 756 static int mod_compress_patch_connection(server *srv, connection *con, plugin_data *p) {
 757         size_t i, j;
 758         plugin_config *s = p->config_storage[0];
 759
 760         PATCH(compress_cache_dir);
 761         PATCH(compress);
 762         PATCH(compress_max_filesize);
 763         PATCH(allowed_encodings);
 764         PATCH(max_loadavg);
 765
 766         /* skip the first, the global context */
 767         for (i = 1; i < srv->config_context->used; i++) {
 768                 data_config *dc = (data_config *)srv->config_context->data[i];
 769                 s = p->config_storage[i];
 770
 771                 /* condition didn't match */
 772                 if (!config_check_cond(srv, con, dc)) continue;
 773
 774                 /* merge config */
 775                 for (j = 0; j < dc->value->used; j++) {
 776                         data_unset *du = dc->value->data[j];
 777
 778                         if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.cache-dir"))) {
 779                                 PATCH(compress_cache_dir);
 780                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.filetype"))) {
 781                                 PATCH(compress);
 782                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.max-filesize"))) {
 783                                 PATCH(compress_max_filesize);
 784                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.allowed-encodings"))) {
 785                                 PATCH(allowed_encodings);
 786                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.max-loadavg"))) {
 787                                 PATCH(max_loadavg);
 788                         }
 789                 }
 790         }
 791
 792         return 0;
 793 }
 794 #undef PATCH
 795
 796 static int mod_compress_contains_encoding(const char *headervalue, const char *encoding, size_t len) {
 797         const char *m = headervalue;
 798         do {
 799                 while (*m == ',' || *m == ' ' || *m == '\t') {
 800                         ++m;
 801                 }
 802                 if (0 == strncasecmp(m, encoding, len)) {
 803                         /*(not a full HTTP field parse: not parsing for q-values and not handling q=0)*/
 804                         m += len;
 805                         if (*m == '\0' || *m == ',' || *m == ';' || *m == ' ' || *m == '\t')
 806                                 return 1;
 807                 } else if (*m != '\0') {
 808                         ++m;
 809                 }
 810         } while ((m = strchr(m, ',')));
 811         return 0;
 812 }
 813
 814 PHYSICALPATH_FUNC(mod_compress_physical) {
 815         plugin_data *p = p_d;
 816         size_t m;
 817         off_t max_fsize;
 818         stat_cache_entry *sce = NULL;
 819         buffer *mtime = NULL;
 820         buffer *content_type;
 821
 822         if (con->mode != DIRECT || con->http_status) return HANDLER_GO_ON;
 823
 824         /* only GET and POST can get compressed */
 825         if (con->request.http_method != HTTP_METHOD_GET &&
 826             con->request.http_method != HTTP_METHOD_POST) {
 827                 return HANDLER_GO_ON;
 828         }
 829
 830         if (buffer_string_is_empty(con->physical.path)) {
 831                 return HANDLER_GO_ON;
 832         }
 833
 834         mod_compress_patch_connection(srv, con, p);
 835
 836         max_fsize = p->conf.compress_max_filesize;
 837
 838         if (con->conf.log_request_handling) {
 839                 log_error_write(srv, __FILE__, __LINE__,  "s",  "-- handling file as static file");
 840         }
 841
 842         if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) {
 843                 con->http_status = 403;
 844
 845                 log_error_write(srv, __FILE__, __LINE__, "sbsb",
 846                                 "not a regular file:", con->uri.path,
 847                                 "->", con->physical.path);
 848
 849                 return HANDLER_FINISHED;
 850         }
 851
 852         /* we only handle regular files */
 853 #ifdef HAVE_LSTAT
 854         if ((sce->is_symlink == 1) && !con->conf.follow_symlink) {
 855                 return HANDLER_GO_ON;
 856         }
 857 #endif
 858         if (!S_ISREG(sce->st.st_mode)) {
 859                 return HANDLER_GO_ON;
 860         }
 861
 862         /* don't compress files that are too large as we need to much time to handle them */
 863         if (max_fsize && (sce->st.st_size >> 10) > max_fsize) return HANDLER_GO_ON;
 864
 865         /* don't try to compress files less than 128 bytes
 866          *
 867          * - extra overhead for compression
 868          * - mmap() fails for st_size = 0 :)
 869          */
 870         if (sce->st.st_size < 128) return HANDLER_GO_ON;
 871
 872         /* check if mimetype is in compress-config */
 873         content_type = NULL;
 874         if (sce->content_type->ptr) {
 875                 char *c;
 876                 if ( (c = strchr(sce->content_type->ptr, ';')) != NULL) {
 877                         content_type = srv->tmp_buf;
 878                         buffer_copy_string_len(content_type, sce->content_type->ptr, c - sce->content_type->ptr);
 879                 }
 880         }
 881
 882         for (m = 0; m < p->conf.compress->used; m++) {
 883                 data_string *compress_ds = (data_string *)p->conf.compress->data[m];
 884
 885                 if (!compress_ds) {
 886                         log_error_write(srv, __FILE__, __LINE__, "sbb", "evil", con->physical.path, con->uri.path);
 887
 888                         return HANDLER_GO_ON;
 889                 }
 890
 891                 if (buffer_is_equal(compress_ds->value, sce->content_type)
 892                     || (content_type && buffer_is_equal(compress_ds->value, content_type))) {
 893                         /* mimetype found */
 894                         data_string *ds;
 895
 896                         /* the response might change according to Accept-Encoding */
 897                         response_header_insert(srv, con, CONST_STR_LEN("Vary"), CONST_STR_LEN("Accept-Encoding"));
 898
 899                         if (NULL != (ds = (data_string *)array_get_element(con->request.headers, "Accept-Encoding"))) {
 900                                 int accept_encoding = 0;
 901                                 char *value = ds->value->ptr;
 902                                 int matched_encodings = 0;
 903                                 int use_etag = sce->etag != NULL && sce->etag->ptr != NULL;
 904
 905                                 /* get client side support encodings */
 906 #ifdef USE_ZLIB
 907                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("gzip"))) accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP;
 908                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("x-gzip"))) accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP;
 909                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("deflate"))) accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE;
 910                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("compress"))) accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS;
 911 #endif
 912 #ifdef USE_BZ2LIB
 913                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("bzip2"))) accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2;
 914                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("x-bzip2"))) accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2;
 915 #endif
 916                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("identity"))) accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY;
 917
 918                                 /* find matching entries */
 919                                 matched_encodings = accept_encoding & p->conf.allowed_encodings;
 920
 921                                 if (matched_encodings) {
 922                                         static const char dflt_gzip[] = "gzip";
 923                                         static const char dflt_x_gzip[] = "x-gzip";
 924                                         static const char dflt_deflate[] = "deflate";
 925                                         static const char dflt_bzip2[] = "bzip2";
 926                                         static const char dflt_x_bzip2[] = "x-bzip2";
 927
 928                                         const char *compression_name = NULL;
 929                                         int compression_type = 0;
 930
 931                                         mtime = strftime_cache_get(srv, sce->st.st_mtime);
 932
 933                                         /* try matching original etag of uncompressed version */
 934                                         if (use_etag) {
 935                                                 etag_mutate(con->physical.etag, sce->etag);
 936                                                 if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) {
 937                                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 938                                                         response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 939                                                         response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 940                                                         return HANDLER_FINISHED;
 941                                                 }
 942                                         }
 943
 944                                         /* select best matching encoding */
 945                                         if (matched_encodings & HTTP_ACCEPT_ENCODING_BZIP2) {
 946                                                 compression_type = HTTP_ACCEPT_ENCODING_BZIP2;
 947                                                 compression_name = dflt_bzip2;
 948                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_X_BZIP2) {
 949                                                 compression_type = HTTP_ACCEPT_ENCODING_X_BZIP2;
 950                                                 compression_name = dflt_x_bzip2;
 951                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_GZIP) {
 952                                                 compression_type = HTTP_ACCEPT_ENCODING_GZIP;
 953                                                 compression_name = dflt_gzip;
 954                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_X_GZIP) {
 955                                                 compression_type = HTTP_ACCEPT_ENCODING_X_GZIP;
 956                                                 compression_name = dflt_x_gzip;
 957                                         } else {
 958                                                 force_assert(matched_encodings & HTTP_ACCEPT_ENCODING_DEFLATE);
 959                                                 compression_type = HTTP_ACCEPT_ENCODING_DEFLATE;
 960                                                 compression_name = dflt_deflate;
 961                                         }
 962
 963                                         if (use_etag) {
 964                                                 /* try matching etag of compressed version */
 965                                                 buffer_copy_buffer(srv->tmp_buf, sce->etag);
 966                                                 buffer_append_string_len(srv->tmp_buf, CONST_STR_LEN("-"));
 967                                                 buffer_append_string(srv->tmp_buf, compression_name);
 968                                                 etag_mutate(con->physical.etag, srv->tmp_buf);
 969                                         }
 970
 971                                         if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) {
 972                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name));
 973                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 974                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 975                                                 if (use_etag) {
 976                                                         response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 977                                                 }
 978                                                 return HANDLER_FINISHED;
 979                                         }
 980
 981                                         /* deflate it */
 982                                         if (use_etag && !buffer_string_is_empty(p->conf.compress_cache_dir)) {
 983                                                 if (0 != deflate_file_to_file(srv, con, p, con->physical.path, sce, compression_type))
 984                                                         return HANDLER_GO_ON;
 985                                         } else {
 986                                                 if (0 != deflate_file_to_buffer(srv, con, p, con->physical.path, sce, compression_type))
 987                                                         return HANDLER_GO_ON;
 988                                         }
 989                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name));
 990                                         response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 991                                         if (use_etag) {
 992                                                 response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 993                                         }
 994                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 995                                         /* let mod_staticfile handle the cached compressed files, physical path was modified */
 996                                         return (use_etag && !buffer_string_is_empty(p->conf.compress_cache_dir)) ? HANDLER_GO_ON : HANDLER_FINISHED;
 997                                 }
 998                         }
 999                 }
1000         }
1001
1002         return HANDLER_GO_ON;
1003 }
1004
1005 int mod_compress_plugin_init(plugin *p);
1006 int mod_compress_plugin_init(plugin *p) {
1007         p->version     = LIGHTTPD_VERSION_ID;
1008         p->name        = buffer_init_string("compress");
1009
1010         p->init        = mod_compress_init;
1011         p->set_defaults = mod_compress_setdefaults;
1012         p->handle_subrequest_start  = mod_compress_physical;
1013         p->cleanup     = mod_compress_free;
1014
1015         p->data        = NULL;
1016
1017         return 0;
1018 }