src/mod_compress.c

   1 #include "first.h"
   2
   3 #include "base.h"
   4 #include "log.h"
   5 #include "buffer.h"
   6 #include "response.h"
   7 #include "stat_cache.h"
   8
   9 #include "plugin.h"
  10
  11 #include "crc32.h"
  12 #include "etag.h"
  13
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include "sys-strings.h"
  17
  18 #include <fcntl.h>
  19 #include <unistd.h>
  20 #include <stdlib.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <time.h>
  24
  25 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ
  26 # define USE_ZLIB
  27 # include <zlib.h>
  28 #endif
  29
  30 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
  31 # define USE_BZ2LIB
  32 /* we don't need stdio interface */
  33 # define BZ_NO_STDIO
  34 # include <bzlib.h>
  35 #endif
  36
  37 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
  38 #define USE_MMAP
  39
  40 #include "sys-mmap.h"
  41 #include <setjmp.h>
  42 #include <signal.h>
  43
  44 static volatile int sigbus_jmp_valid;
  45 static sigjmp_buf sigbus_jmp;
  46
  47 static void sigbus_handler(int sig) {
  48         UNUSED(sig);
  49         if (sigbus_jmp_valid) siglongjmp(sigbus_jmp, 1);
  50         log_failed_assert(__FILE__, __LINE__, "SIGBUS");
  51 }
  52 #endif
  53
  54 /* request: accept-encoding */
  55 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0)
  56 #define HTTP_ACCEPT_ENCODING_GZIP     BV(1)
  57 #define HTTP_ACCEPT_ENCODING_DEFLATE  BV(2)
  58 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3)
  59 #define HTTP_ACCEPT_ENCODING_BZIP2    BV(4)
  60 #define HTTP_ACCEPT_ENCODING_X_GZIP   BV(5)
  61 #define HTTP_ACCEPT_ENCODING_X_BZIP2  BV(6)
  62
  63 #ifdef __WIN32
  64 # define mkdir(x,y) mkdir(x)
  65 #endif
  66
  67 typedef struct {
  68         buffer *compress_cache_dir;
  69         array  *compress;
  70         off_t   compress_max_filesize; /** max filesize in kb */
  71         int     allowed_encodings;
  72         double  max_loadavg;
  73 } plugin_config;
  74
  75 typedef struct {
  76         PLUGIN_DATA;
  77         buffer *ofn;
  78         buffer *b;
  79
  80         plugin_config **config_storage;
  81         plugin_config conf;
  82 } plugin_data;
  83
  84 INIT_FUNC(mod_compress_init) {
  85         plugin_data *p;
  86
  87         p = calloc(1, sizeof(*p));
  88
  89         p->ofn = buffer_init();
  90         p->b = buffer_init();
  91
  92         return p;
  93 }
  94
  95 FREE_FUNC(mod_compress_free) {
  96         plugin_data *p = p_d;
  97
  98         UNUSED(srv);
  99
 100         if (!p) return HANDLER_GO_ON;
 101
 102         buffer_free(p->ofn);
 103         buffer_free(p->b);
 104
 105         if (p->config_storage) {
 106                 size_t i;
 107                 for (i = 0; i < srv->config_context->used; i++) {
 108                         plugin_config *s = p->config_storage[i];
 109
 110                         if (NULL == s) continue;
 111
 112                         array_free(s->compress);
 113                         buffer_free(s->compress_cache_dir);
 114
 115                         free(s);
 116                 }
 117                 free(p->config_storage);
 118         }
 119
 120
 121         free(p);
 122
 123         return HANDLER_GO_ON;
 124 }
 125
 126 /* 0 on success, -1 for error */
 127 static int mkdir_recursive(char *dir) {
 128         char *p = dir;
 129
 130         if (!dir || !dir[0])
 131                 return 0;
 132
 133         while ((p = strchr(p + 1, '/')) != NULL) {
 134
 135                 *p = '\0';
 136                 if ((mkdir(dir, 0700) != 0) && (errno != EEXIST)) {
 137                         *p = '/';
 138                         return -1;
 139                 }
 140
 141                 *p++ = '/';
 142                 if (!*p) return 0; /* Ignore trailing slash */
 143         }
 144
 145         return (mkdir(dir, 0700) != 0) && (errno != EEXIST) ? -1 : 0;
 146 }
 147
 148 /* 0 on success, -1 for error */
 149 static int mkdir_for_file(char *filename) {
 150         char *p = filename;
 151
 152         if (!filename || !filename[0])
 153                 return -1;
 154
 155         while ((p = strchr(p + 1, '/')) != NULL) {
 156
 157                 *p = '\0';
 158                 if ((mkdir(filename, 0700) != 0) && (errno != EEXIST)) {
 159                         *p = '/';
 160                         return -1;
 161                 }
 162
 163                 *p++ = '/';
 164                 if (!*p) return -1; /* Unexpected trailing slash in filename */
 165         }
 166
 167         return 0;
 168 }
 169
 170 SETDEFAULTS_FUNC(mod_compress_setdefaults) {
 171         plugin_data *p = p_d;
 172         size_t i = 0;
 173
 174         config_values_t cv[] = {
 175                 { "compress.cache-dir",             NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
 176                 { "compress.filetype",              NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
 177                 { "compress.max-filesize",          NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
 178                 { "compress.allowed-encodings",     NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
 179                 { "compress.max-loadavg",           NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
 180                 { NULL,                             NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET }
 181         };
 182
 183         p->config_storage = calloc(1, srv->config_context->used * sizeof(plugin_config *));
 184
 185         for (i = 0; i < srv->config_context->used; i++) {
 186                 data_config const* config = (data_config const*)srv->config_context->data[i];
 187                 plugin_config *s;
 188                 array  *encodings_arr = array_init();
 189
 190                 s = calloc(1, sizeof(plugin_config));
 191                 s->compress_cache_dir = buffer_init();
 192                 s->compress = array_init();
 193                 s->compress_max_filesize = 0;
 194                 s->allowed_encodings = 0;
 195                 s->max_loadavg = 0.0;
 196
 197                 cv[0].destination = s->compress_cache_dir;
 198                 cv[1].destination = s->compress;
 199                 cv[2].destination = &(s->compress_max_filesize);
 200                 cv[3].destination = encodings_arr; /* temp array for allowed encodings list */
 201                 cv[4].destination = srv->tmp_buf;
 202                 buffer_string_set_length(srv->tmp_buf, 0);
 203
 204                 p->config_storage[i] = s;
 205
 206                 if (0 != config_insert_values_global(srv, config->value, cv, i == 0 ? T_CONFIG_SCOPE_SERVER : T_CONFIG_SCOPE_CONNECTION)) {
 207                         return HANDLER_ERROR;
 208                 }
 209
 210                 if (!buffer_string_is_empty(srv->tmp_buf)) {
 211                         s->max_loadavg = strtod(srv->tmp_buf->ptr, NULL);
 212                 }
 213
 214                 if (!array_is_vlist(s->compress)) {
 215                         log_error_write(srv, __FILE__, __LINE__, "s",
 216                                         "unexpected value for compress.filetype; expected list of \"mimetype\"");
 217                         return HANDLER_ERROR;
 218                 }
 219
 220                 if (!array_is_vlist(encodings_arr)) {
 221                         log_error_write(srv, __FILE__, __LINE__, "s",
 222                                         "unexpected value for compress.allowed-encodings; expected list of \"encoding\"");
 223                         return HANDLER_ERROR;
 224                 }
 225
 226                 if (encodings_arr->used) {
 227                         size_t j = 0;
 228                         for (j = 0; j < encodings_arr->used; j++) {
 229 #if defined(USE_ZLIB) || defined(USE_BZ2LIB)
 230                                 data_string *ds = (data_string *)encodings_arr->data[j];
 231 #endif
 232 #ifdef USE_ZLIB
 233                                 if (NULL != strstr(ds->value->ptr, "gzip"))
 234                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_GZIP | HTTP_ACCEPT_ENCODING_X_GZIP;
 235                                 if (NULL != strstr(ds->value->ptr, "x-gzip"))
 236                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_X_GZIP;
 237                                 if (NULL != strstr(ds->value->ptr, "deflate"))
 238                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_DEFLATE;
 239                                 /*
 240                                 if (NULL != strstr(ds->value->ptr, "compress"))
 241                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_COMPRESS;
 242                                 */
 243 #endif
 244 #ifdef USE_BZ2LIB
 245                                 if (NULL != strstr(ds->value->ptr, "bzip2"))
 246                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2 | HTTP_ACCEPT_ENCODING_X_BZIP2;
 247                                 if (NULL != strstr(ds->value->ptr, "x-bzip2"))
 248                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_X_BZIP2;
 249 #endif
 250                         }
 251                 } else {
 252                         /* default encodings */
 253                         s->allowed_encodings = 0
 254 #ifdef USE_ZLIB
 255                                 | HTTP_ACCEPT_ENCODING_GZIP | HTTP_ACCEPT_ENCODING_X_GZIP | HTTP_ACCEPT_ENCODING_DEFLATE
 256 #endif
 257 #ifdef USE_BZ2LIB
 258                                 | HTTP_ACCEPT_ENCODING_BZIP2 | HTTP_ACCEPT_ENCODING_X_BZIP2
 259 #endif
 260                                 ;
 261                 }
 262
 263                 array_free(encodings_arr);
 264
 265                 if (!buffer_string_is_empty(s->compress_cache_dir)) {
 266                         struct stat st;
 267                         mkdir_recursive(s->compress_cache_dir->ptr);
 268
 269                         if (0 != stat(s->compress_cache_dir->ptr, &st)) {
 270                                 log_error_write(srv, __FILE__, __LINE__, "sbs", "can't stat compress.cache-dir",
 271                                                 s->compress_cache_dir, strerror(errno));
 272
 273                                 return HANDLER_ERROR;
 274                         }
 275                 }
 276         }
 277
 278         return HANDLER_GO_ON;
 279
 280 }
 281
 282 #ifdef USE_ZLIB
 283 static int deflate_file_to_buffer_gzip(server *srv, connection *con, plugin_data *p, char *start, off_t st_size, time_t mtime) {
 284         unsigned char *c;
 285         unsigned long crc;
 286         z_stream z;
 287         size_t outlen;
 288
 289         UNUSED(srv);
 290         UNUSED(con);
 291
 292         z.zalloc = Z_NULL;
 293         z.zfree = Z_NULL;
 294         z.opaque = Z_NULL;
 295
 296         if (Z_OK != deflateInit2(&z,
 297                                  Z_DEFAULT_COMPRESSION,
 298                                  Z_DEFLATED,
 299                                  -MAX_WBITS,  /* supress zlib-header */
 300                                  8,
 301                                  Z_DEFAULT_STRATEGY)) {
 302                 return -1;
 303         }
 304
 305         z.next_in = (unsigned char *)start;
 306         z.avail_in = st_size;
 307         z.total_in = 0;
 308
 309
 310         buffer_string_prepare_copy(p->b, (z.avail_in * 1.1) + 12 + 18);
 311
 312         /* write gzip header */
 313
 314         c = (unsigned char *)p->b->ptr;
 315         c[0] = 0x1f;
 316         c[1] = 0x8b;
 317         c[2] = Z_DEFLATED;
 318         c[3] = 0; /* options */
 319         c[4] = (mtime >>  0) & 0xff;
 320         c[5] = (mtime >>  8) & 0xff;
 321         c[6] = (mtime >> 16) & 0xff;
 322         c[7] = (mtime >> 24) & 0xff;
 323         c[8] = 0x00; /* extra flags */
 324         c[9] = 0x03; /* UNIX */
 325
 326         outlen = 10;
 327         z.next_out = (unsigned char *)p->b->ptr + outlen;
 328         z.avail_out = p->b->size - outlen - 9;
 329         z.total_out = 0;
 330
 331         if (Z_STREAM_END != deflate(&z, Z_FINISH)) {
 332                 deflateEnd(&z);
 333                 return -1;
 334         }
 335
 336         /* trailer */
 337         outlen += z.total_out;
 338
 339         crc = generate_crc32c(start, st_size);
 340
 341         c = (unsigned char *)p->b->ptr + outlen;
 342
 343         c[0] = (crc >>  0) & 0xff;
 344         c[1] = (crc >>  8) & 0xff;
 345         c[2] = (crc >> 16) & 0xff;
 346         c[3] = (crc >> 24) & 0xff;
 347         c[4] = (z.total_in >>  0) & 0xff;
 348         c[5] = (z.total_in >>  8) & 0xff;
 349         c[6] = (z.total_in >> 16) & 0xff;
 350         c[7] = (z.total_in >> 24) & 0xff;
 351         outlen += 8;
 352         buffer_commit(p->b, outlen);
 353
 354         if (Z_OK != deflateEnd(&z)) {
 355                 return -1;
 356         }
 357
 358         return 0;
 359 }
 360
 361 static int deflate_file_to_buffer_deflate(server *srv, connection *con, plugin_data *p, unsigned char *start, off_t st_size) {
 362         z_stream z;
 363
 364         UNUSED(srv);
 365         UNUSED(con);
 366
 367         z.zalloc = Z_NULL;
 368         z.zfree = Z_NULL;
 369         z.opaque = Z_NULL;
 370
 371         if (Z_OK != deflateInit2(&z,
 372                                  Z_DEFAULT_COMPRESSION,
 373                                  Z_DEFLATED,
 374                                  -MAX_WBITS,  /* supress zlib-header */
 375                                  8,
 376                                  Z_DEFAULT_STRATEGY)) {
 377                 return -1;
 378         }
 379
 380         z.next_in = start;
 381         z.avail_in = st_size;
 382         z.total_in = 0;
 383
 384         buffer_string_prepare_copy(p->b, (z.avail_in * 1.1) + 12);
 385
 386         z.next_out = (unsigned char *)p->b->ptr;
 387         z.avail_out = p->b->size - 1;
 388         z.total_out = 0;
 389
 390         if (Z_STREAM_END != deflate(&z, Z_FINISH)) {
 391                 deflateEnd(&z);
 392                 return -1;
 393         }
 394
 395         if (Z_OK != deflateEnd(&z)) {
 396                 return -1;
 397         }
 398
 399         /* trailer */
 400         buffer_commit(p->b, z.total_out);
 401
 402         return 0;
 403 }
 404
 405 #endif
 406
 407 #ifdef USE_BZ2LIB
 408 static int deflate_file_to_buffer_bzip2(server *srv, connection *con, plugin_data *p, unsigned char *start, off_t st_size) {
 409         bz_stream bz;
 410
 411         UNUSED(srv);
 412         UNUSED(con);
 413
 414         bz.bzalloc = NULL;
 415         bz.bzfree = NULL;
 416         bz.opaque = NULL;
 417
 418         if (BZ_OK != BZ2_bzCompressInit(&bz,
 419                                         9, /* blocksize = 900k */
 420                                         0, /* no output */
 421                                         0)) { /* workFactor: default */
 422                 return -1;
 423         }
 424
 425         bz.next_in = (char *)start;
 426         bz.avail_in = st_size;
 427         bz.total_in_lo32 = 0;
 428         bz.total_in_hi32 = 0;
 429
 430         buffer_string_prepare_copy(p->b, (bz.avail_in * 1.1) + 12);
 431
 432         bz.next_out = p->b->ptr;
 433         bz.avail_out = p->b->size - 1;
 434         bz.total_out_lo32 = 0;
 435         bz.total_out_hi32 = 0;
 436
 437         if (BZ_STREAM_END != BZ2_bzCompress(&bz, BZ_FINISH)) {
 438                 BZ2_bzCompressEnd(&bz);
 439                 return -1;
 440         }
 441
 442         if (BZ_OK != BZ2_bzCompressEnd(&bz)) {
 443                 return -1;
 444         }
 445
 446         /* file is too large for now */
 447         if (bz.total_out_hi32) return -1;
 448
 449         /* trailer */
 450         buffer_commit(p->b, bz.total_out_lo32);
 451
 452         return 0;
 453 }
 454 #endif
 455
 456 static void mod_compress_note_ratio(server *srv, connection *con, off_t in, off_t out) {
 457     /* store compression ratio in con->environment
 458      * for possible logging by mod_accesslog
 459      * (late in response handling, so not seen by most other modules) */
 460     /*(should be called only at end of successful response compression)*/
 461     char ratio[LI_ITOSTRING_LENGTH];
 462     if (0 == in) return;
 463     li_itostrn(ratio, sizeof(ratio), out * 100 / in);
 464     array_set_key_value(con->environment,
 465                         CONST_STR_LEN("ratio"),
 466                         ratio, strlen(ratio));
 467     UNUSED(srv);
 468 }
 469
 470 static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) {
 471         int ifd, ofd;
 472         int ret;
 473 #ifdef USE_MMAP
 474         volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */
 475 #endif
 476         void *start;
 477         const char *filename = fn->ptr;
 478         stat_cache_entry *sce_ofn;
 479         ssize_t r;
 480
 481         /* overflow */
 482         if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1;
 483
 484         /* don't mmap files > 128Mb
 485          *
 486          * we could use a sliding window, but currently there is no need for it
 487          */
 488
 489         if (sce->st.st_size > 128 * 1024 * 1024) return -1;
 490
 491         buffer_reset(p->ofn);
 492         buffer_copy_buffer(p->ofn, p->conf.compress_cache_dir);
 493         buffer_append_slash(p->ofn);
 494
 495         if (0 == strncmp(con->physical.path->ptr, con->physical.doc_root->ptr, buffer_string_length(con->physical.doc_root))) {
 496                 buffer_append_string(p->ofn, con->physical.path->ptr + buffer_string_length(con->physical.doc_root));
 497         } else {
 498                 buffer_append_string_buffer(p->ofn, con->uri.path);
 499         }
 500
 501         switch(type) {
 502         case HTTP_ACCEPT_ENCODING_GZIP:
 503         case HTTP_ACCEPT_ENCODING_X_GZIP:
 504                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-gzip-"));
 505                 break;
 506         case HTTP_ACCEPT_ENCODING_DEFLATE:
 507                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-deflate-"));
 508                 break;
 509         case HTTP_ACCEPT_ENCODING_BZIP2:
 510         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 511                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-bzip2-"));
 512                 break;
 513         default:
 514                 log_error_write(srv, __FILE__, __LINE__, "sd", "unknown compression type", type);
 515                 return -1;
 516         }
 517
 518         buffer_append_string_buffer(p->ofn, sce->etag);
 519
 520         if (HANDLER_ERROR != stat_cache_get_entry(srv, con, p->ofn, &sce_ofn)) {
 521                 if (0 == sce->st.st_size) return -1; /* cache file being created */
 522                 /* cache-entry exists */
 523 #if 0
 524                 log_error_write(srv, __FILE__, __LINE__, "bs", p->ofn, "compress-cache hit");
 525 #endif
 526                 mod_compress_note_ratio(srv, con, sce->st.st_size, sce_ofn->st.st_size);
 527                 buffer_copy_buffer(con->physical.path, p->ofn);
 528                 return 0;
 529         }
 530
 531         if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) {
 532                 return -1;
 533         }
 534
 535         if (-1 == mkdir_for_file(p->ofn->ptr)) {
 536                 log_error_write(srv, __FILE__, __LINE__, "sb", "couldn't create directory for file", p->ofn);
 537                 return -1;
 538         }
 539
 540         if (-1 == (ofd = open(p->ofn->ptr, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0600))) {
 541                 if (errno == EEXIST) {
 542                         return -1; /* cache file being created */
 543                 }
 544
 545                 log_error_write(srv, __FILE__, __LINE__, "sbss", "creating cachefile", p->ofn, "failed", strerror(errno));
 546
 547                 return -1;
 548         }
 549 #if 0
 550         log_error_write(srv, __FILE__, __LINE__, "bs", p->ofn, "compress-cache miss");
 551 #endif
 552         if (-1 == (ifd = open(filename, O_RDONLY | O_BINARY))) {
 553                 log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno));
 554
 555                 close(ofd);
 556
 557                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 558                 if (-1 == unlink(p->ofn->ptr)) {
 559                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 560                 }
 561
 562                 return -1;
 563         }
 564
 565 #ifdef USE_MMAP
 566         if (MAP_FAILED != (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) {
 567                 mapped = 1;
 568                 signal(SIGBUS, sigbus_handler);
 569                 sigbus_jmp_valid = 1;
 570                 if (0 != sigsetjmp(sigbus_jmp, 1)) {
 571                         sigbus_jmp_valid = 0;
 572
 573                         log_error_write(srv, __FILE__, __LINE__, "sbd", "SIGBUS in mmap:",
 574                                 fn, ifd);
 575
 576                         munmap(start, sce->st.st_size);
 577                         close(ofd);
 578                         close(ifd);
 579
 580                         /* Remove the incomplete cache file, so that later hits aren't served from it */
 581                         if (-1 == unlink(p->ofn->ptr)) {
 582                                 log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 583                         }
 584
 585                         return -1;
 586                 }
 587         } else
 588 #endif  /* FIXME: might attempt to read very large file completely into memory; see compress.max-filesize config option */
 589         if (NULL == (start = malloc(sce->st.st_size)) || sce->st.st_size != read(ifd, start, sce->st.st_size)) {
 590                 log_error_write(srv, __FILE__, __LINE__, "sbss", "reading", fn, "failed", strerror(errno));
 591
 592                 close(ofd);
 593                 close(ifd);
 594                 free(start);
 595
 596                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 597                 if (-1 == unlink(p->ofn->ptr)) {
 598                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 599                 }
 600
 601                 return -1;
 602         }
 603
 604         ret = -1;
 605         switch(type) {
 606 #ifdef USE_ZLIB
 607         case HTTP_ACCEPT_ENCODING_GZIP:
 608         case HTTP_ACCEPT_ENCODING_X_GZIP:
 609                 ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime);
 610                 break;
 611         case HTTP_ACCEPT_ENCODING_DEFLATE:
 612                 ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size);
 613                 break;
 614 #endif
 615 #ifdef USE_BZ2LIB
 616         case HTTP_ACCEPT_ENCODING_BZIP2:
 617         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 618                 ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size);
 619                 break;
 620 #endif
 621         }
 622
 623         if (ret == 0) {
 624                 r = write(ofd, CONST_BUF_LEN(p->b));
 625                 if (-1 == r) {
 626                         log_error_write(srv, __FILE__, __LINE__, "sbss", "writing cachefile", p->ofn, "failed:", strerror(errno));
 627                         ret = -1;
 628                 } else if ((size_t)r != buffer_string_length(p->b)) {
 629                         log_error_write(srv, __FILE__, __LINE__, "sbs", "writing cachefile", p->ofn, "failed: not enough bytes written");
 630                         ret = -1;
 631                 }
 632         }
 633
 634 #ifdef USE_MMAP
 635         if (mapped) {
 636                 sigbus_jmp_valid = 0;
 637                 munmap(start, sce->st.st_size);
 638         } else
 639 #endif
 640                 free(start);
 641
 642         close(ifd);
 643
 644         if (0 != close(ofd) || ret != 0) {
 645                 if (0 == ret) {
 646                         log_error_write(srv, __FILE__, __LINE__, "sbss", "writing cachefile", p->ofn, "failed:", strerror(errno));
 647                 }
 648
 649                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 650                 if (-1 == unlink(p->ofn->ptr)) {
 651                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 652                 }
 653
 654                 return -1;
 655         }
 656
 657         buffer_copy_buffer(con->physical.path, p->ofn);
 658         mod_compress_note_ratio(srv, con, sce->st.st_size,
 659                                 (off_t)buffer_string_length(p->b));
 660
 661         return 0;
 662 }
 663
 664 static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) {
 665         int ifd;
 666         int ret = -1;
 667 #ifdef USE_MMAP
 668         volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */
 669 #endif
 670         void *start;
 671
 672         /* overflow */
 673         if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1;
 674
 675         /* don't mmap files > 128M
 676          *
 677          * we could use a sliding window, but currently there is no need for it
 678          */
 679
 680         if (sce->st.st_size > 128 * 1024 * 1024) return -1;
 681
 682         if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) {
 683                 return -1;
 684         }
 685
 686         if (-1 == (ifd = open(fn->ptr, O_RDONLY | O_BINARY))) {
 687                 log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno));
 688
 689                 return -1;
 690         }
 691
 692 #ifdef USE_MMAP
 693         if (MAP_FAILED != (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) {
 694                 mapped = 1;
 695                 signal(SIGBUS, sigbus_handler);
 696                 sigbus_jmp_valid = 1;
 697                 if (0 != sigsetjmp(sigbus_jmp, 1)) {
 698                         sigbus_jmp_valid = 0;
 699
 700                         log_error_write(srv, __FILE__, __LINE__, "sbd", "SIGBUS in mmap:",
 701                                 fn, ifd);
 702
 703                         munmap(start, sce->st.st_size);
 704                         close(ifd);
 705                         return -1;
 706                 }
 707         } else
 708 #endif  /* FIXME: might attempt to read very large file completely into memory; see compress.max-filesize config option */
 709         if (NULL == (start = malloc(sce->st.st_size)) || sce->st.st_size != read(ifd, start, sce->st.st_size)) {
 710                 log_error_write(srv, __FILE__, __LINE__, "sbss", "reading", fn, "failed", strerror(errno));
 711
 712                 close(ifd);
 713                 free(start);
 714                 return -1;
 715         }
 716
 717         switch(type) {
 718 #ifdef USE_ZLIB
 719         case HTTP_ACCEPT_ENCODING_GZIP:
 720         case HTTP_ACCEPT_ENCODING_X_GZIP:
 721                 ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime);
 722                 break;
 723         case HTTP_ACCEPT_ENCODING_DEFLATE:
 724                 ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size);
 725                 break;
 726 #endif
 727 #ifdef USE_BZ2LIB
 728         case HTTP_ACCEPT_ENCODING_BZIP2:
 729         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 730                 ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size);
 731                 break;
 732 #endif
 733         default:
 734                 ret = -1;
 735                 break;
 736         }
 737
 738 #ifdef USE_MMAP
 739         if (mapped) {
 740                 sigbus_jmp_valid = 0;
 741                 munmap(start, sce->st.st_size);
 742         } else
 743 #endif
 744                 free(start);
 745
 746         close(ifd);
 747
 748         if (ret != 0) return -1;
 749
 750         mod_compress_note_ratio(srv, con, sce->st.st_size,
 751                                 (off_t)buffer_string_length(p->b));
 752         chunkqueue_reset(con->write_queue);
 753         chunkqueue_append_buffer(con->write_queue, p->b);
 754
 755         buffer_reset(con->physical.path);
 756
 757         con->file_finished = 1;
 758         con->file_started  = 1;
 759
 760         return 0;
 761 }
 762
 763
 764 #define PATCH(x) \
 765         p->conf.x = s->x;
 766 static int mod_compress_patch_connection(server *srv, connection *con, plugin_data *p) {
 767         size_t i, j;
 768         plugin_config *s = p->config_storage[0];
 769
 770         PATCH(compress_cache_dir);
 771         PATCH(compress);
 772         PATCH(compress_max_filesize);
 773         PATCH(allowed_encodings);
 774         PATCH(max_loadavg);
 775
 776         /* skip the first, the global context */
 777         for (i = 1; i < srv->config_context->used; i++) {
 778                 data_config *dc = (data_config *)srv->config_context->data[i];
 779                 s = p->config_storage[i];
 780
 781                 /* condition didn't match */
 782                 if (!config_check_cond(srv, con, dc)) continue;
 783
 784                 /* merge config */
 785                 for (j = 0; j < dc->value->used; j++) {
 786                         data_unset *du = dc->value->data[j];
 787
 788                         if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.cache-dir"))) {
 789                                 PATCH(compress_cache_dir);
 790                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.filetype"))) {
 791                                 PATCH(compress);
 792                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.max-filesize"))) {
 793                                 PATCH(compress_max_filesize);
 794                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.allowed-encodings"))) {
 795                                 PATCH(allowed_encodings);
 796                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.max-loadavg"))) {
 797                                 PATCH(max_loadavg);
 798                         }
 799                 }
 800         }
 801
 802         return 0;
 803 }
 804 #undef PATCH
 805
 806 static int mod_compress_contains_encoding(const char *headervalue, const char *encoding, size_t len) {
 807         const char *m = headervalue;
 808         do {
 809                 while (*m == ',' || *m == ' ' || *m == '\t') {
 810                         ++m;
 811                 }
 812                 if (0 == strncasecmp(m, encoding, len)) {
 813                         /*(not a full HTTP field parse: not parsing for q-values and not handling q=0)*/
 814                         m += len;
 815                         if (*m == '\0' || *m == ',' || *m == ';' || *m == ' ' || *m == '\t')
 816                                 return 1;
 817                 } else if (*m != '\0') {
 818                         ++m;
 819                 }
 820         } while ((m = strchr(m, ',')));
 821         return 0;
 822 }
 823
 824 PHYSICALPATH_FUNC(mod_compress_physical) {
 825         plugin_data *p = p_d;
 826         size_t m;
 827         off_t max_fsize;
 828         stat_cache_entry *sce = NULL;
 829         buffer *mtime = NULL;
 830         buffer *content_type;
 831
 832         if (con->mode != DIRECT || con->http_status) return HANDLER_GO_ON;
 833
 834         /* only GET and POST can get compressed */
 835         if (con->request.http_method != HTTP_METHOD_GET &&
 836             con->request.http_method != HTTP_METHOD_POST) {
 837                 return HANDLER_GO_ON;
 838         }
 839
 840         if (buffer_string_is_empty(con->physical.path)) {
 841                 return HANDLER_GO_ON;
 842         }
 843
 844         mod_compress_patch_connection(srv, con, p);
 845
 846         max_fsize = p->conf.compress_max_filesize;
 847
 848         if (con->conf.log_request_handling) {
 849                 log_error_write(srv, __FILE__, __LINE__,  "s",  "-- handling file as static file");
 850         }
 851
 852         if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) {
 853                 con->http_status = 403;
 854
 855                 log_error_write(srv, __FILE__, __LINE__, "sbsb",
 856                                 "not a regular file:", con->uri.path,
 857                                 "->", con->physical.path);
 858
 859                 return HANDLER_FINISHED;
 860         }
 861
 862         /* we only handle regular files */
 863 #ifdef HAVE_LSTAT
 864         if ((sce->is_symlink == 1) && !con->conf.follow_symlink) {
 865                 return HANDLER_GO_ON;
 866         }
 867 #endif
 868         if (!S_ISREG(sce->st.st_mode)) {
 869                 return HANDLER_GO_ON;
 870         }
 871
 872         /* don't compress files that are too large as we need to much time to handle them */
 873         if (max_fsize && (sce->st.st_size >> 10) > max_fsize) return HANDLER_GO_ON;
 874
 875         /* don't try to compress files less than 128 bytes
 876          *
 877          * - extra overhead for compression
 878          * - mmap() fails for st_size = 0 :)
 879          */
 880         if (sce->st.st_size < 128) return HANDLER_GO_ON;
 881
 882         /* check if mimetype is in compress-config */
 883         content_type = NULL;
 884         if (sce->content_type->ptr) {
 885                 char *c;
 886                 if ( (c = strchr(sce->content_type->ptr, ';')) != NULL) {
 887                         content_type = srv->tmp_buf;
 888                         buffer_copy_string_len(content_type, sce->content_type->ptr, c - sce->content_type->ptr);
 889                 }
 890         }
 891
 892         for (m = 0; m < p->conf.compress->used; m++) {
 893                 data_string *compress_ds = (data_string *)p->conf.compress->data[m];
 894
 895                 if (buffer_is_equal(compress_ds->value, sce->content_type)
 896                     || (content_type && buffer_is_equal(compress_ds->value, content_type))) {
 897                         /* mimetype found */
 898                         data_string *ds;
 899
 900                         /* the response might change according to Accept-Encoding */
 901                         response_header_insert(srv, con, CONST_STR_LEN("Vary"), CONST_STR_LEN("Accept-Encoding"));
 902
 903                         if (NULL != (ds = (data_string *)array_get_element(con->request.headers, "Accept-Encoding"))) {
 904                                 int accept_encoding = 0;
 905                                 char *value = ds->value->ptr;
 906                                 int matched_encodings = 0;
 907                                 int use_etag = sce->etag != NULL && sce->etag->ptr != NULL;
 908
 909                                 /* get client side support encodings */
 910 #ifdef USE_ZLIB
 911                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("gzip"))) accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP;
 912                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("x-gzip"))) accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP;
 913                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("deflate"))) accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE;
 914                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("compress"))) accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS;
 915 #endif
 916 #ifdef USE_BZ2LIB
 917                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("bzip2"))) accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2;
 918                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("x-bzip2"))) accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2;
 919 #endif
 920                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("identity"))) accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY;
 921
 922                                 /* find matching entries */
 923                                 matched_encodings = accept_encoding & p->conf.allowed_encodings;
 924
 925                                 if (matched_encodings) {
 926                                         static const char dflt_gzip[] = "gzip";
 927                                         static const char dflt_x_gzip[] = "x-gzip";
 928                                         static const char dflt_deflate[] = "deflate";
 929                                         static const char dflt_bzip2[] = "bzip2";
 930                                         static const char dflt_x_bzip2[] = "x-bzip2";
 931
 932                                         const char *compression_name = NULL;
 933                                         int compression_type = 0;
 934
 935                                         mtime = strftime_cache_get(srv, sce->st.st_mtime);
 936
 937                                         /* try matching original etag of uncompressed version */
 938                                         if (use_etag) {
 939                                                 etag_mutate(con->physical.etag, sce->etag);
 940                                                 if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) {
 941                                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 942                                                         response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 943                                                         response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 944                                                         return HANDLER_FINISHED;
 945                                                 }
 946                                         }
 947
 948                                         /* select best matching encoding */
 949                                         if (matched_encodings & HTTP_ACCEPT_ENCODING_BZIP2) {
 950                                                 compression_type = HTTP_ACCEPT_ENCODING_BZIP2;
 951                                                 compression_name = dflt_bzip2;
 952                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_X_BZIP2) {
 953                                                 compression_type = HTTP_ACCEPT_ENCODING_X_BZIP2;
 954                                                 compression_name = dflt_x_bzip2;
 955                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_GZIP) {
 956                                                 compression_type = HTTP_ACCEPT_ENCODING_GZIP;
 957                                                 compression_name = dflt_gzip;
 958                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_X_GZIP) {
 959                                                 compression_type = HTTP_ACCEPT_ENCODING_X_GZIP;
 960                                                 compression_name = dflt_x_gzip;
 961                                         } else {
 962                                                 force_assert(matched_encodings & HTTP_ACCEPT_ENCODING_DEFLATE);
 963                                                 compression_type = HTTP_ACCEPT_ENCODING_DEFLATE;
 964                                                 compression_name = dflt_deflate;
 965                                         }
 966
 967                                         if (use_etag) {
 968                                                 /* try matching etag of compressed version */
 969                                                 buffer_copy_buffer(srv->tmp_buf, sce->etag);
 970                                                 buffer_append_string_len(srv->tmp_buf, CONST_STR_LEN("-"));
 971                                                 buffer_append_string(srv->tmp_buf, compression_name);
 972                                                 etag_mutate(con->physical.etag, srv->tmp_buf);
 973                                         }
 974
 975                                         if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) {
 976                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name));
 977                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 978                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 979                                                 if (use_etag) {
 980                                                         response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 981                                                 }
 982                                                 return HANDLER_FINISHED;
 983                                         }
 984
 985                                         /* deflate it */
 986                                         if (use_etag && !buffer_string_is_empty(p->conf.compress_cache_dir)) {
 987                                                 if (0 != deflate_file_to_file(srv, con, p, con->physical.path, sce, compression_type))
 988                                                         return HANDLER_GO_ON;
 989                                         } else {
 990                                                 if (0 != deflate_file_to_buffer(srv, con, p, con->physical.path, sce, compression_type))
 991                                                         return HANDLER_GO_ON;
 992                                         }
 993                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name));
 994                                         response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 995                                         if (use_etag) {
 996                                                 response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 997                                         }
 998                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 999                                         /* let mod_staticfile handle the cached compressed files, physical path was modified */
1000                                         return (use_etag && !buffer_string_is_empty(p->conf.compress_cache_dir)) ? HANDLER_GO_ON : HANDLER_FINISHED;
1001                                 }
1002                         }
1003                 }
1004         }
1005
1006         return HANDLER_GO_ON;
1007 }
1008
1009 int mod_compress_plugin_init(plugin *p);
1010 int mod_compress_plugin_init(plugin *p) {
1011         p->version     = LIGHTTPD_VERSION_ID;
1012         p->name        = buffer_init_string("compress");
1013
1014         p->init        = mod_compress_init;
1015         p->set_defaults = mod_compress_setdefaults;
1016         p->handle_subrequest_start  = mod_compress_physical;
1017         p->cleanup     = mod_compress_free;
1018
1019         p->data        = NULL;
1020
1021         return 0;
1022 }