src/mod_compress.c

   1 #include "first.h"
   2
   3 #include "base.h"
   4 #include "log.h"
   5 #include "buffer.h"
   6 #include "response.h"
   7 #include "stat_cache.h"
   8
   9 #include "plugin.h"
  10
  11 #include "crc32.h"
  12 #include "etag.h"
  13
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16
  17 #include <assert.h>
  18 #include <fcntl.h>
  19 #include <unistd.h>
  20 #include <ctype.h>
  21 #include <stdlib.h>
  22 #include <string.h>
  23 #include <strings.h>
  24 #include <errno.h>
  25 #include <time.h>
  26
  27 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ
  28 # define USE_ZLIB
  29 # include <zlib.h>
  30 #endif
  31
  32 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
  33 # define USE_BZ2LIB
  34 /* we don't need stdio interface */
  35 # define BZ_NO_STDIO
  36 # include <bzlib.h>
  37 #endif
  38
  39 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
  40 #define USE_MMAP
  41
  42 #include "sys-mmap.h"
  43 #include <setjmp.h>
  44 #include <signal.h>
  45
  46 static volatile int sigbus_jmp_valid;
  47 static sigjmp_buf sigbus_jmp;
  48
  49 static void sigbus_handler(int sig) {
  50         UNUSED(sig);
  51         if (sigbus_jmp_valid) siglongjmp(sigbus_jmp, 1);
  52         log_failed_assert(__FILE__, __LINE__, "SIGBUS");
  53 }
  54 #endif
  55
  56 /* request: accept-encoding */
  57 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0)
  58 #define HTTP_ACCEPT_ENCODING_GZIP     BV(1)
  59 #define HTTP_ACCEPT_ENCODING_DEFLATE  BV(2)
  60 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3)
  61 #define HTTP_ACCEPT_ENCODING_BZIP2    BV(4)
  62 #define HTTP_ACCEPT_ENCODING_X_GZIP   BV(5)
  63 #define HTTP_ACCEPT_ENCODING_X_BZIP2  BV(6)
  64
  65 #ifdef __WIN32
  66 # define mkdir(x,y) mkdir(x)
  67 #endif
  68
  69 typedef struct {
  70         buffer *compress_cache_dir;
  71         array  *compress;
  72         off_t   compress_max_filesize; /** max filesize in kb */
  73         int     allowed_encodings;
  74 } plugin_config;
  75
  76 typedef struct {
  77         PLUGIN_DATA;
  78         buffer *ofn;
  79         buffer *b;
  80
  81         plugin_config **config_storage;
  82         plugin_config conf;
  83 } plugin_data;
  84
  85 INIT_FUNC(mod_compress_init) {
  86         plugin_data *p;
  87
  88         p = calloc(1, sizeof(*p));
  89
  90         p->ofn = buffer_init();
  91         p->b = buffer_init();
  92
  93         return p;
  94 }
  95
  96 FREE_FUNC(mod_compress_free) {
  97         plugin_data *p = p_d;
  98
  99         UNUSED(srv);
 100
 101         if (!p) return HANDLER_GO_ON;
 102
 103         buffer_free(p->ofn);
 104         buffer_free(p->b);
 105
 106         if (p->config_storage) {
 107                 size_t i;
 108                 for (i = 0; i < srv->config_context->used; i++) {
 109                         plugin_config *s = p->config_storage[i];
 110
 111                         if (NULL == s) continue;
 112
 113                         array_free(s->compress);
 114                         buffer_free(s->compress_cache_dir);
 115
 116                         free(s);
 117                 }
 118                 free(p->config_storage);
 119         }
 120
 121
 122         free(p);
 123
 124         return HANDLER_GO_ON;
 125 }
 126
 127 /* 0 on success, -1 for error */
 128 static int mkdir_recursive(char *dir) {
 129         char *p = dir;
 130
 131         if (!dir || !dir[0])
 132                 return 0;
 133
 134         while ((p = strchr(p + 1, '/')) != NULL) {
 135
 136                 *p = '\0';
 137                 if ((mkdir(dir, 0700) != 0) && (errno != EEXIST)) {
 138                         *p = '/';
 139                         return -1;
 140                 }
 141
 142                 *p++ = '/';
 143                 if (!*p) return 0; /* Ignore trailing slash */
 144         }
 145
 146         return (mkdir(dir, 0700) != 0) && (errno != EEXIST) ? -1 : 0;
 147 }
 148
 149 /* 0 on success, -1 for error */
 150 static int mkdir_for_file(char *filename) {
 151         char *p = filename;
 152
 153         if (!filename || !filename[0])
 154                 return -1;
 155
 156         while ((p = strchr(p + 1, '/')) != NULL) {
 157
 158                 *p = '\0';
 159                 if ((mkdir(filename, 0700) != 0) && (errno != EEXIST)) {
 160                         *p = '/';
 161                         return -1;
 162                 }
 163
 164                 *p++ = '/';
 165                 if (!*p) return -1; /* Unexpected trailing slash in filename */
 166         }
 167
 168         return 0;
 169 }
 170
 171 SETDEFAULTS_FUNC(mod_compress_setdefaults) {
 172         plugin_data *p = p_d;
 173         size_t i = 0;
 174
 175         config_values_t cv[] = {
 176                 { "compress.cache-dir",             NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
 177                 { "compress.filetype",              NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
 178                 { "compress.max-filesize",          NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
 179                 { "compress.allowed-encodings",     NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
 180                 { NULL,                             NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET }
 181         };
 182
 183         p->config_storage = calloc(1, srv->config_context->used * sizeof(plugin_config *));
 184
 185         for (i = 0; i < srv->config_context->used; i++) {
 186                 data_config const* config = (data_config const*)srv->config_context->data[i];
 187                 plugin_config *s;
 188                 array  *encodings_arr = array_init();
 189
 190                 s = calloc(1, sizeof(plugin_config));
 191                 s->compress_cache_dir = buffer_init();
 192                 s->compress = array_init();
 193                 s->compress_max_filesize = 0;
 194                 s->allowed_encodings = 0;
 195
 196                 cv[0].destination = s->compress_cache_dir;
 197                 cv[1].destination = s->compress;
 198                 cv[2].destination = &(s->compress_max_filesize);
 199                 cv[3].destination = encodings_arr; /* temp array for allowed encodings list */
 200
 201                 p->config_storage[i] = s;
 202
 203                 if (0 != config_insert_values_global(srv, config->value, cv, i == 0 ? T_CONFIG_SCOPE_SERVER : T_CONFIG_SCOPE_CONNECTION)) {
 204                         return HANDLER_ERROR;
 205                 }
 206
 207                 if (encodings_arr->used) {
 208                         size_t j = 0;
 209                         for (j = 0; j < encodings_arr->used; j++) {
 210                                 data_string *ds = (data_string *)encodings_arr->data[j];
 211 #ifdef USE_ZLIB
 212                                 if (NULL != strstr(ds->value->ptr, "gzip"))
 213                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_GZIP | HTTP_ACCEPT_ENCODING_X_GZIP;
 214                                 if (NULL != strstr(ds->value->ptr, "x-gzip"))
 215                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_X_GZIP;
 216                                 if (NULL != strstr(ds->value->ptr, "deflate"))
 217                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_DEFLATE;
 218                                 /*
 219                                 if (NULL != strstr(ds->value->ptr, "compress"))
 220                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_COMPRESS;
 221                                 */
 222 #endif
 223 #ifdef USE_BZ2LIB
 224                                 if (NULL != strstr(ds->value->ptr, "bzip2"))
 225                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2 | HTTP_ACCEPT_ENCODING_X_BZIP2;
 226                                 if (NULL != strstr(ds->value->ptr, "x-bzip2"))
 227                                         s->allowed_encodings |= HTTP_ACCEPT_ENCODING_X_BZIP2;
 228 #endif
 229                         }
 230                 } else {
 231                         /* default encodings */
 232                         s->allowed_encodings = 0
 233 #ifdef USE_ZLIB
 234                                 | HTTP_ACCEPT_ENCODING_GZIP | HTTP_ACCEPT_ENCODING_X_GZIP | HTTP_ACCEPT_ENCODING_DEFLATE
 235 #endif
 236 #ifdef USE_BZ2LIB
 237                                 | HTTP_ACCEPT_ENCODING_BZIP2 | HTTP_ACCEPT_ENCODING_X_BZIP2
 238 #endif
 239                                 ;
 240                 }
 241
 242                 array_free(encodings_arr);
 243
 244                 if (!buffer_string_is_empty(s->compress_cache_dir)) {
 245                         struct stat st;
 246                         mkdir_recursive(s->compress_cache_dir->ptr);
 247
 248                         if (0 != stat(s->compress_cache_dir->ptr, &st)) {
 249                                 log_error_write(srv, __FILE__, __LINE__, "sbs", "can't stat compress.cache-dir",
 250                                                 s->compress_cache_dir, strerror(errno));
 251
 252                                 return HANDLER_ERROR;
 253                         }
 254                 }
 255         }
 256
 257         return HANDLER_GO_ON;
 258
 259 }
 260
 261 #ifdef USE_ZLIB
 262 static int deflate_file_to_buffer_gzip(server *srv, connection *con, plugin_data *p, char *start, off_t st_size, time_t mtime) {
 263         unsigned char *c;
 264         unsigned long crc;
 265         z_stream z;
 266         size_t outlen;
 267
 268         UNUSED(srv);
 269         UNUSED(con);
 270
 271         z.zalloc = Z_NULL;
 272         z.zfree = Z_NULL;
 273         z.opaque = Z_NULL;
 274
 275         if (Z_OK != deflateInit2(&z,
 276                                  Z_DEFAULT_COMPRESSION,
 277                                  Z_DEFLATED,
 278                                  -MAX_WBITS,  /* supress zlib-header */
 279                                  8,
 280                                  Z_DEFAULT_STRATEGY)) {
 281                 return -1;
 282         }
 283
 284         z.next_in = (unsigned char *)start;
 285         z.avail_in = st_size;
 286         z.total_in = 0;
 287
 288
 289         buffer_string_prepare_copy(p->b, (z.avail_in * 1.1) + 12 + 18);
 290
 291         /* write gzip header */
 292
 293         c = (unsigned char *)p->b->ptr;
 294         c[0] = 0x1f;
 295         c[1] = 0x8b;
 296         c[2] = Z_DEFLATED;
 297         c[3] = 0; /* options */
 298         c[4] = (mtime >>  0) & 0xff;
 299         c[5] = (mtime >>  8) & 0xff;
 300         c[6] = (mtime >> 16) & 0xff;
 301         c[7] = (mtime >> 24) & 0xff;
 302         c[8] = 0x00; /* extra flags */
 303         c[9] = 0x03; /* UNIX */
 304
 305         outlen = 10;
 306         z.next_out = (unsigned char *)p->b->ptr + outlen;
 307         z.avail_out = p->b->size - outlen - 9;
 308         z.total_out = 0;
 309
 310         if (Z_STREAM_END != deflate(&z, Z_FINISH)) {
 311                 deflateEnd(&z);
 312                 return -1;
 313         }
 314
 315         /* trailer */
 316         outlen += z.total_out;
 317
 318         crc = generate_crc32c(start, st_size);
 319
 320         c = (unsigned char *)p->b->ptr + outlen;
 321
 322         c[0] = (crc >>  0) & 0xff;
 323         c[1] = (crc >>  8) & 0xff;
 324         c[2] = (crc >> 16) & 0xff;
 325         c[3] = (crc >> 24) & 0xff;
 326         c[4] = (z.total_in >>  0) & 0xff;
 327         c[5] = (z.total_in >>  8) & 0xff;
 328         c[6] = (z.total_in >> 16) & 0xff;
 329         c[7] = (z.total_in >> 24) & 0xff;
 330         outlen += 8;
 331         buffer_commit(p->b, outlen);
 332
 333         if (Z_OK != deflateEnd(&z)) {
 334                 return -1;
 335         }
 336
 337         return 0;
 338 }
 339
 340 static int deflate_file_to_buffer_deflate(server *srv, connection *con, plugin_data *p, unsigned char *start, off_t st_size) {
 341         z_stream z;
 342
 343         UNUSED(srv);
 344         UNUSED(con);
 345
 346         z.zalloc = Z_NULL;
 347         z.zfree = Z_NULL;
 348         z.opaque = Z_NULL;
 349
 350         if (Z_OK != deflateInit2(&z,
 351                                  Z_DEFAULT_COMPRESSION,
 352                                  Z_DEFLATED,
 353                                  -MAX_WBITS,  /* supress zlib-header */
 354                                  8,
 355                                  Z_DEFAULT_STRATEGY)) {
 356                 return -1;
 357         }
 358
 359         z.next_in = start;
 360         z.avail_in = st_size;
 361         z.total_in = 0;
 362
 363         buffer_string_prepare_copy(p->b, (z.avail_in * 1.1) + 12);
 364
 365         z.next_out = (unsigned char *)p->b->ptr;
 366         z.avail_out = p->b->size - 1;
 367         z.total_out = 0;
 368
 369         if (Z_STREAM_END != deflate(&z, Z_FINISH)) {
 370                 deflateEnd(&z);
 371                 return -1;
 372         }
 373
 374         if (Z_OK != deflateEnd(&z)) {
 375                 return -1;
 376         }
 377
 378         /* trailer */
 379         buffer_commit(p->b, z.total_out);
 380
 381         return 0;
 382 }
 383
 384 #endif
 385
 386 #ifdef USE_BZ2LIB
 387 static int deflate_file_to_buffer_bzip2(server *srv, connection *con, plugin_data *p, unsigned char *start, off_t st_size) {
 388         bz_stream bz;
 389
 390         UNUSED(srv);
 391         UNUSED(con);
 392
 393         bz.bzalloc = NULL;
 394         bz.bzfree = NULL;
 395         bz.opaque = NULL;
 396
 397         if (BZ_OK != BZ2_bzCompressInit(&bz,
 398                                         9, /* blocksize = 900k */
 399                                         0, /* no output */
 400                                         0)) { /* workFactor: default */
 401                 return -1;
 402         }
 403
 404         bz.next_in = (char *)start;
 405         bz.avail_in = st_size;
 406         bz.total_in_lo32 = 0;
 407         bz.total_in_hi32 = 0;
 408
 409         buffer_string_prepare_copy(p->b, (bz.avail_in * 1.1) + 12);
 410
 411         bz.next_out = p->b->ptr;
 412         bz.avail_out = p->b->size - 1;
 413         bz.total_out_lo32 = 0;
 414         bz.total_out_hi32 = 0;
 415
 416         if (BZ_STREAM_END != BZ2_bzCompress(&bz, BZ_FINISH)) {
 417                 BZ2_bzCompressEnd(&bz);
 418                 return -1;
 419         }
 420
 421         if (BZ_OK != BZ2_bzCompressEnd(&bz)) {
 422                 return -1;
 423         }
 424
 425         /* file is too large for now */
 426         if (bz.total_out_hi32) return -1;
 427
 428         /* trailer */
 429         buffer_commit(p->b, bz.total_out_lo32);
 430
 431         return 0;
 432 }
 433 #endif
 434
 435 static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) {
 436         int ifd, ofd;
 437         int ret;
 438 #ifdef USE_MMAP
 439         volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */
 440 #endif
 441         void *start;
 442         const char *filename = fn->ptr;
 443         ssize_t r;
 444
 445         /* overflow */
 446         if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1;
 447
 448         /* don't mmap files > 128Mb
 449          *
 450          * we could use a sliding window, but currently there is no need for it
 451          */
 452
 453         if (sce->st.st_size > 128 * 1024 * 1024) return -1;
 454
 455         buffer_reset(p->ofn);
 456         buffer_copy_buffer(p->ofn, p->conf.compress_cache_dir);
 457         buffer_append_slash(p->ofn);
 458
 459         if (0 == strncmp(con->physical.path->ptr, con->physical.doc_root->ptr, buffer_string_length(con->physical.doc_root))) {
 460                 buffer_append_string(p->ofn, con->physical.path->ptr + buffer_string_length(con->physical.doc_root));
 461         } else {
 462                 buffer_append_string_buffer(p->ofn, con->uri.path);
 463         }
 464
 465         switch(type) {
 466         case HTTP_ACCEPT_ENCODING_GZIP:
 467         case HTTP_ACCEPT_ENCODING_X_GZIP:
 468                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-gzip-"));
 469                 break;
 470         case HTTP_ACCEPT_ENCODING_DEFLATE:
 471                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-deflate-"));
 472                 break;
 473         case HTTP_ACCEPT_ENCODING_BZIP2:
 474         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 475                 buffer_append_string_len(p->ofn, CONST_STR_LEN("-bzip2-"));
 476                 break;
 477         default:
 478                 log_error_write(srv, __FILE__, __LINE__, "sd", "unknown compression type", type);
 479                 return -1;
 480         }
 481
 482         buffer_append_string_buffer(p->ofn, sce->etag);
 483
 484         if (-1 == mkdir_for_file(p->ofn->ptr)) {
 485                 log_error_write(srv, __FILE__, __LINE__, "sb", "couldn't create directory for file", p->ofn);
 486                 return -1;
 487         }
 488
 489         if (-1 == (ofd = open(p->ofn->ptr, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0600))) {
 490                 if (errno == EEXIST) {
 491                         /* cache-entry exists */
 492 #if 0
 493                         log_error_write(srv, __FILE__, __LINE__, "bs", p->ofn, "compress-cache hit");
 494 #endif
 495                         buffer_copy_buffer(con->physical.path, p->ofn);
 496
 497                         return 0;
 498                 }
 499
 500                 log_error_write(srv, __FILE__, __LINE__, "sbss", "creating cachefile", p->ofn, "failed", strerror(errno));
 501
 502                 return -1;
 503         }
 504 #if 0
 505         log_error_write(srv, __FILE__, __LINE__, "bs", p->ofn, "compress-cache miss");
 506 #endif
 507         if (-1 == (ifd = open(filename, O_RDONLY | O_BINARY))) {
 508                 log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno));
 509
 510                 close(ofd);
 511
 512                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 513                 if (-1 == unlink(p->ofn->ptr)) {
 514                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 515                 }
 516
 517                 return -1;
 518         }
 519
 520 #ifdef USE_MMAP
 521         if (MAP_FAILED != (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) {
 522                 mapped = 1;
 523                 signal(SIGBUS, sigbus_handler);
 524                 sigbus_jmp_valid = 1;
 525                 if (0 != sigsetjmp(sigbus_jmp, 1)) {
 526                         sigbus_jmp_valid = 0;
 527
 528                         log_error_write(srv, __FILE__, __LINE__, "sbd", "SIGBUS in mmap:",
 529                                 fn, ifd);
 530
 531                         munmap(start, sce->st.st_size);
 532                         close(ofd);
 533                         close(ifd);
 534
 535                         /* Remove the incomplete cache file, so that later hits aren't served from it */
 536                         if (-1 == unlink(p->ofn->ptr)) {
 537                                 log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 538                         }
 539
 540                         return -1;
 541                 }
 542         } else
 543 #endif  /* FIXME: might attempt to read very large file completely into memory; see compress.max-filesize config option */
 544         if (NULL == (start = malloc(sce->st.st_size)) || sce->st.st_size != read(ifd, start, sce->st.st_size)) {
 545                 log_error_write(srv, __FILE__, __LINE__, "sbss", "reading", fn, "failed", strerror(errno));
 546
 547                 close(ofd);
 548                 close(ifd);
 549                 free(start);
 550
 551                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 552                 if (-1 == unlink(p->ofn->ptr)) {
 553                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 554                 }
 555
 556                 return -1;
 557         }
 558
 559         ret = -1;
 560         switch(type) {
 561 #ifdef USE_ZLIB
 562         case HTTP_ACCEPT_ENCODING_GZIP:
 563         case HTTP_ACCEPT_ENCODING_X_GZIP:
 564                 ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime);
 565                 break;
 566         case HTTP_ACCEPT_ENCODING_DEFLATE:
 567                 ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size);
 568                 break;
 569 #endif
 570 #ifdef USE_BZ2LIB
 571         case HTTP_ACCEPT_ENCODING_BZIP2:
 572         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 573                 ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size);
 574                 break;
 575 #endif
 576         }
 577
 578         if (ret == 0) {
 579                 r = write(ofd, CONST_BUF_LEN(p->b));
 580                 if (-1 == r) {
 581                         log_error_write(srv, __FILE__, __LINE__, "sbss", "writing cachefile", p->ofn, "failed:", strerror(errno));
 582                         ret = -1;
 583                 } else if ((size_t)r != buffer_string_length(p->b)) {
 584                         log_error_write(srv, __FILE__, __LINE__, "sbs", "writing cachefile", p->ofn, "failed: not enough bytes written");
 585                         ret = -1;
 586                 }
 587         }
 588
 589 #ifdef USE_MMAP
 590         if (mapped) {
 591                 sigbus_jmp_valid = 0;
 592                 munmap(start, sce->st.st_size);
 593         } else
 594 #endif
 595                 free(start);
 596
 597         close(ofd);
 598         close(ifd);
 599
 600         if (ret != 0) {
 601                 /* Remove the incomplete cache file, so that later hits aren't served from it */
 602                 if (-1 == unlink(p->ofn->ptr)) {
 603                         log_error_write(srv, __FILE__, __LINE__, "sbss", "unlinking incomplete cachefile", p->ofn, "failed:", strerror(errno));
 604                 }
 605
 606                 return -1;
 607         }
 608
 609         buffer_copy_buffer(con->physical.path, p->ofn);
 610
 611         return 0;
 612 }
 613
 614 static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) {
 615         int ifd;
 616         int ret = -1;
 617 #ifdef USE_MMAP
 618         volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */
 619 #endif
 620         void *start;
 621
 622         /* overflow */
 623         if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1;
 624
 625         /* don't mmap files > 128M
 626          *
 627          * we could use a sliding window, but currently there is no need for it
 628          */
 629
 630         if (sce->st.st_size > 128 * 1024 * 1024) return -1;
 631
 632
 633         if (-1 == (ifd = open(fn->ptr, O_RDONLY | O_BINARY))) {
 634                 log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno));
 635
 636                 return -1;
 637         }
 638
 639 #ifdef USE_MMAP
 640         if (MAP_FAILED != (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) {
 641                 mapped = 1;
 642                 signal(SIGBUS, sigbus_handler);
 643                 sigbus_jmp_valid = 1;
 644                 if (0 != sigsetjmp(sigbus_jmp, 1)) {
 645                         sigbus_jmp_valid = 0;
 646
 647                         log_error_write(srv, __FILE__, __LINE__, "sbd", "SIGBUS in mmap:",
 648                                 fn, ifd);
 649
 650                         munmap(start, sce->st.st_size);
 651                         close(ifd);
 652                         return -1;
 653                 }
 654         } else
 655 #endif  /* FIXME: might attempt to read very large file completely into memory; see compress.max-filesize config option */
 656         if (NULL == (start = malloc(sce->st.st_size)) || sce->st.st_size != read(ifd, start, sce->st.st_size)) {
 657                 log_error_write(srv, __FILE__, __LINE__, "sbss", "reading", fn, "failed", strerror(errno));
 658
 659                 close(ifd);
 660                 free(start);
 661                 return -1;
 662         }
 663
 664         switch(type) {
 665 #ifdef USE_ZLIB
 666         case HTTP_ACCEPT_ENCODING_GZIP:
 667         case HTTP_ACCEPT_ENCODING_X_GZIP:
 668                 ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime);
 669                 break;
 670         case HTTP_ACCEPT_ENCODING_DEFLATE:
 671                 ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size);
 672                 break;
 673 #endif
 674 #ifdef USE_BZ2LIB
 675         case HTTP_ACCEPT_ENCODING_BZIP2:
 676         case HTTP_ACCEPT_ENCODING_X_BZIP2:
 677                 ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size);
 678                 break;
 679 #endif
 680         default:
 681                 ret = -1;
 682                 break;
 683         }
 684
 685 #ifdef USE_MMAP
 686         if (mapped) {
 687                 sigbus_jmp_valid = 0;
 688                 munmap(start, sce->st.st_size);
 689         } else
 690 #endif
 691                 free(start);
 692
 693         close(ifd);
 694
 695         if (ret != 0) return -1;
 696
 697         chunkqueue_reset(con->write_queue);
 698         chunkqueue_append_buffer(con->write_queue, p->b);
 699
 700         buffer_reset(con->physical.path);
 701
 702         con->file_finished = 1;
 703         con->file_started  = 1;
 704
 705         return 0;
 706 }
 707
 708
 709 #define PATCH(x) \
 710         p->conf.x = s->x;
 711 static int mod_compress_patch_connection(server *srv, connection *con, plugin_data *p) {
 712         size_t i, j;
 713         plugin_config *s = p->config_storage[0];
 714
 715         PATCH(compress_cache_dir);
 716         PATCH(compress);
 717         PATCH(compress_max_filesize);
 718         PATCH(allowed_encodings);
 719
 720         /* skip the first, the global context */
 721         for (i = 1; i < srv->config_context->used; i++) {
 722                 data_config *dc = (data_config *)srv->config_context->data[i];
 723                 s = p->config_storage[i];
 724
 725                 /* condition didn't match */
 726                 if (!config_check_cond(srv, con, dc)) continue;
 727
 728                 /* merge config */
 729                 for (j = 0; j < dc->value->used; j++) {
 730                         data_unset *du = dc->value->data[j];
 731
 732                         if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.cache-dir"))) {
 733                                 PATCH(compress_cache_dir);
 734                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.filetype"))) {
 735                                 PATCH(compress);
 736                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.max-filesize"))) {
 737                                 PATCH(compress_max_filesize);
 738                         } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.allowed-encodings"))) {
 739                                 PATCH(allowed_encodings);
 740                         }
 741                 }
 742         }
 743
 744         return 0;
 745 }
 746 #undef PATCH
 747
 748 static int mod_compress_contains_encoding(const char *headervalue, const char *encoding, size_t len) {
 749         const char *m = headervalue;
 750         do {
 751                 while (*m == ',' || *m == ' ' || *m == '\t') {
 752                         ++m;
 753                 }
 754                 if (0 == strncasecmp(m, encoding, len)) {
 755                         /*(not a full HTTP field parse: not parsing for q-values and not handling q=0)*/
 756                         m += len;
 757                         if (*m == '\0' || *m == ',' || *m == ';' || *m == ' ' || *m == '\t')
 758                                 return 1;
 759                 } else if (*m != '\0') {
 760                         ++m;
 761                 }
 762         } while ((m = strchr(m, ',')));
 763         return 0;
 764 }
 765
 766 PHYSICALPATH_FUNC(mod_compress_physical) {
 767         plugin_data *p = p_d;
 768         size_t m;
 769         off_t max_fsize;
 770         stat_cache_entry *sce = NULL;
 771         buffer *mtime = NULL;
 772         buffer *content_type;
 773
 774         if (con->mode != DIRECT || con->http_status) return HANDLER_GO_ON;
 775
 776         /* only GET and POST can get compressed */
 777         if (con->request.http_method != HTTP_METHOD_GET &&
 778             con->request.http_method != HTTP_METHOD_POST) {
 779                 return HANDLER_GO_ON;
 780         }
 781
 782         if (buffer_string_is_empty(con->physical.path)) {
 783                 return HANDLER_GO_ON;
 784         }
 785
 786         mod_compress_patch_connection(srv, con, p);
 787
 788         max_fsize = p->conf.compress_max_filesize;
 789
 790         if (con->conf.log_request_handling) {
 791                 log_error_write(srv, __FILE__, __LINE__,  "s",  "-- handling file as static file");
 792         }
 793
 794         if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) {
 795                 con->http_status = 403;
 796
 797                 log_error_write(srv, __FILE__, __LINE__, "sbsb",
 798                                 "not a regular file:", con->uri.path,
 799                                 "->", con->physical.path);
 800
 801                 return HANDLER_FINISHED;
 802         }
 803
 804         /* we only handle regular files */
 805 #ifdef HAVE_LSTAT
 806         if ((sce->is_symlink == 1) && !con->conf.follow_symlink) {
 807                 return HANDLER_GO_ON;
 808         }
 809 #endif
 810         if (!S_ISREG(sce->st.st_mode)) {
 811                 return HANDLER_GO_ON;
 812         }
 813
 814         /* don't compress files that are too large as we need to much time to handle them */
 815         if (max_fsize && (sce->st.st_size >> 10) > max_fsize) return HANDLER_GO_ON;
 816
 817         /* don't try to compress files less than 128 bytes
 818          *
 819          * - extra overhead for compression
 820          * - mmap() fails for st_size = 0 :)
 821          */
 822         if (sce->st.st_size < 128) return HANDLER_GO_ON;
 823
 824         /* check if mimetype is in compress-config */
 825         content_type = NULL;
 826         if (sce->content_type->ptr) {
 827                 char *c;
 828                 if ( (c = strchr(sce->content_type->ptr, ';')) != NULL) {
 829                         content_type = srv->tmp_buf;
 830                         buffer_copy_string_len(content_type, sce->content_type->ptr, c - sce->content_type->ptr);
 831                 }
 832         }
 833
 834         for (m = 0; m < p->conf.compress->used; m++) {
 835                 data_string *compress_ds = (data_string *)p->conf.compress->data[m];
 836
 837                 if (!compress_ds) {
 838                         log_error_write(srv, __FILE__, __LINE__, "sbb", "evil", con->physical.path, con->uri.path);
 839
 840                         return HANDLER_GO_ON;
 841                 }
 842
 843                 if (buffer_is_equal(compress_ds->value, sce->content_type)
 844                     || (content_type && buffer_is_equal(compress_ds->value, content_type))) {
 845                         /* mimetype found */
 846                         data_string *ds;
 847
 848                         /* the response might change according to Accept-Encoding */
 849                         response_header_insert(srv, con, CONST_STR_LEN("Vary"), CONST_STR_LEN("Accept-Encoding"));
 850
 851                         if (NULL != (ds = (data_string *)array_get_element(con->request.headers, "Accept-Encoding"))) {
 852                                 int accept_encoding = 0;
 853                                 char *value = ds->value->ptr;
 854                                 int matched_encodings = 0;
 855                                 int use_etag = sce->etag != NULL && sce->etag->ptr != NULL;
 856
 857                                 /* get client side support encodings */
 858 #ifdef USE_ZLIB
 859                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("gzip"))) accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP;
 860                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("x-gzip"))) accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP;
 861                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("deflate"))) accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE;
 862                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("compress"))) accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS;
 863 #endif
 864 #ifdef USE_BZ2LIB
 865                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("bzip2"))) accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2;
 866                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("x-bzip2"))) accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2;
 867 #endif
 868                                 if (mod_compress_contains_encoding(value, CONST_STR_LEN("identity"))) accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY;
 869
 870                                 /* find matching entries */
 871                                 matched_encodings = accept_encoding & p->conf.allowed_encodings;
 872
 873                                 if (matched_encodings) {
 874                                         static const char dflt_gzip[] = "gzip";
 875                                         static const char dflt_x_gzip[] = "x-gzip";
 876                                         static const char dflt_deflate[] = "deflate";
 877                                         static const char dflt_bzip2[] = "bzip2";
 878                                         static const char dflt_x_bzip2[] = "x-bzip2";
 879
 880                                         const char *compression_name = NULL;
 881                                         int compression_type = 0;
 882
 883                                         mtime = strftime_cache_get(srv, sce->st.st_mtime);
 884
 885                                         /* try matching original etag of uncompressed version */
 886                                         if (use_etag) {
 887                                                 etag_mutate(con->physical.etag, sce->etag);
 888                                                 if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) {
 889                                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 890                                                         response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 891                                                         response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 892                                                         return HANDLER_FINISHED;
 893                                                 }
 894                                         }
 895
 896                                         /* select best matching encoding */
 897                                         if (matched_encodings & HTTP_ACCEPT_ENCODING_BZIP2) {
 898                                                 compression_type = HTTP_ACCEPT_ENCODING_BZIP2;
 899                                                 compression_name = dflt_bzip2;
 900                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_X_BZIP2) {
 901                                                 compression_type = HTTP_ACCEPT_ENCODING_X_BZIP2;
 902                                                 compression_name = dflt_x_bzip2;
 903                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_GZIP) {
 904                                                 compression_type = HTTP_ACCEPT_ENCODING_GZIP;
 905                                                 compression_name = dflt_gzip;
 906                                         } else if (matched_encodings & HTTP_ACCEPT_ENCODING_X_GZIP) {
 907                                                 compression_type = HTTP_ACCEPT_ENCODING_X_GZIP;
 908                                                 compression_name = dflt_x_gzip;
 909                                         } else {
 910                                                 force_assert(matched_encodings & HTTP_ACCEPT_ENCODING_DEFLATE);
 911                                                 compression_type = HTTP_ACCEPT_ENCODING_DEFLATE;
 912                                                 compression_name = dflt_deflate;
 913                                         }
 914
 915                                         if (use_etag) {
 916                                                 /* try matching etag of compressed version */
 917                                                 buffer_copy_buffer(srv->tmp_buf, sce->etag);
 918                                                 buffer_append_string_len(srv->tmp_buf, CONST_STR_LEN("-"));
 919                                                 buffer_append_string(srv->tmp_buf, compression_name);
 920                                                 etag_mutate(con->physical.etag, srv->tmp_buf);
 921                                         }
 922
 923                                         if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) {
 924                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name));
 925                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 926                                                 response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 927                                                 if (use_etag) {
 928                                                         response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 929                                                 }
 930                                                 return HANDLER_FINISHED;
 931                                         }
 932
 933                                         /* deflate it */
 934                                         if (use_etag && !buffer_string_is_empty(p->conf.compress_cache_dir)) {
 935                                                 if (0 != deflate_file_to_file(srv, con, p, con->physical.path, sce, compression_type))
 936                                                         return HANDLER_GO_ON;
 937                                         } else {
 938                                                 if (0 != deflate_file_to_buffer(srv, con, p, con->physical.path, sce, compression_type))
 939                                                         return HANDLER_GO_ON;
 940                                         }
 941                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name));
 942                                         response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
 943                                         if (use_etag) {
 944                                                 response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
 945                                         }
 946                                         response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
 947                                         /* let mod_staticfile handle the cached compressed files, physical path was modified */
 948                                         return (use_etag && !buffer_string_is_empty(p->conf.compress_cache_dir)) ? HANDLER_GO_ON : HANDLER_FINISHED;
 949                                 }
 950                         }
 951                 }
 952         }
 953
 954         return HANDLER_GO_ON;
 955 }
 956
 957 int mod_compress_plugin_init(plugin *p);
 958 int mod_compress_plugin_init(plugin *p) {
 959         p->version     = LIGHTTPD_VERSION_ID;
 960         p->name        = buffer_init_string("compress");
 961
 962         p->init        = mod_compress_init;
 963         p->set_defaults = mod_compress_setdefaults;
 964         p->handle_subrequest_start  = mod_compress_physical;
 965         p->cleanup     = mod_compress_free;
 966
 967         p->data        = NULL;
 968
 969         return 0;
 970 }