4 * bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate patch
6 * Bug fix and new features:
7 * 1) fix loop bug when content-length is bigger than work-block-size*k
11 * lighttpd-1.4.26.mod_deflate.patch from
12 * https://redmine.lighttpd.net/projects/1/wiki/Docs_ModDeflate
16 * Patch further modified in this incarnation.
18 * Note: this patch only handles completed responses (con->file_finished);
19 * this patch does not currently handle streaming dynamic responses,
20 * and therefore also does not worry about Transfer-Encoding: chunked
21 * (or having separate con->output_queue for chunked-encoded output)
22 * (or using separate buffers per connection instead of p->tmp_buf)
23 * (or handling interactions with block buffering and write timeouts)
26 * - fixed major bug with compressing chunks with offset > 0
28 * "Response breaking in mod_deflate"
29 * https://redmine.lighttpd.net/issues/986
30 * - fix broken (in some cases) chunk accounting in deflate_compress_response()
33 * "mod_deflate's bzip2 broken by default"
34 * https://redmine.lighttpd.net/issues/2035
35 * - fix mismatch with current chunk interfaces
37 * "Weird things in chunk.c (functions only handling specific cases, unexpected behaviour)"
38 * https://redmine.lighttpd.net/issues/1510
40 * Behavior changes from prior patch:
41 * - deflate.mimetypes must now be configured to enable compression
42 * deflate.mimetypes = ( ) # compress nothing (disabled; default)
43 * deflate.mimetypes = ( "" ) # compress all mimetypes
44 * deflate.mimetypes = ( "text/" ) # compress text/... mimetypes
46 * "mod_deflate enabled by default"
47 * https://redmine.lighttpd.net/issues/1394
48 * - deflate.enabled directive removed (see new behavior of deflate.mimetypes)
49 * - deflate.debug removed (was developer debug trace, not end-user debug)
50 * - deflate.bzip2 replaced with deflate.allowed-encodings (like mod_compress)
52 * "mod_deflate should allow limiting of compression algorithm from the configuration file"
53 * https://redmine.lighttpd.net/issues/996
54 * "mod_compress disabling methods"
55 * https://redmine.lighttpd.net/issues/1773
56 * - deflate.nocompress-url removed since disabling compression for a URL
57 * can now easily be done by setting to a blank list either directive
58 * deflate.accept_encodings = () or deflate.mimetypes = () in a conditional
59 * block, e.g. $HTTP["url"] =~ "....." { deflate.mimetypes = ( ) }
60 * - deflate.sync-flush removed; controlled by con->conf.stream_response_body
61 * (though streaming compression not currently implemented in mod_deflate)
62 * - inactive directives in this patch (since con->file_finished required)
63 * deflate.work-block-size
64 * deflate.output-buffer-size
65 * - remove weak file size check; SIGBUS is trapped, file that shrink will error
67 * "mod_deflate: filesize check is too weak"
68 * https://redmine.lighttpd.net/issues/1512
69 * - change default deflate.min-compress-size from 0 to now be 256
70 * http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits
71 * Apache 2.4 mod_deflate minimum is 68 bytes
72 * Akamai recommends minimum 860 bytes
73 * Google recommends minimum be somewhere in range between 150 and 1024 bytes
74 * - deflate.max-compress-size new directive (in kb like compress.max_filesize)
75 * - deflate.mem-level removed (too many knobs for little benefit)
76 * - deflate.window-size removed (too many knobs for little benefit)
79 * - config directives may be changed, renamed, or removed
80 * e.g. A set of reasonable defaults might be chosen
81 * instead of making them configurable.
82 * deflate.min-compress-size
83 * - might add deflate.mimetypes-exclude = ( ... ) for list of mimetypes
84 * to avoid compressing, even if a broader deflate.mimetypes matched,
85 * e.g. to compress all "text/" except "text/special".
86 * - mod_compress and mod_deflate might merge overlapping feature sets
87 * (mod_compress.cache-dir does not yet have an equivalent in mod_deflate)
89 * Implementation notes:
90 * - http_chunk_append_mem() used instead of http_chunk_append_buffer()
91 * so that p->tmp_buf can be large and re-used. This results in an extra copy
92 * of compressed data before data is sent to network, though if the compressed
93 * size is larger than 64k, it ends up being sent to a temporary file on
94 * disk without suffering an extra copy in memory, and without extra chunk
95 * create and destroy. If this is ever changed to give away buffers, then use
96 * a unique hctx->output buffer per hctx; do not reuse p->tmp_buf across
97 * multiple requests being handled in parallel.
101 #include <sys/types.h>
102 #include <sys/stat.h>
103 #include "sys-mmap.h"
110 #include <unistd.h> /* read() */
117 #include "http_chunk.h"
118 #include "response.h"
122 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ
126 #ifndef Z_DEFAULT_COMPRESSION
127 #define Z_DEFAULT_COMPRESSION -1
133 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
135 /* we don't need stdio interface */
140 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
143 #include "sys-mmap.h"
147 static volatile int sigbus_jmp_valid
;
148 static sigjmp_buf sigbus_jmp
;
150 static void sigbus_handler(int sig
) {
152 if (sigbus_jmp_valid
) siglongjmp(sigbus_jmp
, 1);
153 log_failed_assert(__FILE__
, __LINE__
, "SIGBUS");
157 /* request: accept-encoding */
158 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0)
159 #define HTTP_ACCEPT_ENCODING_GZIP BV(1)
160 #define HTTP_ACCEPT_ENCODING_DEFLATE BV(2)
161 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3)
162 #define HTTP_ACCEPT_ENCODING_BZIP2 BV(4)
163 #define HTTP_ACCEPT_ENCODING_X_GZIP BV(5)
164 #define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6)
167 #define MByte * 1024 KByte
168 #define GByte * 1024 MByte
172 int allowed_encodings
;
173 unsigned int max_compress_size
;
174 unsigned short min_compress_size
;
175 unsigned short output_buffer_size
;
176 unsigned short work_block_size
;
177 unsigned short sync_flush
;
178 short compression_level
;
187 plugin_config
**config_storage
;
203 chunkqueue
*in_queue
;
205 plugin_data
*plugin_data
;
206 int compression_type
;
209 static handler_ctx
*handler_ctx_init() {
212 hctx
= calloc(1, sizeof(*hctx
));
213 hctx
->in_queue
= chunkqueue_init();
218 static void handler_ctx_free(handler_ctx
*hctx
) {
220 if (hctx
->output
!= p
->tmp_buf
) {
221 buffer_free(hctx
->output
);
224 chunkqueue_free(hctx
->in_queue
);
228 INIT_FUNC(mod_deflate_init
) {
231 p
= calloc(1, sizeof(*p
));
233 p
->encodings
= array_init();
234 p
->tmp_buf
= buffer_init();
235 buffer_string_prepare_copy(p
->tmp_buf
, 64 KByte
);
240 FREE_FUNC(mod_deflate_free
) {
241 plugin_data
*p
= p_d
;
245 if (!p
) return HANDLER_GO_ON
;
247 if (p
->config_storage
) {
249 for (i
= 0; i
< srv
->config_context
->used
; i
++) {
250 plugin_config
*s
= p
->config_storage
[i
];
254 array_free(s
->mimetypes
);
257 free(p
->config_storage
);
260 buffer_free(p
->tmp_buf
);
261 array_free(p
->encodings
);
265 return HANDLER_GO_ON
;
268 SETDEFAULTS_FUNC(mod_deflate_setdefaults
) {
269 plugin_data
*p
= p_d
;
272 config_values_t cv
[] = {
273 { "deflate.mimetypes", NULL
, T_CONFIG_ARRAY
, T_CONFIG_SCOPE_CONNECTION
},
274 { "deflate.allowed-encodings", NULL
, T_CONFIG_ARRAY
, T_CONFIG_SCOPE_CONNECTION
},
275 { "deflate.max-compress-size", NULL
, T_CONFIG_INT
, T_CONFIG_SCOPE_CONNECTION
},
276 { "deflate.min-compress-size", NULL
, T_CONFIG_SHORT
, T_CONFIG_SCOPE_CONNECTION
},
277 { "deflate.compression-level", NULL
, T_CONFIG_SHORT
, T_CONFIG_SCOPE_CONNECTION
},
278 { "deflate.output-buffer-size", NULL
, T_CONFIG_SHORT
, T_CONFIG_SCOPE_CONNECTION
},
279 { "deflate.work-block-size", NULL
, T_CONFIG_SHORT
, T_CONFIG_SCOPE_CONNECTION
},
280 { "deflate.max-loadavg", NULL
, T_CONFIG_STRING
,T_CONFIG_SCOPE_CONNECTION
},
281 { NULL
, NULL
, T_CONFIG_UNSET
, T_CONFIG_SCOPE_UNSET
}
284 p
->config_storage
= calloc(1, srv
->config_context
->used
* sizeof(plugin_config
*));
286 for (i
= 0; i
< srv
->config_context
->used
; i
++) {
289 s
= calloc(1, sizeof(plugin_config
));
290 s
->mimetypes
= array_init();
291 s
->allowed_encodings
= 0;
292 s
->max_compress_size
= 128*1024; /*(128 MB measured as num KB)*/
293 s
->min_compress_size
= 256;
294 s
->output_buffer_size
= 0;
295 s
->work_block_size
= 2048;
297 s
->compression_level
= -1;
298 s
->max_loadavg
= 0.0;
300 array_reset(p
->encodings
); /* temp array for allowed encodings list */
302 cv
[0].destination
= s
->mimetypes
;
303 cv
[1].destination
= p
->encodings
;
304 cv
[2].destination
= &(s
->max_compress_size
);
305 cv
[3].destination
= &(s
->min_compress_size
);
306 cv
[4].destination
= &(s
->compression_level
);
307 cv
[5].destination
= &(s
->output_buffer_size
);
308 cv
[6].destination
= &(s
->work_block_size
);
309 cv
[7].destination
= p
->tmp_buf
;
310 buffer_string_set_length(p
->tmp_buf
, 0);
312 p
->config_storage
[i
] = s
;
314 if (0 != config_insert_values_global(srv
, ((data_config
*)srv
->config_context
->data
[i
])->value
, cv
, i
== 0 ? T_CONFIG_SCOPE_SERVER
: T_CONFIG_SCOPE_CONNECTION
)) {
315 return HANDLER_ERROR
;
318 if ((s
->compression_level
< 1 || s
->compression_level
> 9) &&
319 s
->compression_level
!= -1) {
320 log_error_write(srv
, __FILE__
, __LINE__
, "sd",
321 "compression-level must be between 1 and 9:", s
->compression_level
);
322 return HANDLER_ERROR
;
325 if (!buffer_string_is_empty(p
->tmp_buf
)) {
326 s
->max_loadavg
= strtod(p
->tmp_buf
->ptr
, NULL
);
329 if (!array_is_vlist(s
->mimetypes
)) {
330 log_error_write(srv
, __FILE__
, __LINE__
, "s",
331 "unexpected value for deflate.mimetypes; expected list of \"mimetype\"");
332 return HANDLER_ERROR
;
335 if (!array_is_vlist(p
->encodings
)) {
336 log_error_write(srv
, __FILE__
, __LINE__
, "s",
337 "unexpected value for deflate.allowed-encodings; expected list of \"encoding\"");
338 return HANDLER_ERROR
;
341 if (p
->encodings
->used
) {
343 for (j
= 0; j
< p
->encodings
->used
; j
++) {
344 #if defined(USE_ZLIB) || defined(USE_BZ2LIB)
345 data_string
*ds
= (data_string
*)p
->encodings
->data
[j
];
348 if (NULL
!= strstr(ds
->value
->ptr
, "gzip"))
349 s
->allowed_encodings
|= HTTP_ACCEPT_ENCODING_GZIP
| HTTP_ACCEPT_ENCODING_X_GZIP
;
350 if (NULL
!= strstr(ds
->value
->ptr
, "x-gzip"))
351 s
->allowed_encodings
|= HTTP_ACCEPT_ENCODING_X_GZIP
;
352 if (NULL
!= strstr(ds
->value
->ptr
, "deflate"))
353 s
->allowed_encodings
|= HTTP_ACCEPT_ENCODING_DEFLATE
;
355 if (NULL != strstr(ds->value->ptr, "compress"))
356 s->allowed_encodings |= HTTP_ACCEPT_ENCODING_COMPRESS;
360 if (NULL
!= strstr(ds
->value
->ptr
, "bzip2"))
361 s
->allowed_encodings
|= HTTP_ACCEPT_ENCODING_BZIP2
| HTTP_ACCEPT_ENCODING_X_BZIP2
;
362 if (NULL
!= strstr(ds
->value
->ptr
, "x-bzip2"))
363 s
->allowed_encodings
|= HTTP_ACCEPT_ENCODING_X_BZIP2
;
367 /* default encodings */
369 s
->allowed_encodings
|= HTTP_ACCEPT_ENCODING_GZIP
370 | HTTP_ACCEPT_ENCODING_X_GZIP
371 | HTTP_ACCEPT_ENCODING_DEFLATE
;
374 s
->allowed_encodings
|= HTTP_ACCEPT_ENCODING_BZIP2
375 | HTTP_ACCEPT_ENCODING_X_BZIP2
;
379 /* mod_deflate matches mimetype as prefix of Content-Type
380 * so ignore '*' at end of mimetype for end-user flexibility
381 * in specifying trailing wildcard to grouping of mimetypes */
382 for (size_t m
= 0; m
< s
->mimetypes
->used
; ++m
) {
383 buffer
*mimetype
= ((data_string
*)s
->mimetypes
->data
[m
])->value
;
384 size_t len
= buffer_string_length(mimetype
);
385 if (len
> 2 && mimetype
->ptr
[len
-1] == '*') {
386 buffer_string_set_length(mimetype
, len
-1);
391 return HANDLER_GO_ON
;
396 #if defined(USE_ZLIB) || defined(USE_BZ2LIB)
397 static int stream_http_chunk_append_mem(server
*srv
, connection
*con
, handler_ctx
*hctx
, size_t len
) {
398 /* future: might also write stream to hctx temporary file in compressed file cache */
399 return http_chunk_append_mem(srv
, con
, hctx
->output
->ptr
, len
);
406 static int stream_deflate_init(handler_ctx
*hctx
) {
407 z_stream
* const z
= &hctx
->u
.z
;
408 const plugin_data
* const p
= hctx
->plugin_data
;
414 z
->next_out
= (unsigned char *)hctx
->output
->ptr
;
415 z
->avail_out
= hctx
->output
->size
;
417 if (Z_OK
!= deflateInit2(z
,
418 p
->conf
.compression_level
> 0
419 ? p
->conf
.compression_level
420 : Z_DEFAULT_COMPRESSION
,
422 (hctx
->compression_type
== HTTP_ACCEPT_ENCODING_GZIP
)
423 ? (MAX_WBITS
| 16) /*(0x10 flags gzip header, trailer)*/
424 : -MAX_WBITS
, /*(negate to suppress zlib header)*/
425 8, /* default memLevel */
426 Z_DEFAULT_STRATEGY
)) {
433 static int stream_deflate_compress(server
*srv
, connection
*con
, handler_ctx
*hctx
, unsigned char *start
, off_t st_size
) {
434 z_stream
* const z
= &(hctx
->u
.z
);
438 z
->avail_in
= st_size
;
439 hctx
->bytes_in
+= st_size
;
443 if (Z_OK
!= deflate(z
, Z_NO_FLUSH
)) return -1;
445 if (z
->avail_out
== 0 || z
->avail_in
> 0) {
446 len
= hctx
->output
->size
- z
->avail_out
;
447 hctx
->bytes_out
+= len
;
448 stream_http_chunk_append_mem(srv
, con
, hctx
, len
);
449 z
->next_out
= (unsigned char *)hctx
->output
->ptr
;
450 z
->avail_out
= hctx
->output
->size
;
452 } while (z
->avail_in
> 0);
457 static int stream_deflate_flush(server
*srv
, connection
*con
, handler_ctx
*hctx
, int end
) {
458 z_stream
* const z
= &(hctx
->u
.z
);
459 const plugin_data
*p
= hctx
->plugin_data
;
468 rc
= deflate(z
, Z_FINISH
);
471 } else if (rc
!= Z_STREAM_END
) {
475 if (p
->conf
.sync_flush
) {
476 rc
= deflate(z
, Z_SYNC_FLUSH
);
477 if (rc
!= Z_OK
) return -1;
478 } else if (z
->avail_in
> 0) {
479 rc
= deflate(z
, Z_NO_FLUSH
);
480 if (rc
!= Z_OK
) return -1;
484 len
= hctx
->output
->size
- z
->avail_out
;
485 if (z
->avail_out
== 0 || (len
> 0 && (end
|| p
->conf
.sync_flush
))) {
486 hctx
->bytes_out
+= len
;
487 stream_http_chunk_append_mem(srv
, con
, hctx
, len
);
488 z
->next_out
= (unsigned char *)hctx
->output
->ptr
;
489 z
->avail_out
= hctx
->output
->size
;
491 } while (z
->avail_in
!= 0 || !done
);
496 static int stream_deflate_end(server
*srv
, handler_ctx
*hctx
) {
497 z_stream
* const z
= &(hctx
->u
.z
);
498 int rc
= deflateEnd(z
);
499 if (Z_OK
== rc
|| Z_DATA_ERROR
== rc
) return 0;
501 if (z
->msg
!= NULL
) {
502 log_error_write(srv
, __FILE__
, __LINE__
, "sdss",
503 "deflateEnd error ret=", rc
, ", msg=", z
->msg
);
505 log_error_write(srv
, __FILE__
, __LINE__
, "sd",
506 "deflateEnd error ret=", rc
);
516 static int stream_bzip2_init(handler_ctx
*hctx
) {
517 bz_stream
* const bz
= &hctx
->u
.bz
;
518 const plugin_data
* const p
= hctx
->plugin_data
;
522 bz
->total_in_lo32
= 0;
523 bz
->total_in_hi32
= 0;
524 bz
->total_out_lo32
= 0;
525 bz
->total_out_hi32
= 0;
526 bz
->next_out
= hctx
->output
->ptr
;
527 bz
->avail_out
= hctx
->output
->size
;
529 if (BZ_OK
!= BZ2_bzCompressInit(bz
,
530 p
->conf
.compression_level
> 0
531 ? p
->conf
.compression_level
532 : 9, /* blocksize = 900k */
534 0)) { /* workFactor: default */
541 static int stream_bzip2_compress(server
*srv
, connection
*con
, handler_ctx
*hctx
, unsigned char *start
, off_t st_size
) {
542 bz_stream
* const bz
= &(hctx
->u
.bz
);
545 bz
->next_in
= (char *)start
;
546 bz
->avail_in
= st_size
;
547 hctx
->bytes_in
+= st_size
;
551 if (BZ_RUN_OK
!= BZ2_bzCompress(bz
, BZ_RUN
)) return -1;
553 if (bz
->avail_out
== 0 || bz
->avail_in
> 0) {
554 len
= hctx
->output
->size
- bz
->avail_out
;
555 hctx
->bytes_out
+= len
;
556 stream_http_chunk_append_mem(srv
, con
, hctx
, len
);
557 bz
->next_out
= hctx
->output
->ptr
;
558 bz
->avail_out
= hctx
->output
->size
;
560 } while (bz
->avail_in
> 0);
565 static int stream_bzip2_flush(server
*srv
, connection
*con
, handler_ctx
*hctx
, int end
) {
566 bz_stream
* const bz
= &(hctx
->u
.bz
);
567 const plugin_data
*p
= hctx
->plugin_data
;
576 rc
= BZ2_bzCompress(bz
, BZ_FINISH
);
577 if (rc
== BZ_FINISH_OK
) {
579 } else if (rc
!= BZ_STREAM_END
) {
582 } else if (bz
->avail_in
> 0) {
583 /* p->conf.sync_flush not implemented here,
584 * which would loop on BZ_FLUSH while BZ_FLUSH_OK
585 * until BZ_RUN_OK returned */
586 rc
= BZ2_bzCompress(bz
, BZ_RUN
);
587 if (rc
!= BZ_RUN_OK
) {
592 len
= hctx
->output
->size
- bz
->avail_out
;
593 if (bz
->avail_out
== 0 || (len
> 0 && (end
|| p
->conf
.sync_flush
))) {
594 hctx
->bytes_out
+= len
;
595 stream_http_chunk_append_mem(srv
, con
, hctx
, len
);
596 bz
->next_out
= hctx
->output
->ptr
;
597 bz
->avail_out
= hctx
->output
->size
;
599 } while (bz
->avail_in
!= 0 || !done
);
604 static int stream_bzip2_end(server
*srv
, handler_ctx
*hctx
) {
605 bz_stream
* const bz
= &(hctx
->u
.bz
);
606 int rc
= BZ2_bzCompressEnd(bz
);
607 if (BZ_OK
== rc
|| BZ_DATA_ERROR
== rc
) return 0;
609 log_error_write(srv
, __FILE__
, __LINE__
, "sd",
610 "BZ2_bzCompressEnd error ret=", rc
);
617 static int mod_deflate_stream_init(handler_ctx
*hctx
) {
618 switch(hctx
->compression_type
) {
620 case HTTP_ACCEPT_ENCODING_GZIP
:
621 case HTTP_ACCEPT_ENCODING_DEFLATE
:
622 return stream_deflate_init(hctx
);
625 case HTTP_ACCEPT_ENCODING_BZIP2
:
626 return stream_bzip2_init(hctx
);
633 static int mod_deflate_compress(server
*srv
, connection
*con
, handler_ctx
*hctx
, unsigned char *start
, off_t st_size
) {
634 if (0 == st_size
) return 0;
635 switch(hctx
->compression_type
) {
637 case HTTP_ACCEPT_ENCODING_GZIP
:
638 case HTTP_ACCEPT_ENCODING_DEFLATE
:
639 return stream_deflate_compress(srv
, con
, hctx
, start
, st_size
);
642 case HTTP_ACCEPT_ENCODING_BZIP2
:
643 return stream_bzip2_compress(srv
, con
, hctx
, start
, st_size
);
653 static int mod_deflate_stream_flush(server
*srv
, connection
*con
, handler_ctx
*hctx
, int end
) {
654 if (0 == hctx
->bytes_in
) return 0;
655 switch(hctx
->compression_type
) {
657 case HTTP_ACCEPT_ENCODING_GZIP
:
658 case HTTP_ACCEPT_ENCODING_DEFLATE
:
659 return stream_deflate_flush(srv
, con
, hctx
, end
);
662 case HTTP_ACCEPT_ENCODING_BZIP2
:
663 return stream_bzip2_flush(srv
, con
, hctx
, end
);
673 static void mod_deflate_note_ratio(server
*srv
, connection
*con
, handler_ctx
*hctx
) {
674 /* store compression ratio in con->environment
675 * for possible logging by mod_accesslog
676 * (late in response handling, so not seen by most other modules) */
677 /*(should be called only at end of successful response compression)*/
678 char ratio
[LI_ITOSTRING_LENGTH
];
679 if (0 == hctx
->bytes_in
) return;
680 li_itostrn(ratio
, sizeof(ratio
), hctx
->bytes_out
* 100 / hctx
->bytes_in
);
681 array_set_key_value(con
->environment
,
682 CONST_STR_LEN("ratio"),
683 ratio
, strlen(ratio
));
687 static int mod_deflate_stream_end(server
*srv
, handler_ctx
*hctx
) {
688 switch(hctx
->compression_type
) {
690 case HTTP_ACCEPT_ENCODING_GZIP
:
691 case HTTP_ACCEPT_ENCODING_DEFLATE
:
692 return stream_deflate_end(srv
, hctx
);
695 case HTTP_ACCEPT_ENCODING_BZIP2
:
696 return stream_bzip2_end(srv
, hctx
);
704 static void deflate_compress_cleanup(server
*srv
, connection
*con
, handler_ctx
*hctx
) {
705 const plugin_data
*p
= hctx
->plugin_data
;
706 con
->plugin_ctx
[p
->id
] = NULL
;
708 if (0 != mod_deflate_stream_end(srv
, hctx
)) {
709 log_error_write(srv
, __FILE__
, __LINE__
, "s", "error closing stream");
712 #if 1 /* unnecessary if deflate.min-compress-size is set to a reasonable value */
713 if (hctx
->bytes_in
< hctx
->bytes_out
) {
714 log_error_write(srv
, __FILE__
, __LINE__
, "sbsdsd",
715 "uri ", con
->uri
.path_raw
, " in=", hctx
->bytes_in
, " smaller than out=", hctx
->bytes_out
);
719 handler_ctx_free(hctx
);
723 static int mod_deflate_file_chunk(server
*srv
, connection
*con
, handler_ctx
*hctx
, chunk
*c
, off_t st_size
) {
728 off_t we_want_to_mmap
= 2 MByte
;
729 off_t we_want_to_send
= st_size
;
730 volatile int mapped
= 0;/* quiet warning: might be clobbered by 'longjmp' */
735 if (-1 == c
->file
.fd
) { /* open the file if not already open */
736 if (-1 == (c
->file
.fd
= fdevent_open_cloexec(c
->file
.name
->ptr
, O_RDONLY
, 0))) {
737 log_error_write(srv
, __FILE__
, __LINE__
, "sbs", "open failed for:", c
->file
.name
, strerror(errno
));
743 abs_offset
= c
->file
.start
+ c
->offset
;
748 * - new mmap as the we are at the end of the last one */
749 if (c
->file
.mmap
.start
== MAP_FAILED
||
750 abs_offset
== (off_t
)(c
->file
.mmap
.offset
+ c
->file
.mmap
.length
)) {
752 /* Optimizations for the future:
754 * adaptive mem-mapping
756 * we mmap() the whole file. If someone has alot large files and 32bit
757 * machine the virtual address area will be unrun and we will have a failing
760 * only mmap 16M in one chunk and move the window as soon as we have finished
763 * read-ahead buffering
765 * sending out several large files in parallel trashes the read-ahead of the
766 * kernel leading to long wait-for-seek times.
767 * solutions: (increasing complexity)
769 * 2. use a internal read-ahead buffer in the chunk-structure
770 * 3. use non-blocking IO for file-transfers
773 /* all mmap()ed areas are 512kb expect the last which might be smaller */
776 /* this is a remap, move the mmap-offset */
777 if (c
->file
.mmap
.start
!= MAP_FAILED
) {
778 munmap(c
->file
.mmap
.start
, c
->file
.mmap
.length
);
779 c
->file
.mmap
.offset
+= we_want_to_mmap
;
781 /* in case the range-offset is after the first mmap()ed area we skip the area */
782 c
->file
.mmap
.offset
= 0;
784 while (c
->file
.mmap
.offset
+ we_want_to_mmap
< c
->file
.start
) {
785 c
->file
.mmap
.offset
+= we_want_to_mmap
;
789 /* length is rel, c->offset too, assume there is no limit at the mmap-boundaries */
790 to_mmap
= (c
->file
.start
+ c
->file
.length
) - c
->file
.mmap
.offset
;
791 if (to_mmap
> we_want_to_mmap
) to_mmap
= we_want_to_mmap
;
792 /* we have more to send than we can mmap() at once */
793 if (we_want_to_send
> to_mmap
) we_want_to_send
= to_mmap
;
795 if (MAP_FAILED
== (c
->file
.mmap
.start
= mmap(0, (size_t)to_mmap
, PROT_READ
, MAP_SHARED
, c
->file
.fd
, c
->file
.mmap
.offset
))) {
796 /* close it here, otherwise we'd have to set FD_CLOEXEC */
798 log_error_write(srv
, __FILE__
, __LINE__
, "ssbd", "mmap failed:",
799 strerror(errno
), c
->file
.name
, c
->file
.fd
);
804 c
->file
.mmap
.length
= to_mmap
;
806 /* don't advise files < 64Kb */
807 if (c
->file
.mmap
.length
> (64 KByte
) &&
808 0 != madvise(c
->file
.mmap
.start
, c
->file
.mmap
.length
, MADV_WILLNEED
)) {
809 log_error_write(srv
, __FILE__
, __LINE__
, "ssbd", "madvise failed:",
810 strerror(errno
), c
->file
.name
, c
->file
.fd
);
814 /* chunk_reset() or chunk_free() will cleanup for us */
817 /* to_send = abs_mmap_end - abs_offset */
818 toSend
= (c
->file
.mmap
.offset
+ c
->file
.mmap
.length
) - abs_offset
;
819 if (toSend
> we_want_to_send
) toSend
= we_want_to_send
;
822 log_error_write(srv
, __FILE__
, __LINE__
, "soooo",
823 "toSend is negative:",
827 c
->file
.mmap
.offset
);
828 force_assert(toSend
< 0);
831 start
= c
->file
.mmap
.start
;
835 if (MAP_FAILED
== c
->file
.mmap
.start
) {
837 if (toSend
> 2 MByte
) toSend
= 2 MByte
;
838 if (NULL
== (start
= malloc((size_t)toSend
)) || -1 == lseek(c
->file
.fd
, abs_offset
, SEEK_SET
) || toSend
!= read(c
->file
.fd
, start
, (size_t)toSend
)) {
839 log_error_write(srv
, __FILE__
, __LINE__
, "sbss", "reading", c
->file
.name
, "failed:", strerror(errno
));
848 signal(SIGBUS
, sigbus_handler
);
849 sigbus_jmp_valid
= 1;
850 if (0 != sigsetjmp(sigbus_jmp
, 1)) {
851 sigbus_jmp_valid
= 0;
853 log_error_write(srv
, __FILE__
, __LINE__
, "sbd", "SIGBUS in mmap:",
854 c
->file
.name
, c
->file
.fd
);
860 if (mod_deflate_compress(srv
, con
, hctx
,
861 (unsigned char *)start
+ (abs_offset
- c
->file
.mmap
.offset
), toSend
) < 0) {
862 log_error_write(srv
, __FILE__
, __LINE__
, "s",
869 sigbus_jmp_valid
= 0;
878 static handler_t
deflate_compress_response(server
*srv
, connection
*con
, handler_ctx
*hctx
) {
882 /* move all chunk from write_queue into our in_queue, then adjust
883 * counters since con->write_queue is reused for compressed output */
884 len
= chunkqueue_length(con
->write_queue
);
885 chunkqueue_remove_finished_chunks(con
->write_queue
);
886 chunkqueue_append_chunkqueue(hctx
->in_queue
, con
->write_queue
);
887 con
->write_queue
->bytes_in
-= len
;
888 con
->write_queue
->bytes_out
-= len
;
890 max
= chunkqueue_length(hctx
->in_queue
);
892 /* calculate max bytes to compress for this call */
893 if (p
->conf
.sync_flush
&& max
> (len
= p
->conf
.work_block_size
<< 10)) {
898 /* Compress chunks from in_queue into chunks for write_queue */
900 chunk
*c
= hctx
->in_queue
->first
;
904 len
= buffer_string_length(c
->mem
) - c
->offset
;
905 if (len
> max
) len
= max
;
906 if (mod_deflate_compress(srv
, con
, hctx
, (unsigned char *)c
->mem
->ptr
+c
->offset
, len
) < 0) {
907 log_error_write(srv
, __FILE__
, __LINE__
, "s",
909 return HANDLER_ERROR
;
913 len
= c
->file
.length
- c
->offset
;
914 if (len
> max
) len
= max
;
915 if ((len
= mod_deflate_file_chunk(srv
, con
, hctx
, c
, len
)) < 0) {
916 log_error_write(srv
, __FILE__
, __LINE__
, "s",
917 "compress file chunk failed.");
918 return HANDLER_ERROR
;
922 log_error_write(srv
, __FILE__
, __LINE__
, "ds", c
, "type not known");
923 return HANDLER_ERROR
;
927 chunkqueue_mark_written(hctx
->in_queue
, len
);
930 /*(currently should always be true)*/
931 /*(current implementation requires response be complete)*/
932 close_stream
= (con
->file_finished
&& chunkqueue_is_empty(hctx
->in_queue
));
933 if (mod_deflate_stream_flush(srv
, con
, hctx
, close_stream
) < 0) {
934 log_error_write(srv
, __FILE__
, __LINE__
, "s", "flush error");
935 return HANDLER_ERROR
;
938 return close_stream
? HANDLER_FINISHED
: HANDLER_GO_ON
;
944 static int mod_deflate_patch_connection(server
*srv
, connection
*con
, plugin_data
*p
) {
946 plugin_config
*s
= p
->config_storage
[0];
949 PATCH(allowed_encodings
);
950 PATCH(max_compress_size
);
951 PATCH(min_compress_size
);
952 PATCH(compression_level
);
953 PATCH(output_buffer_size
);
954 PATCH(work_block_size
);
957 /* skip the first, the global context */
958 for (i
= 1; i
< srv
->config_context
->used
; i
++) {
959 data_config
*dc
= (data_config
*)srv
->config_context
->data
[i
];
960 s
= p
->config_storage
[i
];
962 /* condition didn't match */
963 if (!config_check_cond(srv
, con
, dc
)) continue;
966 for (j
= 0; j
< dc
->value
->used
; j
++) {
967 data_unset
*du
= dc
->value
->data
[j
];
969 if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.mimetypes"))) {
971 } else if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.allowed-encodings"))) {
972 PATCH(allowed_encodings
);
973 } else if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.max-compress-size"))) {
974 PATCH(max_compress_size
);
975 } else if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.min-compress-size"))) {
976 PATCH(min_compress_size
);
977 } else if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.compression-level"))) {
978 PATCH(compression_level
);
979 } else if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.output-buffer-size"))) {
980 PATCH(output_buffer_size
);
981 } else if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.work-block-size"))) {
982 PATCH(work_block_size
);
983 } else if (buffer_is_equal_string(du
->key
, CONST_STR_LEN("deflate.max-loadavg"))) {
993 static int mod_deflate_choose_encoding (const char *value
, plugin_data
*p
, const char **label
) {
994 /* get client side support encodings */
995 int accept_encoding
= 0;
996 #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB)
1000 if (NULL
!= strstr(value
, "gzip")) accept_encoding
|= HTTP_ACCEPT_ENCODING_GZIP
;
1001 else if (NULL
!= strstr(value
, "x-gzip")) accept_encoding
|= HTTP_ACCEPT_ENCODING_X_GZIP
;
1002 if (NULL
!= strstr(value
, "deflate")) accept_encoding
|= HTTP_ACCEPT_ENCODING_DEFLATE
;
1004 /* if (NULL != strstr(value, "compress")) accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; */
1006 if (p
->conf
.allowed_encodings
& (HTTP_ACCEPT_ENCODING_BZIP2
| HTTP_ACCEPT_ENCODING_X_BZIP2
)) {
1007 if (NULL
!= strstr(value
, "bzip2")) accept_encoding
|= HTTP_ACCEPT_ENCODING_BZIP2
;
1008 else if (NULL
!= strstr(value
, "x-bzip2")) accept_encoding
|= HTTP_ACCEPT_ENCODING_X_BZIP2
;
1011 /* if (NULL != strstr(value, "identity")) accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; */
1013 /* mask to limit to allowed_encodings */
1014 accept_encoding
&= p
->conf
.allowed_encodings
;
1016 /* select best matching encoding */
1018 if (accept_encoding
& HTTP_ACCEPT_ENCODING_BZIP2
) {
1020 return HTTP_ACCEPT_ENCODING_BZIP2
;
1021 } else if (accept_encoding
& HTTP_ACCEPT_ENCODING_X_BZIP2
) {
1023 return HTTP_ACCEPT_ENCODING_BZIP2
;
1026 if (accept_encoding
& HTTP_ACCEPT_ENCODING_GZIP
) {
1028 return HTTP_ACCEPT_ENCODING_GZIP
;
1029 } else if (accept_encoding
& HTTP_ACCEPT_ENCODING_X_GZIP
) {
1031 return HTTP_ACCEPT_ENCODING_GZIP
;
1032 } else if (accept_encoding
& HTTP_ACCEPT_ENCODING_DEFLATE
) {
1034 return HTTP_ACCEPT_ENCODING_DEFLATE
;
1040 CONNECTION_FUNC(mod_deflate_handle_response_start
) {
1041 plugin_data
*p
= p_d
;
1047 int compression_type
;
1050 /*(current implementation requires response be complete)*/
1051 if (!con
->file_finished
) return HANDLER_GO_ON
;
1052 if (con
->request
.http_method
== HTTP_METHOD_HEAD
) return HANDLER_GO_ON
;
1053 if (con
->parsed_response
& HTTP_TRANSFER_ENCODING
) return HANDLER_GO_ON
;
1055 /* disable compression for some http status types. */
1056 switch(con
->http_status
) {
1062 /* disable compression as we have no response entity */
1063 return HANDLER_GO_ON
;
1068 mod_deflate_patch_connection(srv
, con
, p
);
1070 /* check if deflate configured for any mimetypes */
1071 if (!p
->conf
.mimetypes
->used
) return HANDLER_GO_ON
;
1073 /* check if size of response is below min-compress-size or exceeds max*/
1074 /* (con->file_finished checked at top of routine) */
1075 len
= chunkqueue_length(con
->write_queue
);
1076 if (len
<= (off_t
)p
->conf
.min_compress_size
) return HANDLER_GO_ON
;
1077 if (p
->conf
.max_compress_size
/*(max_compress_size in KB)*/
1078 && len
> ((off_t
)p
->conf
.max_compress_size
<< 10)) {
1079 return HANDLER_GO_ON
;
1082 /* Check if response has a Content-Encoding. */
1083 ds
= (data_string
*)array_get_element(con
->response
.headers
, "Content-Encoding");
1084 if (NULL
!= ds
) return HANDLER_GO_ON
;
1086 /* Check Accept-Encoding for supported encoding. */
1087 ds
= (data_string
*)array_get_element(con
->request
.headers
, "Accept-Encoding");
1088 if (NULL
== ds
) return HANDLER_GO_ON
;
1090 /* find matching encodings */
1091 compression_type
= mod_deflate_choose_encoding(ds
->value
->ptr
, p
, &label
);
1092 if (!compression_type
) return HANDLER_GO_ON
;
1094 /* Check mimetype in response header "Content-Type" */
1095 if (NULL
!= (ds
= (data_string
*)array_get_element(con
->response
.headers
, "Content-Type"))) {
1098 for (m
= 0; m
< p
->conf
.mimetypes
->used
; ++m
) {
1099 data_string
*mimetype
= (data_string
*)p
->conf
.mimetypes
->data
[m
];
1100 if (0 == strncmp(mimetype
->value
->ptr
, ds
->value
->ptr
, buffer_string_length(mimetype
->value
))) {
1101 /* mimetype found */
1106 if (!found
) return HANDLER_GO_ON
;
1109 if (0 == strncasecmp(ds
->value
->ptr
, "application/x-javascript", 24)) {
1110 /*reset compress type to deflate for javascript
1111 * prevent buggy IE6 SP1 doesn't work for js in IFrame
1113 compression_type
= HTTP_ACCEPT_ENCODING_DEFLATE
;
1117 /* If no Content-Type set, compress only if first p->conf.mimetypes value is "" */
1118 data_string
*mimetype
= (data_string
*)p
->conf
.mimetypes
->data
[0];
1119 if (!buffer_string_is_empty(mimetype
->value
)) return HANDLER_GO_ON
;
1122 /* Vary: Accept-Encoding (response might change according to request Accept-Encoding) */
1123 if (NULL
!= (ds
= (data_string
*)array_get_element(con
->response
.headers
, "Vary"))) {
1124 if (NULL
== strstr(ds
->value
->ptr
, "Accept-Encoding")) {
1125 buffer_append_string_len(ds
->value
, CONST_STR_LEN(",Accept-Encoding"));
1128 response_header_insert(srv
, con
, CONST_STR_LEN("Vary"),
1129 CONST_STR_LEN("Accept-Encoding"));
1132 /* check ETag as is done in http_response_handle_cachable()
1133 * (slightly imperfect (close enough?) match of ETag "000000" to "000000-gzip") */
1134 ds
= (data_string
*)array_get_element(con
->response
.headers
, "ETag");
1136 etaglen
= buffer_string_length(ds
->value
);
1138 && con
->http_status
< 300 /*(want 2xx only)*/
1139 && con
->request
.http_if_none_match
1140 && 0 == strncmp(con
->request
.http_if_none_match
, ds
->value
->ptr
, etaglen
-1)
1141 && con
->request
.http_if_none_match
[etaglen
-1] == '-'
1142 && 0 == strncmp(con
->request
.http_if_none_match
+etaglen
, label
, strlen(label
))) {
1144 if ( HTTP_METHOD_GET
== con
->request
.http_method
1145 || HTTP_METHOD_HEAD
== con
->request
.http_method
) {
1146 /* modify ETag response header in-place to remove '"' and append '-label"' */
1147 ds
->value
->ptr
[etaglen
-1] = '-'; /*(overwrite end '"')*/
1148 buffer_append_string(ds
->value
, label
);
1149 buffer_append_string_len(ds
->value
, CONST_STR_LEN("\""));
1150 /*buffer_copy_buffer(con->physical.etag, ds->value);*//*(keep in sync?)*/
1151 con
->http_status
= 304;
1153 con
->http_status
= 412;
1156 /* response_start hook occurs after error docs have been handled.
1157 * For now, send back empty response body.
1158 * In the future, might extract the error doc code so that it
1159 * might be run again if response_start hooks return with
1160 * changed http_status and con->mode = DIRECT */
1161 con
->response
.transfer_encoding
&= ~HTTP_TRANSFER_ENCODING_CHUNKED
;
1162 con
->parsed_response
&= ~HTTP_CONTENT_LENGTH
;
1163 chunkqueue_reset(con
->write_queue
);
1164 con
->file_finished
= 1;
1167 return HANDLER_GO_ON
;
1171 if (0.0 < p
->conf
.max_loadavg
&& p
->conf
.max_loadavg
< srv
->srvconf
.loadavg
[0]) {
1172 return HANDLER_GO_ON
;
1175 /* update ETag, if ETag response header is set */
1177 /* modify ETag response header in-place to remove '"' and append '-label"' */
1178 ds
->value
->ptr
[etaglen
-1] = '-'; /*(overwrite end '"')*/
1179 buffer_append_string(ds
->value
, label
);
1180 buffer_append_string_len(ds
->value
, CONST_STR_LEN("\""));
1181 /*buffer_copy_buffer(con->physical.etag, ds->value);*//*(keep in sync?)*/
1184 /* set Content-Encoding to show selected compression type */
1185 response_header_overwrite(srv
, con
, CONST_STR_LEN("Content-Encoding"), label
, strlen(label
));
1187 /* clear Content-Length and con->write_queue if HTTP HEAD request
1188 * (alternatively, could return original Content-Length with HEAD
1189 * request if ETag not modified and Content-Encoding not added) */
1190 if (HTTP_METHOD_HEAD
== con
->request
.http_method
) {
1191 /* ensure that uncompressed Content-Length is not sent in HEAD response */
1192 chunkqueue_reset(con
->write_queue
);
1193 if (con
->parsed_response
& HTTP_CONTENT_LENGTH
) {
1194 con
->parsed_response
&= ~HTTP_CONTENT_LENGTH
;
1195 if (NULL
!= (ds
= (data_string
*) array_get_element(con
->response
.headers
, "Content-Length"))) {
1196 buffer_reset(ds
->value
); /* headers with empty values are ignored for output */
1199 return HANDLER_GO_ON
;
1202 /* future: might use ETag to check if compressed content is in compressed file cache */
1203 /*if (etaglen) { ... } *//* return if in file cache after updating con->write_queue */
1205 /* enable compression */
1206 p
->conf
.sync_flush
=
1207 (con
->conf
.stream_response_body
&& 0 == p
->conf
.output_buffer_size
);
1208 hctx
= handler_ctx_init();
1209 hctx
->plugin_data
= p
;
1210 hctx
->compression_type
= compression_type
;
1211 /* setup output buffer */
1212 buffer_string_set_length(p
->tmp_buf
, 0);
1213 hctx
->output
= p
->tmp_buf
;
1214 if (0 != mod_deflate_stream_init(hctx
)) {
1215 /*(should not happen unless ENOMEM)*/
1216 handler_ctx_free(hctx
);
1217 log_error_write(srv
, __FILE__
, __LINE__
, "ss",
1218 "Failed to initialize compression", label
);
1219 /* restore prior Etag and unset Content-Encoding */
1221 ds
->value
->ptr
[etaglen
-1] = '"'; /*(overwrite '-')*/
1222 buffer_string_set_length(ds
->value
, etaglen
);
1224 ds
= (data_string
*)array_get_element(con
->response
.headers
, "Content-Encoding");
1225 if (ds
) buffer_reset(ds
->value
); /* headers with empty values are ignored for output */
1226 return HANDLER_GO_ON
;
1229 con
->parsed_response
&= ~HTTP_CONTENT_LENGTH
;
1230 con
->plugin_ctx
[p
->id
] = hctx
;
1232 rc
= deflate_compress_response(srv
, con
, hctx
);
1233 if (HANDLER_GO_ON
!= rc
) {
1234 if (HANDLER_FINISHED
== rc
) {
1235 mod_deflate_note_ratio(srv
, con
, hctx
);
1237 deflate_compress_cleanup(srv
, con
, hctx
);
1238 if (HANDLER_ERROR
== rc
) return HANDLER_ERROR
;
1241 return HANDLER_GO_ON
;
1244 static handler_t
mod_deflate_cleanup(server
*srv
, connection
*con
, void *p_d
) {
1245 plugin_data
*p
= p_d
;
1246 handler_ctx
*hctx
= con
->plugin_ctx
[p
->id
];
1248 if (NULL
!= hctx
) deflate_compress_cleanup(srv
, con
, hctx
);
1250 return HANDLER_GO_ON
;
1253 int mod_deflate_plugin_init(plugin
*p
);
1254 int mod_deflate_plugin_init(plugin
*p
) {
1255 p
->version
= LIGHTTPD_VERSION_ID
;
1256 p
->name
= buffer_init_string("deflate");
1258 p
->init
= mod_deflate_init
;
1259 p
->cleanup
= mod_deflate_free
;
1260 p
->set_defaults
= mod_deflate_setdefaults
;
1261 p
->connection_reset
= mod_deflate_cleanup
;
1262 p
->handle_response_start
= mod_deflate_handle_response_start
;