From 1f3ad401ba4d7f75f554eaf4defd8ad19582f96c Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Wed, 19 Oct 2016 16:09:29 -0400 Subject: [PATCH] [mod_deflate] skip deflate if loadavg too high (fixes #1505) [mod_deflate] skip deflate if 1 min loadavg too high deflate.max-loadavg = "3.50" # express value as string of float num [mod_compress] skip compression if 1 min loadavg too high compress.max-loadavg = "3.50" # express value as string of float num Feature available on BSD-like systems which have getloadavg() in libc Note: load average calculations are different on different operating systems and different types of system loads, so there is no value that can be recommended for one-size-fits-all. x-ref: "Enable mod_compress to abandon compression when load average is too high" https://redmine.lighttpd.net/issues/1505 --- SConstruct | 2 +- configure.ac | 2 +- src/CMakeLists.txt | 1 + src/base.h | 2 ++ src/mod_compress.c | 19 +++++++++++++++++++ src/mod_deflate.c | 16 ++++++++++++++++ src/server.c | 12 ++++++++++++ 7 files changed, 52 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index cda2f7d3..98fe4d77 100644 --- a/SConstruct +++ b/SConstruct @@ -221,7 +221,7 @@ if 1: getuid select signal pathconf madvise prctl\ writev sigaction sendfile64 send_file kqueue port_create localtime_r posix_fadvise issetugid inet_pton \ memset_s explicit_bzero clock_gettime \ - getentropy arc4random jrand48 srandom')) + getentropy arc4random jrand48 srandom getloadavg')) checkFunc(autoconf, 'getrandom', 'linux/random.h') checkTypes(autoconf, Split('pid_t size_t off_t')) diff --git a/configure.ac b/configure.ac index 189e26ce..9f563b98 100644 --- a/configure.ac +++ b/configure.ac @@ -772,7 +772,7 @@ AC_CHECK_FUNCS([dup2 getcwd inet_ntoa inet_ntop inet_pton issetugid memset mmap getuid select signal pathconf madvise posix_fadvise posix_madvise \ writev sigaction sendfile64 send_file kqueue port_create localtime_r gmtime_r \ memset_s explicit_bzero clock_gettime \ - getentropy arc4random jrand48 srandom]) + getentropy arc4random jrand48 srandom getloadavg]) AC_CHECK_HEADERS([linux/random.h],[ AC_CHECK_FUNC([getrandom], AC_DEFINE([HAVE_GETRANDOM], [1], [getrandom])) ]) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 55e72365..f7e4de6a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -134,6 +134,7 @@ check_function_exists(chroot HAVE_CHROOT) check_function_exists(epoll_ctl HAVE_EPOLL_CTL) check_function_exists(fork HAVE_FORK) check_function_exists(getentropy HAVE_GETENTROPY) +check_function_exists(getloadavg HAVE_GETLOADAVG) check_function_exists(getrlimit HAVE_GETRLIMIT) check_function_exists(getuid HAVE_GETUID) check_function_exists(gmtime_r HAVE_GMTIME_R) diff --git a/src/base.h b/src/base.h index 9e6f25ff..e48c8e0e 100644 --- a/src/base.h +++ b/src/base.h @@ -563,6 +563,8 @@ typedef struct { unsigned short http_host_strict; unsigned short http_host_normalize; unsigned short high_precision_timestamps; + time_t loadts; + double loadavg[3]; } server_config; typedef struct server_socket { diff --git a/src/mod_compress.c b/src/mod_compress.c index 99f92e00..bbd3e71a 100644 --- a/src/mod_compress.c +++ b/src/mod_compress.c @@ -71,6 +71,7 @@ typedef struct { array *compress; off_t compress_max_filesize; /** max filesize in kb */ int allowed_encodings; + double max_loadavg; } plugin_config; typedef struct { @@ -177,6 +178,7 @@ SETDEFAULTS_FUNC(mod_compress_setdefaults) { { "compress.filetype", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, { "compress.max-filesize", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, { "compress.allowed-encodings", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, + { "compress.max-loadavg", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION }, { NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET } }; @@ -192,11 +194,14 @@ SETDEFAULTS_FUNC(mod_compress_setdefaults) { s->compress = array_init(); s->compress_max_filesize = 0; s->allowed_encodings = 0; + s->max_loadavg = 0.0; cv[0].destination = s->compress_cache_dir; cv[1].destination = s->compress; cv[2].destination = &(s->compress_max_filesize); cv[3].destination = encodings_arr; /* temp array for allowed encodings list */ + cv[4].destination = srv->tmp_buf; + buffer_string_set_length(srv->tmp_buf, 0); p->config_storage[i] = s; @@ -204,6 +209,10 @@ SETDEFAULTS_FUNC(mod_compress_setdefaults) { return HANDLER_ERROR; } + if (!buffer_string_is_empty(srv->tmp_buf)) { + s->max_loadavg = strtod(srv->tmp_buf->ptr, NULL); + } + if (encodings_arr->used) { size_t j = 0; for (j = 0; j < encodings_arr->used; j++) { @@ -509,6 +518,10 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu return 0; } + if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) { + return -1; + } + if (-1 == mkdir_for_file(p->ofn->ptr)) { log_error_write(srv, __FILE__, __LINE__, "sb", "couldn't create directory for file", p->ofn); return -1; @@ -656,6 +669,9 @@ static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, if (sce->st.st_size > 128 * 1024 * 1024) return -1; + if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) { + return -1; + } if (-1 == (ifd = open(fn->ptr, O_RDONLY | O_BINARY))) { log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno)); @@ -745,6 +761,7 @@ static int mod_compress_patch_connection(server *srv, connection *con, plugin_da PATCH(compress); PATCH(compress_max_filesize); PATCH(allowed_encodings); + PATCH(max_loadavg); /* skip the first, the global context */ for (i = 1; i < srv->config_context->used; i++) { @@ -766,6 +783,8 @@ static int mod_compress_patch_connection(server *srv, connection *con, plugin_da PATCH(compress_max_filesize); } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.allowed-encodings"))) { PATCH(allowed_encodings); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.max-loadavg"))) { + PATCH(max_loadavg); } } } diff --git a/src/mod_deflate.c b/src/mod_deflate.c index 6eecdb12..99a9a857 100644 --- a/src/mod_deflate.c +++ b/src/mod_deflate.c @@ -174,6 +174,7 @@ typedef struct { unsigned short work_block_size; unsigned short sync_flush; short compression_level; + double max_loadavg; } plugin_config; typedef struct { @@ -274,6 +275,7 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) { { "deflate.compression-level", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, { "deflate.output-buffer-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, { "deflate.work-block-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.max-loadavg", NULL, T_CONFIG_STRING,T_CONFIG_SCOPE_CONNECTION }, { NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET } }; @@ -291,6 +293,7 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) { s->work_block_size = 2048; s->sync_flush = 0; s->compression_level = -1; + s->max_loadavg = 0.0; array_reset(p->encodings); /* temp array for allowed encodings list */ @@ -301,6 +304,8 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) { cv[4].destination = &(s->compression_level); cv[5].destination = &(s->output_buffer_size); cv[6].destination = &(s->work_block_size); + cv[7].destination = p->tmp_buf; + buffer_string_set_length(p->tmp_buf, 0); p->config_storage[i] = s; @@ -315,6 +320,10 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) { return HANDLER_ERROR; } + if (!buffer_string_is_empty(p->tmp_buf)) { + s->max_loadavg = strtod(p->tmp_buf->ptr, NULL); + } + if (p->encodings->used) { size_t j = 0; for (j = 0; j < p->encodings->used; j++) { @@ -929,6 +938,7 @@ static int mod_deflate_patch_connection(server *srv, connection *con, plugin_dat PATCH(compression_level); PATCH(output_buffer_size); PATCH(work_block_size); + PATCH(max_loadavg); /* skip the first, the global context */ for (i = 1; i < srv->config_context->used; i++) { @@ -956,6 +966,8 @@ static int mod_deflate_patch_connection(server *srv, connection *con, plugin_dat PATCH(output_buffer_size); } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.work-block-size"))) { PATCH(work_block_size); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.max-loadavg"))) { + PATCH(max_loadavg); } } } @@ -1142,6 +1154,10 @@ CONNECTION_FUNC(mod_deflate_handle_response_start) { } } + if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) { + return HANDLER_GO_ON; + } + /* update ETag, if ETag response header is set */ if (etaglen) { /* modify ETag response header in-place to remove '"' and append '-label"' */ diff --git a/src/server.c b/src/server.c index 01f10d87..f00a8ea0 100644 --- a/src/server.c +++ b/src/server.c @@ -269,6 +269,9 @@ static server *server_init(void) { srv->srvconf.http_host_normalize = 0; srv->srvconf.high_precision_timestamps = 0; srv->srvconf.max_request_field_size = 8192; + srv->srvconf.loadavg[0] = 0.0; + srv->srvconf.loadavg[1] = 0.0; + srv->srvconf.loadavg[2] = 0.0; /* use syslog */ srv->errorlog_fd = STDERR_FILENO; @@ -1583,6 +1586,15 @@ int main (int argc, char **argv) { graceful_shutdown = 2; /* value 2 indicates idle timeout */ } + #ifdef HAVE_GETLOADAVG + /* refresh loadavg data every 30 seconds */ + if (srv->srvconf.loadts + 30 < min_ts) { + if (-1 != getloadavg(srv->srvconf.loadavg, 3)) { + srv->srvconf.loadts = min_ts; + } + } + #endif + /* cleanup stat-cache */ stat_cache_trigger_cleanup(srv); /** -- 2.11.4.GIT