From 2dcaa7db03284768ecfe20c23f825709c674fd09 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Wed, 30 Mar 2016 18:55:06 -0700 Subject: [PATCH] config: set core.bigFileThreshold based on memory size The default value of core.bigFileThreshold (512 MiB) is unsuitable for low memory systems (e.g. a 512 MiB system). Instead calculate a value to use based on the actual memory size of the machine and allow it to be curtailed with a new $Girocco::Config::max_gc_big_file_threshold_size configuration option that is off by default. Signed-off-by: Kyle J. McKay --- Girocco/Config.pm | 5 +++++ Girocco/Util.pm | 18 +++++++++++++++++- shlib.sh | 24 +++++++++++++++--------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/Girocco/Config.pm b/Girocco/Config.pm index c2544c3..8aca0e7 100644 --- a/Girocco/Config.pm +++ b/Girocco/Config.pm @@ -210,6 +210,11 @@ our $min_gc_interval = 604800; # 1 week # May use a 'k', 'm', or 'g' suffix otherwise value is in bytes. our $max_gc_window_memory_size = undef; +# Maximum big file threshold size when repacking. If this is set, it will be +# used instead of the automatically computed value if it's less than that value. +# May use a 'k', 'm', or 'g' suffix otherwise value is in bytes. +our $max_gc_big_file_threshold_size = undef; + # Whether or not to run the ../bin/update-pwd-db script whenever the etc/passwd # database is changed. This is typically needed (i.e. set to a true value) for # FreeBSD style systems when using an sshd chroot jail for push access. So if diff --git a/Girocco/Util.pm b/Girocco/Util.pm index 5cf4173..c9d9fff 100644 --- a/Girocco/Util.pm +++ b/Girocco/Util.pm @@ -19,7 +19,7 @@ BEGIN { extract_url_hostname is_dns_hostname is_our_hostname get_cmd online_cpus sys_pagesize sys_memsize calc_windowmemory to_utf8 capture_command human_size - has_reserved_suffix noFatalsToBrowser); + calc_bigfilethreshold has_reserved_suffix noFatalsToBrowser); } my $encoder; @@ -837,6 +837,22 @@ sub calc_windowmemory { return _make_suffixed_size($max); } +# Return the value to set as core.bigFileThreshold for git repack +# If the system memory cannot be determined, returns "256m" +# Otherwise returns the available memory divided by 16 +# but never more than 512 megabytes or max_gc_big_file_threshold_size. +sub calc_bigfilethreshold { + my $memsize = sys_memsize; + my $max = 256 * 1024 * 1024; + if ($memsize) { + $max = int($memsize / 16); + $max = 512 * 1024 * 1024 if $max >= 512 * 1024 * 1024; + } + my $maxconf = _get_max_conf_suffixed_size($Girocco::Config::max_gc_big_file_threshold_size); + $max = $maxconf if defined($maxconf) && $maxconf && $max > $maxconf; + return _make_suffixed_size($max); +} + # $1 => thing to test # $2 => optional directory, if given and -e "$2/$1$3", then return false # $3 => optional, defaults to '' diff --git a/shlib.sh b/shlib.sh index 53b487e..aae7cec 100644 --- a/shlib.sh +++ b/shlib.sh @@ -69,6 +69,7 @@ get_girocco_config_var_list() ( # var_have_git_172 Set to 1 if git version >= 1.7.2 otherwise '' # var_have_git_173 Set to 1 if git version >= 1.7.3 otherwise '' # var_window_memory Value to use for repack --window-memory= + # var_big_file_threshold Value to use for core.bigFileThreshold # var_log_window_size Value to use for git-svn --log-window-size= # var_utf8_locale Value to use for a UTF-8 locale if available # var_xargs_r A "-r" if xargs needs it to behave correctly @@ -90,6 +91,9 @@ get_girocco_config_var_list() ( printf "var_window_memory=%s\n" \ "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \ -MGirocco::Util -e 'print calc_windowmemory')" + printf "var_big_file_threshold=%s\n" \ + "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \ + -MGirocco::Util -e 'print calc_bigfilethreshold')" printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}" # We parse the output of `locale -a` and select a suitable UTF-8 locale. _guess_locale="$(locale -a | grep -viE '^(posix|c)(\..*)?$' | \ @@ -125,6 +129,15 @@ else . "@basedir@/shlib_vars.sh" fi +# git_add_config "some.var=value" +# every ' in value must be replaced with the 4-character sequence '\'' before +# calling this function or Git will barf. Will not be effective unless running +# Git version 1.7.3 or later. +git_add_config() { + GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'" + export GIT_CONFIG_PARAMETERS +} + # Make sure we have a reproducible environment by using a controlled HOME dir XDG_CONFIG_HOME="$cfg_chroot/var/empty" HOME="$cfg_chroot/etc/girocco" @@ -149,6 +162,8 @@ if [ -n "$defined_cfg_git_client_ua" ]; then export GIT_HTTP_USER_AGENT fi unset GIT_CONFIG_PARAMETERS +[ -z "$var_big_file_threshold" ] || + git_add_config "core.bigFileThreshold=$var_big_file_threshold" # We cannot use a git() {} or nc_openbsd() {} function to redirect git # and nc_openbsd to the desired executables because when using @@ -181,15 +196,6 @@ git() ( exec "$cfg_git_bin" "$@" ) -# git_add_config "some.var=value" -# every ' in value must be replaced with the 4-character sequence '\'' before -# calling this function or Git will barf. Will not be effective unless running -# Git version 1.7.3 or later. -git_add_config() { - GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'" - export GIT_CONFIG_PARAMETERS -} - nc_openbsd() { "$cfg_nc_openbsd_bin" "$@"; } # Some platforms' broken xargs runs the command always at least once even if -- 2.11.4.GIT