From d739f0b17f1e857b3493362e6c286adeb3c2a459 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Wed, 21 Feb 2018 18:35:21 -0800 Subject: [PATCH] fetch: hide the stash Hide the "stash" ref(s) from remote fetch clients when possible. Git will not allow single-level refs (e.g. "refs/stash") to be pushed in the first place. The only way one could end up in a Girocco repository is when linked working trees (or, gasp, non-bare Girocco repositories) are in use or a non-clean mirror fetch retrieves such a ref. But for the "git log --walk-reflogs" facility to actually work as designed, a copy of the ref log file for the ref in question must be available. There's no mechanism available for a remote to fetch a ref log. Even though Girocco will now attempt to retain anything reachable solely via a ref log entry, there's just no mechanism for remotes to use any of the server's ref logs. Therefore, attempt to hide known "stash" refs by default to avoid confusing remote clients. They would only ever be fetched by a full "mirror" clone anyway and would otherwise bloat the primary pack and virtual bundle unnecessarily. Unless the new "obscure" configuration item has been set to a true value, omit any existing "stash" refs from the primary pack (and therefore the virtual bundle too). They WILL be included in the secondary pack so nothing will get lost for local linked working tree users. Always do this regardless of Git version. In addition tell Git to "hide" those refs from remote fetchers when possible. Since Git 1.8.2 it's possible to "hide" them from remote fetchers over the "git:" and "ssh:" protocols. However, due to an unfortunate Git bug that was not fixed until Git 2.3.5 it's unsafe to attempt to hide them from smart "http:/https:" fetchers before Git version 2.3.5 as it can result in fetches actually failing. It's only the server's Git version that matters for "hide" support. These refs cannot be hidden from non-smart HTTP fetchers or any other non-smart mechanism that may be provided (i.e. rsync/ftp/etc.) as those mechanisms use the $gitdir/info/refs file which does NOT exclude them. This "hiding" should not be considered secure (see the "SECURITY" section of `git help namespaces` for details), but rather an efficiency and "cleanliness" change. A determined pursuer is bound to find a way to your stash even with this change, but everyone else will appreciate the cleanup. Signed-off-by: Kyle J. McKay --- Girocco/Config.pm | 35 +++++++++++++++++++++++++++++++++++ bin/git-daemon-verify | 5 +++++ bin/git-http-backend-verify | 5 +++++ bin/git-shell-verify | 6 ++++++ install.sh | 1 + jobd/gc.sh | 30 ++++++++++++++++++++++++++++++ shlib.sh | 2 ++ 7 files changed, 84 insertions(+) diff --git a/Girocco/Config.pm b/Girocco/Config.pm index 5c5f701..9c8d1bf 100644 --- a/Girocco/Config.pm +++ b/Girocco/Config.pm @@ -864,6 +864,40 @@ our $reflogs_lifetime = 1; # RECOMMENDED VALUE: undef or 5 our $upload_pack_window = undef; +# If this is true then remote fetching of refs/stash and refs/tgstash will +# be allowed. Git does not allow single-level ref names to be pushed so the +# only way they could get in there is if a linked working tree (or, gasp, a +# non-bare Girocco repository) created them or they arrived via a non-clean +# mirror fetch. The client almost certainly does not want to see them. +# Unless this config item is true they will also be left out of the bundle too. +# Since both stash and tgstash are used with their ref logs and there's no way +# for a remote to fetch ref logs, the "log --walk-reflogs" feature could not be +# used with them by a remote that fetched them anyway. +# +# NOTE: The reason this doesn't just control all single-level refs is that the +# "hideRefs" configuration mechanism isn't flexible enough to hide all +# single-level refs without knowing their names. In addition, it hides the +# entire refs hierarchy so refs/stash/foo will also be hidden along with +# refs/stash, but Git doesn't actually support ref names that introduce a +# directory/file confict (aka D/F conflict) and "refs/stash" represents an +# official Git ref name therefore any refs/stash/... names really aren't +# allowed in the first place so it's no problem if they're incidentally hidden +# along with refs/stash itself. +# +# NOTE: Git 1.8.2 or later is required to actually hide the refs from fetchers +# over the "git:" protocol and Git 2.3.5 or later is required to properly hide +# them over the smart "http:" protocol (Girocco will not attempt to "hide" them +# on a smart HTTP fetch if Git is < 2.3.5 to avoid Git bugs.) They will never +# be hidden via the non-smart HTTP fetch or any other non-smart protocols that +# also make use of the $gitdir/info/refs file as they are not excluded from it. +# Nor will they be hidden when accessed via any non-Girocco mechanism. +# They will, however, always be excluded from the primary (aka .bitmap) pack +# and bundle no matter what version of Git is used unless this is set to a +# true value. It's only the server's Git version that matters when hiding refs. +# +# RECOMMENDED VALUE: undef or 0 +our $fetch_stash_refs = undef; + # When set to a true value, Girocco will attempt to pick up ref changes made # outside of Girocco itself and process them using the usual Girocco # notification mechanism. Git lacks any "post-ref-change" hook capability that @@ -1011,6 +1045,7 @@ $jailreporoot =~ s,^/+,,; ($jailreporoot) or die "Girocco::Config \$jailreporoot must be set"; $disable_jailsetup = $disable_jailsetup ? 1 : ''; $notify_single_level = $notify_single_level ? 1 : ''; +$fetch_stash_refs = $fetch_stash_refs ? 1 : ''; (not $mob or $mob eq 'mob') or die "Girocco::Config \$mob must be undef (or '') or 'mob'"; (not $min_key_length or $min_key_length =~ /^[1-9][0-9]*$/) or die "Girocco::Config \$min_key_length must be undef or numeric"; diff --git a/bin/git-daemon-verify b/bin/git-daemon-verify index bfeb41c..1c9c660 100755 --- a/bin/git-daemon-verify +++ b/bin/git-daemon-verify @@ -141,6 +141,11 @@ fi [ -z "$var_upload_window" ] || [ "$type" != "upload-pack" ] || git_add_config "pack.window=$var_upload_window" +if [ "${cfg_fetch_stash_refs:-0}" = "0" ]; then + git_add_config "uploadpack.hiderefs=refs/stash" + git_add_config "uploadpack.hiderefs=refs/tgstash" +fi + exec "$cfg_git_daemon_bin" --inetd --verbose --export-all --enable=upload-archive --base-path="$cfg_reporoot" internalerr "exec failed: $cfg_git_daemon_bin" exit 1 diff --git a/bin/git-http-backend-verify b/bin/git-http-backend-verify index 61ec3cf..50c8943 100755 --- a/bin/git-http-backend-verify +++ b/bin/git-http-backend-verify @@ -342,6 +342,11 @@ if [ -n "$bundle" ]; then exit 1 fi +if [ "${cfg_fetch_stash_refs:-0}" = "0" ] && [ -n "$var_have_git_235" ]; then + git_add_config "uploadpack.hiderefs=refs/stash" + git_add_config "uploadpack.hiderefs=refs/tgstash" +fi + if [ -z "$needsauthcheck" ] || [ -z "$smart" ]; then [ -z "$var_upload_window" ] || [ -z "$smart" ] || git_add_config "pack.window=$var_upload_window" diff --git a/bin/git-shell-verify b/bin/git-shell-verify index d1817fc..5ff46f5 100755 --- a/bin/git-shell-verify +++ b/bin/git-shell-verify @@ -46,6 +46,7 @@ defined_ua=@defined_git_server_ua@ cfg_git_no_mmap=@git_no_mmap@ var_big_file_threshold=@big_file_threshold@ var_upload_window=@upload_pack_window@ +cfg_fetch_stash_refs=@fetch_stash_refs@ export XDG_CONFIG_HOME export HOME @@ -80,6 +81,11 @@ fi git_add_config "core.bigFileThreshold=$var_big_file_threshold" git_add_config "gc.auto=0" +if [ "${cfg_fetch_stash_refs:-0}" = "0" ]; then + git_add_config "uploadpack.hiderefs=refs/stash" + git_add_config "uploadpack.hiderefs=refs/tgstash" +fi + # When accessing Git via ssh, there should never be a terminal on 0, 1 or 2 # The "no-pty" flag should be set in all the ssh keys files to prevent this # However, just in case, check for it here and die as a safety fallback diff --git a/install.sh b/install.sh index 97ac7d7..79af6ca 100755 --- a/install.sh +++ b/install.sh @@ -563,6 +563,7 @@ perl -I. -M$GIROCCO_CONF -i -p \ -e 's/\@git_no_mmap\@/"$Girocco::Config::git_no_mmap"/g;' \ -e 's/\@big_file_threshold\@/"'"$var_big_file_threshold"'"/g;' \ -e 's/\@upload_pack_window\@/"'"$var_upload_window"'"/g;' \ + -e 's/\@fetch_stash_refs\@/"$Girocco::Config::fetch_stash_refs"/g;' \ -e 'close ARGV if eof;' \ "$basedir"/jobs/*.sh "$basedir"/jobd/*.sh \ "$basedir"/taskd/*.sh "$basedir"/gitweb/*.sh \ diff --git a/jobd/gc.sh b/jobd/gc.sh index ae049d8..862a14d 100755 --- a/jobd/gc.sh +++ b/jobd/gc.sh @@ -497,11 +497,35 @@ make_repack_dir() { ln -s ../../refs repack/refs/refs _lines=$(( $(LC_ALL=C wc -l repack/HEAD.orig + >repack/packed-refs.extra + _xtralines=0 cat packed-refs >repack/packed-refs.orig if [ $(LC_ALL=C wc -l &2 "[$proj] error: make_repack_dir failed original packed-refs line count sanity check" exit 1 fi + if [ "${cfg_fetch_stash_refs:-0}" = "0" ]; then + # migrate any refs/stash or refs/tgstash lines to repack/packed-refs.extra + >xtra; next; } + /^[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]+ refs\/(stash|tgstash)(\/|$)/ { + peeling = 1 + print >>xtra + next + } + { peeling = 0; print; } + ' >repack/packed-refs.new + _xtralines="$(( $(LC_ALL=C wc -l &2 "[$proj] error: make_repack_dir failed packed-refs.extra line count sanity check" + exit 1 + fi + mv -f repack/packed-refs.new repack/packed-refs.orig + _lines="$_newlines" + fi # Note: Git v1.5.0 introduced the "# pack-refs with:" header line for the packed-refs file sed '/^# pack-refs/d; s, refs/, refs/!/,' repack/packed-refs nohead= @@ -1122,6 +1146,7 @@ get_detached_head() { # other refs we might like to keep. # the current directory MUST be set to the repository's --git-dir # the following are included: +# * refs mentioned in repack/packed-refs.extra (if it exists) # * refs mentioned in reflogs/... files # * tree(s) created from index file(s) # * detached linked working tree heads @@ -1129,6 +1154,11 @@ get_detached_head() { # one per line under a refs/z* namespace compute_extra_reachables() { { + if [ -s repack/packed-refs.extra ]; then + LC_ALL=C sed = 1.7.10 otherwise '' # var_have_git_185 Set to 1 if git version >= 1.8.5 otherwise '' # var_have_git_210 Set to 1 if git version >= 2.1.0 otherwise '' + # var_have_git_235 Set to 1 if git version >= 2.3.5 otherwise '' # var_have_git_260 Set to 1 if git version >= 2.6.0 otherwise '' # var_have_git_2101 Set to 1 if git version >= 2.10.1 otherwise '' # var_window_memory Value to use for repack --window-memory= @@ -116,6 +117,7 @@ get_girocco_config_var_list() ( printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)" printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)" printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)" + printf 'var_have_git_235=%s\n' "$([ $(vcmp "$_gver" 2.3.5) -ge 0 ] && echo 1)" printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)" printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)" __girocco_conf="$GIROCCO_CONF" -- 2.11.4.GIT