taskd.pl: compute prior heads state and pass to Notify::ref_changes

[girocco.git] / shlib.sh

#!/bin/sh
#
# This is generic shell library for all the scripts used by Girocco;
# most importantly, it introduces all the $cfg_* shell variables.

# SHA-1 patterns
octet='[0-9a-f][0-9a-f]'
octet4="$octet$octet$octet$octet"
octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
octet20="$octet4$octet4$octet4$octet4$octet4"
nullsha="0000000000000000000000000000000000000000"
# tab
tab="$(printf '\t')"

# set a sane umask that never excludes any user or group permissions
umask $(printf '0%03o' $(( $(umask) & ~0770 )) )

vcmp() {
        # Compare $1 to $2 each of which must match \d+(\.\d+)*
        # An empty string ('') for $1 or $2 is treated like 0
        # Outputs:
        #  -1 if $1 < $2
        #   0 if $1 = $2
        #   1 if $1 > $2
        # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
        while
                _a="${1%%.*}"
                _b="${2%%.*}"
                [ -n "$_a" -o -n "$_b" ]
        do
                if [ "${_a:-0}" -lt "${_b:-0}" ]; then
                        echo -1
                        return
                elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
                        echo 1
                        return
                fi
                _a2="${1#$_a}"
                _b2="${2#$_b}"
                set -- "${_a2#.}" "${_b2#.}"
        done
        echo 0
}

get_girocco_config_pm_var_list() {
        # Export all the variables from Girocco::Config to suitable var= lines
        # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
        # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
        __girocco_conf="$GIROCCO_CONF"
        [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
        [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
        perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
                'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
                        my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
                        $val =~ s/([\\"\$\`])/\\$1/gos;
                        $val =~ s/(?:\r\n|\r|\n)$//os;
                        print "cfg_$_=\"$val\"";
                        print "defined_cfg_$_=",
                                (defined(${$Girocco::Config::{$_}})?"1":"");
                }'
}

get_girocco_config_var_list() (
        # Same as get_girocco_config_pm_var_list except that
        # the following variables (all starting with var_) are added:
        # var_group             cfg_owning_group if defined otherwise `id -gn`
        # var_git_ver           The version number part from `git version`
        # var_git_exec_path     The result of $cfg_git_bin --exec-dir
        # var_perl_bin          Full path to the perl interpreter to use
        # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
        # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
        # var_window_memory     Value to use for repack --window-memory=
        # var_big_file_threshold Value to use for core.bigFileThreshold
        # var_redelta_threshold Recompute deltas if no more than this many objs
        # var_log_window_size   Value to use for git-svn --log-window-size=
        # var_utf8_locale       Value to use for a UTF-8 locale if available
        # var_xargs_r           A "-r" if xargs needs it to behave correctly
        # var_du_exclude        Option to exclude PATTERN from du if available
        # var_du_follow         Option to follow command line sym links if available
        _cfg_vars="$(get_girocco_config_pm_var_list)"
        eval "$_cfg_vars"
        printf '%s\n' "$_cfg_vars"
        printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
        _gver="$("$cfg_git_bin" version 2>/dev/null | \
                LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
        printf 'var_git_ver=%s\n' "$_gver"
        printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
        printf 'var_perl_bin="%s"\n' "${cfg_perl_bin:-$(unset -f perl; command -v perl)}"
        printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
        printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
        __girocco_conf="$GIROCCO_CONF"
        [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
        [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
        printf "var_window_memory=%s\n" \
                "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
                -MGirocco::Util -e 'print calc_windowmemory')"
        printf "var_big_file_threshold=%s\n" \
                "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
                -MGirocco::Util -e 'print calc_bigfilethreshold')"
        printf "var_redelta_threshold=%s\n" \
                "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
                -MGirocco::Util -e 'print calc_redeltathreshold')"
        printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
        # We parse the output of `locale -a` and select a suitable UTF-8 locale.
        _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' | \
                LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' | \
                LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort | \
                head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
        [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
        # On some broken platforms running xargs without -r and empty input runs the command
        printf 'var_xargs_r=%s\n' "$(: | command xargs echo -r)"
        # The disk usage report produces better numbers if du has an exclude option
        _x0="$(basename "$0")"
        _x0="${_x0%?}?*"
        for _duopt in --exclude -I; do
                if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
                        printf 'var_du_exclude=%s\n' "$_duopt"
                        break
                fi
        done
        if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
                printf 'var_du_follow=%s\n' "-H"
                break
        fi
)

# If basedir has been replaced, and shlib_vars.sh exists, get the config
# definitions from it rather than running Perl.
if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
        # Import all the variables from Girocco::Config to the local environment,
        eval "$(get_girocco_config_var_list)"
else
        # Import the variables from shlib_vars.sh which avoids needlessly
        # running another copy of Perl
        . "@basedir@/shlib_vars.sh"
fi

# git_add_config "some.var=value"
# every ' in value must be replaced with the 4-character sequence '\'' before
# calling this function or Git will barf.  Will not be effective unless running
# Git version 1.7.3 or later.
git_add_config() {
        GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
        export GIT_CONFIG_PARAMETERS
}

# Make sure we have a reproducible environment by using a controlled HOME dir
XDG_CONFIG_HOME="$cfg_chroot/var/empty"
HOME="$cfg_chroot/etc/girocco"
GIT_CONFIG_NOSYSTEM=1
GIT_ATTR_NOSYSTEM=1
GIT_NO_REPLACE_OBJECTS=1
GIT_TERMINAL_PROMPT=0
GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
export XDG_CONFIG_HOME
export HOME
export GIT_CONFIG_NOSYSTEM
export GIT_ATTR_NOSYSTEM
export GIT_NO_REPLACE_OBJECTS
export GIT_TERMINAL_PROMPT
export GIT_ASKPASS
unset GIT_USER_AGENT
unset GIT_HTTP_USER_AGENT
if [ -n "$defined_cfg_git_client_ua" ]; then
        GIT_USER_AGENT="$cfg_git_client_ua"
        export GIT_USER_AGENT
        GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
        export GIT_HTTP_USER_AGENT
fi
unset GIT_CONFIG_PARAMETERS
[ -z "$var_big_file_threshold" ] ||
        git_add_config "core.bigFileThreshold=$var_big_file_threshold"

# We cannot use a git() {} or nc_openbsd() {} function to redirect git
# and nc_openbsd to the desired executables because when using
# "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
# different and unexpected ways:
#   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
#   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
#   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
# None of these are good.  We want a temporary "export ENV_VAR=xxx"
# setting only while running func which none of the /bin/sh's do.
#
# Instead we'd like to use an alias that provides the desired behavior without
# any of the bad (a), (b) or (c) effects.
#
# However, unfortunately, some of the crazy /bin/sh implementations do not
# recognize alias expansions when preceded by variable assignments!
#
# So we are left with git() {} and nc_openbsd() {} functions and in the
# case of git() {} we can compensate for (b) and (c) failing to export
# but not (a) and (b) persisting the values so the caller will simply
# have to beware and explicitly unset any variables that should not persist
# beyond the function call itself.

git() (
        [ "${GIT_DIR+set}" = "set" ] && export GIT_DIR
        [ "${GIT_SSL_NO_VERIFY+set}" = "set" ] && export GIT_SSL_NO_VERIFY
        [ "${GIT_TRACE_PACKET+set}" = "set" ] && export GIT_TRACE_PACKET
        [ "${GIT_USER_AGENT+set}" = "set" ] && export GIT_USER_AGENT
        [ "${GIT_HTTP_USER_AGENT+set}" = "set" ] && export GIT_HTTP_USER_AGENT
        exec "$cfg_git_bin" "$@"
)

# see comments for git() -- callers must explicitly export all variables
# intended for the perl interpreter before using this function
perl() { command "${var_perl_bin:-perl}" "$@"; }

nc_openbsd() { "$cfg_nc_openbsd_bin" "$@"; }

list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }

strftime() { command "$cfg_basedir/bin/strftime" "$@"; }

# Some platforms' broken xargs runs the command always at least once even if
# there's no input unless given a special option.  Automatically supply the
# option on those platforms by providing an xargs function.
xargs() { command xargs $var_xargs_r "$@"; }

_addrlist() {
        _list=
        for _addr in "$@"; do
                [ -z "$_list" ] || _list="$_list, "
                _list="$_list$_addr"
        done
        echo "$_list"
}

_sendmail() {
        _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
        if [ -n "$cfg_sender" ]; then
                "$_mailer" -i -f "$cfg_sender" "$@"
        else
                "$_mailer" -i "$@"
        fi
}

mail() {
        _subject=
        if [ "$1" = "-s" ]; then
                shift
                _subject="$1"
                shift
        fi
        {
                echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
                echo "To: $(_addrlist "$@")"
                [ -z "$_subject" ] || echo "Subject: $_subject"
                echo "MIME-Version: 1.0"
                echo "Content-Type: text/plain; charset=utf-8"
                echo "Content-Transfer-Encoding: 8bit"
                [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
                echo "Auto-Submitted: auto-generated"
                echo ""
                cat
        } | _sendmail "$@"
}

# bang CMD... will execute the command with well-defined failure mode;
# set bang_action to string of the failed action ('clone', 'update', ...);
# pre-set bang_once=1 to make sure jobs banging on a repo repeatedly will
# not spam the owner; re-define the bang_trap() function to do custom
# cleanup before bailing out
bang() {
        if [ -n "$show_progress" ]; then
                exec 3>&1
                errcode=
                read -r errcode <<-EOT || :
                $(
                        exec 4>&3 3>&1 1>&4 4>&-
                        { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -a "$bang_log"
                )
                EOT
                exec 3>&-
                if [ -z "$errcode" ]; then
                        # All right. Cool.
                        return;
                fi
        else
                if "$@" >>"$bang_log" 2>&1; then
                        # All right. Cool.
                        return;
                else
                        errcode="$?"
                fi
        fi
        if ! [ -e .banged ] || [ -e .bangagain ]; then
                rm -f .bangagain
                bangmailok=true
                ! [ -f HEAD -a -f config -a -d objects ] ||
                bangmailok="$(GIT_DIR=. git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
                bangaddrs=''
                [ "$bangmailok" = "false" -o -z "$mail" ] || bangaddrs="$mail"
                [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] ||
                if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
                [ -z "$bangaddrs" ] ||
                {
                        echo "$* failed with error code $errcode"
                        echo ""
                        [ ! -n "$bang_once" ] || echo "you will not receive any more notifications until recovery"
                        echo "this status message may be disabled on the project admin page"
                        echo ""
                        echo "Log follows:"
                        echo ""
                        cat "$bang_log"
                } | mail -s "[$cfg_name] $proj $bang_action failed" "$bangaddrs"
        fi
        touch .banged
        cat "$bang_log" > .banglog
        bang_trap
        exit 1
}

# bang_eval CMD... will evaluate the command with well-defined failure mode;
# Identical to bang CMD... except the command is eval'd instead of executed.
bang_eval() {
        bang eval "$*"
}

# Default bang settings:
bang_setup() {
        bang_action="lame_programmer"
        bang_once=
        bang_trap() { :; }
        bang_log="$(mktemp -t repomgr-XXXXXX)"
        trap 'rm -f "$bang_log"' EXIT
        trap 'exit 130' INT
        trap 'exit 143' TERM
}

# Remove banged status
bang_reset() {
        rm -f .banged .bangagain .banglog
}

# Check to see if banged status
is_banged() {
        [ -e .banged ]
}


# Progress report - if show_progress is set, shows the given message.
progress() {
        [ ! -n "$show_progress" ] || echo "$@"
}


# Project config accessors; must be run in project directory
config_get() {
        git config "gitweb.$1"
}

config_set() {
        git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
}

config_set_raw() {
        git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
}

config_get_date_seconds() {
        _dt="$(config_get "$1" || :)"
        [ -n "$_dt" ] || return 1
        _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_rfc2822_date('$_dt')")"
        [ -n "$_ds" ] || return 1
        echo "$_ds"
}

# Tool for checking whether given number of seconds has not passed yet
check_interval() {
        os="$(config_get_date_seconds "$1")" || return 1
        ns="$(date +%s)"
        [ $ns -lt $(($os+$2)) ]
}

# Check if we are running with effective root permissions
is_root() {
        [ "$(id -u 2>/dev/null)" = "0" ]
}

# Check to see if the single argument is a Git directory
is_git_dir() {
        # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
        # And we are slightly more picky (must be refs/.+ not refs/.*)
        [ -d "$1/objects" -a -x "$1/objects" ] || return 1
        [ -d "$1/refs" -a -x "$1/refs" ] || return 1
        if [ -L "$1/HEAD" ]; then
                _hr="$(readlink "$1/HEAD")"
                case "$_hr" in "refs/"?*) :;; *) return 1;; esac
        fi
        [ -f "$1/HEAD" -a -r "$1/HEAD" ] || return 1
        read -r _hr <"$1/HEAD" || return 1
        case "$_hr" in
                $octet20)
                        return 0;;
                ref:*)
                        _hr="${_hr##ref:*[ $tab]}"
                        case "$_hr" in "refs/"?*) return 0;; esac
        esac
        return 1
}

# List all Git repositories, with given prefix if specified, one-per-line
# All project names starting with _ are always excluded from the result
get_repo_list() {
        if [ -n "$1" ]; then
                LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
        else
                LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
        fi | while IFS=: read name id; do
                [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
        done
}

# Return success if the given project name has any forks
has_forks() {
        _prj="${1%.git}"
        [ -n "$_prj" ] || return 1
        [ -d "$cfg_reporoot/$_prj" ] || return 1
        is_git_dir "$cfg_reporoot/$_prj.git" || return 1
        test $(get_repo_list "$_prj/[^/][^/]*:" | LC_ALL=C wc -l) -gt 0
}

# returns empty string and error for empty string otherwise one of
#   m  =>  normal Git mirror
#   s  =>  mirror from svn source
#   d  =>  mirror from darcs source
#   b  =>  mirror from bzr source
#   h  =>  mirror from hg source
# note that if the string is non-empty and none of s, d, b or h match the
# return will always be type m regardless of whether it's a valid Git URL
get_url_mirror_type() {
        case "$1" in
                "")
                        return 1
                        ;;
                svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
                        echo 's'
                        ;;
                darcs://*)
                        echo 'd'
                        ;;
                bzr://*)
                        echo 'b'
                        ;;
                hg+http://* | hg+https://*)
                        echo 'h'
                        ;;
                *)
                        echo 'm'
                        ;;
        esac
        return 0
}

# returns get_url_mirror_type for gitweb.base url of git directory
# (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
# will fail if the directory does not have .nofetch and gitweb.baseurl
# comes back empty -- otherwise .nofetch directories succed with a "" return
# automatically strips any leading "disabled " prefix before testing
get_mirror_type() {
        _gitdir="${1:-.}"
        # always return empty for non-mirrors
        [ ! -e "$_gitdir/.nofetch" ] || return 0
        _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null || :)"
        _url="${_url##* }"
        [ -n "$_url" ] || return 1
        get_url_mirror_type "$_url"
}

# returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
is_gfi_mirror() {
        case "$(get_mirror_type "$1" 2>/dev/null || :)" in
                d|b|h)
                        # darcs, bzr and hg mirrors use git fast-import
                        return 0
                        ;;
                *)
                        # Don't think git-svn currently uses git fast-import
                        # And Git mirrors certainly do not
                        return 1
                        ;;
        esac
        # assume it does not use git fast-import
        return 1
}

# returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
is_svn_mirror() {
        [ "$(get_mirror_type "$1" 2>/dev/null || :)" = "s" ]
}

# A well-known UTF-8 locale is required for some of the fast-import providers
# in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
# but that is not reliably UTF-8 but rather usually US-ASCII.
# We parse the output of `locale -a` and select a suitable UTF-8 locale at
# install time and store that in $var_utf8_locale if one is found.
# If we cannot find one in the `locale -a` output then we just use a well-known
# UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
# it.  We only set this temporarily when running the fast-import providers.
set_utf8_locale() {
        LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
        export LC_ALL
}

# hg-fast-export | git fast-import with error handling in current directory GIT_DIR
git_hg_fetch() (
        set_utf8_locale
        _python="${PYTHON:-python}"
        _err1=
        _err2=
        exec 3>&1
        { read -r _err1 || :; read -r _err2 || :; } <<-EOT
        $(
                exec 4>&3 3>&1 1>&4 4>&-
                {
                        _e1=0
                        [ -f hg2git-marks ] || touch hg2git-marks
                        _af="$(git config hg.authorsfile || :)"
                        _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
                                --repo    "$(pwd)/repo.hg" \
                                --marks   "$(pwd)/hg2git-marks" \
                                --mapping "$(pwd)/hg2git-mapping" \
                                --heads   "$(pwd)/hg2git-heads" \
                                --status  "$(pwd)/hg2git-state" \
                                -U unknown --force --flatten --hg-hash'
                        [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
                        eval "$_cmd" 3>&- || _e1=$?
                        echo $_e1 >&3
                } | \
                {
                        _e2=0
                        rm -f hg2git-marks.new
                        git fast-import \
                                --export-marks="$(pwd)/hg2git-marks.new" \
                                --export-pack-edges="$(pwd)/gfi-packs" \
                                --force 3>&- || _e2=$?
                        echo $_e2 >&3
                }
        )
        EOT
        exec 3>&-
        [ "$_err1" = 0 -a "$_err2" = 0 ] || return 1
        if [ -f hg2git-marks ]; then
                rm -f hg2git-marks.old
                mv hg2git-marks hg2git-marks.old
        else
                touch hg2git-marks.old
        fi
        cat hg2git-marks.old hg2git-marks.new | \
                LC_ALL=C sort -t : -k2,2n -u | \
                LC_ALL=C sed -ne "/^:[1-9][0-9]* $octet20\$/p" > hg2git-marks
        rm hg2git-marks.old hg2git-marks.new
        rm -f hg2git-heads
        git branch --no-color | \
        while IFS= read -r _head; do
                echo ":${_head#??} $(git rev-parse "refs/heads/${_head#??}")"
        done > hg2git-heads
)