scripts/git/git-push-all.sh

   1 #!/usr/bin/env bash
   2
   3 SCRIPT_NAME=$(basename "$0")
   4
   5 function usage()
   6 {
   7   if [ "$TOR_PUSH_SAME" ]; then
   8     CURRENT_PUSH_SAME="push"
   9   else
  10     CURRENT_PUSH_SAME="skip"
  11   fi
  12
  13   echo "$SCRIPT_NAME [-h] [-r <remote-name> [-t <test-branch-prefix>]] [-s]"
  14   # The next line looks misaligned, but it lines up in the output
  15   echo "                [-- [-n] [--no-atomic] <git push options>]"
  16   echo
  17   echo "  arguments:"
  18   echo "   -h: show this help text"
  19   echo "   -n: dry run mode"
  20   echo "       (default: run commands)"
  21   echo "   -r: push to remote-name, rather than the default upstream remote."
  22   echo "       (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)"
  23   echo "   -t: test branch mode: push test branches to remote-name. Pushes"
  24   echo "       branches prefix_035, prefix_040,  ... , prefix_main."
  25   echo "       (default: push maint-*, release-*, and main)"
  26   echo "   -s: push branches whose tips match upstream maint, release, or"
  27   echo "       main branches. The default is to skip these branches,"
  28   echo "       because they do not contain any new code. Use -s to test for"
  29   echo "       CI environment failures, using code that previously passed CI."
  30   echo "       (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
  31   echo "   --: pass further arguments to git push."
  32   echo "       All unrecognised arguments are passed to git push, but complex"
  33   echo "       arguments before -- may be mangled by getopt."
  34   echo "       (default: git push --atomic, current: $GIT_PUSH)"
  35   echo
  36   echo " env vars:"
  37   echo "   optional:"
  38   echo "   TOR_GIT_PUSH_PATH: change to this directory before pushing."
  39   echo "       (default: if \$TOR_FULL_GIT_PATH is set,"
  40   echo "       use \$TOR_FULL_GIT_PATH/\$TOR_MASTER;"
  41   echo "       Otherwise, use the current directory for pushes;"
  42   echo "       current: $TOR_GIT_PUSH_PATH)"
  43   echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
  44   echo "       We recommend using \$HOME/git/."
  45   echo "       (default: use the current directory for pushes;"
  46   echo "       current: $TOR_FULL_GIT_PATH)"
  47   echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
  48   echo "       The primary tor git directory is \$GIT_PATH/\$TOR_MASTER"
  49   echo "       (default: tor; current: $TOR_MASTER_NAME)"
  50   echo
  51   echo "   TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
  52   echo "       Overridden by -r."
  53   echo "       (default: upstream; current: $UPSTREAM_REMOTE)"
  54   echo "   TOR_GIT_PUSH: the git push command and default arguments."
  55   echo "       Overridden by <git push options> after --."
  56   echo "       (default: git push --atomic; current: $GIT_PUSH)"
  57   echo "   TOR_PUSH_SAME: push branches whose tips match upstream maint,"
  58   echo "       release, or main branches. Inverted by -s."
  59   echo "       (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
  60   echo "   TOR_PUSH_DELAY: pushes the main and maint branches separately,"
  61   echo "       so that CI runs in a sensible order."
  62   echo "       (default: push all branches immediately; current: $PUSH_DELAY)"
  63   echo "   we recommend that you set these env vars in your ~/.profile"
  64 }
  65
  66 set -e
  67
  68 #################
  69 # Configuration #
  70 #################
  71
  72 # Don't change this configuration - set the env vars in your .profile
  73 #
  74 # The primary tor git repository directory from which all the worktree have
  75 # been created.
  76 TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
  77 # Which directory do we push from?
  78 if [ "$TOR_FULL_GIT_PATH" ]; then
  79   TOR_GIT_PUSH_PATH=${TOR_GIT_PUSH_PATH:-"$TOR_FULL_GIT_PATH/$TOR_MASTER_NAME"}
  80 fi
  81 # git push command and default arguments
  82 GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"}
  83 # The upstream remote which gitlab.torproject.org/tpo/core/tor.git points to.
  84 DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
  85 # Push to a different upstream remote using -r <remote-name>
  86 UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE}
  87 # Add a delay between pushes, so CI runs on the most important branches first
  88 PUSH_DELAY=${TOR_PUSH_DELAY:-0}
  89 # Push (1) or skip (0) test branches that are the same as an upstream
  90 # maint/main branch. Push if you are testing that the CI environment still
  91 # works on old code, skip if you are testing new code in the branch.
  92 # Default: skip unchanged branches.
  93 # Inverted by the -s option.
  94 PUSH_SAME=${TOR_PUSH_SAME:-0}
  95
  96 #######################
  97 # Argument processing #
  98 #######################
  99
 100 # Controlled by the -t <test-branch-prefix> option. The test branch prefix
 101 # option makes git-merge-forward.sh create new test branches:
 102 # <tbp>_035, <tbp>_040, ... , <tbp>_main, and merge each branch forward into
 103 # the next one.
 104 TEST_BRANCH_PREFIX=
 105
 106 while getopts ":hr:st:" opt; do
 107   case "$opt" in
 108     h) usage
 109        exit 0
 110        ;;
 111     r) UPSTREAM_REMOTE="$OPTARG"
 112        echo "    *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***"
 113        shift
 114        shift
 115        OPTIND=$((OPTIND - 2))
 116        ;;
 117     s) PUSH_SAME=$((! PUSH_SAME))
 118        if [ "$PUSH_SAME" -eq 0 ]; then
 119          echo "    *** SKIPPING UNCHANGED TEST BRANCHES ***"
 120        else
 121          echo "    *** PUSHING UNCHANGED TEST BRANCHES ***"
 122        fi
 123        shift
 124        OPTIND=$((OPTIND - 1))
 125        ;;
 126     t) TEST_BRANCH_PREFIX="$OPTARG"
 127        echo "    *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***"
 128        shift
 129        shift
 130        OPTIND=$((OPTIND - 2))
 131        ;;
 132     *)
 133        # Make git push handle the option
 134        # This might mangle options with spaces, use -- for complex options
 135        GIT_PUSH="$GIT_PUSH $1"
 136        shift
 137        OPTIND=$((OPTIND - 1))
 138        ;;
 139   esac
 140 done
 141
 142 # getopts doesn't allow "-" as an option character,
 143 # so we have to handle -- manually
 144 if [ "$1" = "--" ]; then
 145   shift
 146 fi
 147
 148 if [ "$TEST_BRANCH_PREFIX" ]; then
 149   if [ "$UPSTREAM_REMOTE" = "$DEFAULT_UPSTREAM_REMOTE" ]; then
 150     echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \
 151       "the default remote $DEFAULT_UPSTREAM_REMOTE is not allowed."
 152     echo
 153     usage
 154     exit 1
 155   fi
 156 fi
 157
 158 if [ "$TOR_GIT_PUSH_PATH" ]; then
 159   echo "Changing to $TOR_GIT_PUSH_PATH before pushing"
 160   cd "$TOR_GIT_PUSH_PATH"
 161 else
 162   echo "Pushing from the current directory"
 163 fi
 164
 165 echo "Calling $GIT_PUSH" "$@" "<branches>"
 166
 167 ################################
 168 # Git upstream remote branches #
 169 ################################
 170
 171 set -e
 172 DEFAULT_UPSTREAM_BRANCHES=
 173 if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then
 174     for br in $(git-list-tor-branches.sh -l); do
 175         DEFAULT_UPSTREAM_BRANCHES="${DEFAULT_UPSTREAM_BRANCHES} ${DEFAULT_UPSTREAM_REMOTE}/${br}"
 176     done
 177 fi
 178
 179 UPSTREAM_BRANCHES=
 180 for br in $(git-list-tor-branches.sh -l); do
 181     UPSTREAM_BRANCHES="${UPSTREAM_BRANCHES} ${UPSTREAM_REMOTE}/${br}"
 182 done
 183
 184 ########################
 185 # Git branches to push #
 186 ########################
 187
 188 if [ -z "$TEST_BRANCH_PREFIX" ]; then
 189
 190   # maint/release push mode: push all branches.
 191   #
 192   # List of branches to push. Ordering is not important.
 193   PUSH_BRANCHES="$(git-list-tor-branches.sh -l)"
 194 else
 195
 196   # Test branch push mode: push test branches, based on each maint branch.
 197   #
 198   # List of branches to push. Ordering is not important.
 199   PUSH_BRANCHES=""
 200   for suffix in $(git-list-tor-branches.sh -s -R); do
 201       PUSH_BRANCHES="${PUSH_BRANCHES} ${TEST_BRANCH_PREFIX}${suffix}"
 202   done
 203 fi
 204
 205 set +e
 206
 207 ###############
 208 # Entry point #
 209 ###############
 210
 211 if [ "$TEST_BRANCH_PREFIX" ]; then
 212   # Skip the test branches that are the same as the default or current
 213   # upstream branches (they have already been tested)
 214   UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES"
 215 else
 216   # Skip the local maint-*, release-*, main branches that are the same as the
 217   # current upstream branches, but ignore the default upstream
 218   # (we want to update a non-default remote, even if it matches the default)
 219   UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES"
 220 fi
 221
 222 # Skip branches that match the relevant upstream(s)
 223 if [ "$PUSH_SAME" -eq 0 ]; then
 224   NEW_PUSH_BRANCHES=
 225   for b in $PUSH_BRANCHES; do
 226     PUSH_COMMIT=$(git rev-parse "$b")
 227     SKIP_UPSTREAM=
 228     for u in $UPSTREAM_SKIP_SAME_AS; do
 229       # Skip the branch check on error
 230       UPSTREAM_COMMIT=$(git rev-parse "$u" 2>/dev/null) || continue
 231       if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then
 232         SKIP_UPSTREAM="$u"
 233       fi
 234     done
 235     if [ "$SKIP_UPSTREAM" ]; then
 236       printf "Skipping unchanged: %s matching remote: %s\\n" \
 237         "$b" "$SKIP_UPSTREAM"
 238     else
 239       if [ "$NEW_PUSH_BRANCHES" ]; then
 240         NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}"
 241       else
 242         NEW_PUSH_BRANCHES="${b}"
 243       fi
 244     fi
 245   done
 246   PUSH_BRANCHES=${NEW_PUSH_BRANCHES}
 247 fi
 248
 249 if [ ! "$PUSH_BRANCHES" ]; then
 250   echo "No branches to push!"
 251   # We expect the rest of the script to run without errors, even if there
 252   # are no branches
 253 fi
 254
 255 if [ "$PUSH_DELAY" -le 0 ]; then
 256   echo "Pushing $PUSH_BRANCHES"
 257   # We know that there are no spaces in any branch within $PUSH_BRANCHES, so
 258   # it is safe to use it unquoted.  (This also applies to the other shellcheck
 259   # exceptions below.)
 260   #
 261   # Push all the branches at the same time
 262   # shellcheck disable=SC2086
 263   $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES
 264 else
 265   # Push the branches in optimal CI order, with a delay between each push
 266   PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V)
 267   MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep main$) \
 268       || true # Skipped main branch
 269   if [ -z "$TEST_BRANCH_PREFIX" ]; then
 270     MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \
 271         || true # Skipped all maint branches
 272     RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \
 273       tr "\\n" " ") || true # Skipped all release branches
 274   else
 275     # Actually test branches based on maint branches
 276     MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v main$) \
 277         || true # Skipped all maint test branches
 278     # No release branches
 279     RELEASE_BRANCHES=
 280   fi
 281   if [ "$MASTER_BRANCH" ] || [ "$MAINT_BRANCHES" ] \
 282       || [ "$RELEASE_BRANCHES" ]; then
 283     printf "Pushing with %ss delays, so CI runs in this order:\\n" \
 284            "$PUSH_DELAY"
 285     if [ "$MASTER_BRANCH" ]; then
 286       printf "%s\\n" "$MASTER_BRANCH"
 287     fi
 288     if [ "$MAINT_BRANCHES" ]; then
 289       printf "%s\\n" "$MAINT_BRANCHES"
 290     fi
 291     if [ "$RELEASE_BRANCHES" ]; then
 292       printf "%s\\n" "$RELEASE_BRANCHES"
 293     fi
 294   fi
 295   # shellcheck disable=SC2086
 296   for b in $MASTER_BRANCH $MAINT_BRANCHES; do
 297     $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b"
 298     # If we are pushing more than one branch, delay.  In the unlikely scenario
 299     # where we are pushing maint branches without the main branch, or maint
 300     # without release, there may be an extra delay
 301     if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then
 302       sleep "$PUSH_DELAY"
 303     fi
 304   done
 305   if [ "$RELEASE_BRANCHES" ]; then
 306     # shellcheck disable=SC2086
 307     $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES
 308   fi
 309 fi