scripts/git/git-setup-dirs.sh

   1 #!/usr/bin/env bash
   2
   3 SCRIPT_NAME=$(basename "$0")
   4
   5 function usage()
   6 {
   7   echo "$SCRIPT_NAME [-h] [-n] [-u]"
   8   echo
   9   echo "  arguments:"
  10   echo "   -h: show this help text"
  11   echo "   -n: dry run mode"
  12   echo "       (default: run commands)"
  13   echo "   -u: if a directory or worktree already exists, use it"
  14   echo "       (default: fail and exit on existing directories)"
  15   echo
  16   echo " env vars:"
  17   echo "   required:"
  18   echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
  19   echo "       You must set this env var, we recommend \$HOME/git/"
  20   echo "       (default: fail if this env var is not set;"
  21   echo "       current: $GIT_PATH)"
  22   echo
  23   echo "   optional:"
  24   echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
  25   echo "       The primary tor git directory is \$GIT_PATH/\$TOR_MASTER"
  26   echo "       (default: tor; current: $TOR_MASTER_NAME)"
  27   echo "   TOR_WKT_NAME: the name of the directory containing the tor"
  28   echo "       worktrees. The tor worktrees are:"
  29   echo "       \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
  30   echo "       (default: tor-wkt; current: $TOR_WKT_NAME)"
  31   echo "   TOR_GIT_ORIGIN_PULL: the origin remote pull URL."
  32   echo "       (current: $GIT_ORIGIN_PULL)"
  33   echo "   TOR_GIT_ORIGIN_PUSH: the origin remote push URL"
  34   echo "       (current: $GIT_ORIGIN_PUSH)"
  35   echo "   TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
  36   echo "       If \$TOR_UPSTREAM_REMOTE_NAME is not 'origin', we have a"
  37   echo "       separate upstream remote, and we don't push to origin."
  38   echo "       (default: $DEFAULT_UPSTREAM_REMOTE)"
  39   echo "   TOR_GITHUB_PULL: the tor-github remote pull URL"
  40   echo "       (current: $GITHUB_PULL)"
  41   echo "   TOR_GITHUB_PUSH: the tor-github remote push URL"
  42   echo "       (current: $GITHUB_PUSH)"
  43   echo "   TOR_GITLAB_PULL: the tor-gitlab remote pull URL"
  44   echo "       (current: $GITLAB_PULL)"
  45   echo "   TOR_GITLAB_PUSH: the tor-gitlab remote push URL"
  46   echo "       (current: $GITLAB_PUSH)"
  47   echo "   TOR_EXTRA_CLONE_ARGS: extra arguments to git clone"
  48   echo "       (current: $TOR_EXTRA_CLONE_ARGS)"
  49   echo "   TOR_EXTRA_REMOTE_NAME: the name of an extra remote"
  50   echo "       This remote is not pulled by this script or git-pull-all.sh."
  51   echo "       This remote is not pushed by git-push-all.sh."
  52   echo "       (current: $TOR_EXTRA_REMOTE_NAME)"
  53   echo "   TOR_EXTRA_REMOTE_PULL: the extra remote pull URL."
  54   echo "       (current: $TOR_EXTRA_REMOTE_PULL)"
  55   echo "   TOR_EXTRA_REMOTE_PUSH: the extra remote push URL"
  56   echo "       (current: $TOR_EXTRA_REMOTE_PUSH)"
  57   echo "   we recommend that you set these env vars in your ~/.profile"
  58 }
  59
  60 #################
  61 # Configuration #
  62 #################
  63
  64 # Don't change this configuration - set the env vars in your .profile
  65
  66 # Where are all those git repositories?
  67 GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
  68 # The primary tor git repository directory from which all the worktree have
  69 # been created.
  70 TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
  71 # The worktrees location (directory).
  72 TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
  73
  74 # Origin repositories
  75 GIT_ORIGIN_PULL=${TOR_GIT_ORIGIN_PULL:-"https://git.torproject.org/tor.git"}
  76 GIT_ORIGIN_PUSH=${TOR_GIT_ORIGIN_PUSH:-"git@git-rw.torproject.org:tor.git"}
  77 # The upstream remote which git.torproject.org/tor.git points to.
  78 DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
  79 # Copy the URLs from origin
  80 GIT_UPSTREAM_PULL="$GIT_ORIGIN_PULL"
  81 GIT_UPSTREAM_PUSH="$GIT_ORIGIN_PUSH"
  82 # And avoid pushing to origin if we have an upstream
  83 if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
  84   GIT_ORIGIN_PUSH="No pushes to origin, if there is an upstream"
  85 fi
  86 # GitHub repositories
  87 GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"}
  88 GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"}
  89
  90 # GitLab repositories
  91 GITLAB_PULL=${TOR_GITLAB_PULL:-"https://gitlab.torproject.org/tpo/core/tor.git"}
  92 GITLAB_PUSH=${TOR_GITLAB_PUSH:-"No_Pushing_To_GitLab"}
  93
  94 ##########################
  95 # Git branches to manage #
  96 ##########################
  97
  98 # The branches and worktrees need to be modified when there is a new branch,
  99 # and when an old branch is no longer supported.
 100
 101 set -e
 102 eval "$(git-list-tor-branches.sh -b)"
 103 set +e
 104
 105 # The main branch path has to be the main repository thus contains the
 106 # origin that will be used to fetch the updates. All the worktrees are created
 107 # from that repository.
 108 ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
 109
 110 #######################
 111 # Argument processing #
 112 #######################
 113
 114 # Controlled by the -n option. The dry run option will just output the command
 115 # that would have been executed for each worktree.
 116 DRY_RUN=0
 117
 118 # Controlled by the -s option. The use existing option checks for existing
 119 # directories, and re-uses them, rather than creating a new directory.
 120 USE_EXISTING=0
 121 USE_EXISTING_HINT="Use existing: '$SCRIPT_NAME -u'."
 122
 123 while getopts "hnu" opt; do
 124   case "$opt" in
 125     h) usage
 126        exit 0
 127        ;;
 128     n) DRY_RUN=1
 129        echo "    *** DRY RUN MODE ***"
 130        ;;
 131     u) USE_EXISTING=1
 132        echo "    *** USE EXISTING DIRECTORIES MODE ***"
 133        ;;
 134     *)
 135        echo
 136        usage
 137        exit 1
 138        ;;
 139   esac
 140 done
 141
 142 ###########################
 143 # Git worktrees to manage #
 144 ###########################
 145
 146 COUNT=${#WORKTREE[@]}
 147
 148 #############
 149 # Constants #
 150 #############
 151
 152 # Control characters
 153 CNRM=$'\x1b[0;0m'   # Clear color
 154
 155 # Bright color
 156 BGRN=$'\x1b[1;32m'
 157 BBLU=$'\x1b[1;34m'
 158 BRED=$'\x1b[1;31m'
 159 BYEL=$'\x1b[1;33m'
 160 IWTH=$'\x1b[3;37m'
 161
 162 # Strings for the pretty print.
 163 MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
 164 SUCCESS="${BGRN}success${CNRM}"
 165 SKIPPED="${BYEL}skipped${CNRM}"
 166 FAILED="${BRED}failed${CNRM}"
 167
 168 ####################
 169 # Helper functions #
 170 ####################
 171
 172 # Validate the given returned value (error code), print success or failed. The
 173 # second argument is the error output in case of failure, it is printed out.
 174 # On failure, this function exits.
 175 function validate_ret
 176 {
 177   if [ "$1" -eq 0 ]; then
 178     printf "%s\\n" "$SUCCESS"
 179   else
 180     printf "%s\\n" "$FAILED"
 181     printf "      %s\\n" "$2"
 182     exit 1
 183   fi
 184 }
 185
 186 # Validate the given returned value (error code), print success, skipped, or
 187 # failed. If $USE_EXISTING is 0, fail on error, otherwise, skip on error.
 188 # The second argument is the error output in case of failure, it is printed
 189 # out. On failure, this function exits.
 190 function validate_ret_skip
 191 {
 192   if [ "$1" -ne 0 ]; then
 193     if [ "$USE_EXISTING" -eq "0" ]; then
 194       # Fail and exit with error
 195       validate_ret "$1" "$2 $USE_EXISTING_HINT"
 196     else
 197       printf "%s\\n" "$SKIPPED"
 198       printf "      %s\\n" "${IWTH}$2${CNRM}"
 199       # Tell the caller to skip the rest of the function
 200       return 0
 201     fi
 202   fi
 203   # Tell the caller to continue
 204   return 1
 205 }
 206
 207 # Create a directory, and any missing enclosing directories.
 208 # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
 209 function make_directory
 210 {
 211   local cmd="mkdir -p '$1'"
 212   printf "  %s Creating directory %s..." "$MARKER" "$1"
 213   local check_cmd="[ ! -d '$1' ]"
 214   msg=$( eval "$check_cmd" 2>&1 )
 215   if validate_ret_skip $? "Directory already exists."; then
 216     return
 217   fi
 218   if [ $DRY_RUN -eq 0 ]; then
 219     msg=$( eval "$cmd" 2>&1 )
 220     validate_ret $? "$msg"
 221   else
 222     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 223   fi
 224 }
 225
 226 # Create a symlink from the first argument to the second argument
 227 # If the link already exists: fail if $USE_EXISTING is 0, otherwise skip.
 228 function make_symlink
 229 {
 230   local cmd="ln -s '$1' '$2'"
 231   printf "  %s Creating symlink from %s to %s..." "$MARKER" "$1" "$2"
 232   local check_cmd="[ ! -e '$2' ]"
 233   msg=$( eval "$check_cmd" 2>&1 )
 234   if validate_ret_skip $? "File already exists."; then
 235     return
 236   fi
 237   if [ $DRY_RUN -eq 0 ]; then
 238     msg=$( eval "$cmd" 2>&1 )
 239     validate_ret $? "$msg"
 240   else
 241     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 242   fi
 243 }
 244
 245 # Go into the directory or repository, even if $DRY_RUN is non-zero.
 246 # If the directory does not exist, fail and log an error.
 247 # Otherwise, silently succeed.
 248 function goto_dir
 249 {
 250   if ! cd "$1" 1>/dev/null 2>/dev/null ; then
 251     printf "  %s Changing to directory %s..." "$MARKER" "$1"
 252     validate_ret 1 "$1: Not found. Stopping."
 253   fi
 254 }
 255
 256 # Clone a repository into a directory.
 257 # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
 258 function clone_repo
 259 {
 260   local cmd="git clone $TOR_EXTRA_CLONE_ARGS '$1' '$2'"
 261   printf "  %s Cloning %s into %s..." "$MARKER" "$1" "$2"
 262   local check_cmd="[ ! -d '$2' ]"
 263   msg=$( eval "$check_cmd" 2>&1 )
 264   if validate_ret_skip $? "Directory already exists."; then
 265     # If we skip the clone, we need to do a fetch
 266     goto_dir "$ORIGIN_PATH"
 267     fetch_remote "origin"
 268     return
 269   fi
 270   if [ $DRY_RUN -eq 0 ]; then
 271     msg=$( eval "$cmd" 2>&1 )
 272     validate_ret $? "$msg"
 273   else
 274     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 275   fi
 276 }
 277
 278 # Add a remote by name and URL.
 279 # If the remote already exists: fail if $USE_EXISTING is 0, otherwise skip.
 280 function add_remote
 281 {
 282   local cmd="git remote add '$1' '$2'"
 283   printf "  %s Adding remote %s at %s..." "$MARKER" "$1" "$2"
 284   local check_cmd="git remote get-url '$1'"
 285   msg=$( eval "$check_cmd" 2>&1 )
 286   ret=$?
 287   # We don't want a remote, so we invert the exit status
 288   if validate_ret_skip $(( ! ret )) \
 289                        "Remote already exists for $1 at $msg."; then
 290     return
 291   fi
 292   if [ $DRY_RUN -eq 0 ]; then
 293     msg=$( eval "$cmd" 2>&1 )
 294     validate_ret $? "$msg"
 295   else
 296     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 297   fi
 298 }
 299
 300 # Set a remote's push URL by name and URL.
 301 function set_remote_push
 302 {
 303   local cmd="git remote set-url --push '$1' '$2'"
 304   printf "  %s Setting remote %s push URL to '%s'..." "$MARKER" "$1" "$2"
 305   if [ $DRY_RUN -eq 0 ]; then
 306     msg=$( eval "$cmd" 2>&1 )
 307     validate_ret $? "$msg"
 308   else
 309     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 310   fi
 311 }
 312
 313 # Fetch a remote by name.
 314 function fetch_remote
 315 {
 316   local cmd="git fetch '$1'"
 317   printf "  %s Fetching %s..." "$MARKER" "$1"
 318   if [ $DRY_RUN -eq 0 ]; then
 319     msg=$( eval "$cmd" 2>&1 )
 320     validate_ret $? "$msg"
 321   else
 322     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 323   fi
 324 }
 325
 326 # Replace the fetch configs for a remote with config if they match a pattern.
 327 function replace_fetch_config
 328 {
 329   local cmd="git config --replace-all remote.'$1'.fetch '$2' '$3'"
 330   printf "  %s Replacing %s fetch configs for '%s'..." \
 331     "$MARKER" "$1" "$3"
 332   if [ $DRY_RUN -eq 0 ]; then
 333     msg=$( eval "$cmd" 2>&1 )
 334     validate_ret $? "$msg"
 335   else
 336     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 337   fi
 338 }
 339
 340 # Set up the tor-github PR config, so tor-github/pr/NNNN/head points to GitHub
 341 # PR NNNN. In some repositories, "/head" is optional.
 342 function set_tor_github_pr_fetch_config
 343 {
 344   # Standard branches
 345   replace_fetch_config tor-github \
 346     "+refs/heads/*:refs/remotes/tor-github/*" \
 347     "refs/heads"
 348   # PRs
 349   replace_fetch_config "tor-github" \
 350     "+refs/pull/*:refs/remotes/tor-github/pr/*" \
 351     "refs/pull.*pr"
 352 }
 353
 354 # Set up the tor-github PR config, so tor-gitlab/mr/NNNN points to GitHub
 355 # MR NNNN. In some repositories, "/head" is optional.
 356 function set_tor_gitlab_mr_fetch_config
 357 {
 358   # standard branches
 359   replace_fetch_config tor-gitlab \
 360     "+refs/heads/*:refs/remotes/tor-gitlab/*" \
 361     "refs/heads"
 362   # MRs
 363   replace_fetch_config tor-gitlab \
 364     "+refs/merge-requests/*/head:refs/remotes/tor-gitlab/mr/*" \
 365     "refs/merge-requests.*mr"
 366 }
 367
 368 # Add a new worktree for branch at path.
 369 # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
 370 function add_worktree
 371 {
 372   local cmd="git worktree add '$2' '$1'"
 373   printf "  %s Adding worktree for %s at %s..." "$MARKER" "$1" "$2"
 374   local check_cmd="[ ! -d '$2' ]"
 375   msg=$( eval "$check_cmd" 2>&1 )
 376   if validate_ret_skip $? "Directory already exists."; then
 377     return
 378   fi
 379   if [ $DRY_RUN -eq 0 ]; then
 380     msg=$( eval "$cmd" 2>&1 )
 381     validate_ret $? "$msg"
 382   else
 383     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 384   fi
 385 }
 386
 387 # Switch to the given branch name.
 388 # If the branch does not exist: fail.
 389 function switch_branch
 390 {
 391   local cmd="git checkout '$1'"
 392   printf "  %s Switching branch to %s..." "$MARKER" "$1"
 393   if [ $DRY_RUN -eq 0 ]; then
 394     msg=$( eval "$cmd" 2>&1 )
 395     validate_ret $? "$msg"
 396   else
 397     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 398   fi
 399 }
 400
 401 # Checkout a new branch with the given branch name.
 402 # If the branch already exists: fail if $USE_EXISTING is 0, otherwise skip.
 403 function new_branch
 404 {
 405   local cmd="git checkout -b '$1'"
 406   printf "  %s Creating new branch %s..." "$MARKER" "$1"
 407   local check_cmd="git branch --list '$1'"
 408   msg=$( eval "$check_cmd" 2>&1 )
 409   if validate_ret_skip $? "Branch already exists."; then
 410     return
 411   fi
 412   if [ $DRY_RUN -eq 0 ]; then
 413     msg=$( eval "$cmd" 2>&1 )
 414     validate_ret $? "$msg"
 415   else
 416     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 417   fi
 418 }
 419
 420 # Switch to an existing branch, or checkout a new branch with the given
 421 # branch name.
 422 function switch_or_new_branch
 423 {
 424   local cmd="git rev-parse --verify '$1'"
 425   if [ $DRY_RUN -eq 0 ]; then
 426     # Call switch_branch if there is a branch, or new_branch if there is not
 427     msg=$( eval "$cmd" 2>&1 )
 428     RET=$?
 429     if [ $RET -eq 0 ]; then
 430       # Branch: (commit id)
 431       switch_branch "$1"
 432     elif [ $RET -eq 128 ]; then
 433       # Not a branch: "fatal: Needed a single revision"
 434       new_branch "$1"
 435     else
 436       # Unexpected return value
 437       validate_ret $RET "$msg"
 438     fi
 439   else
 440     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:"
 441     switch_branch "$1"
 442     new_branch "$1"
 443   fi
 444 }
 445
 446 # Set the upstream for branch to upstream.
 447 function set_upstream
 448 {
 449   # Note the argument order is swapped
 450   local cmd="git branch --set-upstream-to='$2' '$1'"
 451   printf "  %s Setting upstream for %s to %s..." "$MARKER" "$1" "$2"
 452   if [ $DRY_RUN -eq 0 ]; then
 453     msg=$( eval "$cmd" 2>&1 )
 454     validate_ret $? "$msg"
 455   else
 456     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 457   fi
 458 }
 459
 460 ###############
 461 # Entry point #
 462 ###############
 463
 464 printf "%s Setting up the repository and remote %s\\n" "$MARKER" \
 465   "${BYEL}origin${CNRM}"
 466 # First, fetch the origin.
 467 ORIGIN_PARENT=$(dirname "$ORIGIN_PATH")
 468 make_directory "$ORIGIN_PARENT"
 469 # This is just cd with an error check
 470 goto_dir "$ORIGIN_PARENT"
 471
 472 # clone repository / origin remote
 473 clone_repo "$GIT_ORIGIN_PULL" "$TOR_MASTER_NAME"
 474 goto_dir "$ORIGIN_PATH"
 475 set_remote_push "origin" "$GIT_ORIGIN_PUSH"
 476
 477 # upstream remote, if different to origin
 478 if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
 479   printf "%s Setting up remote %s\\n" "$MARKER" \
 480     "${BYEL}$DEFAULT_UPSTREAM_REMOTE${CNRM}"
 481   add_remote "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PULL"
 482   set_remote_push "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PUSH"
 483   fetch_remote "$DEFAULT_UPSTREAM_REMOTE"
 484 fi
 485
 486 # GitHub remote
 487 printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-github${CNRM}"
 488 # Add remote
 489 add_remote "tor-github" "$GITHUB_PULL"
 490 set_remote_push "tor-github" "$GITHUB_PUSH"
 491 # Add custom fetch for PRs
 492 set_tor_github_pr_fetch_config
 493 # Now fetch them all
 494 fetch_remote "tor-github"
 495
 496 # GitLab remote
 497 printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-gitlab${CNRM}"
 498 add_remote "tor-gitlab" "$GITLAB_PULL"
 499 set_remote_push "tor-gitlab" "$GITLAB_PUSH"
 500 # Add custom fetch for MRs
 501 set_tor_gitlab_mr_fetch_config
 502 # Now fetch them all
 503 fetch_remote "tor-gitlab"
 504
 505 # Extra remote
 506 if [ "$TOR_EXTRA_REMOTE_NAME" ]; then
 507   printf "%s Setting up remote %s\\n" "$MARKER" \
 508     "${BYEL}$TOR_EXTRA_REMOTE_NAME${CNRM}"
 509   # Add remote
 510   add_remote "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PULL"
 511   set_remote_push "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PUSH"
 512   # But leave it to the user to decide if they want to fetch it
 513   #fetch_remote "$TOR_EXTRA_REMOTE_NAME"
 514 fi
 515
 516 # Go over all configured worktree.
 517 for ((i=0; i<COUNT; i++)); do
 518   branch=${!WORKTREE[$i]:0:1}
 519   repo_path=${!WORKTREE[$i]:1:1}
 520
 521   printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}"
 522   # We cloned the repository, and main is the default branch
 523   if [ "$branch" = "main" ]; then
 524     if [ "$TOR_MASTER_NAME" != "main" ]; then
 525       # Set up a main branch link in the worktree directory
 526       make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/main"
 527     fi
 528   else
 529     # git makes worktree directories if they don't exist
 530     add_worktree "origin/$branch" "$repo_path"
 531   fi
 532   goto_dir "$repo_path"
 533   switch_or_new_branch "$branch"
 534   set_upstream "$branch" "origin/$branch"
 535 done
 536
 537 echo
 538 echo "Remember to copy the git hooks from tor/scripts/git/*.git-hook to"
 539 echo "$ORIGIN_PATH/.git/hooks/*"