ci/lib.sh

   1 # Library of functions shared by all CI scripts
   2
   3 if test true = "$GITHUB_ACTIONS"
   4 then
   5         begin_group () {
   6                 need_to_end_group=t
   7                 echo "::group::$1" >&2
   8                 set -x
   9         }
  10
  11         end_group () {
  12                 test -n "$need_to_end_group" || return 0
  13                 set +x
  14                 need_to_end_group=
  15                 echo '::endgroup::' >&2
  16         }
  17 elif test true = "$GITLAB_CI"
  18 then
  19         begin_group () {
  20                 need_to_end_group=t
  21                 printf "\e[0Ksection_start:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K$1\n"
  22                 trap "end_group '$1'" EXIT
  23                 set -x
  24         }
  25
  26         end_group () {
  27                 test -n "$need_to_end_group" || return 0
  28                 set +x
  29                 need_to_end_group=
  30                 printf "\e[0Ksection_end:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K\n"
  31                 trap - EXIT
  32         }
  33 else
  34         begin_group () { :; }
  35         end_group () { :; }
  36
  37         set -x
  38 fi
  39
  40 group () {
  41         group="$1"
  42         shift
  43         begin_group "$group"
  44
  45         # work around `dash` not supporting `set -o pipefail`
  46         (
  47                 "$@" 2>&1
  48                 echo $? >exit.status
  49         ) |
  50         sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/'
  51         res=$(cat exit.status)
  52         rm exit.status
  53
  54         end_group "$group"
  55         return $res
  56 }
  57
  58 begin_group "CI setup"
  59 trap "end_group 'CI setup'" EXIT
  60
  61 # Set 'exit on error' for all CI scripts to let the caller know that
  62 # something went wrong.
  63 #
  64 # We already enabled tracing executed commands earlier. This helps by showing
  65 # how # environment variables are set and and dependencies are installed.
  66 set -e
  67
  68 skip_branch_tip_with_tag () {
  69         # Sometimes, a branch is pushed at the same time the tag that points
  70         # at the same commit as the tip of the branch is pushed, and building
  71         # both at the same time is a waste.
  72         #
  73         # When the build is triggered by a push to a tag, $CI_BRANCH will
  74         # have that tagname, e.g. v2.14.0.  Let's see if $CI_BRANCH is
  75         # exactly at a tag, and if so, if it is different from $CI_BRANCH.
  76         # That way, we can tell if we are building the tip of a branch that
  77         # is tagged and we can skip the build because we won't be skipping a
  78         # build of a tag.
  79
  80         if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) &&
  81                 test "$TAG" != "$CI_BRANCH"
  82         then
  83                 echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)"
  84                 exit 0
  85         fi
  86 }
  87
  88 # Check whether we can use the path passed via the first argument as Git
  89 # repository.
  90 is_usable_git_repository () {
  91         # We require Git in our PATH, otherwise we cannot access repositories
  92         # at all.
  93         if ! command -v git >/dev/null
  94         then
  95                 return 1
  96         fi
  97
  98         # And the target directory needs to be a proper Git repository.
  99         if ! git -C "$1" rev-parse 2>/dev/null
 100         then
 101                 return 1
 102         fi
 103 }
 104
 105 # Save some info about the current commit's tree, so we can skip the build
 106 # job if we encounter the same tree again and can provide a useful info
 107 # message.
 108 save_good_tree () {
 109         if ! is_usable_git_repository .
 110         then
 111                 return
 112         fi
 113
 114         echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file"
 115         # limit the file size
 116         tail -1000 "$good_trees_file" >"$good_trees_file".tmp
 117         mv "$good_trees_file".tmp "$good_trees_file"
 118 }
 119
 120 # Skip the build job if the same tree has already been built and tested
 121 # successfully before (e.g. because the branch got rebased, changing only
 122 # the commit messages).
 123 skip_good_tree () {
 124         if test true = "$GITHUB_ACTIONS"
 125         then
 126                 return
 127         fi
 128
 129         if ! is_usable_git_repository .
 130         then
 131                 return
 132         fi
 133
 134         if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")"
 135         then
 136                 # Haven't seen this tree yet, or no cached good trees file yet.
 137                 # Continue the build job.
 138                 return
 139         fi
 140
 141         echo "$good_tree_info" | {
 142                 read tree prev_good_commit prev_good_job_number prev_good_job_id
 143
 144                 if test "$CI_JOB_ID" = "$prev_good_job_id"
 145                 then
 146                         cat <<-EOF
 147                         $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
 148                         This commit has already been built and tested successfully by this build job.
 149                         To force a re-build delete the branch's cache and then hit 'Restart job'.
 150                         EOF
 151                 else
 152                         cat <<-EOF
 153                         $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
 154                         This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit.
 155                         The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id
 156                         To force a re-build delete the branch's cache and then hit 'Restart job'.
 157                         EOF
 158                 fi
 159         }
 160
 161         exit 0
 162 }
 163
 164 check_unignored_build_artifacts () {
 165         if ! is_usable_git_repository .
 166         then
 167                 return
 168         fi
 169
 170         ! git ls-files --other --exclude-standard --error-unmatch \
 171                 -- ':/*' 2>/dev/null ||
 172         {
 173                 echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
 174                 false
 175         }
 176 }
 177
 178 handle_failed_tests () {
 179         return 1
 180 }
 181
 182 create_failed_test_artifacts () {
 183         mkdir -p t/failed-test-artifacts
 184
 185         for test_exit in t/test-results/*.exit
 186         do
 187                 test 0 != "$(cat "$test_exit")" || continue
 188
 189                 test_name="${test_exit%.exit}"
 190                 test_name="${test_name##*/}"
 191                 printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n"
 192                 echo "The full logs are in the 'print test failures' step below."
 193                 echo "See also the 'failed-tests-*' artifacts attached to this run."
 194                 cat "t/test-results/$test_name.markup"
 195
 196                 trash_dir="t/trash directory.$test_name"
 197                 cp "t/test-results/$test_name.out" t/failed-test-artifacts/
 198                 tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
 199         done
 200 }
 201
 202 # GitHub Action doesn't set TERM, which is required by tput
 203 export TERM=${TERM:-dumb}
 204
 205 # Clear MAKEFLAGS that may come from the outside world.
 206 export MAKEFLAGS=
 207
 208 if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
 209 then
 210         CI_TYPE=azure-pipelines
 211         # We are running in Azure Pipelines
 212         CI_BRANCH="$BUILD_SOURCEBRANCH"
 213         CI_COMMIT="$BUILD_SOURCEVERSION"
 214         CI_JOB_ID="$BUILD_BUILDID"
 215         CI_JOB_NUMBER="$BUILD_BUILDNUMBER"
 216         CI_OS_NAME="$(echo "$AGENT_OS" | tr A-Z a-z)"
 217         test darwin != "$CI_OS_NAME" || CI_OS_NAME=osx
 218         CI_REPO_SLUG="$(expr "$BUILD_REPOSITORY_URI" : '.*/\([^/]*/[^/]*\)$')"
 219         CC="${CC:-gcc}"
 220
 221         # use a subdirectory of the cache dir (because the file share is shared
 222         # among *all* phases)
 223         cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME"
 224
 225         GIT_TEST_OPTS="--write-junit-xml"
 226         JOBS=10
 227 elif test true = "$GITHUB_ACTIONS"
 228 then
 229         CI_TYPE=github-actions
 230         CI_BRANCH="$GITHUB_REF"
 231         CI_COMMIT="$GITHUB_SHA"
 232         CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)"
 233         test macos != "$CI_OS_NAME" || CI_OS_NAME=osx
 234         CI_REPO_SLUG="$GITHUB_REPOSITORY"
 235         CI_JOB_ID="$GITHUB_RUN_ID"
 236         CC="${CC_PACKAGE:-${CC:-gcc}}"
 237         DONT_SKIP_TAGS=t
 238         handle_failed_tests () {
 239                 echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
 240                 create_failed_test_artifacts
 241                 return 1
 242         }
 243
 244         cache_dir="$HOME/none"
 245
 246         GIT_TEST_OPTS="--github-workflow-markup"
 247         JOBS=10
 248 elif test true = "$GITLAB_CI"
 249 then
 250         CI_TYPE=gitlab-ci
 251         CI_BRANCH="$CI_COMMIT_REF_NAME"
 252         CI_COMMIT="$CI_COMMIT_SHA"
 253         case "$CI_JOB_IMAGE" in
 254         macos-*)
 255                 # GitLab CI has Python installed via multiple package managers,
 256                 # most notably via asdf and Homebrew. Ensure that our builds
 257                 # pick up the Homebrew one by prepending it to our PATH as the
 258                 # asdf one breaks tests.
 259                 export PATH="$(brew --prefix)/bin:$PATH"
 260
 261                 CI_OS_NAME=osx
 262                 ;;
 263         alpine:*|fedora:*|ubuntu:*)
 264                 CI_OS_NAME=linux;;
 265         *)
 266                 echo "Could not identify OS image" >&2
 267                 env >&2
 268                 exit 1
 269                 ;;
 270         esac
 271         CI_REPO_SLUG="$CI_PROJECT_PATH"
 272         CI_JOB_ID="$CI_JOB_ID"
 273         CC="${CC_PACKAGE:-${CC:-gcc}}"
 274         DONT_SKIP_TAGS=t
 275         handle_failed_tests () {
 276                 create_failed_test_artifacts
 277                 return 1
 278         }
 279
 280         cache_dir="$HOME/none"
 281
 282         runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -)
 283         JOBS=$(nproc)
 284 else
 285         echo "Could not identify CI type" >&2
 286         env >&2
 287         exit 1
 288 fi
 289
 290 MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS"
 291 GIT_PROVE_OPTS="--timer --jobs $JOBS"
 292
 293 GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x"
 294 case "$CI_OS_NAME" in
 295 windows|windows_nt)
 296         GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers"
 297         ;;
 298 esac
 299
 300 export GIT_TEST_OPTS
 301 export GIT_PROVE_OPTS
 302
 303 good_trees_file="$cache_dir/good-trees"
 304
 305 mkdir -p "$cache_dir"
 306
 307 test -n "${DONT_SKIP_TAGS-}" ||
 308 skip_branch_tip_with_tag
 309 skip_good_tree
 310
 311 if test -z "$jobname"
 312 then
 313         jobname="$CI_OS_NAME-$CC"
 314 fi
 315
 316 export DEVELOPER=1
 317 export DEFAULT_TEST_TARGET=prove
 318 export GIT_TEST_CLONE_2GB=true
 319 export SKIP_DASHED_BUILT_INS=YesPlease
 320
 321 case "$runs_on_pool" in
 322 ubuntu-*)
 323         if test "$jobname" = "linux-gcc-default"
 324         then
 325                 break
 326         fi
 327
 328         PYTHON_PACKAGE=python2
 329         if test "$jobname" = linux-gcc
 330         then
 331                 PYTHON_PACKAGE=python3
 332         fi
 333         MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/$PYTHON_PACKAGE"
 334
 335         export GIT_TEST_HTTPD=true
 336
 337         # The Linux build installs the defined dependency versions below.
 338         # The OS X build installs much more recent versions, whichever
 339         # were recorded in the Homebrew database upon creating the OS X
 340         # image.
 341         # Keep that in mind when you encounter a broken OS X build!
 342         export LINUX_GIT_LFS_VERSION="1.5.2"
 343
 344         P4_PATH="$HOME/custom/p4"
 345         GIT_LFS_PATH="$HOME/custom/git-lfs"
 346         export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
 347         ;;
 348 macos-*)
 349         MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 350         if [ "$jobname" != osx-gcc ]
 351         then
 352                 MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes"
 353         fi
 354
 355         P4_PATH="$HOME/custom/p4"
 356         export PATH="$P4_PATH:$PATH"
 357         ;;
 358 esac
 359
 360 case "$jobname" in
 361 linux32)
 362         CC=gcc
 363         ;;
 364 linux-musl)
 365         CC=gcc
 366         MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes"
 367         MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes"
 368         MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 369         ;;
 370 linux-leaks|linux-reftable-leaks)
 371         export SANITIZE=leak
 372         export GIT_TEST_PASSING_SANITIZE_LEAK=true
 373         export GIT_TEST_SANITIZE_LEAK_LOG=true
 374         ;;
 375 linux-asan-ubsan)
 376         export SANITIZE=address,undefined
 377         export NO_SVN_TESTS=LetsSaveSomeTime
 378         MAKEFLAGS="$MAKEFLAGS NO_PYTHON=YepBecauseP4FlakesTooOften"
 379         ;;
 380 esac
 381
 382 MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
 383
 384 end_group "CI setup"
 385 set -x