third_party/jpeg-xl/ci.sh

   1 #!/usr/bin/env bash
   2 # Copyright (c) the JPEG XL Project Authors. All rights reserved.
   3 #
   4 # Use of this source code is governed by a BSD-style
   5 # license that can be found in the LICENSE file.
   6
   7 # Continuous integration helper module. This module is meant to be called from
   8 # the .gitlab-ci.yml file during the continuous integration build, as well as
   9 # from the command line for developers.
  10
  11 set -eu
  12
  13 OS=`uname -s`
  14
  15 MYDIR=$(dirname $(realpath "$0"))
  16
  17 ### Environment parameters:
  18 TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-256}"
  19 CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
  20 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}
  21 CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-}
  22 CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-}
  23 CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-}
  24 SKIP_TEST="${SKIP_TEST:-0}"
  25 TEST_SELECTOR="${TEST_SELECTOR:-}"
  26 BUILD_TARGET="${BUILD_TARGET:-}"
  27 ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}"
  28 if [[ -n "${BUILD_TARGET}" ]]; then
  29   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}"
  30 else
  31   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}"
  32 fi
  33 # Whether we should post a message in the MR when the build fails.
  34 POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}"
  35
  36 # Set default compilers to clang if not already set
  37 export CC=${CC:-clang}
  38 export CXX=${CXX:-clang++}
  39
  40 # Time limit for the "fuzz" command in seconds (0 means no limit).
  41 FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}"
  42
  43 SANITIZER="none"
  44
  45
  46 if [[ "${BUILD_TARGET%%-*}" == "x86_64" ||
  47     "${BUILD_TARGET%%-*}" == "i686" ]]; then
  48   # Default to building all targets, even if compiler baseline is SSE4
  49   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_EMU128}
  50 else
  51   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-}
  52 fi
  53
  54 # Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
  55 CMAKE_FLAGS=${CMAKE_FLAGS:-}
  56 CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}"
  57 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}"
  58
  59 CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-}
  60 CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-}
  61 CMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH:-}
  62 CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-}
  63 CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-}
  64 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}
  65
  66 if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then
  67   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -msimd128"
  68   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -msimd128"
  69   CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128"
  70 fi
  71
  72 if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then
  73   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2"
  74   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2"
  75 fi
  76
  77 if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then
  78   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}"
  79 fi
  80
  81 # Version inferred from the CI variables.
  82 CI_COMMIT_SHA=${CI_COMMIT_SHA:-${GITHUB_SHA:-}}
  83 JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}}
  84
  85 # Benchmark parameters
  86 STORE_IMAGES=${STORE_IMAGES:-1}
  87 BENCHMARK_CORPORA="${MYDIR}/third_party/corpora"
  88
  89 # Local flags passed to sanitizers.
  90 UBSAN_FLAGS=(
  91   -fsanitize=alignment
  92   -fsanitize=bool
  93   -fsanitize=bounds
  94   -fsanitize=builtin
  95   -fsanitize=enum
  96   -fsanitize=float-cast-overflow
  97   -fsanitize=float-divide-by-zero
  98   -fsanitize=integer-divide-by-zero
  99   -fsanitize=null
 100   -fsanitize=object-size
 101   -fsanitize=pointer-overflow
 102   -fsanitize=return
 103   -fsanitize=returns-nonnull-attribute
 104   -fsanitize=shift-base
 105   -fsanitize=shift-exponent
 106   -fsanitize=unreachable
 107   -fsanitize=vla-bound
 108
 109   -fno-sanitize-recover=undefined
 110   # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning.
 111   -fsanitize-recover=alignment
 112 )
 113 # -fsanitize=function doesn't work on aarch64 and arm.
 114 if [[ "${BUILD_TARGET%%-*}" != "aarch64" &&
 115     "${BUILD_TARGET%%-*}" != "arm" ]]; then
 116   UBSAN_FLAGS+=(
 117     -fsanitize=function
 118   )
 119 fi
 120 if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then
 121   UBSAN_FLAGS+=(
 122     -fsanitize=signed-integer-overflow
 123   )
 124 fi
 125
 126 CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy | head -n 1)
 127 # Default to "cat" if "colordiff" is not installed or if stdout is not a tty.
 128 if [[ -t 1 ]]; then
 129   COLORDIFF_BIN=$(which colordiff cat | head -n 1)
 130 else
 131   COLORDIFF_BIN="cat"
 132 fi
 133 FIND_BIN=$(which gfind find | head -n 1)
 134 # "false" will disable wine64 when not installed. This won't allow
 135 # cross-compiling.
 136 WINE_BIN=$(which wine64 false | head -n 1)
 137
 138 CLANG_VERSION="${CLANG_VERSION:-}"
 139 # Detect the clang version suffix and store it in CLANG_VERSION. For example,
 140 # "6.0" for clang 6 or "7" for clang 7.
 141 detect_clang_version() {
 142   if [[ -n "${CLANG_VERSION}" ]]; then
 143     return 0
 144   fi
 145   local clang_version=$("${CC:-clang}" --version | head -n1)
 146   clang_version=${clang_version#"Debian "}
 147   clang_version=${clang_version#"Ubuntu "}
 148   local llvm_tag
 149   case "${clang_version}" in
 150     "clang version 6."*)
 151       CLANG_VERSION="6.0"
 152       ;;
 153     "clang version "*)
 154       # Any other clang version uses just the major version number.
 155       local suffix="${clang_version#clang version }"
 156       CLANG_VERSION="${suffix%%.*}"
 157       ;;
 158     "emcc"*)
 159       # We can't use asan or msan in the emcc case.
 160       ;;
 161     *)
 162       echo "Unknown clang version: ${clang_version}" >&2
 163       return 1
 164   esac
 165 }
 166
 167 # Temporary files cleanup hooks.
 168 CLEANUP_FILES=()
 169 cleanup() {
 170   if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
 171     rm -fr "${CLEANUP_FILES[@]}"
 172   fi
 173 }
 174
 175 # Executed on exit.
 176 on_exit() {
 177   local retcode="$1"
 178   # Always cleanup the CLEANUP_FILES.
 179   cleanup
 180
 181   # Post a message in the MR when requested with POST_MESSAGE_ON_ERROR but only
 182   # if the run failed and we are not running from a MR pipeline.
 183   if [[ ${retcode} -ne 0 && -n "${CI_BUILD_NAME:-}" &&
 184         -n "${POST_MESSAGE_ON_ERROR}" && -z "${CI_MERGE_REQUEST_ID:-}" &&
 185         "${CI_BUILD_REF_NAME}" = "master" ]]; then
 186     load_mr_vars_from_commit
 187     { set +xeu; } 2>/dev/null
 188     local message="**Run ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} failed.**
 189
 190 Check the output of the job at ${CI_JOB_URL:-} to see if this was your problem.
 191 If it was, please rollback this change or fix the problem ASAP, broken builds
 192 slow down development. Check if the error already existed in the previous build
 193 as well.
 194
 195 Pipeline: ${CI_PIPELINE_URL}
 196
 197 Previous build commit: ${CI_COMMIT_BEFORE_SHA}
 198 "
 199     cmd_post_mr_comment "${message}"
 200   fi
 201 }
 202
 203 trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT
 204
 205
 206 # These variables are populated when calling merge_request_commits().
 207
 208 # The current hash at the top of the current branch or merge request branch (if
 209 # running from a merge request pipeline).
 210 MR_HEAD_SHA=""
 211 # The common ancestor between the current commit and the tracked branch, such
 212 # as master. This includes a list
 213 MR_ANCESTOR_SHA=""
 214
 215 # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA.
 216 merge_request_commits() {
 217   { set +x; } 2>/dev/null
 218   # GITHUB_SHA is the current reference being build in GitHub Actions.
 219   if [[ -n "${GITHUB_SHA:-}" ]]; then
 220     # GitHub normally does a checkout of a merge commit on a shallow repository
 221     # by default. We want to get a bit more of the history to be able to diff
 222     # changes on the Pull Request if needed. This fetches 10 more commits which
 223     # should be enough given that PR normally should have 1 commit.
 224     git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10
 225     MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null ||
 226                    echo "${GITHUB_SHA}")"
 227   else
 228     # CI_BUILD_REF is the reference currently being build in the CI workflow.
 229     MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "${CI_BUILD_REF:-HEAD}")
 230   fi
 231
 232   if [[ -n "${CI_MERGE_REQUEST_IID:-}" ]]; then
 233     # Merge request pipeline in CI. In this case the upstream is called "origin"
 234     # but it refers to the forked project that's the source of the merge
 235     # request. We need to get the target of the merge request, for which we need
 236     # to query that repository using our CI_JOB_TOKEN.
 237     echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" \
 238       >> "${HOME}/.netrc"
 239     git -C "${MYDIR}" fetch "${CI_MERGE_REQUEST_PROJECT_URL}" \
 240       "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
 241     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD)
 242   elif [[ -n "${GITHUB_BASE_REF:-}" ]]; then
 243     # Pull request workflow in GitHub Actions. GitHub checkout action uses
 244     # "origin" as the remote for the git checkout.
 245     git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}"
 246     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD)
 247   else
 248     # We are in a local branch, not a merge request.
 249     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true)
 250   fi
 251
 252   if [[ -z "${MR_ANCESTOR_SHA}" ]]; then
 253     echo "Warning, not tracking any branch, using the last commit in HEAD.">&2
 254     # This prints the return value with just HEAD.
 255     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^")
 256   else
 257     # GitHub runs the pipeline on a merge commit, no need to look for the common
 258     # ancestor in that case.
 259     if [[ -z "${GITHUB_BASE_REF:-}" ]]; then
 260       MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base \
 261         "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}")
 262     fi
 263   fi
 264   set -x
 265 }
 266
 267 # Load the MR iid from the landed commit message when running not from a
 268 # merge request workflow. This is useful to post back results at the merge
 269 # request when running pipelines from master.
 270 load_mr_vars_from_commit() {
 271   { set +x; } 2>/dev/null
 272   if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then
 273     local mr_iid=$(git rev-list --format=%B --max-count=1 HEAD |
 274       grep -F "${CI_PROJECT_URL}" | grep -F "/merge_requests" | head -n 1)
 275     # mr_iid contains a string like this if it matched:
 276     #  Part-of: <https://gitlab.com/wg1/jpeg-xlm/merge_requests/123456>
 277     if [[ -n "${mr_iid}" ]]; then
 278       mr_iid=$(echo "${mr_iid}" |
 279         sed -E 's,^.*merge_requests/([0-9]+)>.*$,\1,')
 280       CI_MERGE_REQUEST_IID="${mr_iid}"
 281       CI_MERGE_REQUEST_PROJECT_ID=${CI_PROJECT_ID}
 282     fi
 283   fi
 284   set -x
 285 }
 286
 287 # Posts a comment to the current merge request.
 288 cmd_post_mr_comment() {
 289   { set +x; } 2>/dev/null
 290   local comment="$1"
 291   if [[ -n "${BOT_TOKEN:-}" && -n "${CI_MERGE_REQUEST_IID:-}" ]]; then
 292     local url="${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes"
 293     curl -X POST -g \
 294       -H "PRIVATE-TOKEN: ${BOT_TOKEN}" \
 295       --data-urlencode "body=${comment}" \
 296       --output /dev/null \
 297       "${url}"
 298   fi
 299   set -x
 300 }
 301
 302 # Set up and export the environment variables needed by the child processes.
 303 export_env() {
 304   if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 305     # Wine needs to know the paths to the mingw dlls. These should be
 306     # separated by ';'.
 307     WINEPATH=$("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \
 308       | grep -F 'libraries: =' | cut -f 2- -d '=' | tr ':' ';')
 309     # We also need our own libraries in the wine path.
 310     local real_build_dir=$(realpath "${BUILD_DIR}")
 311     # Some library .dll dependencies are installed in /bin:
 312     export WINEPATH="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin"
 313
 314     local prefix="${BUILD_DIR}/wineprefix"
 315     mkdir -p "${prefix}"
 316     export WINEPREFIX=$(realpath "${prefix}")
 317   fi
 318   # Sanitizers need these variables to print and properly format the stack
 319   # traces:
 320   LLVM_SYMBOLIZER=$("${CC:-clang}" -print-prog-name=llvm-symbolizer || true)
 321   if [[ -n "${LLVM_SYMBOLIZER}" ]]; then
 322     export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 323     export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 324     export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 325   fi
 326 }
 327
 328 cmake_configure() {
 329   export_env
 330
 331   if [[ "${STACK_SIZE:-0}" == 1 ]]; then
 332     # Dump the stack size of each function in the .stack_sizes section for
 333     # analysis.
 334     CMAKE_C_FLAGS+=" -fstack-size-section"
 335     CMAKE_CXX_FLAGS+=" -fstack-size-section"
 336   fi
 337
 338   local args=(
 339     -B"${BUILD_DIR}" -H"${MYDIR}"
 340     -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
 341     -G Ninja
 342     -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}"
 343     -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}"
 344     -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}"
 345     -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}"
 346     -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}"
 347     -DJPEGXL_VERSION="${JPEGXL_VERSION}"
 348     -DSANITIZER="${SANITIZER}"
 349     # These are not enabled by default in cmake.
 350     -DJPEGXL_ENABLE_VIEWERS=ON
 351     -DJPEGXL_ENABLE_PLUGINS=ON
 352     -DJPEGXL_ENABLE_DEVTOOLS=ON
 353     # We always use libfuzzer in the ci.sh wrapper.
 354     -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer"
 355   )
 356   if [[ "${BUILD_TARGET}" != *mingw32 ]]; then
 357     args+=(
 358       -DJPEGXL_WARNINGS_AS_ERRORS=ON
 359     )
 360   fi
 361   if [[ -n "${BUILD_TARGET}" ]]; then
 362     local system_name="Linux"
 363     if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 364       # When cross-compiling with mingw the target must be set to Windows and
 365       # run programs with wine.
 366       system_name="Windows"
 367       args+=(
 368         -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}"
 369         # Normally CMake automatically defines MINGW=1 when building with the
 370         # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling
 371         # with clang.
 372         -DMINGW=1
 373       )
 374     fi
 375     # EMSCRIPTEN toolchain sets the right values itself
 376     if [[ "${BUILD_TARGET}" != wasm* ]]; then
 377       # If set, BUILD_TARGET must be the target triplet such as
 378       # x86_64-unknown-linux-gnu.
 379       args+=(
 380         -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}"
 381         -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}"
 382         # Only the first element of the target triplet.
 383         -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}"
 384         -DCMAKE_SYSTEM_NAME="${system_name}"
 385         -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}"
 386       )
 387     else
 388       args+=(
 389         # sjpeg confuses WASM SIMD with SSE.
 390         -DSJPEG_ENABLE_SIMD=OFF
 391         # Building shared libs is not very useful for WASM.
 392         -DBUILD_SHARED_LIBS=OFF
 393       )
 394     fi
 395     args+=(
 396       # These are needed to make googletest work when cross-compiling.
 397       -DCMAKE_CROSSCOMPILING=1
 398       -DHAVE_STD_REGEX=0
 399       -DHAVE_POSIX_REGEX=0
 400       -DHAVE_GNU_POSIX_REGEX=0
 401       -DHAVE_STEADY_CLOCK=0
 402       -DHAVE_THREAD_SAFETY_ATTRIBUTES=0
 403     )
 404     if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then
 405       # find_package() will look in this prefix for libraries.
 406       CMAKE_FIND_ROOT_PATH="/usr/${BUILD_TARGET}"
 407     fi
 408     if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then
 409       CMAKE_PREFIX_PATH="/usr/${BUILD_TARGET}"
 410     fi
 411     # Use pkg-config for the target. If there's no pkg-config available for the
 412     # target we can set the PKG_CONFIG_PATH to the appropriate path in most
 413     # linux distributions.
 414     local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true)
 415     if [[ -z "${pkg_config}" ]]; then
 416       pkg_config=$(which pkg-config)
 417       export PKG_CONFIG_LIBDIR="/usr/${BUILD_TARGET}/lib/pkgconfig"
 418     fi
 419     if [[ -n "${pkg_config}" ]]; then
 420       args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}")
 421     fi
 422   fi
 423   if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then
 424     args+=(
 425       -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}"
 426     )
 427   fi
 428   if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then
 429     args+=(
 430       -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}"
 431     )
 432   fi
 433   if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then
 434     args+=(
 435       -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
 436     )
 437   fi
 438   if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then
 439     args+=(
 440       -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}"
 441     )
 442   fi
 443   if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then
 444     args+=(
 445       -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}"
 446     )
 447   fi
 448   if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then
 449     args+=(
 450       -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}"
 451     )
 452   fi
 453   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 454     emcmake cmake "${args[@]}" "$@"
 455   else
 456     cmake "${args[@]}" "$@"
 457   fi
 458 }
 459
 460 cmake_build_and_test() {
 461   # gtest_discover_tests() runs the test binaries to discover the list of tests
 462   # at build time, which fails under qemu.
 463   ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- all doc
 464   # Pack test binaries if requested.
 465   if [[ "${PACK_TEST:-}" == "1" ]]; then
 466     (cd "${BUILD_DIR}"
 467      ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*'
 468      # gtest / gmock / gtest_main shared libs
 469      ${FIND_BIN} lib/ -name 'libg*.so*'
 470      ${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*'
 471     ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \
 472       --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 473     du -h "${BUILD_DIR}/tests.tar.xz"
 474     # Pack coverage data if also available.
 475     touch "${BUILD_DIR}/gcno.sentinel"
 476     (cd "${BUILD_DIR}"; echo gcno.sentinel; ${FIND_BIN} -name '*gcno') | \
 477       tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \
 478         --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 479   fi
 480
 481   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 482     (cd "${BUILD_DIR}"
 483      export UBSAN_OPTIONS=print_stacktrace=1
 484      [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 485      ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure)
 486   fi
 487 }
 488
 489 # Configure the build to strip unused functions. This considerably reduces the
 490 # output size, specially for tests which only use a small part of the whole
 491 # library.
 492 strip_dead_code() {
 493   # Emscripten does tree shaking without any extra flags.
 494   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 495     return 0
 496   fi
 497   # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively
 498   # discard all unreachable code, reducing the code size. For this to work, we
 499   # need to also pass --no-export-dynamic to prevent it from exporting all the
 500   # internal symbols (like functions) making them all reachable and thus not a
 501   # candidate for removal.
 502   CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections"
 503   CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections"
 504   if [[ "${OS}" == "Darwin" ]]; then
 505     CMAKE_EXE_LINKER_FLAGS+=" -dead_strip"
 506     CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip"
 507   else
 508     CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 509     CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 510   fi
 511 }
 512
 513 ### Externally visible commands
 514
 515 cmd_debug() {
 516   CMAKE_BUILD_TYPE="Debug"
 517   cmake_configure "$@"
 518   cmake_build_and_test
 519 }
 520
 521 cmd_release() {
 522   CMAKE_BUILD_TYPE="Release"
 523   strip_dead_code
 524   cmake_configure "$@"
 525   cmake_build_and_test
 526 }
 527
 528 cmd_opt() {
 529   CMAKE_BUILD_TYPE="RelWithDebInfo"
 530   CMAKE_CXX_FLAGS+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR"
 531   cmake_configure "$@"
 532   cmake_build_and_test
 533 }
 534
 535 cmd_coverage() {
 536   # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests.
 537   TEST_STACK_LIMIT="none"
 538
 539   cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@"
 540
 541   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 542     # If we didn't run the test we also don't print a coverage report.
 543     cmd_coverage_report
 544   fi
 545 }
 546
 547 cmd_coverage_report() {
 548   LLVM_COV=$("${CC:-clang}" -print-prog-name=llvm-cov)
 549   local real_build_dir=$(realpath "${BUILD_DIR}")
 550   local gcovr_args=(
 551     -r "${real_build_dir}"
 552     --gcov-executable "${LLVM_COV} gcov"
 553     # Only print coverage information for the libjxl directories. The rest
 554     # is not part of the code under test.
 555     --filter '.*jxl/.*'
 556     --exclude '.*_gbench.cc'
 557     --exclude '.*_test.cc'
 558     --exclude '.*_testonly..*'
 559     --exclude '.*_debug.*'
 560     --exclude '.*test_utils..*'
 561     --object-directory "${real_build_dir}"
 562   )
 563
 564   (
 565    cd "${real_build_dir}"
 566     gcovr "${gcovr_args[@]}" --html --html-details \
 567       --output="${real_build_dir}/coverage.html"
 568     gcovr "${gcovr_args[@]}" --print-summary |
 569       tee "${real_build_dir}/coverage.txt"
 570     gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml"
 571   )
 572 }
 573
 574 cmd_test() {
 575   export_env
 576   # Unpack tests if needed.
 577   if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then
 578     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz"
 579   fi
 580   if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then
 581     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz"
 582   fi
 583   (cd "${BUILD_DIR}"
 584    export UBSAN_OPTIONS=print_stacktrace=1
 585    [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 586    ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure "$@")
 587 }
 588
 589 cmd_gbench() {
 590   export_env
 591   (cd "${BUILD_DIR}"
 592    export UBSAN_OPTIONS=print_stacktrace=1
 593    lib/jxl_gbench \
 594      --benchmark_counters_tabular=true \
 595      --benchmark_out_format=json \
 596      --benchmark_out=gbench.json "$@"
 597   )
 598 }
 599
 600 cmd_asanfuzz() {
 601   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 602   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 603   cmd_asan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 604 }
 605
 606 cmd_msanfuzz() {
 607   # Install msan if needed before changing the flags.
 608   detect_clang_version
 609   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 610   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 611     # Install msan libraries for this version if needed or if an older version
 612     # with libc++abi was installed.
 613     cmd_msan_install
 614   fi
 615
 616   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 617   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 618   cmd_msan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 619 }
 620
 621 cmd_asan() {
 622   SANITIZER="asan"
 623   CMAKE_C_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 624     -fsanitize=address ${UBSAN_FLAGS[@]}"
 625   CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 626     -fsanitize=address ${UBSAN_FLAGS[@]}"
 627   strip_dead_code
 628   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 629   cmake_build_and_test
 630 }
 631
 632 cmd_tsan() {
 633   SANITIZER="tsan"
 634   local tsan_args=(
 635     -DJXL_ENABLE_ASSERT=1
 636     -g
 637     -DTHREAD_SANITIZER
 638     ${UBSAN_FLAGS[@]}
 639     -fsanitize=thread
 640   )
 641   CMAKE_C_FLAGS+=" ${tsan_args[@]}"
 642   CMAKE_CXX_FLAGS+=" ${tsan_args[@]}"
 643
 644   CMAKE_BUILD_TYPE="RelWithDebInfo"
 645   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 646   cmake_build_and_test
 647 }
 648
 649 cmd_msan() {
 650   SANITIZER="msan"
 651   detect_clang_version
 652   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 653   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 654     # Install msan libraries for this version if needed or if an older version
 655     # with libc++abi was installed.
 656     cmd_msan_install
 657   fi
 658
 659   local msan_c_flags=(
 660     -fsanitize=memory
 661     -fno-omit-frame-pointer
 662     -fsanitize-memory-track-origins
 663
 664     -DJXL_ENABLE_ASSERT=1
 665     -g
 666     -DMEMORY_SANITIZER
 667
 668     # Force gtest to not use the cxxbai.
 669     -DGTEST_HAS_CXXABI_H_=0
 670   )
 671   local msan_cxx_flags=(
 672     "${msan_c_flags[@]}"
 673
 674     # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused
 675     # in those cases. Ignore the warning.
 676     -Wno-unused-command-line-argument
 677     -stdlib=libc++
 678
 679     # We include the libc++ from the msan directory instead, so we don't want
 680     # the std includes.
 681     -nostdinc++
 682     -cxx-isystem"${msan_prefix}/include/c++/v1"
 683   )
 684
 685   local msan_linker_flags=(
 686     -L"${msan_prefix}"/lib
 687     -Wl,-rpath -Wl,"${msan_prefix}"/lib/
 688   )
 689
 690   CMAKE_C_FLAGS+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}"
 691   CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}"
 692   CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 693   CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 694   CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 695   strip_dead_code
 696   cmake_configure "$@" \
 697     -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \
 698     -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_WARNINGS_AS_ERRORS=OFF \
 699     -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}"
 700   cmake_build_and_test
 701 }
 702
 703 # Install libc++ libraries compiled with msan in the msan_prefix for the current
 704 # compiler version.
 705 cmd_msan_install() {
 706   local tmpdir=$(mktemp -d)
 707   CLEANUP_FILES+=("${tmpdir}")
 708   # Detect the llvm to install:
 709   export CC="${CC:-clang}"
 710   export CXX="${CXX:-clang++}"
 711   detect_clang_version
 712   # Allow overriding the LLVM checkout.
 713   local llvm_root="${LLVM_ROOT:-}"
 714   if [ -z "${llvm_root}" ]; then
 715     local llvm_tag="llvmorg-${CLANG_VERSION}.0.0"
 716     case "${CLANG_VERSION}" in
 717       "6.0")
 718         llvm_tag="llvmorg-6.0.1"
 719         ;;
 720       "7")
 721         llvm_tag="llvmorg-7.0.1"
 722         ;;
 723     esac
 724     local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz"
 725     curl -L --show-error -o "${llvm_targz}" \
 726       "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz"
 727     tar -C "${tmpdir}" -zxf "${llvm_targz}"
 728     llvm_root="${tmpdir}/llvm-project-${llvm_tag}"
 729   fi
 730
 731   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 732   rm -rf "${msan_prefix}"
 733
 734   declare -A CMAKE_EXTRAS
 735   CMAKE_EXTRAS[libcxx]="\
 736     -DLIBCXX_CXX_ABI=libstdc++ \
 737     -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON"
 738
 739   for project in libcxx; do
 740     local proj_build="${tmpdir}/build-${project}"
 741     local proj_dir="${llvm_root}/${project}"
 742     mkdir -p "${proj_build}"
 743     cmake -B"${proj_build}" -H"${proj_dir}" \
 744       -G Ninja \
 745       -DCMAKE_BUILD_TYPE=Release \
 746       -DLLVM_USE_SANITIZER=Memory \
 747       -DLLVM_PATH="${llvm_root}/llvm" \
 748       -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \
 749                             head -n1)" \
 750       -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
 751       -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
 752       -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
 753       -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \
 754       -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \
 755       ${CMAKE_EXTRAS[${project}]}
 756     cmake --build "${proj_build}"
 757     ninja -C "${proj_build}" install
 758   done
 759 }
 760
 761 # Internal build step shared between all cmd_ossfuzz_* commands.
 762 _cmd_ossfuzz() {
 763   local sanitizer="$1"
 764   shift
 765   mkdir -p "${BUILD_DIR}"
 766   local real_build_dir=$(realpath "${BUILD_DIR}")
 767
 768   # oss-fuzz defines three directories:
 769   # * /work, with the working directory to do re-builds
 770   # * /src, with the source code to build
 771   # * /out, with the output directory where to copy over the built files.
 772   # We use $BUILD_DIR as the /work and the script directory as the /src. The
 773   # /out directory is ignored as developers are used to look for the fuzzers in
 774   # $BUILD_DIR/tools/ directly.
 775
 776   if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then
 777     sudo docker run --rm -i \
 778       --user $(id -u):$(id -g) \
 779       -v "${real_build_dir}":/work \
 780       gcr.io/oss-fuzz-base/msan-libs-builder \
 781       bash -c "cp -r /msan /work"
 782   fi
 783
 784   # Args passed to ninja. These will be evaluated as a string separated by
 785   # spaces.
 786   local jpegxl_extra_args="$@"
 787
 788   sudo docker run --rm -i \
 789     -e JPEGXL_UID=$(id -u) \
 790     -e JPEGXL_GID=$(id -g) \
 791     -e FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" \
 792     -e SANITIZER="${sanitizer}" \
 793     -e ARCHITECTURE=x86_64 \
 794     -e FUZZING_LANGUAGE=c++ \
 795     -e MSAN_LIBS_PATH="/work/msan" \
 796     -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \
 797     -v "${MYDIR}":/src/libjxl \
 798     -v "${MYDIR}/tools/ossfuzz-build.sh":/src/build.sh \
 799     -v "${real_build_dir}":/work \
 800     gcr.io/oss-fuzz/libjxl
 801 }
 802
 803 cmd_ossfuzz_asan() {
 804   _cmd_ossfuzz address "$@"
 805 }
 806 cmd_ossfuzz_msan() {
 807   _cmd_ossfuzz memory "$@"
 808 }
 809 cmd_ossfuzz_ubsan() {
 810   _cmd_ossfuzz undefined "$@"
 811 }
 812
 813 cmd_ossfuzz_ninja() {
 814   [[ -e "${BUILD_DIR}/build.ninja" ]]
 815   local real_build_dir=$(realpath "${BUILD_DIR}")
 816
 817   if [[ -e "${BUILD_DIR}/msan" ]]; then
 818     echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2
 819     exit 1
 820   fi
 821
 822   sudo docker run --rm -i \
 823     --user $(id -u):$(id -g) \
 824     -v "${MYDIR}":/src/libjxl \
 825     -v "${real_build_dir}":/work \
 826     gcr.io/oss-fuzz/libjxl \
 827     ninja -C /work "$@"
 828 }
 829
 830 cmd_fast_benchmark() {
 831   local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar"
 832   mkdir -p "${BENCHMARK_CORPORA}"
 833   curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \
 834     "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
 835
 836   local tmpdir=$(mktemp -d)
 837   CLEANUP_FILES+=("${tmpdir}")
 838   tar -xf "${small_corpus_tar}" -C "${tmpdir}"
 839
 840   run_benchmark "${tmpdir}" 1048576
 841 }
 842
 843 cmd_benchmark() {
 844   local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar"
 845   mkdir -p "${BENCHMARK_CORPORA}"
 846   curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \
 847     "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar"
 848
 849   local tmpdir=$(mktemp -d)
 850   CLEANUP_FILES+=("${tmpdir}")
 851   tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}"
 852
 853   local sem_id="jpegxl_benchmark-$$"
 854   local nprocs=$(nproc --all || echo 1)
 855   images=()
 856   local filename
 857   while IFS= read -r filename; do
 858     # This removes the './'
 859     filename="${filename:2}"
 860     local mode
 861     if [[ "${filename:0:4}" == "srgb" ]]; then
 862       mode="RGB_D65_SRG_Rel_SRG"
 863     elif [[ "${filename:0:5}" == "adobe" ]]; then
 864       mode="RGB_D65_Ado_Rel_Ado"
 865     else
 866       echo "Unknown image colorspace: ${filename}" >&2
 867       exit 1
 868     fi
 869     png_filename="${filename%.ppm}.png"
 870     png_filename=$(echo "${png_filename}" | tr '/' '_')
 871     sem --bg --id "${sem_id}" -j"${nprocs}" -- \
 872       "${BUILD_DIR}/tools/decode_and_encode" \
 873         "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}"
 874     images+=( "${png_filename}" )
 875   done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f)
 876   sem --id "${sem_id}" --wait
 877
 878   # We need about 10 GiB per thread on these images.
 879   run_benchmark "${tmpdir}" 10485760
 880 }
 881
 882 get_mem_available() {
 883   if [[ "${OS}" == "Darwin" ]]; then
 884     echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}')
 885   else
 886     echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}')
 887   fi
 888 }
 889
 890 run_benchmark() {
 891   local src_img_dir="$1"
 892   local mem_per_thread="${2:-10485760}"
 893
 894   local output_dir="${BUILD_DIR}/benchmark_results"
 895   mkdir -p "${output_dir}"
 896
 897   # The memory available at the beginning of the benchmark run in kB. The number
 898   # of threads depends on the available memory, and the passed memory per
 899   # thread. We also add a 2 GiB of constant memory.
 900   local mem_available="$(get_mem_available)"
 901   # Check that we actually have a MemAvailable value.
 902   [[ -n "${mem_available}" ]]
 903   local num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} ))
 904   if [[ ${num_threads} -le 0 ]]; then
 905     num_threads=1
 906   fi
 907
 908   local benchmark_args=(
 909     --input "${src_img_dir}/*.png"
 910     --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2
 911     --output_dir "${output_dir}"
 912     --noprofiler --show_progress
 913     --num_threads="${num_threads}"
 914   )
 915   if [[ "${STORE_IMAGES}" == "1" ]]; then
 916     benchmark_args+=(--save_decompressed --save_compressed)
 917   fi
 918   (
 919     [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 920     "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \
 921        tee "${output_dir}/results.txt"
 922
 923     # Check error code for benckmark_xl command. This will exit if not.
 924     return ${PIPESTATUS[0]}
 925   )
 926
 927   if [[ -n "${CI_BUILD_NAME:-}" ]]; then
 928     { set +x; } 2>/dev/null
 929     local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}):
 930
 931 $(cat "${output_dir}/results.txt")
 932 "
 933     cmd_post_mr_comment "${message}"
 934     set -x
 935   fi
 936 }
 937
 938 # Helper function to wait for the CPU temperature to cool down on ARM.
 939 wait_for_temp() {
 940   { set +x; } 2>/dev/null
 941   local temp_limit=${1:-38000}
 942   if [[ -z "${THERMAL_FILE:-}" ]]; then
 943     echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \
 944       "to read the temperature from. This is normally set in the runner." >&2
 945     exit 1
 946   fi
 947   local org_temp=$(cat "${THERMAL_FILE}")
 948   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 949     echo -n "Waiting for temp to get down from ${org_temp}... "
 950   fi
 951   local temp="${org_temp}"
 952   local secs=0
 953   while [[ "${temp}" -ge "${temp_limit}" ]]; do
 954     sleep 1
 955     temp=$(cat "${THERMAL_FILE}")
 956     echo -n "${temp} "
 957     secs=$((secs + 1))
 958     if [[ ${secs} -ge 5 ]]; then
 959       break
 960     fi
 961   done
 962   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 963     echo "Done, temp=${temp}"
 964   fi
 965   set -x
 966 }
 967
 968 # Helper function to set the cpuset restriction of the current process.
 969 cmd_cpuset() {
 970   [[ "${SKIP_CPUSET:-}" != "1" ]] || return 0
 971   local newset="$1"
 972   local mycpuset=$(cat /proc/self/cpuset)
 973   mycpuset="/dev/cpuset${mycpuset}"
 974   # Check that the directory exists:
 975   [[ -d "${mycpuset}" ]]
 976   if [[ -e "${mycpuset}/cpuset.cpus" ]]; then
 977     echo "${newset}" >"${mycpuset}/cpuset.cpus"
 978   else
 979     echo "${newset}" >"${mycpuset}/cpus"
 980   fi
 981 }
 982
 983 # Return the encoding/decoding speed from the Stats output.
 984 _speed_from_output() {
 985   local speed="$1"
 986   local unit="${2:-MP/s}"
 987   if [[ "${speed}" == *"${unit}"* ]]; then
 988     speed="${speed%% ${unit}*}"
 989     speed="${speed##* }"
 990     echo "${speed}"
 991   fi
 992 }
 993
 994
 995 # Run benchmarks on ARM for the big and little CPUs.
 996 cmd_arm_benchmark() {
 997   # Flags used for cjxl encoder with .png inputs
 998   local jxl_png_benchmarks=(
 999     # Lossy options:
1000     "--epf=0 --distance=1.0 --speed=cheetah"
1001     "--epf=2 --distance=1.0 --speed=cheetah"
1002     "--epf=0 --distance=8.0 --speed=cheetah"
1003     "--epf=1 --distance=8.0 --speed=cheetah"
1004     "--epf=2 --distance=8.0 --speed=cheetah"
1005     "--epf=3 --distance=8.0 --speed=cheetah"
1006     "--modular -Q 90"
1007     "--modular -Q 50"
1008     # Lossless options:
1009     "--modular"
1010     "--modular -E 0 -I 0"
1011     "--modular -P 5"
1012     "--modular --responsive=1"
1013     # Near-lossless options:
1014     "--epf=0 --distance=0.3 --speed=fast"
1015     "--modular -Q 97"
1016   )
1017
1018   # Flags used for cjxl encoder with .jpg inputs. These should do lossless
1019   # JPEG recompression (of pixels or full jpeg).
1020   local jxl_jpeg_benchmarks=(
1021     "--num_reps=3"
1022   )
1023
1024   local images=(
1025     "testdata/jxl/flower/flower.png"
1026   )
1027
1028   local jpg_images=(
1029     "testdata/jxl/flower/flower.png.im_q85_420.jpg"
1030   )
1031
1032   if [[ "${SKIP_CPUSET:-}" == "1" ]]; then
1033     # Use a single cpu config in this case.
1034     local cpu_confs=("?")
1035   else
1036     # Otherwise the CPU config comes from the environment:
1037     local cpu_confs=(
1038       "${RUNNER_CPU_LITTLE}"
1039       "${RUNNER_CPU_BIG}"
1040       # The CPU description is something like 3-7, so these configurations only
1041       # take the first CPU of the group.
1042       "${RUNNER_CPU_LITTLE%%-*}"
1043       "${RUNNER_CPU_BIG%%-*}"
1044     )
1045     # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will
1046     # be ignored but still evaluated when calling cmd_cpuset.
1047     [[ -n "${RUNNER_CPU_ALL}" ]]
1048   fi
1049
1050   local jpg_dirname="third_party/corpora/jpeg"
1051   mkdir -p "${jpg_dirname}"
1052   local jpg_qualities=( 50 80 95 )
1053   for src_img in "${images[@]}"; do
1054     for q in "${jpg_qualities[@]}"; do
1055       local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg"
1056       convert -sampling-factor 1x1 -quality "${q}" \
1057         "${src_img}" "${jpeg_name}"
1058       jpg_images+=("${jpeg_name}")
1059     done
1060   done
1061
1062   local output_dir="${BUILD_DIR}/benchmark_results"
1063   mkdir -p "${output_dir}"
1064   local runs_file="${output_dir}/runs.txt"
1065
1066   if [[ ! -e "${runs_file}" ]]; then
1067     echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" |
1068       tee -a "${runs_file}"
1069   fi
1070
1071   mkdir -p "${BUILD_DIR}/arm_benchmark"
1072   local flags
1073   local src_img
1074   for src_img in "${jpg_images[@]}" "${images[@]}"; do
1075     local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ')
1076     local enc_binaries=("${BUILD_DIR}/tools/cjxl")
1077     local src_ext="${src_img##*.}"
1078     for enc_binary in "${enc_binaries[@]}"; do
1079       local enc_binary_base=$(basename "${enc_binary}")
1080
1081       # Select the list of flags to use for the current encoder/image pair.
1082       local img_benchmarks
1083       if [[ "${src_ext}" == "jpg" ]]; then
1084         img_benchmarks=("${jxl_jpeg_benchmarks[@]}")
1085       else
1086         img_benchmarks=("${jxl_png_benchmarks[@]}")
1087       fi
1088
1089       for flags in "${img_benchmarks[@]}"; do
1090         # Encoding step.
1091         local enc_file_hash="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}"
1092         enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ')
1093         local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl"
1094
1095         for cpu_conf in "${cpu_confs[@]}"; do
1096           cmd_cpuset "${cpu_conf}"
1097           # nproc returns the number of active CPUs, which is given by the cpuset
1098           # mask.
1099           local num_threads="$(nproc)"
1100
1101           echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1102           local enc_output
1103           if [[ "${flags}" == *"modular"* ]]; then
1104             # We don't benchmark encoding speed in this case.
1105             if [[ ! -f "${enc_file}" ]]; then
1106               cmd_cpuset "${RUNNER_CPU_ALL:-}"
1107               "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp"
1108               mv "${enc_file}.tmp" "${enc_file}"
1109               cmd_cpuset "${cpu_conf}"
1110             fi
1111             enc_output=" ?? MP/s"
1112           else
1113             wait_for_temp
1114             enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \
1115               2>&1 | tee /dev/stderr | grep -F "MP/s [")
1116             mv "${enc_file}.tmp" "${enc_file}"
1117           fi
1118           local enc_speed=$(_speed_from_output "${enc_output}")
1119           local enc_size=$(stat -c "%s" "${enc_file}")
1120
1121           echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1122
1123           local dec_output
1124           wait_for_temp
1125           dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1126             --num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr |
1127             grep -E "M[BP]/s \[")
1128           local img_size=$(echo "${dec_output}" | cut -f 1 -d ',')
1129           local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ')
1130           local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ')
1131           local img_size_px=$(( ${img_size_x} * ${img_size_y} ))
1132           local dec_speed=$(_speed_from_output "${dec_output}")
1133
1134           # For JPEG lossless recompression modes (where the original is a JPEG)
1135           # decode to JPG as well.
1136           local jpeg_dec_mps_speed=""
1137           local jpeg_dec_mbs_speed=""
1138           if [[ "${src_ext}" == "jpg" ]]; then
1139             wait_for_temp
1140             local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg"
1141             dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1142               "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \
1143                 tee /dev/stderr | grep -E "M[BP]/s \[")
1144             local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}")
1145             local jpeg_dec_mbs_speed=$(_speed_from_output "${dec_output}" MB/s)
1146             if ! cmp --quiet "${src_img}" "${dec_file}"; then
1147               # Add a start at the end to signal that the files are different.
1148               jpeg_dec_mbs_speed+="*"
1149             fi
1150           fi
1151
1152           # Record entry in a tab-separated file.
1153           local src_img_base=$(basename "${src_img}")
1154           echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" |
1155             tee -a "${runs_file}"
1156         done
1157       done
1158     done
1159   done
1160   cmd_cpuset "${RUNNER_CPU_ALL:-}"
1161   cat "${runs_file}"
1162
1163   if [[ -n "${CI_BUILD_NAME:-}" ]]; then
1164     load_mr_vars_from_commit
1165     { set +x; } 2>/dev/null
1166     local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}):
1167
1168 \`\`\`
1169 $(column -t -s "        " "${runs_file}")
1170 \`\`\`
1171 "
1172     cmd_post_mr_comment "${message}"
1173     set -x
1174   fi
1175 }
1176
1177 # Generate a corpus and run the fuzzer on that corpus.
1178 cmd_fuzz() {
1179   local corpus_dir=$(realpath "${BUILD_DIR}/fuzzer_corpus")
1180   local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash")
1181   mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}"
1182   # Generate step.
1183   "${BUILD_DIR}/tools/fuzzer_corpus" "${corpus_dir}"
1184   # Run step:
1185   local nprocs=$(nproc --all || echo 1)
1186   (
1187    cd "${BUILD_DIR}"
1188    "tools/djxl_fuzzer" "${fuzzer_crash_dir}" "${corpus_dir}" \
1189      -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \
1190      -artifact_prefix="${fuzzer_crash_dir}/"
1191   )
1192 }
1193
1194 # Runs the linter (clang-format) on the pending CLs.
1195 cmd_lint() {
1196   merge_request_commits
1197   { set +x; } 2>/dev/null
1198   local versions=(${1:-6.0 7 8 9 10 11})
1199   local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format)
1200   local tmpdir=$(mktemp -d)
1201   CLEANUP_FILES+=("${tmpdir}")
1202
1203   local ret=0
1204   local build_patch="${tmpdir}/build_cleaner.patch"
1205   if ! "${MYDIR}/tools/build_cleaner.py" >"${build_patch}"; then
1206     ret=1
1207     echo "build_cleaner.py findings:" >&2
1208     "${COLORDIFF_BIN}" <"${build_patch}"
1209     echo "Run \`tools/build_cleaner.py --update\` to apply them" >&2
1210   fi
1211
1212   local installed=()
1213   local clang_patch
1214   local clang_format
1215   for clang_format in "${clang_format_bins[@]}"; do
1216     if ! which "${clang_format}" >/dev/null; then
1217       continue
1218     fi
1219     installed+=("${clang_format}")
1220     local tmppatch="${tmpdir}/${clang_format}.patch"
1221     # We include in this linter all the changes including the uncommitted changes
1222     # to avoid printing changes already applied.
1223     set -x
1224     # Ignoring the error that git-clang-format outputs.
1225     git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \
1226       --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true
1227     { set +x; } 2>/dev/null
1228     if grep -E '^--- ' "${tmppatch}">/dev/null; then
1229       if [[ -n "${LINT_OUTPUT:-}" ]]; then
1230         cp "${tmppatch}" "${LINT_OUTPUT}"
1231       fi
1232       clang_patch="${tmppatch}"
1233     else
1234       echo "clang-format check OK" >&2
1235       return ${ret}
1236     fi
1237   done
1238
1239   if [[ ${#installed[@]} -eq 0 ]]; then
1240     echo "You must install clang-format for \"git clang-format\"" >&2
1241     exit 1
1242   fi
1243
1244   # clang-format is installed but found problems.
1245   echo "clang-format findings:" >&2
1246   "${COLORDIFF_BIN}" < "${clang_patch}"
1247
1248   echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \
1249     "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \
1250     "directory to apply them." >&2
1251   exit 1
1252 }
1253
1254 # Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs
1255 # clang-tidy over all the source files instead.
1256 cmd_tidy() {
1257   local what="${1:-}"
1258
1259   if [[ -z "${CLANG_TIDY_BIN}" ]]; then
1260     echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2
1261     exit 1
1262   fi
1263
1264   local git_args=()
1265   if [[ "${what}" == "all" ]]; then
1266     git_args=(ls-files)
1267     shift
1268   else
1269     merge_request_commits
1270     git_args=(
1271         diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}"
1272         "${MR_HEAD_SHA}"
1273     )
1274   fi
1275
1276   # Clang-tidy needs the compilation database generated by cmake.
1277   if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then
1278     # Generate the build options in debug mode, since we need the debug asserts
1279     # enabled for the clang-tidy analyzer to use them.
1280     CMAKE_BUILD_TYPE="Debug"
1281     cmake_configure
1282     # Build the autogen targets to generate the .h files from the .ui files.
1283     local autogen_targets=(
1284         $(ninja -C "${BUILD_DIR}" -t targets | grep -F _autogen: |
1285           cut -f 1 -d :)
1286     )
1287     if [[ ${#autogen_targets[@]} != 0 ]]; then
1288       ninja -C "${BUILD_DIR}" "${autogen_targets[@]}"
1289     fi
1290   fi
1291
1292   cd "${MYDIR}"
1293   local nprocs=$(nproc --all || echo 1)
1294   local ret=0
1295   if ! parallel -j"${nprocs}" --keep-order -- \
1296       "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \
1297       < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \
1298       >"${BUILD_DIR}/clang-tidy.txt"; then
1299     ret=1
1300   fi
1301   { set +x; } 2>/dev/null
1302   echo "Findings statistics:" >&2
1303   grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \
1304     | uniq -c >&2
1305
1306   if [[ $ret -ne 0 ]]; then
1307     cat >&2 <<EOF
1308 Errors found, see ${BUILD_DIR}/clang-tidy.txt for details.
1309 To automatically fix them, run:
1310
1311   SKIP_TEST=1 ./ci.sh debug
1312   ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$')
1313 EOF
1314   fi
1315
1316   return ${ret}
1317 }
1318
1319 # Print stats about all the packages built in ${BUILD_DIR}/debs/.
1320 cmd_debian_stats() {
1321   { set +x; } 2>/dev/null
1322   local debsdir="${BUILD_DIR}/debs"
1323   local f
1324   while IFS='' read -r -d '' f; do
1325     echo "====================================================================="
1326     echo "Package $f:"
1327     dpkg --info $f
1328     dpkg --contents $f
1329   done < <(find "${BUILD_DIR}/debs" -maxdepth 1 -mindepth 1 -type f \
1330            -name '*.deb' -print0)
1331 }
1332
1333 build_debian_pkg() {
1334   local srcdir="$1"
1335   local srcpkg="$2"
1336
1337   local debsdir="${BUILD_DIR}/debs"
1338   local builddir="${debsdir}/${srcpkg}"
1339
1340   # debuild doesn't have an easy way to build out of tree, so we make a copy
1341   # of with all symlinks on the first level.
1342   mkdir -p "${builddir}"
1343   for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do
1344     if [[ ! -L "${builddir}/$f" ]]; then
1345       rm -f "${builddir}/$f"
1346       ln -s "${srcdir}/$f" "${builddir}/$f"
1347     fi
1348   done
1349   (
1350     cd "${builddir}"
1351     debuild -b -uc -us
1352   )
1353 }
1354
1355 cmd_debian_build() {
1356   local srcpkg="${1:-}"
1357
1358   case "${srcpkg}" in
1359     jpeg-xl)
1360       build_debian_pkg "${MYDIR}" "jpeg-xl"
1361       ;;
1362     highway)
1363       build_debian_pkg "${MYDIR}/third_party/highway" "highway"
1364       ;;
1365     *)
1366       echo "ERROR: Must pass a valid source package name to build." >&2
1367       ;;
1368   esac
1369 }
1370
1371 get_version() {
1372   local varname=$1
1373   local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1)
1374   [[ -n "${line}" ]]
1375   line="${line#set(${varname} }"
1376   line="${line%)}"
1377   echo "${line}"
1378 }
1379
1380 cmd_bump_version() {
1381   local newver="${1:-}"
1382
1383   if ! which dch >/dev/null; then
1384     echo "Run:\n  sudo apt install debhelper"
1385     exit 1
1386   fi
1387
1388   if [[ -z "${newver}" ]]; then
1389     local major=$(get_version JPEGXL_MAJOR_VERSION)
1390     local minor=$(get_version JPEGXL_MINOR_VERSION)
1391     local patch=0
1392     minor=$(( ${minor}  + 1))
1393   else
1394     local major="${newver%%.*}"
1395     newver="${newver#*.}"
1396     local minor="${newver%%.*}"
1397     newver="${newver#${minor}}"
1398     local patch="${newver#.}"
1399     if [[ -z "${patch}" ]]; then
1400       patch=0
1401     fi
1402   fi
1403
1404   newver="${major}.${minor}.${patch}"
1405
1406   echo "Bumping version to ${newver} (${major}.${minor}.${patch})"
1407   sed -E \
1408     -e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \
1409     -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \
1410     -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \
1411     -i lib/CMakeLists.txt
1412   sed -E \
1413     -e "s/(LIBJXL_VERSION: )[0-9\\.]+/\\1 ${major}.${minor}.${patch}/" \
1414     -e "s/(LIBJXL_ABI_VERSION: )[0-9\\.]+/\\1 ${major}.${minor}/" \
1415     -i .github/workflows/conformance.yml
1416
1417   # Update lib.gni
1418   tools/build_cleaner.py --update
1419
1420   # Mark the previous version as "unstable".
1421   DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release ''
1422   DEBCHANGE_RELEASE_HEURISTIC=log dch -M \
1423     --newversion "${newver}" \
1424     "Bump JPEG XL version to ${newver}."
1425 }
1426
1427 # Check that the AUTHORS file contains the email of the committer.
1428 cmd_authors() {
1429   merge_request_commits
1430   local emails
1431   local names
1432   readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1433   readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1434   for i in "${!names[@]}"; do
1435     echo "Checking name '${names[$i]}' with email '${emails[$i]}' ..."
1436     "${MYDIR}"/tools/check_author.py "${emails[$i]}" "${names[$i]}"
1437   done
1438 }
1439
1440 main() {
1441   local cmd="${1:-}"
1442   if [[ -z "${cmd}" ]]; then
1443     cat >&2 <<EOF
1444 Use: $0 CMD
1445
1446 Where cmd is one of:
1447  opt       Build and test a Release with symbols build.
1448  debug     Build and test a Debug build (NDEBUG is not defined).
1449  release   Build and test a striped Release binary without debug information.
1450  asan      Build and test an ASan (AddressSanitizer) build.
1451  msan      Build and test an MSan (MemorySanitizer) build. Needs to have msan
1452            c++ libs installed with msan_install first.
1453  tsan      Build and test a TSan (ThreadSanitizer) build.
1454  asanfuzz  Build and test an ASan (AddressSanitizer) build for fuzzing.
1455  msanfuzz  Build and test an MSan (MemorySanitizer) build for fuzzing.
1456  test      Run the tests build by opt, debug, release, asan or msan. Useful when
1457            building with SKIP_TEST=1.
1458  gbench    Run the Google benchmark tests.
1459  fuzz      Generate the fuzzer corpus and run the fuzzer on it. Useful after
1460            building with asan or msan.
1461  benchmark Run the benchmark over the default corpus.
1462  fast_benchmark Run the benchmark over the small corpus.
1463
1464  coverage  Buils and run tests with coverage support. Runs coverage_report as
1465            well.
1466  coverage_report Generate HTML, XML and text coverage report after a coverage
1467            run.
1468
1469  lint      Run the linter checks on the current commit or merge request.
1470  tidy      Run clang-tidy on the current commit or merge request.
1471  authors   Check that the last commit's author is listed in the AUTHORS file.
1472
1473  msan_install Install the libc++ libraries required to build in msan mode. This
1474               needs to be done once.
1475
1476  debian_build <srcpkg> Build the given source package.
1477  debian_stats  Print stats about the built packages.
1478
1479 oss-fuzz commands:
1480  ossfuzz_asan   Build the local source inside oss-fuzz docker with asan.
1481  ossfuzz_msan   Build the local source inside oss-fuzz docker with msan.
1482  ossfuzz_ubsan  Build the local source inside oss-fuzz docker with ubsan.
1483  ossfuzz_ninja  Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra
1484                 parameters are passed to ninja, for example "djxl_fuzzer" will
1485                 only build that ninja target. Use for faster build iteration
1486                 after one of the ossfuzz_*san commands.
1487
1488 You can pass some optional environment variables as well:
1489  - BUILD_DIR: The output build directory (by default "$$repo/build")
1490  - BUILD_TARGET: The target triplet used when cross-compiling.
1491  - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS.
1492  - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package.
1493  - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only).
1494  - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds.
1495  - LINT_OUTPUT: Path to the output patch from the "lint" command.
1496  - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark.
1497  - SKIP_TEST=1: Skip the test stage.
1498  - STORE_IMAGES=0: Makes the benchmark discard the computed images.
1499  - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB.
1500  - TEST_SELECTOR: pass additional arguments to ctest, e.g. "-R .Resample.".
1501  - STACK_SIZE=1: Generate binaries with the .stack_sizes sections.
1502
1503 These optional environment variables are forwarded to the cmake call as
1504 parameters:
1505  - CMAKE_BUILD_TYPE
1506  - CMAKE_C_FLAGS
1507  - CMAKE_CXX_FLAGS
1508  - CMAKE_C_COMPILER_LAUNCHER
1509  - CMAKE_CXX_COMPILER_LAUNCHER
1510  - CMAKE_CROSSCOMPILING_EMULATOR
1511  - CMAKE_FIND_ROOT_PATH
1512  - CMAKE_EXE_LINKER_FLAGS
1513  - CMAKE_MAKE_PROGRAM
1514  - CMAKE_MODULE_LINKER_FLAGS
1515  - CMAKE_SHARED_LINKER_FLAGS
1516  - CMAKE_TOOLCHAIN_FILE
1517
1518 Example:
1519   BUILD_DIR=/tmp/build $0 opt
1520 EOF
1521     exit 1
1522   fi
1523
1524   cmd="cmd_${cmd}"
1525   shift
1526   set -x
1527   "${cmd}" "$@"
1528 }
1529
1530 main "$@"