third_party/jpeg-xl/ci.sh

   1 #!/usr/bin/env bash
   2 # Copyright (c) the JPEG XL Project Authors. All rights reserved.
   3 #
   4 # Use of this source code is governed by a BSD-style
   5 # license that can be found in the LICENSE file.
   6
   7 # Continuous integration helper module. This module is meant to be called from
   8 # the .gitlab-ci.yml file during the continuous integration build, as well as
   9 # from the command line for developers.
  10
  11 set -eu
  12
  13 OS=`uname -s`
  14
  15 MYDIR=$(dirname $(realpath "$0"))
  16
  17 ### Environment parameters:
  18 TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-256}"
  19 CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
  20 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}
  21 CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-}
  22 CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-}
  23 CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-}
  24 SKIP_BUILD="${SKIP_BUILD:-0}"
  25 SKIP_TEST="${SKIP_TEST:-0}"
  26 FASTER_MSAN_BUILD="${FASTER_MSAN_BUILD:-0}"
  27 TARGETS="${TARGETS:-all doc}"
  28 TEST_SELECTOR="${TEST_SELECTOR:-}"
  29 BUILD_TARGET="${BUILD_TARGET:-}"
  30 ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}"
  31 if [[ -n "${BUILD_TARGET}" ]]; then
  32   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}"
  33 else
  34   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}"
  35 fi
  36 # Whether we should post a message in the MR when the build fails.
  37 POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}"
  38 # By default, do a lightweight debian HWY package build.
  39 HWY_PKG_OPTIONS="${HWY_PKG_OPTIONS:---set-envvar=HWY_EXTRA_CONFIG=-DBUILD_TESTING=OFF -DHWY_ENABLE_EXAMPLES=OFF -DHWY_ENABLE_CONTRIB=OFF}"
  40
  41 # Set default compilers to clang if not already set
  42 export CC=${CC:-clang}
  43 export CXX=${CXX:-clang++}
  44
  45 # Time limit for the "fuzz" command in seconds (0 means no limit).
  46 FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}"
  47
  48 SANITIZER="none"
  49
  50
  51 if [[ "${BUILD_TARGET%%-*}" == "x86_64" ||
  52     "${BUILD_TARGET%%-*}" == "i686" ]]; then
  53   # Default to building all targets, even if compiler baseline is SSE4
  54   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_EMU128}
  55 else
  56   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-}
  57 fi
  58
  59 # Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
  60 CMAKE_FLAGS=${CMAKE_FLAGS:-}
  61 CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}"
  62 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}"
  63
  64 CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-}
  65 CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-}
  66 CMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH:-}
  67 CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-}
  68 CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-}
  69 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}
  70
  71 if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then
  72   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -msimd128"
  73   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -msimd128"
  74   CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128"
  75 fi
  76
  77 if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then
  78   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2"
  79   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2"
  80 fi
  81
  82 if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then
  83   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}"
  84 fi
  85
  86 # Version inferred from the CI variables.
  87 CI_COMMIT_SHA=${CI_COMMIT_SHA:-${GITHUB_SHA:-}}
  88 JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}}
  89
  90 # Benchmark parameters
  91 STORE_IMAGES=${STORE_IMAGES:-1}
  92 BENCHMARK_CORPORA="${MYDIR}/third_party/corpora"
  93
  94 # Local flags passed to sanitizers.
  95 UBSAN_FLAGS=(
  96   -fsanitize=alignment
  97   -fsanitize=bool
  98   -fsanitize=bounds
  99   -fsanitize=builtin
 100   -fsanitize=enum
 101   -fsanitize=float-cast-overflow
 102   -fsanitize=float-divide-by-zero
 103   -fsanitize=integer-divide-by-zero
 104   -fsanitize=null
 105   -fsanitize=object-size
 106   -fsanitize=pointer-overflow
 107   -fsanitize=return
 108   -fsanitize=returns-nonnull-attribute
 109   -fsanitize=shift-base
 110   -fsanitize=shift-exponent
 111   -fsanitize=unreachable
 112   -fsanitize=vla-bound
 113
 114   -fno-sanitize-recover=undefined
 115   # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning.
 116   -fsanitize-recover=alignment
 117 )
 118 # -fsanitize=function doesn't work on aarch64 and arm.
 119 if [[ "${BUILD_TARGET%%-*}" != "aarch64" &&
 120     "${BUILD_TARGET%%-*}" != "arm" ]]; then
 121   UBSAN_FLAGS+=(
 122     -fsanitize=function
 123   )
 124 fi
 125 if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then
 126   UBSAN_FLAGS+=(
 127     -fsanitize=signed-integer-overflow
 128   )
 129 fi
 130
 131 CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy | head -n 1)
 132 # Default to "cat" if "colordiff" is not installed or if stdout is not a tty.
 133 if [[ -t 1 ]]; then
 134   COLORDIFF_BIN=$(which colordiff cat | head -n 1)
 135 else
 136   COLORDIFF_BIN="cat"
 137 fi
 138 FIND_BIN=$(which gfind find | head -n 1)
 139 # "false" will disable wine64 when not installed. This won't allow
 140 # cross-compiling.
 141 WINE_BIN=$(which wine64 false | head -n 1)
 142
 143 CLANG_VERSION="${CLANG_VERSION:-}"
 144 # Detect the clang version suffix and store it in CLANG_VERSION. For example,
 145 # "6.0" for clang 6 or "7" for clang 7.
 146 detect_clang_version() {
 147   if [[ -n "${CLANG_VERSION}" ]]; then
 148     return 0
 149   fi
 150   local clang_version=$("${CC:-clang}" --version | head -n1)
 151   clang_version=${clang_version#"Debian "}
 152   clang_version=${clang_version#"Ubuntu "}
 153   local llvm_tag
 154   case "${clang_version}" in
 155     "clang version 6."*)
 156       CLANG_VERSION="6.0"
 157       ;;
 158     "clang version "*)
 159       # Any other clang version uses just the major version number.
 160       local suffix="${clang_version#clang version }"
 161       CLANG_VERSION="${suffix%%.*}"
 162       ;;
 163     "emcc"*)
 164       # We can't use asan or msan in the emcc case.
 165       ;;
 166     *)
 167       echo "Unknown clang version: ${clang_version}" >&2
 168       return 1
 169   esac
 170 }
 171
 172 # Temporary files cleanup hooks.
 173 CLEANUP_FILES=()
 174 cleanup() {
 175   if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
 176     rm -fr "${CLEANUP_FILES[@]}"
 177   fi
 178 }
 179
 180 # Executed on exit.
 181 on_exit() {
 182   local retcode="$1"
 183   # Always cleanup the CLEANUP_FILES.
 184   cleanup
 185
 186   # Post a message in the MR when requested with POST_MESSAGE_ON_ERROR but only
 187   # if the run failed and we are not running from a MR pipeline.
 188   if [[ ${retcode} -ne 0 && -n "${CI_BUILD_NAME:-}" &&
 189         -n "${POST_MESSAGE_ON_ERROR}" && -z "${CI_MERGE_REQUEST_ID:-}" &&
 190         "${CI_BUILD_REF_NAME}" = "master" ]]; then
 191     load_mr_vars_from_commit
 192     { set +xeu; } 2>/dev/null
 193     local message="**Run ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} failed.**
 194
 195 Check the output of the job at ${CI_JOB_URL:-} to see if this was your problem.
 196 If it was, please rollback this change or fix the problem ASAP, broken builds
 197 slow down development. Check if the error already existed in the previous build
 198 as well.
 199
 200 Pipeline: ${CI_PIPELINE_URL}
 201
 202 Previous build commit: ${CI_COMMIT_BEFORE_SHA}
 203 "
 204     cmd_post_mr_comment "${message}"
 205   fi
 206 }
 207
 208 trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT
 209
 210
 211 # These variables are populated when calling merge_request_commits().
 212
 213 # The current hash at the top of the current branch or merge request branch (if
 214 # running from a merge request pipeline).
 215 MR_HEAD_SHA=""
 216 # The common ancestor between the current commit and the tracked branch, such
 217 # as master. This includes a list
 218 MR_ANCESTOR_SHA=""
 219
 220 # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA.
 221 merge_request_commits() {
 222   { set +x; } 2>/dev/null
 223   # GITHUB_SHA is the current reference being build in GitHub Actions.
 224   if [[ -n "${GITHUB_SHA:-}" ]]; then
 225     # GitHub normally does a checkout of a merge commit on a shallow repository
 226     # by default. We want to get a bit more of the history to be able to diff
 227     # changes on the Pull Request if needed. This fetches 10 more commits which
 228     # should be enough given that PR normally should have 1 commit.
 229     git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10
 230     MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse HEAD)
 231   else
 232     # CI_BUILD_REF is the reference currently being build in the CI workflow.
 233     MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "${CI_BUILD_REF:-HEAD}")
 234   fi
 235
 236   if [[ -n "${CI_MERGE_REQUEST_IID:-}" ]]; then
 237     # Merge request pipeline in CI. In this case the upstream is called "origin"
 238     # but it refers to the forked project that's the source of the merge
 239     # request. We need to get the target of the merge request, for which we need
 240     # to query that repository using our CI_JOB_TOKEN.
 241     echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" \
 242       >> "${HOME}/.netrc"
 243     git -C "${MYDIR}" fetch "${CI_MERGE_REQUEST_PROJECT_URL}" \
 244       "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
 245     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD)
 246   elif [[ -n "${GITHUB_BASE_REF:-}" ]]; then
 247     # Pull request workflow in GitHub Actions. GitHub checkout action uses
 248     # "origin" as the remote for the git checkout.
 249     git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}"
 250     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD)
 251   else
 252     # We are in a local branch, not a merge request.
 253     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true)
 254   fi
 255
 256   if [[ -z "${MR_ANCESTOR_SHA}" ]]; then
 257     echo "Warning, not tracking any branch, using the last commit in HEAD.">&2
 258     # This prints the return value with just HEAD.
 259     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^")
 260   else
 261     # GitHub runs the pipeline on a merge commit, no need to look for the common
 262     # ancestor in that case.
 263     if [[ -z "${GITHUB_BASE_REF:-}" ]]; then
 264       MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base \
 265         "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}")
 266     fi
 267   fi
 268   set -x
 269 }
 270
 271 # Load the MR iid from the landed commit message when running not from a
 272 # merge request workflow. This is useful to post back results at the merge
 273 # request when running pipelines from master.
 274 load_mr_vars_from_commit() {
 275   { set +x; } 2>/dev/null
 276   if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then
 277     local mr_iid=$(git rev-list --format=%B --max-count=1 HEAD |
 278       grep -F "${CI_PROJECT_URL}" | grep -F "/merge_requests" | head -n 1)
 279     # mr_iid contains a string like this if it matched:
 280     #  Part-of: <https://gitlab.com/wg1/jpeg-xlm/merge_requests/123456>
 281     if [[ -n "${mr_iid}" ]]; then
 282       mr_iid=$(echo "${mr_iid}" |
 283         sed -E 's,^.*merge_requests/([0-9]+)>.*$,\1,')
 284       CI_MERGE_REQUEST_IID="${mr_iid}"
 285       CI_MERGE_REQUEST_PROJECT_ID=${CI_PROJECT_ID}
 286     fi
 287   fi
 288   set -x
 289 }
 290
 291 # Posts a comment to the current merge request.
 292 cmd_post_mr_comment() {
 293   { set +x; } 2>/dev/null
 294   local comment="$1"
 295   if [[ -n "${BOT_TOKEN:-}" && -n "${CI_MERGE_REQUEST_IID:-}" ]]; then
 296     local url="${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes"
 297     curl -X POST -g \
 298       -H "PRIVATE-TOKEN: ${BOT_TOKEN}" \
 299       --data-urlencode "body=${comment}" \
 300       --output /dev/null \
 301       "${url}"
 302   fi
 303   set -x
 304 }
 305
 306 # Set up and export the environment variables needed by the child processes.
 307 export_env() {
 308   if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 309     # Wine needs to know the paths to the mingw dlls. These should be
 310     # separated by ';'.
 311     WINEPATH=$("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \
 312       | grep -F 'libraries: =' | cut -f 2- -d '=' | tr ':' ';')
 313     # We also need our own libraries in the wine path.
 314     local real_build_dir=$(realpath "${BUILD_DIR}")
 315     # Some library .dll dependencies are installed in /bin:
 316     export WINEPATH="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin"
 317
 318     local prefix="${BUILD_DIR}/wineprefix"
 319     mkdir -p "${prefix}"
 320     export WINEPREFIX=$(realpath "${prefix}")
 321   fi
 322   # Sanitizers need these variables to print and properly format the stack
 323   # traces:
 324   LLVM_SYMBOLIZER=$("${CC:-clang}" -print-prog-name=llvm-symbolizer || true)
 325   if [[ -n "${LLVM_SYMBOLIZER}" ]]; then
 326     export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 327     export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 328     export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 329   fi
 330 }
 331
 332 cmake_configure() {
 333   export_env
 334
 335   if [[ "${STACK_SIZE:-0}" == 1 ]]; then
 336     # Dump the stack size of each function in the .stack_sizes section for
 337     # analysis.
 338     CMAKE_C_FLAGS+=" -fstack-size-section"
 339     CMAKE_CXX_FLAGS+=" -fstack-size-section"
 340   fi
 341
 342   local args=(
 343     -B"${BUILD_DIR}" -H"${MYDIR}"
 344     -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
 345     -G Ninja
 346     -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}"
 347     -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}"
 348     -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}"
 349     -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}"
 350     -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}"
 351     -DJPEGXL_VERSION="${JPEGXL_VERSION}"
 352     -DSANITIZER="${SANITIZER}"
 353     # These are not enabled by default in cmake.
 354     -DJPEGXL_ENABLE_VIEWERS=ON
 355     -DJPEGXL_ENABLE_PLUGINS=ON
 356     -DJPEGXL_ENABLE_DEVTOOLS=ON
 357     # We always use libfuzzer in the ci.sh wrapper.
 358     -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer"
 359   )
 360   if [[ "${BUILD_TARGET}" != *mingw32 ]]; then
 361     args+=(
 362       -DJPEGXL_WARNINGS_AS_ERRORS=ON
 363     )
 364   fi
 365   if [[ -n "${BUILD_TARGET}" ]]; then
 366     local system_name="Linux"
 367     if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 368       # When cross-compiling with mingw the target must be set to Windows and
 369       # run programs with wine.
 370       system_name="Windows"
 371       args+=(
 372         -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}"
 373         # Normally CMake automatically defines MINGW=1 when building with the
 374         # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling
 375         # with clang.
 376         -DMINGW=1
 377       )
 378     fi
 379     # EMSCRIPTEN toolchain sets the right values itself
 380     if [[ "${BUILD_TARGET}" != wasm* ]]; then
 381       # If set, BUILD_TARGET must be the target triplet such as
 382       # x86_64-unknown-linux-gnu.
 383       args+=(
 384         -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}"
 385         -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}"
 386         # Only the first element of the target triplet.
 387         -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}"
 388         -DCMAKE_SYSTEM_NAME="${system_name}"
 389         -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}"
 390       )
 391     else
 392       args+=(
 393         # sjpeg confuses WASM SIMD with SSE.
 394         -DSJPEG_ENABLE_SIMD=OFF
 395         # Building shared libs is not very useful for WASM.
 396         -DBUILD_SHARED_LIBS=OFF
 397       )
 398     fi
 399     args+=(
 400       # These are needed to make googletest work when cross-compiling.
 401       -DCMAKE_CROSSCOMPILING=1
 402       -DHAVE_STD_REGEX=0
 403       -DHAVE_POSIX_REGEX=0
 404       -DHAVE_GNU_POSIX_REGEX=0
 405       -DHAVE_STEADY_CLOCK=0
 406       -DHAVE_THREAD_SAFETY_ATTRIBUTES=0
 407     )
 408     if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then
 409       # find_package() will look in this prefix for libraries.
 410       CMAKE_FIND_ROOT_PATH="/usr/${BUILD_TARGET}"
 411     fi
 412     if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then
 413       CMAKE_PREFIX_PATH="/usr/${BUILD_TARGET}"
 414     fi
 415     # Use pkg-config for the target. If there's no pkg-config available for the
 416     # target we can set the PKG_CONFIG_PATH to the appropriate path in most
 417     # linux distributions.
 418     local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true)
 419     if [[ -z "${pkg_config}" ]]; then
 420       pkg_config=$(which pkg-config)
 421       export PKG_CONFIG_LIBDIR="/usr/${BUILD_TARGET}/lib/pkgconfig"
 422     fi
 423     if [[ -n "${pkg_config}" ]]; then
 424       args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}")
 425     fi
 426   fi
 427   if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then
 428     args+=(
 429       -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}"
 430     )
 431   fi
 432   if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then
 433     args+=(
 434       -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}"
 435     )
 436   fi
 437   if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then
 438     args+=(
 439       -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
 440     )
 441   fi
 442   if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then
 443     args+=(
 444       -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}"
 445     )
 446   fi
 447   if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then
 448     args+=(
 449       -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}"
 450     )
 451   fi
 452   if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then
 453     args+=(
 454       -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}"
 455     )
 456   fi
 457   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 458     emcmake cmake "${args[@]}" "$@"
 459   else
 460     cmake "${args[@]}" "$@"
 461   fi
 462 }
 463
 464 cmake_build_and_test() {
 465   if [[ "${SKIP_BUILD}" -eq "1" ]]; then
 466       return 0
 467   fi
 468   # gtest_discover_tests() runs the test binaries to discover the list of tests
 469   # at build time, which fails under qemu.
 470   ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- $TARGETS
 471   # Pack test binaries if requested.
 472   if [[ "${PACK_TEST:-}" == "1" ]]; then
 473     (cd "${BUILD_DIR}"
 474      ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*'
 475      # gtest / gmock / gtest_main shared libs
 476      ${FIND_BIN} lib/ -name 'libg*.so*'
 477      ${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*'
 478     ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \
 479       --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 480     du -h "${BUILD_DIR}/tests.tar.xz"
 481     # Pack coverage data if also available.
 482     touch "${BUILD_DIR}/gcno.sentinel"
 483     (cd "${BUILD_DIR}"; echo gcno.sentinel; ${FIND_BIN} -name '*gcno') | \
 484       tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \
 485         --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 486   fi
 487
 488   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 489     (cd "${BUILD_DIR}"
 490      export UBSAN_OPTIONS=print_stacktrace=1
 491      [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 492      ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure)
 493   fi
 494 }
 495
 496 # Configure the build to strip unused functions. This considerably reduces the
 497 # output size, specially for tests which only use a small part of the whole
 498 # library.
 499 strip_dead_code() {
 500   # Emscripten does tree shaking without any extra flags.
 501   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 502     return 0
 503   fi
 504   # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively
 505   # discard all unreachable code, reducing the code size. For this to work, we
 506   # need to also pass --no-export-dynamic to prevent it from exporting all the
 507   # internal symbols (like functions) making them all reachable and thus not a
 508   # candidate for removal.
 509   CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections"
 510   CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections"
 511   if [[ "${OS}" == "Darwin" ]]; then
 512     CMAKE_EXE_LINKER_FLAGS+=" -dead_strip"
 513     CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip"
 514   else
 515     CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 516     CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 517   fi
 518 }
 519
 520 ### Externally visible commands
 521
 522 cmd_debug() {
 523   CMAKE_BUILD_TYPE="Debug"
 524   cmake_configure "$@"
 525   cmake_build_and_test
 526 }
 527
 528 cmd_release() {
 529   CMAKE_BUILD_TYPE="Release"
 530   strip_dead_code
 531   cmake_configure "$@"
 532   cmake_build_and_test
 533 }
 534
 535 cmd_opt() {
 536   CMAKE_BUILD_TYPE="RelWithDebInfo"
 537   CMAKE_CXX_FLAGS+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR"
 538   cmake_configure "$@"
 539   cmake_build_and_test
 540 }
 541
 542 cmd_coverage() {
 543   # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests.
 544   TEST_STACK_LIMIT="none"
 545
 546   cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@"
 547
 548   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 549     # If we didn't run the test we also don't print a coverage report.
 550     cmd_coverage_report
 551   fi
 552 }
 553
 554 cmd_coverage_report() {
 555   LLVM_COV=$("${CC:-clang}" -print-prog-name=llvm-cov)
 556   local real_build_dir=$(realpath "${BUILD_DIR}")
 557   local gcovr_args=(
 558     -r "${real_build_dir}"
 559     --gcov-executable "${LLVM_COV} gcov"
 560     # Only print coverage information for the libjxl directories. The rest
 561     # is not part of the code under test.
 562     --filter '.*jxl/.*'
 563     --exclude '.*_gbench.cc'
 564     --exclude '.*_test.cc'
 565     --exclude '.*_testonly..*'
 566     --exclude '.*_debug.*'
 567     --exclude '.*test_utils..*'
 568     --object-directory "${real_build_dir}"
 569   )
 570
 571   (
 572    cd "${real_build_dir}"
 573     gcovr "${gcovr_args[@]}" --html --html-details \
 574       --output="${real_build_dir}/coverage.html"
 575     gcovr "${gcovr_args[@]}" --print-summary |
 576       tee "${real_build_dir}/coverage.txt"
 577     gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml"
 578   )
 579 }
 580
 581 cmd_test() {
 582   export_env
 583   # Unpack tests if needed.
 584   if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then
 585     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz"
 586   fi
 587   if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then
 588     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz"
 589   fi
 590   (cd "${BUILD_DIR}"
 591    export UBSAN_OPTIONS=print_stacktrace=1
 592    [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 593    ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure "$@")
 594 }
 595
 596 cmd_gbench() {
 597   export_env
 598   (cd "${BUILD_DIR}"
 599    export UBSAN_OPTIONS=print_stacktrace=1
 600    lib/jxl_gbench \
 601      --benchmark_counters_tabular=true \
 602      --benchmark_out_format=json \
 603      --benchmark_out=gbench.json "$@"
 604   )
 605 }
 606
 607 cmd_asanfuzz() {
 608   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 609   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 610   cmd_asan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 611 }
 612
 613 cmd_msanfuzz() {
 614   # Install msan if needed before changing the flags.
 615   detect_clang_version
 616   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 617   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 618     # Install msan libraries for this version if needed or if an older version
 619     # with libc++abi was installed.
 620     cmd_msan_install
 621   fi
 622
 623   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 624   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 625   cmd_msan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 626 }
 627
 628 cmd_asan() {
 629   SANITIZER="asan"
 630   CMAKE_C_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 631     -fsanitize=address ${UBSAN_FLAGS[@]}"
 632   CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 633     -fsanitize=address ${UBSAN_FLAGS[@]}"
 634   strip_dead_code
 635   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 636   cmake_build_and_test
 637 }
 638
 639 cmd_tsan() {
 640   SANITIZER="tsan"
 641   local tsan_args=(
 642     -DJXL_ENABLE_ASSERT=1
 643     -g
 644     -DTHREAD_SANITIZER
 645     ${UBSAN_FLAGS[@]}
 646     -fsanitize=thread
 647   )
 648   CMAKE_C_FLAGS+=" ${tsan_args[@]}"
 649   CMAKE_CXX_FLAGS+=" ${tsan_args[@]}"
 650
 651   CMAKE_BUILD_TYPE="RelWithDebInfo"
 652   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 653   cmake_build_and_test
 654 }
 655
 656 cmd_msan() {
 657   SANITIZER="msan"
 658   detect_clang_version
 659   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 660   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 661     # Install msan libraries for this version if needed or if an older version
 662     # with libc++abi was installed.
 663     cmd_msan_install
 664   fi
 665
 666   local msan_c_flags=(
 667     -fsanitize=memory
 668     -fno-omit-frame-pointer
 669
 670     -DJXL_ENABLE_ASSERT=1
 671     -g
 672     -DMEMORY_SANITIZER
 673
 674     # Force gtest to not use the cxxbai.
 675     -DGTEST_HAS_CXXABI_H_=0
 676   )
 677   if [[ "${FASTER_MSAN_BUILD}" -ne "1" ]]; then
 678     msan_c_flags=(
 679       "${msan_c_flags[@]}"
 680       -fsanitize-memory-track-origins
 681     )
 682   fi
 683
 684   local msan_cxx_flags=(
 685     "${msan_c_flags[@]}"
 686
 687     # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused
 688     # in those cases. Ignore the warning.
 689     -Wno-unused-command-line-argument
 690     -stdlib=libc++
 691
 692     # We include the libc++ from the msan directory instead, so we don't want
 693     # the std includes.
 694     -nostdinc++
 695     -cxx-isystem"${msan_prefix}/include/c++/v1"
 696   )
 697
 698   local msan_linker_flags=(
 699     -L"${msan_prefix}"/lib
 700     -Wl,-rpath -Wl,"${msan_prefix}"/lib/
 701   )
 702
 703   CMAKE_C_FLAGS+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}"
 704   CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}"
 705   CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 706   CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 707   CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 708   strip_dead_code
 709   cmake_configure "$@" \
 710     -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \
 711     -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_WARNINGS_AS_ERRORS=OFF \
 712     -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}"
 713   cmake_build_and_test
 714 }
 715
 716 # Install libc++ libraries compiled with msan in the msan_prefix for the current
 717 # compiler version.
 718 cmd_msan_install() {
 719   local tmpdir=$(mktemp -d)
 720   CLEANUP_FILES+=("${tmpdir}")
 721   # Detect the llvm to install:
 722   export CC="${CC:-clang}"
 723   export CXX="${CXX:-clang++}"
 724   detect_clang_version
 725   # Allow overriding the LLVM checkout.
 726   local llvm_root="${LLVM_ROOT:-}"
 727   if [ -z "${llvm_root}" ]; then
 728     local llvm_tag="llvmorg-${CLANG_VERSION}.0.0"
 729     case "${CLANG_VERSION}" in
 730       "6.0")
 731         llvm_tag="llvmorg-6.0.1"
 732         ;;
 733       "7")
 734         llvm_tag="llvmorg-7.0.1"
 735         ;;
 736     esac
 737     local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz"
 738     curl -L --show-error -o "${llvm_targz}" \
 739       "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz"
 740     tar -C "${tmpdir}" -zxf "${llvm_targz}"
 741     llvm_root="${tmpdir}/llvm-project-${llvm_tag}"
 742   fi
 743
 744   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 745   rm -rf "${msan_prefix}"
 746
 747   local TARGET_OPTS=""
 748   if [[ -n "${BUILD_TARGET}" ]]; then
 749     TARGET_OPTS=" \
 750       -DCMAKE_C_COMPILER_TARGET=\"${BUILD_TARGET}\" \
 751       -DCMAKE_CXX_COMPILER_TARGET=\"${BUILD_TARGET}\" \
 752       -DCMAKE_SYSTEM_PROCESSOR=\"${BUILD_TARGET%%-*}\" \
 753     "
 754   fi
 755
 756   declare -A CMAKE_EXTRAS
 757   CMAKE_EXTRAS[libcxx]="\
 758     -DLIBCXX_CXX_ABI=libstdc++ \
 759     -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON"
 760
 761   for project in libcxx; do
 762     local proj_build="${tmpdir}/build-${project}"
 763     local proj_dir="${llvm_root}/${project}"
 764     mkdir -p "${proj_build}"
 765     cmake -B"${proj_build}" -H"${proj_dir}" \
 766       -G Ninja \
 767       -DCMAKE_BUILD_TYPE=Release \
 768       -DLLVM_USE_SANITIZER=Memory \
 769       -DLLVM_PATH="${llvm_root}/llvm" \
 770       -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \
 771                             head -n1)" \
 772       -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
 773       -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
 774       -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
 775       -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \
 776       -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \
 777       ${TARGET_OPTS} \
 778       ${CMAKE_EXTRAS[${project}]}
 779     cmake --build "${proj_build}"
 780     ninja -C "${proj_build}" install
 781   done
 782 }
 783
 784 # Internal build step shared between all cmd_ossfuzz_* commands.
 785 _cmd_ossfuzz() {
 786   local sanitizer="$1"
 787   shift
 788   mkdir -p "${BUILD_DIR}"
 789   local real_build_dir=$(realpath "${BUILD_DIR}")
 790
 791   # oss-fuzz defines three directories:
 792   # * /work, with the working directory to do re-builds
 793   # * /src, with the source code to build
 794   # * /out, with the output directory where to copy over the built files.
 795   # We use $BUILD_DIR as the /work and the script directory as the /src. The
 796   # /out directory is ignored as developers are used to look for the fuzzers in
 797   # $BUILD_DIR/tools/ directly.
 798
 799   if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then
 800     sudo docker run --rm -i \
 801       --user $(id -u):$(id -g) \
 802       -v "${real_build_dir}":/work \
 803       gcr.io/oss-fuzz-base/msan-libs-builder \
 804       bash -c "cp -r /msan /work"
 805   fi
 806
 807   # Args passed to ninja. These will be evaluated as a string separated by
 808   # spaces.
 809   local jpegxl_extra_args="$@"
 810
 811   sudo docker run --rm -i \
 812     -e JPEGXL_UID=$(id -u) \
 813     -e JPEGXL_GID=$(id -g) \
 814     -e FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" \
 815     -e SANITIZER="${sanitizer}" \
 816     -e ARCHITECTURE=x86_64 \
 817     -e FUZZING_LANGUAGE=c++ \
 818     -e MSAN_LIBS_PATH="/work/msan" \
 819     -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \
 820     -v "${MYDIR}":/src/libjxl \
 821     -v "${MYDIR}/tools/scripts/ossfuzz-build.sh":/src/build.sh \
 822     -v "${real_build_dir}":/work \
 823     gcr.io/oss-fuzz/libjxl
 824 }
 825
 826 cmd_ossfuzz_asan() {
 827   _cmd_ossfuzz address "$@"
 828 }
 829 cmd_ossfuzz_msan() {
 830   _cmd_ossfuzz memory "$@"
 831 }
 832 cmd_ossfuzz_ubsan() {
 833   _cmd_ossfuzz undefined "$@"
 834 }
 835
 836 cmd_ossfuzz_ninja() {
 837   [[ -e "${BUILD_DIR}/build.ninja" ]]
 838   local real_build_dir=$(realpath "${BUILD_DIR}")
 839
 840   if [[ -e "${BUILD_DIR}/msan" ]]; then
 841     echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2
 842     exit 1
 843   fi
 844
 845   sudo docker run --rm -i \
 846     --user $(id -u):$(id -g) \
 847     -v "${MYDIR}":/src/libjxl \
 848     -v "${real_build_dir}":/work \
 849     gcr.io/oss-fuzz/libjxl \
 850     ninja -C /work "$@"
 851 }
 852
 853 cmd_fast_benchmark() {
 854   local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar"
 855   mkdir -p "${BENCHMARK_CORPORA}"
 856   curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \
 857     "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
 858
 859   local tmpdir=$(mktemp -d)
 860   CLEANUP_FILES+=("${tmpdir}")
 861   tar -xf "${small_corpus_tar}" -C "${tmpdir}"
 862
 863   run_benchmark "${tmpdir}" 1048576
 864 }
 865
 866 cmd_benchmark() {
 867   local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar"
 868   mkdir -p "${BENCHMARK_CORPORA}"
 869   curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \
 870     "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar"
 871
 872   local tmpdir=$(mktemp -d)
 873   CLEANUP_FILES+=("${tmpdir}")
 874   tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}"
 875
 876   local sem_id="jpegxl_benchmark-$$"
 877   local nprocs=$(nproc --all || echo 1)
 878   images=()
 879   local filename
 880   while IFS= read -r filename; do
 881     # This removes the './'
 882     filename="${filename:2}"
 883     local mode
 884     if [[ "${filename:0:4}" == "srgb" ]]; then
 885       mode="RGB_D65_SRG_Rel_SRG"
 886     elif [[ "${filename:0:5}" == "adobe" ]]; then
 887       mode="RGB_D65_Ado_Rel_Ado"
 888     else
 889       echo "Unknown image colorspace: ${filename}" >&2
 890       exit 1
 891     fi
 892     png_filename="${filename%.ppm}.png"
 893     png_filename=$(echo "${png_filename}" | tr '/' '_')
 894     sem --bg --id "${sem_id}" -j"${nprocs}" -- \
 895       "${BUILD_DIR}/tools/decode_and_encode" \
 896         "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}"
 897     images+=( "${png_filename}" )
 898   done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f)
 899   sem --id "${sem_id}" --wait
 900
 901   # We need about 10 GiB per thread on these images.
 902   run_benchmark "${tmpdir}" 10485760
 903 }
 904
 905 get_mem_available() {
 906   if [[ "${OS}" == "Darwin" ]]; then
 907     echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}')
 908   else
 909     echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}')
 910   fi
 911 }
 912
 913 run_benchmark() {
 914   local src_img_dir="$1"
 915   local mem_per_thread="${2:-10485760}"
 916
 917   local output_dir="${BUILD_DIR}/benchmark_results"
 918   mkdir -p "${output_dir}"
 919
 920   # The memory available at the beginning of the benchmark run in kB. The number
 921   # of threads depends on the available memory, and the passed memory per
 922   # thread. We also add a 2 GiB of constant memory.
 923   local mem_available="$(get_mem_available)"
 924   # Check that we actually have a MemAvailable value.
 925   [[ -n "${mem_available}" ]]
 926   local num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} ))
 927   if [[ ${num_threads} -le 0 ]]; then
 928     num_threads=1
 929   fi
 930
 931   local benchmark_args=(
 932     --input "${src_img_dir}/*.png"
 933     --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2
 934     --output_dir "${output_dir}"
 935     --show_progress
 936     --num_threads="${num_threads}"
 937   )
 938   if [[ "${STORE_IMAGES}" == "1" ]]; then
 939     benchmark_args+=(--save_decompressed --save_compressed)
 940   fi
 941   (
 942     [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 943     "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \
 944        tee "${output_dir}/results.txt"
 945
 946     # Check error code for benckmark_xl command. This will exit if not.
 947     return ${PIPESTATUS[0]}
 948   )
 949
 950   if [[ -n "${CI_BUILD_NAME:-}" ]]; then
 951     { set +x; } 2>/dev/null
 952     local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}):
 953
 954 $(cat "${output_dir}/results.txt")
 955 "
 956     cmd_post_mr_comment "${message}"
 957     set -x
 958   fi
 959 }
 960
 961 # Helper function to wait for the CPU temperature to cool down on ARM.
 962 wait_for_temp() {
 963   { set +x; } 2>/dev/null
 964   local temp_limit=${1:-38000}
 965   if [[ -z "${THERMAL_FILE:-}" ]]; then
 966     echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \
 967       "to read the temperature from. This is normally set in the runner." >&2
 968     exit 1
 969   fi
 970   local org_temp=$(cat "${THERMAL_FILE}")
 971   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 972     echo -n "Waiting for temp to get down from ${org_temp}... "
 973   fi
 974   local temp="${org_temp}"
 975   local secs=0
 976   while [[ "${temp}" -ge "${temp_limit}" ]]; do
 977     sleep 1
 978     temp=$(cat "${THERMAL_FILE}")
 979     echo -n "${temp} "
 980     secs=$((secs + 1))
 981     if [[ ${secs} -ge 5 ]]; then
 982       break
 983     fi
 984   done
 985   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 986     echo "Done, temp=${temp}"
 987   fi
 988   set -x
 989 }
 990
 991 # Helper function to set the cpuset restriction of the current process.
 992 cmd_cpuset() {
 993   [[ "${SKIP_CPUSET:-}" != "1" ]] || return 0
 994   local newset="$1"
 995   local mycpuset=$(cat /proc/self/cpuset)
 996   mycpuset="/dev/cpuset${mycpuset}"
 997   # Check that the directory exists:
 998   [[ -d "${mycpuset}" ]]
 999   if [[ -e "${mycpuset}/cpuset.cpus" ]]; then
1000     echo "${newset}" >"${mycpuset}/cpuset.cpus"
1001   else
1002     echo "${newset}" >"${mycpuset}/cpus"
1003   fi
1004 }
1005
1006 # Return the encoding/decoding speed from the Stats output.
1007 _speed_from_output() {
1008   local speed="$1"
1009   local unit="${2:-MP/s}"
1010   if [[ "${speed}" == *"${unit}"* ]]; then
1011     speed="${speed%% ${unit}*}"
1012     speed="${speed##* }"
1013     echo "${speed}"
1014   fi
1015 }
1016
1017
1018 # Run benchmarks on ARM for the big and little CPUs.
1019 cmd_arm_benchmark() {
1020   # Flags used for cjxl encoder with .png inputs
1021   local jxl_png_benchmarks=(
1022     # Lossy options:
1023     "--epf=0 --distance=1.0 --speed=cheetah"
1024     "--epf=2 --distance=1.0 --speed=cheetah"
1025     "--epf=0 --distance=8.0 --speed=cheetah"
1026     "--epf=1 --distance=8.0 --speed=cheetah"
1027     "--epf=2 --distance=8.0 --speed=cheetah"
1028     "--epf=3 --distance=8.0 --speed=cheetah"
1029     "--modular -Q 90"
1030     "--modular -Q 50"
1031     # Lossless options:
1032     "--modular"
1033     "--modular -E 0 -I 0"
1034     "--modular -P 5"
1035     "--modular --responsive=1"
1036     # Near-lossless options:
1037     "--epf=0 --distance=0.3 --speed=fast"
1038     "--modular -Q 97"
1039   )
1040
1041   # Flags used for cjxl encoder with .jpg inputs. These should do lossless
1042   # JPEG recompression (of pixels or full jpeg).
1043   local jxl_jpeg_benchmarks=(
1044     "--num_reps=3"
1045   )
1046
1047   local images=(
1048     "testdata/jxl/flower/flower.png"
1049   )
1050
1051   local jpg_images=(
1052     "testdata/jxl/flower/flower.png.im_q85_420.jpg"
1053   )
1054
1055   if [[ "${SKIP_CPUSET:-}" == "1" ]]; then
1056     # Use a single cpu config in this case.
1057     local cpu_confs=("?")
1058   else
1059     # Otherwise the CPU config comes from the environment:
1060     local cpu_confs=(
1061       "${RUNNER_CPU_LITTLE}"
1062       "${RUNNER_CPU_BIG}"
1063       # The CPU description is something like 3-7, so these configurations only
1064       # take the first CPU of the group.
1065       "${RUNNER_CPU_LITTLE%%-*}"
1066       "${RUNNER_CPU_BIG%%-*}"
1067     )
1068     # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will
1069     # be ignored but still evaluated when calling cmd_cpuset.
1070     [[ -n "${RUNNER_CPU_ALL}" ]]
1071   fi
1072
1073   local jpg_dirname="third_party/corpora/jpeg"
1074   mkdir -p "${jpg_dirname}"
1075   local jpg_qualities=( 50 80 95 )
1076   for src_img in "${images[@]}"; do
1077     for q in "${jpg_qualities[@]}"; do
1078       local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg"
1079       convert -sampling-factor 1x1 -quality "${q}" \
1080         "${src_img}" "${jpeg_name}"
1081       jpg_images+=("${jpeg_name}")
1082     done
1083   done
1084
1085   local output_dir="${BUILD_DIR}/benchmark_results"
1086   mkdir -p "${output_dir}"
1087   local runs_file="${output_dir}/runs.txt"
1088
1089   if [[ ! -e "${runs_file}" ]]; then
1090     echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" |
1091       tee -a "${runs_file}"
1092   fi
1093
1094   mkdir -p "${BUILD_DIR}/arm_benchmark"
1095   local flags
1096   local src_img
1097   for src_img in "${jpg_images[@]}" "${images[@]}"; do
1098     local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ')
1099     local enc_binaries=("${BUILD_DIR}/tools/cjxl")
1100     local src_ext="${src_img##*.}"
1101     for enc_binary in "${enc_binaries[@]}"; do
1102       local enc_binary_base=$(basename "${enc_binary}")
1103
1104       # Select the list of flags to use for the current encoder/image pair.
1105       local img_benchmarks
1106       if [[ "${src_ext}" == "jpg" ]]; then
1107         img_benchmarks=("${jxl_jpeg_benchmarks[@]}")
1108       else
1109         img_benchmarks=("${jxl_png_benchmarks[@]}")
1110       fi
1111
1112       for flags in "${img_benchmarks[@]}"; do
1113         # Encoding step.
1114         local enc_file_hash="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}"
1115         enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ')
1116         local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl"
1117
1118         for cpu_conf in "${cpu_confs[@]}"; do
1119           cmd_cpuset "${cpu_conf}"
1120           # nproc returns the number of active CPUs, which is given by the cpuset
1121           # mask.
1122           local num_threads="$(nproc)"
1123
1124           echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1125           local enc_output
1126           if [[ "${flags}" == *"modular"* ]]; then
1127             # We don't benchmark encoding speed in this case.
1128             if [[ ! -f "${enc_file}" ]]; then
1129               cmd_cpuset "${RUNNER_CPU_ALL:-}"
1130               "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp"
1131               mv "${enc_file}.tmp" "${enc_file}"
1132               cmd_cpuset "${cpu_conf}"
1133             fi
1134             enc_output=" ?? MP/s"
1135           else
1136             wait_for_temp
1137             enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \
1138               2>&1 | tee /dev/stderr | grep -F "MP/s [")
1139             mv "${enc_file}.tmp" "${enc_file}"
1140           fi
1141           local enc_speed=$(_speed_from_output "${enc_output}")
1142           local enc_size=$(stat -c "%s" "${enc_file}")
1143
1144           echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1145
1146           local dec_output
1147           wait_for_temp
1148           dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1149             --num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr |
1150             grep -E "M[BP]/s \[")
1151           local img_size=$(echo "${dec_output}" | cut -f 1 -d ',')
1152           local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ')
1153           local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ')
1154           local img_size_px=$(( ${img_size_x} * ${img_size_y} ))
1155           local dec_speed=$(_speed_from_output "${dec_output}")
1156
1157           # For JPEG lossless recompression modes (where the original is a JPEG)
1158           # decode to JPG as well.
1159           local jpeg_dec_mps_speed=""
1160           local jpeg_dec_mbs_speed=""
1161           if [[ "${src_ext}" == "jpg" ]]; then
1162             wait_for_temp
1163             local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg"
1164             dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1165               "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \
1166                 tee /dev/stderr | grep -E "M[BP]/s \[")
1167             local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}")
1168             local jpeg_dec_mbs_speed=$(_speed_from_output "${dec_output}" MB/s)
1169             if ! cmp --quiet "${src_img}" "${dec_file}"; then
1170               # Add a start at the end to signal that the files are different.
1171               jpeg_dec_mbs_speed+="*"
1172             fi
1173           fi
1174
1175           # Record entry in a tab-separated file.
1176           local src_img_base=$(basename "${src_img}")
1177           echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" |
1178             tee -a "${runs_file}"
1179         done
1180       done
1181     done
1182   done
1183   cmd_cpuset "${RUNNER_CPU_ALL:-}"
1184   cat "${runs_file}"
1185
1186   if [[ -n "${CI_BUILD_NAME:-}" ]]; then
1187     load_mr_vars_from_commit
1188     { set +x; } 2>/dev/null
1189     local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}):
1190
1191 \`\`\`
1192 $(column -t -s "        " "${runs_file}")
1193 \`\`\`
1194 "
1195     cmd_post_mr_comment "${message}"
1196     set -x
1197   fi
1198 }
1199
1200 # Generate a corpus and run the fuzzer on that corpus.
1201 cmd_fuzz() {
1202   local corpus_dir=$(realpath "${BUILD_DIR}/fuzzer_corpus")
1203   local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash")
1204   mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}"
1205   # Generate step.
1206   "${BUILD_DIR}/tools/fuzzer_corpus" "${corpus_dir}"
1207   # Run step:
1208   local nprocs=$(nproc --all || echo 1)
1209   (
1210    cd "${BUILD_DIR}"
1211    "tools/djxl_fuzzer" "${fuzzer_crash_dir}" "${corpus_dir}" \
1212      -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \
1213      -artifact_prefix="${fuzzer_crash_dir}/"
1214   )
1215 }
1216
1217 # Runs the linters (clang-format, build_cleaner, buildirier) on the pending CLs.
1218 cmd_lint() {
1219   merge_request_commits
1220   { set +x; } 2>/dev/null
1221   local versions=(${1:-16 15 14 13 12 11 10 9 8 7 6.0})
1222   local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format)
1223   local tmpdir=$(mktemp -d)
1224   CLEANUP_FILES+=("${tmpdir}")
1225
1226   local ret=0
1227   local build_patch="${tmpdir}/build_cleaner.patch"
1228   if ! "${MYDIR}/tools/scripts/build_cleaner.py" >"${build_patch}"; then
1229     ret=1
1230     echo "build_cleaner.py findings:" >&2
1231     "${COLORDIFF_BIN}" <"${build_patch}"
1232     echo "Run \`tools/scripts/build_cleaner.py --update\` to apply them" >&2
1233   fi
1234
1235   # It is ok, if buildifier is not installed.
1236   if which buildifier >/dev/null; then
1237     local buildifier_patch="${tmpdir}/buildifier.patch"
1238     local bazel_files=`git -C ${MYDIR} ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`
1239     set -x
1240     buildifier -d ${bazel_files} >"${buildifier_patch}"|| true
1241     { set +x; } 2>/dev/null
1242     if [ -s "${buildifier_patch}" ]; then
1243       ret=1
1244       echo 'buildifier have found some problems in Bazel build files:' >&2
1245       "${COLORDIFF_BIN}" <"${buildifier_patch}"
1246       echo 'To fix them run (from the base directory):' >&2
1247       echo '  buildifier `git ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`' >&2
1248     fi
1249   fi
1250
1251   local installed=()
1252   local clang_patch
1253   local clang_format
1254   for clang_format in "${clang_format_bins[@]}"; do
1255     if ! which "${clang_format}" >/dev/null; then
1256       continue
1257     fi
1258     installed+=("${clang_format}")
1259     local tmppatch="${tmpdir}/${clang_format}.patch"
1260     # We include in this linter all the changes including the uncommitted changes
1261     # to avoid printing changes already applied.
1262     set -x
1263     # Ignoring the error that git-clang-format outputs.
1264     git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \
1265       --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true
1266     { set +x; } 2>/dev/null
1267     if grep -E '^--- ' "${tmppatch}" | grep -v 'a/third_party' >/dev/null; then
1268       if [[ -n "${LINT_OUTPUT:-}" ]]; then
1269         cp "${tmppatch}" "${LINT_OUTPUT}"
1270       fi
1271       clang_patch="${tmppatch}"
1272     else
1273       echo "clang-format check OK" >&2
1274       return ${ret}
1275     fi
1276   done
1277
1278   if [[ ${#installed[@]} -eq 0 ]]; then
1279     echo "You must install clang-format for \"git clang-format\"" >&2
1280     exit 1
1281   fi
1282
1283   # clang-format is installed but found problems.
1284   echo "clang-format findings:" >&2
1285   "${COLORDIFF_BIN}" < "${clang_patch}"
1286
1287   echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \
1288     "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \
1289     "directory to apply them." >&2
1290   exit 1
1291 }
1292
1293 # Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs
1294 # clang-tidy over all the source files instead.
1295 cmd_tidy() {
1296   local what="${1:-}"
1297
1298   if [[ -z "${CLANG_TIDY_BIN}" ]]; then
1299     echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2
1300     exit 1
1301   fi
1302
1303   local git_args=()
1304   if [[ "${what}" == "all" ]]; then
1305     git_args=(ls-files)
1306     shift
1307   else
1308     merge_request_commits
1309     git_args=(
1310         diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}"
1311         "${MR_HEAD_SHA}"
1312     )
1313   fi
1314
1315   # Clang-tidy needs the compilation database generated by cmake.
1316   if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then
1317     # Generate the build options in debug mode, since we need the debug asserts
1318     # enabled for the clang-tidy analyzer to use them.
1319     CMAKE_BUILD_TYPE="Debug"
1320     cmake_configure
1321     # Build the autogen targets to generate the .h files from the .ui files.
1322     local autogen_targets=(
1323         $(ninja -C "${BUILD_DIR}" -t targets | grep -F _autogen: |
1324           cut -f 1 -d :)
1325     )
1326     if [[ ${#autogen_targets[@]} != 0 ]]; then
1327       ninja -C "${BUILD_DIR}" "${autogen_targets[@]}"
1328     fi
1329   fi
1330
1331   cd "${MYDIR}"
1332   local nprocs=$(nproc --all || echo 1)
1333   local ret=0
1334   if ! parallel -j"${nprocs}" --keep-order -- \
1335       "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \
1336       < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \
1337       >"${BUILD_DIR}/clang-tidy.txt"; then
1338     ret=1
1339   fi
1340   { set +x; } 2>/dev/null
1341   echo "Findings statistics:" >&2
1342   grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \
1343     | uniq -c >&2
1344
1345   if [[ $ret -ne 0 ]]; then
1346     cat >&2 <<EOF
1347 Errors found, see ${BUILD_DIR}/clang-tidy.txt for details.
1348 To automatically fix them, run:
1349
1350   SKIP_TEST=1 ./ci.sh debug
1351   ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$')
1352 EOF
1353   fi
1354
1355   return ${ret}
1356 }
1357
1358 # Print stats about all the packages built in ${BUILD_DIR}/debs/.
1359 cmd_debian_stats() {
1360   { set +x; } 2>/dev/null
1361   local debsdir="${BUILD_DIR}/debs"
1362   local f
1363   while IFS='' read -r -d '' f; do
1364     echo "====================================================================="
1365     echo "Package $f:"
1366     dpkg --info $f
1367     dpkg --contents $f
1368   done < <(find "${BUILD_DIR}/debs" -maxdepth 1 -mindepth 1 -type f \
1369            -name '*.deb' -print0)
1370 }
1371
1372 build_debian_pkg() {
1373   local srcdir="$1"
1374   local srcpkg="$2"
1375   local options="${3:-}"
1376
1377   local debsdir="${BUILD_DIR}/debs"
1378   local builddir="${debsdir}/${srcpkg}"
1379
1380   # debuild doesn't have an easy way to build out of tree, so we make a copy
1381   # of with all symlinks on the first level.
1382   mkdir -p "${builddir}"
1383   for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do
1384     if [[ ! -L "${builddir}/$f" ]]; then
1385       rm -f "${builddir}/$f"
1386       ln -s "${srcdir}/$f" "${builddir}/$f"
1387     fi
1388   done
1389   (
1390     cd "${builddir}"
1391     debuild "${options}" -b -uc -us
1392   )
1393 }
1394
1395 cmd_debian_build() {
1396   local srcpkg="${1:-}"
1397
1398   case "${srcpkg}" in
1399     jpeg-xl)
1400       build_debian_pkg "${MYDIR}" "jpeg-xl"
1401       ;;
1402     highway)
1403       build_debian_pkg "${MYDIR}/third_party/highway" "highway" "${HWY_PKG_OPTIONS}"
1404       ;;
1405     *)
1406       echo "ERROR: Must pass a valid source package name to build." >&2
1407       ;;
1408   esac
1409 }
1410
1411 get_version() {
1412   local varname=$1
1413   local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1)
1414   [[ -n "${line}" ]]
1415   line="${line#set(${varname} }"
1416   line="${line%)}"
1417   echo "${line}"
1418 }
1419
1420 cmd_bump_version() {
1421   local newver="${1:-}"
1422
1423   if ! which dch >/dev/null; then
1424     echo "Missing dch\nTo install it run:\n  sudo apt install devscripts"
1425     exit 1
1426   fi
1427
1428   if [[ -z "${newver}" ]]; then
1429     local major=$(get_version JPEGXL_MAJOR_VERSION)
1430     local minor=$(get_version JPEGXL_MINOR_VERSION)
1431     local patch=0
1432     minor=$(( ${minor}  + 1))
1433   else
1434     local major="${newver%%.*}"
1435     newver="${newver#*.}"
1436     local minor="${newver%%.*}"
1437     newver="${newver#${minor}}"
1438     local patch="${newver#.}"
1439     if [[ -z "${patch}" ]]; then
1440       patch=0
1441     fi
1442   fi
1443
1444   newver="${major}.${minor}.${patch}"
1445
1446   echo "Bumping version to ${newver} (${major}.${minor}.${patch})"
1447   sed -E \
1448     -e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \
1449     -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \
1450     -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \
1451     -i lib/CMakeLists.txt
1452   sed -E \
1453     -e "s/(LIBJXL_VERSION: )[0-9\\.]+/\\1 ${major}.${minor}.${patch}/" \
1454     -e "s/(LIBJXL_ABI_VERSION: )[0-9\\.]+/\\1 ${major}.${minor}/" \
1455     -i .github/workflows/conformance.yml
1456
1457   # Update lib.gni
1458   tools/scripts/build_cleaner.py --update
1459
1460   # Mark the previous version as "unstable".
1461   DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release ''
1462   DEBCHANGE_RELEASE_HEURISTIC=log dch -M \
1463     --newversion "${newver}" \
1464     "Bump JPEG XL version to ${newver}."
1465 }
1466
1467 # Check that the AUTHORS file contains the email of the committer.
1468 cmd_authors() {
1469   merge_request_commits
1470   local emails
1471   local names
1472   readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1473   readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1474   for i in "${!names[@]}"; do
1475     echo "Checking name '${names[$i]}' with email '${emails[$i]}' ..."
1476     "${MYDIR}"/tools/scripts/check_author.py "${emails[$i]}" "${names[$i]}"
1477   done
1478 }
1479
1480 main() {
1481   local cmd="${1:-}"
1482   if [[ -z "${cmd}" ]]; then
1483     cat >&2 <<EOF
1484 Use: $0 CMD
1485
1486 Where cmd is one of:
1487  opt       Build and test a Release with symbols build.
1488  debug     Build and test a Debug build (NDEBUG is not defined).
1489  release   Build and test a striped Release binary without debug information.
1490  asan      Build and test an ASan (AddressSanitizer) build.
1491  msan      Build and test an MSan (MemorySanitizer) build. Needs to have msan
1492            c++ libs installed with msan_install first.
1493  tsan      Build and test a TSan (ThreadSanitizer) build.
1494  asanfuzz  Build and test an ASan (AddressSanitizer) build for fuzzing.
1495  msanfuzz  Build and test an MSan (MemorySanitizer) build for fuzzing.
1496  test      Run the tests build by opt, debug, release, asan or msan. Useful when
1497            building with SKIP_TEST=1.
1498  gbench    Run the Google benchmark tests.
1499  fuzz      Generate the fuzzer corpus and run the fuzzer on it. Useful after
1500            building with asan or msan.
1501  benchmark Run the benchmark over the default corpus.
1502  fast_benchmark Run the benchmark over the small corpus.
1503
1504  coverage  Build and run tests with coverage support. Runs coverage_report as
1505            well.
1506  coverage_report Generate HTML, XML and text coverage report after a coverage
1507            run.
1508
1509  lint      Run the linter checks on the current commit or merge request.
1510  tidy      Run clang-tidy on the current commit or merge request.
1511  authors   Check that the last commit's author is listed in the AUTHORS file.
1512
1513  msan_install Install the libc++ libraries required to build in msan mode. This
1514               needs to be done once.
1515
1516  debian_build <srcpkg> Build the given source package.
1517  debian_stats  Print stats about the built packages.
1518
1519 oss-fuzz commands:
1520  ossfuzz_asan   Build the local source inside oss-fuzz docker with asan.
1521  ossfuzz_msan   Build the local source inside oss-fuzz docker with msan.
1522  ossfuzz_ubsan  Build the local source inside oss-fuzz docker with ubsan.
1523  ossfuzz_ninja  Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra
1524                 parameters are passed to ninja, for example "djxl_fuzzer" will
1525                 only build that ninja target. Use for faster build iteration
1526                 after one of the ossfuzz_*san commands.
1527
1528 You can pass some optional environment variables as well:
1529  - BUILD_DIR: The output build directory (by default "$$repo/build")
1530  - BUILD_TARGET: The target triplet used when cross-compiling.
1531  - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS.
1532  - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package.
1533  - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only).
1534  - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds.
1535  - LINT_OUTPUT: Path to the output patch from the "lint" command.
1536  - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark.
1537  - SKIP_BUILD=1: Skip the build stage, cmake configure only.
1538  - SKIP_TEST=1: Skip the test stage.
1539  - STORE_IMAGES=0: Makes the benchmark discard the computed images.
1540  - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB.
1541  - TEST_SELECTOR: pass additional arguments to ctest, e.g. "-R .Resample.".
1542  - STACK_SIZE=1: Generate binaries with the .stack_sizes sections.
1543
1544 These optional environment variables are forwarded to the cmake call as
1545 parameters:
1546  - CMAKE_BUILD_TYPE
1547  - CMAKE_C_FLAGS
1548  - CMAKE_CXX_FLAGS
1549  - CMAKE_C_COMPILER_LAUNCHER
1550  - CMAKE_CXX_COMPILER_LAUNCHER
1551  - CMAKE_CROSSCOMPILING_EMULATOR
1552  - CMAKE_FIND_ROOT_PATH
1553  - CMAKE_EXE_LINKER_FLAGS
1554  - CMAKE_MAKE_PROGRAM
1555  - CMAKE_MODULE_LINKER_FLAGS
1556  - CMAKE_SHARED_LINKER_FLAGS
1557  - CMAKE_TOOLCHAIN_FILE
1558
1559 Example:
1560   BUILD_DIR=/tmp/build $0 opt
1561 EOF
1562     exit 1
1563   fi
1564
1565   cmd="cmd_${cmd}"
1566   shift
1567   set -x
1568   "${cmd}" "$@"
1569 }
1570
1571 main "$@"