third_party/jpeg-xl/ci.sh

   1 #!/usr/bin/env bash
   2 # Copyright (c) the JPEG XL Project Authors. All rights reserved.
   3 #
   4 # Use of this source code is governed by a BSD-style
   5 # license that can be found in the LICENSE file.
   6
   7 # Continuous integration helper module. This module is meant to be called from
   8 # workflows during the continuous integration build, as well as from the
   9 # command line for developers.
  10
  11 set -eu
  12
  13 OS=`uname -s`
  14
  15 MYDIR=$(dirname $(realpath "$0"))
  16
  17 ### Environment parameters:
  18 TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-256}"
  19 CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
  20 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}
  21 CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-}
  22 CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-}
  23 CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-}
  24 SKIP_BUILD="${SKIP_BUILD:-0}"
  25 SKIP_TEST="${SKIP_TEST:-0}"
  26 FASTER_MSAN_BUILD="${FASTER_MSAN_BUILD:-0}"
  27 TARGETS="${TARGETS:-all doc}"
  28 TEST_SELECTOR="${TEST_SELECTOR:-}"
  29 BUILD_TARGET="${BUILD_TARGET:-}"
  30 ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}"
  31 if [[ -n "${BUILD_TARGET}" ]]; then
  32   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}"
  33 else
  34   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}"
  35 fi
  36 # Whether we should post a message in the MR when the build fails.
  37 POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}"
  38 # By default, do a lightweight debian HWY package build.
  39 HWY_PKG_OPTIONS="${HWY_PKG_OPTIONS:---set-envvar=HWY_EXTRA_CONFIG=-DBUILD_TESTING=OFF -DHWY_ENABLE_EXAMPLES=OFF -DHWY_ENABLE_CONTRIB=OFF}"
  40
  41 # Set default compilers to clang if not already set
  42 export CC=${CC:-clang}
  43 export CXX=${CXX:-clang++}
  44
  45 # Time limit for the "fuzz" command in seconds (0 means no limit).
  46 FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}"
  47
  48 SANITIZER="none"
  49
  50
  51 if [[ "${BUILD_TARGET%%-*}" == "x86_64" ||
  52     "${BUILD_TARGET%%-*}" == "i686" ]]; then
  53   # Default to building all targets, even if compiler baseline is SSE4
  54   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_EMU128}
  55 else
  56   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-}
  57 fi
  58
  59 # Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
  60 CMAKE_FLAGS=${CMAKE_FLAGS:-}
  61 CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}"
  62 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}"
  63
  64 CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-}
  65 CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-}
  66 CMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH:-}
  67 CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-}
  68 CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-}
  69 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}
  70
  71 if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then
  72   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -msimd128"
  73   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -msimd128"
  74   CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128"
  75 fi
  76
  77 if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then
  78   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2"
  79   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2"
  80 fi
  81
  82 if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then
  83   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}"
  84 fi
  85
  86 # Version inferred from the CI variables.
  87 CI_COMMIT_SHA=${GITHUB_SHA:-}
  88 JPEGXL_VERSION=${JPEGXL_VERSION:-}
  89
  90 # Benchmark parameters
  91 STORE_IMAGES=${STORE_IMAGES:-1}
  92 BENCHMARK_CORPORA="${MYDIR}/third_party/corpora"
  93
  94 # Local flags passed to sanitizers.
  95 UBSAN_FLAGS=(
  96   -fsanitize=alignment
  97   -fsanitize=bool
  98   -fsanitize=bounds
  99   -fsanitize=builtin
 100   -fsanitize=enum
 101   -fsanitize=float-cast-overflow
 102   -fsanitize=float-divide-by-zero
 103   -fsanitize=integer-divide-by-zero
 104   -fsanitize=null
 105   -fsanitize=object-size
 106   -fsanitize=pointer-overflow
 107   -fsanitize=return
 108   -fsanitize=returns-nonnull-attribute
 109   -fsanitize=shift-base
 110   -fsanitize=shift-exponent
 111   -fsanitize=unreachable
 112   -fsanitize=vla-bound
 113
 114   -fno-sanitize-recover=undefined
 115   # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning.
 116   -fsanitize-recover=alignment
 117 )
 118 # -fsanitize=function doesn't work on aarch64 and arm.
 119 if [[ "${BUILD_TARGET%%-*}" != "aarch64" &&
 120     "${BUILD_TARGET%%-*}" != "arm" ]]; then
 121   UBSAN_FLAGS+=(
 122     -fsanitize=function
 123   )
 124 fi
 125 if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then
 126   UBSAN_FLAGS+=(
 127     -fsanitize=signed-integer-overflow
 128   )
 129 fi
 130
 131 CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy | head -n 1)
 132 # Default to "cat" if "colordiff" is not installed or if stdout is not a tty.
 133 if [[ -t 1 ]]; then
 134   COLORDIFF_BIN=$(which colordiff cat | head -n 1)
 135 else
 136   COLORDIFF_BIN="cat"
 137 fi
 138 FIND_BIN=$(which gfind find | head -n 1)
 139 # "false" will disable wine64 when not installed. This won't allow
 140 # cross-compiling.
 141 WINE_BIN=$(which wine64 false | head -n 1)
 142
 143 CLANG_VERSION="${CLANG_VERSION:-}"
 144 # Detect the clang version suffix and store it in CLANG_VERSION. For example,
 145 # "6.0" for clang 6 or "7" for clang 7.
 146 detect_clang_version() {
 147   if [[ -n "${CLANG_VERSION}" ]]; then
 148     return 0
 149   fi
 150   local clang_version=$("${CC:-clang}" --version | head -n1)
 151   clang_version=${clang_version#"Debian "}
 152   clang_version=${clang_version#"Ubuntu "}
 153   local llvm_tag
 154   case "${clang_version}" in
 155     "clang version 6."*)
 156       CLANG_VERSION="6.0"
 157       ;;
 158     "clang version "*)
 159       # Any other clang version uses just the major version number.
 160       local suffix="${clang_version#clang version }"
 161       CLANG_VERSION="${suffix%%.*}"
 162       ;;
 163     "emcc"*)
 164       # We can't use asan or msan in the emcc case.
 165       ;;
 166     *)
 167       echo "Unknown clang version: ${clang_version}" >&2
 168       return 1
 169   esac
 170 }
 171
 172 # Temporary files cleanup hooks.
 173 CLEANUP_FILES=()
 174 cleanup() {
 175   if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
 176     rm -fr "${CLEANUP_FILES[@]}"
 177   fi
 178 }
 179
 180 # Executed on exit.
 181 on_exit() {
 182   local retcode="$1"
 183   # Always cleanup the CLEANUP_FILES.
 184   cleanup
 185 }
 186
 187 trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT
 188
 189
 190 # These variables are populated when calling merge_request_commits().
 191
 192 # The current hash at the top of the current branch or merge request branch (if
 193 # running from a merge request pipeline).
 194 MR_HEAD_SHA=""
 195 # The common ancestor between the current commit and the tracked branch, such
 196 # as main. This includes a list
 197 MR_ANCESTOR_SHA=""
 198
 199 # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA.
 200 merge_request_commits() {
 201   { set +x; } 2>/dev/null
 202   # GITHUB_SHA is the current reference being build in GitHub Actions.
 203   if [[ -n "${GITHUB_SHA:-}" ]]; then
 204     # GitHub normally does a checkout of a merge commit on a shallow repository
 205     # by default. We want to get a bit more of the history to be able to diff
 206     # changes on the Pull Request if needed. This fetches 10 more commits which
 207     # should be enough given that PR normally should have 1 commit.
 208     git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10
 209     if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
 210       MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null ||
 211                    echo "${GITHUB_SHA}")"
 212     else
 213       MR_HEAD_SHA="${GITHUB_SHA}"
 214     fi
 215   else
 216     MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "HEAD")
 217   fi
 218
 219   if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
 220     # Pull request workflow in GitHub Actions. GitHub checkout action uses
 221     # "origin" as the remote for the git checkout.
 222     git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}"
 223     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD)
 224   else
 225     # We are in a local branch, not a pull request workflow.
 226     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true)
 227   fi
 228
 229   if [[ -z "${MR_ANCESTOR_SHA}" ]]; then
 230     echo "Warning, not tracking any branch, using the last commit in HEAD.">&2
 231     # This prints the return value with just HEAD.
 232     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^")
 233   else
 234     # GitHub runs the pipeline on a merge commit, no need to look for the common
 235     # ancestor in that case.
 236     if [[ -z "${GITHUB_BASE_REF:-}" ]]; then
 237       MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base \
 238         "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}")
 239     fi
 240   fi
 241   set -x
 242 }
 243
 244
 245 # Set up and export the environment variables needed by the child processes.
 246 export_env() {
 247   if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 248     # Wine needs to know the paths to the mingw dlls. These should be
 249     # separated by ';'.
 250     WINEPATH=$("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \
 251       | grep -F 'libraries: =' | cut -f 2- -d '=' | tr ':' ';')
 252     # We also need our own libraries in the wine path.
 253     local real_build_dir=$(realpath "${BUILD_DIR}")
 254     # Some library .dll dependencies are installed in /bin:
 255     export WINEPATH="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin"
 256
 257     local prefix="${BUILD_DIR}/wineprefix"
 258     mkdir -p "${prefix}"
 259     export WINEPREFIX=$(realpath "${prefix}")
 260   fi
 261   # Sanitizers need these variables to print and properly format the stack
 262   # traces:
 263   LLVM_SYMBOLIZER=$("${CC:-clang}" -print-prog-name=llvm-symbolizer || true)
 264   if [[ -n "${LLVM_SYMBOLIZER}" ]]; then
 265     export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 266     export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 267     export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 268   fi
 269 }
 270
 271 cmake_configure() {
 272   export_env
 273
 274   if [[ "${STACK_SIZE:-0}" == 1 ]]; then
 275     # Dump the stack size of each function in the .stack_sizes section for
 276     # analysis.
 277     CMAKE_C_FLAGS+=" -fstack-size-section"
 278     CMAKE_CXX_FLAGS+=" -fstack-size-section"
 279   fi
 280
 281   local args=(
 282     -B"${BUILD_DIR}" -H"${MYDIR}"
 283     -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
 284     -G Ninja
 285     -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}"
 286     -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}"
 287     -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}"
 288     -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}"
 289     -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}"
 290     -DJPEGXL_VERSION="${JPEGXL_VERSION}"
 291     -DSANITIZER="${SANITIZER}"
 292     # These are not enabled by default in cmake.
 293     -DJPEGXL_ENABLE_VIEWERS=ON
 294     -DJPEGXL_ENABLE_PLUGINS=ON
 295     -DJPEGXL_ENABLE_DEVTOOLS=ON
 296     # We always use libfuzzer in the ci.sh wrapper.
 297     -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer"
 298   )
 299   if [[ "${BUILD_TARGET}" != *mingw32 ]]; then
 300     args+=(
 301       -DJPEGXL_WARNINGS_AS_ERRORS=ON
 302     )
 303   fi
 304   if [[ -n "${BUILD_TARGET}" ]]; then
 305     local system_name="Linux"
 306     if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 307       # When cross-compiling with mingw the target must be set to Windows and
 308       # run programs with wine.
 309       system_name="Windows"
 310       args+=(
 311         -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}"
 312         # Normally CMake automatically defines MINGW=1 when building with the
 313         # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling
 314         # with clang.
 315         -DMINGW=1
 316       )
 317     fi
 318     # EMSCRIPTEN toolchain sets the right values itself
 319     if [[ "${BUILD_TARGET}" != wasm* ]]; then
 320       # If set, BUILD_TARGET must be the target triplet such as
 321       # x86_64-unknown-linux-gnu.
 322       args+=(
 323         -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}"
 324         -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}"
 325         # Only the first element of the target triplet.
 326         -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}"
 327         -DCMAKE_SYSTEM_NAME="${system_name}"
 328         -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}"
 329       )
 330     else
 331       args+=(
 332         # sjpeg confuses WASM SIMD with SSE.
 333         -DSJPEG_ENABLE_SIMD=OFF
 334         # Building shared libs is not very useful for WASM.
 335         -DBUILD_SHARED_LIBS=OFF
 336       )
 337     fi
 338     args+=(
 339       # These are needed to make googletest work when cross-compiling.
 340       -DCMAKE_CROSSCOMPILING=1
 341       -DHAVE_STD_REGEX=0
 342       -DHAVE_POSIX_REGEX=0
 343       -DHAVE_GNU_POSIX_REGEX=0
 344       -DHAVE_STEADY_CLOCK=0
 345       -DHAVE_THREAD_SAFETY_ATTRIBUTES=0
 346     )
 347     if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then
 348       # find_package() will look in this prefix for libraries.
 349       CMAKE_FIND_ROOT_PATH="/usr/${BUILD_TARGET}"
 350     fi
 351     if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then
 352       CMAKE_PREFIX_PATH="/usr/${BUILD_TARGET}"
 353     fi
 354     # Use pkg-config for the target. If there's no pkg-config available for the
 355     # target we can set the PKG_CONFIG_PATH to the appropriate path in most
 356     # linux distributions.
 357     local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true)
 358     if [[ -z "${pkg_config}" ]]; then
 359       pkg_config=$(which pkg-config)
 360       export PKG_CONFIG_LIBDIR="/usr/${BUILD_TARGET}/lib/pkgconfig"
 361     fi
 362     if [[ -n "${pkg_config}" ]]; then
 363       args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}")
 364     fi
 365   fi
 366   if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then
 367     args+=(
 368       -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}"
 369     )
 370   fi
 371   if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then
 372     args+=(
 373       -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}"
 374     )
 375   fi
 376   if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then
 377     args+=(
 378       -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
 379     )
 380   fi
 381   if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then
 382     args+=(
 383       -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}"
 384     )
 385   fi
 386   if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then
 387     args+=(
 388       -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}"
 389     )
 390   fi
 391   if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then
 392     args+=(
 393       -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}"
 394     )
 395   fi
 396   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 397     emcmake cmake "${args[@]}" "$@"
 398   else
 399     cmake "${args[@]}" "$@"
 400   fi
 401 }
 402
 403 cmake_build_and_test() {
 404   if [[ "${SKIP_BUILD}" -eq "1" ]]; then
 405       return 0
 406   fi
 407   # gtest_discover_tests() runs the test binaries to discover the list of tests
 408   # at build time, which fails under qemu.
 409   ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- $TARGETS
 410   # Pack test binaries if requested.
 411   if [[ "${PACK_TEST:-}" == "1" ]]; then
 412     (cd "${BUILD_DIR}"
 413      ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*'
 414      # gtest / gmock / gtest_main shared libs
 415      ${FIND_BIN} lib/ -name 'libg*.so*'
 416      ${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*'
 417     ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \
 418       --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 419     du -h "${BUILD_DIR}/tests.tar.xz"
 420     # Pack coverage data if also available.
 421     touch "${BUILD_DIR}/gcno.sentinel"
 422     (cd "${BUILD_DIR}"; echo gcno.sentinel; ${FIND_BIN} -name '*gcno') | \
 423       tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \
 424         --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 425   fi
 426
 427   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 428     (cd "${BUILD_DIR}"
 429      export UBSAN_OPTIONS=print_stacktrace=1
 430      [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 431      ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure)
 432   fi
 433 }
 434
 435 # Configure the build to strip unused functions. This considerably reduces the
 436 # output size, specially for tests which only use a small part of the whole
 437 # library.
 438 strip_dead_code() {
 439   # Emscripten does tree shaking without any extra flags.
 440   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 441     return 0
 442   fi
 443   # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively
 444   # discard all unreachable code, reducing the code size. For this to work, we
 445   # need to also pass --no-export-dynamic to prevent it from exporting all the
 446   # internal symbols (like functions) making them all reachable and thus not a
 447   # candidate for removal.
 448   CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections"
 449   CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections"
 450   if [[ "${OS}" == "Darwin" ]]; then
 451     CMAKE_EXE_LINKER_FLAGS+=" -dead_strip"
 452     CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip"
 453   else
 454     CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 455     CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 456   fi
 457 }
 458
 459 ### Externally visible commands
 460
 461 cmd_debug() {
 462   CMAKE_BUILD_TYPE="Debug"
 463   cmake_configure "$@"
 464   cmake_build_and_test
 465 }
 466
 467 cmd_release() {
 468   CMAKE_BUILD_TYPE="Release"
 469   strip_dead_code
 470   cmake_configure "$@"
 471   cmake_build_and_test
 472 }
 473
 474 cmd_opt() {
 475   CMAKE_BUILD_TYPE="RelWithDebInfo"
 476   CMAKE_CXX_FLAGS+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR"
 477   cmake_configure "$@"
 478   cmake_build_and_test
 479 }
 480
 481 cmd_coverage() {
 482   # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests.
 483   TEST_STACK_LIMIT="none"
 484
 485   cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@"
 486
 487   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 488     # If we didn't run the test we also don't print a coverage report.
 489     cmd_coverage_report
 490   fi
 491 }
 492
 493 cmd_coverage_report() {
 494   LLVM_COV=$("${CC:-clang}" -print-prog-name=llvm-cov)
 495   local real_build_dir=$(realpath "${BUILD_DIR}")
 496   local gcovr_args=(
 497     -r "${real_build_dir}"
 498     --gcov-executable "${LLVM_COV} gcov"
 499     # Only print coverage information for the libjxl directories. The rest
 500     # is not part of the code under test.
 501     --filter '.*jxl/.*'
 502     --exclude '.*_gbench.cc'
 503     --exclude '.*_test.cc'
 504     --exclude '.*_testonly..*'
 505     --exclude '.*_debug.*'
 506     --exclude '.*test_utils..*'
 507     --object-directory "${real_build_dir}"
 508   )
 509
 510   (
 511    cd "${real_build_dir}"
 512     gcovr "${gcovr_args[@]}" --html --html-details \
 513       --output="${real_build_dir}/coverage.html"
 514     gcovr "${gcovr_args[@]}" --print-summary |
 515       tee "${real_build_dir}/coverage.txt"
 516     gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml"
 517   )
 518 }
 519
 520 cmd_test() {
 521   export_env
 522   # Unpack tests if needed.
 523   if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then
 524     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz"
 525   fi
 526   if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then
 527     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz"
 528   fi
 529   (cd "${BUILD_DIR}"
 530    export UBSAN_OPTIONS=print_stacktrace=1
 531    [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 532    ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure "$@")
 533 }
 534
 535 cmd_gbench() {
 536   export_env
 537   (cd "${BUILD_DIR}"
 538    export UBSAN_OPTIONS=print_stacktrace=1
 539    lib/jxl_gbench \
 540      --benchmark_counters_tabular=true \
 541      --benchmark_out_format=json \
 542      --benchmark_out=gbench.json "$@"
 543   )
 544 }
 545
 546 cmd_asanfuzz() {
 547   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 548   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 549   cmd_asan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 550 }
 551
 552 cmd_msanfuzz() {
 553   # Install msan if needed before changing the flags.
 554   detect_clang_version
 555   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 556   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 557     # Install msan libraries for this version if needed or if an older version
 558     # with libc++abi was installed.
 559     cmd_msan_install
 560   fi
 561
 562   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 563   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 564   cmd_msan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 565 }
 566
 567 cmd_asan() {
 568   SANITIZER="asan"
 569   CMAKE_C_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 570     -fsanitize=address ${UBSAN_FLAGS[@]}"
 571   CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 572     -fsanitize=address ${UBSAN_FLAGS[@]}"
 573   strip_dead_code
 574   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 575   cmake_build_and_test
 576 }
 577
 578 cmd_tsan() {
 579   SANITIZER="tsan"
 580   local tsan_args=(
 581     -DJXL_ENABLE_ASSERT=1
 582     -g
 583     -DTHREAD_SANITIZER
 584     ${UBSAN_FLAGS[@]}
 585     -fsanitize=thread
 586   )
 587   CMAKE_C_FLAGS+=" ${tsan_args[@]}"
 588   CMAKE_CXX_FLAGS+=" ${tsan_args[@]}"
 589
 590   CMAKE_BUILD_TYPE="RelWithDebInfo"
 591   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 592   cmake_build_and_test
 593 }
 594
 595 cmd_msan() {
 596   SANITIZER="msan"
 597   detect_clang_version
 598   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 599   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 600     # Install msan libraries for this version if needed or if an older version
 601     # with libc++abi was installed.
 602     cmd_msan_install
 603   fi
 604
 605   local msan_c_flags=(
 606     -fsanitize=memory
 607     -fno-omit-frame-pointer
 608
 609     -DJXL_ENABLE_ASSERT=1
 610     -g
 611     -DMEMORY_SANITIZER
 612
 613     # Force gtest to not use the cxxbai.
 614     -DGTEST_HAS_CXXABI_H_=0
 615   )
 616   if [[ "${FASTER_MSAN_BUILD}" -ne "1" ]]; then
 617     msan_c_flags=(
 618       "${msan_c_flags[@]}"
 619       -fsanitize-memory-track-origins
 620     )
 621   fi
 622
 623   local msan_cxx_flags=(
 624     "${msan_c_flags[@]}"
 625
 626     # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused
 627     # in those cases. Ignore the warning.
 628     -Wno-unused-command-line-argument
 629     -stdlib=libc++
 630
 631     # We include the libc++ from the msan directory instead, so we don't want
 632     # the std includes.
 633     -nostdinc++
 634     -cxx-isystem"${msan_prefix}/include/c++/v1"
 635   )
 636
 637   local msan_linker_flags=(
 638     -L"${msan_prefix}"/lib
 639     -Wl,-rpath -Wl,"${msan_prefix}"/lib/
 640   )
 641
 642   CMAKE_C_FLAGS+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}"
 643   CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}"
 644   CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 645   CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 646   CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 647   strip_dead_code
 648   cmake_configure "$@" \
 649     -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \
 650     -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_WARNINGS_AS_ERRORS=OFF \
 651     -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}"
 652   cmake_build_and_test
 653 }
 654
 655 # Install libc++ libraries compiled with msan in the msan_prefix for the current
 656 # compiler version.
 657 cmd_msan_install() {
 658   local tmpdir=$(mktemp -d)
 659   CLEANUP_FILES+=("${tmpdir}")
 660   # Detect the llvm to install:
 661   export CC="${CC:-clang}"
 662   export CXX="${CXX:-clang++}"
 663   detect_clang_version
 664   # Allow overriding the LLVM checkout.
 665   local llvm_root="${LLVM_ROOT:-}"
 666   if [ -z "${llvm_root}" ]; then
 667     local llvm_tag="llvmorg-${CLANG_VERSION}.0.0"
 668     case "${CLANG_VERSION}" in
 669       "6.0")
 670         llvm_tag="llvmorg-6.0.1"
 671         ;;
 672       "7")
 673         llvm_tag="llvmorg-7.0.1"
 674         ;;
 675     esac
 676     local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz"
 677     curl -L --show-error -o "${llvm_targz}" \
 678       "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz"
 679     tar -C "${tmpdir}" -zxf "${llvm_targz}"
 680     llvm_root="${tmpdir}/llvm-project-${llvm_tag}"
 681   fi
 682
 683   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 684   rm -rf "${msan_prefix}"
 685
 686   local TARGET_OPTS=""
 687   if [[ -n "${BUILD_TARGET}" ]]; then
 688     TARGET_OPTS=" \
 689       -DCMAKE_C_COMPILER_TARGET=\"${BUILD_TARGET}\" \
 690       -DCMAKE_CXX_COMPILER_TARGET=\"${BUILD_TARGET}\" \
 691       -DCMAKE_SYSTEM_PROCESSOR=\"${BUILD_TARGET%%-*}\" \
 692     "
 693   fi
 694
 695   declare -A CMAKE_EXTRAS
 696   CMAKE_EXTRAS[libcxx]="\
 697     -DLIBCXX_CXX_ABI=libstdc++ \
 698     -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON"
 699
 700   for project in libcxx; do
 701     local proj_build="${tmpdir}/build-${project}"
 702     local proj_dir="${llvm_root}/${project}"
 703     mkdir -p "${proj_build}"
 704     cmake -B"${proj_build}" -H"${proj_dir}" \
 705       -G Ninja \
 706       -DCMAKE_BUILD_TYPE=Release \
 707       -DLLVM_USE_SANITIZER=Memory \
 708       -DLLVM_PATH="${llvm_root}/llvm" \
 709       -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \
 710                             head -n1)" \
 711       -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
 712       -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
 713       -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
 714       -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \
 715       -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \
 716       ${TARGET_OPTS} \
 717       ${CMAKE_EXTRAS[${project}]}
 718     cmake --build "${proj_build}"
 719     ninja -C "${proj_build}" install
 720   done
 721 }
 722
 723 # Internal build step shared between all cmd_ossfuzz_* commands.
 724 _cmd_ossfuzz() {
 725   local sanitizer="$1"
 726   shift
 727   mkdir -p "${BUILD_DIR}"
 728   local real_build_dir=$(realpath "${BUILD_DIR}")
 729
 730   # oss-fuzz defines three directories:
 731   # * /work, with the working directory to do re-builds
 732   # * /src, with the source code to build
 733   # * /out, with the output directory where to copy over the built files.
 734   # We use $BUILD_DIR as the /work and the script directory as the /src. The
 735   # /out directory is ignored as developers are used to look for the fuzzers in
 736   # $BUILD_DIR/tools/ directly.
 737
 738   if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then
 739     sudo docker run --rm -i \
 740       --user $(id -u):$(id -g) \
 741       -v "${real_build_dir}":/work \
 742       gcr.io/oss-fuzz-base/msan-libs-builder \
 743       bash -c "cp -r /msan /work"
 744   fi
 745
 746   # Args passed to ninja. These will be evaluated as a string separated by
 747   # spaces.
 748   local jpegxl_extra_args="$@"
 749
 750   sudo docker run --rm -i \
 751     -e JPEGXL_UID=$(id -u) \
 752     -e JPEGXL_GID=$(id -g) \
 753     -e FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" \
 754     -e SANITIZER="${sanitizer}" \
 755     -e ARCHITECTURE=x86_64 \
 756     -e FUZZING_LANGUAGE=c++ \
 757     -e MSAN_LIBS_PATH="/work/msan" \
 758     -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \
 759     -v "${MYDIR}":/src/libjxl \
 760     -v "${MYDIR}/tools/scripts/ossfuzz-build.sh":/src/build.sh \
 761     -v "${real_build_dir}":/work \
 762     gcr.io/oss-fuzz/libjxl
 763 }
 764
 765 cmd_ossfuzz_asan() {
 766   _cmd_ossfuzz address "$@"
 767 }
 768 cmd_ossfuzz_msan() {
 769   _cmd_ossfuzz memory "$@"
 770 }
 771 cmd_ossfuzz_ubsan() {
 772   _cmd_ossfuzz undefined "$@"
 773 }
 774
 775 cmd_ossfuzz_ninja() {
 776   [[ -e "${BUILD_DIR}/build.ninja" ]]
 777   local real_build_dir=$(realpath "${BUILD_DIR}")
 778
 779   if [[ -e "${BUILD_DIR}/msan" ]]; then
 780     echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2
 781     exit 1
 782   fi
 783
 784   sudo docker run --rm -i \
 785     --user $(id -u):$(id -g) \
 786     -v "${MYDIR}":/src/libjxl \
 787     -v "${real_build_dir}":/work \
 788     gcr.io/oss-fuzz/libjxl \
 789     ninja -C /work "$@"
 790 }
 791
 792 cmd_fast_benchmark() {
 793   local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar"
 794   mkdir -p "${BENCHMARK_CORPORA}"
 795   curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \
 796     "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
 797
 798   local tmpdir=$(mktemp -d)
 799   CLEANUP_FILES+=("${tmpdir}")
 800   tar -xf "${small_corpus_tar}" -C "${tmpdir}"
 801
 802   run_benchmark "${tmpdir}" 1048576
 803 }
 804
 805 cmd_benchmark() {
 806   local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar"
 807   mkdir -p "${BENCHMARK_CORPORA}"
 808   curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \
 809     "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar"
 810
 811   local tmpdir=$(mktemp -d)
 812   CLEANUP_FILES+=("${tmpdir}")
 813   tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}"
 814
 815   local sem_id="jpegxl_benchmark-$$"
 816   local nprocs=$(nproc --all || echo 1)
 817   images=()
 818   local filename
 819   while IFS= read -r filename; do
 820     # This removes the './'
 821     filename="${filename:2}"
 822     local mode
 823     if [[ "${filename:0:4}" == "srgb" ]]; then
 824       mode="RGB_D65_SRG_Rel_SRG"
 825     elif [[ "${filename:0:5}" == "adobe" ]]; then
 826       mode="RGB_D65_Ado_Rel_Ado"
 827     else
 828       echo "Unknown image colorspace: ${filename}" >&2
 829       exit 1
 830     fi
 831     png_filename="${filename%.ppm}.png"
 832     png_filename=$(echo "${png_filename}" | tr '/' '_')
 833     sem --bg --id "${sem_id}" -j"${nprocs}" -- \
 834       "${BUILD_DIR}/tools/decode_and_encode" \
 835         "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}"
 836     images+=( "${png_filename}" )
 837   done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f)
 838   sem --id "${sem_id}" --wait
 839
 840   # We need about 10 GiB per thread on these images.
 841   run_benchmark "${tmpdir}" 10485760
 842 }
 843
 844 get_mem_available() {
 845   if [[ "${OS}" == "Darwin" ]]; then
 846     echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}')
 847   else
 848     echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}')
 849   fi
 850 }
 851
 852 run_benchmark() {
 853   local src_img_dir="$1"
 854   local mem_per_thread="${2:-10485760}"
 855
 856   local output_dir="${BUILD_DIR}/benchmark_results"
 857   mkdir -p "${output_dir}"
 858
 859   # The memory available at the beginning of the benchmark run in kB. The number
 860   # of threads depends on the available memory, and the passed memory per
 861   # thread. We also add a 2 GiB of constant memory.
 862   local mem_available="$(get_mem_available)"
 863   # Check that we actually have a MemAvailable value.
 864   [[ -n "${mem_available}" ]]
 865   local num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} ))
 866   if [[ ${num_threads} -le 0 ]]; then
 867     num_threads=1
 868   fi
 869
 870   local benchmark_args=(
 871     --input "${src_img_dir}/*.png"
 872     --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2
 873     --output_dir "${output_dir}"
 874     --show_progress
 875     --num_threads="${num_threads}"
 876   )
 877   if [[ "${STORE_IMAGES}" == "1" ]]; then
 878     benchmark_args+=(--save_decompressed --save_compressed)
 879   fi
 880   (
 881     [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 882     "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \
 883        tee "${output_dir}/results.txt"
 884
 885     # Check error code for benckmark_xl command. This will exit if not.
 886     return ${PIPESTATUS[0]}
 887   )
 888
 889
 890 }
 891
 892 # Helper function to wait for the CPU temperature to cool down on ARM.
 893 wait_for_temp() {
 894   { set +x; } 2>/dev/null
 895   local temp_limit=${1:-38000}
 896   if [[ -z "${THERMAL_FILE:-}" ]]; then
 897     echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \
 898       "to read the temperature from. This is normally set in the runner." >&2
 899     exit 1
 900   fi
 901   local org_temp=$(cat "${THERMAL_FILE}")
 902   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 903     echo -n "Waiting for temp to get down from ${org_temp}... "
 904   fi
 905   local temp="${org_temp}"
 906   local secs=0
 907   while [[ "${temp}" -ge "${temp_limit}" ]]; do
 908     sleep 1
 909     temp=$(cat "${THERMAL_FILE}")
 910     echo -n "${temp} "
 911     secs=$((secs + 1))
 912     if [[ ${secs} -ge 5 ]]; then
 913       break
 914     fi
 915   done
 916   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 917     echo "Done, temp=${temp}"
 918   fi
 919   set -x
 920 }
 921
 922 # Helper function to set the cpuset restriction of the current process.
 923 cmd_cpuset() {
 924   [[ "${SKIP_CPUSET:-}" != "1" ]] || return 0
 925   local newset="$1"
 926   local mycpuset=$(cat /proc/self/cpuset)
 927   mycpuset="/dev/cpuset${mycpuset}"
 928   # Check that the directory exists:
 929   [[ -d "${mycpuset}" ]]
 930   if [[ -e "${mycpuset}/cpuset.cpus" ]]; then
 931     echo "${newset}" >"${mycpuset}/cpuset.cpus"
 932   else
 933     echo "${newset}" >"${mycpuset}/cpus"
 934   fi
 935 }
 936
 937 # Return the encoding/decoding speed from the Stats output.
 938 _speed_from_output() {
 939   local speed="$1"
 940   local unit="${2:-MP/s}"
 941   if [[ "${speed}" == *"${unit}"* ]]; then
 942     speed="${speed%% ${unit}*}"
 943     speed="${speed##* }"
 944     echo "${speed}"
 945   fi
 946 }
 947
 948
 949 # Run benchmarks on ARM for the big and little CPUs.
 950 cmd_arm_benchmark() {
 951   # Flags used for cjxl encoder with .png inputs
 952   local jxl_png_benchmarks=(
 953     # Lossy options:
 954     "--epf=0 --distance=1.0 --speed=cheetah"
 955     "--epf=2 --distance=1.0 --speed=cheetah"
 956     "--epf=0 --distance=8.0 --speed=cheetah"
 957     "--epf=1 --distance=8.0 --speed=cheetah"
 958     "--epf=2 --distance=8.0 --speed=cheetah"
 959     "--epf=3 --distance=8.0 --speed=cheetah"
 960     "--modular -Q 90"
 961     "--modular -Q 50"
 962     # Lossless options:
 963     "--modular"
 964     "--modular -E 0 -I 0"
 965     "--modular -P 5"
 966     "--modular --responsive=1"
 967     # Near-lossless options:
 968     "--epf=0 --distance=0.3 --speed=fast"
 969     "--modular -Q 97"
 970   )
 971
 972   # Flags used for cjxl encoder with .jpg inputs. These should do lossless
 973   # JPEG recompression (of pixels or full jpeg).
 974   local jxl_jpeg_benchmarks=(
 975     "--num_reps=3"
 976   )
 977
 978   local images=(
 979     "testdata/jxl/flower/flower.png"
 980   )
 981
 982   local jpg_images=(
 983     "testdata/jxl/flower/flower.png.im_q85_420.jpg"
 984   )
 985
 986   if [[ "${SKIP_CPUSET:-}" == "1" ]]; then
 987     # Use a single cpu config in this case.
 988     local cpu_confs=("?")
 989   else
 990     # Otherwise the CPU config comes from the environment:
 991     local cpu_confs=(
 992       "${RUNNER_CPU_LITTLE}"
 993       "${RUNNER_CPU_BIG}"
 994       # The CPU description is something like 3-7, so these configurations only
 995       # take the first CPU of the group.
 996       "${RUNNER_CPU_LITTLE%%-*}"
 997       "${RUNNER_CPU_BIG%%-*}"
 998     )
 999     # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will
1000     # be ignored but still evaluated when calling cmd_cpuset.
1001     [[ -n "${RUNNER_CPU_ALL}" ]]
1002   fi
1003
1004   local jpg_dirname="third_party/corpora/jpeg"
1005   mkdir -p "${jpg_dirname}"
1006   local jpg_qualities=( 50 80 95 )
1007   for src_img in "${images[@]}"; do
1008     for q in "${jpg_qualities[@]}"; do
1009       local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg"
1010       convert -sampling-factor 1x1 -quality "${q}" \
1011         "${src_img}" "${jpeg_name}"
1012       jpg_images+=("${jpeg_name}")
1013     done
1014   done
1015
1016   local output_dir="${BUILD_DIR}/benchmark_results"
1017   mkdir -p "${output_dir}"
1018   local runs_file="${output_dir}/runs.txt"
1019
1020   if [[ ! -e "${runs_file}" ]]; then
1021     echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" |
1022       tee -a "${runs_file}"
1023   fi
1024
1025   mkdir -p "${BUILD_DIR}/arm_benchmark"
1026   local flags
1027   local src_img
1028   for src_img in "${jpg_images[@]}" "${images[@]}"; do
1029     local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ')
1030     local enc_binaries=("${BUILD_DIR}/tools/cjxl")
1031     local src_ext="${src_img##*.}"
1032     for enc_binary in "${enc_binaries[@]}"; do
1033       local enc_binary_base=$(basename "${enc_binary}")
1034
1035       # Select the list of flags to use for the current encoder/image pair.
1036       local img_benchmarks
1037       if [[ "${src_ext}" == "jpg" ]]; then
1038         img_benchmarks=("${jxl_jpeg_benchmarks[@]}")
1039       else
1040         img_benchmarks=("${jxl_png_benchmarks[@]}")
1041       fi
1042
1043       for flags in "${img_benchmarks[@]}"; do
1044         # Encoding step.
1045         local enc_file_hash="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}"
1046         enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ')
1047         local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl"
1048
1049         for cpu_conf in "${cpu_confs[@]}"; do
1050           cmd_cpuset "${cpu_conf}"
1051           # nproc returns the number of active CPUs, which is given by the cpuset
1052           # mask.
1053           local num_threads="$(nproc)"
1054
1055           echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1056           local enc_output
1057           if [[ "${flags}" == *"modular"* ]]; then
1058             # We don't benchmark encoding speed in this case.
1059             if [[ ! -f "${enc_file}" ]]; then
1060               cmd_cpuset "${RUNNER_CPU_ALL:-}"
1061               "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp"
1062               mv "${enc_file}.tmp" "${enc_file}"
1063               cmd_cpuset "${cpu_conf}"
1064             fi
1065             enc_output=" ?? MP/s"
1066           else
1067             wait_for_temp
1068             enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \
1069               2>&1 | tee /dev/stderr | grep -F "MP/s [")
1070             mv "${enc_file}.tmp" "${enc_file}"
1071           fi
1072           local enc_speed=$(_speed_from_output "${enc_output}")
1073           local enc_size=$(stat -c "%s" "${enc_file}")
1074
1075           echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1076
1077           local dec_output
1078           wait_for_temp
1079           dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1080             --num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr |
1081             grep -E "M[BP]/s \[")
1082           local img_size=$(echo "${dec_output}" | cut -f 1 -d ',')
1083           local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ')
1084           local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ')
1085           local img_size_px=$(( ${img_size_x} * ${img_size_y} ))
1086           local dec_speed=$(_speed_from_output "${dec_output}")
1087
1088           # For JPEG lossless recompression modes (where the original is a JPEG)
1089           # decode to JPG as well.
1090           local jpeg_dec_mps_speed=""
1091           local jpeg_dec_mbs_speed=""
1092           if [[ "${src_ext}" == "jpg" ]]; then
1093             wait_for_temp
1094             local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg"
1095             dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1096               "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \
1097                 tee /dev/stderr | grep -E "M[BP]/s \[")
1098             local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}")
1099             local jpeg_dec_mbs_speed=$(_speed_from_output "${dec_output}" MB/s)
1100             if ! cmp --quiet "${src_img}" "${dec_file}"; then
1101               # Add a start at the end to signal that the files are different.
1102               jpeg_dec_mbs_speed+="*"
1103             fi
1104           fi
1105
1106           # Record entry in a tab-separated file.
1107           local src_img_base=$(basename "${src_img}")
1108           echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" |
1109             tee -a "${runs_file}"
1110         done
1111       done
1112     done
1113   done
1114   cmd_cpuset "${RUNNER_CPU_ALL:-}"
1115   cat "${runs_file}"
1116
1117 }
1118
1119 # Generate a corpus and run the fuzzer on that corpus.
1120 cmd_fuzz() {
1121   local corpus_dir=$(realpath "${BUILD_DIR}/fuzzer_corpus")
1122   local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash")
1123   mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}"
1124   # Generate step.
1125   "${BUILD_DIR}/tools/fuzzer_corpus" "${corpus_dir}"
1126   # Run step:
1127   local nprocs=$(nproc --all || echo 1)
1128   (
1129    cd "${BUILD_DIR}"
1130    "tools/djxl_fuzzer" "${fuzzer_crash_dir}" "${corpus_dir}" \
1131      -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \
1132      -artifact_prefix="${fuzzer_crash_dir}/"
1133   )
1134 }
1135
1136 # Runs the linters (clang-format, build_cleaner, buildirier) on the pending CLs.
1137 cmd_lint() {
1138   merge_request_commits
1139   { set +x; } 2>/dev/null
1140   local versions=(${1:-16 15 14 13 12 11 10 9 8 7 6.0})
1141   local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format)
1142   local tmpdir=$(mktemp -d)
1143   CLEANUP_FILES+=("${tmpdir}")
1144
1145   local ret=0
1146   local build_patch="${tmpdir}/build_cleaner.patch"
1147   if ! "${MYDIR}/tools/scripts/build_cleaner.py" >"${build_patch}"; then
1148     ret=1
1149     echo "build_cleaner.py findings:" >&2
1150     "${COLORDIFF_BIN}" <"${build_patch}"
1151     echo "Run \`tools/scripts/build_cleaner.py --update\` to apply them" >&2
1152   fi
1153
1154   # It is ok, if buildifier is not installed.
1155   if which buildifier >/dev/null; then
1156     local buildifier_patch="${tmpdir}/buildifier.patch"
1157     local bazel_files=`git -C ${MYDIR} ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`
1158     set -x
1159     buildifier -d ${bazel_files} >"${buildifier_patch}"|| true
1160     { set +x; } 2>/dev/null
1161     if [ -s "${buildifier_patch}" ]; then
1162       ret=1
1163       echo 'buildifier have found some problems in Bazel build files:' >&2
1164       "${COLORDIFF_BIN}" <"${buildifier_patch}"
1165       echo 'To fix them run (from the base directory):' >&2
1166       echo '  buildifier `git ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`' >&2
1167     fi
1168   fi
1169
1170   local installed=()
1171   local clang_patch
1172   local clang_format
1173   for clang_format in "${clang_format_bins[@]}"; do
1174     if ! which "${clang_format}" >/dev/null; then
1175       continue
1176     fi
1177     installed+=("${clang_format}")
1178     local tmppatch="${tmpdir}/${clang_format}.patch"
1179     # We include in this linter all the changes including the uncommitted changes
1180     # to avoid printing changes already applied.
1181     set -x
1182     # Ignoring the error that git-clang-format outputs.
1183     git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \
1184       --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true
1185     { set +x; } 2>/dev/null
1186     if grep -E '^--- ' "${tmppatch}" | grep -v 'a/third_party' >/dev/null; then
1187       if [[ -n "${LINT_OUTPUT:-}" ]]; then
1188         cp "${tmppatch}" "${LINT_OUTPUT}"
1189       fi
1190       clang_patch="${tmppatch}"
1191     else
1192       echo "clang-format check OK" >&2
1193       return ${ret}
1194     fi
1195   done
1196
1197   if [[ ${#installed[@]} -eq 0 ]]; then
1198     echo "You must install clang-format for \"git clang-format\"" >&2
1199     exit 1
1200   fi
1201
1202   # clang-format is installed but found problems.
1203   echo "clang-format findings:" >&2
1204   "${COLORDIFF_BIN}" < "${clang_patch}"
1205
1206   echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \
1207     "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \
1208     "directory to apply them." >&2
1209   exit 1
1210 }
1211
1212 # Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs
1213 # clang-tidy over all the source files instead.
1214 cmd_tidy() {
1215   local what="${1:-}"
1216
1217   if [[ -z "${CLANG_TIDY_BIN}" ]]; then
1218     echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2
1219     exit 1
1220   fi
1221
1222   local git_args=()
1223   if [[ "${what}" == "all" ]]; then
1224     git_args=(ls-files)
1225     shift
1226   else
1227     merge_request_commits
1228     git_args=(
1229         diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}"
1230         "${MR_HEAD_SHA}"
1231     )
1232   fi
1233
1234   # Clang-tidy needs the compilation database generated by cmake.
1235   if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then
1236     # Generate the build options in debug mode, since we need the debug asserts
1237     # enabled for the clang-tidy analyzer to use them.
1238     CMAKE_BUILD_TYPE="Debug"
1239     cmake_configure
1240     # Build the autogen targets to generate the .h files from the .ui files.
1241     local autogen_targets=(
1242         $(ninja -C "${BUILD_DIR}" -t targets | grep -F _autogen: |
1243           cut -f 1 -d :)
1244     )
1245     if [[ ${#autogen_targets[@]} != 0 ]]; then
1246       ninja -C "${BUILD_DIR}" "${autogen_targets[@]}"
1247     fi
1248   fi
1249
1250   cd "${MYDIR}"
1251   local nprocs=$(nproc --all || echo 1)
1252   local ret=0
1253   if ! parallel -j"${nprocs}" --keep-order -- \
1254       "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \
1255       < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \
1256       >"${BUILD_DIR}/clang-tidy.txt"; then
1257     ret=1
1258   fi
1259   { set +x; } 2>/dev/null
1260   echo "Findings statistics:" >&2
1261   grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \
1262     | uniq -c >&2
1263
1264   if [[ $ret -ne 0 ]]; then
1265     cat >&2 <<EOF
1266 Errors found, see ${BUILD_DIR}/clang-tidy.txt for details.
1267 To automatically fix them, run:
1268
1269   SKIP_TEST=1 ./ci.sh debug
1270   ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$')
1271 EOF
1272   fi
1273
1274   return ${ret}
1275 }
1276
1277 # Print stats about all the packages built in ${BUILD_DIR}/debs/.
1278 cmd_debian_stats() {
1279   { set +x; } 2>/dev/null
1280   local debsdir="${BUILD_DIR}/debs"
1281   local f
1282   while IFS='' read -r -d '' f; do
1283     echo "====================================================================="
1284     echo "Package $f:"
1285     dpkg --info $f
1286     dpkg --contents $f
1287   done < <(find "${BUILD_DIR}/debs" -maxdepth 1 -mindepth 1 -type f \
1288            -name '*.deb' -print0)
1289 }
1290
1291 build_debian_pkg() {
1292   local srcdir="$1"
1293   local srcpkg="$2"
1294   local options="${3:-}"
1295
1296   local debsdir="${BUILD_DIR}/debs"
1297   local builddir="${debsdir}/${srcpkg}"
1298
1299   # debuild doesn't have an easy way to build out of tree, so we make a copy
1300   # of with all symlinks on the first level.
1301   mkdir -p "${builddir}"
1302   for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do
1303     if [[ ! -L "${builddir}/$f" ]]; then
1304       rm -f "${builddir}/$f"
1305       ln -s "${srcdir}/$f" "${builddir}/$f"
1306     fi
1307   done
1308   (
1309     cd "${builddir}"
1310     debuild "${options}" -b -uc -us
1311   )
1312 }
1313
1314 cmd_debian_build() {
1315   local srcpkg="${1:-}"
1316
1317   case "${srcpkg}" in
1318     jpeg-xl)
1319       build_debian_pkg "${MYDIR}" "jpeg-xl"
1320       ;;
1321     highway)
1322       build_debian_pkg "${MYDIR}/third_party/highway" "highway" "${HWY_PKG_OPTIONS}"
1323       ;;
1324     *)
1325       echo "ERROR: Must pass a valid source package name to build." >&2
1326       ;;
1327   esac
1328 }
1329
1330 get_version() {
1331   local varname=$1
1332   local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1)
1333   [[ -n "${line}" ]]
1334   line="${line#set(${varname} }"
1335   line="${line%)}"
1336   echo "${line}"
1337 }
1338
1339 cmd_bump_version() {
1340   local newver="${1:-}"
1341
1342   if ! which dch >/dev/null; then
1343     echo "Missing dch\nTo install it run:\n  sudo apt install devscripts"
1344     exit 1
1345   fi
1346
1347   if [[ -z "${newver}" ]]; then
1348     local major=$(get_version JPEGXL_MAJOR_VERSION)
1349     local minor=$(get_version JPEGXL_MINOR_VERSION)
1350     local patch=0
1351     minor=$(( ${minor}  + 1))
1352   else
1353     local major="${newver%%.*}"
1354     newver="${newver#*.}"
1355     local minor="${newver%%.*}"
1356     newver="${newver#${minor}}"
1357     local patch="${newver#.}"
1358     if [[ -z "${patch}" ]]; then
1359       patch=0
1360     fi
1361   fi
1362
1363   newver="${major}.${minor}.${patch}"
1364
1365   echo "Bumping version to ${newver} (${major}.${minor}.${patch})"
1366   sed -E \
1367     -e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \
1368     -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \
1369     -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \
1370     -i lib/CMakeLists.txt
1371   sed -E \
1372     -e "s/(LIBJXL_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}.${patch}\"/" \
1373     -e "s/(LIBJXL_ABI_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}\"/" \
1374     -i .github/workflows/conformance.yml
1375
1376   # Update lib.gni
1377   tools/scripts/build_cleaner.py --update
1378
1379   # Mark the previous version as "unstable".
1380   DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release ''
1381   DEBCHANGE_RELEASE_HEURISTIC=log dch -M \
1382     --newversion "${newver}" \
1383     "Bump JPEG XL version to ${newver}."
1384 }
1385
1386 # Check that the AUTHORS file contains the email of the committer.
1387 cmd_authors() {
1388   merge_request_commits
1389   local emails
1390   local names
1391   readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1392   readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1393   for i in "${!names[@]}"; do
1394     echo "Checking name '${names[$i]}' with email '${emails[$i]}' ..."
1395     "${MYDIR}"/tools/scripts/check_author.py "${emails[$i]}" "${names[$i]}"
1396   done
1397 }
1398
1399 main() {
1400   local cmd="${1:-}"
1401   if [[ -z "${cmd}" ]]; then
1402     cat >&2 <<EOF
1403 Use: $0 CMD
1404
1405 Where cmd is one of:
1406  opt       Build and test a Release with symbols build.
1407  debug     Build and test a Debug build (NDEBUG is not defined).
1408  release   Build and test a striped Release binary without debug information.
1409  asan      Build and test an ASan (AddressSanitizer) build.
1410  msan      Build and test an MSan (MemorySanitizer) build. Needs to have msan
1411            c++ libs installed with msan_install first.
1412  tsan      Build and test a TSan (ThreadSanitizer) build.
1413  asanfuzz  Build and test an ASan (AddressSanitizer) build for fuzzing.
1414  msanfuzz  Build and test an MSan (MemorySanitizer) build for fuzzing.
1415  test      Run the tests build by opt, debug, release, asan or msan. Useful when
1416            building with SKIP_TEST=1.
1417  gbench    Run the Google benchmark tests.
1418  fuzz      Generate the fuzzer corpus and run the fuzzer on it. Useful after
1419            building with asan or msan.
1420  benchmark Run the benchmark over the default corpus.
1421  fast_benchmark Run the benchmark over the small corpus.
1422
1423  coverage  Build and run tests with coverage support. Runs coverage_report as
1424            well.
1425  coverage_report Generate HTML, XML and text coverage report after a coverage
1426            run.
1427
1428  lint      Run the linter checks on the current commit or merge request.
1429  tidy      Run clang-tidy on the current commit or merge request.
1430  authors   Check that the last commit's author is listed in the AUTHORS file.
1431
1432  msan_install Install the libc++ libraries required to build in msan mode. This
1433               needs to be done once.
1434
1435  debian_build <srcpkg> Build the given source package.
1436  debian_stats  Print stats about the built packages.
1437
1438 oss-fuzz commands:
1439  ossfuzz_asan   Build the local source inside oss-fuzz docker with asan.
1440  ossfuzz_msan   Build the local source inside oss-fuzz docker with msan.
1441  ossfuzz_ubsan  Build the local source inside oss-fuzz docker with ubsan.
1442  ossfuzz_ninja  Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra
1443                 parameters are passed to ninja, for example "djxl_fuzzer" will
1444                 only build that ninja target. Use for faster build iteration
1445                 after one of the ossfuzz_*san commands.
1446
1447 You can pass some optional environment variables as well:
1448  - BUILD_DIR: The output build directory (by default "$$repo/build")
1449  - BUILD_TARGET: The target triplet used when cross-compiling.
1450  - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS.
1451  - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package.
1452  - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only).
1453  - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds.
1454  - LINT_OUTPUT: Path to the output patch from the "lint" command.
1455  - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark.
1456  - SKIP_BUILD=1: Skip the build stage, cmake configure only.
1457  - SKIP_TEST=1: Skip the test stage.
1458  - STORE_IMAGES=0: Makes the benchmark discard the computed images.
1459  - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB.
1460  - TEST_SELECTOR: pass additional arguments to ctest, e.g. "-R .Resample.".
1461  - STACK_SIZE=1: Generate binaries with the .stack_sizes sections.
1462
1463 These optional environment variables are forwarded to the cmake call as
1464 parameters:
1465  - CMAKE_BUILD_TYPE
1466  - CMAKE_C_FLAGS
1467  - CMAKE_CXX_FLAGS
1468  - CMAKE_C_COMPILER_LAUNCHER
1469  - CMAKE_CXX_COMPILER_LAUNCHER
1470  - CMAKE_CROSSCOMPILING_EMULATOR
1471  - CMAKE_FIND_ROOT_PATH
1472  - CMAKE_EXE_LINKER_FLAGS
1473  - CMAKE_MAKE_PROGRAM
1474  - CMAKE_MODULE_LINKER_FLAGS
1475  - CMAKE_SHARED_LINKER_FLAGS
1476  - CMAKE_TOOLCHAIN_FILE
1477
1478 Example:
1479   BUILD_DIR=/tmp/build $0 opt
1480 EOF
1481     exit 1
1482   fi
1483
1484   cmd="cmd_${cmd}"
1485   shift
1486   set -x
1487   "${cmd}" "$@"
1488 }
1489
1490 main "$@"