third_party/jpeg-xl/ci.sh

   1 #!/usr/bin/env bash
   2 # Copyright (c) the JPEG XL Project Authors. All rights reserved.
   3 #
   4 # Use of this source code is governed by a BSD-style
   5 # license that can be found in the LICENSE file.
   6
   7 # Continuous integration helper module. This module is meant to be called from
   8 # workflows during the continuous integration build, as well as from the
   9 # command line for developers.
  10
  11 set -eu
  12
  13 OS=`uname -s`
  14
  15 MYDIR=$(dirname $(realpath "$0"))
  16
  17 ### Environment parameters:
  18 TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-256}"
  19 BENCHMARK_NUM_THREADS="${BENCHMARK_NUM_THREADS:-0}"
  20 BUILD_CONFIG=${BUILD_CONFIG:-}
  21 CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
  22 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}
  23 CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-}
  24 CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-}
  25 CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-}
  26 SKIP_BUILD="${SKIP_BUILD:-0}"
  27 SKIP_TEST="${SKIP_TEST:-0}"
  28 FASTER_MSAN_BUILD="${FASTER_MSAN_BUILD:-0}"
  29 TARGETS="${TARGETS:-all doc}"
  30 TEST_SELECTOR="${TEST_SELECTOR:-}"
  31 BUILD_TARGET="${BUILD_TARGET:-}"
  32 ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}"
  33 if [[ -n "${BUILD_TARGET}" ]]; then
  34   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}"
  35 else
  36   BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}"
  37 fi
  38 # Whether we should post a message in the MR when the build fails.
  39 POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}"
  40 # By default, do a lightweight debian HWY package build.
  41 HWY_PKG_OPTIONS="${HWY_PKG_OPTIONS:---set-envvar=HWY_EXTRA_CONFIG=-DBUILD_TESTING=OFF -DHWY_ENABLE_EXAMPLES=OFF -DHWY_ENABLE_CONTRIB=OFF}"
  42
  43 # Set default compilers to clang if not already set
  44 export CC=${CC:-clang}
  45 export CXX=${CXX:-clang++}
  46
  47 # Time limit for the "fuzz" command in seconds (0 means no limit).
  48 FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}"
  49
  50 SANITIZER="none"
  51
  52
  53 if [[ "${BUILD_TARGET%%-*}" == "x86_64" ||
  54     "${BUILD_TARGET%%-*}" == "i686" ]]; then
  55   # Default to building all targets, even if compiler baseline is SSE4
  56   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_EMU128}
  57 else
  58   HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-}
  59 fi
  60
  61 # Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
  62 CMAKE_FLAGS=${CMAKE_FLAGS:-}
  63 CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}"
  64 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}"
  65
  66 CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-}
  67 CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-}
  68 CMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH:-}
  69 CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-}
  70 CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-}
  71 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}
  72
  73 if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then
  74   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -msimd128"
  75   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -msimd128"
  76   CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128"
  77 fi
  78
  79 if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then
  80   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2"
  81   CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2"
  82 fi
  83
  84 if [[ -z "${BUILD_CONFIG}" ]]; then
  85   TOOLS_DIR="${BUILD_DIR}/tools"
  86 else
  87   TOOLS_DIR="${BUILD_DIR}/tools/${BUILD_CONFIG}"
  88 fi
  89
  90 if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then
  91   CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}"
  92 fi
  93
  94 # Version inferred from the CI variables.
  95 CI_COMMIT_SHA=${GITHUB_SHA:-}
  96 JPEGXL_VERSION=${JPEGXL_VERSION:-}
  97
  98 # Benchmark parameters
  99 STORE_IMAGES=${STORE_IMAGES:-1}
 100 BENCHMARK_CORPORA="${MYDIR}/third_party/corpora"
 101
 102 # Local flags passed to sanitizers.
 103 UBSAN_FLAGS=(
 104   -fsanitize=alignment
 105   -fsanitize=bool
 106   -fsanitize=bounds
 107   -fsanitize=builtin
 108   -fsanitize=enum
 109   -fsanitize=float-cast-overflow
 110   -fsanitize=float-divide-by-zero
 111   -fsanitize=integer-divide-by-zero
 112   -fsanitize=null
 113   -fsanitize=object-size
 114   -fsanitize=pointer-overflow
 115   -fsanitize=return
 116   -fsanitize=returns-nonnull-attribute
 117   -fsanitize=shift-base
 118   -fsanitize=shift-exponent
 119   -fsanitize=unreachable
 120   -fsanitize=vla-bound
 121
 122   -fno-sanitize-recover=undefined
 123   # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning.
 124   -fsanitize-recover=alignment
 125 )
 126 # -fsanitize=function doesn't work on aarch64 and arm.
 127 if [[ "${BUILD_TARGET%%-*}" != "aarch64" &&
 128     "${BUILD_TARGET%%-*}" != "arm" ]]; then
 129   UBSAN_FLAGS+=(
 130     -fsanitize=function
 131   )
 132 fi
 133 if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then
 134   UBSAN_FLAGS+=(
 135     -fsanitize=signed-integer-overflow
 136   )
 137 fi
 138
 139 CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy 2>/dev/null | head -n 1)
 140 # Default to "cat" if "colordiff" is not installed or if stdout is not a tty.
 141 if [[ -t 1 ]]; then
 142   COLORDIFF_BIN=$(which colordiff cat 2>/dev/null | head -n 1)
 143 else
 144   COLORDIFF_BIN="cat"
 145 fi
 146 FIND_BIN=$(which gfind find 2>/dev/null | head -n 1)
 147 # "false" will disable wine64 when not installed. This won't allow
 148 # cross-compiling.
 149 WINE_BIN=$(which wine64 false 2>/dev/null | head -n 1)
 150
 151 CLANG_VERSION="${CLANG_VERSION:-}"
 152 # Detect the clang version suffix and store it in CLANG_VERSION. For example,
 153 # "6.0" for clang 6 or "7" for clang 7.
 154 detect_clang_version() {
 155   if [[ -n "${CLANG_VERSION}" ]]; then
 156     return 0
 157   fi
 158   local clang_version=$("${CC:-clang}" --version | head -n1)
 159   clang_version=${clang_version#"Debian "}
 160   clang_version=${clang_version#"Ubuntu "}
 161   local llvm_tag
 162   case "${clang_version}" in
 163     "clang version 6."*)
 164       CLANG_VERSION="6.0"
 165       ;;
 166     "clang version "*)
 167       # Any other clang version uses just the major version number.
 168       local suffix="${clang_version#clang version }"
 169       CLANG_VERSION="${suffix%%.*}"
 170       ;;
 171     "emcc"*)
 172       # We can't use asan or msan in the emcc case.
 173       ;;
 174     *)
 175       echo "Unknown clang version: ${clang_version}" >&2
 176       return 1
 177   esac
 178 }
 179
 180 # Temporary files cleanup hooks.
 181 CLEANUP_FILES=()
 182 cleanup() {
 183   if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
 184     rm -fr "${CLEANUP_FILES[@]}"
 185   fi
 186 }
 187
 188 # Executed on exit.
 189 on_exit() {
 190   local retcode="$1"
 191   # Always cleanup the CLEANUP_FILES.
 192   cleanup
 193 }
 194
 195 trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT
 196
 197
 198 # These variables are populated when calling merge_request_commits().
 199
 200 # The current hash at the top of the current branch or merge request branch (if
 201 # running from a merge request pipeline).
 202 MR_HEAD_SHA=""
 203 # The common ancestor between the current commit and the tracked branch, such
 204 # as main. This includes a list
 205 MR_ANCESTOR_SHA=""
 206
 207 # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA.
 208 merge_request_commits() {
 209   { set +x; } 2>/dev/null
 210   # GITHUB_SHA is the current reference being build in GitHub Actions.
 211   if [[ -n "${GITHUB_SHA:-}" ]]; then
 212     # GitHub normally does a checkout of a merge commit on a shallow repository
 213     # by default. We want to get a bit more of the history to be able to diff
 214     # changes on the Pull Request if needed. This fetches 10 more commits which
 215     # should be enough given that PR normally should have 1 commit.
 216     git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10
 217     if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
 218       MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null ||
 219                    echo "${GITHUB_SHA}")"
 220     else
 221       MR_HEAD_SHA="${GITHUB_SHA}"
 222     fi
 223   else
 224     MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "HEAD")
 225   fi
 226
 227   if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
 228     # Pull request workflow in GitHub Actions. GitHub checkout action uses
 229     # "origin" as the remote for the git checkout.
 230     git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}"
 231     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD)
 232   else
 233     # We are in a local branch, not a pull request workflow.
 234     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true)
 235   fi
 236
 237   if [[ -z "${MR_ANCESTOR_SHA}" ]]; then
 238     echo "Warning, not tracking any branch, using the last commit in HEAD.">&2
 239     # This prints the return value with just HEAD.
 240     MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^")
 241   else
 242     # GitHub runs the pipeline on a merge commit, no need to look for the common
 243     # ancestor in that case.
 244     if [[ -z "${GITHUB_BASE_REF:-}" ]]; then
 245       MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base \
 246         "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}")
 247     fi
 248   fi
 249   set -x
 250 }
 251
 252
 253 # Set up and export the environment variables needed by the child processes.
 254 export_env() {
 255   if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 256     # Wine needs to know the paths to the mingw dlls. These should be
 257     # separated by ';'.
 258     WINEPATH=$("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \
 259       | grep -F 'libraries: =' | cut -f 2- -d '=' | tr ':' ';')
 260     # We also need our own libraries in the wine path.
 261     local real_build_dir=$(realpath "${BUILD_DIR}")
 262     # Some library .dll dependencies are installed in /bin:
 263     export WINEPATH="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin"
 264
 265     local prefix="${BUILD_DIR}/wineprefix"
 266     mkdir -p "${prefix}"
 267     export WINEPREFIX=$(realpath "${prefix}")
 268   fi
 269   # Sanitizers need these variables to print and properly format the stack
 270   # traces:
 271   LLVM_SYMBOLIZER=$("${CC:-clang}" -print-prog-name=llvm-symbolizer || true)
 272   if [[ -n "${LLVM_SYMBOLIZER}" ]]; then
 273     export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 274     export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 275     export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
 276   fi
 277 }
 278
 279 cmake_configure() {
 280   export_env
 281
 282   if [[ "${STACK_SIZE:-0}" == 1 ]]; then
 283     # Dump the stack size of each function in the .stack_sizes section for
 284     # analysis.
 285     CMAKE_C_FLAGS+=" -fstack-size-section"
 286     CMAKE_CXX_FLAGS+=" -fstack-size-section"
 287   fi
 288
 289   local args=(
 290     -B"${BUILD_DIR}" -H"${MYDIR}"
 291     -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
 292     -G Ninja
 293     -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}"
 294     -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}"
 295     -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}"
 296     -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}"
 297     -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}"
 298     -DJPEGXL_VERSION="${JPEGXL_VERSION}"
 299     -DSANITIZER="${SANITIZER}"
 300     # These are not enabled by default in cmake.
 301     -DJPEGXL_ENABLE_VIEWERS=ON
 302     -DJPEGXL_ENABLE_PLUGINS=ON
 303     -DJPEGXL_ENABLE_DEVTOOLS=ON
 304     # We always use libfuzzer in the ci.sh wrapper.
 305     -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer"
 306   )
 307   if [[ "${BUILD_TARGET}" != *mingw32 ]]; then
 308     args+=(
 309       -DJPEGXL_WARNINGS_AS_ERRORS=ON
 310     )
 311   fi
 312   if [[ -n "${BUILD_TARGET}" ]]; then
 313     local system_name="Linux"
 314     if [[ "${BUILD_TARGET}" == *mingw32 ]]; then
 315       # When cross-compiling with mingw the target must be set to Windows and
 316       # run programs with wine.
 317       system_name="Windows"
 318       args+=(
 319         -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}"
 320         # Normally CMake automatically defines MINGW=1 when building with the
 321         # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling
 322         # with clang.
 323         -DMINGW=1
 324       )
 325     fi
 326     # EMSCRIPTEN toolchain sets the right values itself
 327     if [[ "${BUILD_TARGET}" != wasm* ]]; then
 328       # If set, BUILD_TARGET must be the target triplet such as
 329       # x86_64-unknown-linux-gnu.
 330       args+=(
 331         -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}"
 332         -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}"
 333         # Only the first element of the target triplet.
 334         -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}"
 335         -DCMAKE_SYSTEM_NAME="${system_name}"
 336         -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}"
 337       )
 338     else
 339       args+=(
 340         # sjpeg confuses WASM SIMD with SSE.
 341         -DSJPEG_ENABLE_SIMD=OFF
 342         # Building shared libs is not very useful for WASM.
 343         -DBUILD_SHARED_LIBS=OFF
 344       )
 345     fi
 346     args+=(
 347       # These are needed to make googletest work when cross-compiling.
 348       -DCMAKE_CROSSCOMPILING=1
 349       -DHAVE_STD_REGEX=0
 350       -DHAVE_POSIX_REGEX=0
 351       -DHAVE_GNU_POSIX_REGEX=0
 352       -DHAVE_STEADY_CLOCK=0
 353       -DHAVE_THREAD_SAFETY_ATTRIBUTES=0
 354     )
 355     if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then
 356       # find_package() will look in this prefix for libraries.
 357       CMAKE_FIND_ROOT_PATH="/usr/${BUILD_TARGET}"
 358     fi
 359     if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then
 360       CMAKE_PREFIX_PATH="/usr/${BUILD_TARGET}"
 361     fi
 362     # Use pkg-config for the target. If there's no pkg-config available for the
 363     # target we can set the PKG_CONFIG_PATH to the appropriate path in most
 364     # linux distributions.
 365     local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true)
 366     if [[ -z "${pkg_config}" ]]; then
 367       pkg_config=$(which pkg-config)
 368       export PKG_CONFIG_LIBDIR="/usr/${BUILD_TARGET}/lib/pkgconfig"
 369     fi
 370     if [[ -n "${pkg_config}" ]]; then
 371       args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}")
 372     fi
 373   fi
 374   if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then
 375     args+=(
 376       -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}"
 377     )
 378   fi
 379   if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then
 380     args+=(
 381       -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}"
 382     )
 383   fi
 384   if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then
 385     args+=(
 386       -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
 387     )
 388   fi
 389   if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then
 390     args+=(
 391       -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}"
 392     )
 393   fi
 394   if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then
 395     args+=(
 396       -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}"
 397     )
 398   fi
 399   if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then
 400     args+=(
 401       -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}"
 402     )
 403   fi
 404   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 405     emcmake cmake "${args[@]}" "$@"
 406   else
 407     cmake "${args[@]}" "$@"
 408   fi
 409 }
 410
 411 cmake_build_and_test() {
 412   if [[ "${SKIP_BUILD}" -eq "1" ]]; then
 413       return 0
 414   fi
 415   # gtest_discover_tests() runs the test binaries to discover the list of tests
 416   # at build time, which fails under qemu.
 417   ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- $TARGETS
 418   # Pack test binaries if requested.
 419   if [[ "${PACK_TEST:-}" == "1" ]]; then
 420     (cd "${BUILD_DIR}"
 421      ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*'
 422      # gtest / gtest_main shared libs
 423      ${FIND_BIN} lib/ -name 'libg*.so*'
 424      ${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*'
 425     ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \
 426       --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 427     du -h "${BUILD_DIR}/tests.tar.xz"
 428     # Pack coverage data if also available.
 429     touch "${BUILD_DIR}/gcno.sentinel"
 430     (cd "${BUILD_DIR}"; echo gcno.sentinel; ${FIND_BIN} -name '*gcno') | \
 431       tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \
 432         --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
 433   fi
 434
 435   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 436     (cd "${BUILD_DIR}"
 437      export UBSAN_OPTIONS=print_stacktrace=1
 438      [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 439      ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure)
 440   fi
 441 }
 442
 443 # Configure the build to strip unused functions. This considerably reduces the
 444 # output size, specially for tests which only use a small part of the whole
 445 # library.
 446 strip_dead_code() {
 447   # Emscripten does tree shaking without any extra flags.
 448   if [[ "${BUILD_TARGET}" == wasm* ]]; then
 449     return 0
 450   fi
 451   # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively
 452   # discard all unreachable code, reducing the code size. For this to work, we
 453   # need to also pass --no-export-dynamic to prevent it from exporting all the
 454   # internal symbols (like functions) making them all reachable and thus not a
 455   # candidate for removal.
 456   CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections"
 457   CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections"
 458   if [[ "${OS}" == "Darwin" ]]; then
 459     CMAKE_EXE_LINKER_FLAGS+=" -dead_strip"
 460     CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip"
 461   else
 462     CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 463     CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
 464   fi
 465 }
 466
 467 ### Externally visible commands
 468
 469 cmd_debug() {
 470   CMAKE_BUILD_TYPE="Debug"
 471   cmake_configure "$@"
 472   cmake_build_and_test
 473 }
 474
 475 cmd_release() {
 476   CMAKE_BUILD_TYPE="Release"
 477   strip_dead_code
 478   cmake_configure "$@"
 479   cmake_build_and_test
 480 }
 481
 482 cmd_opt() {
 483   CMAKE_BUILD_TYPE="RelWithDebInfo"
 484   CMAKE_CXX_FLAGS+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR"
 485   cmake_configure "$@"
 486   cmake_build_and_test
 487 }
 488
 489 cmd_coverage() {
 490   # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests.
 491   TEST_STACK_LIMIT="none"
 492
 493   cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@"
 494
 495   if [[ "${SKIP_TEST}" -ne "1" ]]; then
 496     # If we didn't run the test we also don't print a coverage report.
 497     cmd_coverage_report
 498   fi
 499 }
 500
 501 cmd_coverage_report() {
 502   LLVM_COV=$("${CC:-clang}" -print-prog-name=llvm-cov)
 503   local real_build_dir=$(realpath "${BUILD_DIR}")
 504   local gcovr_args=(
 505     -r "${real_build_dir}"
 506     --gcov-executable "${LLVM_COV} gcov"
 507     # Only print coverage information for the libjxl directories. The rest
 508     # is not part of the code under test.
 509     --filter '.*jxl/.*'
 510     --exclude '.*_gbench.cc'
 511     --exclude '.*_test.cc'
 512     --exclude '.*_testonly..*'
 513     --exclude '.*_debug.*'
 514     --exclude '.*test_utils..*'
 515     --object-directory "${real_build_dir}"
 516   )
 517
 518   (
 519    cd "${real_build_dir}"
 520     gcovr "${gcovr_args[@]}" --html --html-details \
 521       --output="${real_build_dir}/coverage.html"
 522     gcovr "${gcovr_args[@]}" --print-summary |
 523       tee "${real_build_dir}/coverage.txt"
 524     gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml"
 525   )
 526 }
 527
 528 cmd_test() {
 529   export_env
 530   # Unpack tests if needed.
 531   if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then
 532     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz"
 533   fi
 534   if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then
 535     tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz"
 536   fi
 537   (cd "${BUILD_DIR}"
 538    export UBSAN_OPTIONS=print_stacktrace=1
 539    [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 540    ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure "$@")
 541 }
 542
 543 cmd_gbench() {
 544   export_env
 545   (cd "${BUILD_DIR}"
 546    export UBSAN_OPTIONS=print_stacktrace=1
 547    lib/jxl_gbench \
 548      --benchmark_counters_tabular=true \
 549      --benchmark_out_format=json \
 550      --benchmark_out=gbench.json "$@"
 551   )
 552 }
 553
 554 cmd_asanfuzz() {
 555   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 556   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 557   cmd_asan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 558 }
 559
 560 cmd_msanfuzz() {
 561   # Install msan if needed before changing the flags.
 562   detect_clang_version
 563   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 564   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 565     # Install msan libraries for this version if needed or if an older version
 566     # with libc++abi was installed.
 567     cmd_msan_install
 568   fi
 569
 570   CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 571   CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
 572   cmd_msan -DJPEGXL_ENABLE_FUZZERS=ON "$@"
 573 }
 574
 575 cmd_asan() {
 576   SANITIZER="asan"
 577   CMAKE_C_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 578     -fsanitize=address ${UBSAN_FLAGS[@]}"
 579   CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
 580     -fsanitize=address ${UBSAN_FLAGS[@]}"
 581   strip_dead_code
 582   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 583   cmake_build_and_test
 584 }
 585
 586 cmd_tsan() {
 587   SANITIZER="tsan"
 588   local tsan_args=(
 589     -DJXL_ENABLE_ASSERT=1
 590     -g
 591     -DTHREAD_SANITIZER
 592     ${UBSAN_FLAGS[@]}
 593     -fsanitize=thread
 594   )
 595   CMAKE_C_FLAGS+=" ${tsan_args[@]}"
 596   CMAKE_CXX_FLAGS+=" ${tsan_args[@]}"
 597
 598   CMAKE_BUILD_TYPE="RelWithDebInfo"
 599   cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
 600   cmake_build_and_test
 601 }
 602
 603 cmd_msan() {
 604   SANITIZER="msan"
 605   detect_clang_version
 606   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 607   if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
 608     # Install msan libraries for this version if needed or if an older version
 609     # with libc++abi was installed.
 610     cmd_msan_install
 611   fi
 612
 613   local msan_c_flags=(
 614     -fsanitize=memory
 615     -fno-omit-frame-pointer
 616
 617     -DJXL_ENABLE_ASSERT=1
 618     -g
 619     -DMEMORY_SANITIZER
 620
 621     # Force gtest to not use the cxxbai.
 622     -DGTEST_HAS_CXXABI_H_=0
 623   )
 624   if [[ "${FASTER_MSAN_BUILD}" -ne "1" ]]; then
 625     msan_c_flags=(
 626       "${msan_c_flags[@]}"
 627       -fsanitize-memory-track-origins
 628     )
 629   fi
 630
 631   local msan_cxx_flags=(
 632     "${msan_c_flags[@]}"
 633
 634     # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused
 635     # in those cases. Ignore the warning.
 636     -Wno-unused-command-line-argument
 637     -stdlib=libc++
 638
 639     # We include the libc++ from the msan directory instead, so we don't want
 640     # the std includes.
 641     -nostdinc++
 642     -cxx-isystem"${msan_prefix}/include/c++/v1"
 643   )
 644
 645   local msan_linker_flags=(
 646     -L"${msan_prefix}"/lib
 647     -Wl,-rpath -Wl,"${msan_prefix}"/lib/
 648   )
 649
 650   CMAKE_C_FLAGS+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}"
 651   CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}"
 652   CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 653   CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 654   CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
 655   strip_dead_code
 656   cmake_configure "$@" \
 657     -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \
 658     -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_WARNINGS_AS_ERRORS=OFF \
 659     -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}"
 660   cmake_build_and_test
 661 }
 662
 663 # Install libc++ libraries compiled with msan in the msan_prefix for the current
 664 # compiler version.
 665 cmd_msan_install() {
 666   local tmpdir=$(mktemp -d)
 667   CLEANUP_FILES+=("${tmpdir}")
 668   # Detect the llvm to install:
 669   export CC="${CC:-clang}"
 670   export CXX="${CXX:-clang++}"
 671   detect_clang_version
 672   # Allow overriding the LLVM checkout.
 673   local llvm_root="${LLVM_ROOT:-}"
 674   if [ -z "${llvm_root}" ]; then
 675     local llvm_tag="llvmorg-${CLANG_VERSION}.0.0"
 676     case "${CLANG_VERSION}" in
 677       "6.0")
 678         llvm_tag="llvmorg-6.0.1"
 679         ;;
 680       "7")
 681         llvm_tag="llvmorg-7.0.1"
 682         ;;
 683     esac
 684     local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz"
 685     curl -L --show-error -o "${llvm_targz}" \
 686       "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz"
 687     tar -C "${tmpdir}" -zxf "${llvm_targz}"
 688     llvm_root="${tmpdir}/llvm-project-${llvm_tag}"
 689   fi
 690
 691   local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
 692   rm -rf "${msan_prefix}"
 693
 694   local TARGET_OPTS=""
 695   if [[ -n "${BUILD_TARGET}" ]]; then
 696     TARGET_OPTS=" \
 697       -DCMAKE_C_COMPILER_TARGET=\"${BUILD_TARGET}\" \
 698       -DCMAKE_CXX_COMPILER_TARGET=\"${BUILD_TARGET}\" \
 699       -DCMAKE_SYSTEM_PROCESSOR=\"${BUILD_TARGET%%-*}\" \
 700     "
 701   fi
 702
 703   declare -A CMAKE_EXTRAS
 704   CMAKE_EXTRAS[libcxx]="\
 705     -DLIBCXX_CXX_ABI=libstdc++ \
 706     -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON"
 707
 708   for project in libcxx; do
 709     local proj_build="${tmpdir}/build-${project}"
 710     local proj_dir="${llvm_root}/${project}"
 711     mkdir -p "${proj_build}"
 712     cmake -B"${proj_build}" -H"${proj_dir}" \
 713       -G Ninja \
 714       -DCMAKE_BUILD_TYPE=Release \
 715       -DLLVM_USE_SANITIZER=Memory \
 716       -DLLVM_PATH="${llvm_root}/llvm" \
 717       -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \
 718                             head -n1)" \
 719       -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
 720       -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
 721       -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
 722       -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \
 723       -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \
 724       ${TARGET_OPTS} \
 725       ${CMAKE_EXTRAS[${project}]}
 726     cmake --build "${proj_build}"
 727     ninja -C "${proj_build}" install
 728   done
 729 }
 730
 731 # Internal build step shared between all cmd_ossfuzz_* commands.
 732 _cmd_ossfuzz() {
 733   local sanitizer="$1"
 734   shift
 735   mkdir -p "${BUILD_DIR}"
 736   local real_build_dir=$(realpath "${BUILD_DIR}")
 737
 738   # oss-fuzz defines three directories:
 739   # * /work, with the working directory to do re-builds
 740   # * /src, with the source code to build
 741   # * /out, with the output directory where to copy over the built files.
 742   # We use $BUILD_DIR as the /work and the script directory as the /src. The
 743   # /out directory is ignored as developers are used to look for the fuzzers in
 744   # $BUILD_DIR/tools/ directly.
 745
 746   if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then
 747     sudo docker run --rm -i \
 748       --user $(id -u):$(id -g) \
 749       -v "${real_build_dir}":/work \
 750       gcr.io/oss-fuzz-base/msan-libs-builder \
 751       bash -c "cp -r /msan /work"
 752   fi
 753
 754   # Args passed to ninja. These will be evaluated as a string separated by
 755   # spaces.
 756   local jpegxl_extra_args="$@"
 757
 758   sudo docker run --rm -i \
 759     -e JPEGXL_UID=$(id -u) \
 760     -e JPEGXL_GID=$(id -g) \
 761     -e FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" \
 762     -e SANITIZER="${sanitizer}" \
 763     -e ARCHITECTURE=x86_64 \
 764     -e FUZZING_LANGUAGE=c++ \
 765     -e MSAN_LIBS_PATH="/work/msan" \
 766     -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \
 767     -v "${MYDIR}":/src/libjxl \
 768     -v "${MYDIR}/tools/scripts/ossfuzz-build.sh":/src/build.sh \
 769     -v "${real_build_dir}":/work \
 770     gcr.io/oss-fuzz/libjxl
 771 }
 772
 773 cmd_ossfuzz_asan() {
 774   _cmd_ossfuzz address "$@"
 775 }
 776 cmd_ossfuzz_msan() {
 777   _cmd_ossfuzz memory "$@"
 778 }
 779 cmd_ossfuzz_ubsan() {
 780   _cmd_ossfuzz undefined "$@"
 781 }
 782
 783 cmd_ossfuzz_ninja() {
 784   [[ -e "${BUILD_DIR}/build.ninja" ]]
 785   local real_build_dir=$(realpath "${BUILD_DIR}")
 786
 787   if [[ -e "${BUILD_DIR}/msan" ]]; then
 788     echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2
 789     exit 1
 790   fi
 791
 792   sudo docker run --rm -i \
 793     --user $(id -u):$(id -g) \
 794     -v "${MYDIR}":/src/libjxl \
 795     -v "${real_build_dir}":/work \
 796     gcr.io/oss-fuzz/libjxl \
 797     ninja -C /work "$@"
 798 }
 799
 800 cmd_fast_benchmark() {
 801   local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar"
 802   local small_corpus_url="https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
 803   mkdir -p "${BENCHMARK_CORPORA}"
 804   if [ -f "${small_corpus_tar}" ]; then
 805     curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" "${small_corpus_url}"
 806   else
 807     curl --show-error -o "${small_corpus_tar}" "${small_corpus_url}"
 808   fi
 809
 810   local tmpdir=$(mktemp -d)
 811   CLEANUP_FILES+=("${tmpdir}")
 812   tar -xf "${small_corpus_tar}" -C "${tmpdir}"
 813
 814   run_benchmark "${tmpdir}" 1048576
 815 }
 816
 817 cmd_benchmark() {
 818   local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar"
 819   mkdir -p "${BENCHMARK_CORPORA}"
 820   curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \
 821     "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar"
 822
 823   local tmpdir=$(mktemp -d)
 824   CLEANUP_FILES+=("${tmpdir}")
 825   tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}"
 826
 827   local sem_id="jpegxl_benchmark-$$"
 828   local nprocs=$(nproc --all || echo 1)
 829   images=()
 830   local filename
 831   while IFS= read -r filename; do
 832     # This removes the './'
 833     filename="${filename:2}"
 834     local mode
 835     if [[ "${filename:0:4}" == "srgb" ]]; then
 836       mode="RGB_D65_SRG_Rel_SRG"
 837     elif [[ "${filename:0:5}" == "adobe" ]]; then
 838       mode="RGB_D65_Ado_Rel_Ado"
 839     else
 840       echo "Unknown image colorspace: ${filename}" >&2
 841       exit 1
 842     fi
 843     png_filename="${filename%.ppm}.png"
 844     png_filename=$(echo "${png_filename}" | tr '/' '_')
 845     sem --bg --id "${sem_id}" -j"${nprocs}" -- \
 846       "${TOOLS_DIR}/decode_and_encode" \
 847         "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}"
 848     images+=( "${png_filename}" )
 849   done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f)
 850   sem --id "${sem_id}" --wait
 851
 852   # We need about 10 GiB per thread on these images.
 853   run_benchmark "${tmpdir}" 10485760
 854 }
 855
 856 get_mem_available() {
 857   if [[ "${OS}" == "Darwin" ]]; then
 858     echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}')
 859   elif [[ "${OS}" == MINGW* ]]; then
 860     echo $(vmstat | tail -n 1 | awk '{print $4 * 4}')
 861   else
 862     echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}')
 863   fi
 864 }
 865
 866 run_benchmark() {
 867   local src_img_dir="$1"
 868   local mem_per_thread="${2:-10485760}"
 869
 870   local output_dir="${BUILD_DIR}/benchmark_results"
 871   mkdir -p "${output_dir}"
 872
 873   if [[ "${OS}" == MINGW* ]]; then
 874     src_img_dir=`cygpath -w "${src_img_dir}"`
 875   fi
 876
 877   local num_threads=1
 878   if [[ ${BENCHMARK_NUM_THREADS} -gt 0 ]]; then
 879     num_threads=${BENCHMARK_NUM_THREADS}
 880   else
 881     # The memory available at the beginning of the benchmark run in kB. The number
 882     # of threads depends on the available memory, and the passed memory per
 883     # thread. We also add a 2 GiB of constant memory.
 884     local mem_available="$(get_mem_available)"
 885     # Check that we actually have a MemAvailable value.
 886     [[ -n "${mem_available}" ]]
 887     num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} ))
 888     if [[ ${num_threads} -le 0 ]]; then
 889       num_threads=1
 890     fi
 891   fi
 892
 893   local benchmark_args=(
 894     --input "${src_img_dir}/*.png"
 895     --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2
 896     --output_dir "${output_dir}"
 897     --show_progress
 898     --num_threads="${num_threads}"
 899     --decode_reps=11
 900     --encode_reps=11
 901   )
 902   if [[ "${STORE_IMAGES}" == "1" ]]; then
 903     benchmark_args+=(--save_decompressed --save_compressed)
 904   fi
 905   (
 906     [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
 907     "${TOOLS_DIR}/benchmark_xl" "${benchmark_args[@]}" | \
 908        tee "${output_dir}/results.txt"
 909
 910     # Check error code for benckmark_xl command. This will exit if not.
 911     return ${PIPESTATUS[0]}
 912   )
 913 }
 914
 915 # Helper function to wait for the CPU temperature to cool down on ARM.
 916 wait_for_temp() {
 917   { set +x; } 2>/dev/null
 918   local temp_limit=${1:-38000}
 919   if [[ -z "${THERMAL_FILE:-}" ]]; then
 920     echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \
 921       "to read the temperature from. This is normally set in the runner." >&2
 922     exit 1
 923   fi
 924   local org_temp=$(cat "${THERMAL_FILE}")
 925   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 926     echo -n "Waiting for temp to get down from ${org_temp}... "
 927   fi
 928   local temp="${org_temp}"
 929   local secs=0
 930   while [[ "${temp}" -ge "${temp_limit}" ]]; do
 931     sleep 1
 932     temp=$(cat "${THERMAL_FILE}")
 933     echo -n "${temp} "
 934     secs=$((secs + 1))
 935     if [[ ${secs} -ge 5 ]]; then
 936       break
 937     fi
 938   done
 939   if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
 940     echo "Done, temp=${temp}"
 941   fi
 942   set -x
 943 }
 944
 945 # Helper function to set the cpuset restriction of the current process.
 946 cmd_cpuset() {
 947   [[ "${SKIP_CPUSET:-}" != "1" ]] || return 0
 948   local newset="$1"
 949   local mycpuset=$(cat /proc/self/cpuset)
 950   mycpuset="/dev/cpuset${mycpuset}"
 951   # Check that the directory exists:
 952   [[ -d "${mycpuset}" ]]
 953   if [[ -e "${mycpuset}/cpuset.cpus" ]]; then
 954     echo "${newset}" >"${mycpuset}/cpuset.cpus"
 955   else
 956     echo "${newset}" >"${mycpuset}/cpus"
 957   fi
 958 }
 959
 960 # Return the encoding/decoding speed from the Stats output.
 961 _speed_from_output() {
 962   local speed="$1"
 963   local unit="${2:-MP/s}"
 964   if [[ "${speed}" == *"${unit}"* ]]; then
 965     speed="${speed%% ${unit}*}"
 966     speed="${speed##* }"
 967     echo "${speed}"
 968   fi
 969 }
 970
 971
 972 # Run benchmarks on ARM for the big and little CPUs.
 973 cmd_arm_benchmark() {
 974   # Flags used for cjxl encoder with .png inputs
 975   local jxl_png_benchmarks=(
 976     # Lossy options:
 977     "--epf=0 --distance=1.0 --speed=cheetah"
 978     "--epf=2 --distance=1.0 --speed=cheetah"
 979     "--epf=0 --distance=8.0 --speed=cheetah"
 980     "--epf=1 --distance=8.0 --speed=cheetah"
 981     "--epf=2 --distance=8.0 --speed=cheetah"
 982     "--epf=3 --distance=8.0 --speed=cheetah"
 983     "--modular -Q 90"
 984     "--modular -Q 50"
 985     # Lossless options:
 986     "--modular"
 987     "--modular -E 0 -I 0"
 988     "--modular -P 5"
 989     "--modular --responsive=1"
 990     # Near-lossless options:
 991     "--epf=0 --distance=0.3 --speed=fast"
 992     "--modular -Q 97"
 993   )
 994
 995   # Flags used for cjxl encoder with .jpg inputs. These should do lossless
 996   # JPEG recompression (of pixels or full jpeg).
 997   local jxl_jpeg_benchmarks=(
 998     "--num_reps=3"
 999   )
1000
1001   local images=(
1002     "testdata/jxl/flower/flower.png"
1003   )
1004
1005   local jpg_images=(
1006     "testdata/jxl/flower/flower.png.im_q85_420.jpg"
1007   )
1008
1009   if [[ "${SKIP_CPUSET:-}" == "1" ]]; then
1010     # Use a single cpu config in this case.
1011     local cpu_confs=("?")
1012   else
1013     # Otherwise the CPU config comes from the environment:
1014     local cpu_confs=(
1015       "${RUNNER_CPU_LITTLE}"
1016       "${RUNNER_CPU_BIG}"
1017       # The CPU description is something like 3-7, so these configurations only
1018       # take the first CPU of the group.
1019       "${RUNNER_CPU_LITTLE%%-*}"
1020       "${RUNNER_CPU_BIG%%-*}"
1021     )
1022     # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will
1023     # be ignored but still evaluated when calling cmd_cpuset.
1024     [[ -n "${RUNNER_CPU_ALL}" ]]
1025   fi
1026
1027   local jpg_dirname="third_party/corpora/jpeg"
1028   mkdir -p "${jpg_dirname}"
1029   local jpg_qualities=( 50 80 95 )
1030   for src_img in "${images[@]}"; do
1031     for q in "${jpg_qualities[@]}"; do
1032       local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg"
1033       convert -sampling-factor 1x1 -quality "${q}" \
1034         "${src_img}" "${jpeg_name}"
1035       jpg_images+=("${jpeg_name}")
1036     done
1037   done
1038
1039   local output_dir="${BUILD_DIR}/benchmark_results"
1040   mkdir -p "${output_dir}"
1041   local runs_file="${output_dir}/runs.txt"
1042
1043   if [[ ! -e "${runs_file}" ]]; then
1044     echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" |
1045       tee -a "${runs_file}"
1046   fi
1047
1048   mkdir -p "${BUILD_DIR}/arm_benchmark"
1049   local flags
1050   local src_img
1051   for src_img in "${jpg_images[@]}" "${images[@]}"; do
1052     local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ')
1053     local enc_binaries=("${TOOLS_DIR}/cjxl")
1054     local src_ext="${src_img##*.}"
1055     for enc_binary in "${enc_binaries[@]}"; do
1056       local enc_binary_base=$(basename "${enc_binary}")
1057
1058       # Select the list of flags to use for the current encoder/image pair.
1059       local img_benchmarks
1060       if [[ "${src_ext}" == "jpg" ]]; then
1061         img_benchmarks=("${jxl_jpeg_benchmarks[@]}")
1062       else
1063         img_benchmarks=("${jxl_png_benchmarks[@]}")
1064       fi
1065
1066       for flags in "${img_benchmarks[@]}"; do
1067         # Encoding step.
1068         local enc_file_hash="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}"
1069         enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ')
1070         local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl"
1071
1072         for cpu_conf in "${cpu_confs[@]}"; do
1073           cmd_cpuset "${cpu_conf}"
1074           # nproc returns the number of active CPUs, which is given by the cpuset
1075           # mask.
1076           local num_threads="$(nproc)"
1077
1078           echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1079           local enc_output
1080           if [[ "${flags}" == *"modular"* ]]; then
1081             # We don't benchmark encoding speed in this case.
1082             if [[ ! -f "${enc_file}" ]]; then
1083               cmd_cpuset "${RUNNER_CPU_ALL:-}"
1084               "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp"
1085               mv "${enc_file}.tmp" "${enc_file}"
1086               cmd_cpuset "${cpu_conf}"
1087             fi
1088             enc_output=" ?? MP/s"
1089           else
1090             wait_for_temp
1091             enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \
1092               2>&1 | tee /dev/stderr | grep -F "MP/s [")
1093             mv "${enc_file}.tmp" "${enc_file}"
1094           fi
1095           local enc_speed=$(_speed_from_output "${enc_output}")
1096           local enc_size=$(stat -c "%s" "${enc_file}")
1097
1098           echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1099
1100           local dec_output
1101           wait_for_temp
1102           dec_output=$("${TOOLS_DIR}/djxl" "${enc_file}" \
1103             --num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr |
1104             grep -E "M[BP]/s \[")
1105           local img_size=$(echo "${dec_output}" | cut -f 1 -d ',')
1106           local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ')
1107           local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ')
1108           local img_size_px=$(( ${img_size_x} * ${img_size_y} ))
1109           local dec_speed=$(_speed_from_output "${dec_output}")
1110
1111           # For JPEG lossless recompression modes (where the original is a JPEG)
1112           # decode to JPG as well.
1113           local jpeg_dec_mps_speed=""
1114           local jpeg_dec_mbs_speed=""
1115           if [[ "${src_ext}" == "jpg" ]]; then
1116             wait_for_temp
1117             local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg"
1118             dec_output=$("${TOOLS_DIR}/djxl" "${enc_file}" \
1119               "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \
1120                 tee /dev/stderr | grep -E "M[BP]/s \[")
1121             local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}")
1122             local jpeg_dec_mbs_speed=$(_speed_from_output "${dec_output}" MB/s)
1123             if ! cmp --quiet "${src_img}" "${dec_file}"; then
1124               # Add a start at the end to signal that the files are different.
1125               jpeg_dec_mbs_speed+="*"
1126             fi
1127           fi
1128
1129           # Record entry in a tab-separated file.
1130           local src_img_base=$(basename "${src_img}")
1131           echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" |
1132             tee -a "${runs_file}"
1133         done
1134       done
1135     done
1136   done
1137   cmd_cpuset "${RUNNER_CPU_ALL:-}"
1138   cat "${runs_file}"
1139
1140 }
1141
1142 # Generate a corpus and run the fuzzer on that corpus.
1143 cmd_fuzz() {
1144   local corpus_dir=$(realpath "${BUILD_DIR}/fuzzer_corpus")
1145   local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash")
1146   mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}"
1147   # Generate step.
1148   "${TOOLS_DIR}/fuzzer_corpus" "${corpus_dir}"
1149   # Run step:
1150   local nprocs=$(nproc --all || echo 1)
1151   (
1152    cd "${TOOLS_DIR}"
1153    djxl_fuzzer "${fuzzer_crash_dir}" "${corpus_dir}" \
1154      -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \
1155      -artifact_prefix="${fuzzer_crash_dir}/"
1156   )
1157 }
1158
1159 # Runs the linters (clang-format, build_cleaner, buildirier) on the pending CLs.
1160 cmd_lint() {
1161   merge_request_commits
1162   { set +x; } 2>/dev/null
1163   local versions=(${1:-16 15 14 13 12 11 10 9 8 7 6.0})
1164   local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format)
1165   local tmpdir=$(mktemp -d)
1166   CLEANUP_FILES+=("${tmpdir}")
1167
1168   local ret=0
1169   local build_patch="${tmpdir}/build_cleaner.patch"
1170   if ! "${MYDIR}/tools/scripts/build_cleaner.py" >"${build_patch}"; then
1171     ret=1
1172     echo "build_cleaner.py findings:" >&2
1173     "${COLORDIFF_BIN}" <"${build_patch}"
1174     echo "Run \`tools/scripts/build_cleaner.py --update\` to apply them" >&2
1175   fi
1176
1177   # It is ok, if buildifier is not installed.
1178   if which buildifier >/dev/null; then
1179     local buildifier_patch="${tmpdir}/buildifier.patch"
1180     local bazel_files=`git -C ${MYDIR} ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`
1181     set -x
1182     buildifier -d ${bazel_files} >"${buildifier_patch}"|| true
1183     { set +x; } 2>/dev/null
1184     if [ -s "${buildifier_patch}" ]; then
1185       ret=1
1186       echo 'buildifier have found some problems in Bazel build files:' >&2
1187       "${COLORDIFF_BIN}" <"${buildifier_patch}"
1188       echo 'To fix them run (from the base directory):' >&2
1189       echo '  buildifier `git ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`' >&2
1190     fi
1191   fi
1192
1193   local installed=()
1194   local clang_patch
1195   local clang_format
1196   for clang_format in "${clang_format_bins[@]}"; do
1197     if ! which "${clang_format}" >/dev/null; then
1198       continue
1199     fi
1200     installed+=("${clang_format}")
1201     local tmppatch="${tmpdir}/${clang_format}.patch"
1202     # We include in this linter all the changes including the uncommitted changes
1203     # to avoid printing changes already applied.
1204     set -x
1205     # Ignoring the error that git-clang-format outputs.
1206     git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \
1207       --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true
1208     { set +x; } 2>/dev/null
1209     if grep -E '^--- ' "${tmppatch}" | grep -v 'a/third_party' >/dev/null; then
1210       if [[ -n "${LINT_OUTPUT:-}" ]]; then
1211         cp "${tmppatch}" "${LINT_OUTPUT}"
1212       fi
1213       clang_patch="${tmppatch}"
1214     else
1215       echo "clang-format check OK" >&2
1216       return ${ret}
1217     fi
1218   done
1219
1220   if [[ ${#installed[@]} -eq 0 ]]; then
1221     echo "You must install clang-format for \"git clang-format\"" >&2
1222     exit 1
1223   fi
1224
1225   # clang-format is installed but found problems.
1226   echo "clang-format findings:" >&2
1227   "${COLORDIFF_BIN}" < "${clang_patch}"
1228
1229   echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \
1230     "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \
1231     "directory to apply them." >&2
1232   exit 1
1233 }
1234
1235 # Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs
1236 # clang-tidy over all the source files instead.
1237 cmd_tidy() {
1238   local what="${1:-}"
1239
1240   if [[ -z "${CLANG_TIDY_BIN}" ]]; then
1241     echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2
1242     exit 1
1243   fi
1244
1245   local git_args=()
1246   if [[ "${what}" == "all" ]]; then
1247     git_args=(ls-files)
1248     shift
1249   else
1250     merge_request_commits
1251     git_args=(
1252         diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}"
1253         "${MR_HEAD_SHA}"
1254     )
1255   fi
1256
1257   # Clang-tidy needs the compilation database generated by cmake.
1258   if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then
1259     # Generate the build options in debug mode, since we need the debug asserts
1260     # enabled for the clang-tidy analyzer to use them.
1261     CMAKE_BUILD_TYPE="Debug"
1262     cmake_configure
1263     # Build the autogen targets to generate the .h files from the .ui files.
1264     local autogen_targets=(
1265         $(ninja -C "${BUILD_DIR}" -t targets | grep -F _autogen: |
1266           cut -f 1 -d :)
1267     )
1268     if [[ ${#autogen_targets[@]} != 0 ]]; then
1269       ninja -C "${BUILD_DIR}" "${autogen_targets[@]}"
1270     fi
1271   fi
1272
1273   cd "${MYDIR}"
1274   local nprocs=$(nproc --all || echo 1)
1275   local ret=0
1276   if ! parallel -j"${nprocs}" --keep-order -- \
1277       "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \
1278       < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \
1279       >"${BUILD_DIR}/clang-tidy.txt"; then
1280     ret=1
1281   fi
1282   { set +x; } 2>/dev/null
1283   echo "Findings statistics:" >&2
1284   grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \
1285     | uniq -c >&2
1286
1287   if [[ $ret -ne 0 ]]; then
1288     cat >&2 <<EOF
1289 Errors found, see ${BUILD_DIR}/clang-tidy.txt for details.
1290 To automatically fix them, run:
1291
1292   SKIP_TEST=1 ./ci.sh debug
1293   ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$')
1294 EOF
1295   fi
1296
1297   return ${ret}
1298 }
1299
1300 # Print stats about all the packages built in ${BUILD_DIR}/debs/.
1301 cmd_debian_stats() {
1302   { set +x; } 2>/dev/null
1303   local debsdir="${BUILD_DIR}/debs"
1304   local f
1305   while IFS='' read -r -d '' f; do
1306     echo "====================================================================="
1307     echo "Package $f:"
1308     dpkg --info $f
1309     dpkg --contents $f
1310   done < <(find "${BUILD_DIR}/debs" -maxdepth 1 -mindepth 1 -type f \
1311            -name '*.deb' -print0)
1312 }
1313
1314 build_debian_pkg() {
1315   local srcdir="$1"
1316   local srcpkg="$2"
1317   local options="${3:-}"
1318
1319   local debsdir="${BUILD_DIR}/debs"
1320   local builddir="${debsdir}/${srcpkg}"
1321
1322   # debuild doesn't have an easy way to build out of tree, so we make a copy
1323   # of with all symlinks on the first level.
1324   mkdir -p "${builddir}"
1325   for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do
1326     if [[ ! -L "${builddir}/$f" ]]; then
1327       rm -f "${builddir}/$f"
1328       ln -s "${srcdir}/$f" "${builddir}/$f"
1329     fi
1330   done
1331   (
1332     cd "${builddir}"
1333     debuild "${options}" -b -uc -us
1334   )
1335 }
1336
1337 cmd_debian_build() {
1338   local srcpkg="${1:-}"
1339
1340   case "${srcpkg}" in
1341     jpeg-xl)
1342       build_debian_pkg "${MYDIR}" "jpeg-xl"
1343       ;;
1344     highway)
1345       build_debian_pkg "${MYDIR}/third_party/highway" "highway" "${HWY_PKG_OPTIONS}"
1346       ;;
1347     *)
1348       echo "ERROR: Must pass a valid source package name to build." >&2
1349       ;;
1350   esac
1351 }
1352
1353 get_version() {
1354   local varname=$1
1355   local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1)
1356   [[ -n "${line}" ]]
1357   line="${line#set(${varname} }"
1358   line="${line%)}"
1359   echo "${line}"
1360 }
1361
1362 cmd_bump_version() {
1363   local newver="${1:-}"
1364
1365   if ! which dch >/dev/null; then
1366     echo "Missing dch\nTo install it run:\n  sudo apt install devscripts"
1367     exit 1
1368   fi
1369
1370   if [[ -z "${newver}" ]]; then
1371     local major=$(get_version JPEGXL_MAJOR_VERSION)
1372     local minor=$(get_version JPEGXL_MINOR_VERSION)
1373     local patch=0
1374     minor=$(( ${minor}  + 1))
1375   else
1376     local major="${newver%%.*}"
1377     newver="${newver#*.}"
1378     local minor="${newver%%.*}"
1379     newver="${newver#${minor}}"
1380     local patch="${newver#.}"
1381     if [[ -z "${patch}" ]]; then
1382       patch=0
1383     fi
1384   fi
1385
1386   newver="${major}.${minor}.${patch}"
1387
1388   echo "Bumping version to ${newver} (${major}.${minor}.${patch})"
1389   sed -E \
1390     -e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \
1391     -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \
1392     -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \
1393     -i lib/CMakeLists.txt
1394   sed -E \
1395     -e "s/(LIBJXL_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}.${patch}\"/" \
1396     -e "s/(LIBJXL_ABI_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}\"/" \
1397     -i .github/workflows/conformance.yml
1398
1399   # Update lib.gni
1400   tools/scripts/build_cleaner.py --update
1401
1402   # Mark the previous version as "unstable".
1403   DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release ''
1404   DEBCHANGE_RELEASE_HEURISTIC=log dch -M \
1405     --newversion "${newver}" \
1406     "Bump JPEG XL version to ${newver}."
1407 }
1408
1409 # Check that the AUTHORS file contains the email of the committer.
1410 cmd_authors() {
1411   merge_request_commits
1412   local emails
1413   local names
1414   readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1415   readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}")
1416   for i in "${!names[@]}"; do
1417     echo "Checking name '${names[$i]}' with email '${emails[$i]}' ..."
1418     "${MYDIR}"/tools/scripts/check_author.py "${emails[$i]}" "${names[$i]}"
1419   done
1420 }
1421
1422 main() {
1423   local cmd="${1:-}"
1424   if [[ -z "${cmd}" ]]; then
1425     cat >&2 <<EOF
1426 Use: $0 CMD
1427
1428 Where cmd is one of:
1429  opt       Build and test a Release with symbols build.
1430  debug     Build and test a Debug build (NDEBUG is not defined).
1431  release   Build and test a striped Release binary without debug information.
1432  asan      Build and test an ASan (AddressSanitizer) build.
1433  msan      Build and test an MSan (MemorySanitizer) build. Needs to have msan
1434            c++ libs installed with msan_install first.
1435  tsan      Build and test a TSan (ThreadSanitizer) build.
1436  asanfuzz  Build and test an ASan (AddressSanitizer) build for fuzzing.
1437  msanfuzz  Build and test an MSan (MemorySanitizer) build for fuzzing.
1438  test      Run the tests build by opt, debug, release, asan or msan. Useful when
1439            building with SKIP_TEST=1.
1440  gbench    Run the Google benchmark tests.
1441  fuzz      Generate the fuzzer corpus and run the fuzzer on it. Useful after
1442            building with asan or msan.
1443  benchmark Run the benchmark over the default corpus.
1444  fast_benchmark Run the benchmark over the small corpus.
1445
1446  coverage  Build and run tests with coverage support. Runs coverage_report as
1447            well.
1448  coverage_report Generate HTML, XML and text coverage report after a coverage
1449            run.
1450
1451  lint      Run the linter checks on the current commit or merge request.
1452  tidy      Run clang-tidy on the current commit or merge request.
1453  authors   Check that the last commit's author is listed in the AUTHORS file.
1454
1455  msan_install Install the libc++ libraries required to build in msan mode. This
1456               needs to be done once.
1457
1458  debian_build <srcpkg> Build the given source package.
1459  debian_stats  Print stats about the built packages.
1460
1461 oss-fuzz commands:
1462  ossfuzz_asan   Build the local source inside oss-fuzz docker with asan.
1463  ossfuzz_msan   Build the local source inside oss-fuzz docker with msan.
1464  ossfuzz_ubsan  Build the local source inside oss-fuzz docker with ubsan.
1465  ossfuzz_ninja  Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra
1466                 parameters are passed to ninja, for example "djxl_fuzzer" will
1467                 only build that ninja target. Use for faster build iteration
1468                 after one of the ossfuzz_*san commands.
1469
1470 You can pass some optional environment variables as well:
1471  - BUILD_DIR: The output build directory (by default "$$repo/build")
1472  - BUILD_TARGET: The target triplet used when cross-compiling.
1473  - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS.
1474  - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package.
1475  - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only).
1476  - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds.
1477  - LINT_OUTPUT: Path to the output patch from the "lint" command.
1478  - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark.
1479  - SKIP_BUILD=1: Skip the build stage, cmake configure only.
1480  - SKIP_TEST=1: Skip the test stage.
1481  - STORE_IMAGES=0: Makes the benchmark discard the computed images.
1482  - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB.
1483  - TEST_SELECTOR: pass additional arguments to ctest, e.g. "-R .Resample.".
1484  - STACK_SIZE=1: Generate binaries with the .stack_sizes sections.
1485
1486 These optional environment variables are forwarded to the cmake call as
1487 parameters:
1488  - CMAKE_BUILD_TYPE
1489  - CMAKE_C_FLAGS
1490  - CMAKE_CXX_FLAGS
1491  - CMAKE_C_COMPILER_LAUNCHER
1492  - CMAKE_CXX_COMPILER_LAUNCHER
1493  - CMAKE_CROSSCOMPILING_EMULATOR
1494  - CMAKE_FIND_ROOT_PATH
1495  - CMAKE_EXE_LINKER_FLAGS
1496  - CMAKE_MAKE_PROGRAM
1497  - CMAKE_MODULE_LINKER_FLAGS
1498  - CMAKE_SHARED_LINKER_FLAGS
1499  - CMAKE_TOOLCHAIN_FILE
1500
1501 Example:
1502   BUILD_DIR=/tmp/build $0 opt
1503 EOF
1504     exit 1
1505   fi
1506
1507   cmd="cmd_${cmd}"
1508   shift
1509   set -x
1510   "${cmd}" "$@"
1511 }
1512
1513 main "$@"