2 # Copyright (c) the JPEG XL Project Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style
5 # license that can be found in the LICENSE file.
7 # Continuous integration helper module. This module is meant to be called from
8 # workflows during the continuous integration build, as well as from the
9 # command line for developers.
15 MYDIR
=$
(dirname $
(realpath
"$0"))
17 ### Environment parameters:
18 TEST_STACK_LIMIT
="${TEST_STACK_LIMIT:-256}"
19 CMAKE_BUILD_TYPE
=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
20 CMAKE_PREFIX_PATH
=${CMAKE_PREFIX_PATH:-}
21 CMAKE_C_COMPILER_LAUNCHER
=${CMAKE_C_COMPILER_LAUNCHER:-}
22 CMAKE_CXX_COMPILER_LAUNCHER
=${CMAKE_CXX_COMPILER_LAUNCHER:-}
23 CMAKE_MAKE_PROGRAM
=${CMAKE_MAKE_PROGRAM:-}
24 SKIP_BUILD
="${SKIP_BUILD:-0}"
25 SKIP_TEST
="${SKIP_TEST:-0}"
26 FASTER_MSAN_BUILD
="${FASTER_MSAN_BUILD:-0}"
27 TARGETS
="${TARGETS:-all doc}"
28 TEST_SELECTOR
="${TEST_SELECTOR:-}"
29 BUILD_TARGET
="${BUILD_TARGET:-}"
30 ENABLE_WASM_SIMD
="${ENABLE_WASM_SIMD:-0}"
31 if [[ -n "${BUILD_TARGET}" ]]; then
32 BUILD_DIR
="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}"
34 BUILD_DIR
="${BUILD_DIR:-${MYDIR}/build}"
36 # Whether we should post a message in the MR when the build fails.
37 POST_MESSAGE_ON_ERROR
="${POST_MESSAGE_ON_ERROR:-1}"
38 # By default, do a lightweight debian HWY package build.
39 HWY_PKG_OPTIONS
="${HWY_PKG_OPTIONS:---set-envvar=HWY_EXTRA_CONFIG=-DBUILD_TESTING=OFF -DHWY_ENABLE_EXAMPLES=OFF -DHWY_ENABLE_CONTRIB=OFF}"
41 # Set default compilers to clang if not already set
42 export CC
=${CC:-clang}
43 export CXX
=${CXX:-clang++}
45 # Time limit for the "fuzz" command in seconds (0 means no limit).
46 FUZZER_MAX_TIME
="${FUZZER_MAX_TIME:-0}"
51 if [[ "${BUILD_TARGET%%-*}" == "x86_64" ||
52 "${BUILD_TARGET%%-*}" == "i686" ]]; then
53 # Default to building all targets, even if compiler baseline is SSE4
54 HWY_BASELINE_TARGETS
=${HWY_BASELINE_TARGETS:-HWY_EMU128}
56 HWY_BASELINE_TARGETS
=${HWY_BASELINE_TARGETS:-}
59 # Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
60 CMAKE_FLAGS
=${CMAKE_FLAGS:-}
61 CMAKE_C_FLAGS
="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}"
62 CMAKE_CXX_FLAGS
="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}"
64 CMAKE_CROSSCOMPILING_EMULATOR
=${CMAKE_CROSSCOMPILING_EMULATOR:-}
65 CMAKE_EXE_LINKER_FLAGS
=${CMAKE_EXE_LINKER_FLAGS:-}
66 CMAKE_FIND_ROOT_PATH
=${CMAKE_FIND_ROOT_PATH:-}
67 CMAKE_MODULE_LINKER_FLAGS
=${CMAKE_MODULE_LINKER_FLAGS:-}
68 CMAKE_SHARED_LINKER_FLAGS
=${CMAKE_SHARED_LINKER_FLAGS:-}
69 CMAKE_TOOLCHAIN_FILE
=${CMAKE_TOOLCHAIN_FILE:-}
71 if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then
72 CMAKE_CXX_FLAGS
="${CMAKE_CXX_FLAGS} -msimd128"
73 CMAKE_C_FLAGS
="${CMAKE_C_FLAGS} -msimd128"
74 CMAKE_EXE_LINKER_FLAGS
="${CMAKE_EXE_LINKER_FLAGS} -msimd128"
77 if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then
78 CMAKE_CXX_FLAGS
="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2"
79 CMAKE_C_FLAGS
="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2"
82 if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then
83 CMAKE_CXX_FLAGS
="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}"
86 # Version inferred from the CI variables.
87 CI_COMMIT_SHA
=${GITHUB_SHA:-}
88 JPEGXL_VERSION
=${JPEGXL_VERSION:-}
90 # Benchmark parameters
91 STORE_IMAGES
=${STORE_IMAGES:-1}
92 BENCHMARK_CORPORA
="${MYDIR}/third_party/corpora"
94 # Local flags passed to sanitizers.
101 -fsanitize=float-cast-overflow
102 -fsanitize=float-divide-by-zero
103 -fsanitize=integer-divide-by-zero
105 -fsanitize=object-size
106 -fsanitize=pointer-overflow
108 -fsanitize=returns-nonnull-attribute
109 -fsanitize=shift-base
110 -fsanitize=shift-exponent
111 -fsanitize=unreachable
114 -fno-sanitize-recover=undefined
115 # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning.
116 -fsanitize-recover=alignment
118 # -fsanitize=function doesn't work on aarch64 and arm.
119 if [[ "${BUILD_TARGET%%-*}" != "aarch64" &&
120 "${BUILD_TARGET%%-*}" != "arm" ]]; then
125 if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then
127 -fsanitize=signed-integer-overflow
131 CLANG_TIDY_BIN
=$
(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy |
head -n 1)
132 # Default to "cat" if "colordiff" is not installed or if stdout is not a tty.
134 COLORDIFF_BIN
=$
(which colordiff
cat |
head -n 1)
138 FIND_BIN
=$
(which gfind
find |
head -n 1)
139 # "false" will disable wine64 when not installed. This won't allow
141 WINE_BIN
=$
(which wine64 false |
head -n 1)
143 CLANG_VERSION
="${CLANG_VERSION:-}"
144 # Detect the clang version suffix and store it in CLANG_VERSION. For example,
145 # "6.0" for clang 6 or "7" for clang 7.
146 detect_clang_version
() {
147 if [[ -n "${CLANG_VERSION}" ]]; then
150 local clang_version
=$
("${CC:-clang}" --version |
head -n1)
151 clang_version
=${clang_version#"Debian "}
152 clang_version
=${clang_version#"Ubuntu "}
154 case "${clang_version}" in
159 # Any other clang version uses just the major version number.
160 local suffix
="${clang_version#clang version }"
161 CLANG_VERSION
="${suffix%%.*}"
164 # We can't use asan or msan in the emcc case.
167 echo "Unknown clang version: ${clang_version}" >&2
172 # Temporary files cleanup hooks.
175 if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
176 rm -fr "${CLEANUP_FILES[@]}"
183 # Always cleanup the CLEANUP_FILES.
187 trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT
190 # These variables are populated when calling merge_request_commits().
192 # The current hash at the top of the current branch or merge request branch (if
193 # running from a merge request pipeline).
195 # The common ancestor between the current commit and the tracked branch, such
196 # as main. This includes a list
199 # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA.
200 merge_request_commits
() {
201 { set +x
; } 2>/dev
/null
202 # GITHUB_SHA is the current reference being build in GitHub Actions.
203 if [[ -n "${GITHUB_SHA:-}" ]]; then
204 # GitHub normally does a checkout of a merge commit on a shallow repository
205 # by default. We want to get a bit more of the history to be able to diff
206 # changes on the Pull Request if needed. This fetches 10 more commits which
207 # should be enough given that PR normally should have 1 commit.
208 git
-C "${MYDIR}" fetch
-q origin
"${GITHUB_SHA}" --depth 10
209 if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
210 MR_HEAD_SHA
="$(git rev-parse "FETCH_HEAD^
2" 2>/dev/null ||
211 echo "${GITHUB_SHA}")"
213 MR_HEAD_SHA
="${GITHUB_SHA}"
216 MR_HEAD_SHA
=$
(git
-C "${MYDIR}" rev-parse
-q "HEAD")
219 if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
220 # Pull request workflow in GitHub Actions. GitHub checkout action uses
221 # "origin" as the remote for the git checkout.
222 git
-C "${MYDIR}" fetch
-q origin
"${GITHUB_BASE_REF}"
223 MR_ANCESTOR_SHA
=$
(git
-C "${MYDIR}" rev-parse
-q FETCH_HEAD
)
225 # We are in a local branch, not a pull request workflow.
226 MR_ANCESTOR_SHA
=$
(git
-C "${MYDIR}" rev-parse
-q HEAD@
{upstream
} || true
)
229 if [[ -z "${MR_ANCESTOR_SHA}" ]]; then
230 echo "Warning, not tracking any branch, using the last commit in HEAD.">&2
231 # This prints the return value with just HEAD.
232 MR_ANCESTOR_SHA
=$
(git
-C "${MYDIR}" rev-parse
-q "${MR_HEAD_SHA}^")
234 # GitHub runs the pipeline on a merge commit, no need to look for the common
235 # ancestor in that case.
236 if [[ -z "${GITHUB_BASE_REF:-}" ]]; then
237 MR_ANCESTOR_SHA
=$
(git
-C "${MYDIR}" merge-base \
238 "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}")
245 # Set up and export the environment variables needed by the child processes.
247 if [[ "${BUILD_TARGET}" == *mingw32
]]; then
248 # Wine needs to know the paths to the mingw dlls. These should be
250 WINEPATH
=$
("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \
251 |
grep -F 'libraries: =' | cut
-f 2- -d '=' |
tr ':' ';')
252 # We also need our own libraries in the wine path.
253 local real_build_dir
=$
(realpath
"${BUILD_DIR}")
254 # Some library .dll dependencies are installed in /bin:
255 export WINEPATH
="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin"
257 local prefix
="${BUILD_DIR}/wineprefix"
259 export WINEPREFIX
=$
(realpath
"${prefix}")
261 # Sanitizers need these variables to print and properly format the stack
263 LLVM_SYMBOLIZER
=$
("${CC:-clang}" -print-prog-name=llvm-symbolizer || true
)
264 if [[ -n "${LLVM_SYMBOLIZER}" ]]; then
265 export ASAN_SYMBOLIZER_PATH
="${LLVM_SYMBOLIZER}"
266 export MSAN_SYMBOLIZER_PATH
="${LLVM_SYMBOLIZER}"
267 export UBSAN_SYMBOLIZER_PATH
="${LLVM_SYMBOLIZER}"
274 if [[ "${STACK_SIZE:-0}" == 1 ]]; then
275 # Dump the stack size of each function in the .stack_sizes section for
277 CMAKE_C_FLAGS
+=" -fstack-size-section"
278 CMAKE_CXX_FLAGS
+=" -fstack-size-section"
282 -B"${BUILD_DIR}" -H"${MYDIR}"
283 -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
285 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}"
286 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}"
287 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}"
288 -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}"
289 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}"
290 -DJPEGXL_VERSION="${JPEGXL_VERSION}"
291 -DSANITIZER="${SANITIZER}"
292 # These are not enabled by default in cmake.
293 -DJPEGXL_ENABLE_VIEWERS=ON
294 -DJPEGXL_ENABLE_PLUGINS=ON
295 -DJPEGXL_ENABLE_DEVTOOLS=ON
296 # We always use libfuzzer in the ci.sh wrapper.
297 -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer"
299 if [[ "${BUILD_TARGET}" != *mingw32
]]; then
301 -DJPEGXL_WARNINGS_AS_ERRORS=ON
304 if [[ -n "${BUILD_TARGET}" ]]; then
305 local system_name
="Linux"
306 if [[ "${BUILD_TARGET}" == *mingw32
]]; then
307 # When cross-compiling with mingw the target must be set to Windows and
308 # run programs with wine.
309 system_name
="Windows"
311 -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}"
312 # Normally CMake automatically defines MINGW=1 when building with the
313 # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling
318 # EMSCRIPTEN toolchain sets the right values itself
319 if [[ "${BUILD_TARGET}" != wasm
* ]]; then
320 # If set, BUILD_TARGET must be the target triplet such as
321 # x86_64-unknown-linux-gnu.
323 -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}"
324 -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}"
325 # Only the first element of the target triplet.
326 -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}"
327 -DCMAKE_SYSTEM_NAME="${system_name}"
328 -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}"
332 # sjpeg confuses WASM SIMD with SSE.
333 -DSJPEG_ENABLE_SIMD=OFF
334 # Building shared libs is not very useful for WASM.
335 -DBUILD_SHARED_LIBS=OFF
339 # These are needed to make googletest work when cross-compiling.
340 -DCMAKE_CROSSCOMPILING=1
343 -DHAVE_GNU_POSIX_REGEX=0
344 -DHAVE_STEADY_CLOCK=0
345 -DHAVE_THREAD_SAFETY_ATTRIBUTES=0
347 if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then
348 # find_package() will look in this prefix for libraries.
349 CMAKE_FIND_ROOT_PATH
="/usr/${BUILD_TARGET}"
351 if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then
352 CMAKE_PREFIX_PATH
="/usr/${BUILD_TARGET}"
354 # Use pkg-config for the target. If there's no pkg-config available for the
355 # target we can set the PKG_CONFIG_PATH to the appropriate path in most
356 # linux distributions.
357 local pkg_config
=$
(which "${BUILD_TARGET}-pkg-config" || true
)
358 if [[ -z "${pkg_config}" ]]; then
359 pkg_config
=$
(which pkg-config
)
360 export PKG_CONFIG_LIBDIR
="/usr/${BUILD_TARGET}/lib/pkgconfig"
362 if [[ -n "${pkg_config}" ]]; then
363 args
+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}")
366 if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then
368 -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}"
371 if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then
373 -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}"
376 if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then
378 -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
381 if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then
383 -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}"
386 if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then
388 -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}"
391 if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then
393 -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}"
396 if [[ "${BUILD_TARGET}" == wasm
* ]]; then
397 emcmake cmake
"${args[@]}" "$@"
399 cmake
"${args[@]}" "$@"
403 cmake_build_and_test
() {
404 if [[ "${SKIP_BUILD}" -eq "1" ]]; then
407 # gtest_discover_tests() runs the test binaries to discover the list of tests
408 # at build time, which fails under qemu.
409 ASAN_OPTIONS
=detect_leaks
=0 cmake
--build "${BUILD_DIR}" -- $TARGETS
410 # Pack test binaries if requested.
411 if [[ "${PACK_TEST:-}" == "1" ]]; then
413 ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*'
414 # gtest / gmock / gtest_main shared libs
415 ${FIND_BIN} lib
/ -name 'libg*.so*'
416 ${FIND_BIN} -type d
-name tests
-a '!' -path '*CMakeFiles*'
417 ) |
tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \
418 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
419 du
-h "${BUILD_DIR}/tests.tar.xz"
420 # Pack coverage data if also available.
421 touch "${BUILD_DIR}/gcno.sentinel"
422 (cd "${BUILD_DIR}"; echo gcno.sentinel
; ${FIND_BIN} -name '*gcno') | \
423 tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \
424 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
427 if [[ "${SKIP_TEST}" -ne "1" ]]; then
429 export UBSAN_OPTIONS
=print_stacktrace
=1
430 [[ "${TEST_STACK_LIMIT}" == "none" ]] ||
ulimit -s "${TEST_STACK_LIMIT}"
431 ctest
-j $
(nproc
--all ||
echo 1) ${TEST_SELECTOR} --output-on-failure)
435 # Configure the build to strip unused functions. This considerably reduces the
436 # output size, specially for tests which only use a small part of the whole
439 # Emscripten does tree shaking without any extra flags.
440 if [[ "${BUILD_TARGET}" == wasm
* ]]; then
443 # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively
444 # discard all unreachable code, reducing the code size. For this to work, we
445 # need to also pass --no-export-dynamic to prevent it from exporting all the
446 # internal symbols (like functions) making them all reachable and thus not a
447 # candidate for removal.
448 CMAKE_CXX_FLAGS
+=" -ffunction-sections -fdata-sections"
449 CMAKE_C_FLAGS
+=" -ffunction-sections -fdata-sections"
450 if [[ "${OS}" == "Darwin" ]]; then
451 CMAKE_EXE_LINKER_FLAGS
+=" -dead_strip"
452 CMAKE_SHARED_LINKER_FLAGS
+=" -dead_strip"
454 CMAKE_EXE_LINKER_FLAGS
+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
455 CMAKE_SHARED_LINKER_FLAGS
+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
459 ### Externally visible commands
462 CMAKE_BUILD_TYPE
="Debug"
468 CMAKE_BUILD_TYPE
="Release"
475 CMAKE_BUILD_TYPE
="RelWithDebInfo"
476 CMAKE_CXX_FLAGS
+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR"
482 # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests.
483 TEST_STACK_LIMIT
="none"
485 cmd_release
-DJPEGXL_ENABLE_COVERAGE=ON
"$@"
487 if [[ "${SKIP_TEST}" -ne "1" ]]; then
488 # If we didn't run the test we also don't print a coverage report.
493 cmd_coverage_report
() {
494 LLVM_COV
=$
("${CC:-clang}" -print-prog-name=llvm-cov
)
495 local real_build_dir
=$
(realpath
"${BUILD_DIR}")
497 -r "${real_build_dir}"
498 --gcov-executable "${LLVM_COV} gcov"
499 # Only print coverage information for the libjxl directories. The rest
500 # is not part of the code under test.
502 --exclude '.*_gbench.cc'
503 --exclude '.*_test.cc'
504 --exclude '.*_testonly..*'
505 --exclude '.*_debug.*'
506 --exclude '.*test_utils..*'
507 --object-directory "${real_build_dir}"
511 cd "${real_build_dir}"
512 gcovr
"${gcovr_args[@]}" --html --html-details \
513 --output="${real_build_dir}/coverage.html"
514 gcovr
"${gcovr_args[@]}" --print-summary |
515 tee "${real_build_dir}/coverage.txt"
516 gcovr
"${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml"
522 # Unpack tests if needed.
523 if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then
524 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz"
526 if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then
527 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz"
530 export UBSAN_OPTIONS
=print_stacktrace
=1
531 [[ "${TEST_STACK_LIMIT}" == "none" ]] ||
ulimit -s "${TEST_STACK_LIMIT}"
532 ctest
-j $
(nproc
--all ||
echo 1) ${TEST_SELECTOR} --output-on-failure "$@")
538 export UBSAN_OPTIONS
=print_stacktrace
=1
540 --benchmark_counters_tabular=true \
541 --benchmark_out_format=json \
542 --benchmark_out=gbench.json
"$@"
547 CMAKE_CXX_FLAGS
+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
548 CMAKE_C_FLAGS
+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
549 cmd_asan
-DJPEGXL_ENABLE_FUZZERS=ON
"$@"
553 # Install msan if needed before changing the flags.
555 local msan_prefix
="${HOME}/.msan/${CLANG_VERSION}"
556 if [[ ! -d "${msan_prefix}" ||
-e "${msan_prefix}/lib/libc++abi.a" ]]; then
557 # Install msan libraries for this version if needed or if an older version
558 # with libc++abi was installed.
562 CMAKE_CXX_FLAGS
+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
563 CMAKE_C_FLAGS
+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"
564 cmd_msan
-DJPEGXL_ENABLE_FUZZERS=ON
"$@"
569 CMAKE_C_FLAGS
+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
570 -fsanitize=address ${UBSAN_FLAGS[@]}"
571 CMAKE_CXX_FLAGS
+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
572 -fsanitize=address ${UBSAN_FLAGS[@]}"
574 cmake_configure
"$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
581 -DJXL_ENABLE_ASSERT=1
587 CMAKE_C_FLAGS
+=" ${tsan_args[@]}"
588 CMAKE_CXX_FLAGS
+=" ${tsan_args[@]}"
590 CMAKE_BUILD_TYPE
="RelWithDebInfo"
591 cmake_configure
"$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
598 local msan_prefix
="${HOME}/.msan/${CLANG_VERSION}"
599 if [[ ! -d "${msan_prefix}" ||
-e "${msan_prefix}/lib/libc++abi.a" ]]; then
600 # Install msan libraries for this version if needed or if an older version
601 # with libc++abi was installed.
607 -fno-omit-frame-pointer
609 -DJXL_ENABLE_ASSERT=1
613 # Force gtest to not use the cxxbai.
614 -DGTEST_HAS_CXXABI_H_=0
616 if [[ "${FASTER_MSAN_BUILD}" -ne "1" ]]; then
619 -fsanitize-memory-track-origins
623 local msan_cxx_flags
=(
626 # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused
627 # in those cases. Ignore the warning.
628 -Wno-unused-command-line-argument
631 # We include the libc++ from the msan directory instead, so we don't want
634 -cxx-isystem"${msan_prefix}/include/c++/v1"
637 local msan_linker_flags
=(
638 -L"${msan_prefix}"/lib
639 -Wl,-rpath -Wl,"${msan_prefix}"/lib
/
642 CMAKE_C_FLAGS
+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}"
643 CMAKE_CXX_FLAGS
+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}"
644 CMAKE_EXE_LINKER_FLAGS
+=" ${msan_linker_flags[@]}"
645 CMAKE_MODULE_LINKER_FLAGS
+=" ${msan_linker_flags[@]}"
646 CMAKE_SHARED_LINKER_FLAGS
+=" ${msan_linker_flags[@]}"
648 cmake_configure
"$@" \
649 -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \
650 -DJPEGXL_ENABLE_TCMALLOC=OFF
-DJPEGXL_WARNINGS_AS_ERRORS=OFF \
651 -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}"
655 # Install libc++ libraries compiled with msan in the msan_prefix for the current
658 local tmpdir
=$
(mktemp
-d)
659 CLEANUP_FILES
+=("${tmpdir}")
660 # Detect the llvm to install:
661 export CC
="${CC:-clang}"
662 export CXX
="${CXX:-clang++}"
664 # Allow overriding the LLVM checkout.
665 local llvm_root
="${LLVM_ROOT:-}"
666 if [ -z "${llvm_root}" ]; then
667 local llvm_tag
="llvmorg-${CLANG_VERSION}.0.0"
668 case "${CLANG_VERSION}" in
670 llvm_tag
="llvmorg-6.0.1"
673 llvm_tag
="llvmorg-7.0.1"
676 local llvm_targz
="${tmpdir}/${llvm_tag}.tar.gz"
677 curl
-L --show-error -o "${llvm_targz}" \
678 "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz"
679 tar -C "${tmpdir}" -zxf "${llvm_targz}"
680 llvm_root
="${tmpdir}/llvm-project-${llvm_tag}"
683 local msan_prefix
="${HOME}/.msan/${CLANG_VERSION}"
684 rm -rf "${msan_prefix}"
687 if [[ -n "${BUILD_TARGET}" ]]; then
689 -DCMAKE_C_COMPILER_TARGET=\"${BUILD_TARGET}\" \
690 -DCMAKE_CXX_COMPILER_TARGET=\"${BUILD_TARGET}\" \
691 -DCMAKE_SYSTEM_PROCESSOR=\"${BUILD_TARGET%%-*}\" \
695 declare -A CMAKE_EXTRAS
696 CMAKE_EXTRAS
[libcxx
]="\
697 -DLIBCXX_CXX_ABI=libstdc++ \
698 -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON"
700 for project
in libcxx
; do
701 local proj_build
="${tmpdir}/build-${project}"
702 local proj_dir
="${llvm_root}/${project}"
703 mkdir
-p "${proj_build}"
704 cmake
-B"${proj_build}" -H"${proj_dir}" \
706 -DCMAKE_BUILD_TYPE=Release \
707 -DLLVM_USE_SANITIZER=Memory \
708 -DLLVM_PATH="${llvm_root}/llvm" \
709 -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \
711 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
712 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
713 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
714 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \
715 -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \
717 ${CMAKE_EXTRAS[${project}]}
718 cmake
--build "${proj_build}"
719 ninja
-C "${proj_build}" install
723 # Internal build step shared between all cmd_ossfuzz_* commands.
727 mkdir
-p "${BUILD_DIR}"
728 local real_build_dir
=$
(realpath
"${BUILD_DIR}")
730 # oss-fuzz defines three directories:
731 # * /work, with the working directory to do re-builds
732 # * /src, with the source code to build
733 # * /out, with the output directory where to copy over the built files.
734 # We use $BUILD_DIR as the /work and the script directory as the /src. The
735 # /out directory is ignored as developers are used to look for the fuzzers in
736 # $BUILD_DIR/tools/ directly.
738 if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then
739 sudo docker run
--rm -i \
740 --user $
(id
-u):$
(id
-g) \
741 -v "${real_build_dir}":/work \
742 gcr.io
/oss-fuzz-base
/msan-libs-builder \
743 bash
-c "cp -r /msan /work"
746 # Args passed to ninja. These will be evaluated as a string separated by
748 local jpegxl_extra_args
="$@"
750 sudo docker run
--rm -i \
751 -e JPEGXL_UID
=$
(id
-u) \
752 -e JPEGXL_GID
=$
(id
-g) \
753 -e FUZZING_ENGINE
="${FUZZING_ENGINE:-libfuzzer}" \
754 -e SANITIZER
="${sanitizer}" \
755 -e ARCHITECTURE
=x86_64 \
756 -e FUZZING_LANGUAGE
=c
++ \
757 -e MSAN_LIBS_PATH
="/work/msan" \
758 -e JPEGXL_EXTRA_ARGS
="${jpegxl_extra_args}" \
759 -v "${MYDIR}":/src
/libjxl \
760 -v "${MYDIR}/tools/scripts/ossfuzz-build.sh":/src
/build.sh \
761 -v "${real_build_dir}":/work \
762 gcr.io
/oss-fuzz
/libjxl
766 _cmd_ossfuzz address
"$@"
769 _cmd_ossfuzz memory
"$@"
771 cmd_ossfuzz_ubsan
() {
772 _cmd_ossfuzz undefined
"$@"
775 cmd_ossfuzz_ninja
() {
776 [[ -e "${BUILD_DIR}/build.ninja" ]]
777 local real_build_dir
=$
(realpath
"${BUILD_DIR}")
779 if [[ -e "${BUILD_DIR}/msan" ]]; then
780 echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2
784 sudo docker run
--rm -i \
785 --user $
(id
-u):$
(id
-g) \
786 -v "${MYDIR}":/src
/libjxl \
787 -v "${real_build_dir}":/work \
788 gcr.io
/oss-fuzz
/libjxl \
792 cmd_fast_benchmark
() {
793 local small_corpus_tar
="${BENCHMARK_CORPORA}/jyrki-full.tar"
794 mkdir
-p "${BENCHMARK_CORPORA}"
795 curl
--show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \
796 "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
798 local tmpdir
=$
(mktemp
-d)
799 CLEANUP_FILES
+=("${tmpdir}")
800 tar -xf "${small_corpus_tar}" -C "${tmpdir}"
802 run_benchmark
"${tmpdir}" 1048576
806 local nikon_corpus_tar
="${BENCHMARK_CORPORA}/nikon-subset.tar"
807 mkdir
-p "${BENCHMARK_CORPORA}"
808 curl
--show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \
809 "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar"
811 local tmpdir
=$
(mktemp
-d)
812 CLEANUP_FILES
+=("${tmpdir}")
813 tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}"
815 local sem_id
="jpegxl_benchmark-$$"
816 local nprocs
=$
(nproc
--all ||
echo 1)
819 while IFS
= read -r filename
; do
820 # This removes the './'
821 filename
="${filename:2}"
823 if [[ "${filename:0:4}" == "srgb" ]]; then
824 mode
="RGB_D65_SRG_Rel_SRG"
825 elif [[ "${filename:0:5}" == "adobe" ]]; then
826 mode
="RGB_D65_Ado_Rel_Ado"
828 echo "Unknown image colorspace: ${filename}" >&2
831 png_filename
="${filename%.ppm}.png"
832 png_filename
=$
(echo "${png_filename}" |
tr '/' '_')
833 sem
--bg --id "${sem_id}" -j"${nprocs}" -- \
834 "${BUILD_DIR}/tools/decode_and_encode" \
835 "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}"
836 images
+=( "${png_filename}" )
837 done < <(cd "${tmpdir}"; ${FIND_BIN} .
-name '*.ppm' -type f
)
838 sem
--id "${sem_id}" --wait
840 # We need about 10 GiB per thread on these images.
841 run_benchmark
"${tmpdir}" 10485760
844 get_mem_available
() {
845 if [[ "${OS}" == "Darwin" ]]; then
846 echo $
(vm_stat |
grep -F 'Pages free:' |
awk '{print $3 * 4}')
848 echo $
(grep -F MemAvailable
: /proc
/meminfo |
awk '{print $2}')
853 local src_img_dir
="$1"
854 local mem_per_thread
="${2:-10485760}"
856 local output_dir
="${BUILD_DIR}/benchmark_results"
857 mkdir
-p "${output_dir}"
859 # The memory available at the beginning of the benchmark run in kB. The number
860 # of threads depends on the available memory, and the passed memory per
861 # thread. We also add a 2 GiB of constant memory.
862 local mem_available
="$(get_mem_available)"
863 # Check that we actually have a MemAvailable value.
864 [[ -n "${mem_available}" ]]
865 local num_threads
=$
(( (${mem_available} - 1048576) / ${mem_per_thread} ))
866 if [[ ${num_threads} -le 0 ]]; then
870 local benchmark_args
=(
871 --input "${src_img_dir}/*.png"
872 --codec=jpeg
:yuv420
:q85
,webp
:q80
,jxl
:d1
:6,jxl
:d1
:6:downsampling
=8,jxl
:d5
:6,jxl
:d5
:6:downsampling
=8,jxl
:m
:d0
:2,jxl
:m
:d0
:3,jxl
:m
:d2
:2
873 --output_dir "${output_dir}"
875 --num_threads="${num_threads}"
877 if [[ "${STORE_IMAGES}" == "1" ]]; then
878 benchmark_args
+=(--save_decompressed --save_compressed)
881 [[ "${TEST_STACK_LIMIT}" == "none" ]] ||
ulimit -s "${TEST_STACK_LIMIT}"
882 "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \
883 tee "${output_dir}/results.txt"
885 # Check error code for benckmark_xl command. This will exit if not.
886 return ${PIPESTATUS[0]}
892 # Helper function to wait for the CPU temperature to cool down on ARM.
894 { set +x
; } 2>/dev
/null
895 local temp_limit
=${1:-38000}
896 if [[ -z "${THERMAL_FILE:-}" ]]; then
897 echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \
898 "to read the temperature from. This is normally set in the runner." >&2
901 local org_temp
=$
(cat "${THERMAL_FILE}")
902 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
903 echo -n "Waiting for temp to get down from ${org_temp}... "
905 local temp
="${org_temp}"
907 while [[ "${temp}" -ge "${temp_limit}" ]]; do
909 temp
=$
(cat "${THERMAL_FILE}")
912 if [[ ${secs} -ge 5 ]]; then
916 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
917 echo "Done, temp=${temp}"
922 # Helper function to set the cpuset restriction of the current process.
924 [[ "${SKIP_CPUSET:-}" != "1" ]] ||
return 0
926 local mycpuset
=$
(cat /proc
/self
/cpuset
)
927 mycpuset
="/dev/cpuset${mycpuset}"
928 # Check that the directory exists:
929 [[ -d "${mycpuset}" ]]
930 if [[ -e "${mycpuset}/cpuset.cpus" ]]; then
931 echo "${newset}" >"${mycpuset}/cpuset.cpus"
933 echo "${newset}" >"${mycpuset}/cpus"
937 # Return the encoding/decoding speed from the Stats output.
938 _speed_from_output
() {
940 local unit
="${2:-MP/s}"
941 if [[ "${speed}" == *"${unit}"* ]]; then
942 speed
="${speed%% ${unit}*}"
949 # Run benchmarks on ARM for the big and little CPUs.
950 cmd_arm_benchmark
() {
951 # Flags used for cjxl encoder with .png inputs
952 local jxl_png_benchmarks
=(
954 "--epf=0 --distance=1.0 --speed=cheetah"
955 "--epf=2 --distance=1.0 --speed=cheetah"
956 "--epf=0 --distance=8.0 --speed=cheetah"
957 "--epf=1 --distance=8.0 --speed=cheetah"
958 "--epf=2 --distance=8.0 --speed=cheetah"
959 "--epf=3 --distance=8.0 --speed=cheetah"
964 "--modular -E 0 -I 0"
966 "--modular --responsive=1"
967 # Near-lossless options:
968 "--epf=0 --distance=0.3 --speed=fast"
972 # Flags used for cjxl encoder with .jpg inputs. These should do lossless
973 # JPEG recompression (of pixels or full jpeg).
974 local jxl_jpeg_benchmarks
=(
979 "testdata/jxl/flower/flower.png"
983 "testdata/jxl/flower/flower.png.im_q85_420.jpg"
986 if [[ "${SKIP_CPUSET:-}" == "1" ]]; then
987 # Use a single cpu config in this case.
988 local cpu_confs
=("?")
990 # Otherwise the CPU config comes from the environment:
992 "${RUNNER_CPU_LITTLE}"
994 # The CPU description is something like 3-7, so these configurations only
995 # take the first CPU of the group.
996 "${RUNNER_CPU_LITTLE%%-*}"
997 "${RUNNER_CPU_BIG%%-*}"
999 # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will
1000 # be ignored but still evaluated when calling cmd_cpuset.
1001 [[ -n "${RUNNER_CPU_ALL}" ]]
1004 local jpg_dirname
="third_party/corpora/jpeg"
1005 mkdir
-p "${jpg_dirname}"
1006 local jpg_qualities
=( 50 80 95 )
1007 for src_img
in "${images[@]}"; do
1008 for q
in "${jpg_qualities[@]}"; do
1009 local jpeg_name
="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg"
1010 convert
-sampling-factor 1x1
-quality "${q}" \
1011 "${src_img}" "${jpeg_name}"
1012 jpg_images
+=("${jpeg_name}")
1016 local output_dir
="${BUILD_DIR}/benchmark_results"
1017 mkdir
-p "${output_dir}"
1018 local runs_file
="${output_dir}/runs.txt"
1020 if [[ ! -e "${runs_file}" ]]; then
1021 echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" |
1022 tee -a "${runs_file}"
1025 mkdir
-p "${BUILD_DIR}/arm_benchmark"
1028 for src_img
in "${jpg_images[@]}" "${images[@]}"; do
1029 local src_img_hash
=$
(sha1sum "${src_img}" | cut
-f 1 -d ' ')
1030 local enc_binaries
=("${BUILD_DIR}/tools/cjxl")
1031 local src_ext
="${src_img##*.}"
1032 for enc_binary
in "${enc_binaries[@]}"; do
1033 local enc_binary_base
=$
(basename "${enc_binary}")
1035 # Select the list of flags to use for the current encoder/image pair.
1036 local img_benchmarks
1037 if [[ "${src_ext}" == "jpg" ]]; then
1038 img_benchmarks
=("${jxl_jpeg_benchmarks[@]}")
1040 img_benchmarks
=("${jxl_png_benchmarks[@]}")
1043 for flags
in "${img_benchmarks[@]}"; do
1045 local enc_file_hash
="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}"
1046 enc_file_hash
=$
(echo "${enc_file_hash}" |
sha1sum | cut
-f 1 -d ' ')
1047 local enc_file
="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl"
1049 for cpu_conf
in "${cpu_confs[@]}"; do
1050 cmd_cpuset
"${cpu_conf}"
1051 # nproc returns the number of active CPUs, which is given by the cpuset
1053 local num_threads
="$(nproc)"
1055 echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1057 if [[ "${flags}" == *"modular"* ]]; then
1058 # We don't benchmark encoding speed in this case.
1059 if [[ ! -f "${enc_file}" ]]; then
1060 cmd_cpuset
"${RUNNER_CPU_ALL:-}"
1061 "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp"
1062 mv "${enc_file}.tmp" "${enc_file}"
1063 cmd_cpuset
"${cpu_conf}"
1065 enc_output
=" ?? MP/s"
1068 enc_output
=$
("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \
1069 2>&1 |
tee /dev
/stderr |
grep -F "MP/s [")
1070 mv "${enc_file}.tmp" "${enc_file}"
1072 local enc_speed
=$
(_speed_from_output
"${enc_output}")
1073 local enc_size
=$
(stat
-c "%s" "${enc_file}")
1075 echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
1079 dec_output
=$
("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1080 --num_reps=5 --num_threads="${num_threads}" 2>&1 |
tee /dev
/stderr |
1081 grep -E "M[BP]/s \[")
1082 local img_size
=$
(echo "${dec_output}" | cut
-f 1 -d ',')
1083 local img_size_x
=$
(echo "${img_size}" | cut
-f 1 -d ' ')
1084 local img_size_y
=$
(echo "${img_size}" | cut
-f 3 -d ' ')
1085 local img_size_px
=$
(( ${img_size_x} * ${img_size_y} ))
1086 local dec_speed
=$
(_speed_from_output
"${dec_output}")
1088 # For JPEG lossless recompression modes (where the original is a JPEG)
1089 # decode to JPG as well.
1090 local jpeg_dec_mps_speed
=""
1091 local jpeg_dec_mbs_speed
=""
1092 if [[ "${src_ext}" == "jpg" ]]; then
1094 local dec_file
="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg"
1095 dec_output
=$
("${BUILD_DIR}/tools/djxl" "${enc_file}" \
1096 "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \
1097 tee /dev
/stderr |
grep -E "M[BP]/s \[")
1098 local jpeg_dec_mps_speed
=$
(_speed_from_output
"${dec_output}")
1099 local jpeg_dec_mbs_speed
=$
(_speed_from_output
"${dec_output}" MB
/s
)
1100 if ! cmp --quiet "${src_img}" "${dec_file}"; then
1101 # Add a start at the end to signal that the files are different.
1102 jpeg_dec_mbs_speed
+="*"
1106 # Record entry in a tab-separated file.
1107 local src_img_base
=$
(basename "${src_img}")
1108 echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" |
1109 tee -a "${runs_file}"
1114 cmd_cpuset
"${RUNNER_CPU_ALL:-}"
1119 # Generate a corpus and run the fuzzer on that corpus.
1121 local corpus_dir
=$
(realpath
"${BUILD_DIR}/fuzzer_corpus")
1122 local fuzzer_crash_dir
=$
(realpath
"${BUILD_DIR}/fuzzer_crash")
1123 mkdir
-p "${corpus_dir}" "${fuzzer_crash_dir}"
1125 "${BUILD_DIR}/tools/fuzzer_corpus" "${corpus_dir}"
1127 local nprocs
=$
(nproc
--all ||
echo 1)
1130 "tools/djxl_fuzzer" "${fuzzer_crash_dir}" "${corpus_dir}" \
1131 -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \
1132 -artifact_prefix="${fuzzer_crash_dir}/"
1136 # Runs the linters (clang-format, build_cleaner, buildirier) on the pending CLs.
1138 merge_request_commits
1139 { set +x
; } 2>/dev
/null
1140 local versions
=(${1:-16 15 14 13 12 11 10 9 8 7 6.0})
1141 local clang_format_bins
=("${versions[@]/#/clang-format-}" clang-format
)
1142 local tmpdir
=$
(mktemp
-d)
1143 CLEANUP_FILES
+=("${tmpdir}")
1146 local build_patch
="${tmpdir}/build_cleaner.patch"
1147 if ! "${MYDIR}/tools/scripts/build_cleaner.py" >"${build_patch}"; then
1149 echo "build_cleaner.py findings:" >&2
1150 "${COLORDIFF_BIN}" <"${build_patch}"
1151 echo "Run \`tools/scripts/build_cleaner.py --update\` to apply them" >&2
1154 # It is ok, if buildifier is not installed.
1155 if which buildifier
>/dev
/null
; then
1156 local buildifier_patch
="${tmpdir}/buildifier.patch"
1157 local bazel_files
=`git -C ${MYDIR} ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`
1159 buildifier
-d ${bazel_files} >"${buildifier_patch}"|| true
1160 { set +x
; } 2>/dev
/null
1161 if [ -s "${buildifier_patch}" ]; then
1163 echo 'buildifier have found some problems in Bazel build files:' >&2
1164 "${COLORDIFF_BIN}" <"${buildifier_patch}"
1165 echo 'To fix them run (from the base directory):' >&2
1166 echo ' buildifier `git ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`' >&2
1173 for clang_format
in "${clang_format_bins[@]}"; do
1174 if ! which "${clang_format}" >/dev
/null
; then
1177 installed
+=("${clang_format}")
1178 local tmppatch
="${tmpdir}/${clang_format}.patch"
1179 # We include in this linter all the changes including the uncommitted changes
1180 # to avoid printing changes already applied.
1182 # Ignoring the error that git-clang-format outputs.
1183 git
-C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \
1184 --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true
1185 { set +x
; } 2>/dev
/null
1186 if grep -E '^--- ' "${tmppatch}" |
grep -v 'a/third_party' >/dev
/null
; then
1187 if [[ -n "${LINT_OUTPUT:-}" ]]; then
1188 cp "${tmppatch}" "${LINT_OUTPUT}"
1190 clang_patch
="${tmppatch}"
1192 echo "clang-format check OK" >&2
1197 if [[ ${#installed[@]} -eq 0 ]]; then
1198 echo "You must install clang-format for \"git clang-format\"" >&2
1202 # clang-format is installed but found problems.
1203 echo "clang-format findings:" >&2
1204 "${COLORDIFF_BIN}" < "${clang_patch}"
1206 echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \
1207 "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \
1208 "directory to apply them." >&2
1212 # Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs
1213 # clang-tidy over all the source files instead.
1217 if [[ -z "${CLANG_TIDY_BIN}" ]]; then
1218 echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2
1223 if [[ "${what}" == "all" ]]; then
1227 merge_request_commits
1229 diff-tree
--no-commit-id --name-only -r "${MR_ANCESTOR_SHA}"
1234 # Clang-tidy needs the compilation database generated by cmake.
1235 if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then
1236 # Generate the build options in debug mode, since we need the debug asserts
1237 # enabled for the clang-tidy analyzer to use them.
1238 CMAKE_BUILD_TYPE
="Debug"
1240 # Build the autogen targets to generate the .h files from the .ui files.
1241 local autogen_targets
=(
1242 $
(ninja
-C "${BUILD_DIR}" -t targets |
grep -F _autogen
: |
1245 if [[ ${#autogen_targets[@]} != 0 ]]; then
1246 ninja
-C "${BUILD_DIR}" "${autogen_targets[@]}"
1251 local nprocs
=$
(nproc
--all ||
echo 1)
1253 if ! parallel
-j"${nprocs}" --keep-order -- \
1254 "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \
1255 < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \
1256 >"${BUILD_DIR}/clang-tidy.txt
"; then
1259 { set +x; } 2>/dev/null
1260 echo "Findings statistics
:" >&2
1261 grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt
" | sort \
1264 if [[ $ret -ne 0 ]]; then
1266 Errors found, see ${BUILD_DIR}/clang-tidy.txt for details.
1267 To automatically fix them, run:
1269 SKIP_TEST=1 ./ci.sh debug
1270 ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$')
1277 # Print stats about all the packages built in ${BUILD_DIR}/debs/.
1278 cmd_debian_stats() {
1279 { set +x; } 2>/dev/null
1280 local debsdir="${BUILD_DIR}/debs
"
1282 while IFS='' read -r -d '' f; do
1283 echo "====================================================================="
1287 done < <(find "${BUILD_DIR}/debs
" -maxdepth 1 -mindepth 1 -type f \
1288 -name '*.deb' -print0)
1291 build_debian_pkg() {
1294 local options="${3:-}"
1296 local debsdir="${BUILD_DIR}/debs
"
1297 local builddir="${debsdir}/${srcpkg}"
1299 # debuild doesn't have an easy way to build out of tree, so we make a copy
1300 # of with all symlinks on the first level.
1301 mkdir -p "${builddir}"
1302 for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do
1303 if [[ ! -L "${builddir}/$f" ]]; then
1304 rm -f "${builddir}/$f"
1305 ln -s "${srcdir}/$f" "${builddir}/$f"
1310 debuild "${options}" -b -uc -us
1314 cmd_debian_build() {
1315 local srcpkg="${1:-}"
1319 build_debian_pkg "${MYDIR}" "jpeg-xl
"
1322 build_debian_pkg "${MYDIR}/third_party
/highway
" "highway
" "${HWY_PKG_OPTIONS}"
1325 echo "ERROR
: Must pass a valid
source package name to build.
" >&2
1332 local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1)
1334 line="${line#set(${varname} }"
1339 cmd_bump_version() {
1340 local newver="${1:-}"
1342 if ! which dch >/dev/null; then
1343 echo "Missing dch
\nTo
install it run
:\n sudo apt
install devscripts
"
1347 if [[ -z "${newver}" ]]; then
1348 local major=$(get_version JPEGXL_MAJOR_VERSION)
1349 local minor=$(get_version JPEGXL_MINOR_VERSION)
1351 minor=$(( ${minor} + 1))
1353 local major="${newver%%.*}"
1354 newver="${newver#*.}"
1355 local minor="${newver%%.*}"
1356 newver="${newver#${minor}}"
1357 local patch="${newver#.}"
1358 if [[ -z "${patch}" ]]; then
1363 newver="${major}.${minor}.${patch}"
1365 echo "Bumping version to
${newver} (${major}.${minor}.${patch})"
1367 -e "s
/(set\\(JPEGXL_MAJOR_VERSION
) [0-9]+\\)/\\1 ${major})/" \
1368 -e "s
/(set\\(JPEGXL_MINOR_VERSION
) [0-9]+\\)/\\1 ${minor})/" \
1369 -e "s
/(set\\(JPEGXL_PATCH_VERSION
) [0-9]+\\)/\\1 ${patch})/" \
1370 -i lib/CMakeLists.txt
1372 -e "s
/(LIBJXL_VERSION
: )\"[0-9.
]+\"/\\1\"${major}.${minor}.${patch}\"/" \
1373 -e "s
/(LIBJXL_ABI_VERSION
: )\"[0-9.
]+\"/\\1\"${major}.
${minor}\"/" \
1374 -i .github/workflows/conformance.yml
1377 tools/scripts/build_cleaner.py --update
1379 # Mark the previous version as "unstable
".
1380 DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release ''
1381 DEBCHANGE_RELEASE_HEURISTIC=log dch -M \
1382 --newversion "${newver}" \
1383 "Bump JPEG XL version to
${newver}.
"
1386 # Check that the AUTHORS file contains the email of the committer.
1388 merge_request_commits
1391 readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..
${MR_HEAD_SHA}")
1392 readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..
${MR_HEAD_SHA}")
1393 for i in "${!names[@]}"; do
1394 echo "Checking name
'${names[$i]}' with email
'${emails[$i]}' ...
"
1395 "${MYDIR}"/tools/scripts/check_author.py "${emails[$i]}" "${names[$i]}"
1401 if [[ -z "${cmd}" ]]; then
1405 Where cmd is one of:
1406 opt Build and test a Release with symbols build.
1407 debug Build and test a Debug build (NDEBUG is not defined).
1408 release Build and test a striped Release binary without debug information.
1409 asan Build and test an ASan (AddressSanitizer) build.
1410 msan Build and test an MSan (MemorySanitizer) build. Needs to have msan
1411 c++ libs installed with msan_install first.
1412 tsan Build and test a TSan (ThreadSanitizer) build.
1413 asanfuzz Build and test an ASan (AddressSanitizer) build for fuzzing.
1414 msanfuzz Build and test an MSan (MemorySanitizer) build for fuzzing.
1415 test Run the tests build by opt, debug, release, asan or msan. Useful when
1416 building with SKIP_TEST=1.
1417 gbench Run the Google benchmark tests.
1418 fuzz Generate the fuzzer corpus and run the fuzzer on it. Useful after
1419 building with asan or msan.
1420 benchmark Run the benchmark over the default corpus.
1421 fast_benchmark Run the benchmark over the small corpus.
1423 coverage Build and run tests with coverage support. Runs coverage_report as
1425 coverage_report Generate HTML, XML and text coverage report after a coverage
1428 lint Run the linter checks on the current commit or merge request.
1429 tidy Run clang-tidy on the current commit or merge request.
1430 authors Check that the last commit's author is listed in the AUTHORS file.
1432 msan_install Install the libc++ libraries required to build in msan mode. This
1433 needs to be done once.
1435 debian_build <srcpkg> Build the given source package.
1436 debian_stats Print stats about the built packages.
1439 ossfuzz_asan Build the local source inside oss-fuzz docker with asan.
1440 ossfuzz_msan Build the local source inside oss-fuzz docker with msan.
1441 ossfuzz_ubsan Build the local source inside oss-fuzz docker with ubsan.
1442 ossfuzz_ninja Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra
1443 parameters are passed to ninja, for example "djxl_fuzzer
" will
1444 only build that ninja target. Use for faster build iteration
1445 after one of the ossfuzz_*san commands.
1447 You can pass some optional environment variables as well:
1448 - BUILD_DIR: The output build directory (by default "$
$repo/build
")
1449 - BUILD_TARGET: The target triplet used when cross-compiling.
1450 - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS.
1451 - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package.
1452 - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only).
1453 - FUZZER_MAX_TIME: "fuzz
" command fuzzer running timeout in seconds.
1454 - LINT_OUTPUT: Path to the output patch from the "lint
" command.
1455 - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark.
1456 - SKIP_BUILD=1: Skip the build stage, cmake configure only.
1457 - SKIP_TEST=1: Skip the test stage.
1458 - STORE_IMAGES=0: Makes the benchmark discard the computed images.
1459 - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB.
1460 - TEST_SELECTOR: pass additional arguments to ctest, e.g. "-R .Resample.
".
1461 - STACK_SIZE=1: Generate binaries with the .stack_sizes sections.
1463 These optional environment variables are forwarded to the cmake call as
1468 - CMAKE_C_COMPILER_LAUNCHER
1469 - CMAKE_CXX_COMPILER_LAUNCHER
1470 - CMAKE_CROSSCOMPILING_EMULATOR
1471 - CMAKE_FIND_ROOT_PATH
1472 - CMAKE_EXE_LINKER_FLAGS
1473 - CMAKE_MAKE_PROGRAM
1474 - CMAKE_MODULE_LINKER_FLAGS
1475 - CMAKE_SHARED_LINKER_FLAGS
1476 - CMAKE_TOOLCHAIN_FILE
1479 BUILD_DIR=/tmp/build $0 opt