1 # CI configuration file for CI utilizing cirrus-ci.org
3 # For instructions on how to enable the CI integration in a repository and
4 # further details, see src/tools/ci/README
8 # Source of images / containers
9 GCP_PROJECT: pg-ci-images
10 IMAGE_PROJECT: $GCP_PROJECT
11 CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci
13 # The lower depth accelerates git clone. Use a bit of depth so that
14 # concurrent tasks and retrying older jobs has a chance of working.
15 CIRRUS_CLONE_DEPTH: 500
16 # Useful to be able to analyse what in a script takes long
17 CIRRUS_LOG_TIMESTAMP: true
19 CCACHE_MAXSIZE: "250M"
21 # target to test, for all but windows
22 CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
25 MTEST_ARGS: --print-errorlogs --no-rebuild -C build
26 PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
27 TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
28 PG_TEST_EXTRA: kerberos ldap ssl
31 # What files to preserve in case tests fail
32 on_failure_ac: &on_failure_ac
40 on_failure_meson: &on_failure_meson
43 - "build*/testrun/**/*.log"
44 - "build*/testrun/**/*.diffs"
45 - "build*/testrun/**/regress_log_*"
48 # In theory it'd be nice to upload the junit files meson generates, so that
49 # cirrus will nicely annotate the commit. Unfortunately the files don't
50 # contain identifieable file + line numbers right now, so the annotations
51 # don't end up useful. We could probably improve on that with a some custom
52 # conversion script, but ...
54 path: "build*/meson-logs/*.txt"
58 # To avoid unnecessarily spinning up a lot of VMs / containers for entirely
59 # broken commits, have a minimal task that all others depend on.
63 # If a specific OS is requested, don't run the sanity check. This shortens
64 # push-wait-for-ci cycle time a bit when debugging operating system specific
65 # failures. Uses skip instead of only_if, as cirrus otherwise warns about
66 # only_if conditions not matching.
67 skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*'
73 CCACHE_DIR: ${CIRRUS_WORKING_DIR}/ccache_dir
74 # no options enabled, should be small
75 CCACHE_MAXSIZE: "150M"
77 # Container starts up quickly, but is slower at runtime, particularly for
78 # tests. Good for the briefly running sanity check.
80 image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
88 chown -R postgres:postgres .
89 mkdir -p ${CCACHE_DIR}
90 chown -R postgres:postgres ${CCACHE_DIR}
91 echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
92 su postgres -c "ulimit -l -H && ulimit -l -S"
93 # Can't change container's kernel.core_pattern. Postgres user can't write
94 # to / normally. Change that.
102 --auto-features=disabled \
103 -Dtap_tests=enabled \
108 ninja -C build -j${BUILD_JOBS}
110 upload_caches: ccache
112 # Run a minimal set of tests. The main regression tests take too long for
113 # this purpose. For now this is a random quick pg_regress style test, and a
114 # tap test that exercises both a frontend binary and the backend.
115 test_minimal_script: |
118 meson test $MTEST_ARGS --num-processes ${TEST_JOBS} \
119 tmp_install cube/regress pg_ctl/001_start_stop
123 <<: *on_failure_meson
125 mkdir -m 770 /tmp/cores
126 find / -maxdepth 1 -type f -name 'core*' -exec mv '{}' /tmp/cores/ \;
127 src/tools/ci/cores_backtrace.sh linux /tmp/cores
131 name: FreeBSD - 13 - Meson
134 # FreeBSD on GCP is slow when running with larger number of CPUS /
135 # jobs. Using one more job than cpus seems to work best.
140 CCACHE_DIR: /tmp/ccache_dir
141 CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
144 depends_on: SanityCheck
145 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
147 compute_engine_instance:
148 image_project: $IMAGE_PROJECT
149 image: family/pg-ci-freebsd-13
158 ulimit -a -H && ulimit -a -S
163 # Workaround around performance issues due to 32KB block size
164 repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
165 create_user_script: |
167 chown -R postgres:postgres .
168 mkdir -p ${CCACHE_DIR}
169 chown -R postgres:postgres ${CCACHE_DIR}
170 setup_core_files_script: |
171 mkdir -m 770 /tmp/cores
172 chown root:postgres /tmp/cores
173 sysctl kern.corefile='/tmp/cores/%N.%P.core'
174 setup_additional_packages_script: |
177 # NB: Intentionally build without -Dllvm. The freebsd image size is already
178 # large enough to make VM startup slow, and even without llvm freebsd
179 # already takes longer than other platforms except for windows.
184 -Dcassert=true -Dssl=openssl -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
185 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
186 -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
189 build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
190 upload_caches: ccache
195 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
199 <<: *on_failure_meson
200 cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores
203 # configure feature flags, shared between the task running the linux tests and
204 # the CompilerWarnings task
205 LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
219 --with-tcl --with-tclconfig=/usr/lib/tcl8.6/
223 LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >-
233 TEST_JOBS: 8 # experimentally derived to be a decent choice
235 CCACHE_DIR: /tmp/ccache_dir
236 DEBUGINFOD_URLS: "https://debuginfod.debian.net"
238 # Enable a reasonable set of sanitizers. Use the linux task for that, as
239 # it one of the fastest tasks (without sanitizers). Also several of the
240 # sanitizers work best on linux.
242 # The overhead of alignment sanitizer is low, undefined behaviour has
243 # moderate overhead. Test alignment sanitizer in the meson task, as it
244 # does both 32 and 64 bit builds and is thus more likely to expose
247 # Address sanitizer in contrast is somewhat expensive. Enable it in the
248 # autoconf task, as the meson task tests both 32 and 64bit.
250 # disable_coredump=0, abort_on_error=1: for useful backtraces in case of crashes
251 # print_stacktraces=1,verbosity=2, duh
252 # detect_leaks=0: too many uninteresting leak errors in short-lived binaries
253 UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2
254 ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0
256 # SANITIZER_FLAGS is set in the tasks below
257 CFLAGS: -Og -ggdb -fno-sanitize-recover=all $SANITIZER_FLAGS
259 LDFLAGS: $SANITIZER_FLAGS
263 LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
264 LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
266 depends_on: SanityCheck
267 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
269 compute_engine_instance:
270 image_project: $IMAGE_PROJECT
271 image: family/pg-ci-bullseye
277 folder: ${CCACHE_DIR}
283 ulimit -a -H && ulimit -a -S
285 create_user_script: |
287 chown -R postgres:postgres .
288 mkdir -p ${CCACHE_DIR}
289 chown -R postgres:postgres ${CCACHE_DIR}
290 echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
291 su postgres -c "ulimit -l -H && ulimit -l -S"
292 setup_core_files_script: |
293 mkdir -m 770 /tmp/cores
294 chown root:postgres /tmp/cores
295 sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core'
296 setup_additional_packages_script: |
298 #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
301 - name: Linux - Debian Bullseye - Autoconf
304 SANITIZER_FLAGS: -fsanitize=address
309 --enable-cassert --enable-debug --enable-tap-tests \
312 ${LINUX_CONFIGURE_FEATURES} \
316 build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin"
317 upload_caches: ccache
321 ulimit -c unlimited # default is 0
322 make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
328 - name: Linux - Debian Bullseye - Meson
331 CCACHE_MAXSIZE: "400M" # tests two different builds
332 SANITIZER_FLAGS: -fsanitize=alignment,undefined
339 ${LINUX_MESON_FEATURES} \
340 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
344 # Also build & test in a 32bit build - it's gotten rare to test that
346 configure_32_script: |
348 export CC='ccache gcc -m32'
352 ${LINUX_MESON_FEATURES} \
354 --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \
355 -DPERL=perl5.32-i386-linux-gnu \
356 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
360 build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
361 build_32_script: su postgres -c 'ninja -C build-32 -j${BUILD_JOBS}'
363 upload_caches: ccache
368 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
370 # so that we don't upload 64bit logs if 32bit fails
373 # There's currently no coverage of icu with LANG=C in the buildfarm. We
374 # can easily provide some here by running one of the sets of tests that
375 # way. Newer versions of python insist on changing the LC_CTYPE away
376 # from C, prevent that with PYTHONCOERCECLOCALE.
377 test_world_32_script: |
380 PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
384 <<: *on_failure_meson
387 cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
391 name: macOS - Monterey - Meson
394 CPUS: 12 # always get that much for cirrusci macOS instances
396 # Test performance regresses noticably when using all cores. 8 seems to
398 # https://postgr.es/m/20220927040208.l3shfcidovpzqxfh%40awork3.anarazel.de
401 CIRRUS_WORKING_DIR: ${HOME}/pgsql/
402 CCACHE_DIR: ${HOME}/ccache
403 HOMEBREW_CACHE: ${HOME}/homebrew-cache
404 PERL5LIB: ${HOME}/perl5/lib/perl5
408 CPPFLAGS: -DRANDOMIZE_ALLOCATED_MEMORY
412 depends_on: SanityCheck
413 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
421 ulimit -a -H && ulimit -a -S
424 setup_core_files_script:
425 - mkdir ${HOME}/cores
426 - sudo sysctl kern.corefile="${HOME}/cores/core.%P"
431 - perl -mIPC::Run -e 1 || cpan -T IPC::Run
432 - perl -mIO::Pty -e 1 || cpan -T IO::Pty
436 # XXX: Could we instead install homebrew into a cached directory? The
437 # homebrew installation takes a good bit of time every time, even if the
438 # packages do not need to be downloaded.
440 folder: $HOMEBREW_CACHE
441 setup_additional_packages_script: |
456 brew cleanup -s # to reduce cache size
457 upload_caches: homebrew
462 brewpath="/usr/local"
463 PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
465 for pkg in icu4c krb5 openldap openssl zstd ; do
466 pkgpath="${brewpath}/opt/${pkg}"
467 PKG_CONFIG_PATH="${pkgpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
468 PATH="${pkgpath}/bin:${pkgpath}/sbin:$PATH"
471 export PKG_CONFIG_PATH PATH
475 -Dextra_include_dirs=${brewpath}/include \
476 -Dextra_lib_dirs=${brewpath}/lib \
478 -Dssl=openssl -Duuid=e2fs -Ddtrace=auto \
479 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
482 build_script: ninja -C build -j${BUILD_JOBS}
483 upload_caches: ccache
486 ulimit -c unlimited # default is 0
487 ulimit -n 1024 # default is 256, pretty low
488 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
491 <<: *on_failure_meson
492 cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
495 WINDOWS_ENVIRONMENT_BASE: &WINDOWS_ENVIRONMENT_BASE
497 # Half the allowed per-user CPU cores
500 # The default cirrus working dir is in a directory msbuild complains about
501 CIRRUS_WORKING_DIR: "c:/cirrus"
502 # git's tar doesn't deal with drive letters, see
503 # https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
504 TAR: "c:/windows/system32/tar.exe"
505 # Avoids port conflicts between concurrent tap test runs
506 PG_TEST_USE_UNIX_SOCKETS: 1
507 PG_REGRESS_SOCK_DIR: "c:/cirrus/"
512 powershell -Command get-psdrive -psprovider filesystem
517 name: Windows - Server 2019, VS 2019 - Meson & ninja
518 << : *WINDOWS_ENVIRONMENT_BASE
521 TEST_JOBS: 8 # wild guess, data based value welcome
523 # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
524 # prevents crash reporting from working unless binaries do SetErrorMode()
525 # themselves. Furthermore, it appears that either python or, more likely,
526 # the C runtime has a bug where SEM_NOGPFAULTERRORBOX can very
527 # occasionally *trigger* a crash on process exit - which is hard to debug,
528 # given that it explicitly prevents crash dumps from working...
529 # 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
530 CIRRUS_WINDOWS_ERROR_MODE: 0x8001
532 depends_on: SanityCheck
533 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
536 image: $CONTAINER_REPO/windows_ci_vs_2019:latest
540 setup_additional_packages_script: |
541 REM choco install -y --no-progress ...
543 # Use /DEBUG:FASTLINK to avoid high memory usage during linking
546 meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dssl=openssl -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
552 check_world_script: |
554 meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
557 <<: *on_failure_meson
559 path: "crashlog-*.txt"
564 << : *WINDOWS_ENVIRONMENT_BASE
565 name: Windows - Server 2019, MinGW64 - Meson
567 # due to resource constraints we don't run this task by default for now
569 # worth using only_if despite being manual, otherwise this task will show up
570 # when e.g. ci-os-only: linux is used.
571 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*'
572 # otherwise it'll be sorted before other tasks
573 depends_on: SanityCheck
576 image: $CONTAINER_REPO/windows_ci_mingw64:latest
581 TEST_JOBS: 4 # higher concurrency causes occasional failures
582 CCACHE_DIR: C:/msys64/ccache
583 CCACHE_MAXSIZE: "500M"
584 CCACHE_SLOPPINESS: pch_defines,time_macros
586 # for some reason mingw plpython cannot find it's installation without this
587 PYTHONHOME: C:/msys64/ucrt64
588 # prevents MSYS bash from resetting error mode
590 # Start bash in current working directory
592 BASH: C:\msys64\usr\bin\bash.exe -l
595 folder: ${CCACHE_DIR}
597 setup_additional_packages_script: |
598 REM C:\msys64\usr\bin\pacman.exe -S --noconfirm ...
601 %BASH% -c "where gcc"
602 %BASH% -c "gcc --version"
603 %BASH% -c "where perl"
604 %BASH% -c "perl --version"
606 # disable -Dnls as the number of files it creates cause a noticable slowdown
608 %BASH% -c "meson setup -Ddebug=true -Doptimization=g -Dcassert=true -Db_pch=true -Dnls=disabled -DTAR=%TAR% build"
611 %BASH% -c "ninja -C build"
613 upload_caches: ccache
616 %BASH% -c "meson test %MTEST_ARGS% --num-processes %TEST_JOBS%"
619 <<: *on_failure_meson
621 path: "crashlog-*.txt"
626 name: CompilerWarnings
628 # To limit unnecessary work only run this once the SanityCheck
629 # succeeds. This is particularly important for this task as we intentionally
630 # use always: to continue after failures. Task that did not run count as a
631 # success, so we need to recheck SanityChecks's condition here ...
632 depends_on: SanityCheck
633 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
639 # Use larger ccache cache, as this task compiles with multiple compilers /
641 CCACHE_MAXSIZE: "1GB"
642 CCACHE_DIR: "/tmp/ccache_dir"
644 LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
645 LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
648 image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
655 ulimit -a -H && ulimit -a -S
663 setup_additional_packages_script: |
665 #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
668 # Test that code can be built with gcc/clang without warnings
671 setup_script: echo "COPT=-Werror" > src/Makefile.custom
673 # Trace probes have a history of getting accidentally broken. Use the
674 # different compilers to build with different combinations of dtrace on/off
675 # and cassert on/off.
677 # gcc, cassert off, dtrace on
679 gcc_warning_script: |
683 ${LINUX_CONFIGURE_FEATURES} \
684 CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
685 make -s -j${BUILD_JOBS} clean
686 time make -s -j${BUILD_JOBS} world-bin
688 # gcc, cassert on, dtrace off
690 gcc_a_warning_script: |
694 ${LINUX_CONFIGURE_FEATURES} \
695 CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
696 make -s -j${BUILD_JOBS} clean
697 time make -s -j${BUILD_JOBS} world-bin
699 # clang, cassert off, dtrace off
701 clang_warning_script: |
703 --cache clang.cache \
704 ${LINUX_CONFIGURE_FEATURES} \
705 CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
706 make -s -j${BUILD_JOBS} clean
707 time make -s -j${BUILD_JOBS} world-bin
709 # clang, cassert on, dtrace on
711 clang_a_warning_script: |
713 --cache clang.cache \
716 ${LINUX_CONFIGURE_FEATURES} \
717 CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
718 make -s -j${BUILD_JOBS} clean
719 time make -s -j${BUILD_JOBS} world-bin
721 # cross-compile to windows
723 mingw_cross_warning_script: |
725 --host=x86_64-w64-mingw32 \
727 CC="ccache x86_64-w64-mingw32-gcc" \
728 CXX="ccache x86_64-w64-mingw32-g++"
729 make -s -j${BUILD_JOBS} clean
730 time make -s -j${BUILD_JOBS} world-bin
733 # Verify docs can be built
735 # XXX: Only do this if there have been changes in doc/ since last build
743 make -s -j${BUILD_JOBS} clean
744 time make -s -j${BUILD_JOBS} -C doc
747 # Verify headerscheck / cpluspluscheck succeed
749 # - Don't use ccache, the files are uncacheable, polluting ccache's
751 # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
752 # - XXX have to disable ICU to avoid errors:
753 # https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
756 headers_headerscheck_script: |
758 ${LINUX_CONFIGURE_FEATURES} \
761 CC="gcc" CXX"=g++" CLANG="clang"
762 make -s -j${BUILD_JOBS} clean
763 time make -s headerscheck EXTRAFLAGS='-fmax-errors=10'
764 headers_cpluspluscheck_script: |
765 time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
768 upload_caches: ccache