1 # CI configuration file for CI utilizing cirrus-ci.org
3 # For instructions on how to enable the CI integration in a repository and
4 # further details, see src/tools/ci/README
8 # Source of images / containers
9 GCP_PROJECT: pg-ci-images
10 IMAGE_PROJECT: $GCP_PROJECT
11 CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci
13 # The lower depth accelerates git clone. Use a bit of depth so that
14 # concurrent tasks and retrying older jobs have a chance of working.
15 CIRRUS_CLONE_DEPTH: 500
16 # Useful to be able to analyse what in a script takes long
17 CIRRUS_LOG_TIMESTAMP: true
19 CCACHE_MAXSIZE: "250M"
21 # target to test, for all but windows
22 CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
25 MTEST_ARGS: --print-errorlogs --no-rebuild -C build
26 PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
27 TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
28 PG_TEST_EXTRA: kerberos ldap ssl load_balance
31 # What files to preserve in case tests fail
32 on_failure_ac: &on_failure_ac
40 on_failure_meson: &on_failure_meson
43 - "build*/testrun/**/*.log"
44 - "build*/testrun/**/*.diffs"
45 - "build*/testrun/**/regress_log_*"
48 # In theory it'd be nice to upload the junit files meson generates, so that
49 # cirrus will nicely annotate the commit. Unfortunately the files don't
50 # contain identifiable file + line numbers right now, so the annotations
51 # don't end up useful. We could probably improve on that with a some custom
52 # conversion script, but ...
54 path: "build*/meson-logs/*.txt"
58 # To avoid unnecessarily spinning up a lot of VMs / containers for entirely
59 # broken commits, have a minimal task that all others depend on.
63 # If a specific OS is requested, don't run the sanity check. This shortens
64 # push-wait-for-ci cycle time a bit when debugging operating system specific
65 # failures. Uses skip instead of only_if, as cirrus otherwise warns about
66 # only_if conditions not matching.
67 skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*'
73 CCACHE_DIR: ${CIRRUS_WORKING_DIR}/ccache_dir
74 # no options enabled, should be small
75 CCACHE_MAXSIZE: "150M"
77 # Container starts up quickly, but is slower at runtime, particularly for
78 # tests. Good for the briefly running sanity check.
80 image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
88 chown -R postgres:postgres .
89 mkdir -p ${CCACHE_DIR}
90 chown -R postgres:postgres ${CCACHE_DIR}
91 echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
92 su postgres -c "ulimit -l -H && ulimit -l -S"
93 # Can't change container's kernel.core_pattern. Postgres user can't write
94 # to / normally. Change that.
102 --auto-features=disabled \
103 -Dtap_tests=enabled \
108 ninja -C build -j${BUILD_JOBS}
110 upload_caches: ccache
112 # Run a minimal set of tests. The main regression tests take too long for
113 # this purpose. For now this is a random quick pg_regress style test, and a
114 # tap test that exercises both a frontend binary and the backend.
115 test_minimal_script: |
118 meson test $MTEST_ARGS --num-processes ${TEST_JOBS} \
119 tmp_install cube/regress pg_ctl/001_start_stop
123 <<: *on_failure_meson
125 mkdir -m 770 /tmp/cores
126 find / -maxdepth 1 -type f -name 'core*' -exec mv '{}' /tmp/cores/ \;
127 src/tools/ci/cores_backtrace.sh linux /tmp/cores
131 name: FreeBSD - 13 - Meson
134 # FreeBSD on GCP is slow when running with larger number of CPUS /
135 # jobs. Using one more job than cpus seems to work best.
140 CCACHE_DIR: /tmp/ccache_dir
141 CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
144 depends_on: SanityCheck
145 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
147 compute_engine_instance:
148 image_project: $IMAGE_PROJECT
149 image: family/pg-ci-freebsd-13
158 ulimit -a -H && ulimit -a -S
163 # Work around performance issues due to 32KB block size
164 repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
165 create_user_script: |
167 chown -R postgres:postgres .
168 mkdir -p ${CCACHE_DIR}
169 chown -R postgres:postgres ${CCACHE_DIR}
170 setup_core_files_script: |
171 mkdir -m 770 /tmp/cores
172 chown root:postgres /tmp/cores
173 sysctl kern.corefile='/tmp/cores/%N.%P.core'
174 setup_additional_packages_script: |
177 # NB: Intentionally build without -Dllvm. The freebsd image size is already
178 # large enough to make VM startup slow, and even without llvm freebsd
179 # already takes longer than other platforms except for windows.
184 -Dcassert=true -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
185 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
186 -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
189 build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
190 upload_caches: ccache
195 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
198 # test runningcheck, freebsd chosen because it's currently fast enough
199 test_running_script: |
203 meson test $MTEST_ARGS --quiet --suite setup
204 export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
205 mkdir -p build/testrun
206 build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
207 echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
208 build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
209 meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
210 build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
214 # if the server continues running, it often causes cirrus-ci to fail
215 # during upload, as it doesn't expect artifacts to change size
216 stop_running_script: |
218 build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop || true
220 <<: *on_failure_meson
221 cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores
224 # configure feature flags, shared between the task running the linux tests and
225 # the CompilerWarnings task
226 LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
240 --with-tcl --with-tclconfig=/usr/lib/tcl8.6/
244 LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >-
253 TEST_JOBS: 8 # experimentally derived to be a decent choice
255 CCACHE_DIR: /tmp/ccache_dir
256 DEBUGINFOD_URLS: "https://debuginfod.debian.net"
258 # Enable a reasonable set of sanitizers. Use the linux task for that, as
259 # it's one of the fastest tasks (without sanitizers). Also several of the
260 # sanitizers work best on linux.
262 # The overhead of alignment sanitizer is low, undefined behaviour has
263 # moderate overhead. Test alignment sanitizer in the meson task, as it
264 # does both 32 and 64 bit builds and is thus more likely to expose
267 # Address sanitizer in contrast is somewhat expensive. Enable it in the
268 # autoconf task, as the meson task tests both 32 and 64bit.
270 # disable_coredump=0, abort_on_error=1: for useful backtraces in case of crashes
271 # print_stacktraces=1,verbosity=2, duh
272 # detect_leaks=0: too many uninteresting leak errors in short-lived binaries
273 UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2
274 ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0
276 # SANITIZER_FLAGS is set in the tasks below
277 CFLAGS: -Og -ggdb -fno-sanitize-recover=all $SANITIZER_FLAGS
279 LDFLAGS: $SANITIZER_FLAGS
283 LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
284 LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
286 depends_on: SanityCheck
287 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
289 compute_engine_instance:
290 image_project: $IMAGE_PROJECT
291 image: family/pg-ci-bullseye
297 folder: ${CCACHE_DIR}
303 ulimit -a -H && ulimit -a -S
305 create_user_script: |
307 chown -R postgres:postgres .
308 mkdir -p ${CCACHE_DIR}
309 chown -R postgres:postgres ${CCACHE_DIR}
310 echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
311 su postgres -c "ulimit -l -H && ulimit -l -S"
312 setup_core_files_script: |
313 mkdir -m 770 /tmp/cores
314 chown root:postgres /tmp/cores
315 sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core'
317 setup_hosts_file_script: |
318 cat >> /etc/hosts <<-EOF
319 127.0.0.1 pg-loadbalancetest
320 127.0.0.2 pg-loadbalancetest
321 127.0.0.3 pg-loadbalancetest
324 setup_additional_packages_script: |
326 #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
329 - name: Linux - Debian Bullseye - Autoconf
332 SANITIZER_FLAGS: -fsanitize=address
337 --enable-cassert --enable-debug --enable-tap-tests \
339 --with-segsize-blocks=8 \
341 ${LINUX_CONFIGURE_FEATURES} \
345 build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin"
346 upload_caches: ccache
350 ulimit -c unlimited # default is 0
351 make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
357 - name: Linux - Debian Bullseye - Meson
360 CCACHE_MAXSIZE: "400M" # tests two different builds
361 SANITIZER_FLAGS: -fsanitize=alignment,undefined
368 ${LINUX_MESON_FEATURES} \
369 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
373 # Also build & test in a 32bit build - it's gotten rare to test that
375 configure_32_script: |
377 export CC='ccache gcc -m32'
381 ${LINUX_MESON_FEATURES} \
383 --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \
384 -DPERL=perl5.32-i386-linux-gnu \
385 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
389 build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
390 build_32_script: su postgres -c 'ninja -C build-32 -j${BUILD_JOBS}'
392 upload_caches: ccache
397 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
399 # so that we don't upload 64bit logs if 32bit fails
402 # There's currently no coverage of icu with LANG=C in the buildfarm. We
403 # can easily provide some here by running one of the sets of tests that
404 # way. Newer versions of python insist on changing the LC_CTYPE away
405 # from C, prevent that with PYTHONCOERCECLOCALE.
406 test_world_32_script: |
409 PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
413 <<: *on_failure_meson
416 cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
420 name: macOS - Ventura - Meson
423 CPUS: 4 # always get that much for cirrusci macOS instances
425 # Test performance regresses noticably when using all cores. 8 seems to
427 # https://postgr.es/m/20220927040208.l3shfcidovpzqxfh%40awork3.anarazel.de
430 CIRRUS_WORKING_DIR: ${HOME}/pgsql/
431 CCACHE_DIR: ${HOME}/ccache
432 HOMEBREW_CACHE: ${HOME}/homebrew-cache
433 PERL5LIB: ${HOME}/perl5/lib/perl5
437 CPPFLAGS: -DRANDOMIZE_ALLOCATED_MEMORY
441 depends_on: SanityCheck
442 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
445 image: ghcr.io/cirruslabs/macos-ventura-base:latest
450 ulimit -a -H && ulimit -a -S
453 setup_core_files_script:
454 - mkdir ${HOME}/cores
455 - sudo sysctl kern.corefile="${HOME}/cores/core.%P"
460 - perl -mIPC::Run -e 1 || cpan -T IPC::Run
461 - perl -mIO::Pty -e 1 || cpan -T IO::Pty
465 # XXX: Could we instead install homebrew into a cached directory? The
466 # homebrew installation takes a good bit of time every time, even if the
467 # packages do not need to be downloaded.
469 folder: $HOMEBREW_CACHE
470 setup_additional_packages_script: |
485 brew cleanup -s # to reduce cache size
486 upload_caches: homebrew
491 brewpath="/opt/homebrew"
492 PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
494 for pkg in icu4c krb5 openldap openssl zstd ; do
495 pkgpath="${brewpath}/opt/${pkg}"
496 PKG_CONFIG_PATH="${pkgpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
497 PATH="${pkgpath}/bin:${pkgpath}/sbin:$PATH"
500 export PKG_CONFIG_PATH PATH
504 -Dextra_include_dirs=${brewpath}/include \
505 -Dextra_lib_dirs=${brewpath}/lib \
507 -Duuid=e2fs -Ddtrace=auto \
509 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
512 build_script: ninja -C build -j${BUILD_JOBS}
513 upload_caches: ccache
516 ulimit -c unlimited # default is 0
517 ulimit -n 1024 # default is 256, pretty low
518 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
521 <<: *on_failure_meson
522 cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
525 WINDOWS_ENVIRONMENT_BASE: &WINDOWS_ENVIRONMENT_BASE
527 # Half the allowed per-user CPU cores
530 # The default cirrus working dir is in a directory msbuild complains about
531 CIRRUS_WORKING_DIR: "c:/cirrus"
532 # git's tar doesn't deal with drive letters, see
533 # https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
534 TAR: "c:/windows/system32/tar.exe"
535 # Avoids port conflicts between concurrent tap test runs
536 PG_TEST_USE_UNIX_SOCKETS: 1
537 PG_REGRESS_SOCK_DIR: "c:/cirrus/"
542 powershell -Command get-psdrive -psprovider filesystem
547 name: Windows - Server 2019, VS 2019 - Meson & ninja
548 << : *WINDOWS_ENVIRONMENT_BASE
551 TEST_JOBS: 8 # wild guess, data based value welcome
553 # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
554 # prevents crash reporting from working unless binaries do SetErrorMode()
555 # themselves. Furthermore, it appears that either python or, more likely,
556 # the C runtime has a bug where SEM_NOGPFAULTERRORBOX can very
557 # occasionally *trigger* a crash on process exit - which is hard to debug,
558 # given that it explicitly prevents crash dumps from working...
559 # 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
560 CIRRUS_WINDOWS_ERROR_MODE: 0x8001
562 depends_on: SanityCheck
563 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
565 compute_engine_instance:
566 image_project: $IMAGE_PROJECT
567 image: family/pg-ci-windows-ci-vs-2019
572 setup_additional_packages_script: |
573 REM choco install -y --no-progress ...
575 setup_hosts_file_script: |
576 echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
577 echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
578 echo 127.0.0.3 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
579 type c:\Windows\System32\Drivers\etc\hosts
581 # Use /DEBUG:FASTLINK to avoid high memory usage during linking
584 meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
590 check_world_script: |
592 meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
595 <<: *on_failure_meson
597 path: "crashlog-*.txt"
602 << : *WINDOWS_ENVIRONMENT_BASE
603 name: Windows - Server 2019, MinGW64 - Meson
605 # due to resource constraints we don't run this task by default for now
607 # worth using only_if despite being manual, otherwise this task will show up
608 # when e.g. ci-os-only: linux is used.
609 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*'
610 # otherwise it'll be sorted before other tasks
611 depends_on: SanityCheck
613 compute_engine_instance:
614 image_project: $IMAGE_PROJECT
615 image: family/pg-ci-windows-ci-mingw64
621 TEST_JOBS: 4 # higher concurrency causes occasional failures
622 CCACHE_DIR: C:/msys64/ccache
623 CCACHE_MAXSIZE: "500M"
624 CCACHE_SLOPPINESS: pch_defines,time_macros
626 # for some reason mingw plpython cannot find its installation without this
627 PYTHONHOME: C:/msys64/ucrt64
628 # prevents MSYS bash from resetting error mode
630 # Start bash in current working directory
632 BASH: C:\msys64\usr\bin\bash.exe -l
635 folder: ${CCACHE_DIR}
637 setup_additional_packages_script: |
638 REM C:\msys64\usr\bin\pacman.exe -S --noconfirm ...
641 %BASH% -c "where gcc"
642 %BASH% -c "gcc --version"
643 %BASH% -c "where perl"
644 %BASH% -c "perl --version"
646 # disable -Dnls as the number of files it creates cause a noticable slowdown
648 %BASH% -c "meson setup -Ddebug=true -Doptimization=g -Dcassert=true -Db_pch=true -Dnls=disabled -DTAR=%TAR% build"
651 %BASH% -c "ninja -C build"
653 upload_caches: ccache
656 %BASH% -c "meson test %MTEST_ARGS% --num-processes %TEST_JOBS%"
659 <<: *on_failure_meson
661 path: "crashlog-*.txt"
666 name: CompilerWarnings
668 # To limit unnecessary work only run this once the SanityCheck
669 # succeeds. This is particularly important for this task as we intentionally
670 # use always: to continue after failures. Task that did not run count as a
671 # success, so we need to recheck SanityChecks's condition here ...
672 depends_on: SanityCheck
673 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
679 # Use larger ccache cache, as this task compiles with multiple compilers /
682 CCACHE_DIR: "/tmp/ccache_dir"
684 LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
685 LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
688 image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
695 ulimit -a -H && ulimit -a -S
703 setup_additional_packages_script: |
705 #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
708 # Test that code can be built with gcc/clang without warnings
711 setup_script: echo "COPT=-Werror" > src/Makefile.custom
713 # Trace probes have a history of getting accidentally broken. Use the
714 # different compilers to build with different combinations of dtrace on/off
715 # and cassert on/off.
717 # gcc, cassert off, dtrace on
719 gcc_warning_script: |
723 ${LINUX_CONFIGURE_FEATURES} \
724 CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
725 make -s -j${BUILD_JOBS} clean
726 time make -s -j${BUILD_JOBS} world-bin
728 # gcc, cassert on, dtrace off
730 gcc_a_warning_script: |
734 ${LINUX_CONFIGURE_FEATURES} \
735 CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
736 make -s -j${BUILD_JOBS} clean
737 time make -s -j${BUILD_JOBS} world-bin
739 # clang, cassert off, dtrace off
741 clang_warning_script: |
743 --cache clang.cache \
744 ${LINUX_CONFIGURE_FEATURES} \
745 CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
746 make -s -j${BUILD_JOBS} clean
747 time make -s -j${BUILD_JOBS} world-bin
749 # clang, cassert on, dtrace on
751 clang_a_warning_script: |
753 --cache clang.cache \
756 ${LINUX_CONFIGURE_FEATURES} \
757 CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
758 make -s -j${BUILD_JOBS} clean
759 time make -s -j${BUILD_JOBS} world-bin
761 # cross-compile to windows
763 mingw_cross_warning_script: |
765 --host=x86_64-w64-mingw32 \
768 CC="ccache x86_64-w64-mingw32-gcc" \
769 CXX="ccache x86_64-w64-mingw32-g++"
770 make -s -j${BUILD_JOBS} clean
771 time make -s -j${BUILD_JOBS} world-bin
774 # Verify docs can be built
776 # XXX: Only do this if there have been changes in doc/ since last build
784 make -s -j${BUILD_JOBS} clean
785 time make -s -j${BUILD_JOBS} -C doc
788 # Verify headerscheck / cpluspluscheck succeed
790 # - Don't use ccache, the files are uncacheable, polluting ccache's
792 # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
793 # - XXX have to disable ICU to avoid errors:
794 # https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
797 headers_headerscheck_script: |
799 ${LINUX_CONFIGURE_FEATURES} \
802 CC="gcc" CXX"=g++" CLANG="clang"
803 make -s -j${BUILD_JOBS} clean
804 time make -s headerscheck EXTRAFLAGS='-fmax-errors=10'
805 headers_cpluspluscheck_script: |
806 time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
809 upload_caches: ccache