.gitlab-ci-main.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build a few things first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # test_only: Tests using the build from prior stages, these typically
  11 #            have an explicit dependency defined to a specific build job,
  12 #            which means that start as soon as the build job finished.
  13 # test_private: Like test_only, but running on private runners
  14 # report: Code coverage reporting
  15
  16 stages:
  17   - images
  18   - build_first
  19   - build
  20   - test_only
  21   - test_private
  22   - report
  23
  24 variables:
  25   # We want to be resilient to runner failures
  26   ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
  27   EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
  28   GET_SOURCES_ATTEMPTS: "3"
  29   RESTORE_CACHE_ATTEMPTS: "3"
  30   #
  31   GIT_STRATEGY: fetch
  32   GIT_DEPTH: "3"
  33   #
  34   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  35   # overwrite this variable if you want use your own image registry.
  36   #
  37   # Or better ask for access to the shared development repository, see
  38   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  39   #
  40   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  41   #
  42   # Set this to the contents of bootstrap/sha1sum.txt
  43   # which is generated by bootstrap/template.py --render
  44   #
  45   SAMBA_CI_CONTAINER_TAG: fa3eeb92fb5447524a057a4c377e6960dff626ce
  46   #
  47   # We use the ubuntu1804 image as default as
  48   # it matches what we have on sn-devel-184.
  49   #
  50   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  51   #
  52   # The following images are available
  53   # Please see the samba-o3 sections at the end of this file!
  54   # We should run that for each available image
  55   #
  56   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ubuntu1604
  57   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ubuntu1804
  58   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ubuntu2004
  59   SAMBA_CI_CONTAINER_IMAGE_debian9: debian9
  60   SAMBA_CI_CONTAINER_IMAGE_debian10: debian10
  61   SAMBA_CI_CONTAINER_IMAGE_opensuse151: opensuse151
  62   SAMBA_CI_CONTAINER_IMAGE_opensuse152: opensuse152
  63   SAMBA_CI_CONTAINER_IMAGE_fedora33: fedora33
  64   SAMBA_CI_CONTAINER_IMAGE_fedora34: fedora34
  65   SAMBA_CI_CONTAINER_IMAGE_centos7: centos7
  66   SAMBA_CI_CONTAINER_IMAGE_centos8: centos8
  67
  68 include:
  69   # The image creation details are specified in a separate file
  70   # See bootstrap/README.md for details
  71   - 'bootstrap/.gitlab-ci.yml'
  72
  73 .shared_runner_build_image:
  74   extends: .shared_runner_build
  75   variables:
  76     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE}
  77   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_JOB_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  78
  79 .shared_template:
  80   extends: .shared_runner_build_image
  81   # All Samba jobs are interruptible, this avoids burning CPU when a
  82   # newer branch is pushed.
  83   interruptible: true
  84   timeout: 2h
  85
  86   variables:
  87     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  88   stage: build
  89   cache:
  90     key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}.${SAMBA_CI_FLAVOR}
  91     paths:
  92       - ccache
  93   before_script:
  94     - uname -a
  95     - lsb_release -a
  96     - cat /etc/os-release
  97     - lscpu
  98     - cat /proc/cpuinfo
  99     - mount
 100     - df -h
 101     - cat /proc/swaps
 102     - free -h
 103       # ld will fail if coverage enabled, force link ld to ld.bfd
 104     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
 105       # See bootstrap/.gitlab-ci.yml how to generate a new image
 106     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
 107     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
 108     - echo "SAMBA_CI_JOB_IMAGE[${SAMBA_CI_JOB_IMAGE}]"
 109     - echo "CI_JOB_IMAGE[${CI_JOB_IMAGE}]"
 110     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
 111     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
 112     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
 113     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
 114     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
 115     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 116     - export CCACHE_BASEDIR="${PWD}"
 117     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 118     - export CC="ccache cc"
 119     - export CXX="ccache c++"
 120     - ccache -z -M 500M
 121     - ccache -s
 122   after_script:
 123     - mount
 124     - df -h
 125     - cat /proc/swaps
 126     - free -h
 127     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 128   artifacts:
 129     expire_in: 1 week
 130     paths:
 131       - "*.stdout"
 132       - "*.stderr"
 133       - "*.info"
 134       - public
 135       - system-info.txt
 136   retry:
 137     max: 2
 138     when:
 139       - runner_system_failure
 140       - stuck_or_timeout_failure
 141       - api_failure
 142       - runner_unsupported
 143       - stale_schedule
 144       - job_execution_timeout
 145       - archived_failure
 146       - scheduler_failure
 147       - data_integrity_failure
 148
 149   script:
 150     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 151     # autobuild name, which means we can define a default template that runs most autobuild jobs
 152     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 153
 154 # Ensure when adding a new job below that you also add it to
 155 # the dependencies for 'pages' below for the code coverage page
 156 # generation.
 157
 158 others:
 159   extends: .shared_template
 160   script:
 161     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 162     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 163     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 164     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 165     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 166     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 167     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 168     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 169
 170 .shared_template_build_only:
 171   extends: .shared_template
 172   timeout: 45m
 173   artifacts:
 174     expire_in: 1 week
 175     paths:
 176       - "*.stdout"
 177       - "*.stderr"
 178       - "*.info"
 179       - system-info.txt
 180       - samba-testbase.tar.gz
 181   script:
 182     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 183     # autobuild name, which means we can define a default template that runs most autobuild jobs
 184     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 185     # On success we need to pack everything into an artifacts file
 186     # which needs to be in the git checkout.
 187     # As tar doesn't handle hardlink of read-only files,
 188     # we remember the acls and add write permissions
 189     # before creating the archive. The consumer will apply
 190     # the acls again.
 191     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 192     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 193     - ln -s /tmp/samba-testbase/${AUTOBUILD_JOB_NAME}/ /tmp/samba-testbase/build_subdir_link
 194     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 195     - chmod -R +w /tmp/samba-testbase
 196     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 197     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 198     - ls -la samba-testbase.tar.gz
 199     - sha1sum samba-testbase.tar.gz
 200
 201 .shared_template_test_only:
 202   extends:
 203     - .shared_template
 204     - .shared_runner_test
 205   stage: test_only
 206   script:
 207     # We unpack the artifacts file created by the .shared_template_build_only
 208     # run we depend on
 209     - ls -la samba-testbase.tar.gz
 210     - sha1sum samba-testbase.tar.gz
 211     - tar xfz samba-testbase.tar.gz -C /
 212     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 213     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 214     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 215     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 216     - ls -la /tmp/samba-testbase/
 217     - ls -la /tmp/samba-testbase/build_subdir_link
 218     - ls -la /tmp/samba-testbase/build_subdir_link/
 219     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then find /tmp/samba-testbase/build_subdir_link/ -type d -printf "'%p'\n" | xargs chmod u+w; fi
 220     - ls -la /tmp/samba-testbase/build_subdir_link/
 221     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 222     # autobuild name, which means we can define a default template that runs most autobuild jobs
 223     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 224
 225 samba-def-build:
 226   extends: .shared_template_build_only
 227   stage: build_first
 228
 229 .needs_samba-def-build:
 230   extends: .shared_template_test_only
 231   needs:
 232     - job: samba-def-build
 233       artifacts: true
 234
 235 samba-mit-build:
 236   extends: .shared_template_build_only
 237   stage: build_first
 238
 239 .needs_samba-mit-build:
 240   extends: .shared_template_test_only
 241   needs:
 242     - job: samba-mit-build
 243       artifacts: true
 244
 245 samba-h5l-build:
 246   extends: .shared_template_build_only
 247
 248 .needs_samba-h5l-build:
 249   extends: .shared_template_test_only
 250   needs:
 251     - job: samba-h5l-build
 252       artifacts: true
 253
 254 samba-nt4-build:
 255   extends: .shared_template_build_only
 256
 257 .needs_samba-nt4-build:
 258   extends: .shared_template_test_only
 259   needs:
 260     - job: samba-nt4-build
 261       artifacts: true
 262
 263 samba-no-opath-build:
 264   extends: .shared_template_build_only
 265
 266 .needs_samba-no-opath-build:
 267   extends: .shared_template_test_only
 268   needs:
 269     - job: samba-no-opath-build
 270       artifacts: true
 271
 272 samba:
 273   extends: .shared_template
 274
 275 samba-mitkrb5:
 276   extends: .shared_template
 277
 278 samba-minimal-smbd:
 279   extends: .shared_template
 280
 281 samba-nopython:
 282   extends: .shared_template
 283
 284 samba-admem:
 285   extends: .needs_samba-def-build
 286
 287 samba-ad-dc-2:
 288   extends: .needs_samba-def-build
 289
 290 samba-ad-dc-3:
 291   extends: .needs_samba-def-build
 292
 293 samba-ad-dc-4a:
 294   extends: .needs_samba-def-build
 295
 296 samba-ad-dc-4b:
 297   extends: .needs_samba-def-build
 298
 299 samba-ad-dc-5:
 300   extends: .needs_samba-def-build
 301
 302 samba-ad-dc-6:
 303   extends: .needs_samba-def-build
 304
 305 samba-ad-back1:
 306   extends: .needs_samba-def-build
 307
 308 samba-ad-back2:
 309   extends: .needs_samba-def-build
 310
 311 samba-schemaupgrade:
 312   extends: .needs_samba-def-build
 313
 314 samba-libs:
 315   extends: .shared_template
 316
 317 samba-fuzz:
 318   extends: .shared_template
 319   variables:
 320     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1604}
 321
 322 ctdb:
 323   extends: .shared_template
 324
 325 samba-ctdb:
 326   extends: .shared_template
 327
 328 samba-ad-dc-ntvfs:
 329   extends: .needs_samba-def-build
 330
 331 samba-admem-mit:
 332   extends: .needs_samba-mit-build
 333
 334 samba-ad-dc-4a-mitkrb5:
 335   extends: .needs_samba-mit-build
 336
 337 samba-ad-dc-4b-mitkrb5:
 338   extends: .needs_samba-mit-build
 339
 340 # This task is run first to ensure we compile before we start the
 341 # main run as it is the fastest full compile of Samba.
 342 samba-fips:
 343   extends: .shared_template
 344   variables:
 345     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
 346
 347 .private_test_only:
 348   extends: .private_runner_test
 349   stage: test_private
 350   only:
 351     variables:
 352       # These jobs are only run if the gitlab repo has private runners available.
 353       # To enable private jobs, you must add the following var and value to
 354       # your gitlab repo by navigating to:
 355       # settings -> CI/CD -> Environment variables
 356       - $SUPPORT_PRIVATE_TEST == "yes"
 357
 358 .needs_samba-def-build-private:
 359   extends:
 360     - .needs_samba-def-build
 361     - .private_test_only
 362
 363 .needs_samba-mit-build-private:
 364   extends:
 365     - .needs_samba-mit-build
 366     - .private_test_only
 367
 368 .needs_samba-h5l-build-private:
 369   extends:
 370     - .needs_samba-h5l-build
 371     - .private_test_only
 372
 373 .needs_samba-nt4-build-private:
 374   extends:
 375     - .needs_samba-nt4-build
 376     - .private_test_only
 377
 378 .needs_samba-no-opath-build-private:
 379   extends:
 380     - .needs_samba-no-opath-build
 381     - .private_test_only
 382
 383 samba-fileserver:
 384   extends: .needs_samba-h5l-build-private
 385
 386 samba-ad-dc-1:
 387   extends: .needs_samba-def-build-private
 388
 389 samba-nt4:
 390   extends: .needs_samba-nt4-build-private
 391
 392 samba-ad-dc-1-mitkrb5:
 393   extends: .needs_samba-mit-build-private
 394
 395 samba-no-opath1:
 396   extends: .needs_samba-no-opath-build-private
 397
 398 samba-no-opath2:
 399   extends: .needs_samba-no-opath-build-private
 400
 401 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 402 pages:
 403   extends: .shared_runner_build_image
 404   stage: report
 405   dependencies:  # tell gitlab to download artifacts for these jobs
 406     - others
 407     - samba
 408     - samba-mitkrb5
 409     - samba-admem
 410     - samba-ad-dc-2
 411     - samba-ad-dc-3
 412     - samba-ad-dc-4a
 413     - samba-ad-dc-4b
 414     - samba-ad-dc-5
 415     - samba-ad-dc-6
 416     - samba-libs
 417     - samba-minimal-smbd
 418     - samba-nopython
 419     - samba-fuzz
 420     # - ctdb  # TODO
 421     - samba-ctdb
 422     - samba-ad-dc-ntvfs
 423     - samba-admem-mit
 424     - samba-ad-dc-4a-mitkrb5
 425     - samba-ad-dc-4b-mitkrb5
 426     - samba-ad-back1
 427     - samba-ad-back2
 428     - samba-fileserver
 429     - samba-ad-dc-1
 430     - samba-nt4
 431     - samba-schemaupgrade
 432     - samba-ad-dc-1-mitkrb5
 433     - samba-fips
 434     - samba-no-opath1
 435     - samba-no-opath2
 436     - ubuntu1804-samba-o3
 437   script:
 438     - ls -la *.info
 439     - ./configure.developer
 440     - make -j
 441     - ls -la *.info
 442     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 443     - ls -la *.info
 444     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 445   artifacts:
 446     expire_in: 30 days
 447     paths:
 448       - public
 449   only:
 450     variables:
 451       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 452
 453 # Coverity Scan
 454 coverity:
 455   extends: .shared_runner_build_image
 456   variables:
 457     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
 458   stage: build
 459   script:
 460     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 461     - tar xf /tmp/coverity_tool.tgz
 462     - ./configure.developer --with-cluster-support --with-system-mitkrb5 --with-experimental-mit-ad-dc
 463     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 464     - tar czf cov-int.tar.gz cov-int
 465     - curl
 466       --form token=$COVERITY_SCAN_TOKEN
 467       --form email=$COVERITY_SCAN_EMAIL
 468       --form file=@cov-int.tar.gz
 469       --form version="`git describe --tags`"
 470       --form description="CI build"
 471       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 472   only:
 473     refs:
 474       - master
 475       - schedules
 476     variables:
 477       - $COVERITY_SCAN_TOKEN != null
 478       - $COVERITY_SCAN_PROJECT_NAME != null
 479       - $COVERITY_SCAN_EMAIL != null
 480   artifacts:
 481     expire_in: 1 week
 482     when: on_failure
 483     paths:
 484       - cov-int/*.txt
 485
 486 #
 487 # We build samba-o3 on all supported distributions
 488 #
 489
 490 # This job, which matches the main CI, needs to still do coverage so
 491 # we show the coverage on the "none" environment tests
 492 #
 493 # We want --enable-coverage specified here otherwise we will have a
 494 # different set of build options on the coverage build and can fail
 495 # when -O3 gets combined with --enable-coverage in the scheduled
 496 # builds.
 497
 498 ubuntu1804-samba-o3:
 499   extends: .shared_template
 500   variables:
 501     AUTOBUILD_JOB_NAME: samba-o3
 502     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1804}
 503     SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "--enable-coverage"
 504
 505 # All other jobs do not want code coverage.
 506 .samba-o3-template:
 507   extends: .shared_template
 508   variables:
 509     AUTOBUILD_JOB_NAME: samba-o3
 510   only:
 511     variables:
 512       # do not run o3 for coverage since they are using different images
 513       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 514
 515 ubuntu2004-samba-o3:
 516   extends: .samba-o3-template
 517   variables:
 518     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
 519
 520 debian10-samba-o3:
 521   extends: .samba-o3-template
 522   variables:
 523     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian10}
 524
 525 opensuse151-samba-o3:
 526   extends: .samba-o3-template
 527   variables:
 528     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse151}
 529
 530 opensuse152-samba-o3:
 531   extends: .samba-o3-template
 532   variables:
 533     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse152}
 534
 535 centos7-samba-o3:
 536   extends: .samba-o3-template
 537   variables:
 538     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos7}
 539     # Git on CentOS doesn't support shallow git cloning
 540     GIT_DEPTH: ""
 541     # We need a newer GnuTLS version on CentOS7
 542     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 543
 544 centos8-samba-o3:
 545   extends: .samba-o3-template
 546   variables:
 547     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos8}
 548
 549 fedora33-samba-o3:
 550   extends: .samba-o3-template
 551   variables:
 552     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 553
 554 fedora34-samba-o3:
 555   extends: .samba-o3-template
 556   variables:
 557     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
 558
 559 #
 560 # Keep the samba-o3 sections at the end ...
 561 #