.gitlab-ci.yml

   1 # Types of CI pipelines:
   2 # | pipeline name        | context   | description                                                 |
   3 # |----------------------|-----------|-------------------------------------------------------------|
   4 # | merge pipeline       | mesa/mesa | pipeline running for an MR; if it passes the MR gets merged |
   5 # | pre-merge pipeline   | mesa/mesa | same as above, except its status doesn't affect the MR      |
   6 # | post-merge pipeline  | mesa/mesa | pipeline immediately after merging                          |
   7 # | fork pipeline        | fork      | pipeline running in a user fork                             |
   8 # | scheduled pipeline   | mesa/mesa | nightly pipelines, running every morning at 4am UTC         |
   9 # | direct-push pipeline | mesa/mesa | when commits are pushed directly to mesa/mesa, bypassing Marge and its gating pipeline |
  10 #
  11 # Note that the release branches maintained by the release manager fall under
  12 # the "direct push" category.
  13 #
  14 # "context" indicates the permissions that the jobs get; notably, any
  15 # container created in mesa/mesa gets pushed immediately for everyone to use
  16 # as soon as the image tag change is merged.
  17 #
  18 # Merge pipelines contain all jobs that must pass before the MR can be merged.
  19 # Pre-merge pipelines contain the exact same jobs as merge pipelines.
  20 # Post-merge pipelines contain *only* the `pages` job that deploys the new
  21 # version of the website.
  22 # Fork pipelines contain everything.
  23 # Scheduled pipelines only contain the container+build jobs, and some extra
  24 # test jobs (typically "full" variants of pre-merge jobs that only run 1/X
  25 # test cases), but not a repeat of the merge pipeline jobs.
  26 # Direct-push pipelines contain the same jobs as merge pipelines.
  27
  28 workflow:
  29   rules:
  30     # do not duplicate pipelines on merge pipelines
  31     - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
  32       when: never
  33     # merge pipeline
  34     - if: &is-merge-attempt $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "merge_request_event"
  35       variables:
  36         KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
  37         MESA_CI_PERFORMANCE_ENABLED: 1
  38         VALVE_INFRA_VANGOGH_JOB_PRIORITY: ""  # Empty tags are ignored by gitlab
  39     # post-merge pipeline
  40     - if: &is-post-merge $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "push"
  41     # nightly pipeline
  42     - if: &is-scheduled-pipeline $CI_PIPELINE_SOURCE == "schedule"
  43       variables:
  44         KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
  45         JOB_PRIORITY: 50
  46         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
  47     # pipeline for direct pushes that bypassed the CI
  48     - if: &is-direct-push $CI_PROJECT_NAMESPACE == "mesa" && $CI_PIPELINE_SOURCE == "push" && $GITLAB_USER_LOGIN != "marge-bot"
  49       variables:
  50         KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
  51         JOB_PRIORITY: 40
  52         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
  53     # pre-merge or fork pipeline
  54     - if: $FORCE_KERNEL_TAG != null
  55       variables:
  56         KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${FORCE_KERNEL_TAG}
  57         JOB_PRIORITY: 50
  58         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
  59     - if: $FORCE_KERNEL_TAG == null
  60       variables:
  61         KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
  62         JOB_PRIORITY: 50
  63         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
  64
  65
  66 variables:
  67   FDO_UPSTREAM_REPO: mesa/mesa
  68   MESA_TEMPLATES_COMMIT: &ci-templates-commit d5aa3941aa03c2f716595116354fb81eb8012acb
  69   CI_PRE_CLONE_SCRIPT: |-
  70           set -o xtrace
  71           wget -q -O download-git-cache.sh ${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/download-git-cache.sh
  72           bash download-git-cache.sh
  73           rm download-git-cache.sh
  74           set +o xtrace
  75   S3_JWT_FILE: /s3_jwt
  76   S3_HOST: s3.freedesktop.org
  77   # This bucket is used to fetch the kernel image
  78   S3_KERNEL_BUCKET: mesa-rootfs
  79   # Bucket for git cache
  80   S3_GITCACHE_BUCKET: git-cache
  81   # Bucket for the pipeline artifacts pushed to S3
  82   S3_ARTIFACTS_BUCKET: artifacts
  83   # Buckets for traces
  84   S3_TRACIE_RESULTS_BUCKET: mesa-tracie-results
  85   S3_TRACIE_PUBLIC_BUCKET: mesa-tracie-public
  86   S3_TRACIE_PRIVATE_BUCKET: mesa-tracie-private
  87   # per-pipeline artifact storage on MinIO
  88   PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/${S3_ARTIFACTS_BUCKET}/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
  89   # per-job artifact storage on MinIO
  90   JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
  91   # reference images stored for traces
  92   PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/${S3_TRACIE_RESULTS_BUCKET}/$FDO_UPSTREAM_REPO"
  93   # For individual CI farm status see .ci-farms folder
  94   # Disable farm with   `git mv .ci-farms{,-disabled}/$farm_name`
  95   # Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name`
  96   # NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST!
  97   ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts
  98   # Python scripts for structured logger
  99   PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install"
 100   # Drop once deqp-runner is upgraded to > 0.18.0
 101   MESA_VK_ABORT_ON_DEVICE_LOSS: 1
 102   # Avoid the wall of "Unsupported SPIR-V capability" warnings in CI job log, hiding away useful output
 103   MESA_SPIRV_LOG_LEVEL: error
 104
 105 default:
 106   id_tokens:
 107     S3_JWT:
 108       aud: https://s3.freedesktop.org
 109   before_script:
 110     - >
 111       export SCRIPTS_DIR=$(mktemp -d) &&
 112       curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh" &&
 113       . ${SCRIPTS_DIR}/setup-test-env.sh &&
 114       echo -n "${S3_JWT}" > "${S3_JWT_FILE}" &&
 115       unset CI_JOB_JWT S3_JWT  # Unsetting vulnerable env variables
 116
 117   after_script:
 118     # Work around https://gitlab.com/gitlab-org/gitlab/-/issues/20338
 119     - find -name '*.log' -exec mv {} {}.txt \;
 120
 121     - >
 122       set +x
 123
 124       test -e "${S3_JWT_FILE}" &&
 125       export S3_JWT="$(<${S3_JWT_FILE})" &&
 126       rm "${S3_JWT_FILE}"
 127
 128   # Retry when job fails. Failed jobs can be found in the Mesa CI Daily Reports:
 129   # https://gitlab.freedesktop.org/mesa/mesa/-/issues/?sort=created_date&state=opened&label_name%5B%5D=CI%20daily
 130   retry:
 131     max: 1
 132     # Ignore runner_unsupported, stale_schedule, archived_failure, or
 133     # unmet_prerequisites
 134     when:
 135       - api_failure
 136       - runner_system_failure
 137       - script_failure
 138       - job_execution_timeout
 139       - scheduler_failure
 140       - data_integrity_failure
 141       - unknown_failure
 142
 143 stages:
 144   - sanity
 145   - container
 146   - git-archive
 147   - build-x86_64
 148   - build-misc
 149   - code-validation
 150   - amd
 151   - intel
 152   - nouveau
 153   - arm
 154   - broadcom
 155   - freedreno
 156   - etnaviv
 157   - software-renderer
 158   - layered-backends
 159   - deploy
 160
 161 include:
 162   - project: 'freedesktop/ci-templates'
 163     ref: 16bc29078de5e0a067ff84a1a199a3760d3b3811
 164     file:
 165       - '/templates/ci-fairy.yml'
 166   - project: 'freedesktop/ci-templates'
 167     ref: *ci-templates-commit
 168     file:
 169       - '/templates/alpine.yml'
 170       - '/templates/debian.yml'
 171       - '/templates/fedora.yml'
 172   - local: '.gitlab-ci/image-tags.yml'
 173   - local: '.gitlab-ci/lava/lava-gitlab-ci.yml'
 174   - local: '.gitlab-ci/container/gitlab-ci.yml'
 175   - local: '.gitlab-ci/build/gitlab-ci.yml'
 176   - local: '.gitlab-ci/test/gitlab-ci.yml'
 177   - local: '.gitlab-ci/farm-rules.yml'
 178   - local: '.gitlab-ci/test-source-dep.yml'
 179   - local: 'docs/gitlab-ci.yml'
 180   - local: 'src/**/ci/gitlab-ci.yml'
 181
 182
 183 # YAML anchors for rule conditions
 184 # --------------------------------
 185 .rules-anchors:
 186   # Pre-merge pipeline
 187   - &is-pre-merge '$CI_PIPELINE_SOURCE == "merge_request_event"'
 188
 189
 190 .never-post-merge-rules:
 191   rules:
 192     - if: *is-post-merge
 193       when: never
 194
 195
 196 .container+build-rules:
 197   rules:
 198     # Run when re-enabling a disabled farm, but not when disabling it
 199     - !reference [.disable-farm-mr-rules, rules]
 200     # Never run immediately after merging, as we just ran everything
 201     - !reference [.never-post-merge-rules, rules]
 202     # Build everything in merge pipelines, if any files affecting the pipeline
 203     # were changed
 204     - if: *is-merge-attempt
 205       changes: &all_paths
 206         - VERSION
 207         - bin/git_sha1_gen.py
 208         - bin/install_megadrivers.py
 209         - bin/symbols-check.py
 210         # GitLab CI
 211         - .gitlab-ci.yml
 212         - .gitlab-ci/**/*
 213         - .ci-farms/*
 214         # Meson
 215         - meson*
 216         - build-support/**/*
 217         - subprojects/**/*
 218         # clang format
 219         - .clang-format
 220         - .clang-format-include
 221         - .clang-format-ignore
 222         # Source code
 223         - include/**/*
 224         - src/**/*
 225       when: on_success
 226     # Same as above, but for pre-merge pipelines
 227     - if: *is-pre-merge
 228       changes:
 229         *all_paths
 230       when: manual
 231     # Skip everything for pre-merge and merge pipelines which don't change
 232     # anything in the build
 233     - if: *is-merge-attempt
 234       when: never
 235     - if: *is-pre-merge
 236       when: never
 237     # Build everything after someone bypassed the CI
 238     - if: *is-direct-push
 239       when: on_success
 240     # Build everything in scheduled pipelines
 241     - if: *is-scheduled-pipeline
 242       when: on_success
 243     # Allow building everything in fork pipelines, but build nothing unless
 244     # manually triggered
 245     - when: manual
 246
 247
 248 .ci-deqp-artifacts:
 249   artifacts:
 250     name: "mesa_${CI_JOB_NAME}"
 251     when: always
 252     untracked: false
 253     paths:
 254       # Watch out!  Artifacts are relative to the build dir.
 255       # https://gitlab.com/gitlab-org/gitlab-ce/commit/8788fb925706cad594adf6917a6c5f6587dd1521
 256       - artifacts
 257       - _build/meson-logs/*.txt
 258       - _build/meson-logs/strace
 259
 260 # Git archive
 261
 262 make git archive:
 263   extends:
 264     - .fdo.ci-fairy
 265   stage: git-archive
 266   rules:
 267     - !reference [.scheduled_pipeline-rules, rules]
 268   # ensure we are running on packet
 269   tags:
 270     - packet.net
 271   script:
 272     # Compactify the .git directory
 273     - git gc --aggressive
 274     # Download & cache the perfetto subproject as well.
 275     - rm -rf subprojects/perfetto ; mkdir -p subprojects/perfetto && curl https://android.googlesource.com/platform/external/perfetto/+archive/$(grep 'revision =' subprojects/perfetto.wrap | cut -d ' ' -f3).tar.gz | tar zxf - -C subprojects/perfetto
 276     # compress the current folder
 277     - tar -cvzf ../$CI_PROJECT_NAME.tar.gz .
 278
 279     - ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
 280
 281 # Sanity checks of MR settings and commit logs
 282 sanity:
 283   extends:
 284     - .fdo.ci-fairy
 285   stage: sanity
 286   rules:
 287     - if: *is-pre-merge
 288       when: on_success
 289     - when: never
 290   variables:
 291     GIT_STRATEGY: none
 292   script:
 293     # ci-fairy check-commits --junit-xml=check-commits.xml
 294     - ci-fairy check-merge-request --require-allow-collaboration --junit-xml=check-merge-request.xml
 295     - |
 296       set -eu
 297       image_tags=(
 298         DEBIAN_BASE_TAG
 299         DEBIAN_BUILD_TAG
 300         DEBIAN_TEST_ANDROID_TAG
 301         DEBIAN_TEST_GL_TAG
 302         DEBIAN_TEST_VK_TAG
 303         ALPINE_X86_64_BUILD_TAG
 304         ALPINE_X86_64_LAVA_SSH_TAG
 305         FEDORA_X86_64_BUILD_TAG
 306         KERNEL_ROOTFS_TAG
 307         KERNEL_TAG
 308         PKG_REPO_REV
 309         WINDOWS_X64_MSVC_TAG
 310         WINDOWS_X64_BUILD_TAG
 311         WINDOWS_X64_TEST_TAG
 312       )
 313       for var in "${image_tags[@]}"
 314       do
 315         if [ "$(echo -n "${!var}" | wc -c)" -gt 20 ]
 316         then
 317           echo "$var is too long; please make sure it is at most 20 chars."
 318           exit 1
 319         fi
 320       done
 321   artifacts:
 322     when: on_failure
 323     reports:
 324       junit: check-*.xml
 325   tags:
 326     - placeholder-job
 327
 328
 329 mr-label-maker-test:
 330   extends:
 331     - .fdo.ci-fairy
 332   stage: sanity
 333   rules:
 334     - !reference [.mr-label-maker-rules, rules]
 335   variables:
 336     GIT_STRATEGY: fetch
 337   timeout: 10m
 338   script:
 339     - set -eu
 340     - python3 -m venv .venv
 341     - source .venv/bin/activate
 342     - pip install git+https://gitlab.freedesktop.org/freedesktop/mr-label-maker
 343     - mr-label-maker --dry-run --mr $CI_MERGE_REQUEST_IID
 344
 345 # Jobs that need to pass before spending hardware resources on further testing
 346 .required-for-hardware-jobs:
 347   needs:
 348     - job: clang-format
 349       optional: true
 350     - job: rustfmt
 351       optional: true