.gitlab-ci.yml

   1 stages:
   2   - test
   3   - build-website
   4   - deploy-website
   5
   6 workflow:
   7   rules:
   8     - if: $CI_FORCE_RUN
   9     - if: $CI_MERGE_REQUEST_IID
  10     - if: $CI_COMMIT_TAG
  11     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
  12
  13 image: debian:bookworm
  14
  15 variables:
  16     GET_SOURCES_ATTEMPTS: 10
  17
  18 before_script:
  19   - export DEBIAN_FRONTEND=noninteractive
  20   - apt-get update -qq
  21
  22 .prepare-lint-po: &prepare-lint-po
  23   - apt-get -qy install git i18nspector
  24   - git clone https://gitlab.tails.boum.org/tails/jenkins-tools.git /tmp/jenkins-tools
  25
  26 build-website:
  27   stage: build-website
  28   rules:
  29     - if: '$CI_COMMIT_BRANCH == "master"'
  30     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
  31     - changes:
  32         - .gitlab-ci.yml
  33   cache:
  34     key: website-$CI_COMMIT_REF_SLUG
  35     paths:
  36       - config/chroot_local-includes/usr/share/doc/tails/website
  37       - wiki/src/.ikiwiki
  38   script:
  39     - apt-get -y install wget git
  40     - wget -q https://gitlab.tails.boum.org/tails/puppet-tails/-/raw/master/files/D68F87149EBA77541573C1C12453AA9CE4123A9A.asc -O /etc/apt/trusted.gpg.d/tails.asc
  41     - |
  42       echo -e 'Explanation: tails: po4a
  43       Package: po4a
  44       Pin: release n=bullseye, o=Debian
  45       Pin-Priority: 1000' > /etc/apt/preferences.d/po4a.pref
  46     - echo 'deb https://deb.tails.boum.org/ ikiwiki main' > /etc/apt/sources.list.d/tails.list
  47     - echo 'deb https://deb.debian.org/debian bullseye main' > /etc/apt/sources.list.d/bullseye.list
  48     - apt-get update && apt-get -y install ikiwiki po4a libyaml-perl libyaml-libyaml-perl libyaml-syck-perl perlmagick
  49     - mkdir /underlays
  50     - git clone --depth=1 https://gitlab.tails.boum.org/tails/etcher-binary.git /underlays/etcher-binary
  51     - git clone --depth=1 https://gitlab.tails.boum.org/tails/promotion-material.git /underlays/promotion-material
  52     - |
  53       ./build-website \
  54         --url "https://tails.net" \
  55         --usedirs \
  56         --sslcookie \
  57         --historyurl "https://gitlab.tails.boum.org/tails/tails/-/commits/master/wiki/src/[[file]]" \
  58         --diffurl "https://gitlab.tails.boum.org/tails/tails/-/commit/[[sha1_commit]]" \
  59         --rss \
  60         --atom \
  61         --set-yaml add_underlays="['/underlays/etcher-binary', '/underlays/promotion-material']"
  62
  63 deploy-website:
  64   stage: deploy-website
  65   rules:
  66     - if: '$CI_COMMIT_BRANCH == "master"'
  67   cache:
  68     key: website-$CI_COMMIT_REF_SLUG
  69     paths:
  70       - config/chroot_local-includes/usr/share/doc/tails/website
  71       - wiki/src/.ikiwiki
  72     policy: pull
  73   script:
  74     - apt-get install -y openssh-client rsync
  75     - test -e .ssh || mkdir .ssh
  76     - cp "$WEBSITE_DEPLOY_SSH_PRIVATE_KEY" .ssh/private_key
  77     - cp "$WEBSITE_DEPLOY_SSH_KNOWN_HOSTS" .ssh/known_hosts
  78     - chmod 400 .ssh/known_hosts .ssh/private_key
  79     - echo "variables often lack a trailing newline, which breaks SSH, detect and fix"
  80     - ssh-keygen -y -f .ssh/private_key || echo >> .ssh/private_key
  81     - echo "here is the SSH key we will deploy with"
  82     - ssh-keygen -y -f .ssh/private_key
  83     - echo -n "Begin rsync, time is " && date '+%Y-%m-%d %H-%M-%S%z'
  84     - rsync --rsh="ssh -p 3004 -o UserKnownHostsFile=.ssh/known_hosts -i .ssh/private_key" --checksum --archive --no-times --verbose --mkpath --delete "config/chroot_local-includes/usr/share/doc/tails/website/" www-data@chameleon.tails.net:/
  85
  86 lint-po:
  87   image: debian:testing
  88   rules:
  89     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
  90     - changes:
  91         - .gitlab-ci.yml
  92         - ./**.po
  93   script:
  94     - *prepare-lint-po
  95     - /tmp/jenkins-tools/slaves/lint_po
  96
  97 ruff-lint-changed-files:
  98   only:
  99     - merge_requests
 100
 101   image: debian:sid
 102
 103   script:
 104     - apt-get -qy install ruff findutils git python3
 105     - ruff --version
 106     - git fetch origin "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 107     - ./bin/test-utils/ruff "origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}" check --output-format=junit --output-file=ruff.xml
 108   artifacts:
 109     when: always
 110     reports:
 111       junit: ruff.xml
 112
 113 ruff-format-changed-files:
 114   only:
 115     - merge_requests
 116
 117   image: debian:sid
 118
 119   script:
 120     - apt-get -qy install ruff findutils git python3
 121     - ruff --version
 122     - git fetch origin "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 123     - ./bin/test-utils/ruff "origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}" format --check
 124
 125 check-website-core-pages:
 126   script:
 127     - apt-get -qy install git
 128     - ./bin/check-core-pages
 129
 130 check-po-msgfmt:
 131   rules:
 132     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 133     - changes:
 134         - .gitlab-ci.yml
 135         - ./**.po
 136   script:
 137     - apt-get -qy install python3 gettext
 138     - ./bin/check-po-msgfmt
 139
 140 check-po-meta-date:
 141   rules:
 142     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 143     - changes:
 144         - .gitlab-ci.yml
 145         - ./**.po
 146   script:
 147     - apt-get -qy install git ruby
 148     - ./bin/sanity-check-website
 149
 150 check-translatable-live-website-urls:
 151   script:
 152     - apt-get -qy install python3-polib
 153     - ./bin/check-translatable-live-website-urls po/tails.pot
 154
 155 check-locale-descriptions:
 156   script:
 157     - apt-get -qy install python3 python3-requests python3-toml python3-bs4
 158     - echo 'If this fails, look at https://tails.net/contribute/release_process/update_locale_descriptions/'
 159     - ./bin/locale-descriptions suggest
 160
 161 rubocop:
 162   image: debian:bookworm
 163   script:
 164   - apt-get -qy install rubocop
 165   - rubocop --version
 166   - rubocop --format junit --out rubocop.xml --format markdown
 167   artifacts:
 168     when: always
 169     reports:
 170       junit: rubocop.xml
 171
 172 test-iuk:
 173   rules:
 174     - if: '$CI_COMMIT_BRANCH != "master"'
 175   script:
 176   - './bin/test-utils/test-iuk'
 177
 178 test-perl5lib:
 179   rules:
 180     - if: '$CI_COMMIT_BRANCH != "master"'
 181   script:
 182   - 'cat config/chroot_local-packageslists/tails-perl5lib.list
 183        | grep -E -v "^#"
 184        | xargs apt-get -qy install'
 185   - 'apt-get -qy install
 186        apt-file
 187        libdist-zilla-plugin-test-notabs-perl
 188        libdist-zilla-plugin-test-perl-critic-perl
 189        libdist-zilla-app-command-authordebs-perl
 190        libmodule-build-perl
 191        sudo'
 192   - apt-get update -qq # Take into account APT configuration added by apt-file
 193   # Otherwise, apt-get called by "dzil authordebs --install" asks confirmation
 194   - echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/yes
 195   - cd $CI_PROJECT_DIR/config/chroot_local-includes/usr/src/perl5lib
 196   - dzil authordebs --install
 197   - dzil test --all
 198
 199 shellcheck:
 200   image: debian:testing
 201   script:
 202   - apt-get -qy install python3 shellcheck xmlstarlet git
 203   - shellcheck --version
 204   - 'git ls-files -z | ./bin/test-utils/is-file-type filter --zero shell | xargs --verbose --no-run-if-empty -0 shellcheck --format=checkstyle
 205        | xmlstarlet tr config/ci/shellcheck/checkstyle2junit.xslt
 206        > shellcheck.xml'
 207   artifacts:
 208     when: always
 209     reports:
 210       junit: shellcheck.xml
 211
 212 test-persistent-storage-config-file:
 213   script:
 214     - apt-get -qy install python3 python3-gi acl
 215     - config/chroot_local-includes/usr/lib/python3/dist-packages/tps/configuration/config_file_test.py
 216
 217 test-python-doctest:
 218   script:
 219     - apt-get -qy install python3 python3-sh python3-toml python3-requests python3-bs4
 220     - config/chroot_local-includes/usr/local/lib/tails-gdm-error-message doctest --verbose
 221     - env PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages python3 config/chroot_local-includes/usr/local/bin/tails-documentation --doctest
 222     - ./bin/locale-descriptions doctest
 223
 224 test-tca:
 225   rules:
 226     - if: '$CI_COMMIT_BRANCH != "master"'
 227   script:
 228     - 'cat config/chroot_local-packageslists/tor-connection-assistant.list
 229        | grep -E -v "^#"
 230        | xargs apt-get -qy install'
 231     - 'cd config/chroot_local-includes/usr/lib/python3/dist-packages ; find tca -name "*.py" -print0 | xargs -0 -L1 env PYTHONPATH=. python3 -m doctest'
 232
 233 test-tca-portal:
 234   rules:
 235     - if: '$CI_COMMIT_BRANCH != "master"'
 236   script:
 237     - 'cat config/chroot_local-packageslists/tor-connection-assistant.list
 238        | grep -E -v "^#"
 239        | xargs apt-get -qy install'
 240     - 'PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages env python3 ./config/chroot_local-includes/usr/local/lib/tca-portal --doctest-only --log-level DEBUG'
 241
 242
 243 test-tailslib:
 244   rules:
 245     - if: '$CI_COMMIT_BRANCH != "master"'
 246   script:
 247     - apt-get -qy install python3 python3-atomicwrites python3-sh python3-gi git
 248     - 'cd config/chroot_local-includes/usr/lib/python3/dist-packages ; find tailslib -name "*.py" -print0 | grep --null-data -v -e netnsdrop.py -e gnome.py | xargs -0 -L1 env PYTHONPATH=. python3 -m doctest'
 249
 250 test-whisperback:
 251   rules:
 252     - if: '$CI_COMMIT_BRANCH != "master"'
 253   script:
 254     - 'cat config/chroot_local-packageslists/whisperback.list | grep -E -v "^#"
 255          | xargs apt-get -qy install'
 256     - apt-get -qy install python3-pytest
 257     - 'PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages
 258          pytest-3 --verbose --junit-xml=report.xml
 259          config/chroot_local-includes/usr/lib/python3/dist-packages/whisperBack/test.py'
 260   artifacts:
 261     when: always
 262     reports:
 263       junit: report.xml
 264
 265 apt-snapshots-expiry:
 266   script:
 267     - apt-get -qy install curl git
 268     - ./bin/apt-snapshots-expiry
 269   rules:
 270     - if: '$CI_COMMIT_BRANCH =~ /^stable|testing|devel$/'
 271     - changes:
 272         - .gitlab-ci.yml
 273         - config/APT_snapshots.d/*/serial
 274         - vagrant/definitions/tails-builder/config/APT_snapshots.d/*/serial
 275
 276 .install-https-get-expired-build-deps: &install-https-get-expired-build-deps
 277   - apt-get -qy install --no-install-recommends golang-go ca-certificates
 278
 279 .build-https-get-expired: &build-https-get-expired
 280   - go build -o ./https-get-expired config/chroot_local-includes/usr/src/https-get-expired.go
 281
 282 .test-https-get-expired: &test-https-get-expired
 283   - echo "Basic check:"
 284   - ./https-get-expired -reject-expired https://tails.net/
 285   - echo "Let's pretend we are in the past. Then, this certificate is still good."
 286   - ./https-get-expired -current-time 2000-01-01 -reject-expired https://tails.net/
 287   - echo "Let's pretend we are in the future. Then, this certificate is expired"
 288   - "! ./https-get-expired -current-time 2090-01-01 -reject-expired https://tails.net/"
 289   - "! ./https-get-expired -reject-expired https://wrong.host.badssl.com/"
 290   - "! ./https-get-expired -reject-expired https://self-signed.badssl.com/"
 291   - "! ./https-get-expired -reject-expired https://untrusted-root.badssl.com/"
 292   - "! ./https-get-expired -reject-expired https://expired.badssl.com/"
 293   - echo "Invalid host"
 294   - "! ./https-get-expired -reject-expired https://nxdomain.tails.net/"
 295   - "./bin/test-utils/https-get-expired-test-all"
 296
 297 https-get-expired:
 298   rules:
 299     - if: '$CI_COMMIT_BRANCH =~ /^stable|testing|devel$/'
 300     - changes:
 301         - .gitlab-ci.yml
 302         - config/chroot_local-includes/usr/src/https-get-expired.go
 303         - config/chroot_local-includes/etc/default/htpdate.pools
 304   script:
 305     - *install-https-get-expired-build-deps
 306     - *build-https-get-expired
 307     - *test-https-get-expired
 308
 309 https-get-expired-sid:
 310   # this job gives us results using a future version of Golang compared to the one we actually use
 311   image: debian:sid
 312   rules:
 313     - if: '$CI_COMMIT_BRANCH == "devel"'
 314     - changes:
 315         - .gitlab-ci.yml
 316         - config/chroot_local-includes/usr/src/https-get-expired.go
 317         - config/chroot_local-includes/etc/default/htpdate.pools
 318   script:
 319     - *install-https-get-expired-build-deps
 320     - *build-https-get-expired
 321     - *test-https-get-expired