s4: smbtorture: Add fsync_resource_fork test to fruit tests.
[Samba.git] / bootstrap / config.py
blobbf2ce5207bb29c04ba3c552e9f6b4ad2cdd7509f
1 #!/usr/bin/env python3
3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 """
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Update the lists in this file to require new packages in the
24 container images used in GitLab CI
26 Author: Joe Guo <joeg@catalyst.net.nz>
27 """
28 import os
29 from os.path import abspath, dirname, join
30 HERE = abspath(dirname(__file__))
31 # output dir for rendered files
32 OUT = join(HERE, 'generated-dists')
35 # pkgs with same name in all packaging systems
36 COMMON = [
37 'acl',
38 'attr',
39 'autoconf',
40 'binutils',
41 'bison',
42 'ccache',
43 'curl',
44 'chrpath',
45 'flex',
46 'gcc',
47 'gdb',
48 'git',
49 'gzip',
50 'hostname',
51 'htop',
52 'jq',
53 'lcov',
54 'make',
55 'patch',
56 'perl',
57 'psmisc', # for pstree in test
58 'rng-tools',
59 'rsync',
60 'sed',
61 'sudo', # docker images has no sudo by default
62 'tar',
63 'tree',
64 'wget',
68 # define pkgs for all packaging systems in parallel
69 # make it easier to find missing ones
70 # use latest ubuntu and fedora as defaults
71 # deb, rpm, ...
72 PKGS = [
73 # NAME1-dev, NAME2-devel
74 ('lmdb-utils', 'lmdb'),
75 ('mingw-w64', 'mingw64-gcc'),
76 ('zlib1g-dev', 'zlib-devel'),
77 ('libbsd-dev', 'libbsd-devel'),
78 ('liburing-dev', 'liburing-devel'),
79 ('libarchive-dev', 'libarchive-devel'),
80 ('libblkid-dev', 'libblkid-devel'),
81 ('libcap-dev', 'libcap-devel'),
82 ('libacl1-dev', 'libacl-devel'),
83 ('libattr1-dev', 'libattr-devel'),
85 # libNAME1-dev, NAME2-devel
86 ('libpopt-dev', 'popt-devel'),
87 ('libreadline-dev', 'readline-devel'),
88 ('libjansson-dev', 'jansson-devel'),
89 ('liblmdb-dev', 'lmdb-devel'),
90 ('libncurses5-dev', 'ncurses-devel'),
91 # NOTE: Debian 7+ or Ubuntu 16.04+
92 ('libsystemd-dev', 'systemd-devel'),
93 ('libkrb5-dev', 'krb5-devel'),
94 ('libldap2-dev', 'openldap-devel'),
95 ('libcups2-dev', 'cups-devel'),
96 ('libpam0g-dev', 'pam-devel'),
97 ('libgpgme11-dev', 'gpgme-devel'),
98 # NOTE: Debian 8+ and Ubuntu 14.04+
99 ('libgnutls28-dev', 'gnutls-devel'),
100 ('libtasn1-bin', 'libtasn1-tools'),
101 ('libtasn1-dev', 'libtasn1-devel'),
102 ('', 'quota-devel'),
103 ('uuid-dev', 'libuuid-devel'),
104 ('libjs-jquery', ''),
105 ('libavahi-common-dev', 'avahi-devel'),
106 ('libdbus-1-dev', 'dbus-devel'),
107 ('libpcap-dev', 'libpcap-devel'),
108 ('libunwind-dev', 'libunwind-devel'), # for back trace
109 ('libglib2.0-dev', 'glib2-devel'),
110 ('libicu-dev', 'libicu-devel'),
111 ('heimdal-multidev', ''),
113 # NAME1, NAME2
114 # for debian, locales provide locale support with language packs
115 # ubuntu split language packs to language-pack-xx
116 # for centos, glibc-common provide locale support with language packs
117 # fedora split language packs to glibc-langpack-xx
118 ('locales', 'glibc-common'), # required for locale
119 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
120 ('bind9utils', 'bind-utils'),
121 ('dnsutils', ''),
122 ('xsltproc', 'libxslt'),
123 ('krb5-user', 'krb5-workstation'),
124 ('krb5-config', ''),
125 ('krb5-kdc', 'krb5-server'),
126 ('apt-utils', 'yum-utils'),
127 ('pkg-config', 'pkgconfig'),
128 ('procps', 'procps-ng'), # required for the free cmd in tests
129 ('lsb-release', 'lsb-release'), # we need lsb_relase to show info
130 ('', 'rpcgen'), # required for test
131 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
132 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
133 ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
134 ('mawk', 'gawk'),
135 ('', 'mold'),
136 ('', 'ShellCheck'),
137 ('', 'shfmt'),
139 ('python3', 'python3'),
140 ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
141 ('python3-dev', 'python3-devel'),
142 ('python3-dbg', ''),
143 ('python3-iso8601', 'python3-iso8601'),
144 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
145 ('python3-markdown', 'python3-markdown'),
146 ('python3-dnspython', 'python3-dns'),
147 ('python3-pexpect', ''), # for wintest only
148 ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
149 ('python3-setproctitle', 'python3-setproctitle'),
150 ('python3-requests', 'python3-requests'), # for cert auto enroll
152 ('', 'python3-libsemanage'),
153 ('', 'python3-policycoreutils'),
155 # perl
156 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
157 ('libjson-perl', 'perl-JSON'),
158 ('', 'perl-JSON-Parse'),
159 ('perl-modules', ''),
160 ('', 'perl-FindBin'),
161 ('', 'perl-Archive-Tar'),
162 ('', 'perl-ExtUtils-MakeMaker'),
163 ('', 'perl-Test-Base'),
164 ('', 'perl-generators'),
165 ('', 'perl-interpreter'),
167 # fs
168 ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
169 ('', 'glusterfs-api-devel'),
170 ('glusterfs-common', 'glusterfs-devel'),
171 ('libcephfs-dev', 'libcephfs-devel'),
173 # spotlight
174 ('libtracker-sparql-2.0-dev', 'tracker-devel'),
176 # misc
177 # @ means group for rpm, use fedora as rpm default
178 ('build-essential', '@development-tools'),
179 ('debhelper', ''),
180 # rpm has no pkg for docbook-xml
181 ('docbook-xml', 'docbook-dtds'),
182 ('docbook-xsl', 'docbook-style-xsl'),
183 ('', 'keyutils-libs-devel'),
184 ('', 'which'),
185 ('xz-utils', 'xz')
189 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
190 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
192 GENERATED_MARKER = r"""
194 # This file is generated by 'bootstrap/template.py --render'
195 # See also bootstrap/config.py
200 APT_BOOTSTRAP = r"""
201 #!/bin/bash
202 {GENERATED_MARKER}
203 set -xueo pipefail
205 export DEBIAN_FRONTEND=noninteractive
206 apt-get -y update
208 apt-get -y install \
209 {pkgs}
211 apt-get -y autoremove
212 apt-get -y autoclean
213 apt-get -y clean
217 YUM_BOOTSTRAP = r"""
218 #!/bin/bash
219 {GENERATED_MARKER}
220 set -xueo pipefail
222 yum update -y
223 yum install -y epel-release
224 yum install -y yum-plugin-copr
225 yum copr enable -y sergiomb/SambaAD
226 yum update -y
228 yum install -y \
229 {pkgs}
231 yum clean all
233 if [ ! -f /usr/bin/python3 ]; then
234 ln -sf /usr/bin/python3.6 /usr/bin/python3
238 CENTOS8S_YUM_BOOTSTRAP = r"""
239 #!/bin/bash
240 {GENERATED_MARKER}
241 set -xueo pipefail
243 yum update -y
244 yum install -y dnf-plugins-core
245 yum install -y epel-release
247 yum -v repolist all
248 yum config-manager --set-enabled powertools -y || \
249 yum config-manager --set-enabled powertools -y
251 yum update -y
253 yum install -y \
254 --setopt=install_weak_deps=False \
255 {pkgs}
257 yum clean all
260 DNF_BOOTSTRAP = r"""
261 #!/bin/bash
262 {GENERATED_MARKER}
263 set -xueo pipefail
265 dnf update -y
267 dnf install -y \
268 --setopt=install_weak_deps=False \
269 {pkgs}
271 dnf clean all
274 DNF_BOOTSTRAP_MIT = r"""
275 #!/bin/bash
276 {GENERATED_MARKER}
277 set -xueo pipefail
279 dnf update -y
280 dnf install -y dnf-plugins-core
281 dnf copr -y enable abbra/krb5-test
282 dnf update -y
284 dnf install -y \
285 --setopt=install_weak_deps=False \
286 {pkgs}
288 dnf clean all
291 ZYPPER_BOOTSTRAP = r"""
292 #!/bin/bash
293 {GENERATED_MARKER}
294 set -xueo pipefail
296 zypper --non-interactive refresh
297 zypper --non-interactive update
298 zypper --non-interactive install \
299 --no-recommends \
300 system-user-nobody \
301 {pkgs}
303 zypper --non-interactive clean
305 if [ -f /usr/lib/mit/bin/krb5-config ]; then
306 ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
310 # A generic shell script to setup locale
311 LOCALE_SETUP = r"""
312 #!/bin/bash
313 {GENERATED_MARKER}
314 set -xueo pipefail
316 # refer to /usr/share/i18n/locales
317 INPUTFILE=en_US
318 # refer to /usr/share/i18n/charmaps
319 CHARMAP=UTF-8
320 # locale to generate in /usr/lib/locale
321 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
322 LOCALE=$INPUTFILE.utf8
324 # if locale is already correct, exit
325 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
327 # if locale not available, generate locale into /usr/lib/locale
328 if ! ( locale --all-locales | grep -i $LOCALE )
329 then
330 # no-archive means create its own dir
331 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
334 # update locale conf and global env file
335 # set both LC_ALL and LANG for safe
337 # update conf for Debian family
338 FILE=/etc/default/locale
339 if [ -f $FILE ]
340 then
341 echo LC_ALL="$LOCALE" > $FILE
342 echo LANG="$LOCALE" >> $FILE
345 # update conf for RedHat family
346 FILE=/etc/locale.conf
347 if [ -f $FILE ]
348 then
349 # LC_ALL is not valid in this file, set LANG only
350 echo LANG="$LOCALE" > $FILE
353 # update global env file
354 FILE=/etc/environment
355 if [ -f $FILE ]
356 then
357 # append LC_ALL if not exist
358 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
359 # append LANG if not exist
360 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
365 DOCKERFILE = r"""
366 {GENERATED_MARKER}
367 FROM {docker_image}
369 # pass in with --build-arg while build
370 ARG SHA1SUM
371 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
373 ADD *.sh /tmp/
374 # need root permission, do it before USER samba
375 RUN /tmp/bootstrap.sh && /tmp/locale.sh
377 # if ld.gold exists, force link it to ld
378 RUN set -x; ! LD_GOLD=$(which ld.gold) || {{ LD=$(which ld) && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"; }}
379 # if ld.mold exists, force link it to ld (prefer mold over gold! ;-)
380 RUN set -x; ! LD_MOLD=$(which ld.mold) || {{ LD=$(which ld) && ln -sf $LD_MOLD $LD && test -x $LD && echo "$LD is now $LD_MOLD"; }}
382 # make test can not work with root, so we have to create a new user
383 RUN useradd -m -U -s /bin/bash samba && \
384 mkdir -p /etc/sudoers.d && \
385 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
387 USER samba
388 WORKDIR /home/samba
389 # samba tests rely on this
390 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
393 # Vagrantfile snippet for each dist
394 VAGRANTFILE_SNIPPET = r"""
395 config.vm.define "{name}" do |v|
396 v.vm.box = "{vagrant_box}"
397 v.vm.hostname = "{name}"
398 v.vm.provision :shell, path: "{name}/bootstrap.sh"
399 v.vm.provision :shell, path: "{name}/locale.sh"
403 # global Vagrantfile with snippets for all dists
404 VAGRANTFILE_GLOBAL = r"""
405 {GENERATED_MARKER}
407 Vagrant.configure("2") do |config|
408 config.ssh.insert_key = false
410 {vagrantfile_snippets}
416 DEB_DISTS = {
417 'debian11': {
418 'docker_image': 'debian:11',
419 'vagrant_box': 'debian/bullseye64',
420 'replace': {
421 'language-pack-en': '', # included in locales
424 'ubuntu1804': {
425 'docker_image': 'ubuntu:18.04',
426 'vagrant_box': 'ubuntu/bionic64',
427 'replace': {
428 'liburing-dev': '', # not available
431 'ubuntu2004': {
432 'docker_image': 'ubuntu:20.04',
433 'vagrant_box': 'ubuntu/focal64',
434 'replace': {
435 'liburing-dev': '', # not available
441 RPM_DISTS = {
442 'centos7': {
443 'docker_image': 'centos:7',
444 'vagrant_box': 'centos/7',
445 'bootstrap': YUM_BOOTSTRAP,
446 'replace': {
447 'lsb-release': 'redhat-lsb',
448 'python3': 'python36',
449 'python3-cryptography': 'python36-cryptography',
450 'python3-devel': 'python36-devel',
451 'python3-dns': 'python36-dns',
452 'python3-pyasn1': 'python36-pyasn1',
453 'python3-gpg': 'python36-gpg',
454 'python3-iso8601' : 'python36-iso8601',
455 'python3-markdown': 'python36-markdown',
456 'python3-requests': 'python36-requests',
457 # although python36-devel is available
458 # after epel-release installed
459 # however, all other python3 pkgs are still python36-ish
460 'python2-gpg': 'pygpgme',
461 '@development-tools': '"@Development Tools"', # add quotes
462 'glibc-langpack-en': '', # included in glibc-common
463 'glibc-locale-source': '', # included in glibc-common
464 # update perl core modules on centos
465 # fix: Can't locate Archive/Tar.pm in @INC
466 'perl': 'perl-core',
467 'perl-FindBin': '',
468 'rpcsvc-proto-devel': '',
469 'glusterfs-api-devel': '',
470 'glusterfs-devel': '',
471 'libcephfs-devel': '',
472 'gnutls-devel': 'compat-gnutls37-devel',
473 'liburing-devel': '', # not available
474 'python3-setproctitle': 'python36-setproctitle',
475 'tracker-devel': '', # do not install
476 'mold': '',
477 'ShellCheck': '',
478 'shfmt': '',
481 'centos8s': {
482 'docker_image': 'quay.io/centos/centos:stream8',
483 'vagrant_box': 'centos/stream8',
484 'bootstrap': CENTOS8S_YUM_BOOTSTRAP,
485 'replace': {
486 'lsb-release': 'redhat-lsb',
487 '@development-tools': '"@Development Tools"', # add quotes
488 'lcov': '', # does not exist
489 'perl-JSON-Parse': '', # does not exist?
490 'perl-Test-Base': 'perl-Test-Simple',
491 'perl-FindBin': '',
492 'liburing-devel': '', # not available yet, Add me back, once available!
493 'mold': '',
494 'ShellCheck': '',
495 'shfmt': '',
498 'fedora36': {
499 'docker_image': 'quay.io/fedora/fedora:36',
500 'vagrant_box': 'fedora/36-cloud-base',
501 'bootstrap': DNF_BOOTSTRAP,
502 'replace': {
503 'lsb-release': 'redhat-lsb',
504 'perl-FindBin': '',
505 'python3-iso8601': 'python3-dateutil',
506 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
509 'f36mit120': {
510 'docker_image': 'quay.io/fedora/fedora:36',
511 'vagrant_box': 'fedora/36-cloud-base',
512 'bootstrap': DNF_BOOTSTRAP_MIT,
513 'replace': {
514 'lsb-release': 'redhat-lsb',
515 'perl-FindBin': '',
516 'python3-iso8601': 'python3-dateutil',
517 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
520 'opensuse153': {
521 'docker_image': 'opensuse/leap:15.3',
522 'vagrant_box': 'opensuse/openSUSE-15.3-x86_64',
523 'bootstrap': ZYPPER_BOOTSTRAP,
524 'replace': {
525 '@development-tools': '',
526 'dbus-devel': 'dbus-1-devel',
527 'docbook-style-xsl': 'docbook-xsl-stylesheets',
528 'glibc-common': 'glibc-locale',
529 'glibc-locale-source': 'glibc-i18ndata',
530 'glibc-langpack-en': '',
531 'jansson-devel': 'libjansson-devel',
532 'keyutils-libs-devel': 'keyutils-devel',
533 'krb5-workstation': 'krb5-client',
534 'python3-libsemanage': 'python2-semanage',
535 'openldap-devel': 'openldap2-devel',
536 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
537 'perl-JSON-Parse': 'perl-JSON-XS',
538 'perl-generators': '',
539 'perl-interpreter': '',
540 'perl-FindBin': '',
541 'procps-ng': 'procps',
542 'python3-iso8601': 'python3-python-dateutil',
543 'python3-dns': 'python3-dnspython',
544 'python3-markdown': 'python3-Markdown',
545 'quota-devel': '',
546 'glusterfs-api-devel': '',
547 'libtasn1-tools': '', # asn1Parser is part of libtasn1
548 'mold': '',
549 'ShellCheck': '',
550 'shfmt': '',
556 DEB_FAMILY = {
557 'name': 'deb',
558 'pkgs': DEB_PKGS,
559 'bootstrap': APT_BOOTSTRAP, # family default
560 'dists': DEB_DISTS,
564 RPM_FAMILY = {
565 'name': 'rpm',
566 'pkgs': RPM_PKGS,
567 'bootstrap': YUM_BOOTSTRAP, # family default
568 'dists': RPM_DISTS,
572 YML_HEADER = r"""
574 packages:
578 def expand_family_dists(family):
579 dists = {}
580 for name, config in family['dists'].items():
581 config = config.copy()
582 config['name'] = name
583 config['home'] = join(OUT, name)
584 config['family'] = family['name']
585 config['GENERATED_MARKER'] = GENERATED_MARKER
587 # replace dist specific pkgs
588 replace = config.get('replace', {})
589 pkgs = []
590 for pkg in family['pkgs']:
591 pkg = replace.get(pkg, pkg) # replace if exists or get self
592 if pkg:
593 pkgs.append(pkg)
594 pkgs.sort()
596 lines = [' - {}'.format(pkg) for pkg in pkgs]
597 config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
599 sep = ' \\' + os.linesep + ' '
600 config['pkgs'] = sep.join(pkgs)
602 # get dist bootstrap template or fall back to family default
603 bootstrap_template = config.get('bootstrap', family['bootstrap'])
604 config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
605 config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
607 config['Dockerfile'] = DOCKERFILE.format(**config).strip()
608 # keep the indent, no strip
609 config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
611 dists[name] = config
612 return dists
615 # expanded config for dists
616 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
617 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
619 # assemble all together
620 DISTS = {}
621 DISTS.update(DEB_DISTS_EXP)
622 DISTS.update(RPM_DISTS_EXP)
625 def render_vagrantfile(dists):
627 Render all snippets for each dist into global Vagrantfile.
629 Vagrant supports multiple vms in one Vagrantfile.
630 This make it easier to manage the fleet, e.g:
632 start all: vagrant up
633 start one: vagrant up ubuntu1804
635 All other commands apply to above syntax, e.g.: status, destroy, provision
637 # sort dists by name and put all vagrantfile snippets together
638 snippets = [
639 dists[dist]['vagrantfile_snippet']
640 for dist in sorted(dists.keys())]
642 return VAGRANTFILE_GLOBAL.format(
643 vagrantfile_snippets=''.join(snippets),
644 GENERATED_MARKER=GENERATED_MARKER
648 VAGRANTFILE = render_vagrantfile(DISTS)
651 # data we need to expose
652 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']