3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Update the lists in this file to require new packages in the
24 container images used in GitLab CI
26 Author: Joe Guo <joeg@catalyst.net.nz>
29 from os
.path
import abspath
, dirname
, join
30 HERE
= abspath(dirname(__file__
))
31 # output dir for rendered files
32 OUT
= join(HERE
, 'generated-dists')
35 # pkgs with same name in all packaging systems
57 'psmisc', # for pstree in test
61 'sudo', # docker images has no sudo by default
68 # define pkgs for all packaging systems in parallel
69 # make it easier to find missing ones
70 # use latest ubuntu and fedora as defaults
73 # NAME1-dev, NAME2-devel
74 ('lmdb-utils', 'lmdb'),
75 ('mingw-w64', 'mingw64-gcc'),
76 ('zlib1g-dev', 'zlib-devel'),
77 ('libbsd-dev', 'libbsd-devel'),
78 ('liburing-dev', 'liburing-devel'),
79 ('libarchive-dev', 'libarchive-devel'),
80 ('libblkid-dev', 'libblkid-devel'),
81 ('libcap-dev', 'libcap-devel'),
82 ('libacl1-dev', 'libacl-devel'),
83 ('libattr1-dev', 'libattr-devel'),
85 # libNAME1-dev, NAME2-devel
86 ('libpopt-dev', 'popt-devel'),
87 ('libreadline-dev', 'readline-devel'),
88 ('libjansson-dev', 'jansson-devel'),
89 ('liblmdb-dev', 'lmdb-devel'),
90 ('libncurses5-dev', 'ncurses-devel'),
91 # NOTE: Debian 7+ or Ubuntu 16.04+
92 ('libsystemd-dev', 'systemd-devel'),
93 ('libkrb5-dev', 'krb5-devel'),
94 ('libldap2-dev', 'openldap-devel'),
95 ('libcups2-dev', 'cups-devel'),
96 ('libpam0g-dev', 'pam-devel'),
97 ('libgpgme11-dev', 'gpgme-devel'),
98 # NOTE: Debian 8+ and Ubuntu 14.04+
99 ('libgnutls28-dev', 'gnutls-devel'),
100 ('gnutls-bin', 'gnutls-utils'),
101 ('libtasn1-bin', 'libtasn1-tools'),
102 ('libtasn1-dev', 'libtasn1-devel'),
104 ('uuid-dev', 'libuuid-devel'),
105 ('libjs-jquery', ''),
106 ('libavahi-common-dev', 'avahi-devel'),
107 ('libdbus-1-dev', 'dbus-devel'),
108 ('libpcap-dev', 'libpcap-devel'),
109 ('libunwind-dev', 'libunwind-devel'), # for back trace
110 ('libglib2.0-dev', 'glib2-devel'),
111 ('libicu-dev', 'libicu-devel'),
112 ('heimdal-multidev', ''),
115 # for debian, locales provide locale support with language packs
116 # ubuntu split language packs to language-pack-xx
117 # for centos, glibc-common provide locale support with language packs
118 # fedora split language packs to glibc-langpack-xx
119 ('locales', 'glibc-common'), # required for locale
120 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
121 ('bind9utils', 'bind-utils'),
123 ('xsltproc', 'libxslt'),
124 ('krb5-user', 'krb5-workstation'),
126 ('krb5-kdc', 'krb5-server'),
127 ('apt-utils', 'yum-utils'),
128 ('pkg-config', 'pkgconfig'),
129 ('procps', 'procps-ng'), # required for the free cmd in tests
130 ('lsb-release', 'lsb-release'), # we need lsb_release to show info
131 ('', 'rpcgen'), # required for test
132 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
133 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
134 ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
139 ('', 'crypto-policies-scripts'),
141 ('python3', 'python3'),
142 ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
143 ('python3-dev', 'python3-devel'),
145 ('python3-iso8601', 'python3-iso8601'),
146 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
147 ('python3-markdown', 'python3-markdown'),
148 ('python3-dnspython', 'python3-dns'),
149 ('python3-pexpect', ''), # for wintest only
150 ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
151 ('python3-setproctitle', 'python3-setproctitle'),
152 ('python3-requests', 'python3-requests'), # for cert auto enroll
154 ('', 'python3-libsemanage'),
155 ('', 'python3-policycoreutils'),
158 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
159 ('libjson-perl', 'perl-JSON'),
160 ('', 'perl-JSON-Parse'),
161 ('perl-modules', ''),
162 ('', 'perl-FindBin'),
163 ('', 'perl-Archive-Tar'),
164 ('', 'perl-ExtUtils-MakeMaker'),
165 ('', 'perl-Test-Base'),
166 ('', 'perl-generators'),
167 ('', 'perl-interpreter'),
170 ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
171 ('', 'glusterfs-api-devel'),
172 ('glusterfs-common', 'glusterfs-devel'),
173 ('libcephfs-dev', 'libcephfs-devel'),
176 ('libtracker-sparql-2.0-dev', 'tracker-devel'),
179 # @ means group for rpm, use fedora as rpm default
180 ('build-essential', '@development-tools'),
182 # rpm has no pkg for docbook-xml
183 ('docbook-xml', 'docbook-dtds'),
184 ('docbook-xsl', 'docbook-style-xsl'),
185 ('libkeyutils-dev', 'keyutils-libs-devel'),
191 DEB_PKGS
= COMMON
+ [pkg
for pkg
, _
in PKGS
if pkg
]
192 RPM_PKGS
= COMMON
+ [pkg
for _
, pkg
in PKGS
if pkg
]
194 GENERATED_MARKER
= r
"""
196 # This file is generated by 'bootstrap/template.py --render'
197 # See also bootstrap/config.py
207 export DEBIAN_FRONTEND=noninteractive
213 apt-get -y autoremove
225 yum install -y epel-release
226 yum install -y yum-plugin-copr
227 yum copr enable -y sergiomb/SambaAD
235 if [ ! -f /usr/bin/python3 ]; then
236 ln -sf /usr/bin/python3.6 /usr/bin/python3
240 CENTOS8S_YUM_BOOTSTRAP
= r
"""
246 yum install -y dnf-plugins-core
247 yum install -y epel-release
250 yum config-manager --set-enabled powertools -y
255 --setopt=install_weak_deps=False \
269 --setopt=install_weak_deps=False \
274 update-crypto-policies --set DEFAULT:AD-SUPPORT
277 DNF_BOOTSTRAP_MIT
= r
"""
283 dnf install -y dnf-plugins-core
284 dnf copr -y enable abbra/krb5-test
288 --setopt=install_weak_deps=False \
294 ZYPPER_BOOTSTRAP
= r
"""
299 zypper --non-interactive refresh
300 zypper --non-interactive update
301 zypper --non-interactive install \
306 zypper --non-interactive clean
308 if [ -f /usr/lib/mit/bin/krb5-config ]; then
309 ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
313 # A generic shell script to setup locale
319 # refer to /usr/share/i18n/locales
321 # refer to /usr/share/i18n/charmaps
323 # locale to generate in /usr/lib/locale
324 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
325 LOCALE=$INPUTFILE.utf8
327 # if locale is already correct, exit
328 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
330 # if locale not available, generate locale into /usr/lib/locale
331 if ! ( locale --all-locales | grep -i $LOCALE )
333 # no-archive means create its own dir
334 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
337 # update locale conf and global env file
338 # set both LC_ALL and LANG for safe
340 # update conf for Debian family
341 FILE=/etc/default/locale
344 echo LC_ALL="$LOCALE" > $FILE
345 echo LANG="$LOCALE" >> $FILE
348 # update conf for RedHat family
349 FILE=/etc/locale.conf
352 # LC_ALL is not valid in this file, set LANG only
353 echo LANG="$LOCALE" > $FILE
356 # update global env file
357 FILE=/etc/environment
360 # append LC_ALL if not exist
361 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
362 # append LANG if not exist
363 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
372 # pass in with --build-arg while build
374 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
377 # need root permission, do it before USER samba
378 RUN /tmp/bootstrap.sh && /tmp/locale.sh
380 # if ld.gold exists, force link it to ld
381 RUN set -x; ! LD_GOLD=$(which ld.gold) || {{ LD=$(which ld) && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"; }}
382 # if ld.mold exists, force link it to ld (prefer mold over gold! ;-)
383 RUN set -x; ! LD_MOLD=$(which ld.mold) || {{ LD=$(which ld) && ln -sf $LD_MOLD $LD && test -x $LD && echo "$LD is now $LD_MOLD"; }}
385 # make test can not work with root, so we have to create a new user
386 RUN useradd -m -U -s /bin/bash samba && \
387 mkdir -p /etc/sudoers.d && \
388 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
392 # samba tests rely on this
393 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8 LANGUAGE=en_US
396 # Vagrantfile snippet for each dist
397 VAGRANTFILE_SNIPPET
= r
"""
398 config.vm.define "{name}" do |v|
399 v.vm.box = "{vagrant_box}"
400 v.vm.hostname = "{name}"
401 v.vm.provision :shell, path: "{name}/bootstrap.sh"
402 v.vm.provision :shell, path: "{name}/locale.sh"
406 # global Vagrantfile with snippets for all dists
407 VAGRANTFILE_GLOBAL
= r
"""
410 Vagrant.configure("2") do |config|
411 config.ssh.insert_key = false
413 {vagrantfile_snippets}
420 'docker_image': 'debian:11',
421 'vagrant_box': 'debian/bullseye64',
423 'language-pack-en': '', # included in locales
427 'docker_image': 'registry-1.docker.io/i386/debian:11',
428 'vagrant_box': 'debian/bullseye32',
430 'language-pack-en': '', # included in locales
434 'docker_image': 'debian:12',
435 'vagrant_box': 'debian/bookworm64',
437 'language-pack-en': '', # included in locales
438 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
442 'docker_image': 'registry-1.docker.io/i386/debian:12',
443 'vagrant_box': 'debian/bookworm32',
445 'language-pack-en': '', # included in locales
446 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
450 'docker_image': 'ubuntu:18.04',
451 'vagrant_box': 'ubuntu/bionic64',
453 'liburing-dev': '', # not available
456 'ubuntu1804-32bit': {
457 'docker_image': 'registry-1.docker.io/i386/ubuntu:18.04',
458 'vagrant_box': 'ubuntu/bionic32',
460 'liburing-dev': '', # not available
464 'docker_image': 'ubuntu:20.04',
465 'vagrant_box': 'ubuntu/focal64',
467 'liburing-dev': '', # not available
471 'docker_image': 'ubuntu:22.04',
472 'vagrant_box': 'ubuntu/jammy64',
474 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
482 'docker_image': 'centos:7',
483 'vagrant_box': 'centos/7',
484 'bootstrap': YUM_BOOTSTRAP
,
486 'lsb-release': 'redhat-lsb',
487 'python3': 'python36',
488 'python3-cryptography': 'python36-cryptography',
489 'python3-devel': 'python36-devel',
490 'python3-dns': 'python36-dns',
491 'python3-pyasn1': 'python36-pyasn1',
492 'python3-gpg': 'python36-gpg',
493 'python3-iso8601' : 'python36-iso8601',
494 'python3-markdown': 'python36-markdown',
495 'python3-requests': 'python36-requests',
496 # although python36-devel is available
497 # after epel-release installed
498 # however, all other python3 pkgs are still python36-ish
499 'python2-gpg': 'pygpgme',
500 '@development-tools': '"@Development Tools"', # add quotes
501 'glibc-langpack-en': '', # included in glibc-common
502 'glibc-locale-source': '', # included in glibc-common
503 # update perl core modules on centos
504 # fix: Can't locate Archive/Tar.pm in @INC
507 'rpcsvc-proto-devel': '',
508 'glusterfs-api-devel': '',
509 'glusterfs-devel': '',
510 'libcephfs-devel': '',
511 'gnutls-devel': 'compat-gnutls37-devel',
512 'gnutls-utils': 'compat-gnutls37-utils',
513 'liburing-devel': '', # not available
514 'python3-setproctitle': 'python36-setproctitle',
515 'tracker-devel': '', # do not install
522 'docker_image': 'quay.io/centos/centos:stream8',
523 'vagrant_box': 'centos/stream8',
524 'bootstrap': CENTOS8S_YUM_BOOTSTRAP
,
526 'lsb-release': 'redhat-lsb',
527 '@development-tools': '"@Development Tools"', # add quotes
528 'lcov': '', # does not exist
529 'perl-JSON-Parse': '', # does not exist?
530 'perl-Test-Base': 'perl-Test-Simple',
532 'liburing-devel': '', # not available yet, Add me back, once available!
539 'docker_image': 'quay.io/fedora/fedora:38',
540 'vagrant_box': 'fedora/38-cloud-base',
541 'bootstrap': DNF_BOOTSTRAP
,
543 'lsb-release': 'redhat-lsb',
545 'python3-iso8601': 'python3-dateutil',
546 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
550 'docker_image': 'opensuse/leap:15.5',
551 'vagrant_box': 'opensuse/openSUSE-15.5-x86_64',
552 'bootstrap': ZYPPER_BOOTSTRAP
,
554 '@development-tools': '',
555 'dbus-devel': 'dbus-1-devel',
556 'docbook-style-xsl': 'docbook-xsl-stylesheets',
557 'glibc-common': 'glibc-locale',
558 'glibc-locale-source': 'glibc-i18ndata',
559 'glibc-langpack-en': '',
560 'jansson-devel': 'libjansson-devel',
561 'keyutils-libs-devel': 'keyutils-devel',
562 'krb5-workstation': 'krb5-client',
563 'python3-libsemanage': 'python3-semanage',
564 'openldap-devel': 'openldap2-devel',
565 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
566 'perl-JSON-Parse': 'perl-JSON-XS',
567 'perl-generators': '',
568 'perl-interpreter': '',
570 'procps-ng': 'procps',
571 'python3-iso8601': 'python3-python-dateutil',
572 'python3-dns': 'python3-dnspython',
573 'python3-markdown': 'python3-Markdown',
575 'glusterfs-api-devel': '',
576 'gnutls-utils': 'gnutls',
577 'libtasn1-tools': '', # asn1Parser is part of libtasn1
589 'bootstrap': APT_BOOTSTRAP
, # family default
597 'bootstrap': YUM_BOOTSTRAP
, # family default
608 def expand_family_dists(family
):
610 for name
, config
in family
['dists'].items():
611 config
= config
.copy()
612 config
['name'] = name
613 config
['home'] = join(OUT
, name
)
614 config
['family'] = family
['name']
615 config
['GENERATED_MARKER'] = GENERATED_MARKER
617 # replace dist specific pkgs
618 replace
= config
.get('replace', {})
620 for pkg
in family
['pkgs']:
621 pkg
= replace
.get(pkg
, pkg
) # replace if exists or get self
626 lines
= [' - {}'.format(pkg
) for pkg
in pkgs
]
627 config
['packages.yml'] = YML_HEADER
.lstrip() + os
.linesep
.join(lines
)
629 sep
= ' \\' + os
.linesep
+ ' '
630 config
['pkgs'] = sep
.join(pkgs
)
632 # get dist bootstrap template or fall back to family default
633 bootstrap_template
= config
.get('bootstrap', family
['bootstrap'])
634 config
['bootstrap.sh'] = bootstrap_template
.format(**config
).strip()
635 config
['locale.sh'] = LOCALE_SETUP
.format(**config
).strip()
637 config
['Dockerfile'] = DOCKERFILE
.format(**config
).strip()
638 # keep the indent, no strip
639 config
['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET
.format(**config
)
645 # expanded config for dists
646 DEB_DISTS_EXP
= expand_family_dists(DEB_FAMILY
)
647 RPM_DISTS_EXP
= expand_family_dists(RPM_FAMILY
)
649 # assemble all together
651 DISTS
.update(DEB_DISTS_EXP
)
652 DISTS
.update(RPM_DISTS_EXP
)
655 def render_vagrantfile(dists
):
657 Render all snippets for each dist into global Vagrantfile.
659 Vagrant supports multiple vms in one Vagrantfile.
660 This make it easier to manage the fleet, e.g:
662 start all: vagrant up
663 start one: vagrant up ubuntu1804
665 All other commands apply to above syntax, e.g.: status, destroy, provision
667 # sort dists by name and put all vagrantfile snippets together
669 dists
[dist
]['vagrantfile_snippet']
670 for dist
in sorted(dists
.keys())]
672 return VAGRANTFILE_GLOBAL
.format(
673 vagrantfile_snippets
=''.join(snippets
),
674 GENERATED_MARKER
=GENERATED_MARKER
678 VAGRANTFILE
= render_vagrantfile(DISTS
)
681 # data we need to expose
682 __all__
= ['DISTS', 'VAGRANTFILE', 'OUT']