3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Update the lists in this file to require new packages in the
24 container images used in GitLab CI
26 Author: Joe Guo <joeg@catalyst.net.nz>
29 from os
.path
import abspath
, dirname
, join
30 HERE
= abspath(dirname(__file__
))
31 # output dir for rendered files
32 OUT
= join(HERE
, 'generated-dists')
35 # pkgs with same name in all packaging systems
57 'psmisc', # for pstree in test
61 'sudo', # docker images has no sudo by default
68 # define pkgs for all packaging systems in parallel
69 # make it easier to find missing ones
70 # use latest ubuntu and fedora as defaults
73 # NAME1-dev, NAME2-devel
74 ('lmdb-utils', 'lmdb'),
75 ('mingw-w64', 'mingw64-gcc'),
76 ('zlib1g-dev', 'zlib-devel'),
77 ('libbsd-dev', 'libbsd-devel'),
78 ('liburing-dev', 'liburing-devel'),
79 ('libarchive-dev', 'libarchive-devel'),
80 ('libblkid-dev', 'libblkid-devel'),
81 ('libcap-dev', 'libcap-devel'),
82 ('libacl1-dev', 'libacl-devel'),
83 ('libattr1-dev', 'libattr-devel'),
85 # libNAME1-dev, NAME2-devel
86 ('libpopt-dev', 'popt-devel'),
87 ('libreadline-dev', 'readline-devel'),
88 ('libjansson-dev', 'jansson-devel'),
89 ('liblmdb-dev', 'lmdb-devel'),
90 ('libncurses5-dev', 'ncurses-devel'),
91 # NOTE: Debian 7+ or Ubuntu 16.04+
92 ('libsystemd-dev', 'systemd-devel'),
93 ('libkrb5-dev', 'krb5-devel'),
94 ('libldap2-dev', 'openldap-devel'),
95 ('libcups2-dev', 'cups-devel'),
96 ('libpam0g-dev', 'pam-devel'),
97 ('libgpgme11-dev', 'gpgme-devel'),
98 # NOTE: Debian 8+ and Ubuntu 14.04+
99 ('libgnutls28-dev', 'gnutls-devel'),
100 ('gnutls-bin', 'gnutls-utils'),
101 ('libtasn1-bin', 'libtasn1-tools'),
102 ('libtasn1-dev', 'libtasn1-devel'),
104 ('uuid-dev', 'libuuid-devel'),
105 ('libjs-jquery', ''),
106 ('libavahi-common-dev', 'avahi-devel'),
107 ('libdbus-1-dev', 'dbus-devel'),
108 ('libpcap-dev', 'libpcap-devel'),
109 ('libunwind-dev', 'libunwind-devel'), # for back trace
110 ('libglib2.0-dev', 'glib2-devel'),
111 ('libicu-dev', 'libicu-devel'),
112 ('heimdal-multidev', ''),
115 # for debian, locales provide locale support with language packs
116 # ubuntu split language packs to language-pack-xx
117 # for centos, glibc-common provide locale support with language packs
118 # fedora split language packs to glibc-langpack-xx
119 ('locales', 'glibc-common'), # required for locale
120 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
121 ('bind9utils', 'bind-utils'),
123 ('xsltproc', 'libxslt'),
124 ('krb5-user', 'krb5-workstation'),
126 ('krb5-kdc', 'krb5-server'),
127 ('apt-utils', 'yum-utils'),
128 ('pkg-config', 'pkgconfig'),
129 ('procps', 'procps-ng'), # required for the free cmd in tests
130 ('lsb-release', 'lsb-release'), # we need lsb_release to show info
131 ('', 'rpcgen'), # required for test
132 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
133 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
134 ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
139 ('', 'crypto-policies-scripts'),
142 ('python3', 'python3'),
143 ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
144 ('python3-dev', 'python3-devel'),
146 ('python3-iso8601', 'python3-iso8601'),
147 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
148 ('python3-markdown', 'python3-markdown'),
149 ('python3-dnspython', 'python3-dns'),
150 ('python3-pexpect', ''), # for wintest only
151 ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
152 ('python3-setproctitle', 'python3-setproctitle'),
153 ('python3-requests', 'python3-requests'), # for cert auto enroll
155 ('', 'python3-libsemanage'),
156 ('', 'python3-policycoreutils'),
159 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
160 ('perl-modules', ''),
161 ('', 'perl-FindBin'),
162 ('', 'perl-Archive-Tar'),
163 ('', 'perl-ExtUtils-MakeMaker'),
164 ('', 'perl-Test-Base'),
165 ('', 'perl-generators'),
166 ('', 'perl-interpreter'),
169 ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
170 ('', 'glusterfs-api-devel'),
171 ('glusterfs-common', 'glusterfs-devel'),
172 ('libcephfs-dev', 'libcephfs-devel'),
175 ('libtracker-sparql-2.0-dev', 'tracker-devel'),
178 # @ means group for rpm, use fedora as rpm default
179 ('build-essential', '@development-tools'),
181 # rpm has no pkg for docbook-xml
182 ('docbook-xml', 'docbook-dtds'),
183 ('docbook-xsl', 'docbook-style-xsl'),
184 ('libkeyutils-dev', 'keyutils-libs-devel'),
190 DEB_PKGS
= COMMON
+ [pkg
for pkg
, _
in PKGS
if pkg
]
191 RPM_PKGS
= COMMON
+ [pkg
for _
, pkg
in PKGS
if pkg
]
193 GENERATED_MARKER
= r
"""
195 # This file is generated by 'bootstrap/template.py --render'
196 # See also bootstrap/config.py
206 export DEBIAN_FRONTEND=noninteractive
212 apt-get -y autoremove
224 yum install -y epel-release
225 yum install -y yum-plugin-copr
226 yum copr enable -y sergiomb/SambaAD
234 if [ ! -f /usr/bin/python3 ]; then
235 ln -sf /usr/bin/python3.6 /usr/bin/python3
239 CENTOS8S_YUM_BOOTSTRAP
= r
"""
245 yum install -y dnf-plugins-core
246 yum install -y epel-release
249 yum config-manager --set-enabled powertools -y
254 --setopt=install_weak_deps=False \
268 --setopt=install_weak_deps=False \
273 update-crypto-policies --set DEFAULT:AD-SUPPORT
276 DNF_BOOTSTRAP_MIT
= r
"""
282 dnf install -y dnf-plugins-core
283 dnf copr -y enable abbra/krb5-test
287 --setopt=install_weak_deps=False \
293 ZYPPER_BOOTSTRAP
= r
"""
298 zypper --non-interactive refresh
299 zypper --non-interactive update
300 zypper --non-interactive install \
305 zypper --non-interactive clean
307 if [ -f /usr/lib/mit/bin/krb5-config ]; then
308 ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
312 # A generic shell script to setup locale
318 # refer to /usr/share/i18n/locales
320 # refer to /usr/share/i18n/charmaps
322 # locale to generate in /usr/lib/locale
323 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
324 LOCALE=$INPUTFILE.utf8
326 # if locale is already correct, exit
327 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
329 # if locale not available, generate locale into /usr/lib/locale
330 if ! ( locale --all-locales | grep -i $LOCALE )
332 # no-archive means create its own dir
333 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
336 # update locale conf and global env file
337 # set both LC_ALL and LANG for safe
339 # update conf for Debian family
340 FILE=/etc/default/locale
343 echo LC_ALL="$LOCALE" > $FILE
344 echo LANG="$LOCALE" >> $FILE
347 # update conf for RedHat family
348 FILE=/etc/locale.conf
351 # LC_ALL is not valid in this file, set LANG only
352 echo LANG="$LOCALE" > $FILE
355 # update global env file
356 FILE=/etc/environment
359 # append LC_ALL if not exist
360 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
361 # append LANG if not exist
362 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
371 # pass in with --build-arg while build
373 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
376 # need root permission, do it before USER samba
377 RUN /tmp/bootstrap.sh && /tmp/locale.sh
379 # if ld.gold exists, force link it to ld
380 RUN set -x; ! LD_GOLD=$(which ld.gold) || {{ LD=$(which ld) && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"; }}
381 # if ld.mold exists, force link it to ld (prefer mold over gold! ;-)
382 RUN set -x; ! LD_MOLD=$(which ld.mold) || {{ LD=$(which ld) && ln -sf $LD_MOLD $LD && test -x $LD && echo "$LD is now $LD_MOLD"; }}
384 # make test can not work with root, so we have to create a new user
385 RUN useradd -m -U -s /bin/bash samba && \
386 mkdir -p /etc/sudoers.d && \
387 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
391 # samba tests rely on this
392 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8 LANGUAGE=en_US
395 # Vagrantfile snippet for each dist
396 VAGRANTFILE_SNIPPET
= r
"""
397 config.vm.define "{name}" do |v|
398 v.vm.box = "{vagrant_box}"
399 v.vm.hostname = "{name}"
400 v.vm.provision :shell, path: "{name}/bootstrap.sh"
401 v.vm.provision :shell, path: "{name}/locale.sh"
405 # global Vagrantfile with snippets for all dists
406 VAGRANTFILE_GLOBAL
= r
"""
409 Vagrant.configure("2") do |config|
410 config.ssh.insert_key = false
412 {vagrantfile_snippets}
419 'docker_image': 'debian:11',
420 'vagrant_box': 'debian/bullseye64',
422 'language-pack-en': '', # included in locales
426 'docker_image': 'registry-1.docker.io/i386/debian:11',
427 'vagrant_box': 'debian/bullseye32',
429 'language-pack-en': '', # included in locales
433 'docker_image': 'debian:12',
434 'vagrant_box': 'debian/bookworm64',
436 'language-pack-en': '', # included in locales
437 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
441 'docker_image': 'registry-1.docker.io/i386/debian:12',
442 'vagrant_box': 'debian/bookworm32',
444 'language-pack-en': '', # included in locales
445 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
449 'docker_image': 'ubuntu:18.04',
450 'vagrant_box': 'ubuntu/bionic64',
452 'liburing-dev': '', # not available
455 'ubuntu1804-32bit': {
456 'docker_image': 'registry-1.docker.io/i386/ubuntu:18.04',
457 'vagrant_box': 'ubuntu/bionic32',
459 'liburing-dev': '', # not available
463 'docker_image': 'ubuntu:20.04',
464 'vagrant_box': 'ubuntu/focal64',
466 'liburing-dev': '', # not available
470 'docker_image': 'ubuntu:22.04',
471 'vagrant_box': 'ubuntu/jammy64',
473 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
481 'docker_image': 'centos:7',
482 'vagrant_box': 'centos/7',
483 'bootstrap': YUM_BOOTSTRAP
,
485 'lsb-release': 'redhat-lsb',
486 'python3': 'python36',
487 'python3-cryptography': 'python36-cryptography',
488 'python3-devel': 'python36-devel',
489 'python3-dns': 'python36-dns',
490 'python3-pyasn1': 'python36-pyasn1',
491 'python3-gpg': 'python36-gpg',
492 'python3-iso8601' : 'python36-iso8601',
493 'python3-markdown': 'python36-markdown',
494 'python3-requests': 'python36-requests',
495 # although python36-devel is available
496 # after epel-release installed
497 # however, all other python3 pkgs are still python36-ish
498 'python2-gpg': 'pygpgme',
499 '@development-tools': '"@Development Tools"', # add quotes
500 'glibc-langpack-en': '', # included in glibc-common
501 'glibc-locale-source': '', # included in glibc-common
502 # update perl core modules on centos
503 # fix: Can't locate Archive/Tar.pm in @INC
506 'rpcsvc-proto-devel': '',
507 'glusterfs-api-devel': '',
508 'glusterfs-devel': '',
509 'libcephfs-devel': '',
510 'gnutls-devel': 'compat-gnutls37-devel',
511 'gnutls-utils': 'compat-gnutls37-utils',
512 'liburing-devel': '', # not available
513 'python3-setproctitle': 'python36-setproctitle',
514 'tracker-devel': '', # do not install
522 'docker_image': 'quay.io/centos/centos:stream8',
523 'vagrant_box': 'centos/stream8',
524 'bootstrap': CENTOS8S_YUM_BOOTSTRAP
,
526 'lsb-release': 'redhat-lsb',
527 '@development-tools': '"@Development Tools"', # add quotes
528 'lcov': '', # does not exist
529 'perl-JSON-Parse': '', # does not exist?
530 'perl-Test-Base': 'perl-Test-Simple',
532 'liburing-devel': '', # not available yet, Add me back, once available!
540 'docker_image': 'quay.io/fedora/fedora:38',
541 'vagrant_box': 'fedora/38-cloud-base',
542 'bootstrap': DNF_BOOTSTRAP
,
544 'lsb-release': 'redhat-lsb',
546 'python3-iso8601': 'python3-dateutil',
547 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
551 'docker_image': 'opensuse/leap:15.5',
552 'vagrant_box': 'opensuse/openSUSE-15.5-x86_64',
553 'bootstrap': ZYPPER_BOOTSTRAP
,
555 '@development-tools': '',
556 'dbus-devel': 'dbus-1-devel',
557 'docbook-style-xsl': 'docbook-xsl-stylesheets',
558 'glibc-common': 'glibc-locale',
559 'glibc-locale-source': 'glibc-i18ndata',
560 'glibc-langpack-en': '',
561 'jansson-devel': 'libjansson-devel',
562 'keyutils-libs-devel': 'keyutils-devel',
563 'krb5-workstation': 'krb5-client',
564 'python3-libsemanage': 'python3-semanage',
565 'openldap-devel': 'openldap2-devel',
566 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
567 'perl-JSON-Parse': 'perl-JSON-XS',
568 'perl-generators': '',
569 'perl-interpreter': '',
571 'procps-ng': 'procps',
572 'python3-iso8601': 'python3-python-dateutil',
573 'python3-dns': 'python3-dnspython',
574 'python3-markdown': 'python3-Markdown',
576 'glusterfs-api-devel': '',
577 'gnutls-utils': 'gnutls',
578 'libtasn1-tools': '', # asn1Parser is part of libtasn1
590 'bootstrap': APT_BOOTSTRAP
, # family default
598 'bootstrap': YUM_BOOTSTRAP
, # family default
609 def expand_family_dists(family
):
611 for name
, config
in family
['dists'].items():
612 config
= config
.copy()
613 config
['name'] = name
614 config
['home'] = join(OUT
, name
)
615 config
['family'] = family
['name']
616 config
['GENERATED_MARKER'] = GENERATED_MARKER
618 # replace dist specific pkgs
619 replace
= config
.get('replace', {})
621 for pkg
in family
['pkgs']:
622 pkg
= replace
.get(pkg
, pkg
) # replace if exists or get self
627 lines
= [' - {}'.format(pkg
) for pkg
in pkgs
]
628 config
['packages.yml'] = YML_HEADER
.lstrip() + os
.linesep
.join(lines
)
630 sep
= ' \\' + os
.linesep
+ ' '
631 config
['pkgs'] = sep
.join(pkgs
)
633 # get dist bootstrap template or fall back to family default
634 bootstrap_template
= config
.get('bootstrap', family
['bootstrap'])
635 config
['bootstrap.sh'] = bootstrap_template
.format(**config
).strip()
636 config
['locale.sh'] = LOCALE_SETUP
.format(**config
).strip()
638 config
['Dockerfile'] = DOCKERFILE
.format(**config
).strip()
639 # keep the indent, no strip
640 config
['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET
.format(**config
)
646 # expanded config for dists
647 DEB_DISTS_EXP
= expand_family_dists(DEB_FAMILY
)
648 RPM_DISTS_EXP
= expand_family_dists(RPM_FAMILY
)
650 # assemble all together
652 DISTS
.update(DEB_DISTS_EXP
)
653 DISTS
.update(RPM_DISTS_EXP
)
656 def render_vagrantfile(dists
):
658 Render all snippets for each dist into global Vagrantfile.
660 Vagrant supports multiple vms in one Vagrantfile.
661 This make it easier to manage the fleet, e.g:
663 start all: vagrant up
664 start one: vagrant up ubuntu1804
666 All other commands apply to above syntax, e.g.: status, destroy, provision
668 # sort dists by name and put all vagrantfile snippets together
670 dists
[dist
]['vagrantfile_snippet']
671 for dist
in sorted(dists
.keys())]
673 return VAGRANTFILE_GLOBAL
.format(
674 vagrantfile_snippets
=''.join(snippets
),
675 GENERATED_MARKER
=GENERATED_MARKER
679 VAGRANTFILE
= render_vagrantfile(DISTS
)
682 # data we need to expose
683 __all__
= ['DISTS', 'VAGRANTFILE', 'OUT']