3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Update the lists in this file to require new packages in the
24 container images used in GitLab CI
26 Author: Joe Guo <joeg@catalyst.net.nz>
29 from os
.path
import abspath
, dirname
, join
30 HERE
= abspath(dirname(__file__
))
31 # output dir for rendered files
32 OUT
= join(HERE
, 'generated-dists')
35 # pkgs with same name in all packaging systems
57 'psmisc', # for pstree in test
61 'sudo', # docker images has no sudo by default
68 # define pkgs for all packaging systems in parallel
69 # make it easier to find missing ones
70 # use latest ubuntu and fedora as defaults
73 # NAME1-dev, NAME2-devel
74 ('lmdb-utils', 'lmdb'),
75 ('mingw-w64', 'mingw64-gcc'),
76 ('zlib1g-dev', 'zlib-devel'),
77 ('libbsd-dev', 'libbsd-devel'),
78 ('liburing-dev', 'liburing-devel'),
79 ('libarchive-dev', 'libarchive-devel'),
80 ('libblkid-dev', 'libblkid-devel'),
81 ('libcap-dev', 'libcap-devel'),
82 ('libacl1-dev', 'libacl-devel'),
83 ('libattr1-dev', 'libattr-devel'),
85 # libNAME1-dev, NAME2-devel
86 ('libpopt-dev', 'popt-devel'),
87 ('libreadline-dev', 'readline-devel'),
88 ('libjansson-dev', 'jansson-devel'),
89 ('liblmdb-dev', 'lmdb-devel'),
90 ('libncurses5-dev', 'ncurses-devel'),
91 # NOTE: Debian 7+ or Ubuntu 16.04+
92 ('libsystemd-dev', 'systemd-devel'),
93 ('libkrb5-dev', 'krb5-devel'),
94 ('libldap2-dev', 'openldap-devel'),
95 ('libcups2-dev', 'cups-devel'),
96 ('libpam0g-dev', 'pam-devel'),
97 ('libgpgme11-dev', 'gpgme-devel'),
98 # NOTE: Debian 8+ and Ubuntu 14.04+
99 ('libgnutls28-dev', 'gnutls-devel'),
100 ('libtasn1-bin', 'libtasn1-tools'),
101 ('libtasn1-dev', 'libtasn1-devel'),
103 ('uuid-dev', 'libuuid-devel'),
104 ('libjs-jquery', ''),
105 ('libavahi-common-dev', 'avahi-devel'),
106 ('libdbus-1-dev', 'dbus-devel'),
107 ('libpcap-dev', 'libpcap-devel'),
108 ('libunwind-dev', 'libunwind-devel'), # for back trace
109 ('libglib2.0-dev', 'glib2-devel'),
110 ('libicu-dev', 'libicu-devel'),
111 ('heimdal-multidev', ''),
114 # for debian, locales provide locale support with language packs
115 # ubuntu split language packs to language-pack-xx
116 # for centos, glibc-common provide locale support with language packs
117 # fedora split language packs to glibc-langpack-xx
118 ('locales', 'glibc-common'), # required for locale
119 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
120 ('bind9utils', 'bind-utils'),
122 ('xsltproc', 'libxslt'),
123 ('krb5-user', 'krb5-workstation'),
125 ('krb5-kdc', 'krb5-server'),
126 ('apt-utils', 'yum-utils'),
127 ('pkg-config', 'pkgconfig'),
128 ('procps', 'procps-ng'), # required for the free cmd in tests
129 ('lsb-release', 'lsb-release'), # we need lsb_relase to show info
130 ('', 'rpcgen'), # required for test
131 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
132 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
133 ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
139 ('python3', 'python3'),
140 ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
141 ('python3-dev', 'python3-devel'),
143 ('python3-iso8601', 'python3-iso8601'),
144 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
145 ('python3-markdown', 'python3-markdown'),
146 ('python3-dnspython', 'python3-dns'),
147 ('python3-pexpect', ''), # for wintest only
148 ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
149 ('python3-setproctitle', 'python3-setproctitle'),
150 ('python3-requests', 'python3-requests'), # for cert auto enroll
152 ('', 'python3-libsemanage'),
153 ('', 'python3-policycoreutils'),
156 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
157 ('libjson-perl', 'perl-JSON'),
158 ('', 'perl-JSON-Parse'),
159 ('perl-modules', ''),
160 ('', 'perl-FindBin'),
161 ('', 'perl-Archive-Tar'),
162 ('', 'perl-ExtUtils-MakeMaker'),
163 ('', 'perl-Test-Base'),
164 ('', 'perl-generators'),
165 ('', 'perl-interpreter'),
168 ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
169 ('', 'glusterfs-api-devel'),
170 ('glusterfs-common', 'glusterfs-devel'),
171 ('libcephfs-dev', 'libcephfs-devel'),
174 ('libtracker-sparql-2.0-dev', 'tracker-devel'),
177 # @ means group for rpm, use fedora as rpm default
178 ('build-essential', '@development-tools'),
180 # rpm has no pkg for docbook-xml
181 ('docbook-xml', 'docbook-dtds'),
182 ('docbook-xsl', 'docbook-style-xsl'),
183 ('', 'keyutils-libs-devel'),
189 DEB_PKGS
= COMMON
+ [pkg
for pkg
, _
in PKGS
if pkg
]
190 RPM_PKGS
= COMMON
+ [pkg
for _
, pkg
in PKGS
if pkg
]
192 GENERATED_MARKER
= r
"""
194 # This file is generated by 'bootstrap/template.py --render'
195 # See also bootstrap/config.py
205 export DEBIAN_FRONTEND=noninteractive
211 apt-get -y autoremove
223 yum install -y epel-release
224 yum install -y yum-plugin-copr
225 yum copr enable -y sergiomb/SambaAD
233 if [ ! -f /usr/bin/python3 ]; then
234 ln -sf /usr/bin/python3.6 /usr/bin/python3
238 CENTOS8S_YUM_BOOTSTRAP
= r
"""
244 yum install -y dnf-plugins-core
245 yum install -y epel-release
248 yum config-manager --set-enabled powertools -y || \
249 yum config-manager --set-enabled powertools -y
254 --setopt=install_weak_deps=False \
268 --setopt=install_weak_deps=False \
274 DNF_BOOTSTRAP_MIT
= r
"""
280 dnf install -y dnf-plugins-core
281 dnf copr -y enable abbra/krb5-test
285 --setopt=install_weak_deps=False \
291 ZYPPER_BOOTSTRAP
= r
"""
296 zypper --non-interactive refresh
297 zypper --non-interactive update
298 zypper --non-interactive install \
303 zypper --non-interactive clean
305 if [ -f /usr/lib/mit/bin/krb5-config ]; then
306 ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
310 # A generic shell script to setup locale
316 # refer to /usr/share/i18n/locales
318 # refer to /usr/share/i18n/charmaps
320 # locale to generate in /usr/lib/locale
321 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
322 LOCALE=$INPUTFILE.utf8
324 # if locale is already correct, exit
325 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
327 # if locale not available, generate locale into /usr/lib/locale
328 if ! ( locale --all-locales | grep -i $LOCALE )
330 # no-archive means create its own dir
331 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
334 # update locale conf and global env file
335 # set both LC_ALL and LANG for safe
337 # update conf for Debian family
338 FILE=/etc/default/locale
341 echo LC_ALL="$LOCALE" > $FILE
342 echo LANG="$LOCALE" >> $FILE
345 # update conf for RedHat family
346 FILE=/etc/locale.conf
349 # LC_ALL is not valid in this file, set LANG only
350 echo LANG="$LOCALE" > $FILE
353 # update global env file
354 FILE=/etc/environment
357 # append LC_ALL if not exist
358 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
359 # append LANG if not exist
360 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
369 # pass in with --build-arg while build
371 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
374 # need root permission, do it before USER samba
375 RUN /tmp/bootstrap.sh && /tmp/locale.sh
377 # if ld.gold exists, force link it to ld
378 RUN set -x; ! LD_GOLD=$(which ld.gold) || {{ LD=$(which ld) && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"; }}
379 # if ld.mold exists, force link it to ld (prefer mold over gold! ;-)
380 RUN set -x; ! LD_MOLD=$(which ld.mold) || {{ LD=$(which ld) && ln -sf $LD_MOLD $LD && test -x $LD && echo "$LD is now $LD_MOLD"; }}
382 # make test can not work with root, so we have to create a new user
383 RUN useradd -m -U -s /bin/bash samba && \
384 mkdir -p /etc/sudoers.d && \
385 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
389 # samba tests rely on this
390 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
393 # Vagrantfile snippet for each dist
394 VAGRANTFILE_SNIPPET
= r
"""
395 config.vm.define "{name}" do |v|
396 v.vm.box = "{vagrant_box}"
397 v.vm.hostname = "{name}"
398 v.vm.provision :shell, path: "{name}/bootstrap.sh"
399 v.vm.provision :shell, path: "{name}/locale.sh"
403 # global Vagrantfile with snippets for all dists
404 VAGRANTFILE_GLOBAL
= r
"""
407 Vagrant.configure("2") do |config|
408 config.ssh.insert_key = false
410 {vagrantfile_snippets}
418 'docker_image': 'debian:11',
419 'vagrant_box': 'debian/bullseye64',
421 'language-pack-en': '', # included in locales
425 'docker_image': 'ubuntu:18.04',
426 'vagrant_box': 'ubuntu/bionic64',
428 'liburing-dev': '', # not available
432 'docker_image': 'ubuntu:20.04',
433 'vagrant_box': 'ubuntu/focal64',
435 'liburing-dev': '', # not available
443 'docker_image': 'centos:7',
444 'vagrant_box': 'centos/7',
445 'bootstrap': YUM_BOOTSTRAP
,
447 'lsb-release': 'redhat-lsb',
448 'python3': 'python36',
449 'python3-cryptography': 'python36-cryptography',
450 'python3-devel': 'python36-devel',
451 'python3-dns': 'python36-dns',
452 'python3-pyasn1': 'python36-pyasn1',
453 'python3-gpg': 'python36-gpg',
454 'python3-iso8601' : 'python36-iso8601',
455 'python3-markdown': 'python36-markdown',
456 'python3-requests': 'python36-requests',
457 # although python36-devel is available
458 # after epel-release installed
459 # however, all other python3 pkgs are still python36-ish
460 'python2-gpg': 'pygpgme',
461 '@development-tools': '"@Development Tools"', # add quotes
462 'glibc-langpack-en': '', # included in glibc-common
463 'glibc-locale-source': '', # included in glibc-common
464 # update perl core modules on centos
465 # fix: Can't locate Archive/Tar.pm in @INC
468 'rpcsvc-proto-devel': '',
469 'glusterfs-api-devel': '',
470 'glusterfs-devel': '',
471 'libcephfs-devel': '',
472 'gnutls-devel': 'compat-gnutls37-devel',
473 'liburing-devel': '', # not available
474 'python3-setproctitle': 'python36-setproctitle',
475 'tracker-devel': '', # do not install
482 'docker_image': 'quay.io/centos/centos:stream8',
483 'vagrant_box': 'centos/stream8',
484 'bootstrap': CENTOS8S_YUM_BOOTSTRAP
,
486 'lsb-release': 'redhat-lsb',
487 '@development-tools': '"@Development Tools"', # add quotes
488 'lcov': '', # does not exist
489 'perl-JSON-Parse': '', # does not exist?
490 'perl-Test-Base': 'perl-Test-Simple',
492 'liburing-devel': '', # not available yet, Add me back, once available!
499 'docker_image': 'quay.io/fedora/fedora:36',
500 'vagrant_box': 'fedora/36-cloud-base',
501 'bootstrap': DNF_BOOTSTRAP
,
503 'lsb-release': 'redhat-lsb',
505 'python3-iso8601': 'python3-dateutil',
506 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
510 'docker_image': 'quay.io/fedora/fedora:36',
511 'vagrant_box': 'fedora/36-cloud-base',
512 'bootstrap': DNF_BOOTSTRAP_MIT
,
514 'lsb-release': 'redhat-lsb',
516 'python3-iso8601': 'python3-dateutil',
517 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
521 'docker_image': 'opensuse/leap:15.3',
522 'vagrant_box': 'opensuse/openSUSE-15.3-x86_64',
523 'bootstrap': ZYPPER_BOOTSTRAP
,
525 '@development-tools': '',
526 'dbus-devel': 'dbus-1-devel',
527 'docbook-style-xsl': 'docbook-xsl-stylesheets',
528 'glibc-common': 'glibc-locale',
529 'glibc-locale-source': 'glibc-i18ndata',
530 'glibc-langpack-en': '',
531 'jansson-devel': 'libjansson-devel',
532 'keyutils-libs-devel': 'keyutils-devel',
533 'krb5-workstation': 'krb5-client',
534 'python3-libsemanage': 'python2-semanage',
535 'openldap-devel': 'openldap2-devel',
536 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
537 'perl-JSON-Parse': 'perl-JSON-XS',
538 'perl-generators': '',
539 'perl-interpreter': '',
541 'procps-ng': 'procps',
542 'python3-iso8601': 'python3-python-dateutil',
543 'python3-dns': 'python3-dnspython',
544 'python3-markdown': 'python3-Markdown',
546 'glusterfs-api-devel': '',
547 'libtasn1-tools': '', # asn1Parser is part of libtasn1
559 'bootstrap': APT_BOOTSTRAP
, # family default
567 'bootstrap': YUM_BOOTSTRAP
, # family default
578 def expand_family_dists(family
):
580 for name
, config
in family
['dists'].items():
581 config
= config
.copy()
582 config
['name'] = name
583 config
['home'] = join(OUT
, name
)
584 config
['family'] = family
['name']
585 config
['GENERATED_MARKER'] = GENERATED_MARKER
587 # replace dist specific pkgs
588 replace
= config
.get('replace', {})
590 for pkg
in family
['pkgs']:
591 pkg
= replace
.get(pkg
, pkg
) # replace if exists or get self
596 lines
= [' - {}'.format(pkg
) for pkg
in pkgs
]
597 config
['packages.yml'] = YML_HEADER
.lstrip() + os
.linesep
.join(lines
)
599 sep
= ' \\' + os
.linesep
+ ' '
600 config
['pkgs'] = sep
.join(pkgs
)
602 # get dist bootstrap template or fall back to family default
603 bootstrap_template
= config
.get('bootstrap', family
['bootstrap'])
604 config
['bootstrap.sh'] = bootstrap_template
.format(**config
).strip()
605 config
['locale.sh'] = LOCALE_SETUP
.format(**config
).strip()
607 config
['Dockerfile'] = DOCKERFILE
.format(**config
).strip()
608 # keep the indent, no strip
609 config
['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET
.format(**config
)
615 # expanded config for dists
616 DEB_DISTS_EXP
= expand_family_dists(DEB_FAMILY
)
617 RPM_DISTS_EXP
= expand_family_dists(RPM_FAMILY
)
619 # assemble all together
621 DISTS
.update(DEB_DISTS_EXP
)
622 DISTS
.update(RPM_DISTS_EXP
)
625 def render_vagrantfile(dists
):
627 Render all snippets for each dist into global Vagrantfile.
629 Vagrant supports multiple vms in one Vagrantfile.
630 This make it easier to manage the fleet, e.g:
632 start all: vagrant up
633 start one: vagrant up ubuntu1804
635 All other commands apply to above syntax, e.g.: status, destroy, provision
637 # sort dists by name and put all vagrantfile snippets together
639 dists
[dist
]['vagrantfile_snippet']
640 for dist
in sorted(dists
.keys())]
642 return VAGRANTFILE_GLOBAL
.format(
643 vagrantfile_snippets
=''.join(snippets
),
644 GENERATED_MARKER
=GENERATED_MARKER
648 VAGRANTFILE
= render_vagrantfile(DISTS
)
651 # data we need to expose
652 __all__
= ['DISTS', 'VAGRANTFILE', 'OUT']