3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Update the lists in this file to require new packages in the
24 container images used in GitLab CI
26 Author: Joe Guo <joeg@catalyst.net.nz>
29 from os
.path
import abspath
, dirname
, join
30 HERE
= abspath(dirname(__file__
))
31 # output dir for rendered files
32 OUT
= join(HERE
, 'generated-dists')
35 # pkgs with same name in all packaging systems
56 'psmisc', # for pstree in test
60 'sudo', # docker images has no sudo by default
67 # define pkgs for all packaging systems in parallel
68 # make it easier to find missing ones
69 # use latest ubuntu and fedora as defaults
72 # NAME1-dev, NAME2-devel
73 ('lmdb-utils', 'lmdb'),
74 ('mingw-w64', 'mingw64-gcc'),
75 ('zlib1g-dev', 'zlib-devel'),
76 ('libbsd-dev', 'libbsd-devel'),
77 ('liburing-dev', 'liburing-devel'),
78 ('libarchive-dev', 'libarchive-devel'),
79 ('libblkid-dev', 'libblkid-devel'),
80 ('libcap-dev', 'libcap-devel'),
81 ('libacl1-dev', 'libacl-devel'),
82 ('libattr1-dev', 'libattr-devel'),
84 # libNAME1-dev, NAME2-devel
85 ('libpopt-dev', 'popt-devel'),
86 ('libreadline-dev', 'readline-devel'),
87 ('libjansson-dev', 'jansson-devel'),
88 ('liblmdb-dev', 'lmdb-devel'),
89 ('libncurses5-dev', 'ncurses-devel'),
90 # NOTE: Debian 7+ or Ubuntu 16.04+
91 ('libsystemd-dev', 'systemd-devel'),
92 ('libkrb5-dev', 'krb5-devel'),
93 ('libldap2-dev', 'openldap-devel'),
94 ('libcups2-dev', 'cups-devel'),
95 ('libpam0g-dev', 'pam-devel'),
96 ('libgpgme11-dev', 'gpgme-devel'),
97 # NOTE: Debian 8+ and Ubuntu 14.04+
98 ('libgnutls28-dev', 'gnutls-devel'),
99 ('libtasn1-bin', 'libtasn1-tools'),
100 ('libtasn1-dev', 'libtasn1-devel'),
102 ('uuid-dev', 'libuuid-devel'),
103 ('libjs-jquery', ''),
104 ('libavahi-common-dev', 'avahi-devel'),
105 ('libdbus-1-dev', 'dbus-devel'),
106 ('libpcap-dev', 'libpcap-devel'),
107 ('libunwind-dev', 'libunwind-devel'), # for back trace
108 ('libglib2.0-dev', 'glib2-devel'),
109 ('libicu-dev', 'libicu-devel'),
110 ('heimdal-multidev', ''),
113 # for debian, locales provide locale support with language packs
114 # ubuntu split language packs to language-pack-xx
115 # for centos, glibc-common provide locale support with language packs
116 # fedora split language packs to glibc-langpack-xx
117 ('locales', 'glibc-common'), # required for locale
118 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
119 ('bind9utils', 'bind-utils'),
121 ('xsltproc', 'libxslt'),
122 ('krb5-user', 'krb5-workstation'),
124 ('krb5-kdc', 'krb5-server'),
125 ('apt-utils', 'yum-utils'),
126 ('pkg-config', 'pkgconfig'),
127 ('procps', 'procps-ng'), # required for the free cmd in tests
128 ('lsb-release', 'lsb-release'), # we need lsb_relase to show info
129 ('', 'rpcgen'), # required for test
130 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
131 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
132 ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
135 ('python3', 'python3'),
136 ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
137 ('python3-dev', 'python3-devel'),
139 ('python3-iso8601', 'python3-iso8601'),
140 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
141 ('python3-markdown', 'python3-markdown'),
142 ('python3-matplotlib', ''),
143 ('python3-dnspython', 'python3-dns'),
144 ('python3-pexpect', ''), # for wintest only
145 ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
146 ('python3-setproctitle', 'python3-setproctitle'),
148 ('', 'libsemanage-python'),
149 ('', 'policycoreutils-python'),
152 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
153 ('libjson-perl', 'perl-JSON'),
154 ('', 'perl-JSON-Parse'),
155 ('perl-modules', ''),
156 ('', 'perl-FindBin'),
157 ('', 'perl-Archive-Tar'),
158 ('', 'perl-ExtUtils-MakeMaker'),
159 ('', 'perl-Test-Base'),
160 ('', 'perl-generators'),
161 ('', 'perl-interpreter'),
164 ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
165 ('', 'glusterfs-api-devel'),
166 ('glusterfs-common', 'glusterfs-devel'),
167 ('libcephfs-dev', 'libcephfs-devel'),
170 ('libtracker-sparql-2.0-dev', 'tracker-devel'),
173 # @ means group for rpm, use fedora as rpm default
174 ('build-essential', '@development-tools'),
176 # rpm has no pkg for docbook-xml
177 ('docbook-xml', 'docbook-dtds'),
178 ('docbook-xsl', 'docbook-style-xsl'),
179 ('', 'keyutils-libs-devel'),
184 DEB_PKGS
= COMMON
+ [pkg
for pkg
, _
in PKGS
if pkg
]
185 RPM_PKGS
= COMMON
+ [pkg
for _
, pkg
in PKGS
if pkg
]
187 GENERATED_MARKER
= r
"""
189 # This file is generated by 'bootstrap/template.py --render'
190 # See also bootstrap/config.py
200 export DEBIAN_FRONTEND=noninteractive
206 apt-get -y autoremove
218 yum install -y epel-release
219 yum install -y yum-plugin-copr
220 yum copr enable -y sergiomb/SambaAD
228 if [ ! -f /usr/bin/python3 ]; then
229 ln -sf /usr/bin/python3.6 /usr/bin/python3
233 CENTOS8_YUM_BOOTSTRAP
= r
"""
239 yum install -y dnf-plugins-core
240 yum install -y epel-release
243 yum config-manager --set-enabled PowerTools -y || \
244 yum config-manager --set-enabled powertools -y
245 yum config-manager --set-enabled Devel -y || \
246 yum config-manager --set-enabled devel -y
250 --setopt=install_weak_deps=False \
264 --setopt=install_weak_deps=False \
270 ZYPPER_BOOTSTRAP
= r
"""
275 zypper --non-interactive refresh
276 zypper --non-interactive update
277 zypper --non-interactive install \
282 zypper --non-interactive clean
284 if [ -f /usr/lib/mit/bin/krb5-config ]; then
285 ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
289 # A generic shell script to setup locale
295 # refer to /usr/share/i18n/locales
297 # refer to /usr/share/i18n/charmaps
299 # locale to generate in /usr/lib/locale
300 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
301 LOCALE=$INPUTFILE.utf8
303 # if locale is already correct, exit
304 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
306 # if locale not available, generate locale into /usr/lib/locale
307 if ! ( locale --all-locales | grep -i $LOCALE )
309 # no-archive means create its own dir
310 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
313 # update locale conf and global env file
314 # set both LC_ALL and LANG for safe
316 # update conf for Debian family
317 FILE=/etc/default/locale
320 echo LC_ALL="$LOCALE" > $FILE
321 echo LANG="$LOCALE" >> $FILE
324 # update conf for RedHat family
325 FILE=/etc/locale.conf
328 # LC_ALL is not valid in this file, set LANG only
329 echo LANG="$LOCALE" > $FILE
332 # update global env file
333 FILE=/etc/environment
336 # append LC_ALL if not exist
337 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
338 # append LANG if not exist
339 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
348 # pass in with --build-arg while build
350 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
353 # need root permission, do it before USER samba
354 RUN /tmp/bootstrap.sh && /tmp/locale.sh
356 # if ld.gold exists, force link it to ld
357 RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"
359 # make test can not work with root, so we have to create a new user
360 RUN useradd -m -U -s /bin/bash samba && \
361 mkdir -p /etc/sudoers.d && \
362 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
366 # samba tests rely on this
367 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
370 # Vagrantfile snippet for each dist
371 VAGRANTFILE_SNIPPET
= r
"""
372 config.vm.define "{name}" do |v|
373 v.vm.box = "{vagrant_box}"
374 v.vm.hostname = "{name}"
375 v.vm.provision :shell, path: "{name}/bootstrap.sh"
376 v.vm.provision :shell, path: "{name}/locale.sh"
380 # global Vagrantfile with snippets for all dists
381 VAGRANTFILE_GLOBAL
= r
"""
384 Vagrant.configure("2") do |config|
385 config.ssh.insert_key = false
387 {vagrantfile_snippets}
395 'docker_image': 'debian:10',
396 'vagrant_box': 'debian/buster64',
398 'language-pack-en': '', # included in locales
399 'liburing-dev': '', # not available
403 'docker_image': 'ubuntu:18.04',
404 'vagrant_box': 'ubuntu/bionic64',
406 'liburing-dev': '', # not available
410 'docker_image': 'ubuntu:20.04',
411 'vagrant_box': 'ubuntu/focal64',
413 'liburing-dev': '', # not available
421 'docker_image': 'centos:7',
422 'vagrant_box': 'centos/7',
423 'bootstrap': YUM_BOOTSTRAP
,
425 'lsb-release': 'redhat-lsb',
426 'python3': 'python36',
427 'python3-cryptography': 'python36-cryptography',
428 'python3-devel': 'python36-devel',
429 'python3-dns': 'python36-dns',
430 'python3-pyasn1': 'python36-pyasn1',
431 'python3-gpg': 'python36-gpg',
432 'python3-iso8601' : 'python36-iso8601',
433 'python3-markdown': 'python36-markdown',
434 # although python36-devel is available
435 # after epel-release installed
436 # however, all other python3 pkgs are still python36-ish
437 'python2-gpg': 'pygpgme',
438 'python3-gpg': '', # no python3-gpg yet
439 '@development-tools': '"@Development Tools"', # add quotes
440 'glibc-langpack-en': '', # included in glibc-common
441 'glibc-locale-source': '', # included in glibc-common
442 # update perl core modules on centos
443 # fix: Can't locate Archive/Tar.pm in @INC
446 'rpcsvc-proto-devel': '',
447 'glusterfs-api-devel': '',
448 'glusterfs-devel': '',
449 'libcephfs-devel': '',
450 'gnutls-devel': 'compat-gnutls34-devel',
451 'liburing-devel': '', # not available
452 'python3-setproctitle': 'python36-setproctitle',
453 'tracker-devel': '', # do not install
457 'docker_image': 'centos:8',
458 'vagrant_box': 'centos/8',
459 'bootstrap': CENTOS8_YUM_BOOTSTRAP
,
461 'lsb-release': 'redhat-lsb',
462 '@development-tools': '"@Development Tools"', # add quotes
463 'libsemanage-python': 'python3-libsemanage',
464 'lcov': '', # does not exist
465 'perl-JSON-Parse': '', # does not exist?
466 'perl-Test-Base': 'perl-Test-Simple',
468 'policycoreutils-python': 'python3-policycoreutils',
469 'liburing-devel': '', # not available yet, Add me back, once available!
473 'docker_image': 'fedora:33',
474 'vagrant_box': 'fedora/33-cloud-base',
475 'bootstrap': DNF_BOOTSTRAP
,
477 'lsb-release': 'redhat-lsb',
478 'libsemanage-python': 'python3-libsemanage',
479 'policycoreutils-python': 'python3-policycoreutils',
480 'python3-iso8601': 'python3-dateutil',
484 'docker_image': 'fedora:34',
485 'vagrant_box': 'fedora/34-cloud-base',
486 'bootstrap': DNF_BOOTSTRAP
,
488 'lsb-release': 'redhat-lsb',
489 'libsemanage-python': 'python3-libsemanage',
490 'policycoreutils-python': 'python3-policycoreutils',
492 'python3-iso8601': 'python3-dateutil',
493 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available
497 'docker_image': 'opensuse/leap:15.1',
498 'vagrant_box': 'opensuse/openSUSE-15.1-x86_64',
499 'bootstrap': ZYPPER_BOOTSTRAP
,
501 '@development-tools': '',
502 'dbus-devel': 'dbus-1-devel',
503 'docbook-style-xsl': 'docbook-xsl-stylesheets',
504 'glibc-common': 'glibc-locale',
505 'glibc-locale-source': 'glibc-i18ndata',
506 'glibc-langpack-en': '',
507 'jansson-devel': 'libjansson-devel',
508 'keyutils-libs-devel': 'keyutils-devel',
509 'krb5-workstation': 'krb5-client',
510 'libsemanage-python': 'python2-semanage',
511 'openldap-devel': 'openldap2-devel',
512 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
513 'perl-JSON-Parse': 'perl-JSON-XS',
514 'perl-generators': '',
515 'perl-interpreter': '',
517 'procps-ng': 'procps',
518 'python3-dns': 'python3-dnspython',
519 'python3-markdown': 'python3-Markdown',
521 'glusterfs-api-devel': '',
522 'libtasn1-tools': '', # asn1Parser is part of libtasn1
523 'mingw64-gcc': '', # doesn't exist
524 'liburing-devel': '', # not available
528 'docker_image': 'opensuse/leap:15.2',
529 'vagrant_box': 'opensuse/openSUSE-15.2-x86_64',
530 'bootstrap': ZYPPER_BOOTSTRAP
,
532 '@development-tools': '',
533 'dbus-devel': 'dbus-1-devel',
534 'docbook-style-xsl': 'docbook-xsl-stylesheets',
535 'glibc-common': 'glibc-locale',
536 'glibc-locale-source': 'glibc-i18ndata',
537 'glibc-langpack-en': '',
538 'jansson-devel': 'libjansson-devel',
539 'keyutils-libs-devel': 'keyutils-devel',
540 'krb5-workstation': 'krb5-client',
541 'libsemanage-python': 'python2-semanage',
542 'openldap-devel': 'openldap2-devel',
543 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
544 'perl-JSON-Parse': 'perl-JSON-XS',
545 'perl-generators': '',
546 'perl-interpreter': '',
548 'procps-ng': 'procps',
549 'python3-iso8601': 'python3-python-dateutil',
550 'python3-dns': 'python3-dnspython',
551 'python3-markdown': 'python3-Markdown',
553 'glusterfs-api-devel': '',
554 'libtasn1-tools': '', # asn1Parser is part of libtasn1
563 'bootstrap': APT_BOOTSTRAP
, # family default
571 'bootstrap': YUM_BOOTSTRAP
, # family default
582 def expand_family_dists(family
):
584 for name
, config
in family
['dists'].items():
585 config
= config
.copy()
586 config
['name'] = name
587 config
['home'] = join(OUT
, name
)
588 config
['family'] = family
['name']
589 config
['GENERATED_MARKER'] = GENERATED_MARKER
591 # replace dist specific pkgs
592 replace
= config
.get('replace', {})
594 for pkg
in family
['pkgs']:
595 pkg
= replace
.get(pkg
, pkg
) # replace if exists or get self
600 lines
= [' - {}'.format(pkg
) for pkg
in pkgs
]
601 config
['packages.yml'] = YML_HEADER
.lstrip() + os
.linesep
.join(lines
)
603 sep
= ' \\' + os
.linesep
+ ' '
604 config
['pkgs'] = sep
.join(pkgs
)
606 # get dist bootstrap template or fall back to family default
607 bootstrap_template
= config
.get('bootstrap', family
['bootstrap'])
608 config
['bootstrap.sh'] = bootstrap_template
.format(**config
).strip()
609 config
['locale.sh'] = LOCALE_SETUP
.format(**config
).strip()
611 config
['Dockerfile'] = DOCKERFILE
.format(**config
).strip()
612 # keep the indent, no strip
613 config
['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET
.format(**config
)
619 # expanded config for dists
620 DEB_DISTS_EXP
= expand_family_dists(DEB_FAMILY
)
621 RPM_DISTS_EXP
= expand_family_dists(RPM_FAMILY
)
623 # assemble all together
625 DISTS
.update(DEB_DISTS_EXP
)
626 DISTS
.update(RPM_DISTS_EXP
)
629 def render_vagrantfile(dists
):
631 Render all snippets for each dist into global Vagrantfile.
633 Vagrant supports multiple vms in one Vagrantfile.
634 This make it easier to manage the fleet, e.g:
636 start all: vagrant up
637 start one: vagrant up ubuntu1804
639 All other commands apply to above syntax, e.g.: status, destroy, provision
641 # sort dists by name and put all vagrantfile snippets together
643 dists
[dist
]['vagrantfile_snippet']
644 for dist
in sorted(dists
.keys())]
646 return VAGRANTFILE_GLOBAL
.format(
647 vagrantfile_snippets
=''.join(snippets
),
648 GENERATED_MARKER
=GENERATED_MARKER
652 VAGRANTFILE
= render_vagrantfile(DISTS
)
655 # data we need to expose
656 __all__
= ['DISTS', 'VAGRANTFILE', 'OUT']