CVE-2020-10730: dsdb: Ban the combination of paged_results and VLV
[Samba.git] / bootstrap / config.py
blob24f21a3c7490b1b6e4d698a96256050acf568bdb
1 #!/usr/bin/env python3
3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 """
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Author: Joe Guo <joeg@catalyst.net.nz>
24 """
25 import os
26 from os.path import abspath, dirname, join
27 HERE = abspath(dirname(__file__))
28 # output dir for rendered files
29 OUT = join(HERE, 'generated-dists')
32 # pkgs with same name in all packaging systems
33 COMMON = [
34 'acl',
35 'attr',
36 'autoconf',
37 'binutils',
38 'bison',
39 'ccache',
40 'curl',
41 'chrpath',
42 'flex',
43 'gcc',
44 'gdb',
45 'git',
46 'gzip',
47 'hostname',
48 'htop',
49 'lcov',
50 'make',
51 'patch',
52 'perl',
53 'psmisc', # for pstree in test
54 'rng-tools',
55 'rsync',
56 'sed',
57 'sudo', # docker images has no sudo by default
58 'tar',
59 'tree',
60 'wget',
64 # define pkgs for all packaging systems in parallel
65 # make it easier to find missing ones
66 # use latest ubuntu and fedora as defaults
67 # deb, rpm, ...
68 PKGS = [
69 # NAME1-dev, NAME2-devel
70 ('lmdb-utils', 'lmdb'),
71 ('mingw-w64', 'mingw64-gcc'),
72 ('zlib1g-dev', 'zlib-devel'),
73 ('libbsd-dev', 'libbsd-devel'),
74 ('liburing-dev', 'liburing-devel'),
75 ('libarchive-dev', 'libarchive-devel'),
76 ('libblkid-dev', 'libblkid-devel'),
77 ('libcap-dev', 'libcap-devel'),
78 ('libacl1-dev', 'libacl-devel'),
79 ('libattr1-dev', 'libattr-devel'),
81 # libNAME1-dev, NAME2-devel
82 ('libpopt-dev', 'popt-devel'),
83 ('libreadline-dev', 'readline-devel'),
84 ('libjansson-dev', 'jansson-devel'),
85 ('liblmdb-dev', 'lmdb-devel'),
86 ('libncurses5-dev', 'ncurses-devel'),
87 # NOTE: Debian 7+ or Ubuntu 16.04+
88 ('libsystemd-dev', 'systemd-devel'),
89 ('libkrb5-dev', 'krb5-devel'),
90 ('libldap2-dev', 'openldap-devel'),
91 ('libcups2-dev', 'cups-devel'),
92 ('libpam0g-dev', 'pam-devel'),
93 ('libgpgme11-dev', 'gpgme-devel'),
94 # NOTE: Debian 8+ and Ubuntu 14.04+
95 ('libgnutls28-dev', 'gnutls-devel'),
96 ('libtasn1-bin', 'libtasn1-tools'),
97 ('libtasn1-dev', 'libtasn1-devel'),
98 ('', 'quota-devel'),
99 ('uuid-dev', 'libuuid-devel'),
100 ('libjs-jquery', ''),
101 ('libavahi-common-dev', 'avahi-devel'),
102 ('libdbus-1-dev', 'dbus-devel'),
103 ('libpcap-dev', 'libpcap-devel'),
104 ('libunwind-dev', 'libunwind-devel'), # for back trace
105 ('libglib2.0-dev', 'glib2-devel'),
106 ('libicu-dev', 'libicu-devel'),
107 ('heimdal-multidev', ''),
109 # NAME1, NAME2
110 # for debian, locales provide locale support with language packs
111 # ubuntu split language packs to language-pack-xx
112 # for centos, glibc-common provide locale support with language packs
113 # fedora split language packs to glibc-langpack-xx
114 ('locales', 'glibc-common'), # required for locale
115 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
116 ('bind9utils', 'bind-utils'),
117 ('dnsutils', ''),
118 ('xsltproc', 'libxslt'),
119 ('krb5-user', ''),
120 ('krb5-config', ''),
121 ('krb5-kdc', 'krb5-server'),
122 ('apt-utils', 'yum-utils'),
123 ('pkg-config', 'pkgconfig'),
124 ('procps', 'procps-ng'), # required for the free cmd in tests
125 ('lsb-release', 'lsb-release'), # we need lsb_relase to show info
126 ('', 'rpcgen'), # required for test
127 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
128 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
129 ('', 'libnsl2-devel'), # for <rpcsvc/yp_prot.h> header on fedora
130 ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
131 ('mawk', 'gawk'),
133 ('python3', 'python3'),
134 ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
135 ('python3-dev', 'python3-devel'),
136 ('python3-dbg', ''),
137 ('python3-iso8601', ''),
138 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
139 ('python3-markdown', 'python3-markdown'),
140 ('python3-matplotlib', ''),
141 ('python3-dnspython', 'python3-dns'),
142 ('python3-pexpect', ''), # for wintest only
143 ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
145 ('', 'libsemanage-python'),
146 ('', 'policycoreutils-python'),
148 # perl
149 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
150 ('libjson-perl', 'perl-JSON-Parse'),
151 ('perl-modules', ''),
152 ('', 'perl-Archive-Tar'),
153 ('', 'perl-ExtUtils-MakeMaker'),
154 ('', 'perl-Test-Base'),
155 ('', 'perl-generators'),
156 ('', 'perl-interpreter'),
158 # fs
159 ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
160 ('', 'glusterfs-api-devel'),
161 ('glusterfs-common', 'glusterfs-devel'),
162 ('libcephfs-dev', 'libcephfs-devel'),
164 # misc
165 # @ means group for rpm, use fedora as rpm default
166 ('build-essential', '@development-tools'),
167 ('debhelper', ''),
168 # rpm has no pkg for docbook-xml
169 ('docbook-xml', 'docbook-dtds'),
170 ('docbook-xsl', 'docbook-style-xsl'),
171 ('', 'keyutils-libs-devel'),
172 ('', 'which'),
176 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
177 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
179 GENERATED_MARKER = r"""
181 # This file is generated by 'bootstrap/template.py --render'
182 # See also bootstrap/config.py
187 APT_BOOTSTRAP = r"""
188 #!/bin/bash
189 {GENERATED_MARKER}
190 set -xueo pipefail
192 export DEBIAN_FRONTEND=noninteractive
193 apt-get -y update
195 apt-get -y install \
196 {pkgs}
198 apt-get -y autoremove
199 apt-get -y autoclean
200 apt-get -y clean
204 YUM_BOOTSTRAP = r"""
205 #!/bin/bash
206 {GENERATED_MARKER}
207 set -xueo pipefail
209 yum update -y
210 yum install -y epel-release
211 yum install -y yum-plugin-copr
212 yum copr enable -y sergiomb/SambaAD
213 yum update -y
215 yum install -y \
216 {pkgs}
218 yum clean all
220 if [ ! -f /usr/bin/python3 ]; then
221 ln -sf /usr/bin/python3.6 /usr/bin/python3
225 CENTOS8_YUM_BOOTSTRAP = r"""
226 #!/bin/bash
227 {GENERATED_MARKER}
228 set -xueo pipefail
230 yum update -y
231 yum install -y dnf-plugins-core
232 yum install -y epel-release
234 yum -v repolist all
235 yum config-manager --set-enabled PowerTools -y
236 yum config-manager --set-enabled Devel -y
237 yum update -y
239 yum install -y \
240 --setopt=install_weak_deps=False \
241 {pkgs}
243 yum clean all
246 DNF_BOOTSTRAP = r"""
247 #!/bin/bash
248 {GENERATED_MARKER}
249 set -xueo pipefail
251 dnf update -y
253 dnf install -y \
254 --setopt=install_weak_deps=False \
255 {pkgs}
257 dnf clean all
260 ZYPPER_BOOTSTRAP = r"""
261 #!/bin/bash
262 {GENERATED_MARKER}
263 set -xueo pipefail
265 zypper --non-interactive refresh
266 zypper --non-interactive update
267 zypper --non-interactive install \
268 --no-recommends \
269 system-user-nobody \
270 {pkgs}
272 zypper --non-interactive clean
274 if [ -f /usr/lib/mit/bin/krb5-config ]; then
275 ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
279 # A generic shell script to setup locale
280 LOCALE_SETUP = r"""
281 #!/bin/bash
282 {GENERATED_MARKER}
283 set -xueo pipefail
285 # refer to /usr/share/i18n/locales
286 INPUTFILE=en_US
287 # refer to /usr/share/i18n/charmaps
288 CHARMAP=UTF-8
289 # locale to generate in /usr/lib/locale
290 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
291 LOCALE=$INPUTFILE.utf8
293 # if locale is already correct, exit
294 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
296 # if locale not available, generate locale into /usr/lib/locale
297 if ! ( locale --all-locales | grep -i $LOCALE )
298 then
299 # no-archive means create its own dir
300 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
303 # update locale conf and global env file
304 # set both LC_ALL and LANG for safe
306 # update conf for Debian family
307 FILE=/etc/default/locale
308 if [ -f $FILE ]
309 then
310 echo LC_ALL="$LOCALE" > $FILE
311 echo LANG="$LOCALE" >> $FILE
314 # update conf for RedHat family
315 FILE=/etc/locale.conf
316 if [ -f $FILE ]
317 then
318 # LC_ALL is not valid in this file, set LANG only
319 echo LANG="$LOCALE" > $FILE
322 # update global env file
323 FILE=/etc/environment
324 if [ -f $FILE ]
325 then
326 # append LC_ALL if not exist
327 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
328 # append LANG if not exist
329 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
334 DOCKERFILE = r"""
335 {GENERATED_MARKER}
336 FROM {docker_image}
338 # pass in with --build-arg while build
339 ARG SHA1SUM
340 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
342 ADD *.sh /tmp/
343 # need root permission, do it before USER samba
344 RUN /tmp/bootstrap.sh && /tmp/locale.sh
346 # if ld.gold exists, force link it to ld
347 RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"
349 # make test can not work with root, so we have to create a new user
350 RUN useradd -m -U -s /bin/bash samba && \
351 mkdir -p /etc/sudoers.d && \
352 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
354 USER samba
355 WORKDIR /home/samba
356 # samba tests rely on this
357 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
360 # Vagrantfile snippet for each dist
361 VAGRANTFILE_SNIPPET = r"""
362 config.vm.define "{name}" do |v|
363 v.vm.box = "{vagrant_box}"
364 v.vm.hostname = "{name}"
365 v.vm.provision :shell, path: "{name}/bootstrap.sh"
366 v.vm.provision :shell, path: "{name}/locale.sh"
370 # global Vagrantfile with snippets for all dists
371 VAGRANTFILE_GLOBAL = r"""
372 {GENERATED_MARKER}
374 Vagrant.configure("2") do |config|
375 config.ssh.insert_key = false
377 {vagrantfile_snippets}
383 DEB_DISTS = {
384 'debian10': {
385 'docker_image': 'debian:10',
386 'vagrant_box': 'debian/buster64',
387 'replace': {
388 'language-pack-en': '', # included in locales
389 'liburing-dev': '', # not available
392 'ubuntu1604': {
393 'docker_image': 'ubuntu:16.04',
394 'vagrant_box': 'ubuntu/xenial64',
395 'replace': {
396 'python-gpg': 'python-gpgme',
397 'python3-gpg': 'python3-gpgme',
398 'glusterfs-common': '',
399 'libcephfs-dev': '',
400 'liburing-dev': '', # not available
403 'ubuntu1804': {
404 'docker_image': 'ubuntu:18.04',
405 'vagrant_box': 'ubuntu/bionic64',
406 'replace': {
407 'liburing-dev': '', # not available
410 'ubuntu2004': {
411 'docker_image': 'ubuntu:20.04',
412 'vagrant_box': 'ubuntu/focal64',
413 'replace': {
414 'liburing-dev': '', # not available
420 RPM_DISTS = {
421 'centos7': {
422 'docker_image': 'centos:7',
423 'vagrant_box': 'centos/7',
424 'bootstrap': YUM_BOOTSTRAP,
425 'replace': {
426 'lsb-release': 'redhat-lsb',
427 'python3': 'python36',
428 'python3-cryptography': 'python36-cryptography',
429 'python3-devel': 'python36-devel',
430 'python3-dns': 'python36-dns',
431 'python3-pyasn1': 'python36-pyasn1',
432 'python3-gpg': 'python36-gpg',
433 'python3-iso8601' : 'python36-iso8601',
434 'python3-markdown': 'python36-markdown',
435 # although python36-devel is available
436 # after epel-release installed
437 # however, all other python3 pkgs are still python36-ish
438 'python2-gpg': 'pygpgme',
439 'python3-gpg': '', # no python3-gpg yet
440 '@development-tools': '"@Development Tools"', # add quotes
441 'glibc-langpack-en': '', # included in glibc-common
442 'glibc-locale-source': '', # included in glibc-common
443 # update perl core modules on centos
444 # fix: Can't locate Archive/Tar.pm in @INC
445 'perl': 'perl-core',
446 'rpcsvc-proto-devel': '',
447 'glusterfs-api-devel': '',
448 'glusterfs-devel': '',
449 'libcephfs-devel': '',
450 'gnutls-devel': 'compat-gnutls34-devel',
451 'liburing-devel': '', # not available
454 'centos8': {
455 'docker_image': 'centos:8',
456 'vagrant_box': 'centos/8',
457 'bootstrap': CENTOS8_YUM_BOOTSTRAP,
458 'replace': {
459 'lsb-release': 'redhat-lsb',
460 '@development-tools': '"@Development Tools"', # add quotes
461 'libsemanage-python': 'python3-libsemanage',
462 'lcov': '', # does not exist
463 'perl-JSON-Parse': '', # does not exist?
464 'perl-Test-Base': 'perl-Test-Simple',
465 'policycoreutils-python': 'python3-policycoreutils',
466 'liburing-devel': '', # not available yet, Add me back, once available!
469 'fedora31': {
470 'docker_image': 'fedora:31',
471 'vagrant_box': 'fedora/31-cloud-base',
472 'bootstrap': DNF_BOOTSTRAP,
473 'replace': {
474 'lsb-release': 'redhat-lsb',
475 'libsemanage-python': 'python3-libsemanage',
476 'policycoreutils-python': 'python3-policycoreutils',
479 'fedora32': {
480 'docker_image': 'fedora:32',
481 'vagrant_box': 'fedora/32-cloud-base',
482 'bootstrap': DNF_BOOTSTRAP,
483 'replace': {
484 'lsb-release': 'redhat-lsb',
485 'libsemanage-python': 'python3-libsemanage',
486 'policycoreutils-python': 'python3-policycoreutils',
489 'opensuse150': {
490 'docker_image': 'opensuse/leap:15.0',
491 'vagrant_box': 'opensuse/openSUSE-15.0-x86_64',
492 'bootstrap': ZYPPER_BOOTSTRAP,
493 'replace': {
494 '@development-tools': '',
495 'dbus-devel': 'dbus-1-devel',
496 'docbook-style-xsl': 'docbook-xsl-stylesheets',
497 'glibc-common': 'glibc-locale',
498 'glibc-locale-source': 'glibc-i18ndata',
499 'glibc-langpack-en': '',
500 'jansson-devel': 'libjansson-devel',
501 'keyutils-libs-devel': 'keyutils-devel',
502 'krb5-workstation': 'krb5-client',
503 'libnsl2-devel': 'libnsl-devel',
504 'libsemanage-python': 'python2-semanage',
505 'openldap-devel': 'openldap2-devel',
506 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
507 'perl-JSON-Parse': 'perl-JSON-XS',
508 'perl-generators': '',
509 'perl-interpreter': '',
510 'procps-ng': 'procps',
511 'python-dns': 'python2-dnspython',
512 'python3-dns': 'python3-dnspython',
513 'python3-markdown': 'python3-Markdown',
514 'quota-devel': '',
515 'glusterfs-api-devel': '',
516 'libtasn1-tools': '', # asn1Parser is part of libtasn1
517 'mingw64-gcc': '', # doesn't exist
518 'liburing-devel': '', # not available
521 'opensuse151': {
522 'docker_image': 'opensuse/leap:15.1',
523 'vagrant_box': 'opensuse/openSUSE-15.1-x86_64',
524 'bootstrap': ZYPPER_BOOTSTRAP,
525 'replace': {
526 '@development-tools': '',
527 'dbus-devel': 'dbus-1-devel',
528 'docbook-style-xsl': 'docbook-xsl-stylesheets',
529 'glibc-common': 'glibc-locale',
530 'glibc-locale-source': 'glibc-i18ndata',
531 'glibc-langpack-en': '',
532 'jansson-devel': 'libjansson-devel',
533 'keyutils-libs-devel': 'keyutils-devel',
534 'krb5-workstation': 'krb5-client',
535 'libnsl2-devel': 'libnsl-devel',
536 'libsemanage-python': 'python2-semanage',
537 'openldap-devel': 'openldap2-devel',
538 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
539 'perl-JSON-Parse': 'perl-JSON-XS',
540 'perl-generators': '',
541 'perl-interpreter': '',
542 'procps-ng': 'procps',
543 'python-dns': 'python2-dnspython',
544 'python3-dns': 'python3-dnspython',
545 'python3-markdown': 'python3-Markdown',
546 'quota-devel': '',
547 'glusterfs-api-devel': '',
548 'libtasn1-tools': '', # asn1Parser is part of libtasn1
549 'mingw64-gcc': '', # doesn't exist
550 'liburing-devel': '', # not available, will be added in 15.2
556 DEB_FAMILY = {
557 'name': 'deb',
558 'pkgs': DEB_PKGS,
559 'bootstrap': APT_BOOTSTRAP, # family default
560 'dists': DEB_DISTS,
564 RPM_FAMILY = {
565 'name': 'rpm',
566 'pkgs': RPM_PKGS,
567 'bootstrap': YUM_BOOTSTRAP, # family default
568 'dists': RPM_DISTS,
572 YML_HEADER = r"""
574 packages:
578 def expand_family_dists(family):
579 dists = {}
580 for name, config in family['dists'].items():
581 config = config.copy()
582 config['name'] = name
583 config['home'] = join(OUT, name)
584 config['family'] = family['name']
585 config['GENERATED_MARKER'] = GENERATED_MARKER
587 # replace dist specific pkgs
588 replace = config.get('replace', {})
589 pkgs = []
590 for pkg in family['pkgs']:
591 pkg = replace.get(pkg, pkg) # replace if exists or get self
592 if pkg:
593 pkgs.append(pkg)
594 pkgs.sort()
596 lines = [' - {}'.format(pkg) for pkg in pkgs]
597 config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
599 sep = ' \\' + os.linesep + ' '
600 config['pkgs'] = sep.join(pkgs)
602 # get dist bootstrap template or fall back to family default
603 bootstrap_template = config.get('bootstrap', family['bootstrap'])
604 config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
605 config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
607 config['Dockerfile'] = DOCKERFILE.format(**config).strip()
608 # keep the indent, no strip
609 config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
611 dists[name] = config
612 return dists
615 # expanded config for dists
616 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
617 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
619 # assemble all together
620 DISTS = {}
621 DISTS.update(DEB_DISTS_EXP)
622 DISTS.update(RPM_DISTS_EXP)
625 def render_vagrantfile(dists):
627 Render all snippets for each dist into global Vagrantfile.
629 Vagrant supports multiple vms in one Vagrantfile.
630 This make it easier to manage the fleet, e.g:
632 start all: vagrant up
633 start one: vagrant up ubuntu1804
635 All other commands apply to above syntax, e.g.: status, destroy, provision
637 # sort dists by name and put all vagrantfile snippets together
638 snippets = [
639 dists[dist]['vagrantfile_snippet']
640 for dist in sorted(dists.keys())]
642 return VAGRANTFILE_GLOBAL.format(
643 vagrantfile_snippets=''.join(snippets),
644 GENERATED_MARKER=GENERATED_MARKER
648 VAGRANTFILE = render_vagrantfile(DISTS)
651 # data we need to expose
652 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']