tstream_npa: Avoid an unnecessary ZERO_STRUCTP
[Samba.git] / bootstrap / config.py
blob3cb14556130a78465eac7eae29089f8bf176d489
1 #!/usr/bin/env python3
3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 """
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Author: Joe Guo <joeg@catalyst.net.nz>
24 """
25 import os
26 from os.path import abspath, dirname, join
27 HERE = abspath(dirname(__file__))
28 # output dir for rendered files
29 OUT = join(HERE, 'dists')
32 # pkgs with same name in all packaging systems
33 COMMON = [
34 'attr',
35 'autoconf',
36 'binutils',
37 'bison',
38 'ccache',
39 'curl',
40 'gcc',
41 'gdb',
42 'git',
43 'make',
44 'perl',
45 'psmisc', # for pstree in test
46 'sudo', # docker images has no sudo by default
47 'vim',
48 'wget',
52 # define pkgs for all packaging systems in parallel
53 # make it easier to find missing ones
54 # use latest ubuntu and fedora as defaults
55 # deb, rpm, ...
56 PKGS = [
57 # NAME1-dev, NAME2-devel
58 ('lmdb-utils', 'lmdb-devel'),
59 ('nettle-dev', 'nettle-devel'),
60 ('zlib1g-dev', 'zlib-devel'),
61 ('libbsd-dev', 'libbsd-devel'),
62 ('libaio-dev', 'libaio-devel'),
63 ('libarchive-dev', 'libarchive-devel'),
64 ('libblkid-dev', 'libblkid-devel'),
65 ('libxml2-dev', 'libxml2-devel'),
66 ('libcap-dev', 'libpcap-devel'),
67 ('libacl1-dev', 'libacl-devel'),
68 ('libattr1-dev', 'libattr-devel'),
70 # libNAME1-dev, NAME2-devel
71 ('libpopt-dev', 'popt-devel'),
72 ('libreadline-dev', 'readline-devel'),
73 ('libjansson-dev', 'jansson-devel'),
74 ('liblmdb-dev', 'lmdb-devel'),
75 ('libncurses5-dev', 'ncurses-devel'),
76 # NOTE: Debian 7+ or Ubuntu 16.04+
77 ('libsystemd-dev', 'systemd-devel'),
78 ('libkrb5-dev', 'krb5-devel'),
79 ('libldap2-dev', 'openldap-devel'),
80 ('libcups2-dev', 'cups-devel'),
81 ('libpam0g-dev', 'pam-devel'),
82 ('libgpgme11-dev', 'gpgme-devel'),
83 # NOTE: Debian 8+ and Ubuntu 14.04+
84 ('libgnutls28-dev', 'gnutls-devel'),
85 ('libdbus-1-dev', 'dbus-devel'),
87 # NAME1, NAME2
88 # for debian, locales provide locale support with language packs
89 # ubuntu split language packs to language-pack-xx
90 # for centos, glibc-common provide locale support with language packs
91 # fedora split language packs to glibc-langpack-xx
92 ('locales', 'glibc-common'), # required for locale
93 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
94 ('', 'glibc-locale-source'), # for localedef
95 ('bind9', 'bind'),
96 ('bind9utils', 'bind-utils'),
97 ('dnsutils', ''),
98 ('locate', 'mlocate'),
99 ('xsltproc', 'libxslt'),
100 ('krb5-kdc', 'krb5-workstation'),
101 ('apt-utils', 'yum-utils'),
102 ('pkg-config', 'pkgconfig'),
103 ('procps', 'procps-ng'), # required for the free cmd in tests
104 ('lsb-core', 'redhat-lsb'), # we need lsb_relase to show info
105 ('', 'rpcgen'), # required for test
106 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
107 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
108 ('', 'libnsl2-devel'), # for <rpcsvc/yp_prot.h> header on fedora
110 # python
111 ('python-dev', 'python-devel'),
112 ('python-gpg', 'python2-gpg'), # defaults to ubuntu/fedora latest
113 ('python-crypto', 'python-crypto'),
114 ('python-markdown', 'python-markdown'),
115 ('python-dnspython', 'python-dns'),
117 ('python3-dev', 'python3-devel'),
118 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
119 ('python3-crypto', 'python3-crypto'),
120 ('python3-markdown', 'python3-markdown'),
121 ('python3-dnspython', 'python3-dns'),
123 ('', 'libsemanage-python'),
124 ('', 'policycoreutils-python'),
126 # perl
127 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
128 # not strict equivalents
129 ('perl-modules', 'perl-ExtUtils-MakeMaker'),
130 ('libjson-perl', 'perl-Test-Base'),
132 # misc
133 # @ means group for rpm, use fedora as rpm default
134 ('build-essential', '@development-tools'),
135 ('debhelper', ''),
136 # rpm has no pkg for docbook-xml
137 ('docbook-xml', 'docbook-dtds'),
138 ('docbook-xsl', 'docbook-style-xsl'),
139 ('flex', ''),
140 ('', 'keyutils-libs-devel'),
145 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
146 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
149 APT_BOOTSTRAP = r"""
150 #!/bin/bash
151 set -xueo pipefail
153 export DEBIAN_FRONTEND=noninteractive
154 apt-get -y update
156 apt-get -y install \
157 {pkgs}
159 apt-get -y autoremove
160 apt-get -y autoclean
161 apt-get -y clean
163 # uncomment locale
164 # this file doesn't exist on ubuntu1404 even locales installed
165 if [ -f /etc/locale.gen ]; then
166 sed -i '/^#\s*en_US.UTF-8 UTF-8/s/^#\s*//' /etc/locale.gen
169 locale-gen
171 # update /etc/default/locale
172 update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
174 # set both for safe
175 echo LC_ALL="en_US.UTF-8" >> /etc/environment
176 echo LANG="en_US.UTF-8" >> /etc/environment
180 YUM_BOOTSTRAP = r"""
181 #!/bin/bash
182 set -xueo pipefail
184 yum -y -q update
185 yum -y -q install epel-release
186 yum -y -q update
188 yum -y -q --verbose install \
189 {pkgs}
191 yum clean all
193 # gen locale
194 localedef -c -i en_US -f UTF-8 en_US.UTF-8
196 # no update-locale, diy
197 # LC_ALL is not valid in this file
198 echo LANG="en_US.UTF-8" > /etc/locale.conf
200 # set both for safe
201 echo LC_ALL="en_US.UTF-8" >> /etc/environment
202 echo LANG="en_US.UTF-8" >> /etc/environment
206 DNF_BOOTSTRAP = r"""
207 #!/bin/bash
208 set -xueo pipefail
210 dnf -y -q update
212 dnf -y -q --verbose install \
213 {pkgs}
215 dnf clean all
217 # gen locale
218 localedef -c -i en_US -f UTF-8 en_US.UTF-8
220 # no update-locale, diy
221 # LC_ALL is not valid in this file
222 echo LANG="en_US.UTF-8" > /etc/locale.conf
224 # set both for safe
225 echo LC_ALL="en_US.UTF-8" >> /etc/environment
226 echo LANG="en_US.UTF-8" >> /etc/environment
230 DOCKERFILE = r"""
231 FROM {docker_image}
233 # we will use this image to run ci, these ENV vars are important
234 ENV CC="ccache gcc"
236 ADD bootstrap.sh /tmp/bootstrap.sh
237 # need root permission, do it before USER samba
238 RUN bash /tmp/bootstrap.sh
240 # make test can not work with root, so we have to create a new user
241 RUN useradd -m -s /bin/bash samba && \
242 mkdir -p /etc/sudoers.d && \
243 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
245 USER samba
246 WORKDIR /home/samba
247 # samba tests rely on this
248 ENV USER=samba LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
251 # Vagrantfile snippet for each dist
252 VAGRANTFILE_SNIPPET = r"""
253 config.vm.define "{name}" do |v|
254 v.vm.box = "{vagrant_box}"
255 v.vm.hostname = "{name}"
256 v.vm.provision :shell, path: "{name}/bootstrap.sh"
260 # global Vagrantfile with snippets for all dists
261 VAGRANTFILE_GLOBAL = r"""
262 Vagrant.configure("2") do |config|
263 config.ssh.insert_key = false
265 {vagrantfile_snippets}
271 DEB_DISTS = {
272 'debian7': {
273 'docker_image': 'debian:7',
274 'vagrant_box': 'debian/wheezy64',
275 'replace': {
276 'libgnutls28-dev': 'libgnutls-dev',
277 'libsystemd-dev': '', # not available, remove
278 'lmdb-utils': '', # not available, remove
279 'liblmdb-dev': '', # not available, remove
280 'python-gpg': 'python-gpgme',
281 'python3-gpg': '', # no python3 gpg pkg available, remove
282 'language-pack-en': '', # included in locales
285 'debian8': {
286 'docker_image': 'debian:8',
287 'vagrant_box': 'debian/jessie64',
288 'replace': {
289 'python-gpg': 'python-gpgme',
290 'python3-gpg': 'python3-gpgme',
291 'language-pack-en': '', # included in locales
294 'debian9': {
295 'docker_image': 'debian:9',
296 'vagrant_box': 'debian/stretch64',
297 'replace': {
298 'language-pack-en': '', # included in locales
301 'ubuntu1404': {
302 'docker_image': 'ubuntu:14.04',
303 'vagrant_box': 'ubuntu/trusty64',
304 'replace': {
305 'libsystemd-dev': '', # remove
306 'libgnutls28-dev': 'libgnutls-dev',
307 'python-gpg': 'python-gpgme',
308 'python3-gpg': 'python3-gpgme',
309 'lmdb-utils': 'lmdb-utils/trusty-backports',
310 'liblmdb-dev': 'liblmdb-dev/trusty-backports',
313 'ubuntu1604': {
314 'docker_image': 'ubuntu:16.04',
315 'vagrant_box': 'ubuntu/xenial64',
316 'replace': {
317 'python-gpg': 'python-gpgme',
318 'python3-gpg': 'python3-gpgme',
321 'ubuntu1804': {
322 'docker_image': 'ubuntu:18.04',
323 'vagrant_box': 'ubuntu/bionic64',
328 RPM_DISTS = {
329 'centos6': {
330 'docker_image': 'centos:6',
331 'vagrant_box': 'centos/6',
332 'bootstrap': YUM_BOOTSTRAP,
333 'replace': {
334 'python3-devel': 'python34-devel',
335 'python2-gpg': 'pygpgme',
336 'python3-gpg': '', # no python3-gpg yet
337 '@development-tools': '"@Development Tools"', # add quotes
338 'glibc-langpack-en': '', # included in glibc-common
339 'glibc-locale-source': '', # included in glibc-common
340 'procps-ng': 'procps', # centos6 still use old name
341 # update perl core modules on centos
342 # fix: Can't locate Archive/Tar.pm in @INC
343 'perl': 'perl-core',
346 'centos7': {
347 'docker_image': 'centos:7',
348 'vagrant_box': 'centos/7',
349 'bootstrap': YUM_BOOTSTRAP,
350 'replace': {
351 'python3-devel': 'python34-devel',
352 # although python36-devel is available
353 # after epel-release installed
354 # however, all other python3 pkgs are still python34-ish
355 'python2-gpg': 'pygpgme',
356 'python3-gpg': '', # no python3-gpg yet
357 '@development-tools': '"@Development Tools"', # add quotes
358 'glibc-langpack-en': '', # included in glibc-common
359 'glibc-locale-source': '', # included in glibc-common
360 # update perl core modules on centos
361 # fix: Can't locate Archive/Tar.pm in @INC
362 'perl': 'perl-core',
365 'fedora28': {
366 'docker_image': 'fedora:28',
367 'vagrant_box': 'fedora/28-cloud-base',
368 'bootstrap': DNF_BOOTSTRAP,
370 'fedora29': {
371 'docker_image': 'fedora:29',
372 'vagrant_box': 'fedora/29-cloud-base',
373 'bootstrap': DNF_BOOTSTRAP,
378 DEB_FAMILY = {
379 'name': 'deb',
380 'pkgs': DEB_PKGS,
381 'bootstrap': APT_BOOTSTRAP, # family default
382 'dists': DEB_DISTS,
386 RPM_FAMILY = {
387 'name': 'rpm',
388 'pkgs': RPM_PKGS,
389 'bootstrap': YUM_BOOTSTRAP, # family default
390 'dists': RPM_DISTS,
394 YML_HEADER = r"""
396 packages:
400 def expand_family_dists(family):
401 dists = {}
402 for name, config in family['dists'].items():
403 config = config.copy()
404 config['name'] = name
405 config['home'] = join(OUT, name)
406 config['family'] = family['name']
408 # replace dist specific pkgs
409 replace = config.get('replace', {})
410 pkgs = []
411 for pkg in family['pkgs']:
412 pkg = replace.get(pkg, pkg) # replace if exists or get self
413 if pkg:
414 pkgs.append(pkg)
415 pkgs.sort()
417 lines = [' - {}'.format(pkg) for pkg in pkgs]
418 config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
420 sep = ' \\' + os.linesep + ' '
421 config['pkgs'] = sep.join(pkgs)
423 # get dist bootstrap template or fall back to family default
424 bootstrap_template = config.get('bootstrap', family['bootstrap'])
425 config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
427 config['Dockerfile'] = DOCKERFILE.format(**config).strip()
428 # keep the indent, no strip
429 config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
431 dists[name] = config
432 return dists
435 # expanded config for dists
436 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
437 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
439 # assemble all together
440 DISTS = {}
441 DISTS.update(DEB_DISTS_EXP)
442 DISTS.update(RPM_DISTS_EXP)
445 def render_vagrantfile(dists):
447 Render all snippets for each dist into global Vagrantfile.
449 Vagrant supports multiple vms in one Vagrantfile.
450 This make it easier to manage the fleet, e.g:
452 start all: vagrant up
453 start one: vagrant up ubuntu1804
455 All other commands apply to above syntax, e.g.: status, destroy, provision
457 # sort dists by name and put all vagrantfile snippets together
458 snippets = [
459 dists[dist]['vagrantfile_snippet']
460 for dist in sorted(dists.keys())]
462 return VAGRANTFILE_GLOBAL.format(vagrantfile_snippets=''.join(snippets))
465 VAGRANTFILE = render_vagrantfile(DISTS)
468 # data we need to expose
469 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']