bootstrap/config.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) Catalyst.Net Ltd 2019
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """
  19 Manage dependencies and bootstrap environments for Samba.
  20
  21 Config file for packages and templates.
  22
  23 Author: Joe Guo <joeg@catalyst.net.nz>
  24 """
  25 import os
  26 from os.path import abspath, dirname, join
  27 HERE = abspath(dirname(__file__))
  28 # output dir for rendered files
  29 OUT = join(HERE, 'dists')
  30
  31
  32 # pkgs with same name in all packaging systems
  33 COMMON = [
  34     'attr',
  35     'autoconf',
  36     'binutils',
  37     'bison',
  38     'ccache',
  39     'curl',
  40     'gcc',
  41     'gdb',
  42     'git',
  43     'make',
  44     'perl',
  45     'psmisc',  # for pstree in test
  46     'sudo',  # docker images has no sudo by default
  47     'vim',
  48     'wget',
  49 ]
  50
  51
  52 # define pkgs for all packaging systems in parallel
  53 # make it easier to find missing ones
  54 # use latest ubuntu and fedora as defaults
  55 # deb, rpm, ...
  56 PKGS = [
  57     # NAME1-dev, NAME2-devel
  58     ('lmdb-utils', 'lmdb-devel'),
  59     ('nettle-dev', 'nettle-devel'),
  60     ('zlib1g-dev', 'zlib-devel'),
  61     ('libbsd-dev', 'libbsd-devel'),
  62     ('libaio-dev', 'libaio-devel'),
  63     ('libarchive-dev', 'libarchive-devel'),
  64     ('libblkid-dev', 'libblkid-devel'),
  65     ('libxml2-dev', 'libxml2-devel'),
  66     ('libcap-dev', 'libpcap-devel'),
  67     ('libacl1-dev', 'libacl-devel'),
  68     ('libattr1-dev', 'libattr-devel'),
  69
  70     # libNAME1-dev, NAME2-devel
  71     ('libpopt-dev', 'popt-devel'),
  72     ('libreadline-dev', 'readline-devel'),
  73     ('libjansson-dev', 'jansson-devel'),
  74     ('liblmdb-dev', 'lmdb-devel'),
  75     ('libncurses5-dev', 'ncurses-devel'),
  76     # NOTE: Debian 7+ or Ubuntu 16.04+
  77     ('libsystemd-dev', 'systemd-devel'),
  78     ('libkrb5-dev', 'krb5-devel'),
  79     ('libldap2-dev', 'openldap-devel'),
  80     ('libcups2-dev', 'cups-devel'),
  81     ('libpam0g-dev', 'pam-devel'),
  82     ('libgpgme11-dev', 'gpgme-devel'),
  83     # NOTE: Debian 8+ and Ubuntu 14.04+
  84     ('libgnutls28-dev', 'gnutls-devel'),
  85     ('libdbus-1-dev', 'dbus-devel'),
  86
  87     # NAME1, NAME2
  88     # for debian, locales provide locale support with language packs
  89     # ubuntu split language packs to language-pack-xx
  90     # for centos, glibc-common provide locale support with language packs
  91     # fedora split language packs  to glibc-langpack-xx
  92     ('locales', 'glibc-common'),  # required for locale
  93     ('language-pack-en', 'glibc-langpack-en'),  # we need en_US.UTF-8
  94     ('', 'glibc-locale-source'),  # for localedef
  95     ('bind9', 'bind'),
  96     ('bind9utils', 'bind-utils'),
  97     ('dnsutils', ''),
  98     ('locate', 'mlocate'),
  99     ('xsltproc', 'libxslt'),
 100     ('krb5-kdc', 'krb5-workstation'),
 101     ('apt-utils', 'yum-utils'),
 102     ('pkg-config', 'pkgconfig'),
 103     ('procps', 'procps-ng'),  # required for the free cmd in tests
 104     ('lsb-core', 'redhat-lsb'),  # we need lsb_relase to show info
 105     ('', 'rpcgen'),  # required for test
 106     # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
 107     ('', 'libtirpc-devel'),  # for <rpc/rpc.h> header on fedora
 108     ('', 'libnsl2-devel'),  # for <rpcsvc/yp_prot.h> header on fedora
 109
 110     # python
 111     ('python-dev', 'python-devel'),
 112     ('python-gpg', 'python2-gpg'),  # defaults to ubuntu/fedora latest
 113     ('python-crypto', 'python-crypto'),
 114     ('python-markdown', 'python-markdown'),
 115     ('python-dnspython', 'python-dns'),
 116
 117     ('python3-dev', 'python3-devel'),
 118     ('python3-gpg', 'python3-gpg'),  # defaults to ubuntu/fedora latest
 119     ('python3-crypto', 'python3-crypto'),
 120     ('python3-markdown', 'python3-markdown'),
 121     ('python3-dnspython', 'python3-dns'),
 122
 123     ('', 'libsemanage-python'),
 124     ('', 'policycoreutils-python'),
 125
 126     # perl
 127     ('libparse-yapp-perl', 'perl-Parse-Yapp'),
 128     # not strict equivalents
 129     ('perl-modules', 'perl-ExtUtils-MakeMaker'),
 130     ('libjson-perl', 'perl-Test-Base'),
 131
 132     # misc
 133     # @ means group for rpm, use fedora as rpm default
 134     ('build-essential', '@development-tools'),
 135     ('debhelper', ''),
 136     # rpm has no pkg for docbook-xml
 137     ('docbook-xml', 'docbook-dtds'),
 138     ('docbook-xsl', 'docbook-style-xsl'),
 139     ('flex', ''),
 140     ('', 'keyutils-libs-devel'),
 141
 142 ]
 143
 144
 145 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
 146 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
 147
 148
 149 APT_BOOTSTRAP = r"""
 150 #!/bin/bash
 151 set -xueo pipefail
 152
 153 export DEBIAN_FRONTEND=noninteractive
 154 apt-get -y update
 155
 156 apt-get -y install \
 157     {pkgs}
 158
 159 apt-get -y autoremove
 160 apt-get -y autoclean
 161 apt-get -y clean
 162
 163 # uncomment locale
 164 # this file doesn't exist on ubuntu1404 even locales installed
 165 if [ -f /etc/locale.gen ]; then
 166     sed -i '/^#\s*en_US.UTF-8 UTF-8/s/^#\s*//' /etc/locale.gen
 167 fi
 168
 169 locale-gen
 170
 171 # update /etc/default/locale
 172 update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
 173
 174 # set both for safe
 175 echo LC_ALL="en_US.UTF-8" >> /etc/environment
 176 echo LANG="en_US.UTF-8" >> /etc/environment
 177 """
 178
 179
 180 YUM_BOOTSTRAP = r"""
 181 #!/bin/bash
 182 set -xueo pipefail
 183
 184 yum -y -q update
 185 yum -y -q install epel-release
 186 yum -y -q update
 187
 188 yum -y -q --verbose install \
 189     {pkgs}
 190
 191 yum clean all
 192
 193 # gen locale
 194 localedef -c -i en_US -f UTF-8 en_US.UTF-8
 195
 196 # no update-locale, diy
 197 # LC_ALL is not valid in this file
 198 echo LANG="en_US.UTF-8" > /etc/locale.conf
 199
 200 # set both for safe
 201 echo LC_ALL="en_US.UTF-8" >> /etc/environment
 202 echo LANG="en_US.UTF-8" >> /etc/environment
 203 """
 204
 205
 206 DNF_BOOTSTRAP = r"""
 207 #!/bin/bash
 208 set -xueo pipefail
 209
 210 dnf -y -q update
 211
 212 dnf -y -q --verbose install \
 213     {pkgs}
 214
 215 dnf clean all
 216
 217 # gen locale
 218 localedef -c -i en_US -f UTF-8 en_US.UTF-8
 219
 220 # no update-locale, diy
 221 # LC_ALL is not valid in this file
 222 echo LANG="en_US.UTF-8" > /etc/locale.conf
 223
 224 # set both for safe
 225 echo LC_ALL="en_US.UTF-8" >> /etc/environment
 226 echo LANG="en_US.UTF-8" >> /etc/environment
 227 """
 228
 229
 230 DOCKERFILE = r"""
 231 FROM {docker_image}
 232
 233 # we will use this image to run ci, these ENV vars are important
 234 ENV CC="ccache gcc"
 235
 236 ADD bootstrap.sh /tmp/bootstrap.sh
 237 # need root permission, do it before USER samba
 238 RUN bash /tmp/bootstrap.sh
 239
 240 # make test can not work with root, so we have to create a new user
 241 RUN useradd -m -s /bin/bash samba && \
 242     mkdir -p /etc/sudoers.d && \
 243     echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
 244
 245 USER samba
 246 WORKDIR /home/samba
 247 # samba tests rely on this
 248 ENV USER=samba LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
 249 """
 250
 251 # Vagrantfile snippet for each dist
 252 VAGRANTFILE_SNIPPET = r"""
 253     config.vm.define "{name}" do |v|
 254         v.vm.box = "{vagrant_box}"
 255         v.vm.hostname = "{name}"
 256         v.vm.provision :shell, path: "{name}/bootstrap.sh"
 257     end
 258 """
 259
 260 # global Vagrantfile with snippets for all dists
 261 VAGRANTFILE_GLOBAL = r"""
 262 Vagrant.configure("2") do |config|
 263     config.ssh.insert_key = false
 264
 265 {vagrantfile_snippets}
 266
 267 end
 268 """
 269
 270
 271 DEB_DISTS = {
 272     'debian7': {
 273         'docker_image': 'debian:7',
 274         'vagrant_box': 'debian/wheezy64',
 275         'replace': {
 276             'libgnutls28-dev': 'libgnutls-dev',
 277             'libsystemd-dev': '',  # not available, remove
 278             'lmdb-utils': '',  # not available, remove
 279             'liblmdb-dev': '',  # not available, remove
 280             'python-gpg': 'python-gpgme',
 281             'python3-gpg': '',  # no python3 gpg pkg available, remove
 282             'language-pack-en': '',   # included in locales
 283         }
 284     },
 285     'debian8': {
 286         'docker_image': 'debian:8',
 287         'vagrant_box': 'debian/jessie64',
 288         'replace': {
 289             'python-gpg': 'python-gpgme',
 290             'python3-gpg': 'python3-gpgme',
 291             'language-pack-en': '',   # included in locales
 292         }
 293     },
 294     'debian9': {
 295         'docker_image': 'debian:9',
 296         'vagrant_box': 'debian/stretch64',
 297         'replace': {
 298             'language-pack-en': '',   # included in locales
 299         }
 300     },
 301     'ubuntu1404': {
 302         'docker_image': 'ubuntu:14.04',
 303         'vagrant_box': 'ubuntu/trusty64',
 304         'replace': {
 305             'libsystemd-dev': '',  # remove
 306             'libgnutls28-dev': 'libgnutls-dev',
 307             'python-gpg': 'python-gpgme',
 308             'python3-gpg': 'python3-gpgme',
 309             'lmdb-utils': 'lmdb-utils/trusty-backports',
 310             'liblmdb-dev': 'liblmdb-dev/trusty-backports',
 311         }
 312     },
 313     'ubuntu1604': {
 314         'docker_image': 'ubuntu:16.04',
 315         'vagrant_box': 'ubuntu/xenial64',
 316         'replace': {
 317             'python-gpg': 'python-gpgme',
 318             'python3-gpg': 'python3-gpgme',
 319         }
 320     },
 321     'ubuntu1804': {
 322         'docker_image': 'ubuntu:18.04',
 323         'vagrant_box': 'ubuntu/bionic64',
 324     },
 325 }
 326
 327
 328 RPM_DISTS = {
 329     'centos6': {
 330         'docker_image': 'centos:6',
 331         'vagrant_box': 'centos/6',
 332         'bootstrap': YUM_BOOTSTRAP,
 333         'replace': {
 334             'python3-devel': 'python34-devel',
 335             'python2-gpg': 'pygpgme',
 336             'python3-gpg': '',  # no python3-gpg yet
 337             '@development-tools': '"@Development Tools"',  # add quotes
 338             'glibc-langpack-en': '',  # included in glibc-common
 339             'glibc-locale-source': '',  # included in glibc-common
 340             'procps-ng': 'procps',  # centos6 still use old name
 341             # update perl core modules on centos
 342             # fix: Can't locate Archive/Tar.pm in @INC
 343             'perl': 'perl-core',
 344         }
 345     },
 346     'centos7': {
 347         'docker_image': 'centos:7',
 348         'vagrant_box': 'centos/7',
 349         'bootstrap': YUM_BOOTSTRAP,
 350         'replace': {
 351             'python3-devel': 'python34-devel',
 352             # although python36-devel is available
 353             # after epel-release installed
 354             # however, all other python3 pkgs are still python34-ish
 355             'python2-gpg': 'pygpgme',
 356             'python3-gpg': '',  # no python3-gpg yet
 357             '@development-tools': '"@Development Tools"',  # add quotes
 358             'glibc-langpack-en': '',  # included in glibc-common
 359             'glibc-locale-source': '',  # included in glibc-common
 360             # update perl core modules on centos
 361             # fix: Can't locate Archive/Tar.pm in @INC
 362             'perl': 'perl-core',
 363         }
 364     },
 365     'fedora28': {
 366         'docker_image': 'fedora:28',
 367         'vagrant_box': 'fedora/28-cloud-base',
 368         'bootstrap': DNF_BOOTSTRAP,
 369     },
 370     'fedora29': {
 371         'docker_image': 'fedora:29',
 372         'vagrant_box': 'fedora/29-cloud-base',
 373         'bootstrap': DNF_BOOTSTRAP,
 374     },
 375 }
 376
 377
 378 DEB_FAMILY = {
 379     'name': 'deb',
 380     'pkgs': DEB_PKGS,
 381     'bootstrap': APT_BOOTSTRAP,  # family default
 382     'dists': DEB_DISTS,
 383 }
 384
 385
 386 RPM_FAMILY = {
 387     'name': 'rpm',
 388     'pkgs': RPM_PKGS,
 389     'bootstrap': YUM_BOOTSTRAP,  # family default
 390     'dists': RPM_DISTS,
 391 }
 392
 393
 394 YML_HEADER = r"""
 395 ---
 396 packages:
 397 """
 398
 399
 400 def expand_family_dists(family):
 401     dists = {}
 402     for name, config in family['dists'].items():
 403         config = config.copy()
 404         config['name'] = name
 405         config['home'] = join(OUT, name)
 406         config['family'] = family['name']
 407
 408         # replace dist specific pkgs
 409         replace = config.get('replace', {})
 410         pkgs = []
 411         for pkg in family['pkgs']:
 412             pkg = replace.get(pkg, pkg)  # replace if exists or get self
 413             if pkg:
 414                 pkgs.append(pkg)
 415         pkgs.sort()
 416
 417         lines = ['  - {}'.format(pkg) for pkg in pkgs]
 418         config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
 419
 420         sep = ' \\' + os.linesep + '    '
 421         config['pkgs'] = sep.join(pkgs)
 422
 423         # get dist bootstrap template or fall back to family default
 424         bootstrap_template = config.get('bootstrap', family['bootstrap'])
 425         config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
 426
 427         config['Dockerfile'] = DOCKERFILE.format(**config).strip()
 428         # keep the indent, no strip
 429         config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
 430
 431         dists[name] = config
 432     return dists
 433
 434
 435 # expanded config for dists
 436 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
 437 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
 438
 439 # assemble all together
 440 DISTS = {}
 441 DISTS.update(DEB_DISTS_EXP)
 442 DISTS.update(RPM_DISTS_EXP)
 443
 444
 445 def render_vagrantfile(dists):
 446     """
 447     Render all snippets for each dist into global Vagrantfile.
 448
 449     Vagrant supports multiple vms in one Vagrantfile.
 450     This make it easier to manage the fleet, e.g:
 451
 452     start all: vagrant up
 453     start one: vagrant up ubuntu1804
 454
 455     All other commands apply to above syntax, e.g.: status, destroy, provision
 456     """
 457     # sort dists by name and put all vagrantfile snippets together
 458     snippets = [
 459         dists[dist]['vagrantfile_snippet']
 460         for dist in sorted(dists.keys())]
 461
 462     return VAGRANTFILE_GLOBAL.format(vagrantfile_snippets=''.join(snippets))
 463
 464
 465 VAGRANTFILE = render_vagrantfile(DISTS)
 466
 467
 468 # data we need to expose
 469 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']