bootstrap/config.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) Catalyst.Net Ltd 2019
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """
  19 Manage dependencies and bootstrap environments for Samba.
  20
  21 Config file for packages and templates.
  22
  23 Author: Joe Guo <joeg@catalyst.net.nz>
  24 """
  25 import os
  26 from os.path import abspath, dirname, join
  27 HERE = abspath(dirname(__file__))
  28 # output dir for rendered files
  29 OUT = join(HERE, 'dists')
  30
  31
  32 # pkgs with same name in all packaging systems
  33 COMMON = [
  34     'acl',
  35     'attr',
  36     'autoconf',
  37     'binutils',
  38     'bison',
  39     'curl',
  40     'gcc',
  41     'gdb',
  42     'git',
  43     'gzip',
  44     'hostname',
  45     'htop',
  46     'make',
  47     'patch',
  48     'perl',
  49     'psmisc',  # for pstree in test
  50     'rng-tools',
  51     'rsync',
  52     'sed',
  53     'sudo',  # docker images has no sudo by default
  54     'tar',
  55     'tree',
  56 ]
  57
  58
  59 # define pkgs for all packaging systems in parallel
  60 # make it easier to find missing ones
  61 # use latest ubuntu and fedora as defaults
  62 # deb, rpm, ...
  63 PKGS = [
  64     # NAME1-dev, NAME2-devel
  65     ('lmdb-utils', 'lmdb'),
  66     ('nettle-dev', 'nettle-devel'),
  67     ('zlib1g-dev', 'zlib-devel'),
  68     ('libbsd-dev', 'libbsd-devel'),
  69     ('libaio-dev', 'libaio-devel'),
  70     ('libarchive-dev', 'libarchive-devel'),
  71     ('libblkid-dev', 'libblkid-devel'),
  72     ('libcap-dev', 'libcap-devel'),
  73     ('libacl1-dev', 'libacl-devel'),
  74     ('libattr1-dev', 'libattr-devel'),
  75
  76     # libNAME1-dev, NAME2-devel
  77     ('libpopt-dev', 'popt-devel'),
  78     ('libreadline-dev', 'readline-devel'),
  79     ('libjansson-dev', 'jansson-devel'),
  80     ('liblmdb-dev', 'lmdb-devel'),
  81     ('libncurses5-dev', 'ncurses-devel'),
  82     # NOTE: Debian 7+ or Ubuntu 16.04+
  83     ('libsystemd-dev', 'systemd-devel'),
  84     ('libkrb5-dev', 'krb5-devel'),
  85     ('libldap2-dev', 'openldap-devel'),
  86     ('libcups2-dev', 'cups-devel'),
  87     ('libpam0g-dev', 'pam-devel'),
  88     ('libgpgme11-dev', 'gpgme-devel'),
  89     # NOTE: Debian 8+ and Ubuntu 14.04+
  90     ('libgnutls28-dev', 'gnutls-devel'),
  91     ('libtasn1-bin', ''),
  92     ('libtasn1-dev', 'libtasn1-devel'),
  93     ('', 'quota-devel'),
  94     ('uuid-dev', 'libuuid-devel'),
  95     ('libjs-jquery', ''),
  96     ('libavahi-common-dev', 'avahi-devel'),
  97     ('libdbus-1-dev', 'dbus-devel'),
  98     ('libpcap-dev', 'libpcap-devel'),
  99     ('libunwind-dev', 'libunwind-devel'),  # for back trace
 100     ('libglib2.0-dev', 'glib2-devel'),
 101     ('libicu-dev', 'libicu-devel'),
 102
 103     # NAME1, NAME2
 104     # for debian, locales provide locale support with language packs
 105     # ubuntu split language packs to language-pack-xx
 106     # for centos, glibc-common provide locale support with language packs
 107     # fedora split language packs  to glibc-langpack-xx
 108     ('locales', 'glibc-common'),  # required for locale
 109     ('language-pack-en', 'glibc-langpack-en'),  # we need en_US.UTF-8
 110     ('bind9utils', 'bind-utils'),
 111     ('dnsutils', ''),
 112     ('xsltproc', 'libxslt'),
 113     ('krb5-user', ''),
 114     ('krb5-config', ''),
 115     ('', 'krb5-server'),
 116     ('apt-utils', 'yum-utils'),
 117     ('pkg-config', 'pkgconfig'),
 118     ('procps', 'procps-ng'),  # required for the free cmd in tests
 119     ('lsb-release', 'lsb-release'),  # we need lsb_relase to show info
 120     ('', 'rpcgen'),  # required for test
 121     # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
 122     ('', 'libtirpc-devel'),  # for <rpc/rpc.h> header on fedora
 123     ('', 'libnsl2-devel'),  # for <rpcsvc/yp_prot.h> header on fedora
 124     ('mawk', 'gawk'),
 125
 126     # python
 127     ('python-dev', 'python-devel'),
 128     ('python-dbg', ''),
 129     ('python-iso8601', ''),
 130     ('python-gpg', 'python2-gpg'),  # defaults to ubuntu/fedora latest
 131     ('python-crypto', 'python-crypto'),
 132     ('python-markdown', 'python-markdown'),
 133     ('python-dnspython', 'python-dns'),
 134     ('python-pexpect', ''),  # for wintest only
 135
 136     ('python3-dev', 'python3-devel'),
 137     ('python3-dbg', ''),
 138     ('python3-iso8601', ''),
 139     ('python3-gpg', 'python3-gpg'),  # defaults to ubuntu/fedora latest
 140     ('python3-crypto', 'python3-crypto'),
 141     ('python3-markdown', 'python3-markdown'),
 142     ('python3-matplotlib', ''),
 143     ('python3-dnspython', 'python3-dns'),
 144     ('python3-pexpect', ''),  # for wintest only
 145
 146     ('', 'libsemanage-python'),
 147     ('', 'policycoreutils-python'),
 148
 149     # perl
 150     ('libparse-yapp-perl', 'perl-Parse-Yapp'),
 151     ('libjson-perl', 'perl-JSON-Parse'),
 152     ('perl-modules', ''),
 153     ('', 'perl-Archive-Tar'),
 154     ('', 'perl-ExtUtils-MakeMaker'),
 155     ('', 'perl-Test-Base'),
 156     ('', 'perl-generators'),
 157     ('', 'perl-interpreter'),
 158
 159     # misc
 160     # @ means group for rpm, use fedora as rpm default
 161     ('build-essential', '@development-tools'),
 162     ('debhelper', ''),
 163     # rpm has no pkg for docbook-xml
 164     ('docbook-xml', 'docbook-dtds'),
 165     ('docbook-xsl', 'docbook-style-xsl'),
 166     ('flex', ''),
 167     ('', 'keyutils-libs-devel'),
 168     ('', 'which'),
 169 ]
 170
 171
 172 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
 173 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
 174
 175
 176 APT_BOOTSTRAP = r"""
 177 #!/bin/bash
 178 set -xueo pipefail
 179
 180 export DEBIAN_FRONTEND=noninteractive
 181 apt-get -y update
 182
 183 apt-get -y install \
 184     {pkgs}
 185
 186 apt-get -y autoremove
 187 apt-get -y autoclean
 188 apt-get -y clean
 189 """
 190
 191
 192 YUM_BOOTSTRAP = r"""
 193 #!/bin/bash
 194 set -xueo pipefail
 195
 196 yum -y -q update
 197 yum -y -q install epel-release
 198 yum -y -q update
 199
 200 yum -y -q --verbose install \
 201     {pkgs}
 202
 203 yum clean all
 204 """
 205
 206
 207 DNF_BOOTSTRAP = r"""
 208 #!/bin/bash
 209 set -xueo pipefail
 210
 211 dnf -y -q update
 212
 213 dnf -y -q --verbose install \
 214     {pkgs}
 215
 216 dnf clean all
 217 """
 218
 219 ZYPPER_BOOTSTRAP = r"""
 220 #!/bin/bash
 221 set -xueo pipefail
 222
 223 zypper --non-interactive refresh
 224 zypper --non-interactive update
 225 zypper --non-interactive install \
 226     {pkgs} \
 227     system-user-nobody
 228
 229 zypper --non-interactive clean
 230
 231 if [ -f /usr/lib/mit/bin/krb5-config ]; then
 232     ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
 233 fi
 234 """
 235
 236 # A generic shell script to setup locale
 237 LOCALE_SETUP = r"""
 238 #!/bin/bash
 239 set -xueo pipefail
 240
 241 # refer to /usr/share/i18n/locales
 242 INPUTFILE=en_US
 243 # refer to /usr/share/i18n/charmaps
 244 CHARMAP=UTF-8
 245 # locale to generate in /usr/lib/locale
 246 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
 247 LOCALE=$INPUTFILE.utf8
 248
 249 # if locale is already correct, exit
 250 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
 251
 252 # if locale not available, generate locale into /usr/lib/locale
 253 if ! ( locale --all-locales | grep -i $LOCALE )
 254 then
 255     # no-archive means create its own dir
 256     localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
 257 fi
 258
 259 # update locale conf and global env file
 260 # set both LC_ALL and LANG for safe
 261
 262 # update conf for Debian family
 263 FILE=/etc/default/locale
 264 if [ -f $FILE ]
 265 then
 266     echo LC_ALL="$LOCALE" > $FILE
 267     echo LANG="$LOCALE" >> $FILE
 268 fi
 269
 270 # update conf for RedHat family
 271 FILE=/etc/locale.conf
 272 if [ -f $FILE ]
 273 then
 274     # LC_ALL is not valid in this file, set LANG only
 275     echo LANG="$LOCALE" > $FILE
 276 fi
 277
 278 # update global env file
 279 FILE=/etc/environment
 280 if [ -f $FILE ]
 281 then
 282     # append LC_ALL if not exist
 283     grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
 284     # append LANG if not exist
 285     grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
 286 fi
 287 """
 288
 289
 290 DOCKERFILE = r"""
 291 FROM {docker_image}
 292
 293 # pass in with --build-arg while build
 294 ARG SHA1SUM
 295 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
 296
 297 ADD *.sh /tmp/
 298 # need root permission, do it before USER samba
 299 RUN /tmp/bootstrap.sh && /tmp/locale.sh
 300
 301 # if ld.gold exists, force link it to ld
 302 RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"
 303
 304 # make test can not work with root, so we have to create a new user
 305 RUN useradd -m -U -s /bin/bash samba && \
 306     mkdir -p /etc/sudoers.d && \
 307     echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
 308
 309 USER samba
 310 WORKDIR /home/samba
 311 # samba tests rely on this
 312 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
 313 """
 314
 315 # Vagrantfile snippet for each dist
 316 VAGRANTFILE_SNIPPET = r"""
 317     config.vm.define "{name}" do |v|
 318         v.vm.box = "{vagrant_box}"
 319         v.vm.hostname = "{name}"
 320         v.vm.provision :shell, path: "{name}/bootstrap.sh"
 321         v.vm.provision :shell, path: "{name}/locale.sh"
 322     end
 323 """
 324
 325 # global Vagrantfile with snippets for all dists
 326 VAGRANTFILE_GLOBAL = r"""
 327 Vagrant.configure("2") do |config|
 328     config.ssh.insert_key = false
 329
 330 {vagrantfile_snippets}
 331
 332 end
 333 """
 334
 335
 336 DEB_DISTS = {
 337     'debian7': {
 338         'docker_image': 'debian:7',
 339         'vagrant_box': 'debian/wheezy64',
 340         'replace': {
 341             'libgnutls28-dev': 'libgnutls-dev',
 342             'libsystemd-dev': '',  # not available, remove
 343             'lmdb-utils': '',  # not available, remove
 344             'liblmdb-dev': '',  # not available, remove
 345             'python-gpg': 'python-gpgme',
 346             'python3-gpg': '',  # no python3 gpg pkg available, remove
 347             'language-pack-en': '',   # included in locales
 348         }
 349     },
 350     'debian8': {
 351         'docker_image': 'debian:8',
 352         'vagrant_box': 'debian/jessie64',
 353         'replace': {
 354             'python-gpg': 'python-gpgme',
 355             'python3-gpg': 'python3-gpgme',
 356             'language-pack-en': '',   # included in locales
 357         }
 358     },
 359     'debian9': {
 360         'docker_image': 'debian:9',
 361         'vagrant_box': 'debian/stretch64',
 362         'replace': {
 363             'language-pack-en': '',   # included in locales
 364         }
 365     },
 366     'ubuntu1404': {
 367         'docker_image': 'ubuntu:14.04',
 368         'vagrant_box': 'ubuntu/trusty64',
 369         'replace': {
 370             'libsystemd-dev': '',  # remove
 371             'libgnutls28-dev': 'libgnutls-dev',
 372             'python-gpg': 'python-gpgme',
 373             'python3-gpg': 'python3-gpgme',
 374             'lmdb-utils': 'lmdb-utils/trusty-backports',
 375             'liblmdb-dev': 'liblmdb-dev/trusty-backports',
 376             'libunwind-dev': 'libunwind8-dev',
 377         }
 378     },
 379     'ubuntu1604': {
 380         'docker_image': 'ubuntu:16.04',
 381         'vagrant_box': 'ubuntu/xenial64',
 382         'replace': {
 383             'python-gpg': 'python-gpgme',
 384             'python3-gpg': 'python3-gpgme',
 385         }
 386     },
 387     'ubuntu1804': {
 388         'docker_image': 'ubuntu:18.04',
 389         'vagrant_box': 'ubuntu/bionic64',
 390     },
 391 }
 392
 393
 394 RPM_DISTS = {
 395     'centos6': {
 396         'docker_image': 'centos:6',
 397         'vagrant_box': 'centos/6',
 398         'bootstrap': YUM_BOOTSTRAP,
 399         'replace': {
 400             'lsb-release': 'redhat-lsb',
 401             'python3-devel': 'python34-devel',
 402             'python2-gpg': 'pygpgme',
 403             'python3-gpg': '',  # no python3-gpg yet
 404             '@development-tools': '"@Development Tools"',  # add quotes
 405             'glibc-langpack-en': '',  # included in glibc-common
 406             'glibc-locale-source': '',  # included in glibc-common
 407             'procps-ng': 'procps',  # centos6 still use old name
 408             # update perl core modules on centos
 409             # fix: Can't locate Archive/Tar.pm in @INC
 410             'perl': 'perl-core',
 411         }
 412     },
 413     'centos7': {
 414         'docker_image': 'centos:7',
 415         'vagrant_box': 'centos/7',
 416         'bootstrap': YUM_BOOTSTRAP,
 417         'replace': {
 418             'lsb-release': 'redhat-lsb',
 419             'python3-devel': 'python34-devel',
 420             # although python36-devel is available
 421             # after epel-release installed
 422             # however, all other python3 pkgs are still python34-ish
 423             'python2-gpg': 'pygpgme',
 424             'python3-gpg': '',  # no python3-gpg yet
 425             '@development-tools': '"@Development Tools"',  # add quotes
 426             'glibc-langpack-en': '',  # included in glibc-common
 427             'glibc-locale-source': '',  # included in glibc-common
 428             # update perl core modules on centos
 429             # fix: Can't locate Archive/Tar.pm in @INC
 430             'perl': 'perl-core',
 431         }
 432     },
 433     'fedora28': {
 434         'docker_image': 'fedora:28',
 435         'vagrant_box': 'fedora/28-cloud-base',
 436         'bootstrap': DNF_BOOTSTRAP,
 437         'replace': {
 438             'lsb-release': 'redhat-lsb',
 439         }
 440     },
 441     'fedora29': {
 442         'docker_image': 'fedora:29',
 443         'vagrant_box': 'fedora/29-cloud-base',
 444         'bootstrap': DNF_BOOTSTRAP,
 445         'replace': {
 446             'lsb-release': 'redhat-lsb',
 447         }
 448     },
 449     'opensuse150': {
 450         'docker_image': 'opensuse/leap:15.0',
 451         'vagrant_box': 'opensuse/openSUSE-15.0-x86_64',
 452         'bootstrap': ZYPPER_BOOTSTRAP,
 453         'replace': {
 454             '@development-tools': '',
 455             'dbus-devel': 'dbus-1-devel',
 456             'docbook-style-xsl': 'docbook-xsl-stylesheets',
 457             'glibc-common': 'glibc-locale',
 458             'glibc-locale-source': 'glibc-i18ndata',
 459             'glibc-langpack-en': '',
 460             'jansson-devel': 'libjansson-devel',
 461             'keyutils-libs-devel': 'keyutils-devel',
 462             'krb5-workstation': 'krb5-client',
 463             'libnsl2-devel': 'libnsl-devel',
 464             'libsemanage-python': 'python2-semanage',
 465             'nettle-devel': 'libnettle-devel',
 466             'openldap-devel': 'openldap2-devel',
 467             'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
 468             'perl-JSON-Parse': 'perl-JSON-XS',
 469             'perl-generators': '',
 470             'perl-interpreter': '',
 471             'procps-ng': 'procps',
 472             'python-dns': 'python2-dnspython',
 473             'python3-crypto': 'python3-pycrypto',
 474             'python3-dns': 'python3-dnspython',
 475             'python3-markdown': 'python3-Markdown',
 476             'quota-devel': '',
 477         }
 478     }
 479 }
 480
 481
 482 DEB_FAMILY = {
 483     'name': 'deb',
 484     'pkgs': DEB_PKGS,
 485     'bootstrap': APT_BOOTSTRAP,  # family default
 486     'dists': DEB_DISTS,
 487 }
 488
 489
 490 RPM_FAMILY = {
 491     'name': 'rpm',
 492     'pkgs': RPM_PKGS,
 493     'bootstrap': YUM_BOOTSTRAP,  # family default
 494     'dists': RPM_DISTS,
 495 }
 496
 497
 498 YML_HEADER = r"""
 499 ---
 500 packages:
 501 """
 502
 503
 504 def expand_family_dists(family):
 505     dists = {}
 506     for name, config in family['dists'].items():
 507         config = config.copy()
 508         config['name'] = name
 509         config['home'] = join(OUT, name)
 510         config['family'] = family['name']
 511
 512         # replace dist specific pkgs
 513         replace = config.get('replace', {})
 514         pkgs = []
 515         for pkg in family['pkgs']:
 516             pkg = replace.get(pkg, pkg)  # replace if exists or get self
 517             if pkg:
 518                 pkgs.append(pkg)
 519         pkgs.sort()
 520
 521         lines = ['  - {}'.format(pkg) for pkg in pkgs]
 522         config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
 523
 524         sep = ' \\' + os.linesep + '    '
 525         config['pkgs'] = sep.join(pkgs)
 526
 527         # get dist bootstrap template or fall back to family default
 528         bootstrap_template = config.get('bootstrap', family['bootstrap'])
 529         config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
 530         config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
 531
 532         config['Dockerfile'] = DOCKERFILE.format(**config).strip()
 533         # keep the indent, no strip
 534         config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
 535
 536         dists[name] = config
 537     return dists
 538
 539
 540 # expanded config for dists
 541 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
 542 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
 543
 544 # assemble all together
 545 DISTS = {}
 546 DISTS.update(DEB_DISTS_EXP)
 547 DISTS.update(RPM_DISTS_EXP)
 548
 549
 550 def render_vagrantfile(dists):
 551     """
 552     Render all snippets for each dist into global Vagrantfile.
 553
 554     Vagrant supports multiple vms in one Vagrantfile.
 555     This make it easier to manage the fleet, e.g:
 556
 557     start all: vagrant up
 558     start one: vagrant up ubuntu1804
 559
 560     All other commands apply to above syntax, e.g.: status, destroy, provision
 561     """
 562     # sort dists by name and put all vagrantfile snippets together
 563     snippets = [
 564         dists[dist]['vagrantfile_snippet']
 565         for dist in sorted(dists.keys())]
 566
 567     return VAGRANTFILE_GLOBAL.format(vagrantfile_snippets=''.join(snippets))
 568
 569
 570 VAGRANTFILE = render_vagrantfile(DISTS)
 571
 572
 573 # data we need to expose
 574 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']