bootstrap/config.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) Catalyst.Net Ltd 2019
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """
  19 Manage dependencies and bootstrap environments for Samba.
  20
  21 Config file for packages and templates.
  22
  23 Author: Joe Guo <joeg@catalyst.net.nz>
  24 """
  25 import os
  26 from os.path import abspath, dirname, join
  27 HERE = abspath(dirname(__file__))
  28 # output dir for rendered files
  29 OUT = join(HERE, 'generated-dists')
  30
  31
  32 # pkgs with same name in all packaging systems
  33 COMMON = [
  34     'acl',
  35     'attr',
  36     'autoconf',
  37     'binutils',
  38     'bison',
  39     'curl',
  40     'gcc',
  41     'gdb',
  42     'git',
  43     'gzip',
  44     'hostname',
  45     'htop',
  46     'make',
  47     'patch',
  48     'perl',
  49     'psmisc',  # for pstree in test
  50     'rng-tools',
  51     'rsync',
  52     'sed',
  53     'sudo',  # docker images has no sudo by default
  54     'tar',
  55     'tree',
  56 ]
  57
  58
  59 # define pkgs for all packaging systems in parallel
  60 # make it easier to find missing ones
  61 # use latest ubuntu and fedora as defaults
  62 # deb, rpm, ...
  63 PKGS = [
  64     # NAME1-dev, NAME2-devel
  65     ('lmdb-utils', 'lmdb'),
  66     ('nettle-dev', 'nettle-devel'),
  67     ('zlib1g-dev', 'zlib-devel'),
  68     ('libbsd-dev', 'libbsd-devel'),
  69     ('libaio-dev', 'libaio-devel'),
  70     ('libarchive-dev', 'libarchive-devel'),
  71     ('libblkid-dev', 'libblkid-devel'),
  72     ('libcap-dev', 'libcap-devel'),
  73     ('libacl1-dev', 'libacl-devel'),
  74     ('libattr1-dev', 'libattr-devel'),
  75
  76     # libNAME1-dev, NAME2-devel
  77     ('libpopt-dev', 'popt-devel'),
  78     ('libreadline-dev', 'readline-devel'),
  79     ('libjansson-dev', 'jansson-devel'),
  80     ('liblmdb-dev', 'lmdb-devel'),
  81     ('libncurses5-dev', 'ncurses-devel'),
  82     # NOTE: Debian 7+ or Ubuntu 16.04+
  83     ('libsystemd-dev', 'systemd-devel'),
  84     ('libkrb5-dev', 'krb5-devel'),
  85     ('libldap2-dev', 'openldap-devel'),
  86     ('libcups2-dev', 'cups-devel'),
  87     ('libpam0g-dev', 'pam-devel'),
  88     ('libgpgme11-dev', 'gpgme-devel'),
  89     # NOTE: Debian 8+ and Ubuntu 14.04+
  90     ('libgnutls28-dev', 'gnutls-devel'),
  91     ('libtasn1-bin', ''),
  92     ('libtasn1-dev', 'libtasn1-devel'),
  93     ('', 'quota-devel'),
  94     ('uuid-dev', 'libuuid-devel'),
  95     ('libjs-jquery', ''),
  96     ('libavahi-common-dev', 'avahi-devel'),
  97     ('libdbus-1-dev', 'dbus-devel'),
  98     ('libpcap-dev', 'libpcap-devel'),
  99     ('libunwind-dev', 'libunwind-devel'),  # for back trace
 100     ('libglib2.0-dev', 'glib2-devel'),
 101     ('libicu-dev', 'libicu-devel'),
 102
 103     # NAME1, NAME2
 104     # for debian, locales provide locale support with language packs
 105     # ubuntu split language packs to language-pack-xx
 106     # for centos, glibc-common provide locale support with language packs
 107     # fedora split language packs  to glibc-langpack-xx
 108     ('locales', 'glibc-common'),  # required for locale
 109     ('language-pack-en', 'glibc-langpack-en'),  # we need en_US.UTF-8
 110     ('bind9utils', 'bind-utils'),
 111     ('dnsutils', ''),
 112     ('xsltproc', 'libxslt'),
 113     ('krb5-user', ''),
 114     ('krb5-config', ''),
 115     ('', 'krb5-server'),
 116     ('apt-utils', 'yum-utils'),
 117     ('pkg-config', 'pkgconfig'),
 118     ('procps', 'procps-ng'),  # required for the free cmd in tests
 119     ('lsb-release', 'lsb-release'),  # we need lsb_relase to show info
 120     ('', 'rpcgen'),  # required for test
 121     # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
 122     ('', 'libtirpc-devel'),  # for <rpc/rpc.h> header on fedora
 123     ('', 'libnsl2-devel'),  # for <rpcsvc/yp_prot.h> header on fedora
 124     ('mawk', 'gawk'),
 125
 126     # python
 127     ('python-dev', 'python-devel'),
 128     ('python-dbg', ''),
 129     ('python-iso8601', ''),
 130     ('python-gpg', 'python2-gpg'),  # defaults to ubuntu/fedora latest
 131     ('python-crypto', 'python-crypto'),
 132     ('python-markdown', 'python-markdown'),
 133     ('python-dnspython', 'python-dns'),
 134     ('python-pexpect', ''),  # for wintest only
 135
 136     ('python3-dev', 'python3-devel'),
 137     ('python3-dbg', ''),
 138     ('python3-iso8601', ''),
 139     ('python3-gpg', 'python3-gpg'),  # defaults to ubuntu/fedora latest
 140     ('python3-crypto', 'python3-crypto'),
 141     ('python3-markdown', 'python3-markdown'),
 142     ('python3-matplotlib', ''),
 143     ('python3-dnspython', 'python3-dns'),
 144     ('python3-pexpect', ''),  # for wintest only
 145
 146     ('', 'libsemanage-python'),
 147     ('', 'policycoreutils-python'),
 148
 149     # perl
 150     ('libparse-yapp-perl', 'perl-Parse-Yapp'),
 151     ('libjson-perl', 'perl-JSON-Parse'),
 152     ('perl-modules', ''),
 153     ('', 'perl-Archive-Tar'),
 154     ('', 'perl-ExtUtils-MakeMaker'),
 155     ('', 'perl-Test-Base'),
 156     ('', 'perl-generators'),
 157     ('', 'perl-interpreter'),
 158
 159     # misc
 160     # @ means group for rpm, use fedora as rpm default
 161     ('build-essential', '@development-tools'),
 162     ('debhelper', ''),
 163     # rpm has no pkg for docbook-xml
 164     ('docbook-xml', 'docbook-dtds'),
 165     ('docbook-xsl', 'docbook-style-xsl'),
 166     ('flex', ''),
 167     ('', 'keyutils-libs-devel'),
 168     ('', 'which'),
 169 ]
 170
 171
 172 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
 173 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
 174
 175 GENERATED_MARKER = r"""
 176 #
 177 # This file is generated by 'bootstrap/template.py --render'
 178 # See also bootstrap/config.py
 179 #
 180 """
 181
 182
 183 APT_BOOTSTRAP = r"""
 184 #!/bin/bash
 185 {GENERATED_MARKER}
 186 set -xueo pipefail
 187
 188 export DEBIAN_FRONTEND=noninteractive
 189 apt-get -y update
 190
 191 apt-get -y install \
 192     {pkgs}
 193
 194 apt-get -y autoremove
 195 apt-get -y autoclean
 196 apt-get -y clean
 197 """
 198
 199
 200 YUM_BOOTSTRAP = r"""
 201 #!/bin/bash
 202 {GENERATED_MARKER}
 203 set -xueo pipefail
 204
 205 yum -y -q update
 206 yum -y -q install epel-release
 207 yum -y -q update
 208
 209 yum -y -q --verbose install \
 210     {pkgs}
 211
 212 yum clean all
 213 """
 214
 215
 216 DNF_BOOTSTRAP = r"""
 217 #!/bin/bash
 218 {GENERATED_MARKER}
 219 set -xueo pipefail
 220
 221 dnf -y -q update
 222
 223 dnf -y -q --verbose install \
 224     {pkgs}
 225
 226 dnf clean all
 227 """
 228
 229 ZYPPER_BOOTSTRAP = r"""
 230 #!/bin/bash
 231 {GENERATED_MARKER}
 232 set -xueo pipefail
 233
 234 zypper --non-interactive refresh
 235 zypper --non-interactive update
 236 zypper --non-interactive install \
 237     {pkgs} \
 238     system-user-nobody
 239
 240 zypper --non-interactive clean
 241
 242 if [ -f /usr/lib/mit/bin/krb5-config ]; then
 243     ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
 244 fi
 245 """
 246
 247 # A generic shell script to setup locale
 248 LOCALE_SETUP = r"""
 249 #!/bin/bash
 250 {GENERATED_MARKER}
 251 set -xueo pipefail
 252
 253 # refer to /usr/share/i18n/locales
 254 INPUTFILE=en_US
 255 # refer to /usr/share/i18n/charmaps
 256 CHARMAP=UTF-8
 257 # locale to generate in /usr/lib/locale
 258 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
 259 LOCALE=$INPUTFILE.utf8
 260
 261 # if locale is already correct, exit
 262 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
 263
 264 # if locale not available, generate locale into /usr/lib/locale
 265 if ! ( locale --all-locales | grep -i $LOCALE )
 266 then
 267     # no-archive means create its own dir
 268     localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
 269 fi
 270
 271 # update locale conf and global env file
 272 # set both LC_ALL and LANG for safe
 273
 274 # update conf for Debian family
 275 FILE=/etc/default/locale
 276 if [ -f $FILE ]
 277 then
 278     echo LC_ALL="$LOCALE" > $FILE
 279     echo LANG="$LOCALE" >> $FILE
 280 fi
 281
 282 # update conf for RedHat family
 283 FILE=/etc/locale.conf
 284 if [ -f $FILE ]
 285 then
 286     # LC_ALL is not valid in this file, set LANG only
 287     echo LANG="$LOCALE" > $FILE
 288 fi
 289
 290 # update global env file
 291 FILE=/etc/environment
 292 if [ -f $FILE ]
 293 then
 294     # append LC_ALL if not exist
 295     grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
 296     # append LANG if not exist
 297     grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
 298 fi
 299 """
 300
 301
 302 DOCKERFILE = r"""
 303 {GENERATED_MARKER}
 304 FROM {docker_image}
 305
 306 # pass in with --build-arg while build
 307 ARG SHA1SUM
 308 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
 309
 310 ADD *.sh /tmp/
 311 # need root permission, do it before USER samba
 312 RUN /tmp/bootstrap.sh && /tmp/locale.sh
 313
 314 # if ld.gold exists, force link it to ld
 315 RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"
 316
 317 # make test can not work with root, so we have to create a new user
 318 RUN useradd -m -U -s /bin/bash samba && \
 319     mkdir -p /etc/sudoers.d && \
 320     echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
 321
 322 USER samba
 323 WORKDIR /home/samba
 324 # samba tests rely on this
 325 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
 326 """
 327
 328 # Vagrantfile snippet for each dist
 329 VAGRANTFILE_SNIPPET = r"""
 330     config.vm.define "{name}" do |v|
 331         v.vm.box = "{vagrant_box}"
 332         v.vm.hostname = "{name}"
 333         v.vm.provision :shell, path: "{name}/bootstrap.sh"
 334         v.vm.provision :shell, path: "{name}/locale.sh"
 335     end
 336 """
 337
 338 # global Vagrantfile with snippets for all dists
 339 VAGRANTFILE_GLOBAL = r"""
 340 {GENERATED_MARKER}
 341
 342 Vagrant.configure("2") do |config|
 343     config.ssh.insert_key = false
 344
 345 {vagrantfile_snippets}
 346
 347 end
 348 """
 349
 350
 351 DEB_DISTS = {
 352     'debian7': {
 353         'docker_image': 'debian:7',
 354         'vagrant_box': 'debian/wheezy64',
 355         'replace': {
 356             'libgnutls28-dev': 'libgnutls-dev',
 357             'libsystemd-dev': '',  # not available, remove
 358             'lmdb-utils': '',  # not available, remove
 359             'liblmdb-dev': '',  # not available, remove
 360             'python-gpg': 'python-gpgme',
 361             'python3-gpg': '',  # no python3 gpg pkg available, remove
 362             'language-pack-en': '',   # included in locales
 363         }
 364     },
 365     'debian8': {
 366         'docker_image': 'debian:8',
 367         'vagrant_box': 'debian/jessie64',
 368         'replace': {
 369             'python-gpg': 'python-gpgme',
 370             'python3-gpg': 'python3-gpgme',
 371             'language-pack-en': '',   # included in locales
 372         }
 373     },
 374     'debian9': {
 375         'docker_image': 'debian:9',
 376         'vagrant_box': 'debian/stretch64',
 377         'replace': {
 378             'language-pack-en': '',   # included in locales
 379         }
 380     },
 381     'ubuntu1404': {
 382         'docker_image': 'ubuntu:14.04',
 383         'vagrant_box': 'ubuntu/trusty64',
 384         'replace': {
 385             'libsystemd-dev': '',  # remove
 386             'libgnutls28-dev': 'libgnutls-dev',
 387             'python-gpg': 'python-gpgme',
 388             'python3-gpg': 'python3-gpgme',
 389             'lmdb-utils': 'lmdb-utils/trusty-backports',
 390             'liblmdb-dev': 'liblmdb-dev/trusty-backports',
 391             'libunwind-dev': 'libunwind8-dev',
 392         }
 393     },
 394     'ubuntu1604': {
 395         'docker_image': 'ubuntu:16.04',
 396         'vagrant_box': 'ubuntu/xenial64',
 397         'replace': {
 398             'python-gpg': 'python-gpgme',
 399             'python3-gpg': 'python3-gpgme',
 400         }
 401     },
 402     'ubuntu1804': {
 403         'docker_image': 'ubuntu:18.04',
 404         'vagrant_box': 'ubuntu/bionic64',
 405     },
 406 }
 407
 408
 409 RPM_DISTS = {
 410     'centos6': {
 411         'docker_image': 'centos:6',
 412         'vagrant_box': 'centos/6',
 413         'bootstrap': YUM_BOOTSTRAP,
 414         'replace': {
 415             'lsb-release': 'redhat-lsb',
 416             'python3-devel': 'python34-devel',
 417             'python2-gpg': 'pygpgme',
 418             'python3-gpg': '',  # no python3-gpg yet
 419             '@development-tools': '"@Development Tools"',  # add quotes
 420             'glibc-langpack-en': '',  # included in glibc-common
 421             'glibc-locale-source': '',  # included in glibc-common
 422             'procps-ng': 'procps',  # centos6 still use old name
 423             # update perl core modules on centos
 424             # fix: Can't locate Archive/Tar.pm in @INC
 425             'perl': 'perl-core',
 426         }
 427     },
 428     'centos7': {
 429         'docker_image': 'centos:7',
 430         'vagrant_box': 'centos/7',
 431         'bootstrap': YUM_BOOTSTRAP,
 432         'replace': {
 433             'lsb-release': 'redhat-lsb',
 434             'python3-devel': 'python34-devel',
 435             # although python36-devel is available
 436             # after epel-release installed
 437             # however, all other python3 pkgs are still python34-ish
 438             'python2-gpg': 'pygpgme',
 439             'python3-gpg': '',  # no python3-gpg yet
 440             '@development-tools': '"@Development Tools"',  # add quotes
 441             'glibc-langpack-en': '',  # included in glibc-common
 442             'glibc-locale-source': '',  # included in glibc-common
 443             # update perl core modules on centos
 444             # fix: Can't locate Archive/Tar.pm in @INC
 445             'perl': 'perl-core',
 446         }
 447     },
 448     'fedora28': {
 449         'docker_image': 'fedora:28',
 450         'vagrant_box': 'fedora/28-cloud-base',
 451         'bootstrap': DNF_BOOTSTRAP,
 452         'replace': {
 453             'lsb-release': 'redhat-lsb',
 454         }
 455     },
 456     'fedora29': {
 457         'docker_image': 'fedora:29',
 458         'vagrant_box': 'fedora/29-cloud-base',
 459         'bootstrap': DNF_BOOTSTRAP,
 460         'replace': {
 461             'lsb-release': 'redhat-lsb',
 462         }
 463     },
 464     'opensuse150': {
 465         'docker_image': 'opensuse/leap:15.0',
 466         'vagrant_box': 'opensuse/openSUSE-15.0-x86_64',
 467         'bootstrap': ZYPPER_BOOTSTRAP,
 468         'replace': {
 469             '@development-tools': '',
 470             'dbus-devel': 'dbus-1-devel',
 471             'docbook-style-xsl': 'docbook-xsl-stylesheets',
 472             'glibc-common': 'glibc-locale',
 473             'glibc-locale-source': 'glibc-i18ndata',
 474             'glibc-langpack-en': '',
 475             'jansson-devel': 'libjansson-devel',
 476             'keyutils-libs-devel': 'keyutils-devel',
 477             'krb5-workstation': 'krb5-client',
 478             'libnsl2-devel': 'libnsl-devel',
 479             'libsemanage-python': 'python2-semanage',
 480             'nettle-devel': 'libnettle-devel',
 481             'openldap-devel': 'openldap2-devel',
 482             'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
 483             'perl-JSON-Parse': 'perl-JSON-XS',
 484             'perl-generators': '',
 485             'perl-interpreter': '',
 486             'procps-ng': 'procps',
 487             'python-dns': 'python2-dnspython',
 488             'python3-crypto': 'python3-pycrypto',
 489             'python3-dns': 'python3-dnspython',
 490             'python3-markdown': 'python3-Markdown',
 491             'quota-devel': '',
 492         }
 493     }
 494 }
 495
 496
 497 DEB_FAMILY = {
 498     'name': 'deb',
 499     'pkgs': DEB_PKGS,
 500     'bootstrap': APT_BOOTSTRAP,  # family default
 501     'dists': DEB_DISTS,
 502 }
 503
 504
 505 RPM_FAMILY = {
 506     'name': 'rpm',
 507     'pkgs': RPM_PKGS,
 508     'bootstrap': YUM_BOOTSTRAP,  # family default
 509     'dists': RPM_DISTS,
 510 }
 511
 512
 513 YML_HEADER = r"""
 514 ---
 515 packages:
 516 """
 517
 518
 519 def expand_family_dists(family):
 520     dists = {}
 521     for name, config in family['dists'].items():
 522         config = config.copy()
 523         config['name'] = name
 524         config['home'] = join(OUT, name)
 525         config['family'] = family['name']
 526         config['GENERATED_MARKER'] = GENERATED_MARKER
 527
 528         # replace dist specific pkgs
 529         replace = config.get('replace', {})
 530         pkgs = []
 531         for pkg in family['pkgs']:
 532             pkg = replace.get(pkg, pkg)  # replace if exists or get self
 533             if pkg:
 534                 pkgs.append(pkg)
 535         pkgs.sort()
 536
 537         lines = ['  - {}'.format(pkg) for pkg in pkgs]
 538         config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
 539
 540         sep = ' \\' + os.linesep + '    '
 541         config['pkgs'] = sep.join(pkgs)
 542
 543         # get dist bootstrap template or fall back to family default
 544         bootstrap_template = config.get('bootstrap', family['bootstrap'])
 545         config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
 546         config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
 547
 548         config['Dockerfile'] = DOCKERFILE.format(**config).strip()
 549         # keep the indent, no strip
 550         config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
 551
 552         dists[name] = config
 553     return dists
 554
 555
 556 # expanded config for dists
 557 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
 558 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
 559
 560 # assemble all together
 561 DISTS = {}
 562 DISTS.update(DEB_DISTS_EXP)
 563 DISTS.update(RPM_DISTS_EXP)
 564
 565
 566 def render_vagrantfile(dists):
 567     """
 568     Render all snippets for each dist into global Vagrantfile.
 569
 570     Vagrant supports multiple vms in one Vagrantfile.
 571     This make it easier to manage the fleet, e.g:
 572
 573     start all: vagrant up
 574     start one: vagrant up ubuntu1804
 575
 576     All other commands apply to above syntax, e.g.: status, destroy, provision
 577     """
 578     # sort dists by name and put all vagrantfile snippets together
 579     snippets = [
 580         dists[dist]['vagrantfile_snippet']
 581         for dist in sorted(dists.keys())]
 582
 583     return VAGRANTFILE_GLOBAL.format(
 584             vagrantfile_snippets=''.join(snippets),
 585             GENERATED_MARKER=GENERATED_MARKER
 586             )
 587
 588
 589 VAGRANTFILE = render_vagrantfile(DISTS)
 590
 591
 592 # data we need to expose
 593 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']