bootstrap/config.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) Catalyst.Net Ltd 2019
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """
  19 Manage dependencies and bootstrap environments for Samba.
  20
  21 Config file for packages and templates.
  22
  23 Update the lists in this file to require new packages in the
  24 container images used in GitLab CI
  25
  26 Author: Joe Guo <joeg@catalyst.net.nz>
  27 """
  28 import os
  29 from os.path import abspath, dirname, join
  30 HERE = abspath(dirname(__file__))
  31 # output dir for rendered files
  32 OUT = join(HERE, 'generated-dists')
  33
  34
  35 # pkgs with same name in all packaging systems
  36 COMMON = [
  37     'acl',
  38     'attr',
  39     'autoconf',
  40     'binutils',
  41     'bison',
  42     'ccache',
  43     'curl',
  44     'chrpath',
  45     'flex',
  46     'gcc',
  47     'gdb',
  48     'git',
  49     'gzip',
  50     'hostname',
  51     'htop',
  52     'jq',
  53     'lcov',
  54     'make',
  55     'patch',
  56     'perl',
  57     'psmisc',  # for pstree in test
  58     'rng-tools',
  59     'rsync',
  60     'sed',
  61     'sudo',  # docker images has no sudo by default
  62     'tar',
  63     'tree',
  64     'wget',
  65 ]
  66
  67
  68 # define pkgs for all packaging systems in parallel
  69 # make it easier to find missing ones
  70 # use latest ubuntu and fedora as defaults
  71 # deb, rpm, ...
  72 PKGS = [
  73     # NAME1-dev, NAME2-devel
  74     ('lmdb-utils', 'lmdb'),
  75     ('mingw-w64', 'mingw64-gcc'),
  76     ('zlib1g-dev', 'zlib-devel'),
  77     ('libbsd-dev', 'libbsd-devel'),
  78     ('liburing-dev', 'liburing-devel'),
  79     ('libarchive-dev', 'libarchive-devel'),
  80     ('libblkid-dev', 'libblkid-devel'),
  81     ('libcap-dev', 'libcap-devel'),
  82     ('libacl1-dev', 'libacl-devel'),
  83     ('libattr1-dev', 'libattr-devel'),
  84
  85     # libNAME1-dev, NAME2-devel
  86     ('libpopt-dev', 'popt-devel'),
  87     ('libreadline-dev', 'readline-devel'),
  88     ('libjansson-dev', 'jansson-devel'),
  89     ('liblmdb-dev', 'lmdb-devel'),
  90     ('libncurses5-dev', 'ncurses-devel'),
  91     # NOTE: Debian 7+ or Ubuntu 16.04+
  92     ('libsystemd-dev', 'systemd-devel'),
  93     ('libkrb5-dev', 'krb5-devel'),
  94     ('libldap2-dev', 'openldap-devel'),
  95     ('libcups2-dev', 'cups-devel'),
  96     ('libpam0g-dev', 'pam-devel'),
  97     ('libgpgme11-dev', 'gpgme-devel'),
  98     # NOTE: Debian 8+ and Ubuntu 14.04+
  99     ('libgnutls28-dev', 'gnutls-devel'),
 100     ('libtasn1-bin', 'libtasn1-tools'),
 101     ('libtasn1-dev', 'libtasn1-devel'),
 102     ('', 'quota-devel'),
 103     ('uuid-dev', 'libuuid-devel'),
 104     ('libjs-jquery', ''),
 105     ('libavahi-common-dev', 'avahi-devel'),
 106     ('libdbus-1-dev', 'dbus-devel'),
 107     ('libpcap-dev', 'libpcap-devel'),
 108     ('libunwind-dev', 'libunwind-devel'),  # for back trace
 109     ('libglib2.0-dev', 'glib2-devel'),
 110     ('libicu-dev', 'libicu-devel'),
 111     ('heimdal-multidev', ''),
 112
 113     # NAME1, NAME2
 114     # for debian, locales provide locale support with language packs
 115     # ubuntu split language packs to language-pack-xx
 116     # for centos, glibc-common provide locale support with language packs
 117     # fedora split language packs  to glibc-langpack-xx
 118     ('locales', 'glibc-common'),  # required for locale
 119     ('language-pack-en', 'glibc-langpack-en'),  # we need en_US.UTF-8
 120     ('bind9utils', 'bind-utils'),
 121     ('dnsutils', ''),
 122     ('xsltproc', 'libxslt'),
 123     ('krb5-user', 'krb5-workstation'),
 124     ('krb5-config', ''),
 125     ('krb5-kdc', 'krb5-server'),
 126     ('apt-utils', 'yum-utils'),
 127     ('pkg-config', 'pkgconfig'),
 128     ('procps', 'procps-ng'),  # required for the free cmd in tests
 129     ('lsb-release', 'lsb-release'),  # we need lsb_relase to show info
 130     ('', 'rpcgen'),  # required for test
 131     # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
 132     ('', 'libtirpc-devel'),  # for <rpc/rpc.h> header on fedora
 133     ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
 134     ('mawk', 'gawk'),
 135     ('', 'mold'),
 136
 137     ('python3', 'python3'),
 138     ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
 139     ('python3-dev', 'python3-devel'),
 140     ('python3-dbg', ''),
 141     ('python3-iso8601', 'python3-iso8601'),
 142     ('python3-gpg', 'python3-gpg'),  # defaults to ubuntu/fedora latest
 143     ('python3-markdown', 'python3-markdown'),
 144     ('python3-dnspython', 'python3-dns'),
 145     ('python3-pexpect', ''),  # for wintest only
 146     ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
 147     ('python3-setproctitle', 'python3-setproctitle'),
 148     ('python3-requests', 'python3-requests'), # for cert auto enroll
 149
 150     ('', 'python3-libsemanage'),
 151     ('', 'python3-policycoreutils'),
 152
 153     # perl
 154     ('libparse-yapp-perl', 'perl-Parse-Yapp'),
 155     ('libjson-perl', 'perl-JSON'),
 156     ('', 'perl-JSON-Parse'),
 157     ('perl-modules', ''),
 158     ('', 'perl-FindBin'),
 159     ('', 'perl-Archive-Tar'),
 160     ('', 'perl-ExtUtils-MakeMaker'),
 161     ('', 'perl-Test-Base'),
 162     ('', 'perl-generators'),
 163     ('', 'perl-interpreter'),
 164
 165     # fs
 166     ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
 167     ('', 'glusterfs-api-devel'),
 168     ('glusterfs-common', 'glusterfs-devel'),
 169     ('libcephfs-dev', 'libcephfs-devel'),
 170
 171     # spotlight
 172     ('libtracker-sparql-2.0-dev', 'tracker-devel'),
 173
 174     # misc
 175     # @ means group for rpm, use fedora as rpm default
 176     ('build-essential', '@development-tools'),
 177     ('debhelper', ''),
 178     # rpm has no pkg for docbook-xml
 179     ('docbook-xml', 'docbook-dtds'),
 180     ('docbook-xsl', 'docbook-style-xsl'),
 181     ('', 'keyutils-libs-devel'),
 182     ('', 'which'),
 183     ('xz-utils', 'xz')
 184 ]
 185
 186
 187 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
 188 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
 189
 190 GENERATED_MARKER = r"""
 191 #
 192 # This file is generated by 'bootstrap/template.py --render'
 193 # See also bootstrap/config.py
 194 #
 195 """
 196
 197
 198 APT_BOOTSTRAP = r"""
 199 #!/bin/bash
 200 {GENERATED_MARKER}
 201 set -xueo pipefail
 202
 203 export DEBIAN_FRONTEND=noninteractive
 204 apt-get -y update
 205
 206 apt-get -y install \
 207     {pkgs}
 208
 209 apt-get -y autoremove
 210 apt-get -y autoclean
 211 apt-get -y clean
 212 """
 213
 214
 215 YUM_BOOTSTRAP = r"""
 216 #!/bin/bash
 217 {GENERATED_MARKER}
 218 set -xueo pipefail
 219
 220 yum update -y
 221 yum install -y epel-release
 222 yum install -y yum-plugin-copr
 223 yum copr enable -y sergiomb/SambaAD
 224 yum update -y
 225
 226 yum install -y \
 227     {pkgs}
 228
 229 yum clean all
 230
 231 if [ ! -f /usr/bin/python3 ]; then
 232     ln -sf /usr/bin/python3.6 /usr/bin/python3
 233 fi
 234 """
 235
 236 CENTOS8S_YUM_BOOTSTRAP = r"""
 237 #!/bin/bash
 238 {GENERATED_MARKER}
 239 set -xueo pipefail
 240
 241 yum update -y
 242 yum install -y dnf-plugins-core
 243 yum install -y epel-release
 244
 245 yum -v repolist all
 246 yum config-manager --set-enabled powertools -y || \
 247     yum config-manager --set-enabled powertools -y
 248
 249 yum update -y
 250
 251 yum install -y \
 252     --setopt=install_weak_deps=False \
 253     {pkgs}
 254
 255 yum clean all
 256 """
 257
 258 DNF_BOOTSTRAP = r"""
 259 #!/bin/bash
 260 {GENERATED_MARKER}
 261 set -xueo pipefail
 262
 263 dnf update -y
 264
 265 dnf install -y \
 266     --setopt=install_weak_deps=False \
 267     {pkgs}
 268
 269 dnf clean all
 270 """
 271
 272 DNF_BOOTSTRAP_MIT = r"""
 273 #!/bin/bash
 274 {GENERATED_MARKER}
 275 set -xueo pipefail
 276
 277 dnf update -y
 278 dnf install -y dnf-plugins-core
 279 dnf copr -y enable abbra/krb5-test
 280 dnf update -y
 281
 282 dnf install -y \
 283     --setopt=install_weak_deps=False \
 284     {pkgs}
 285
 286 dnf clean all
 287 """
 288
 289 ZYPPER_BOOTSTRAP = r"""
 290 #!/bin/bash
 291 {GENERATED_MARKER}
 292 set -xueo pipefail
 293
 294 zypper --non-interactive refresh
 295 zypper --non-interactive update
 296 zypper --non-interactive install \
 297     --no-recommends \
 298     system-user-nobody \
 299     {pkgs}
 300
 301 zypper --non-interactive clean
 302
 303 if [ -f /usr/lib/mit/bin/krb5-config ]; then
 304     ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
 305 fi
 306 """
 307
 308 # A generic shell script to setup locale
 309 LOCALE_SETUP = r"""
 310 #!/bin/bash
 311 {GENERATED_MARKER}
 312 set -xueo pipefail
 313
 314 # refer to /usr/share/i18n/locales
 315 INPUTFILE=en_US
 316 # refer to /usr/share/i18n/charmaps
 317 CHARMAP=UTF-8
 318 # locale to generate in /usr/lib/locale
 319 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
 320 LOCALE=$INPUTFILE.utf8
 321
 322 # if locale is already correct, exit
 323 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
 324
 325 # if locale not available, generate locale into /usr/lib/locale
 326 if ! ( locale --all-locales | grep -i $LOCALE )
 327 then
 328     # no-archive means create its own dir
 329     localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
 330 fi
 331
 332 # update locale conf and global env file
 333 # set both LC_ALL and LANG for safe
 334
 335 # update conf for Debian family
 336 FILE=/etc/default/locale
 337 if [ -f $FILE ]
 338 then
 339     echo LC_ALL="$LOCALE" > $FILE
 340     echo LANG="$LOCALE" >> $FILE
 341 fi
 342
 343 # update conf for RedHat family
 344 FILE=/etc/locale.conf
 345 if [ -f $FILE ]
 346 then
 347     # LC_ALL is not valid in this file, set LANG only
 348     echo LANG="$LOCALE" > $FILE
 349 fi
 350
 351 # update global env file
 352 FILE=/etc/environment
 353 if [ -f $FILE ]
 354 then
 355     # append LC_ALL if not exist
 356     grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
 357     # append LANG if not exist
 358     grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
 359 fi
 360 """
 361
 362
 363 DOCKERFILE = r"""
 364 {GENERATED_MARKER}
 365 FROM {docker_image}
 366
 367 # pass in with --build-arg while build
 368 ARG SHA1SUM
 369 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
 370
 371 ADD *.sh /tmp/
 372 # need root permission, do it before USER samba
 373 RUN /tmp/bootstrap.sh && /tmp/locale.sh
 374
 375 # if ld.gold exists, force link it to ld
 376 RUN set -x; ! LD_GOLD=$(which ld.gold) || {{ LD=$(which ld) && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"; }}
 377 # if ld.mold exists, force link it to ld (prefer mold over gold! ;-)
 378 RUN set -x; ! LD_MOLD=$(which ld.mold) || {{ LD=$(which ld) && ln -sf $LD_MOLD $LD && test -x $LD && echo "$LD is now $LD_MOLD"; }}
 379
 380 # make test can not work with root, so we have to create a new user
 381 RUN useradd -m -U -s /bin/bash samba && \
 382     mkdir -p /etc/sudoers.d && \
 383     echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
 384
 385 USER samba
 386 WORKDIR /home/samba
 387 # samba tests rely on this
 388 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
 389 """
 390
 391 # Vagrantfile snippet for each dist
 392 VAGRANTFILE_SNIPPET = r"""
 393     config.vm.define "{name}" do |v|
 394         v.vm.box = "{vagrant_box}"
 395         v.vm.hostname = "{name}"
 396         v.vm.provision :shell, path: "{name}/bootstrap.sh"
 397         v.vm.provision :shell, path: "{name}/locale.sh"
 398     end
 399 """
 400
 401 # global Vagrantfile with snippets for all dists
 402 VAGRANTFILE_GLOBAL = r"""
 403 {GENERATED_MARKER}
 404
 405 Vagrant.configure("2") do |config|
 406     config.ssh.insert_key = false
 407
 408 {vagrantfile_snippets}
 409
 410 end
 411 """
 412
 413
 414 DEB_DISTS = {
 415     'debian11': {
 416         'docker_image': 'debian:11',
 417         'vagrant_box': 'debian/bullseye64',
 418         'replace': {
 419             'language-pack-en': '',   # included in locales
 420         }
 421     },
 422     'ubuntu1804': {
 423         'docker_image': 'ubuntu:18.04',
 424         'vagrant_box': 'ubuntu/bionic64',
 425         'replace': {
 426             'liburing-dev': '',   # not available
 427         }
 428     },
 429     'ubuntu2004': {
 430         'docker_image': 'ubuntu:20.04',
 431         'vagrant_box': 'ubuntu/focal64',
 432         'replace': {
 433             'liburing-dev': '',   # not available
 434         }
 435     },
 436 }
 437
 438
 439 RPM_DISTS = {
 440     'centos7': {
 441         'docker_image': 'centos:7',
 442         'vagrant_box': 'centos/7',
 443         'bootstrap': YUM_BOOTSTRAP,
 444         'replace': {
 445             'lsb-release': 'redhat-lsb',
 446             'python3': 'python36',
 447             'python3-cryptography': 'python36-cryptography',
 448             'python3-devel': 'python36-devel',
 449             'python3-dns': 'python36-dns',
 450             'python3-pyasn1': 'python36-pyasn1',
 451             'python3-gpg': 'python36-gpg',
 452             'python3-iso8601' : 'python36-iso8601',
 453             'python3-markdown': 'python36-markdown',
 454             'python3-requests': 'python36-requests',
 455             # although python36-devel is available
 456             # after epel-release installed
 457             # however, all other python3 pkgs are still python36-ish
 458             'python2-gpg': 'pygpgme',
 459             '@development-tools': '"@Development Tools"',  # add quotes
 460             'glibc-langpack-en': '',  # included in glibc-common
 461             'glibc-locale-source': '',  # included in glibc-common
 462             # update perl core modules on centos
 463             # fix: Can't locate Archive/Tar.pm in @INC
 464             'perl': 'perl-core',
 465             'perl-FindBin': '',
 466             'rpcsvc-proto-devel': '',
 467             'glusterfs-api-devel': '',
 468             'glusterfs-devel': '',
 469             'libcephfs-devel': '',
 470             'gnutls-devel': 'compat-gnutls37-devel',
 471             'liburing-devel': '',   # not available
 472             'python3-setproctitle': 'python36-setproctitle',
 473             'tracker-devel': '', # do not install
 474             'mold': '',
 475         }
 476     },
 477     'centos8s': {
 478         'docker_image': 'quay.io/centos/centos:stream8',
 479         'vagrant_box': 'centos/stream8',
 480         'bootstrap': CENTOS8S_YUM_BOOTSTRAP,
 481         'replace': {
 482             'lsb-release': 'redhat-lsb',
 483             '@development-tools': '"@Development Tools"',  # add quotes
 484             'lcov': '', # does not exist
 485             'perl-JSON-Parse': '', # does not exist?
 486             'perl-Test-Base': 'perl-Test-Simple',
 487             'perl-FindBin': '',
 488             'liburing-devel': '', # not available yet, Add me back, once available!
 489             'mold': '',
 490         }
 491     },
 492     'fedora36': {
 493         'docker_image': 'quay.io/fedora/fedora:36',
 494         'vagrant_box': 'fedora/36-cloud-base',
 495         'bootstrap': DNF_BOOTSTRAP,
 496         'replace': {
 497             'lsb-release': 'redhat-lsb',
 498             'perl-FindBin': '',
 499             'python3-iso8601': 'python3-dateutil',
 500             'libtracker-sparql-2.0-dev': '',  # only tracker 3.x is available
 501         }
 502     },
 503     'f36mit120': {
 504         'docker_image': 'quay.io/fedora/fedora:36',
 505         'vagrant_box': 'fedora/36-cloud-base',
 506         'bootstrap': DNF_BOOTSTRAP_MIT,
 507         'replace': {
 508             'lsb-release': 'redhat-lsb',
 509             'perl-FindBin': '',
 510             'python3-iso8601': 'python3-dateutil',
 511             'libtracker-sparql-2.0-dev': '',  # only tracker 3.x is available
 512         }
 513     },
 514     'opensuse153': {
 515         'docker_image': 'opensuse/leap:15.3',
 516         'vagrant_box': 'opensuse/openSUSE-15.3-x86_64',
 517         'bootstrap': ZYPPER_BOOTSTRAP,
 518         'replace': {
 519             '@development-tools': '',
 520             'dbus-devel': 'dbus-1-devel',
 521             'docbook-style-xsl': 'docbook-xsl-stylesheets',
 522             'glibc-common': 'glibc-locale',
 523             'glibc-locale-source': 'glibc-i18ndata',
 524             'glibc-langpack-en': '',
 525             'jansson-devel': 'libjansson-devel',
 526             'keyutils-libs-devel': 'keyutils-devel',
 527             'krb5-workstation': 'krb5-client',
 528             'python3-libsemanage': 'python2-semanage',
 529             'openldap-devel': 'openldap2-devel',
 530             'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
 531             'perl-JSON-Parse': 'perl-JSON-XS',
 532             'perl-generators': '',
 533             'perl-interpreter': '',
 534             'perl-FindBin': '',
 535             'procps-ng': 'procps',
 536             'python3-iso8601': 'python3-python-dateutil',
 537             'python3-dns': 'python3-dnspython',
 538             'python3-markdown': 'python3-Markdown',
 539             'quota-devel': '',
 540             'glusterfs-api-devel': '',
 541             'libtasn1-tools': '', # asn1Parser is part of libtasn1
 542             'mold': '',
 543         }
 544     }
 545 }
 546
 547
 548 DEB_FAMILY = {
 549     'name': 'deb',
 550     'pkgs': DEB_PKGS,
 551     'bootstrap': APT_BOOTSTRAP,  # family default
 552     'dists': DEB_DISTS,
 553 }
 554
 555
 556 RPM_FAMILY = {
 557     'name': 'rpm',
 558     'pkgs': RPM_PKGS,
 559     'bootstrap': YUM_BOOTSTRAP,  # family default
 560     'dists': RPM_DISTS,
 561 }
 562
 563
 564 YML_HEADER = r"""
 565 ---
 566 packages:
 567 """
 568
 569
 570 def expand_family_dists(family):
 571     dists = {}
 572     for name, config in family['dists'].items():
 573         config = config.copy()
 574         config['name'] = name
 575         config['home'] = join(OUT, name)
 576         config['family'] = family['name']
 577         config['GENERATED_MARKER'] = GENERATED_MARKER
 578
 579         # replace dist specific pkgs
 580         replace = config.get('replace', {})
 581         pkgs = []
 582         for pkg in family['pkgs']:
 583             pkg = replace.get(pkg, pkg)  # replace if exists or get self
 584             if pkg:
 585                 pkgs.append(pkg)
 586         pkgs.sort()
 587
 588         lines = ['  - {}'.format(pkg) for pkg in pkgs]
 589         config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
 590
 591         sep = ' \\' + os.linesep + '    '
 592         config['pkgs'] = sep.join(pkgs)
 593
 594         # get dist bootstrap template or fall back to family default
 595         bootstrap_template = config.get('bootstrap', family['bootstrap'])
 596         config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
 597         config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
 598
 599         config['Dockerfile'] = DOCKERFILE.format(**config).strip()
 600         # keep the indent, no strip
 601         config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
 602
 603         dists[name] = config
 604     return dists
 605
 606
 607 # expanded config for dists
 608 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
 609 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
 610
 611 # assemble all together
 612 DISTS = {}
 613 DISTS.update(DEB_DISTS_EXP)
 614 DISTS.update(RPM_DISTS_EXP)
 615
 616
 617 def render_vagrantfile(dists):
 618     """
 619     Render all snippets for each dist into global Vagrantfile.
 620
 621     Vagrant supports multiple vms in one Vagrantfile.
 622     This make it easier to manage the fleet, e.g:
 623
 624     start all: vagrant up
 625     start one: vagrant up ubuntu1804
 626
 627     All other commands apply to above syntax, e.g.: status, destroy, provision
 628     """
 629     # sort dists by name and put all vagrantfile snippets together
 630     snippets = [
 631         dists[dist]['vagrantfile_snippet']
 632         for dist in sorted(dists.keys())]
 633
 634     return VAGRANTFILE_GLOBAL.format(
 635             vagrantfile_snippets=''.join(snippets),
 636             GENERATED_MARKER=GENERATED_MARKER
 637             )
 638
 639
 640 VAGRANTFILE = render_vagrantfile(DISTS)
 641
 642
 643 # data we need to expose
 644 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']