bootstrap/config.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) Catalyst.Net Ltd 2019
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """
  19 Manage dependencies and bootstrap environments for Samba.
  20
  21 Config file for packages and templates.
  22
  23 Author: Joe Guo <joeg@catalyst.net.nz>
  24 """
  25 import os
  26 from os.path import abspath, dirname, join
  27 HERE = abspath(dirname(__file__))
  28 # output dir for rendered files
  29 OUT = join(HERE, 'generated-dists')
  30
  31
  32 # pkgs with same name in all packaging systems
  33 COMMON = [
  34     'acl',
  35     'attr',
  36     'autoconf',
  37     'binutils',
  38     'bison',
  39     'curl',
  40     'chrpath',
  41     'flex',
  42     'gcc',
  43     'gdb',
  44     'git',
  45     'gzip',
  46     'hostname',
  47     'htop',
  48     'lcov',
  49     'make',
  50     'patch',
  51     'perl',
  52     'psmisc',  # for pstree in test
  53     'rng-tools',
  54     'rsync',
  55     'sed',
  56     'sudo',  # docker images has no sudo by default
  57     'tar',
  58     'tree',
  59 ]
  60
  61
  62 # define pkgs for all packaging systems in parallel
  63 # make it easier to find missing ones
  64 # use latest ubuntu and fedora as defaults
  65 # deb, rpm, ...
  66 PKGS = [
  67     # NAME1-dev, NAME2-devel
  68     ('lmdb-utils', 'lmdb'),
  69     ('mingw-w64', 'mingw64-gcc'),
  70     ('zlib1g-dev', 'zlib-devel'),
  71     ('libbsd-dev', 'libbsd-devel'),
  72     ('liburing-dev', 'liburing-devel'),
  73     ('libarchive-dev', 'libarchive-devel'),
  74     ('libblkid-dev', 'libblkid-devel'),
  75     ('libcap-dev', 'libcap-devel'),
  76     ('libacl1-dev', 'libacl-devel'),
  77     ('libattr1-dev', 'libattr-devel'),
  78
  79     # libNAME1-dev, NAME2-devel
  80     ('libpopt-dev', 'popt-devel'),
  81     ('libreadline-dev', 'readline-devel'),
  82     ('libjansson-dev', 'jansson-devel'),
  83     ('liblmdb-dev', 'lmdb-devel'),
  84     ('libncurses5-dev', 'ncurses-devel'),
  85     # NOTE: Debian 7+ or Ubuntu 16.04+
  86     ('libsystemd-dev', 'systemd-devel'),
  87     ('libkrb5-dev', 'krb5-devel'),
  88     ('libldap2-dev', 'openldap-devel'),
  89     ('libcups2-dev', 'cups-devel'),
  90     ('libpam0g-dev', 'pam-devel'),
  91     ('libgpgme11-dev', 'gpgme-devel'),
  92     # NOTE: Debian 8+ and Ubuntu 14.04+
  93     ('libgnutls28-dev', 'gnutls-devel'),
  94     ('libtasn1-bin', 'libtasn1-tools'),
  95     ('libtasn1-dev', 'libtasn1-devel'),
  96     ('', 'quota-devel'),
  97     ('uuid-dev', 'libuuid-devel'),
  98     ('libjs-jquery', ''),
  99     ('libavahi-common-dev', 'avahi-devel'),
 100     ('libdbus-1-dev', 'dbus-devel'),
 101     ('libpcap-dev', 'libpcap-devel'),
 102     ('libunwind-dev', 'libunwind-devel'),  # for back trace
 103     ('libglib2.0-dev', 'glib2-devel'),
 104     ('libicu-dev', 'libicu-devel'),
 105     ('heimdal-multidev', ''),
 106
 107     # NAME1, NAME2
 108     # for debian, locales provide locale support with language packs
 109     # ubuntu split language packs to language-pack-xx
 110     # for centos, glibc-common provide locale support with language packs
 111     # fedora split language packs  to glibc-langpack-xx
 112     ('locales', 'glibc-common'),  # required for locale
 113     ('language-pack-en', 'glibc-langpack-en'),  # we need en_US.UTF-8
 114     ('bind9utils', 'bind-utils'),
 115     ('dnsutils', ''),
 116     ('xsltproc', 'libxslt'),
 117     ('krb5-user', ''),
 118     ('krb5-config', ''),
 119     ('krb5-kdc', 'krb5-server'),
 120     ('apt-utils', 'yum-utils'),
 121     ('pkg-config', 'pkgconfig'),
 122     ('procps', 'procps-ng'),  # required for the free cmd in tests
 123     ('lsb-release', 'lsb-release'),  # we need lsb_relase to show info
 124     ('', 'rpcgen'),  # required for test
 125     # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
 126     ('', 'libtirpc-devel'),  # for <rpc/rpc.h> header on fedora
 127     ('', 'libnsl2-devel'),  # for <rpcsvc/yp_prot.h> header on fedora
 128     ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
 129     ('mawk', 'gawk'),
 130
 131     ('python3', 'python3'),
 132     ('python3-dev', 'python3-devel'),
 133     ('python3-dbg', ''),
 134     ('python3-iso8601', ''),
 135     ('python3-gpg', 'python3-gpg'),  # defaults to ubuntu/fedora latest
 136     ('python3-markdown', 'python3-markdown'),
 137     ('python3-matplotlib', ''),
 138     ('python3-dnspython', 'python3-dns'),
 139     ('python3-pexpect', ''),  # for wintest only
 140
 141     ('', 'libsemanage-python'),
 142     ('', 'policycoreutils-python'),
 143
 144     # perl
 145     ('libparse-yapp-perl', 'perl-Parse-Yapp'),
 146     ('libjson-perl', 'perl-JSON-Parse'),
 147     ('perl-modules', ''),
 148     ('', 'perl-Archive-Tar'),
 149     ('', 'perl-ExtUtils-MakeMaker'),
 150     ('', 'perl-Test-Base'),
 151     ('', 'perl-generators'),
 152     ('', 'perl-interpreter'),
 153
 154     # fs
 155     ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
 156     ('', 'glusterfs-api-devel'),
 157     ('glusterfs-common', 'glusterfs-devel'),
 158     ('libcephfs-dev', 'libcephfs-devel'),
 159
 160     # misc
 161     # @ means group for rpm, use fedora as rpm default
 162     ('build-essential', '@development-tools'),
 163     ('debhelper', ''),
 164     # rpm has no pkg for docbook-xml
 165     ('docbook-xml', 'docbook-dtds'),
 166     ('docbook-xsl', 'docbook-style-xsl'),
 167     ('', 'keyutils-libs-devel'),
 168     ('', 'which'),
 169 ]
 170
 171
 172 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
 173 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
 174
 175 GENERATED_MARKER = r"""
 176 #
 177 # This file is generated by 'bootstrap/template.py --render'
 178 # See also bootstrap/config.py
 179 #
 180 """
 181
 182
 183 APT_BOOTSTRAP = r"""
 184 #!/bin/bash
 185 {GENERATED_MARKER}
 186 set -xueo pipefail
 187
 188 export DEBIAN_FRONTEND=noninteractive
 189 apt-get -y update
 190
 191 apt-get -y install \
 192     {pkgs}
 193
 194 apt-get -y autoremove
 195 apt-get -y autoclean
 196 apt-get -y clean
 197 """
 198
 199
 200 YUM_BOOTSTRAP = r"""
 201 #!/bin/bash
 202 {GENERATED_MARKER}
 203 set -xueo pipefail
 204
 205 yum update -y
 206 yum install -y epel-release
 207 yum install -y yum-plugin-copr
 208 yum copr enable -y sergiomb/SambaAD
 209 yum update -y
 210
 211 yum install -y \
 212     {pkgs}
 213
 214 yum clean all
 215
 216 if [ ! -f /usr/bin/python3 ]; then
 217     ln -sf /usr/bin/python3.6 /usr/bin/python3
 218 fi
 219 """
 220
 221 CENTOS8_YUM_BOOTSTRAP = r"""
 222 #!/bin/bash
 223 {GENERATED_MARKER}
 224 set -xueo pipefail
 225
 226 yum update -y
 227 yum install -y dnf-plugins-core
 228 yum install -y epel-release
 229 yum config-manager --set-enabled PowerTools -y
 230 yum update -y
 231
 232 yum install -y \
 233     --setopt=install_weak_deps=False \
 234     {pkgs}
 235
 236 yum clean all
 237 """
 238
 239 DNF_BOOTSTRAP = r"""
 240 #!/bin/bash
 241 {GENERATED_MARKER}
 242 set -xueo pipefail
 243
 244 dnf update -y
 245
 246 dnf install -y \
 247     --setopt=install_weak_deps=False \
 248     {pkgs}
 249
 250 dnf clean all
 251 """
 252
 253 ZYPPER_BOOTSTRAP = r"""
 254 #!/bin/bash
 255 {GENERATED_MARKER}
 256 set -xueo pipefail
 257
 258 zypper --non-interactive refresh
 259 zypper --non-interactive update
 260 zypper --non-interactive install \
 261     --no-recommends \
 262     system-user-nobody \
 263     {pkgs}
 264
 265 zypper --non-interactive clean
 266
 267 if [ -f /usr/lib/mit/bin/krb5-config ]; then
 268     ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
 269 fi
 270 """
 271
 272 # A generic shell script to setup locale
 273 LOCALE_SETUP = r"""
 274 #!/bin/bash
 275 {GENERATED_MARKER}
 276 set -xueo pipefail
 277
 278 # refer to /usr/share/i18n/locales
 279 INPUTFILE=en_US
 280 # refer to /usr/share/i18n/charmaps
 281 CHARMAP=UTF-8
 282 # locale to generate in /usr/lib/locale
 283 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
 284 LOCALE=$INPUTFILE.utf8
 285
 286 # if locale is already correct, exit
 287 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
 288
 289 # if locale not available, generate locale into /usr/lib/locale
 290 if ! ( locale --all-locales | grep -i $LOCALE )
 291 then
 292     # no-archive means create its own dir
 293     localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
 294 fi
 295
 296 # update locale conf and global env file
 297 # set both LC_ALL and LANG for safe
 298
 299 # update conf for Debian family
 300 FILE=/etc/default/locale
 301 if [ -f $FILE ]
 302 then
 303     echo LC_ALL="$LOCALE" > $FILE
 304     echo LANG="$LOCALE" >> $FILE
 305 fi
 306
 307 # update conf for RedHat family
 308 FILE=/etc/locale.conf
 309 if [ -f $FILE ]
 310 then
 311     # LC_ALL is not valid in this file, set LANG only
 312     echo LANG="$LOCALE" > $FILE
 313 fi
 314
 315 # update global env file
 316 FILE=/etc/environment
 317 if [ -f $FILE ]
 318 then
 319     # append LC_ALL if not exist
 320     grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
 321     # append LANG if not exist
 322     grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
 323 fi
 324 """
 325
 326
 327 DOCKERFILE = r"""
 328 {GENERATED_MARKER}
 329 FROM {docker_image}
 330
 331 # pass in with --build-arg while build
 332 ARG SHA1SUM
 333 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
 334
 335 ADD *.sh /tmp/
 336 # need root permission, do it before USER samba
 337 RUN /tmp/bootstrap.sh && /tmp/locale.sh
 338
 339 # if ld.gold exists, force link it to ld
 340 RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"
 341
 342 # make test can not work with root, so we have to create a new user
 343 RUN useradd -m -U -s /bin/bash samba && \
 344     mkdir -p /etc/sudoers.d && \
 345     echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
 346
 347 USER samba
 348 WORKDIR /home/samba
 349 # samba tests rely on this
 350 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
 351 """
 352
 353 # Vagrantfile snippet for each dist
 354 VAGRANTFILE_SNIPPET = r"""
 355     config.vm.define "{name}" do |v|
 356         v.vm.box = "{vagrant_box}"
 357         v.vm.hostname = "{name}"
 358         v.vm.provision :shell, path: "{name}/bootstrap.sh"
 359         v.vm.provision :shell, path: "{name}/locale.sh"
 360     end
 361 """
 362
 363 # global Vagrantfile with snippets for all dists
 364 VAGRANTFILE_GLOBAL = r"""
 365 {GENERATED_MARKER}
 366
 367 Vagrant.configure("2") do |config|
 368     config.ssh.insert_key = false
 369
 370 {vagrantfile_snippets}
 371
 372 end
 373 """
 374
 375
 376 DEB_DISTS = {
 377     'debian7': {
 378         'docker_image': 'debian:7',
 379         'vagrant_box': 'debian/wheezy64',
 380         'replace': {
 381             'libgnutls28-dev': 'libgnutls-dev',
 382             'libsystemd-dev': '',  # not available, remove
 383             'lmdb-utils': '',  # not available, remove
 384             'liblmdb-dev': '',  # not available, remove
 385             'python-gpg': 'python-gpgme',
 386             'python3-gpg': '',  # no python3 gpg pkg available, remove
 387             'language-pack-en': '',   # included in locales
 388             'liburing-dev': '',   # not available
 389         }
 390     },
 391     'debian8': {
 392         'docker_image': 'debian:8',
 393         'vagrant_box': 'debian/jessie64',
 394         'replace': {
 395             'python-gpg': 'python-gpgme',
 396             'python3-gpg': 'python3-gpgme',
 397             'language-pack-en': '',   # included in locales
 398             'liburing-dev': '',   # not available
 399         }
 400     },
 401     'debian9': {
 402         'docker_image': 'debian:9',
 403         'vagrant_box': 'debian/stretch64',
 404         'replace': {
 405             'language-pack-en': '',   # included in locales
 406             'liburing-dev': '',   # not available
 407         }
 408     },
 409     'debian10': {
 410         'docker_image': 'debian:10',
 411         'vagrant_box': 'debian/buster64',
 412         'replace': {
 413             'language-pack-en': '',   # included in locales
 414             'liburing-dev': '',   # not available
 415         }
 416     },
 417     'ubuntu1404': {
 418         'docker_image': 'ubuntu:14.04',
 419         'vagrant_box': 'ubuntu/trusty64',
 420         'replace': {
 421             'libsystemd-dev': '',  # remove
 422             'libgnutls28-dev': 'libgnutls-dev',
 423             'python-gpg': 'python-gpgme',
 424             'python3-gpg': 'python3-gpgme',
 425             'lmdb-utils': 'lmdb-utils/trusty-backports',
 426             'liblmdb-dev': 'liblmdb-dev/trusty-backports',
 427             'libunwind-dev': 'libunwind8-dev',
 428             'glusterfs-common': '',
 429             'libcephfs-dev': '',
 430             'liburing-dev': '',   # not available
 431         }
 432     },
 433     'ubuntu1604': {
 434         'docker_image': 'ubuntu:16.04',
 435         'vagrant_box': 'ubuntu/xenial64',
 436         'replace': {
 437             'python-gpg': 'python-gpgme',
 438             'python3-gpg': 'python3-gpgme',
 439             'glusterfs-common': '',
 440             'libcephfs-dev': '',
 441             'liburing-dev': '',   # not available
 442         }
 443     },
 444     'ubuntu1804': {
 445         'docker_image': 'ubuntu:18.04',
 446         'vagrant_box': 'ubuntu/bionic64',
 447         'replace': {
 448             'liburing-dev': '',   # not available
 449         }
 450     },
 451 }
 452
 453
 454 RPM_DISTS = {
 455     'centos6': {
 456         'docker_image': 'centos:6',
 457         'vagrant_box': 'centos/6',
 458         'bootstrap': YUM_BOOTSTRAP,
 459         'replace': {
 460             'lsb-release': 'redhat-lsb',
 461             'python3': 'python36',
 462             'python3-devel': 'python36-devel',
 463             'python2-gpg': 'pygpgme',
 464             'python3-gpg': '',  # no python3-gpg yet
 465             '@development-tools': '"@Development Tools"',  # add quotes
 466             'glibc-langpack-en': '',  # included in glibc-common
 467             'glibc-locale-source': '',  # included in glibc-common
 468             'procps-ng': 'procps',  # centos6 still use old name
 469             # update perl core modules on centos
 470             # fix: Can't locate Archive/Tar.pm in @INC
 471             'perl': 'perl-core',
 472             'rpcsvc-proto-devel': '',
 473             'glusterfs-api-devel': '',
 474             'glusterfs-devel': '',
 475             'libcephfs-devel': '',
 476             'liburing-devel': '',   # not available
 477         }
 478     },
 479     'centos7': {
 480         'docker_image': 'centos:7',
 481         'vagrant_box': 'centos/7',
 482         'bootstrap': YUM_BOOTSTRAP,
 483         'replace': {
 484             'lsb-release': 'redhat-lsb',
 485             'python3': 'python36',
 486             'python3-crypto': 'python36-crypto',
 487             'python3-devel': 'python36-devel',
 488             'python3-dns': 'python36-dns',
 489             'python3-gpg': 'python36-gpg',
 490             'python3-iso8601' : 'python36-iso8601',
 491             'python3-markdown': 'python36-markdown',
 492             # although python36-devel is available
 493             # after epel-release installed
 494             # however, all other python3 pkgs are still python36-ish
 495             'python2-gpg': 'pygpgme',
 496             'python3-gpg': '',  # no python3-gpg yet
 497             '@development-tools': '"@Development Tools"',  # add quotes
 498             'glibc-langpack-en': '',  # included in glibc-common
 499             'glibc-locale-source': '',  # included in glibc-common
 500             # update perl core modules on centos
 501             # fix: Can't locate Archive/Tar.pm in @INC
 502             'perl': 'perl-core',
 503             'rpcsvc-proto-devel': '',
 504             'glusterfs-api-devel': '',
 505             'glusterfs-devel': '',
 506             'libcephfs-devel': '',
 507             'gnutls-devel': 'compat-gnutls34-devel',
 508             'liburing-devel': '',   # not available
 509         }
 510     },
 511     'centos8': {
 512         'docker_image': 'centos:8',
 513         'vagrant_box': 'centos/8',
 514         'bootstrap': CENTOS8_YUM_BOOTSTRAP,
 515         'replace': {
 516             'lsb-release': 'redhat-lsb',
 517             '@development-tools': '"@Development Tools"',  # add quotes
 518             'libsemanage-python': 'python3-libsemanage',
 519             'lcov': '', # does not exist
 520             'perl-JSON-Parse': '', # does not exist?
 521             'perl-Test-Base': 'perl-Test-Simple',
 522             'policycoreutils-python': 'python3-policycoreutils',
 523             'python3-crypto': '',
 524             'quota-devel': '', # FIXME: Add me back, once available!
 525             'liburing-devel': '', # not available yet, Add me back, once available!
 526         }
 527     },
 528     'fedora29': {
 529         'docker_image': 'fedora:29',
 530         'vagrant_box': 'fedora/29-cloud-base',
 531         'bootstrap': DNF_BOOTSTRAP,
 532         'replace': {
 533             'lsb-release': 'redhat-lsb',
 534             'liburing-devel': '',   # not available
 535         }
 536     },
 537     'fedora30': {
 538         'docker_image': 'fedora:30',
 539         'vagrant_box': 'fedora/30-cloud-base',
 540         'bootstrap': DNF_BOOTSTRAP,
 541         'replace': {
 542             'lsb-release': 'redhat-lsb',
 543             'liburing-devel': '',   # not available
 544         }
 545     },
 546     'fedora31': {
 547         'docker_image': 'fedora:31',
 548         'vagrant_box': 'fedora/31-cloud-base',
 549         'bootstrap': DNF_BOOTSTRAP,
 550         'replace': {
 551             'lsb-release': 'redhat-lsb',
 552             'libsemanage-python': 'python3-libsemanage',
 553             'policycoreutils-python': 'python3-policycoreutils',
 554         }
 555     },
 556     'opensuse150': {
 557         'docker_image': 'opensuse/leap:15.0',
 558         'vagrant_box': 'opensuse/openSUSE-15.0-x86_64',
 559         'bootstrap': ZYPPER_BOOTSTRAP,
 560         'replace': {
 561             '@development-tools': '',
 562             'dbus-devel': 'dbus-1-devel',
 563             'docbook-style-xsl': 'docbook-xsl-stylesheets',
 564             'glibc-common': 'glibc-locale',
 565             'glibc-locale-source': 'glibc-i18ndata',
 566             'glibc-langpack-en': '',
 567             'jansson-devel': 'libjansson-devel',
 568             'keyutils-libs-devel': 'keyutils-devel',
 569             'krb5-workstation': 'krb5-client',
 570             'libnsl2-devel': 'libnsl-devel',
 571             'libsemanage-python': 'python2-semanage',
 572             'openldap-devel': 'openldap2-devel',
 573             'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
 574             'perl-JSON-Parse': 'perl-JSON-XS',
 575             'perl-generators': '',
 576             'perl-interpreter': '',
 577             'procps-ng': 'procps',
 578             'python-dns': 'python2-dnspython',
 579             'python3-crypto': 'python3-pycrypto',
 580             'python3-dns': 'python3-dnspython',
 581             'python3-markdown': 'python3-Markdown',
 582             'quota-devel': '',
 583             'glusterfs-api-devel': '',
 584             'libtasn1-tools': '', # asn1Parser is part of libtasn1
 585             'mingw64-gcc': '', # doesn't exist
 586             'liburing-devel': '',   # not available
 587         }
 588     },
 589     'opensuse151': {
 590         'docker_image': 'opensuse/leap:15.1',
 591         'vagrant_box': 'opensuse/openSUSE-15.1-x86_64',
 592         'bootstrap': ZYPPER_BOOTSTRAP,
 593         'replace': {
 594             '@development-tools': '',
 595             'dbus-devel': 'dbus-1-devel',
 596             'docbook-style-xsl': 'docbook-xsl-stylesheets',
 597             'glibc-common': 'glibc-locale',
 598             'glibc-locale-source': 'glibc-i18ndata',
 599             'glibc-langpack-en': '',
 600             'jansson-devel': 'libjansson-devel',
 601             'keyutils-libs-devel': 'keyutils-devel',
 602             'krb5-workstation': 'krb5-client',
 603             'libnsl2-devel': 'libnsl-devel',
 604             'libsemanage-python': 'python2-semanage',
 605             'openldap-devel': 'openldap2-devel',
 606             'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
 607             'perl-JSON-Parse': 'perl-JSON-XS',
 608             'perl-generators': '',
 609             'perl-interpreter': '',
 610             'procps-ng': 'procps',
 611             'python-dns': 'python2-dnspython',
 612             'python3-crypto': 'python3-pycrypto',
 613             'python3-dns': 'python3-dnspython',
 614             'python3-markdown': 'python3-Markdown',
 615             'quota-devel': '',
 616             'glusterfs-api-devel': '',
 617             'libtasn1-tools': '', # asn1Parser is part of libtasn1
 618             'mingw64-gcc': '', # doesn't exist
 619             'liburing-devel': '',   # not available, will be added in 15.2
 620         }
 621     }
 622 }
 623
 624
 625 DEB_FAMILY = {
 626     'name': 'deb',
 627     'pkgs': DEB_PKGS,
 628     'bootstrap': APT_BOOTSTRAP,  # family default
 629     'dists': DEB_DISTS,
 630 }
 631
 632
 633 RPM_FAMILY = {
 634     'name': 'rpm',
 635     'pkgs': RPM_PKGS,
 636     'bootstrap': YUM_BOOTSTRAP,  # family default
 637     'dists': RPM_DISTS,
 638 }
 639
 640
 641 YML_HEADER = r"""
 642 ---
 643 packages:
 644 """
 645
 646
 647 def expand_family_dists(family):
 648     dists = {}
 649     for name, config in family['dists'].items():
 650         config = config.copy()
 651         config['name'] = name
 652         config['home'] = join(OUT, name)
 653         config['family'] = family['name']
 654         config['GENERATED_MARKER'] = GENERATED_MARKER
 655
 656         # replace dist specific pkgs
 657         replace = config.get('replace', {})
 658         pkgs = []
 659         for pkg in family['pkgs']:
 660             pkg = replace.get(pkg, pkg)  # replace if exists or get self
 661             if pkg:
 662                 pkgs.append(pkg)
 663         pkgs.sort()
 664
 665         lines = ['  - {}'.format(pkg) for pkg in pkgs]
 666         config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
 667
 668         sep = ' \\' + os.linesep + '    '
 669         config['pkgs'] = sep.join(pkgs)
 670
 671         # get dist bootstrap template or fall back to family default
 672         bootstrap_template = config.get('bootstrap', family['bootstrap'])
 673         config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
 674         config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
 675
 676         config['Dockerfile'] = DOCKERFILE.format(**config).strip()
 677         # keep the indent, no strip
 678         config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
 679
 680         dists[name] = config
 681     return dists
 682
 683
 684 # expanded config for dists
 685 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
 686 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
 687
 688 # assemble all together
 689 DISTS = {}
 690 DISTS.update(DEB_DISTS_EXP)
 691 DISTS.update(RPM_DISTS_EXP)
 692
 693
 694 def render_vagrantfile(dists):
 695     """
 696     Render all snippets for each dist into global Vagrantfile.
 697
 698     Vagrant supports multiple vms in one Vagrantfile.
 699     This make it easier to manage the fleet, e.g:
 700
 701     start all: vagrant up
 702     start one: vagrant up ubuntu1804
 703
 704     All other commands apply to above syntax, e.g.: status, destroy, provision
 705     """
 706     # sort dists by name and put all vagrantfile snippets together
 707     snippets = [
 708         dists[dist]['vagrantfile_snippet']
 709         for dist in sorted(dists.keys())]
 710
 711     return VAGRANTFILE_GLOBAL.format(
 712             vagrantfile_snippets=''.join(snippets),
 713             GENERATED_MARKER=GENERATED_MARKER
 714             )
 715
 716
 717 VAGRANTFILE = render_vagrantfile(DISTS)
 718
 719
 720 # data we need to expose
 721 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']