tests/lcitool/refresh

   1 #!/usr/bin/env python3
   2 #
   3 # Re-generate container recipes
   4 #
   5 # This script uses the "lcitool" available from
   6 #
   7 #   https://gitlab.com/libvirt/libvirt-ci
   8 #
   9 # Copyright (c) 2020 Red Hat Inc.
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2
  12 # or (at your option) any later version. See the COPYING file in
  13 # the top-level directory.
  14
  15 import sys
  16 import subprocess
  17
  18 from pathlib import Path
  19
  20 if len(sys.argv) != 1:
  21     print("syntax: %s" % sys.argv[0], file=sys.stderr)
  22     sys.exit(1)
  23
  24 self_dir = Path(__file__).parent
  25 src_dir = self_dir.parent.parent
  26 dockerfiles_dir = Path(src_dir, "tests", "docker", "dockerfiles")
  27
  28 lcitool_path = Path(self_dir, "libvirt-ci", "bin", "lcitool")
  29
  30 lcitool_cmd = [lcitool_path, "--data-dir", self_dir]
  31
  32
  33 def atomic_write(filename, content):
  34     tmp = filename.with_suffix(filename.suffix + ".tmp")
  35     try:
  36         with tmp.open("w") as fp:
  37             print(content, file=fp, end="")
  38             tmp.rename(filename)
  39     except Exception as ex:
  40         tmp.unlink()
  41         raise
  42
  43
  44 def generate(filename, cmd, trailer):
  45     print("Generate %s" % filename)
  46     lcitool = subprocess.run(cmd, capture_output=True)
  47
  48     if lcitool.returncode != 0:
  49         raise Exception("Failed to generate %s: %s" % (filename, lcitool.stderr))
  50
  51     content = lcitool.stdout.decode("utf8")
  52     if trailer is not None:
  53         content += trailer
  54     atomic_write(filename, content)
  55
  56 # Optional user setting, this will always be the last thing added
  57 # so maximise the number of layers that are cached
  58 add_user_mapping = [
  59     "# As a final step configure the user (if env is defined)",
  60     "ARG USER",
  61     "ARG UID",
  62     "RUN if [ \"${USER}\" ]; then \\",
  63     "  id ${USER} 2>/dev/null || useradd -u ${UID} -U ${USER}; fi\n"
  64 ]
  65
  66 def generate_dockerfile(host, target, project="qemu", cross=None, trailer=None):
  67     filename = Path(src_dir, "tests", "docker", "dockerfiles", host + ".docker")
  68     cmd = lcitool_cmd + ["dockerfile"]
  69     if cross is not None:
  70         cmd.extend(["--cross", cross])
  71     cmd.extend([target, project])
  72
  73     if trailer is not None:
  74         trailer += "\n".join(add_user_mapping)
  75     else:
  76         trailer = "\n".join(add_user_mapping)
  77
  78     generate(filename, cmd, trailer)
  79
  80
  81 def generate_cirrus(target, trailer=None):
  82     filename = Path(src_dir, ".gitlab-ci.d", "cirrus", target + ".vars")
  83     cmd = lcitool_cmd + ["variables", target, "qemu"]
  84     generate(filename, cmd, trailer)
  85
  86
  87 def generate_pkglist(vm, target):
  88     filename = Path(src_dir, "tests", "vm", "generated", vm + ".json")
  89     cmd = lcitool_cmd + ["variables", "--format", "json", target, "qemu"]
  90     generate(filename, cmd, None)
  91
  92
  93 # Netmap still needs to be manually built as it is yet to be packaged
  94 # into a distro. We also add cscope and gtags which are used in the CI
  95 # test
  96 debian12_extras = [
  97     "# netmap/cscope/global\n",
  98     "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n",
  99     "  apt install -y --no-install-recommends \\\n",
 100     "  cscope\\\n",
 101     "  global\\\n",
 102     "  linux-headers-generic\n",
 103     "RUN git clone https://github.com/luigirizzo/netmap.git /usr/src/netmap\n",
 104     "RUN cd /usr/src/netmap && git checkout v11.3\n",
 105     "RUN cd /usr/src/netmap/LINUX && \\\n",
 106     "  ./configure --no-drivers --no-apps \\\n",
 107     "  --kernel-dir=$(ls -d /usr/src/linux-headers-*-$(dpkg --print-architecture)) \\\n",
 108     "  && make install\n",
 109     "ENV QEMU_CONFIGURE_OPTS --enable-netmap\n"
 110 ]
 111
 112
 113 def cross_build(prefix, targets):
 114     conf = "ENV QEMU_CONFIGURE_OPTS --cross-prefix=%s\n" % (prefix)
 115     targets = "ENV DEF_TARGET_LIST %s\n" % (targets)
 116     return "".join([conf, targets])
 117
 118 #
 119 # Update all the various build configurations.
 120 # Please keep each group sorted alphabetically for easy reading.
 121 #
 122
 123 try:
 124     #
 125     # Standard native builds
 126     #
 127     generate_dockerfile("alpine", "alpine-318")
 128     generate_dockerfile("centos8", "centos-stream-8")
 129     generate_dockerfile("debian", "debian-12",
 130                         trailer="".join(debian12_extras))
 131     generate_dockerfile("fedora", "fedora-38")
 132     generate_dockerfile("opensuse-leap", "opensuse-leap-15")
 133     generate_dockerfile("ubuntu2004", "ubuntu-2004")
 134     generate_dockerfile("ubuntu2204", "ubuntu-2204")
 135
 136     #
 137     # Cross compiling builds
 138     #
 139     generate_dockerfile("debian-amd64-cross", "debian-12",
 140                         cross="x86_64",
 141                         trailer=cross_build("x86_64-linux-gnu-",
 142                                             "x86_64-softmmu,"
 143                                             "x86_64-linux-user,"
 144                                             "i386-softmmu,i386-linux-user"))
 145
 146     generate_dockerfile("debian-arm64-cross", "debian-12",
 147                         cross="aarch64",
 148                         trailer=cross_build("aarch64-linux-gnu-",
 149                                             "aarch64-softmmu,aarch64-linux-user"))
 150
 151     # migration to bookworm stalled: https://lists.debian.org/debian-arm/2023/09/msg00006.html
 152     generate_dockerfile("debian-armel-cross", "debian-11",
 153                         cross="armv6l",
 154                         trailer=cross_build("arm-linux-gnueabi-",
 155                                             "arm-softmmu,arm-linux-user,armeb-linux-user"))
 156
 157     generate_dockerfile("debian-armhf-cross", "debian-12",
 158                         cross="armv7l",
 159                         trailer=cross_build("arm-linux-gnueabihf-",
 160                                             "arm-softmmu,arm-linux-user"))
 161
 162     generate_dockerfile("debian-i686-cross", "debian-11",
 163                         cross="i686",
 164                         trailer=cross_build("x86_64-linux-gnu-",
 165                                             "x86_64-softmmu,"
 166                                             "x86_64-linux-user,"
 167                                             "i386-softmmu,i386-linux-user"))
 168
 169     generate_dockerfile("debian-mips64el-cross", "debian-11",
 170                         cross="mips64el",
 171                         trailer=cross_build("mips64el-linux-gnuabi64-",
 172                                             "mips64el-softmmu,mips64el-linux-user"))
 173
 174     generate_dockerfile("debian-mipsel-cross", "debian-11",
 175                         cross="mipsel",
 176                         trailer=cross_build("mipsel-linux-gnu-",
 177                                             "mipsel-softmmu,mipsel-linux-user"))
 178
 179     generate_dockerfile("debian-ppc64el-cross", "debian-12",
 180                         cross="ppc64le",
 181                         trailer=cross_build("powerpc64le-linux-gnu-",
 182                                             "ppc64-softmmu,ppc64-linux-user"))
 183
 184     generate_dockerfile("debian-riscv64-cross", "debian-sid",
 185                         project="qemu-minimal",
 186                         cross="riscv64",
 187                         trailer=cross_build("riscv64-linux-gnu-",
 188                                             "riscv64-softmmu,riscv64-linux-user"))
 189
 190     generate_dockerfile("debian-s390x-cross", "debian-12",
 191                         cross="s390x",
 192                         trailer=cross_build("s390x-linux-gnu-",
 193                                             "s390x-softmmu,s390x-linux-user"))
 194
 195     generate_dockerfile("fedora-win32-cross", "fedora-38",
 196                         cross="mingw32",
 197                         trailer=cross_build("i686-w64-mingw32-",
 198                                             "i386-softmmu"))
 199
 200     generate_dockerfile("fedora-win64-cross", "fedora-38",
 201                         cross="mingw64",
 202                         trailer=cross_build("x86_64-w64-mingw32-",
 203                                             "x86_64-softmmu"))
 204
 205     #
 206     # Cirrus packages lists for GitLab
 207     #
 208     generate_cirrus("freebsd-13")
 209     generate_cirrus("macos-13")
 210     generate_cirrus("macos-14")
 211
 212     #
 213     # VM packages lists
 214     #
 215     generate_pkglist("freebsd", "freebsd-13")
 216
 217     sys.exit(0)
 218 except Exception as ex:
 219     print(str(ex), file=sys.stderr)
 220     sys.exit(1)