python/scripts/mkvenv.py

   1 """
   2 mkvenv - QEMU pyvenv bootstrapping utility
   3
   4 usage: mkvenv [-h] command ...
   5
   6 QEMU pyvenv bootstrapping utility
   7
   8 options:
   9   -h, --help  show this help message and exit
  10
  11 Commands:
  12   command     Description
  13     create    create a venv
  14     post_init
  15               post-venv initialization
  16     ensure    Ensure that the specified package is installed.
  17     ensuregroup
  18               Ensure that the specified package group is installed.
  19
  20 --------------------------------------------------
  21
  22 usage: mkvenv create [-h] target
  23
  24 positional arguments:
  25   target      Target directory to install virtual environment into.
  26
  27 options:
  28   -h, --help  show this help message and exit
  29
  30 --------------------------------------------------
  31
  32 usage: mkvenv post_init [-h]
  33
  34 options:
  35   -h, --help         show this help message and exit
  36
  37 --------------------------------------------------
  38
  39 usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec...
  40
  41 positional arguments:
  42   dep_spec    PEP 508 Dependency specification, e.g. 'meson>=0.61.5'
  43
  44 options:
  45   -h, --help  show this help message and exit
  46   --online    Install packages from PyPI, if necessary.
  47   --dir DIR   Path to vendored packages where we may install from.
  48
  49 --------------------------------------------------
  50
  51 usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group...
  52
  53 positional arguments:
  54   file        pointer to a TOML file
  55   group       section name in the TOML file
  56
  57 options:
  58   -h, --help  show this help message and exit
  59   --online    Install packages from PyPI, if necessary.
  60   --dir DIR   Path to vendored packages where we may install from.
  61
  62 """
  63
  64 # Copyright (C) 2022-2023 Red Hat, Inc.
  65 #
  66 # Authors:
  67 #  John Snow <jsnow@redhat.com>
  68 #  Paolo Bonzini <pbonzini@redhat.com>
  69 #
  70 # This work is licensed under the terms of the GNU GPL, version 2 or
  71 # later. See the COPYING file in the top-level directory.
  72
  73 import argparse
  74 from importlib.metadata import (
  75     Distribution,
  76     EntryPoint,
  77     PackageNotFoundError,
  78     distribution,
  79     version,
  80 )
  81 from importlib.util import find_spec
  82 import logging
  83 import os
  84 from pathlib import Path
  85 import re
  86 import shutil
  87 import site
  88 import subprocess
  89 import sys
  90 import sysconfig
  91 from types import SimpleNamespace
  92 from typing import (
  93     Any,
  94     Dict,
  95     Iterator,
  96     Optional,
  97     Sequence,
  98     Tuple,
  99     Union,
 100 )
 101 import venv
 102
 103
 104 # Try to load distlib, with a fallback to pip's vendored version.
 105 # HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail
 106 # outside the venv or before a potential call to ensurepip in checkpip().
 107 HAVE_DISTLIB = True
 108 try:
 109     import distlib.scripts
 110     import distlib.version
 111 except ImportError:
 112     try:
 113         # Reach into pip's cookie jar.  pylint and flake8 don't understand
 114         # that these imports will be used via distlib.xxx.
 115         from pip._vendor import distlib
 116         import pip._vendor.distlib.scripts  # noqa, pylint: disable=unused-import
 117         import pip._vendor.distlib.version  # noqa, pylint: disable=unused-import
 118     except ImportError:
 119         HAVE_DISTLIB = False
 120
 121 # Try to load tomllib, with a fallback to tomli.
 122 # HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail
 123 # outside the venv or before a potential call to ensurepip in checkpip().
 124 HAVE_TOMLLIB = True
 125 try:
 126     import tomllib
 127 except ImportError:
 128     try:
 129         import tomli as tomllib
 130     except ImportError:
 131         HAVE_TOMLLIB = False
 132
 133 # Do not add any mandatory dependencies from outside the stdlib:
 134 # This script *must* be usable standalone!
 135
 136 DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
 137 logger = logging.getLogger("mkvenv")
 138
 139
 140 def inside_a_venv() -> bool:
 141     """Returns True if it is executed inside of a virtual environment."""
 142     return sys.prefix != sys.base_prefix
 143
 144
 145 class Ouch(RuntimeError):
 146     """An Exception class we can't confuse with a builtin."""
 147
 148
 149 class QemuEnvBuilder(venv.EnvBuilder):
 150     """
 151     An extension of venv.EnvBuilder for building QEMU's configure-time venv.
 152
 153     The primary difference is that it emulates a "nested" virtual
 154     environment when invoked from inside of an existing virtual
 155     environment by including packages from the parent.  Also,
 156     "ensurepip" is replaced if possible with just recreating pip's
 157     console_scripts inside the virtual environment.
 158
 159     Parameters for base class init:
 160       - system_site_packages: bool = False
 161       - clear: bool = False
 162       - symlinks: bool = False
 163       - upgrade: bool = False
 164       - with_pip: bool = False
 165       - prompt: Optional[str] = None
 166       - upgrade_deps: bool = False             (Since 3.9)
 167     """
 168
 169     def __init__(self, *args: Any, **kwargs: Any) -> None:
 170         logger.debug("QemuEnvBuilder.__init__(...)")
 171
 172         # For nested venv emulation:
 173         self.use_parent_packages = False
 174         if inside_a_venv():
 175             # Include parent packages only if we're in a venv and
 176             # system_site_packages was True.
 177             self.use_parent_packages = kwargs.pop(
 178                 "system_site_packages", False
 179             )
 180             # Include system_site_packages only when the parent,
 181             # The venv we are currently in, also does so.
 182             kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES
 183
 184         # ensurepip is slow: venv creation can be very fast for cases where
 185         # we allow the use of system_site_packages. Therefore, ensurepip is
 186         # replaced with our own script generation once the virtual environment
 187         # is setup.
 188         self.want_pip = kwargs.get("with_pip", False)
 189         if self.want_pip:
 190             if (
 191                 kwargs.get("system_site_packages", False)
 192                 and not need_ensurepip()
 193             ):
 194                 kwargs["with_pip"] = False
 195             else:
 196                 check_ensurepip()
 197
 198         super().__init__(*args, **kwargs)
 199
 200         # Make the context available post-creation:
 201         self._context: Optional[SimpleNamespace] = None
 202
 203     def get_parent_libpath(self) -> Optional[str]:
 204         """Return the libpath of the parent venv, if applicable."""
 205         if self.use_parent_packages:
 206             return sysconfig.get_path("purelib")
 207         return None
 208
 209     @staticmethod
 210     def compute_venv_libpath(context: SimpleNamespace) -> str:
 211         """
 212         Compatibility wrapper for context.lib_path for Python < 3.12
 213         """
 214         # Python 3.12+, not strictly necessary because it's documented
 215         # to be the same as 3.10 code below:
 216         if sys.version_info >= (3, 12):
 217             return context.lib_path
 218
 219         # Python 3.10+
 220         if "venv" in sysconfig.get_scheme_names():
 221             lib_path = sysconfig.get_path(
 222                 "purelib", scheme="venv", vars={"base": context.env_dir}
 223             )
 224             assert lib_path is not None
 225             return lib_path
 226
 227         # For Python <= 3.9 we need to hardcode this. Fortunately the
 228         # code below was the same in Python 3.6-3.10, so there is only
 229         # one case.
 230         if sys.platform == "win32":
 231             return os.path.join(context.env_dir, "Lib", "site-packages")
 232         return os.path.join(
 233             context.env_dir,
 234             "lib",
 235             "python%d.%d" % sys.version_info[:2],
 236             "site-packages",
 237         )
 238
 239     def ensure_directories(self, env_dir: DirType) -> SimpleNamespace:
 240         logger.debug("ensure_directories(env_dir=%s)", env_dir)
 241         self._context = super().ensure_directories(env_dir)
 242         return self._context
 243
 244     def create(self, env_dir: DirType) -> None:
 245         logger.debug("create(env_dir=%s)", env_dir)
 246         super().create(env_dir)
 247         assert self._context is not None
 248         self.post_post_setup(self._context)
 249
 250     def post_post_setup(self, context: SimpleNamespace) -> None:
 251         """
 252         The final, final hook. Enter the venv and run commands inside of it.
 253         """
 254         if self.use_parent_packages:
 255             # We're inside of a venv and we want to include the parent
 256             # venv's packages.
 257             parent_libpath = self.get_parent_libpath()
 258             assert parent_libpath is not None
 259             logger.debug("parent_libpath: %s", parent_libpath)
 260
 261             our_libpath = self.compute_venv_libpath(context)
 262             logger.debug("our_libpath: %s", our_libpath)
 263
 264             pth_file = os.path.join(our_libpath, "nested.pth")
 265             with open(pth_file, "w", encoding="UTF-8") as file:
 266                 file.write(parent_libpath + os.linesep)
 267
 268         if self.want_pip:
 269             args = [
 270                 context.env_exe,
 271                 __file__,
 272                 "post_init",
 273             ]
 274             subprocess.run(args, check=True)
 275
 276     def get_value(self, field: str) -> str:
 277         """
 278         Get a string value from the context namespace after a call to build.
 279
 280         For valid field names, see:
 281         https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories
 282         """
 283         ret = getattr(self._context, field)
 284         assert isinstance(ret, str)
 285         return ret
 286
 287
 288 def need_ensurepip() -> bool:
 289     """
 290     Tests for the presence of setuptools and pip.
 291
 292     :return: `True` if we do not detect both packages.
 293     """
 294     # Don't try to actually import them, it's fraught with danger:
 295     # https://github.com/pypa/setuptools/issues/2993
 296     if find_spec("setuptools") and find_spec("pip"):
 297         return False
 298     return True
 299
 300
 301 def check_ensurepip() -> None:
 302     """
 303     Check that we have ensurepip.
 304
 305     Raise a fatal exception with a helpful hint if it isn't available.
 306     """
 307     if not find_spec("ensurepip"):
 308         msg = (
 309             "Python's ensurepip module is not found.\n"
 310             "It's normally part of the Python standard library, "
 311             "maybe your distribution packages it separately?\n"
 312             "Either install ensurepip, or alleviate the need for it in the "
 313             "first place by installing pip and setuptools for "
 314             f"'{sys.executable}'.\n"
 315             "(Hint: Debian puts ensurepip in its python3-venv package.)"
 316         )
 317         raise Ouch(msg)
 318
 319     # ensurepip uses pyexpat, which can also go missing on us:
 320     if not find_spec("pyexpat"):
 321         msg = (
 322             "Python's pyexpat module is not found.\n"
 323             "It's normally part of the Python standard library, "
 324             "maybe your distribution packages it separately?\n"
 325             "Either install pyexpat, or alleviate the need for it in the "
 326             "first place by installing pip and setuptools for "
 327             f"'{sys.executable}'.\n\n"
 328             "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)"
 329         )
 330         raise Ouch(msg)
 331
 332
 333 def make_venv(  # pylint: disable=too-many-arguments
 334     env_dir: Union[str, Path],
 335     system_site_packages: bool = False,
 336     clear: bool = True,
 337     symlinks: Optional[bool] = None,
 338     with_pip: bool = True,
 339 ) -> None:
 340     """
 341     Create a venv using `QemuEnvBuilder`.
 342
 343     This is analogous to the `venv.create` module-level convenience
 344     function that is part of the Python stdblib, except it uses
 345     `QemuEnvBuilder` instead.
 346
 347     :param env_dir: The directory to create/install to.
 348     :param system_site_packages:
 349         Allow inheriting packages from the system installation.
 350     :param clear: When True, fully remove any prior venv and files.
 351     :param symlinks:
 352         Whether to use symlinks to the target interpreter or not. If
 353         left unspecified, it will use symlinks except on Windows to
 354         match behavior with the "venv" CLI tool.
 355     :param with_pip:
 356         Whether to install "pip" binaries or not.
 357     """
 358     logger.debug(
 359         "%s: make_venv(env_dir=%s, system_site_packages=%s, "
 360         "clear=%s, symlinks=%s, with_pip=%s)",
 361         __file__,
 362         str(env_dir),
 363         system_site_packages,
 364         clear,
 365         symlinks,
 366         with_pip,
 367     )
 368
 369     if symlinks is None:
 370         # Default behavior of standard venv CLI
 371         symlinks = os.name != "nt"
 372
 373     builder = QemuEnvBuilder(
 374         system_site_packages=system_site_packages,
 375         clear=clear,
 376         symlinks=symlinks,
 377         with_pip=with_pip,
 378     )
 379
 380     style = "non-isolated" if builder.system_site_packages else "isolated"
 381     nested = ""
 382     if builder.use_parent_packages:
 383         nested = f"(with packages from '{builder.get_parent_libpath()}') "
 384     print(
 385         f"mkvenv: Creating {style} virtual environment"
 386         f" {nested}at '{str(env_dir)}'",
 387         file=sys.stderr,
 388     )
 389
 390     try:
 391         logger.debug("Invoking builder.create()")
 392         try:
 393             builder.create(str(env_dir))
 394         except SystemExit as exc:
 395             # Some versions of the venv module raise SystemExit; *nasty*!
 396             # We want the exception that prompted it. It might be a subprocess
 397             # error that has output we *really* want to see.
 398             logger.debug("Intercepted SystemExit from EnvBuilder.create()")
 399             raise exc.__cause__ or exc.__context__ or exc
 400         logger.debug("builder.create() finished")
 401     except subprocess.CalledProcessError as exc:
 402         logger.error("mkvenv subprocess failed:")
 403         logger.error("cmd: %s", exc.cmd)
 404         logger.error("returncode: %d", exc.returncode)
 405
 406         def _stringify(data: Union[str, bytes]) -> str:
 407             if isinstance(data, bytes):
 408                 return data.decode()
 409             return data
 410
 411         lines = []
 412         if exc.stdout:
 413             lines.append("========== stdout ==========")
 414             lines.append(_stringify(exc.stdout))
 415             lines.append("============================")
 416         if exc.stderr:
 417             lines.append("========== stderr ==========")
 418             lines.append(_stringify(exc.stderr))
 419             lines.append("============================")
 420         if lines:
 421             logger.error(os.linesep.join(lines))
 422
 423         raise Ouch("VENV creation subprocess failed.") from exc
 424
 425     # print the python executable to stdout for configure.
 426     print(builder.get_value("env_exe"))
 427
 428
 429 def _get_entry_points(packages: Sequence[str]) -> Iterator[str]:
 430
 431     def _generator() -> Iterator[str]:
 432         for package in packages:
 433             try:
 434                 entry_points: Iterator[EntryPoint] = \
 435                     iter(distribution(package).entry_points)
 436             except PackageNotFoundError:
 437                 continue
 438
 439             # The EntryPoints type is only available in 3.10+,
 440             # treat this as a vanilla list and filter it ourselves.
 441             entry_points = filter(
 442                 lambda ep: ep.group == "console_scripts", entry_points
 443             )
 444
 445             for entry_point in entry_points:
 446                 yield f"{entry_point.name} = {entry_point.value}"
 447
 448     return _generator()
 449
 450
 451 def generate_console_scripts(
 452     packages: Sequence[str],
 453     python_path: Optional[str] = None,
 454     bin_path: Optional[str] = None,
 455 ) -> None:
 456     """
 457     Generate script shims for console_script entry points in @packages.
 458     """
 459     if python_path is None:
 460         python_path = sys.executable
 461     if bin_path is None:
 462         bin_path = sysconfig.get_path("scripts")
 463         assert bin_path is not None
 464
 465     logger.debug(
 466         "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)",
 467         packages,
 468         python_path,
 469         bin_path,
 470     )
 471
 472     if not packages:
 473         return
 474
 475     maker = distlib.scripts.ScriptMaker(None, bin_path)
 476     maker.variants = {""}
 477     maker.clobber = False
 478
 479     for entry_point in _get_entry_points(packages):
 480         for filename in maker.make(entry_point):
 481             logger.debug("wrote console_script '%s'", filename)
 482
 483
 484 def pkgname_from_depspec(dep_spec: str) -> str:
 485     """
 486     Parse package name out of a PEP-508 depspec.
 487
 488     See https://peps.python.org/pep-0508/#names
 489     """
 490     match = re.match(
 491         r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE
 492     )
 493     if not match:
 494         raise ValueError(
 495             f"dep_spec '{dep_spec}'"
 496             " does not appear to contain a valid package name"
 497         )
 498     return match.group(0)
 499
 500
 501 def _path_is_prefix(prefix: Optional[str], path: str) -> bool:
 502     try:
 503         return (
 504             prefix is not None and os.path.commonpath([prefix, path]) == prefix
 505         )
 506     except ValueError:
 507         return False
 508
 509
 510 def _is_system_package(dist: Distribution) -> bool:
 511     path = str(dist.locate_file("."))
 512     return not (
 513         _path_is_prefix(sysconfig.get_path("purelib"), path)
 514         or _path_is_prefix(sysconfig.get_path("platlib"), path)
 515     )
 516
 517
 518 def diagnose(
 519     dep_spec: str,
 520     online: bool,
 521     wheels_dir: Optional[Union[str, Path]],
 522     prog: Optional[str],
 523 ) -> Tuple[str, bool]:
 524     """
 525     Offer a summary to the user as to why a package failed to be installed.
 526
 527     :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5'
 528     :param online: Did we allow PyPI access?
 529     :param prog:
 530         Optionally, a shell program name that can be used as a
 531         bellwether to detect if this program is installed elsewhere on
 532         the system. This is used to offer advice when a program is
 533         detected for a different python version.
 534     :param wheels_dir:
 535         Optionally, a directory that was searched for vendored packages.
 536     """
 537     # pylint: disable=too-many-branches
 538
 539     # Some errors are not particularly serious
 540     bad = False
 541
 542     pkg_name = pkgname_from_depspec(dep_spec)
 543     pkg_version: Optional[str] = None
 544     try:
 545         pkg_version = version(pkg_name)
 546     except PackageNotFoundError:
 547         pass
 548
 549     lines = []
 550
 551     if pkg_version:
 552         lines.append(
 553             f"Python package '{pkg_name}' version '{pkg_version}' was found,"
 554             " but isn't suitable."
 555         )
 556     else:
 557         lines.append(
 558             f"Python package '{pkg_name}' was not found nor installed."
 559         )
 560
 561     if wheels_dir:
 562         lines.append(
 563             "No suitable version found in, or failed to install from"
 564             f" '{wheels_dir}'."
 565         )
 566         bad = True
 567
 568     if online:
 569         lines.append("A suitable version could not be obtained from PyPI.")
 570         bad = True
 571     else:
 572         lines.append(
 573             "mkvenv was configured to operate offline and did not check PyPI."
 574         )
 575
 576     if prog and not pkg_version:
 577         which = shutil.which(prog)
 578         if which:
 579             if sys.base_prefix in site.PREFIXES:
 580                 pypath = Path(sys.executable).resolve()
 581                 lines.append(
 582                     f"'{prog}' was detected on your system at '{which}', "
 583                     f"but the Python package '{pkg_name}' was not found by "
 584                     f"this Python interpreter ('{pypath}'). "
 585                     f"Typically this means that '{prog}' has been installed "
 586                     "against a different Python interpreter on your system."
 587                 )
 588             else:
 589                 lines.append(
 590                     f"'{prog}' was detected on your system at '{which}', "
 591                     "but the build is using an isolated virtual environment."
 592                 )
 593             bad = True
 594
 595     lines = [f" • {line}" for line in lines]
 596     if bad:
 597         lines.insert(0, f"Could not provide build dependency '{dep_spec}':")
 598     else:
 599         lines.insert(0, f"'{dep_spec}' not found:")
 600     return os.linesep.join(lines), bad
 601
 602
 603 def pip_install(
 604     args: Sequence[str],
 605     online: bool = False,
 606     wheels_dir: Optional[Union[str, Path]] = None,
 607 ) -> None:
 608     """
 609     Use pip to install a package or package(s) as specified in @args.
 610     """
 611     loud = bool(
 612         os.environ.get("DEBUG")
 613         or os.environ.get("GITLAB_CI")
 614         or os.environ.get("V")
 615     )
 616
 617     full_args = [
 618         sys.executable,
 619         "-m",
 620         "pip",
 621         "install",
 622         "--disable-pip-version-check",
 623         "-v" if loud else "-q",
 624     ]
 625     if not online:
 626         full_args += ["--no-index"]
 627     if wheels_dir:
 628         full_args += ["--find-links", f"file://{str(wheels_dir)}"]
 629     full_args += list(args)
 630     subprocess.run(
 631         full_args,
 632         check=True,
 633     )
 634
 635
 636 def _make_version_constraint(info: Dict[str, str], install: bool) -> str:
 637     """
 638     Construct the version constraint part of a PEP 508 dependency
 639     specification (for example '>=0.61.5') from the accepted and
 640     installed keys of the provided dictionary.
 641
 642     :param info: A dictionary corresponding to a TOML key-value list.
 643     :param install: True generates install constraints, False generates
 644         presence constraints
 645     """
 646     if install and "installed" in info:
 647         return "==" + info["installed"]
 648
 649     dep_spec = info.get("accepted", "")
 650     dep_spec = dep_spec.strip()
 651     # Double check that they didn't just use a version number
 652     if dep_spec and dep_spec[0] not in "!~><=(":
 653         raise Ouch(
 654             "invalid dependency specifier " + dep_spec + " in dependency file"
 655         )
 656
 657     return dep_spec
 658
 659
 660 def _do_ensure(
 661     group: Dict[str, Dict[str, str]],
 662     online: bool = False,
 663     wheels_dir: Optional[Union[str, Path]] = None,
 664 ) -> Optional[Tuple[str, bool]]:
 665     """
 666     Use pip to ensure we have the packages specified in @group.
 667
 668     If the packages are already installed, do nothing. If online and
 669     wheels_dir are both provided, prefer packages found in wheels_dir
 670     first before connecting to PyPI.
 671
 672     :param group: A dictionary of dictionaries, corresponding to a
 673         section in a pythondeps.toml file.
 674     :param online: If True, fall back to PyPI.
 675     :param wheels_dir: If specified, search this path for packages.
 676     """
 677     absent = []
 678     present = []
 679     canary = None
 680     for name, info in group.items():
 681         constraint = _make_version_constraint(info, False)
 682         matcher = distlib.version.LegacyMatcher(name + constraint)
 683         print(f"mkvenv: checking for {matcher}", file=sys.stderr)
 684
 685         dist: Optional[Distribution] = None
 686         try:
 687             dist = distribution(matcher.name)
 688         except PackageNotFoundError:
 689             pass
 690
 691         if (
 692             dist is None
 693             # Always pass installed package to pip, so that they can be
 694             # updated if the requested version changes
 695             or not _is_system_package(dist)
 696             or not matcher.match(distlib.version.LegacyVersion(dist.version))
 697         ):
 698             absent.append(name + _make_version_constraint(info, True))
 699             if len(absent) == 1:
 700                 canary = info.get("canary", None)
 701         else:
 702             logger.info("found %s %s", name, dist.version)
 703             present.append(name)
 704
 705     if present:
 706         generate_console_scripts(present)
 707
 708     if absent:
 709         if online or wheels_dir:
 710             # Some packages are missing or aren't a suitable version,
 711             # install a suitable (possibly vendored) package.
 712             print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr)
 713             try:
 714                 pip_install(args=absent, online=online, wheels_dir=wheels_dir)
 715                 return None
 716             except subprocess.CalledProcessError:
 717                 pass
 718
 719         return diagnose(
 720             absent[0],
 721             online,
 722             wheels_dir,
 723             canary,
 724         )
 725
 726     return None
 727
 728
 729 def ensure(
 730     dep_specs: Sequence[str],
 731     online: bool = False,
 732     wheels_dir: Optional[Union[str, Path]] = None,
 733     prog: Optional[str] = None,
 734 ) -> None:
 735     """
 736     Use pip to ensure we have the package specified by @dep_specs.
 737
 738     If the package is already installed, do nothing. If online and
 739     wheels_dir are both provided, prefer packages found in wheels_dir
 740     first before connecting to PyPI.
 741
 742     :param dep_specs:
 743         PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
 744     :param online: If True, fall back to PyPI.
 745     :param wheels_dir: If specified, search this path for packages.
 746     :param prog:
 747         If specified, use this program name for error diagnostics that will
 748         be presented to the user. e.g., 'sphinx-build' can be used as a
 749         bellwether for the presence of 'sphinx'.
 750     """
 751
 752     if not HAVE_DISTLIB:
 753         raise Ouch("a usable distlib could not be found, please install it")
 754
 755     # Convert the depspecs to a dictionary, as if they came
 756     # from a section in a pythondeps.toml file
 757     group: Dict[str, Dict[str, str]] = {}
 758     for spec in dep_specs:
 759         name = distlib.version.LegacyMatcher(spec).name
 760         group[name] = {}
 761
 762         spec = spec.strip()
 763         pos = len(name)
 764         ver = spec[pos:].strip()
 765         if ver:
 766             group[name]["accepted"] = ver
 767
 768         if prog:
 769             group[name]["canary"] = prog
 770             prog = None
 771
 772     result = _do_ensure(group, online, wheels_dir)
 773     if result:
 774         # Well, that's not good.
 775         if result[1]:
 776             raise Ouch(result[0])
 777         raise SystemExit(f"\n{result[0]}\n\n")
 778
 779
 780 def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]:
 781     if not HAVE_TOMLLIB:
 782         if sys.version_info < (3, 11):
 783             raise Ouch("found no usable tomli, please install it")
 784
 785         raise Ouch(
 786             "Python >=3.11 does not have tomllib... what have you done!?"
 787         )
 788
 789     # Use loads() to support both tomli v1.2.x (Ubuntu 22.04,
 790     # Debian bullseye-backports) and v2.0.x
 791     with open(file, "r", encoding="ascii") as depfile:
 792         contents = depfile.read()
 793         return tomllib.loads(contents)  # type: ignore
 794
 795
 796 def ensure_group(
 797     file: str,
 798     groups: Sequence[str],
 799     online: bool = False,
 800     wheels_dir: Optional[Union[str, Path]] = None,
 801 ) -> None:
 802     """
 803     Use pip to ensure we have the package specified by @dep_specs.
 804
 805     If the package is already installed, do nothing. If online and
 806     wheels_dir are both provided, prefer packages found in wheels_dir
 807     first before connecting to PyPI.
 808
 809     :param dep_specs:
 810         PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
 811     :param online: If True, fall back to PyPI.
 812     :param wheels_dir: If specified, search this path for packages.
 813     """
 814
 815     if not HAVE_DISTLIB:
 816         raise Ouch("found no usable distlib, please install it")
 817
 818     parsed_deps = _parse_groups(file)
 819
 820     to_install: Dict[str, Dict[str, str]] = {}
 821     for group in groups:
 822         try:
 823             to_install.update(parsed_deps[group])
 824         except KeyError as exc:
 825             raise Ouch(f"group {group} not defined") from exc
 826
 827     result = _do_ensure(to_install, online, wheels_dir)
 828     if result:
 829         # Well, that's not good.
 830         if result[1]:
 831             raise Ouch(result[0])
 832         raise SystemExit(f"\n{result[0]}\n\n")
 833
 834
 835 def post_venv_setup() -> None:
 836     """
 837     This is intended to be run *inside the venv* after it is created.
 838     """
 839     logger.debug("post_venv_setup()")
 840     # Generate a 'pip' script so the venv is usable in a normal
 841     # way from the CLI. This only happens when we inherited pip from a
 842     # parent/system-site and haven't run ensurepip in some way.
 843     generate_console_scripts(["pip"])
 844
 845
 846 def _add_create_subcommand(subparsers: Any) -> None:
 847     subparser = subparsers.add_parser("create", help="create a venv")
 848     subparser.add_argument(
 849         "target",
 850         type=str,
 851         action="store",
 852         help="Target directory to install virtual environment into.",
 853     )
 854
 855
 856 def _add_post_init_subcommand(subparsers: Any) -> None:
 857     subparsers.add_parser("post_init", help="post-venv initialization")
 858
 859
 860 def _add_ensuregroup_subcommand(subparsers: Any) -> None:
 861     subparser = subparsers.add_parser(
 862         "ensuregroup",
 863         help="Ensure that the specified package group is installed.",
 864     )
 865     subparser.add_argument(
 866         "--online",
 867         action="store_true",
 868         help="Install packages from PyPI, if necessary.",
 869     )
 870     subparser.add_argument(
 871         "--dir",
 872         type=str,
 873         action="store",
 874         help="Path to vendored packages where we may install from.",
 875     )
 876     subparser.add_argument(
 877         "file",
 878         type=str,
 879         action="store",
 880         help=("Path to a TOML file describing package groups"),
 881     )
 882     subparser.add_argument(
 883         "group",
 884         type=str,
 885         action="store",
 886         help="One or more package group names",
 887         nargs="+",
 888     )
 889
 890
 891 def _add_ensure_subcommand(subparsers: Any) -> None:
 892     subparser = subparsers.add_parser(
 893         "ensure", help="Ensure that the specified package is installed."
 894     )
 895     subparser.add_argument(
 896         "--online",
 897         action="store_true",
 898         help="Install packages from PyPI, if necessary.",
 899     )
 900     subparser.add_argument(
 901         "--dir",
 902         type=str,
 903         action="store",
 904         help="Path to vendored packages where we may install from.",
 905     )
 906     subparser.add_argument(
 907         "--diagnose",
 908         type=str,
 909         action="store",
 910         help=(
 911             "Name of a shell utility to use for "
 912             "diagnostics if this command fails."
 913         ),
 914     )
 915     subparser.add_argument(
 916         "dep_specs",
 917         type=str,
 918         action="store",
 919         help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'",
 920         nargs="+",
 921     )
 922
 923
 924 def main() -> int:
 925     """CLI interface to make_qemu_venv. See module docstring."""
 926     if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"):
 927         # You're welcome.
 928         logging.basicConfig(level=logging.DEBUG)
 929     else:
 930         if os.environ.get("V"):
 931             logging.basicConfig(level=logging.INFO)
 932
 933     parser = argparse.ArgumentParser(
 934         prog="mkvenv",
 935         description="QEMU pyvenv bootstrapping utility",
 936     )
 937     subparsers = parser.add_subparsers(
 938         title="Commands",
 939         dest="command",
 940         required=True,
 941         metavar="command",
 942         help="Description",
 943     )
 944
 945     _add_create_subcommand(subparsers)
 946     _add_post_init_subcommand(subparsers)
 947     _add_ensure_subcommand(subparsers)
 948     _add_ensuregroup_subcommand(subparsers)
 949
 950     args = parser.parse_args()
 951     try:
 952         if args.command == "create":
 953             make_venv(
 954                 args.target,
 955                 system_site_packages=True,
 956                 clear=True,
 957             )
 958         if args.command == "post_init":
 959             post_venv_setup()
 960         if args.command == "ensure":
 961             ensure(
 962                 dep_specs=args.dep_specs,
 963                 online=args.online,
 964                 wheels_dir=args.dir,
 965                 prog=args.diagnose,
 966             )
 967         if args.command == "ensuregroup":
 968             ensure_group(
 969                 file=args.file,
 970                 groups=args.group,
 971                 online=args.online,
 972                 wheels_dir=args.dir,
 973             )
 974         logger.debug("mkvenv.py %s: exiting", args.command)
 975     except Ouch as exc:
 976         print("\n*** Ouch! ***\n", file=sys.stderr)
 977         print(str(exc), "\n\n", file=sys.stderr)
 978         return 1
 979     except SystemExit:
 980         raise
 981     except:  # pylint: disable=bare-except
 982         logger.exception("mkvenv did not complete successfully:")
 983         return 2
 984     return 0
 985
 986
 987 if __name__ == "__main__":
 988     sys.exit(main())