configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import codecs
   8 import errno
   9 import io
  10 import itertools
  11 import logging
  12 import os
  13 import sys
  14 import textwrap
  15
  16 from collections.abc import Iterable
  17
  18 base_dir = os.path.abspath(os.path.dirname(__file__))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  20 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  21 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  23 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
  24 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  25 from mach.site import (
  26     CommandSiteManager,
  27     ExternalPythonSite,
  28     MachSiteManager,
  29     MozSiteMetadata,
  30     SitePackagesSource,
  31 )
  32 from mach.requirements import MachEnvRequirements
  33 from mozbuild.configure import (
  34     ConfigureSandbox,
  35     TRACE,
  36 )
  37 from mozbuild.pythonutil import iter_modules_in_path
  38 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  39 from mozbuild.util import write_indented_repr
  40 import mozpack.path as mozpath
  41 import six
  42
  43
  44 def main(argv):
  45     _activate_build_virtualenv()
  46     config = {}
  47
  48     if "OLD_CONFIGURE" not in os.environ:
  49         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  50
  51     sandbox = ConfigureSandbox(config, os.environ, argv)
  52
  53     clobber_file = "CLOBBER"
  54     if not os.path.exists(clobber_file):
  55         # Simply touch the file.
  56         with open(clobber_file, "a"):
  57             pass
  58
  59     if os.environ.get("MOZ_CONFIGURE_TRACE"):
  60         sandbox._logger.setLevel(TRACE)
  61
  62     sandbox.run(os.path.join(os.path.dirname(__file__), "moz.configure"))
  63
  64     if sandbox._help:
  65         return 0
  66
  67     logging.getLogger("moz.configure").info("Creating config.status")
  68
  69     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
  70     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
  71     if old_js_configure_substs or old_js_configure_defines:
  72         js_config = config.copy()
  73         pwd = os.getcwd()
  74         try:
  75             try:
  76                 os.makedirs("js/src")
  77             except OSError as e:
  78                 if e.errno != errno.EEXIST:
  79                     raise
  80
  81             os.chdir("js/src")
  82             js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
  83             js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
  84             # The build system frontend expects $objdir/js/src/config.status
  85             # to have $objdir/js/src as topobjdir.
  86             # We want forward slashes on all platforms.
  87             js_config["TOPOBJDIR"] += "/js/src"
  88             config_status(js_config, execute=False)
  89         finally:
  90             os.chdir(pwd)
  91
  92     return config_status(config)
  93
  94
  95 def check_unicode(obj):
  96     """Recursively check that all strings in the object are unicode strings."""
  97     if isinstance(obj, dict):
  98         result = True
  99         for k, v in six.iteritems(obj):
 100             if not check_unicode(k):
 101                 print("%s key is not unicode." % k, file=sys.stderr)
 102                 result = False
 103             elif not check_unicode(v):
 104                 print("%s value is not unicode." % k, file=sys.stderr)
 105                 result = False
 106         return result
 107     if isinstance(obj, bytes):
 108         return False
 109     if isinstance(obj, six.text_type):
 110         return True
 111     if isinstance(obj, Iterable):
 112         return all(check_unicode(o) for o in obj)
 113     return True
 114
 115
 116 def config_status(config, execute=True):
 117     # Sanitize config data to feed config.status
 118     # Ideally, all the backend and frontend code would handle the booleans, but
 119     # there are so many things involved, that it's easier to keep config.status
 120     # untouched for now.
 121     def sanitize_config(v):
 122         if v is True:
 123             return "1"
 124         if v is False:
 125             return ""
 126         # Serialize types that look like lists and tuples as lists.
 127         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 128             return list(v)
 129         return v
 130
 131     sanitized_config = {}
 132     sanitized_config["substs"] = {
 133         k: sanitize_config(v)
 134         for k, v in six.iteritems(config)
 135         if k
 136         not in (
 137             "DEFINES",
 138             "TOPSRCDIR",
 139             "TOPOBJDIR",
 140             "CONFIG_STATUS_DEPS",
 141             "OLD_CONFIGURE_SUBSTS",
 142             "OLD_CONFIGURE_DEFINES",
 143         )
 144     }
 145     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 146         sanitized_config["substs"][k] = sanitize_config(v)
 147     sanitized_config["defines"] = {
 148         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 149     }
 150     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 151         sanitized_config["defines"][k] = sanitize_config(v)
 152     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 153     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 154     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 155
 156     if not check_unicode(sanitized_config):
 157         print("Configuration should be all unicode.", file=sys.stderr)
 158         print("Please file a bug for the above.", file=sys.stderr)
 159         sys.exit(1)
 160
 161     # Some values in sanitized_config also have more complex types, such as
 162     # EnumString, which using when calling config_status would currently
 163     # break the build, as well as making it inconsistent with re-running
 164     # config.status, for which they are normalized to plain strings via
 165     # indented_repr. Likewise for non-dict non-string iterables being
 166     # converted to lists.
 167     def normalize(obj):
 168         if isinstance(obj, dict):
 169             return {k: normalize(v) for k, v in six.iteritems(obj)}
 170         if isinstance(obj, six.text_type):
 171             return six.text_type(obj)
 172         if isinstance(obj, Iterable):
 173             return [normalize(o) for o in obj]
 174         return obj
 175
 176     sanitized_config = normalize(sanitized_config)
 177
 178     # Create config.status. Eventually, we'll want to just do the work it does
 179     # here, when we're able to skip configure tests/use cached results/not rely
 180     # on autoconf.
 181     with codecs.open("config.status", "w", "utf-8") as fh:
 182         fh.write(
 183             textwrap.dedent(
 184                 """\
 185             #!%(python)s
 186             # coding=utf-8
 187             from __future__ import unicode_literals
 188         """
 189             )
 190             % {"python": config["PYTHON3"]}
 191         )
 192         for k, v in sorted(six.iteritems(sanitized_config)):
 193             fh.write("%s = " % k)
 194             write_indented_repr(fh, v)
 195         fh.write(
 196             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 197         )
 198
 199         if execute:
 200             fh.write(
 201                 textwrap.dedent(
 202                     """
 203                 if __name__ == '__main__':
 204                     from mozbuild.config_status import config_status
 205                     args = dict([(name, globals()[name]) for name in __all__])
 206                     config_status(**args)
 207             """
 208                 )
 209             )
 210
 211     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 212     partial_config.write_vars(sanitized_config)
 213
 214     # Write out a file so the build backend knows to re-run configure when
 215     # relevant Python changes.
 216     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 217         for f in sorted(
 218             itertools.chain(
 219                 config["CONFIG_STATUS_DEPS"],
 220                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 221             )
 222         ):
 223             fh.write("%s\n" % mozpath.normpath(f))
 224
 225     # Other things than us are going to run this file, so we need to give it
 226     # executable permissions.
 227     os.chmod("config.status", 0o755)
 228     if execute:
 229         from mozbuild.config_status import config_status
 230
 231         return config_status(args=[], **sanitized_config)
 232     return 0
 233
 234
 235 def _activate_build_virtualenv():
 236     """Ensure that the build virtualenv is activated
 237
 238     configure.py may be executed through Mach, or via "./configure, make".
 239     In the first case, the build virtualenv should already be activated.
 240     In the second case, we're likely being executed with the system Python, and must
 241     prepare the virtualenv and activate it ourselves.
 242     """
 243
 244     version = ".".join(str(i) for i in sys.version_info[0:3])
 245     print(f"Using Python {version} from {sys.executable}")
 246
 247     active_site = MozSiteMetadata.from_runtime()
 248     if active_site and active_site.site_name == "build":
 249         # We're already running within the "build" virtualenv, no additional work is
 250         # needed.
 251         return
 252
 253     # We're using the system python (or are nested within a non-build mach-managed
 254     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 255     # configure.
 256
 257     topobjdir = os.path.realpath(".")
 258     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 259
 260     mach_site = MachSiteManager(
 261         topsrcdir,
 262         None,
 263         MachEnvRequirements(),
 264         ExternalPythonSite(sys.executable),
 265         SitePackagesSource.NONE,
 266     )
 267     mach_site.activate()
 268     build_site = CommandSiteManager.from_environment(
 269         topsrcdir,
 270         None,
 271         "build",
 272         os.path.join(topobjdir, "_virtualenvs"),
 273     )
 274     if not build_site.ensure():
 275         print("Created Python 3 virtualenv")
 276     build_site.activate()
 277
 278
 279 if __name__ == "__main__":
 280     sys.exit(main(sys.argv))