configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import codecs
   8 import errno
   9 import io
  10 import itertools
  11 import logging
  12 import os
  13 import sys
  14 import textwrap
  15
  16 from collections.abc import Iterable
  17
  18 base_dir = os.path.abspath(os.path.dirname(__file__))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  20 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  21 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  23 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
  24 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  25 from mach.site import (
  26     CommandSiteManager,
  27     ExternalPythonSite,
  28     MachSiteManager,
  29     MozSiteMetadata,
  30     SitePackagesSource,
  31 )
  32 from mach.requirements import MachEnvRequirements
  33 from mozbuild.configure import (
  34     ConfigureSandbox,
  35     TRACE,
  36 )
  37 from mozbuild.pythonutil import iter_modules_in_path
  38 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  39 from mozbuild.util import write_indented_repr
  40 import mozpack.path as mozpath
  41 import six
  42
  43
  44 def main(argv):
  45     # Check for CRLF line endings.
  46     with open(__file__, "r") as fh:
  47         data = fh.read()
  48         if "\r" in data:
  49             print(
  50                 "\n ***\n"
  51                 " * The source tree appears to have Windows-style line endings.\n"
  52                 " *\n"
  53                 " * If using Git, Git is likely configured to use Windows-style\n"
  54                 " * line endings.\n"
  55                 " *\n"
  56                 " * To convert the working copy to UNIX-style line endings, run\n"
  57                 " * the following:\n"
  58                 " *\n"
  59                 " * $ git config core.autocrlf false\n"
  60                 " * $ git config core.eof lf\n"
  61                 " * $ git rm --cached -r .\n"
  62                 " * $ git reset --hard\n"
  63                 " *\n"
  64                 " * If not using Git, the tool you used to obtain the source\n"
  65                 " * code likely converted files to Windows line endings. See\n"
  66                 " * usage information for that tool for more.\n"
  67                 " ***",
  68                 file=sys.stderr,
  69             )
  70             return 1
  71
  72     config = {}
  73
  74     if "OLD_CONFIGURE" not in os.environ:
  75         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  76
  77     sandbox = ConfigureSandbox(config, os.environ, argv)
  78
  79     if not sandbox._help:
  80         # This limitation has mostly to do with GNU make. Since make can't represent
  81         # variables with spaces without correct quoting and many paths are used
  82         # without proper quoting, using paths with spaces commonly results in
  83         # targets or dependencies being treated as multiple paths. This, of course,
  84         # undermines the ability for make to perform up-to-date checks and makes
  85         # the build system not work very efficiently. In theory, a non-make build
  86         # backend will make this limitation go away. But there is likely a long tail
  87         # of things that will need fixing due to e.g. lack of proper path quoting.
  88         topsrcdir = os.path.realpath(os.path.dirname(__file__))
  89         if len(topsrcdir.split()) > 1:
  90             print(
  91                 "Source directory cannot be located in a path with spaces: %s"
  92                 % topsrcdir,
  93                 file=sys.stderr,
  94             )
  95             return 1
  96         topobjdir = os.path.realpath(os.curdir)
  97         if len(topobjdir.split()) > 1:
  98             print(
  99                 "Object directory cannot be located in a path with spaces: %s"
 100                 % topobjdir,
 101                 file=sys.stderr,
 102             )
 103             return 1
 104
 105         # Do not allow topobjdir == topsrcdir
 106         if os.path.samefile(topsrcdir, topobjdir):
 107             print(
 108                 "  ***\n"
 109                 "  * Building directly in the main source directory is not allowed.\n"
 110                 "  *\n"
 111                 "  * To build, you must run configure from a separate directory\n"
 112                 "  * (referred to as an object directory).\n"
 113                 "  *\n"
 114                 "  * If you are building with a mozconfig, you will need to change your\n"
 115                 "  * mozconfig to point to a different object directory.\n"
 116                 "  ***",
 117                 file=sys.stderr,
 118             )
 119             return 1
 120         _activate_build_virtualenv()
 121
 122     clobber_file = "CLOBBER"
 123     if not os.path.exists(clobber_file):
 124         # Simply touch the file.
 125         with open(clobber_file, "a"):
 126             pass
 127
 128     if os.environ.get("MOZ_CONFIGURE_TRACE"):
 129         sandbox._logger.setLevel(TRACE)
 130
 131     sandbox.run(os.path.join(os.path.dirname(__file__), "moz.configure"))
 132
 133     if sandbox._help:
 134         return 0
 135
 136     logging.getLogger("moz.configure").info("Creating config.status")
 137
 138     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
 139     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
 140     if old_js_configure_substs or old_js_configure_defines:
 141         js_config = config.copy()
 142         pwd = os.getcwd()
 143         try:
 144             try:
 145                 os.makedirs("js/src")
 146             except OSError as e:
 147                 if e.errno != errno.EEXIST:
 148                     raise
 149
 150             os.chdir("js/src")
 151             js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
 152             js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
 153             # The build system frontend expects $objdir/js/src/config.status
 154             # to have $objdir/js/src as topobjdir.
 155             # We want forward slashes on all platforms.
 156             js_config["TOPOBJDIR"] += "/js/src"
 157             config_status(js_config, execute=False)
 158         finally:
 159             os.chdir(pwd)
 160
 161     return config_status(config)
 162
 163
 164 def check_unicode(obj):
 165     """Recursively check that all strings in the object are unicode strings."""
 166     if isinstance(obj, dict):
 167         result = True
 168         for k, v in six.iteritems(obj):
 169             if not check_unicode(k):
 170                 print("%s key is not unicode." % k, file=sys.stderr)
 171                 result = False
 172             elif not check_unicode(v):
 173                 print("%s value is not unicode." % k, file=sys.stderr)
 174                 result = False
 175         return result
 176     if isinstance(obj, bytes):
 177         return False
 178     if isinstance(obj, six.text_type):
 179         return True
 180     if isinstance(obj, Iterable):
 181         return all(check_unicode(o) for o in obj)
 182     return True
 183
 184
 185 def config_status(config, execute=True):
 186     # Sanitize config data to feed config.status
 187     # Ideally, all the backend and frontend code would handle the booleans, but
 188     # there are so many things involved, that it's easier to keep config.status
 189     # untouched for now.
 190     def sanitize_config(v):
 191         if v is True:
 192             return "1"
 193         if v is False:
 194             return ""
 195         # Serialize types that look like lists and tuples as lists.
 196         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 197             return list(v)
 198         return v
 199
 200     sanitized_config = {}
 201     sanitized_config["substs"] = {
 202         k: sanitize_config(v)
 203         for k, v in six.iteritems(config)
 204         if k
 205         not in (
 206             "DEFINES",
 207             "TOPSRCDIR",
 208             "TOPOBJDIR",
 209             "CONFIG_STATUS_DEPS",
 210             "OLD_CONFIGURE_SUBSTS",
 211             "OLD_CONFIGURE_DEFINES",
 212         )
 213     }
 214     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 215         sanitized_config["substs"][k] = sanitize_config(v)
 216     sanitized_config["defines"] = {
 217         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 218     }
 219     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 220         sanitized_config["defines"][k] = sanitize_config(v)
 221     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 222     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 223     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 224
 225     if not check_unicode(sanitized_config):
 226         print("Configuration should be all unicode.", file=sys.stderr)
 227         print("Please file a bug for the above.", file=sys.stderr)
 228         sys.exit(1)
 229
 230     # Some values in sanitized_config also have more complex types, such as
 231     # EnumString, which using when calling config_status would currently
 232     # break the build, as well as making it inconsistent with re-running
 233     # config.status, for which they are normalized to plain strings via
 234     # indented_repr. Likewise for non-dict non-string iterables being
 235     # converted to lists.
 236     def normalize(obj):
 237         if isinstance(obj, dict):
 238             return {k: normalize(v) for k, v in six.iteritems(obj)}
 239         if isinstance(obj, six.text_type):
 240             return six.text_type(obj)
 241         if isinstance(obj, Iterable):
 242             return [normalize(o) for o in obj]
 243         return obj
 244
 245     sanitized_config = normalize(sanitized_config)
 246
 247     # Create config.status. Eventually, we'll want to just do the work it does
 248     # here, when we're able to skip configure tests/use cached results/not rely
 249     # on autoconf.
 250     with codecs.open("config.status", "w", "utf-8") as fh:
 251         fh.write(
 252             textwrap.dedent(
 253                 """\
 254             #!%(python)s
 255             # coding=utf-8
 256             from __future__ import unicode_literals
 257         """
 258             )
 259             % {"python": config["PYTHON3"]}
 260         )
 261         for k, v in sorted(six.iteritems(sanitized_config)):
 262             fh.write("%s = " % k)
 263             write_indented_repr(fh, v)
 264         fh.write(
 265             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 266         )
 267
 268         if execute:
 269             fh.write(
 270                 textwrap.dedent(
 271                     """
 272                 if __name__ == '__main__':
 273                     from mozbuild.config_status import config_status
 274                     args = dict([(name, globals()[name]) for name in __all__])
 275                     config_status(**args)
 276             """
 277                 )
 278             )
 279
 280     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 281     partial_config.write_vars(sanitized_config)
 282
 283     # Write out a file so the build backend knows to re-run configure when
 284     # relevant Python changes.
 285     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 286         for f in sorted(
 287             itertools.chain(
 288                 config["CONFIG_STATUS_DEPS"],
 289                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 290             )
 291         ):
 292             fh.write("%s\n" % mozpath.normpath(f))
 293
 294     # Other things than us are going to run this file, so we need to give it
 295     # executable permissions.
 296     os.chmod("config.status", 0o755)
 297     if execute:
 298         from mozbuild.config_status import config_status
 299
 300         return config_status(args=[], **sanitized_config)
 301     return 0
 302
 303
 304 def _activate_build_virtualenv():
 305     """Ensure that the build virtualenv is activated
 306
 307     configure.py may be executed through Mach, or via "./configure, make".
 308     In the first case, the build virtualenv should already be activated.
 309     In the second case, we're likely being executed with the system Python, and must
 310     prepare the virtualenv and activate it ourselves.
 311     """
 312
 313     version = ".".join(str(i) for i in sys.version_info[0:3])
 314     print(f"Using Python {version} from {sys.executable}")
 315
 316     active_site = MozSiteMetadata.from_runtime()
 317     if active_site and active_site.site_name == "build":
 318         # We're already running within the "build" virtualenv, no additional work is
 319         # needed.
 320         return
 321
 322     # We're using the system python (or are nested within a non-build mach-managed
 323     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 324     # configure.
 325
 326     topobjdir = os.path.realpath(".")
 327     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 328
 329     mach_site = MachSiteManager(
 330         topsrcdir,
 331         None,
 332         MachEnvRequirements(),
 333         ExternalPythonSite(sys.executable),
 334         SitePackagesSource.NONE,
 335     )
 336     mach_site.activate()
 337     build_site = CommandSiteManager.from_environment(
 338         topsrcdir,
 339         None,
 340         "build",
 341         os.path.join(topobjdir, "_virtualenvs"),
 342     )
 343     if not build_site.ensure():
 344         print("Created Python 3 virtualenv")
 345     build_site.activate()
 346
 347
 348 if __name__ == "__main__":
 349     sys.exit(main(sys.argv))