configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import codecs
   8 import errno
   9 import io
  10 import itertools
  11 import logging
  12 import os
  13 import sys
  14 import textwrap
  15
  16 from collections.abc import Iterable
  17
  18 base_dir = os.path.abspath(os.path.dirname(__file__))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  20 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  21 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  23 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
  24 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  25 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "looseversion"))
  26 from mach.site import (
  27     CommandSiteManager,
  28     ExternalPythonSite,
  29     MachSiteManager,
  30     MozSiteMetadata,
  31     SitePackagesSource,
  32 )
  33 from mach.requirements import MachEnvRequirements
  34 from mozbuild.configure import (
  35     ConfigureSandbox,
  36     TRACE,
  37 )
  38 from mozbuild.pythonutil import iter_modules_in_path
  39 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  40 from mozbuild.util import write_indented_repr
  41 import mozpack.path as mozpath
  42 import six
  43
  44
  45 def main(argv):
  46     # Check for CRLF line endings.
  47     with open(__file__, "r") as fh:
  48         data = fh.read()
  49         if "\r" in data:
  50             print(
  51                 "\n ***\n"
  52                 " * The source tree appears to have Windows-style line endings.\n"
  53                 " *\n"
  54                 " * If using Git, Git is likely configured to use Windows-style\n"
  55                 " * line endings.\n"
  56                 " *\n"
  57                 " * To convert the working copy to UNIX-style line endings, run\n"
  58                 " * the following:\n"
  59                 " *\n"
  60                 " * $ git config core.autocrlf false\n"
  61                 " * $ git config core.eof lf\n"
  62                 " * $ git rm --cached -r .\n"
  63                 " * $ git reset --hard\n"
  64                 " *\n"
  65                 " * If not using Git, the tool you used to obtain the source\n"
  66                 " * code likely converted files to Windows line endings. See\n"
  67                 " * usage information for that tool for more.\n"
  68                 " ***",
  69                 file=sys.stderr,
  70             )
  71             return 1
  72
  73     config = {}
  74
  75     if "OLD_CONFIGURE" not in os.environ:
  76         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  77
  78     sandbox = ConfigureSandbox(config, os.environ, argv)
  79
  80     if not sandbox._help:
  81         # This limitation has mostly to do with GNU make. Since make can't represent
  82         # variables with spaces without correct quoting and many paths are used
  83         # without proper quoting, using paths with spaces commonly results in
  84         # targets or dependencies being treated as multiple paths. This, of course,
  85         # undermines the ability for make to perform up-to-date checks and makes
  86         # the build system not work very efficiently. In theory, a non-make build
  87         # backend will make this limitation go away. But there is likely a long tail
  88         # of things that will need fixing due to e.g. lack of proper path quoting.
  89         topsrcdir = os.path.realpath(os.path.dirname(__file__))
  90         if len(topsrcdir.split()) > 1:
  91             print(
  92                 "Source directory cannot be located in a path with spaces: %s"
  93                 % topsrcdir,
  94                 file=sys.stderr,
  95             )
  96             return 1
  97         topobjdir = os.path.realpath(os.curdir)
  98         if len(topobjdir.split()) > 1:
  99             print(
 100                 "Object directory cannot be located in a path with spaces: %s"
 101                 % topobjdir,
 102                 file=sys.stderr,
 103             )
 104             return 1
 105
 106         # Do not allow topobjdir == topsrcdir
 107         if os.path.samefile(topsrcdir, topobjdir):
 108             print(
 109                 "  ***\n"
 110                 "  * Building directly in the main source directory is not allowed.\n"
 111                 "  *\n"
 112                 "  * To build, you must run configure from a separate directory\n"
 113                 "  * (referred to as an object directory).\n"
 114                 "  *\n"
 115                 "  * If you are building with a mozconfig, you will need to change your\n"
 116                 "  * mozconfig to point to a different object directory.\n"
 117                 "  ***",
 118                 file=sys.stderr,
 119             )
 120             return 1
 121         _activate_build_virtualenv()
 122
 123     clobber_file = "CLOBBER"
 124     if not os.path.exists(clobber_file):
 125         # Simply touch the file.
 126         with open(clobber_file, "a"):
 127             pass
 128
 129     if os.environ.get("MOZ_CONFIGURE_TRACE"):
 130         sandbox._logger.setLevel(TRACE)
 131
 132     sandbox.run(os.path.join(os.path.dirname(__file__), "moz.configure"))
 133
 134     if sandbox._help:
 135         return 0
 136
 137     logging.getLogger("moz.configure").info("Creating config.status")
 138
 139     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
 140     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
 141     if old_js_configure_substs or old_js_configure_defines:
 142         js_config = config.copy()
 143         pwd = os.getcwd()
 144         try:
 145             try:
 146                 os.makedirs("js/src")
 147             except OSError as e:
 148                 if e.errno != errno.EEXIST:
 149                     raise
 150
 151             os.chdir("js/src")
 152             js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
 153             js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
 154             # The build system frontend expects $objdir/js/src/config.status
 155             # to have $objdir/js/src as topobjdir.
 156             # We want forward slashes on all platforms.
 157             js_config["TOPOBJDIR"] += "/js/src"
 158             config_status(js_config, execute=False)
 159         finally:
 160             os.chdir(pwd)
 161
 162     return config_status(config)
 163
 164
 165 def check_unicode(obj):
 166     """Recursively check that all strings in the object are unicode strings."""
 167     if isinstance(obj, dict):
 168         result = True
 169         for k, v in six.iteritems(obj):
 170             if not check_unicode(k):
 171                 print("%s key is not unicode." % k, file=sys.stderr)
 172                 result = False
 173             elif not check_unicode(v):
 174                 print("%s value is not unicode." % k, file=sys.stderr)
 175                 result = False
 176         return result
 177     if isinstance(obj, bytes):
 178         return False
 179     if isinstance(obj, six.text_type):
 180         return True
 181     if isinstance(obj, Iterable):
 182         return all(check_unicode(o) for o in obj)
 183     return True
 184
 185
 186 def config_status(config, execute=True):
 187     # Sanitize config data to feed config.status
 188     # Ideally, all the backend and frontend code would handle the booleans, but
 189     # there are so many things involved, that it's easier to keep config.status
 190     # untouched for now.
 191     def sanitize_config(v):
 192         if v is True:
 193             return "1"
 194         if v is False:
 195             return ""
 196         # Serialize types that look like lists and tuples as lists.
 197         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 198             return list(v)
 199         return v
 200
 201     sanitized_config = {}
 202     sanitized_config["substs"] = {
 203         k: sanitize_config(v)
 204         for k, v in six.iteritems(config)
 205         if k
 206         not in (
 207             "DEFINES",
 208             "TOPSRCDIR",
 209             "TOPOBJDIR",
 210             "CONFIG_STATUS_DEPS",
 211             "OLD_CONFIGURE_SUBSTS",
 212             "OLD_CONFIGURE_DEFINES",
 213         )
 214     }
 215     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 216         sanitized_config["substs"][k] = sanitize_config(v)
 217     sanitized_config["defines"] = {
 218         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 219     }
 220     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 221         sanitized_config["defines"][k] = sanitize_config(v)
 222     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 223     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 224     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 225
 226     if not check_unicode(sanitized_config):
 227         print("Configuration should be all unicode.", file=sys.stderr)
 228         print("Please file a bug for the above.", file=sys.stderr)
 229         sys.exit(1)
 230
 231     # Some values in sanitized_config also have more complex types, such as
 232     # EnumString, which using when calling config_status would currently
 233     # break the build, as well as making it inconsistent with re-running
 234     # config.status, for which they are normalized to plain strings via
 235     # indented_repr. Likewise for non-dict non-string iterables being
 236     # converted to lists.
 237     def normalize(obj):
 238         if isinstance(obj, dict):
 239             return {k: normalize(v) for k, v in six.iteritems(obj)}
 240         if isinstance(obj, six.text_type):
 241             return six.text_type(obj)
 242         if isinstance(obj, Iterable):
 243             return [normalize(o) for o in obj]
 244         return obj
 245
 246     sanitized_config = normalize(sanitized_config)
 247
 248     # Create config.status. Eventually, we'll want to just do the work it does
 249     # here, when we're able to skip configure tests/use cached results/not rely
 250     # on autoconf.
 251     with codecs.open("config.status", "w", "utf-8") as fh:
 252         fh.write(
 253             textwrap.dedent(
 254                 """\
 255             #!%(python)s
 256             # coding=utf-8
 257             from __future__ import unicode_literals
 258         """
 259             )
 260             % {"python": config["PYTHON3"]}
 261         )
 262         for k, v in sorted(six.iteritems(sanitized_config)):
 263             fh.write("%s = " % k)
 264             write_indented_repr(fh, v)
 265         fh.write(
 266             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 267         )
 268
 269         if execute:
 270             fh.write(
 271                 textwrap.dedent(
 272                     """
 273                 if __name__ == '__main__':
 274                     from mozbuild.config_status import config_status
 275                     args = dict([(name, globals()[name]) for name in __all__])
 276                     config_status(**args)
 277             """
 278                 )
 279             )
 280
 281     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 282     partial_config.write_vars(sanitized_config)
 283
 284     # Write out a file so the build backend knows to re-run configure when
 285     # relevant Python changes.
 286     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 287         for f in sorted(
 288             itertools.chain(
 289                 config["CONFIG_STATUS_DEPS"],
 290                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 291             )
 292         ):
 293             fh.write("%s\n" % mozpath.normpath(f))
 294
 295     # Other things than us are going to run this file, so we need to give it
 296     # executable permissions.
 297     os.chmod("config.status", 0o755)
 298     if execute:
 299         from mozbuild.config_status import config_status
 300
 301         return config_status(args=[], **sanitized_config)
 302     return 0
 303
 304
 305 def _activate_build_virtualenv():
 306     """Ensure that the build virtualenv is activated
 307
 308     configure.py may be executed through Mach, or via "./configure, make".
 309     In the first case, the build virtualenv should already be activated.
 310     In the second case, we're likely being executed with the system Python, and must
 311     prepare the virtualenv and activate it ourselves.
 312     """
 313
 314     version = ".".join(str(i) for i in sys.version_info[0:3])
 315     print(f"Using Python {version} from {sys.executable}")
 316
 317     active_site = MozSiteMetadata.from_runtime()
 318     if active_site and active_site.site_name == "build":
 319         # We're already running within the "build" virtualenv, no additional work is
 320         # needed.
 321         return
 322
 323     # We're using the system python (or are nested within a non-build mach-managed
 324     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 325     # configure.
 326
 327     topobjdir = os.path.realpath(".")
 328     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 329
 330     mach_site = MachSiteManager(
 331         topsrcdir,
 332         None,
 333         MachEnvRequirements(),
 334         ExternalPythonSite(sys.executable),
 335         SitePackagesSource.NONE,
 336     )
 337     mach_site.activate()
 338     build_site = CommandSiteManager.from_environment(
 339         topsrcdir,
 340         None,
 341         "build",
 342         os.path.join(topobjdir, "_virtualenvs"),
 343     )
 344     if not build_site.ensure():
 345         print("Created Python 3 virtualenv")
 346     build_site.activate()
 347
 348
 349 if __name__ == "__main__":
 350     sys.exit(main(sys.argv))