configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import codecs
   8 import errno
   9 import io
  10 import itertools
  11 import logging
  12 import os
  13 import sys
  14 import textwrap
  15 from collections.abc import Iterable
  16
  17 base_dir = os.path.abspath(os.path.dirname(__file__))
  18 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  20 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  21 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
  23 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  24 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "looseversion"))
  25 import mozpack.path as mozpath
  26 import six
  27 from mach.requirements import MachEnvRequirements
  28 from mach.site import (
  29     CommandSiteManager,
  30     ExternalPythonSite,
  31     MachSiteManager,
  32     MozSiteMetadata,
  33     SitePackagesSource,
  34 )
  35 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  36 from mozbuild.configure import TRACE, ConfigureSandbox
  37 from mozbuild.pythonutil import iter_modules_in_path
  38 from mozbuild.util import write_indented_repr
  39
  40
  41 def main(argv):
  42     # Check for CRLF line endings.
  43     with open(__file__, "r") as fh:
  44         data = fh.read()
  45         if "\r" in data:
  46             print(
  47                 "\n ***\n"
  48                 " * The source tree appears to have Windows-style line endings.\n"
  49                 " *\n"
  50                 " * If using Git, Git is likely configured to use Windows-style\n"
  51                 " * line endings.\n"
  52                 " *\n"
  53                 " * To convert the working copy to UNIX-style line endings, run\n"
  54                 " * the following:\n"
  55                 " *\n"
  56                 " * $ git config core.autocrlf false\n"
  57                 " * $ git config core.eof lf\n"
  58                 " * $ git rm --cached -r .\n"
  59                 " * $ git reset --hard\n"
  60                 " *\n"
  61                 " * If not using Git, the tool you used to obtain the source\n"
  62                 " * code likely converted files to Windows line endings. See\n"
  63                 " * usage information for that tool for more.\n"
  64                 " ***",
  65                 file=sys.stderr,
  66             )
  67             return 1
  68
  69     config = {}
  70
  71     if "OLD_CONFIGURE" not in os.environ:
  72         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  73
  74     sandbox = ConfigureSandbox(config, os.environ, argv)
  75
  76     if not sandbox._help:
  77         # This limitation has mostly to do with GNU make. Since make can't represent
  78         # variables with spaces without correct quoting and many paths are used
  79         # without proper quoting, using paths with spaces commonly results in
  80         # targets or dependencies being treated as multiple paths. This, of course,
  81         # undermines the ability for make to perform up-to-date checks and makes
  82         # the build system not work very efficiently. In theory, a non-make build
  83         # backend will make this limitation go away. But there is likely a long tail
  84         # of things that will need fixing due to e.g. lack of proper path quoting.
  85         topsrcdir = os.path.realpath(os.path.dirname(__file__))
  86         if len(topsrcdir.split()) > 1:
  87             print(
  88                 "Source directory cannot be located in a path with spaces: %s"
  89                 % topsrcdir,
  90                 file=sys.stderr,
  91             )
  92             return 1
  93         topobjdir = os.path.realpath(os.curdir)
  94         if len(topobjdir.split()) > 1:
  95             print(
  96                 "Object directory cannot be located in a path with spaces: %s"
  97                 % topobjdir,
  98                 file=sys.stderr,
  99             )
 100             return 1
 101
 102         # Do not allow topobjdir == topsrcdir
 103         if os.path.samefile(topsrcdir, topobjdir):
 104             print(
 105                 "  ***\n"
 106                 "  * Building directly in the main source directory is not allowed.\n"
 107                 "  *\n"
 108                 "  * To build, you must run configure from a separate directory\n"
 109                 "  * (referred to as an object directory).\n"
 110                 "  *\n"
 111                 "  * If you are building with a mozconfig, you will need to change your\n"
 112                 "  * mozconfig to point to a different object directory.\n"
 113                 "  ***",
 114                 file=sys.stderr,
 115             )
 116             return 1
 117         _activate_build_virtualenv()
 118
 119     clobber_file = "CLOBBER"
 120     if not os.path.exists(clobber_file):
 121         # Simply touch the file.
 122         with open(clobber_file, "a"):
 123             pass
 124
 125     if os.environ.get("MOZ_CONFIGURE_TRACE"):
 126         sandbox._logger.setLevel(TRACE)
 127
 128     sandbox.run(os.path.join(os.path.dirname(__file__), "moz.configure"))
 129
 130     if sandbox._help:
 131         return 0
 132
 133     logging.getLogger("moz.configure").info("Creating config.status")
 134
 135     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
 136     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
 137     if old_js_configure_substs or old_js_configure_defines:
 138         js_config = config.copy()
 139         pwd = os.getcwd()
 140         try:
 141             try:
 142                 os.makedirs("js/src")
 143             except OSError as e:
 144                 if e.errno != errno.EEXIST:
 145                     raise
 146
 147             os.chdir("js/src")
 148             js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
 149             js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
 150             # The build system frontend expects $objdir/js/src/config.status
 151             # to have $objdir/js/src as topobjdir.
 152             # We want forward slashes on all platforms.
 153             js_config["TOPOBJDIR"] += "/js/src"
 154             config_status(js_config, execute=False)
 155         finally:
 156             os.chdir(pwd)
 157
 158     return config_status(config)
 159
 160
 161 def check_unicode(obj):
 162     """Recursively check that all strings in the object are unicode strings."""
 163     if isinstance(obj, dict):
 164         result = True
 165         for k, v in six.iteritems(obj):
 166             if not check_unicode(k):
 167                 print("%s key is not unicode." % k, file=sys.stderr)
 168                 result = False
 169             elif not check_unicode(v):
 170                 print("%s value is not unicode." % k, file=sys.stderr)
 171                 result = False
 172         return result
 173     if isinstance(obj, bytes):
 174         return False
 175     if isinstance(obj, six.text_type):
 176         return True
 177     if isinstance(obj, Iterable):
 178         return all(check_unicode(o) for o in obj)
 179     return True
 180
 181
 182 def config_status(config, execute=True):
 183     # Sanitize config data to feed config.status
 184     # Ideally, all the backend and frontend code would handle the booleans, but
 185     # there are so many things involved, that it's easier to keep config.status
 186     # untouched for now.
 187     def sanitize_config(v):
 188         if v is True:
 189             return "1"
 190         if v is False:
 191             return ""
 192         # Serialize types that look like lists and tuples as lists.
 193         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 194             return list(v)
 195         return v
 196
 197     sanitized_config = {}
 198     sanitized_config["substs"] = {
 199         k: sanitize_config(v)
 200         for k, v in six.iteritems(config)
 201         if k
 202         not in (
 203             "DEFINES",
 204             "TOPSRCDIR",
 205             "TOPOBJDIR",
 206             "CONFIG_STATUS_DEPS",
 207             "OLD_CONFIGURE_SUBSTS",
 208             "OLD_CONFIGURE_DEFINES",
 209         )
 210     }
 211     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 212         sanitized_config["substs"][k] = sanitize_config(v)
 213     sanitized_config["defines"] = {
 214         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 215     }
 216     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 217         sanitized_config["defines"][k] = sanitize_config(v)
 218     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 219     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 220     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 221
 222     if not check_unicode(sanitized_config):
 223         print("Configuration should be all unicode.", file=sys.stderr)
 224         print("Please file a bug for the above.", file=sys.stderr)
 225         sys.exit(1)
 226
 227     # Some values in sanitized_config also have more complex types, such as
 228     # EnumString, which using when calling config_status would currently
 229     # break the build, as well as making it inconsistent with re-running
 230     # config.status, for which they are normalized to plain strings via
 231     # indented_repr. Likewise for non-dict non-string iterables being
 232     # converted to lists.
 233     def normalize(obj):
 234         if isinstance(obj, dict):
 235             return {k: normalize(v) for k, v in six.iteritems(obj)}
 236         if isinstance(obj, six.text_type):
 237             return six.text_type(obj)
 238         if isinstance(obj, Iterable):
 239             return [normalize(o) for o in obj]
 240         return obj
 241
 242     sanitized_config = normalize(sanitized_config)
 243
 244     # Create config.status. Eventually, we'll want to just do the work it does
 245     # here, when we're able to skip configure tests/use cached results/not rely
 246     # on autoconf.
 247     with codecs.open("config.status", "w", "utf-8") as fh:
 248         fh.write(
 249             textwrap.dedent(
 250                 """\
 251             #!%(python)s
 252             # coding=utf-8
 253             from __future__ import unicode_literals
 254         """
 255             )
 256             % {"python": config["PYTHON3"]}
 257         )
 258         for k, v in sorted(six.iteritems(sanitized_config)):
 259             fh.write("%s = " % k)
 260             write_indented_repr(fh, v)
 261         fh.write(
 262             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 263         )
 264
 265         if execute:
 266             fh.write(
 267                 textwrap.dedent(
 268                     """
 269                 if __name__ == '__main__':
 270                     from mozbuild.config_status import config_status
 271                     args = dict([(name, globals()[name]) for name in __all__])
 272                     config_status(**args)
 273             """
 274                 )
 275             )
 276
 277     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 278     partial_config.write_vars(sanitized_config)
 279
 280     # Write out a file so the build backend knows to re-run configure when
 281     # relevant Python changes.
 282     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 283         for f in sorted(
 284             itertools.chain(
 285                 config["CONFIG_STATUS_DEPS"],
 286                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 287             )
 288         ):
 289             fh.write("%s\n" % mozpath.normpath(f))
 290
 291     # Other things than us are going to run this file, so we need to give it
 292     # executable permissions.
 293     os.chmod("config.status", 0o755)
 294     if execute:
 295         from mozbuild.config_status import config_status
 296
 297         return config_status(args=[], **sanitized_config)
 298     return 0
 299
 300
 301 def _activate_build_virtualenv():
 302     """Ensure that the build virtualenv is activated
 303
 304     configure.py may be executed through Mach, or via "./configure, make".
 305     In the first case, the build virtualenv should already be activated.
 306     In the second case, we're likely being executed with the system Python, and must
 307     prepare the virtualenv and activate it ourselves.
 308     """
 309
 310     version = ".".join(str(i) for i in sys.version_info[0:3])
 311     print(f"Using Python {version} from {sys.executable}")
 312
 313     active_site = MozSiteMetadata.from_runtime()
 314     if active_site and active_site.site_name == "build":
 315         # We're already running within the "build" virtualenv, no additional work is
 316         # needed.
 317         return
 318
 319     # We're using the system python (or are nested within a non-build mach-managed
 320     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 321     # configure.
 322
 323     topobjdir = os.path.realpath(".")
 324     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 325
 326     mach_site = MachSiteManager(
 327         topsrcdir,
 328         None,
 329         MachEnvRequirements(),
 330         ExternalPythonSite(sys.executable),
 331         SitePackagesSource.NONE,
 332     )
 333     mach_site.activate()
 334     build_site = CommandSiteManager.from_environment(
 335         topsrcdir,
 336         None,
 337         "build",
 338         os.path.join(topobjdir, "_virtualenvs"),
 339     )
 340     if not build_site.ensure():
 341         print("Created Python 3 virtualenv")
 342     build_site.activate()
 343
 344
 345 if __name__ == "__main__":
 346     sys.exit(main(sys.argv))