configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import codecs
   6 import errno
   7 import io
   8 import itertools
   9 import logging
  10 import os
  11 import pprint
  12 import sys
  13 import textwrap
  14 from collections.abc import Iterable
  15
  16 base_dir = os.path.abspath(os.path.dirname(__file__))
  17 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  18 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  20 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  21 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  23 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "looseversion"))
  24 import mozpack.path as mozpath
  25 import six
  26 from mach.requirements import MachEnvRequirements
  27 from mach.site import (
  28     CommandSiteManager,
  29     ExternalPythonSite,
  30     MachSiteManager,
  31     MozSiteMetadata,
  32     SitePackagesSource,
  33 )
  34 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  35 from mozbuild.configure import TRACE, ConfigureSandbox
  36 from mozbuild.pythonutil import iter_modules_in_path
  37
  38
  39 def main(argv):
  40     # Check for CRLF line endings.
  41     with open(__file__, "r") as fh:
  42         data = fh.read()
  43         if "\r" in data:
  44             print(
  45                 "\n ***\n"
  46                 " * The source tree appears to have Windows-style line endings.\n"
  47                 " *\n"
  48                 " * If using Git, Git is likely configured to use Windows-style\n"
  49                 " * line endings.\n"
  50                 " *\n"
  51                 " * To convert the working copy to UNIX-style line endings, run\n"
  52                 " * the following:\n"
  53                 " *\n"
  54                 " * $ git config core.autocrlf false\n"
  55                 " * $ git config core.eof lf\n"
  56                 " * $ git rm --cached -r .\n"
  57                 " * $ git reset --hard\n"
  58                 " *\n"
  59                 " * If not using Git, the tool you used to obtain the source\n"
  60                 " * code likely converted files to Windows line endings. See\n"
  61                 " * usage information for that tool for more.\n"
  62                 " ***",
  63                 file=sys.stderr,
  64             )
  65             return 1
  66
  67     config = {}
  68
  69     if "OLD_CONFIGURE" not in os.environ:
  70         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  71
  72     sandbox = ConfigureSandbox(config, os.environ, argv)
  73
  74     if not sandbox._help:
  75         # This limitation has mostly to do with GNU make. Since make can't represent
  76         # variables with spaces without correct quoting and many paths are used
  77         # without proper quoting, using paths with spaces commonly results in
  78         # targets or dependencies being treated as multiple paths. This, of course,
  79         # undermines the ability for make to perform up-to-date checks and makes
  80         # the build system not work very efficiently. In theory, a non-make build
  81         # backend will make this limitation go away. But there is likely a long tail
  82         # of things that will need fixing due to e.g. lack of proper path quoting.
  83         topsrcdir = os.path.realpath(os.path.dirname(__file__))
  84         if len(topsrcdir.split()) > 1:
  85             print(
  86                 "Source directory cannot be located in a path with spaces: %s"
  87                 % topsrcdir,
  88                 file=sys.stderr,
  89             )
  90             return 1
  91         topobjdir = os.path.realpath(os.curdir)
  92         if len(topobjdir.split()) > 1:
  93             print(
  94                 "Object directory cannot be located in a path with spaces: %s"
  95                 % topobjdir,
  96                 file=sys.stderr,
  97             )
  98             return 1
  99
 100         # Do not allow topobjdir == topsrcdir
 101         if os.path.samefile(topsrcdir, topobjdir):
 102             print(
 103                 "  ***\n"
 104                 "  * Building directly in the main source directory is not allowed.\n"
 105                 "  *\n"
 106                 "  * To build, you must run configure from a separate directory\n"
 107                 "  * (referred to as an object directory).\n"
 108                 "  *\n"
 109                 "  * If you are building with a mozconfig, you will need to change your\n"
 110                 "  * mozconfig to point to a different object directory.\n"
 111                 "  ***",
 112                 file=sys.stderr,
 113             )
 114             return 1
 115         _activate_build_virtualenv()
 116
 117     clobber_file = "CLOBBER"
 118     if not os.path.exists(clobber_file):
 119         # Simply touch the file.
 120         with open(clobber_file, "a"):
 121             pass
 122
 123     if os.environ.get("MOZ_CONFIGURE_TRACE"):
 124         sandbox._logger.setLevel(TRACE)
 125
 126     sandbox.run(os.path.join(os.path.dirname(__file__), "moz.configure"))
 127
 128     if sandbox._help:
 129         return 0
 130
 131     logging.getLogger("moz.configure").info("Creating config.status")
 132
 133     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
 134     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
 135     if old_js_configure_substs or old_js_configure_defines:
 136         js_config = config.copy()
 137         pwd = os.getcwd()
 138         try:
 139             try:
 140                 os.makedirs("js/src")
 141             except OSError as e:
 142                 if e.errno != errno.EEXIST:
 143                     raise
 144
 145             os.chdir("js/src")
 146             js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
 147             js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
 148             # The build system frontend expects $objdir/js/src/config.status
 149             # to have $objdir/js/src as topobjdir.
 150             # We want forward slashes on all platforms.
 151             js_config["TOPOBJDIR"] += "/js/src"
 152             config_status(js_config, execute=False)
 153         finally:
 154             os.chdir(pwd)
 155
 156     return config_status(config)
 157
 158
 159 def check_unicode(obj):
 160     """Recursively check that all strings in the object are unicode strings."""
 161     if isinstance(obj, dict):
 162         result = True
 163         for k, v in six.iteritems(obj):
 164             if not check_unicode(k):
 165                 print("%s key is not unicode." % k, file=sys.stderr)
 166                 result = False
 167             elif not check_unicode(v):
 168                 print("%s value is not unicode." % k, file=sys.stderr)
 169                 result = False
 170         return result
 171     if isinstance(obj, bytes):
 172         return False
 173     if isinstance(obj, six.text_type):
 174         return True
 175     if isinstance(obj, Iterable):
 176         return all(check_unicode(o) for o in obj)
 177     return True
 178
 179
 180 def config_status(config, execute=True):
 181     # Sanitize config data to feed config.status
 182     # Ideally, all the backend and frontend code would handle the booleans, but
 183     # there are so many things involved, that it's easier to keep config.status
 184     # untouched for now.
 185     def sanitize_config(v):
 186         if v is True:
 187             return "1"
 188         if v is False:
 189             return ""
 190         # Serialize types that look like lists and tuples as lists.
 191         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 192             return list(v)
 193         return v
 194
 195     sanitized_config = {}
 196     sanitized_config["substs"] = {
 197         k: sanitize_config(v)
 198         for k, v in six.iteritems(config)
 199         if k
 200         not in (
 201             "DEFINES",
 202             "TOPSRCDIR",
 203             "TOPOBJDIR",
 204             "CONFIG_STATUS_DEPS",
 205             "OLD_CONFIGURE_SUBSTS",
 206             "OLD_CONFIGURE_DEFINES",
 207         )
 208     }
 209     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 210         sanitized_config["substs"][k] = sanitize_config(v)
 211     sanitized_config["defines"] = {
 212         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 213     }
 214     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 215         sanitized_config["defines"][k] = sanitize_config(v)
 216     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 217     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 218     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 219
 220     if not check_unicode(sanitized_config):
 221         print("Configuration should be all unicode.", file=sys.stderr)
 222         print("Please file a bug for the above.", file=sys.stderr)
 223         sys.exit(1)
 224
 225     # Some values in sanitized_config also have more complex types, such as
 226     # EnumString, which using when calling config_status would currently
 227     # break the build, as well as making it inconsistent with re-running
 228     # config.status, for which they are normalized to plain strings via
 229     # indented_repr. Likewise for non-dict non-string iterables being
 230     # converted to lists.
 231     def normalize(obj):
 232         if isinstance(obj, dict):
 233             return {k: normalize(v) for k, v in six.iteritems(obj)}
 234         if isinstance(obj, six.text_type):
 235             return six.text_type(obj)
 236         if isinstance(obj, Iterable):
 237             return [normalize(o) for o in obj]
 238         return obj
 239
 240     sanitized_config = normalize(sanitized_config)
 241
 242     # Create config.status. Eventually, we'll want to just do the work it does
 243     # here, when we're able to skip configure tests/use cached results/not rely
 244     # on autoconf.
 245     with codecs.open("config.status", "w", "utf-8") as fh:
 246         fh.write(
 247             textwrap.dedent(
 248                 """\
 249             #!%(python)s
 250             # coding=utf-8
 251         """
 252             )
 253             % {"python": config["PYTHON3"]}
 254         )
 255         for k, v in sorted(six.iteritems(sanitized_config)):
 256             fh.write("%s = " % k)
 257             pprint.pprint(v, stream=fh, indent=4)
 258         fh.write(
 259             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 260         )
 261
 262         if execute:
 263             fh.write(
 264                 textwrap.dedent(
 265                     """
 266                 if __name__ == '__main__':
 267                     from mozbuild.config_status import config_status
 268                     args = dict([(name, globals()[name]) for name in __all__])
 269                     config_status(**args)
 270             """
 271                 )
 272             )
 273
 274     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 275     partial_config.write_vars(sanitized_config)
 276
 277     # Write out a file so the build backend knows to re-run configure when
 278     # relevant Python changes.
 279     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 280         for f in sorted(
 281             itertools.chain(
 282                 config["CONFIG_STATUS_DEPS"],
 283                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 284             )
 285         ):
 286             fh.write("%s\n" % mozpath.normpath(f))
 287
 288     # Other things than us are going to run this file, so we need to give it
 289     # executable permissions.
 290     os.chmod("config.status", 0o755)
 291     if execute:
 292         from mozbuild.config_status import config_status
 293
 294         return config_status(args=[], **sanitized_config)
 295     return 0
 296
 297
 298 def _activate_build_virtualenv():
 299     """Ensure that the build virtualenv is activated
 300
 301     configure.py may be executed through Mach, or via "./configure, make".
 302     In the first case, the build virtualenv should already be activated.
 303     In the second case, we're likely being executed with the system Python, and must
 304     prepare the virtualenv and activate it ourselves.
 305     """
 306
 307     version = ".".join(str(i) for i in sys.version_info[0:3])
 308     print(f"Using Python {version} from {sys.executable}")
 309
 310     active_site = MozSiteMetadata.from_runtime()
 311     if active_site and active_site.site_name == "build":
 312         # We're already running within the "build" virtualenv, no additional work is
 313         # needed.
 314         return
 315
 316     # We're using the system python (or are nested within a non-build mach-managed
 317     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 318     # configure.
 319
 320     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 321
 322     mach_site = MachSiteManager(
 323         topsrcdir,
 324         None,
 325         MachEnvRequirements(),
 326         ExternalPythonSite(sys.executable),
 327         SitePackagesSource.NONE,
 328     )
 329     mach_site.activate()
 330
 331     from mach.util import get_virtualenv_base_dir
 332
 333     build_site = CommandSiteManager.from_environment(
 334         topsrcdir,
 335         None,
 336         "build",
 337         get_virtualenv_base_dir(topsrcdir),
 338     )
 339     if not build_site.ensure():
 340         print("Created Python 3 virtualenv")
 341     build_site.activate()
 342
 343
 344 if __name__ == "__main__":
 345     sys.exit(main(sys.argv))