configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import codecs
   6 import errno
   7 import io
   8 import itertools
   9 import logging
  10 import os
  11 import pprint
  12 import sys
  13 import textwrap
  14 from collections.abc import Iterable
  15
  16 base_dir = os.path.abspath(os.path.dirname(__file__))
  17 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  18 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  20 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  21 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  23 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "looseversion"))
  24 import mozpack.path as mozpath
  25 import six
  26 from mach.requirements import MachEnvRequirements
  27 from mach.site import (
  28     CommandSiteManager,
  29     ExternalPythonSite,
  30     MachSiteManager,
  31     MozSiteMetadata,
  32     SitePackagesSource,
  33 )
  34 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  35 from mozbuild.configure import TRACE, ConfigureSandbox
  36 from mozbuild.pythonutil import iter_modules_in_path
  37
  38
  39 def main(argv):
  40     # Check for CRLF line endings.
  41     with open(__file__, "r") as fh:
  42         data = fh.read()
  43         if "\r" in data:
  44             print(
  45                 "\n ***\n"
  46                 " * The source tree appears to have Windows-style line endings.\n"
  47                 " *\n"
  48                 " * If using Git, Git is likely configured to use Windows-style\n"
  49                 " * line endings.\n"
  50                 " *\n"
  51                 " * To convert the working copy to UNIX-style line endings, run\n"
  52                 " * the following:\n"
  53                 " *\n"
  54                 " * $ git config core.autocrlf false\n"
  55                 " * $ git config core.eof lf\n"
  56                 " * $ git rm --cached -r .\n"
  57                 " * $ git reset --hard\n"
  58                 " *\n"
  59                 " * If not using Git, the tool you used to obtain the source\n"
  60                 " * code likely converted files to Windows line endings. See\n"
  61                 " * usage information for that tool for more.\n"
  62                 " ***",
  63                 file=sys.stderr,
  64             )
  65             return 1
  66
  67     config = {}
  68
  69     if "OLD_CONFIGURE" not in os.environ:
  70         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  71
  72     sandbox = ConfigureSandbox(config, os.environ, argv)
  73
  74     if not sandbox._help:
  75         # This limitation has mostly to do with GNU make. Since make can't represent
  76         # variables with spaces without correct quoting and many paths are used
  77         # without proper quoting, using paths with spaces commonly results in
  78         # targets or dependencies being treated as multiple paths. This, of course,
  79         # undermines the ability for make to perform up-to-date checks and makes
  80         # the build system not work very efficiently. In theory, a non-make build
  81         # backend will make this limitation go away. But there is likely a long tail
  82         # of things that will need fixing due to e.g. lack of proper path quoting.
  83         topsrcdir = os.path.realpath(os.path.dirname(__file__))
  84         if len(topsrcdir.split()) > 1:
  85             print(
  86                 "Source directory cannot be located in a path with spaces: %s"
  87                 % topsrcdir,
  88                 file=sys.stderr,
  89             )
  90             return 1
  91         topobjdir = os.path.realpath(os.curdir)
  92         if len(topobjdir.split()) > 1:
  93             print(
  94                 "Object directory cannot be located in a path with spaces: %s"
  95                 % topobjdir,
  96                 file=sys.stderr,
  97             )
  98             return 1
  99
 100         # Do not allow topobjdir == topsrcdir
 101         if os.path.samefile(topsrcdir, topobjdir):
 102             print(
 103                 "  ***\n"
 104                 "  * Building directly in the main source directory is not allowed.\n"
 105                 "  *\n"
 106                 "  * To build, you must run configure from a separate directory\n"
 107                 "  * (referred to as an object directory).\n"
 108                 "  *\n"
 109                 "  * If you are building with a mozconfig, you will need to change your\n"
 110                 "  * mozconfig to point to a different object directory.\n"
 111                 "  ***",
 112                 file=sys.stderr,
 113             )
 114             return 1
 115         _activate_build_virtualenv()
 116
 117     clobber_file = "CLOBBER"
 118     if not os.path.exists(clobber_file):
 119         # Simply touch the file.
 120         with open(clobber_file, "a"):
 121             pass
 122
 123     if os.environ.get("MOZ_CONFIGURE_TRACE"):
 124         sandbox._logger.setLevel(TRACE)
 125
 126     sandbox.run(os.path.join(os.path.dirname(__file__), "moz.configure"))
 127
 128     if sandbox._help:
 129         return 0
 130
 131     logging.getLogger("moz.configure").info("Creating config.status")
 132
 133     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
 134     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
 135     if old_js_configure_substs or old_js_configure_defines:
 136         js_config = config.copy()
 137         pwd = os.getcwd()
 138         try:
 139             try:
 140                 os.makedirs("js/src")
 141             except OSError as e:
 142                 if e.errno != errno.EEXIST:
 143                     raise
 144
 145             os.chdir("js/src")
 146             js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
 147             js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
 148             # The build system frontend expects $objdir/js/src/config.status
 149             # to have $objdir/js/src as topobjdir.
 150             # We want forward slashes on all platforms.
 151             js_config["TOPOBJDIR"] += "/js/src"
 152             config_status(js_config, execute=False)
 153         finally:
 154             os.chdir(pwd)
 155
 156     return config_status(config)
 157
 158
 159 def check_unicode(obj):
 160     """Recursively check that all strings in the object are unicode strings."""
 161     if isinstance(obj, dict):
 162         result = True
 163         for k, v in six.iteritems(obj):
 164             if not check_unicode(k):
 165                 print("%s key is not unicode." % k, file=sys.stderr)
 166                 result = False
 167             elif not check_unicode(v):
 168                 print("%s value is not unicode." % k, file=sys.stderr)
 169                 result = False
 170         return result
 171     if isinstance(obj, bytes):
 172         return False
 173     if isinstance(obj, six.text_type):
 174         return True
 175     if isinstance(obj, Iterable):
 176         return all(check_unicode(o) for o in obj)
 177     return True
 178
 179
 180 def config_status(config, execute=True):
 181     # Sanitize config data to feed config.status
 182     # Ideally, all the backend and frontend code would handle the booleans, but
 183     # there are so many things involved, that it's easier to keep config.status
 184     # untouched for now.
 185     def sanitize_config(v):
 186         if v is True:
 187             return "1"
 188         if v is False:
 189             return ""
 190         # Serialize types that look like lists and tuples as lists.
 191         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 192             return list(v)
 193         return v
 194
 195     sanitized_config = {}
 196     sanitized_config["substs"] = {
 197         k: sanitize_config(v)
 198         for k, v in six.iteritems(config)
 199         if k
 200         not in (
 201             "DEFINES",
 202             "TOPSRCDIR",
 203             "TOPOBJDIR",
 204             "CONFIG_STATUS_DEPS",
 205             "OLD_CONFIGURE_SUBSTS",
 206             "OLD_CONFIGURE_DEFINES",
 207         )
 208     }
 209     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 210         sanitized_config["substs"][k] = sanitize_config(v)
 211     sanitized_config["defines"] = {
 212         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 213     }
 214     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 215         sanitized_config["defines"][k] = sanitize_config(v)
 216     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 217     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 218     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 219
 220     if not check_unicode(sanitized_config):
 221         print("Configuration should be all unicode.", file=sys.stderr)
 222         print("Please file a bug for the above.", file=sys.stderr)
 223         sys.exit(1)
 224
 225     # Create config.status. Eventually, we'll want to just do the work it does
 226     # here, when we're able to skip configure tests/use cached results/not rely
 227     # on autoconf.
 228     with codecs.open("config.status", "w", "utf-8") as fh:
 229         fh.write(
 230             textwrap.dedent(
 231                 """\
 232             #!%(python)s
 233             # coding=utf-8
 234             from mozbuild.configure.constants import *
 235         """
 236             )
 237             % {"python": config["PYTHON3"]}
 238         )
 239         for k, v in sorted(six.iteritems(sanitized_config)):
 240             fh.write("%s = " % k)
 241             pprint.pprint(v, stream=fh, indent=4)
 242         fh.write(
 243             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 244         )
 245
 246         if execute:
 247             fh.write(
 248                 textwrap.dedent(
 249                     """
 250                 if __name__ == '__main__':
 251                     from mozbuild.config_status import config_status
 252                     args = dict([(name, globals()[name]) for name in __all__])
 253                     config_status(**args)
 254             """
 255                 )
 256             )
 257
 258     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 259     partial_config.write_vars(sanitized_config)
 260
 261     # Write out a file so the build backend knows to re-run configure when
 262     # relevant Python changes.
 263     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 264         for f in sorted(
 265             itertools.chain(
 266                 config["CONFIG_STATUS_DEPS"],
 267                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 268             )
 269         ):
 270             fh.write("%s\n" % mozpath.normpath(f))
 271
 272     # Other things than us are going to run this file, so we need to give it
 273     # executable permissions.
 274     os.chmod("config.status", 0o755)
 275     if execute:
 276         from mozbuild.config_status import config_status
 277
 278         return config_status(args=[], **sanitized_config)
 279     return 0
 280
 281
 282 def _activate_build_virtualenv():
 283     """Ensure that the build virtualenv is activated
 284
 285     configure.py may be executed through Mach, or via "./configure, make".
 286     In the first case, the build virtualenv should already be activated.
 287     In the second case, we're likely being executed with the system Python, and must
 288     prepare the virtualenv and activate it ourselves.
 289     """
 290
 291     version = ".".join(str(i) for i in sys.version_info[0:3])
 292     print(f"Using Python {version} from {sys.executable}")
 293
 294     active_site = MozSiteMetadata.from_runtime()
 295     if active_site and active_site.site_name == "build":
 296         # We're already running within the "build" virtualenv, no additional work is
 297         # needed.
 298         return
 299
 300     # We're using the system python (or are nested within a non-build mach-managed
 301     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 302     # configure.
 303
 304     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 305
 306     mach_site = MachSiteManager(
 307         topsrcdir,
 308         None,
 309         MachEnvRequirements(),
 310         ExternalPythonSite(sys.executable),
 311         SitePackagesSource.NONE,
 312     )
 313     mach_site.activate()
 314
 315     from mach.util import get_virtualenv_base_dir
 316
 317     build_site = CommandSiteManager.from_environment(
 318         topsrcdir,
 319         None,
 320         "build",
 321         get_virtualenv_base_dir(topsrcdir),
 322     )
 323     if not build_site.ensure():
 324         print("Created Python 3 virtualenv")
 325     build_site.activate()
 326
 327
 328 if __name__ == "__main__":
 329     sys.exit(main(sys.argv))