configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import codecs
   6 import errno
   7 import io
   8 import itertools
   9 import logging
  10 import os
  11 import pprint
  12 import sys
  13 import textwrap
  14 from collections.abc import Iterable
  15
  16 base_dir = os.path.abspath(os.path.dirname(__file__))
  17 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  18 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  20 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  21 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "looseversion"))
  23 import mozpack.path as mozpath
  24 import six
  25 from mach.requirements import MachEnvRequirements
  26 from mach.site import (
  27     CommandSiteManager,
  28     ExternalPythonSite,
  29     MachSiteManager,
  30     MozSiteMetadata,
  31     SitePackagesSource,
  32 )
  33 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  34 from mozbuild.configure import TRACE, ConfigureSandbox
  35 from mozbuild.pythonutil import iter_modules_in_path
  36
  37
  38 def main(argv):
  39     # Check for CRLF line endings.
  40     with open(__file__, "r") as fh:
  41         data = fh.read()
  42         if "\r" in data:
  43             print(
  44                 "\n ***\n"
  45                 " * The source tree appears to have Windows-style line endings.\n"
  46                 " *\n"
  47                 " * If using Git, Git is likely configured to use Windows-style\n"
  48                 " * line endings.\n"
  49                 " *\n"
  50                 " * To convert the working copy to UNIX-style line endings, run\n"
  51                 " * the following:\n"
  52                 " *\n"
  53                 " * $ git config core.autocrlf false\n"
  54                 " * $ git config core.eof lf\n"
  55                 " * $ git rm --cached -r .\n"
  56                 " * $ git reset --hard\n"
  57                 " *\n"
  58                 " * If not using Git, the tool you used to obtain the source\n"
  59                 " * code likely converted files to Windows line endings. See\n"
  60                 " * usage information for that tool for more.\n"
  61                 " ***",
  62                 file=sys.stderr,
  63             )
  64             return 1
  65
  66     config = {}
  67
  68     if "OLD_CONFIGURE" not in os.environ:
  69         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  70
  71     sandbox = ConfigureSandbox(config, os.environ, argv)
  72
  73     if not sandbox._help:
  74         # This limitation has mostly to do with GNU make. Since make can't represent
  75         # variables with spaces without correct quoting and many paths are used
  76         # without proper quoting, using paths with spaces commonly results in
  77         # targets or dependencies being treated as multiple paths. This, of course,
  78         # undermines the ability for make to perform up-to-date checks and makes
  79         # the build system not work very efficiently. In theory, a non-make build
  80         # backend will make this limitation go away. But there is likely a long tail
  81         # of things that will need fixing due to e.g. lack of proper path quoting.
  82         topsrcdir = os.path.realpath(os.path.dirname(__file__))
  83         if len(topsrcdir.split()) > 1:
  84             print(
  85                 "Source directory cannot be located in a path with spaces: %s"
  86                 % topsrcdir,
  87                 file=sys.stderr,
  88             )
  89             return 1
  90         topobjdir = os.path.realpath(os.curdir)
  91         if len(topobjdir.split()) > 1:
  92             print(
  93                 "Object directory cannot be located in a path with spaces: %s"
  94                 % topobjdir,
  95                 file=sys.stderr,
  96             )
  97             return 1
  98
  99         # Do not allow topobjdir == topsrcdir
 100         if os.path.samefile(topsrcdir, topobjdir):
 101             print(
 102                 "  ***\n"
 103                 "  * Building directly in the main source directory is not allowed.\n"
 104                 "  *\n"
 105                 "  * To build, you must run configure from a separate directory\n"
 106                 "  * (referred to as an object directory).\n"
 107                 "  *\n"
 108                 "  * If you are building with a mozconfig, you will need to change your\n"
 109                 "  * mozconfig to point to a different object directory.\n"
 110                 "  ***",
 111                 file=sys.stderr,
 112             )
 113             return 1
 114         _activate_build_virtualenv()
 115
 116     clobber_file = "CLOBBER"
 117     if not os.path.exists(clobber_file):
 118         # Simply touch the file.
 119         with open(clobber_file, "a"):
 120             pass
 121
 122     if os.environ.get("MOZ_CONFIGURE_TRACE"):
 123         sandbox._logger.setLevel(TRACE)
 124
 125     sandbox.run(os.path.join(os.path.dirname(__file__), "moz.configure"))
 126
 127     if sandbox._help:
 128         return 0
 129
 130     logging.getLogger("moz.configure").info("Creating config.status")
 131
 132     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
 133     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
 134     if old_js_configure_substs or old_js_configure_defines:
 135         js_config = config.copy()
 136         pwd = os.getcwd()
 137         try:
 138             try:
 139                 os.makedirs("js/src")
 140             except OSError as e:
 141                 if e.errno != errno.EEXIST:
 142                     raise
 143
 144             os.chdir("js/src")
 145             js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
 146             js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
 147             # The build system frontend expects $objdir/js/src/config.status
 148             # to have $objdir/js/src as topobjdir.
 149             # We want forward slashes on all platforms.
 150             js_config["TOPOBJDIR"] += "/js/src"
 151             config_status(js_config, execute=False)
 152         finally:
 153             os.chdir(pwd)
 154
 155     return config_status(config)
 156
 157
 158 def check_unicode(obj):
 159     """Recursively check that all strings in the object are unicode strings."""
 160     if isinstance(obj, dict):
 161         result = True
 162         for k, v in six.iteritems(obj):
 163             if not check_unicode(k):
 164                 print("%s key is not unicode." % k, file=sys.stderr)
 165                 result = False
 166             elif not check_unicode(v):
 167                 print("%s value is not unicode." % k, file=sys.stderr)
 168                 result = False
 169         return result
 170     if isinstance(obj, bytes):
 171         return False
 172     if isinstance(obj, six.text_type):
 173         return True
 174     if isinstance(obj, Iterable):
 175         return all(check_unicode(o) for o in obj)
 176     return True
 177
 178
 179 def config_status(config, execute=True):
 180     # Sanitize config data to feed config.status
 181     # Ideally, all the backend and frontend code would handle the booleans, but
 182     # there are so many things involved, that it's easier to keep config.status
 183     # untouched for now.
 184     def sanitize_config(v):
 185         if v is True:
 186             return "1"
 187         if v is False:
 188             return ""
 189         # Serialize types that look like lists and tuples as lists.
 190         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 191             return list(v)
 192         return v
 193
 194     sanitized_config = {}
 195     sanitized_config["substs"] = {
 196         k: sanitize_config(v)
 197         for k, v in six.iteritems(config)
 198         if k
 199         not in (
 200             "DEFINES",
 201             "TOPSRCDIR",
 202             "TOPOBJDIR",
 203             "CONFIG_STATUS_DEPS",
 204             "OLD_CONFIGURE_SUBSTS",
 205             "OLD_CONFIGURE_DEFINES",
 206         )
 207     }
 208     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 209         sanitized_config["substs"][k] = sanitize_config(v)
 210     sanitized_config["defines"] = {
 211         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 212     }
 213     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 214         sanitized_config["defines"][k] = sanitize_config(v)
 215     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 216     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 217     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 218
 219     if not check_unicode(sanitized_config):
 220         print("Configuration should be all unicode.", file=sys.stderr)
 221         print("Please file a bug for the above.", file=sys.stderr)
 222         sys.exit(1)
 223
 224     # Create config.status. Eventually, we'll want to just do the work it does
 225     # here, when we're able to skip configure tests/use cached results/not rely
 226     # on autoconf.
 227     with codecs.open("config.status", "w", "utf-8") as fh:
 228         fh.write(
 229             textwrap.dedent(
 230                 """\
 231             #!%(python)s
 232             # coding=utf-8
 233             from mozbuild.configure.constants import *
 234         """
 235             )
 236             % {"python": config["PYTHON3"]}
 237         )
 238         for k, v in sorted(six.iteritems(sanitized_config)):
 239             fh.write("%s = " % k)
 240             pprint.pprint(v, stream=fh, indent=4)
 241         fh.write(
 242             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 243         )
 244
 245         if execute:
 246             fh.write(
 247                 textwrap.dedent(
 248                     """
 249                 if __name__ == '__main__':
 250                     from mozbuild.config_status import config_status
 251                     args = dict([(name, globals()[name]) for name in __all__])
 252                     config_status(**args)
 253             """
 254                 )
 255             )
 256
 257     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 258     partial_config.write_vars(sanitized_config)
 259
 260     # Write out a file so the build backend knows to re-run configure when
 261     # relevant Python changes.
 262     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 263         for f in sorted(
 264             itertools.chain(
 265                 config["CONFIG_STATUS_DEPS"],
 266                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 267             )
 268         ):
 269             fh.write("%s\n" % mozpath.normpath(f))
 270
 271     # Other things than us are going to run this file, so we need to give it
 272     # executable permissions.
 273     os.chmod("config.status", 0o755)
 274     if execute:
 275         from mozbuild.config_status import config_status
 276
 277         return config_status(args=[], **sanitized_config)
 278     return 0
 279
 280
 281 def _activate_build_virtualenv():
 282     """Ensure that the build virtualenv is activated
 283
 284     configure.py may be executed through Mach, or via "./configure, make".
 285     In the first case, the build virtualenv should already be activated.
 286     In the second case, we're likely being executed with the system Python, and must
 287     prepare the virtualenv and activate it ourselves.
 288     """
 289
 290     version = ".".join(str(i) for i in sys.version_info[0:3])
 291     print(f"Using Python {version} from {sys.executable}")
 292
 293     active_site = MozSiteMetadata.from_runtime()
 294     if active_site and active_site.site_name == "build":
 295         # We're already running within the "build" virtualenv, no additional work is
 296         # needed.
 297         return
 298
 299     # We're using the system python (or are nested within a non-build mach-managed
 300     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 301     # configure.
 302
 303     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 304
 305     mach_site = MachSiteManager(
 306         topsrcdir,
 307         None,
 308         MachEnvRequirements(),
 309         ExternalPythonSite(sys.executable),
 310         SitePackagesSource.NONE,
 311     )
 312     mach_site.activate()
 313
 314     from mach.util import get_virtualenv_base_dir
 315
 316     build_site = CommandSiteManager.from_environment(
 317         topsrcdir,
 318         None,
 319         "build",
 320         get_virtualenv_base_dir(topsrcdir),
 321     )
 322     if not build_site.ensure():
 323         print("Created Python 3 virtualenv")
 324     build_site.activate()
 325
 326
 327 if __name__ == "__main__":
 328     sys.exit(main(sys.argv))