configure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import codecs
   6 import errno
   7 import io
   8 import itertools
   9 import logging
  10 import os
  11 import pprint
  12 import sys
  13 import textwrap
  14 from collections.abc import Iterable
  15
  16 base_dir = os.path.abspath(os.path.dirname(__file__))
  17 sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
  18 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
  19 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
  20 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
  21 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
  22 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "looseversion"))
  23 import mozpack.path as mozpath
  24 import six
  25 from mach.requirements import MachEnvRequirements
  26 from mach.site import (
  27     CommandSiteManager,
  28     ExternalPythonSite,
  29     MachSiteManager,
  30     MozSiteMetadata,
  31     SitePackagesSource,
  32 )
  33 from mozbuild.backend.configenvironment import PartialConfigEnvironment
  34 from mozbuild.configure import TRACE, ConfigureSandbox
  35 from mozbuild.pythonutil import iter_modules_in_path
  36
  37 if "MOZ_CONFIGURE_BUILDSTATUS" in os.environ:
  38
  39     def buildstatus(message):
  40         print("BUILDSTATUS", message)
  41
  42 else:
  43
  44     def buildstatus(message):
  45         return
  46
  47
  48 def main(argv):
  49     # Check for CRLF line endings.
  50     with open(__file__, "r") as fh:
  51         data = fh.read()
  52         if "\r" in data:
  53             print(
  54                 "\n ***\n"
  55                 " * The source tree appears to have Windows-style line endings.\n"
  56                 " *\n"
  57                 " * If using Git, Git is likely configured to use Windows-style\n"
  58                 " * line endings.\n"
  59                 " *\n"
  60                 " * To convert the working copy to UNIX-style line endings, run\n"
  61                 " * the following:\n"
  62                 " *\n"
  63                 " * $ git config core.autocrlf false\n"
  64                 " * $ git config core.eof lf\n"
  65                 " * $ git rm --cached -r .\n"
  66                 " * $ git reset --hard\n"
  67                 " *\n"
  68                 " * If not using Git, the tool you used to obtain the source\n"
  69                 " * code likely converted files to Windows line endings. See\n"
  70                 " * usage information for that tool for more.\n"
  71                 " ***",
  72                 file=sys.stderr,
  73             )
  74             return 1
  75
  76     config = {}
  77
  78     if "OLD_CONFIGURE" not in os.environ:
  79         os.environ["OLD_CONFIGURE"] = os.path.join(base_dir, "old-configure")
  80
  81     sandbox = ConfigureSandbox(config, os.environ, argv)
  82
  83     if not sandbox._help:
  84         # This limitation has mostly to do with GNU make. Since make can't represent
  85         # variables with spaces without correct quoting and many paths are used
  86         # without proper quoting, using paths with spaces commonly results in
  87         # targets or dependencies being treated as multiple paths. This, of course,
  88         # undermines the ability for make to perform up-to-date checks and makes
  89         # the build system not work very efficiently. In theory, a non-make build
  90         # backend will make this limitation go away. But there is likely a long tail
  91         # of things that will need fixing due to e.g. lack of proper path quoting.
  92         topsrcdir = os.path.realpath(os.path.dirname(__file__))
  93         if len(topsrcdir.split()) > 1:
  94             print(
  95                 "Source directory cannot be located in a path with spaces: %s"
  96                 % topsrcdir,
  97                 file=sys.stderr,
  98             )
  99             return 1
 100         topobjdir = os.path.realpath(os.curdir)
 101         if len(topobjdir.split()) > 1:
 102             print(
 103                 "Object directory cannot be located in a path with spaces: %s"
 104                 % topobjdir,
 105                 file=sys.stderr,
 106             )
 107             return 1
 108
 109         # Do not allow topobjdir == topsrcdir
 110         if os.path.samefile(topsrcdir, topobjdir):
 111             print(
 112                 "  ***\n"
 113                 "  * Building directly in the main source directory is not allowed.\n"
 114                 "  *\n"
 115                 "  * To build, you must run configure from a separate directory\n"
 116                 "  * (referred to as an object directory).\n"
 117                 "  *\n"
 118                 "  * If you are building with a mozconfig, you will need to change your\n"
 119                 "  * mozconfig to point to a different object directory.\n"
 120                 "  ***",
 121                 file=sys.stderr,
 122             )
 123             return 1
 124         buildstatus("START_configure activate virtualenv")
 125         _activate_build_virtualenv()
 126         buildstatus("END_configure activate virtualenv")
 127
 128     clobber_file = "CLOBBER"
 129     if not os.path.exists(clobber_file):
 130         # Simply touch the file.
 131         with open(clobber_file, "a"):
 132             pass
 133
 134     if os.environ.get("MOZ_CONFIGURE_TRACE"):
 135         sandbox._logger.setLevel(TRACE)
 136
 137     buildstatus("START_configure read moz.configure")
 138     sandbox.include_file(os.path.join(os.path.dirname(__file__), "moz.configure"))
 139     buildstatus("END_configure read moz.configure")
 140     buildstatus("START_configure run moz.configure")
 141     sandbox.run()
 142     buildstatus("END_configure run moz.configure")
 143
 144     if sandbox._help:
 145         return 0
 146
 147     buildstatus("START_configure config.status")
 148     logging.getLogger("moz.configure").info("Creating config.status")
 149
 150     old_js_configure_substs = config.pop("OLD_JS_CONFIGURE_SUBSTS", None)
 151     old_js_configure_defines = config.pop("OLD_JS_CONFIGURE_DEFINES", None)
 152     try:
 153         if old_js_configure_substs or old_js_configure_defines:
 154             js_config = config.copy()
 155             pwd = os.getcwd()
 156             try:
 157                 try:
 158                     os.makedirs("js/src")
 159                 except OSError as e:
 160                     if e.errno != errno.EEXIST:
 161                         raise
 162
 163                 os.chdir("js/src")
 164                 js_config["OLD_CONFIGURE_SUBSTS"] = old_js_configure_substs
 165                 js_config["OLD_CONFIGURE_DEFINES"] = old_js_configure_defines
 166                 # The build system frontend expects $objdir/js/src/config.status
 167                 # to have $objdir/js/src as topobjdir.
 168                 # We want forward slashes on all platforms.
 169                 js_config["TOPOBJDIR"] += "/js/src"
 170                 ret = config_status(js_config, execute=False)
 171                 if ret:
 172                     return ret
 173             finally:
 174                 os.chdir(pwd)
 175
 176         return config_status(config)
 177     finally:
 178         buildstatus("END_configure config.status")
 179
 180
 181 def check_unicode(obj):
 182     """Recursively check that all strings in the object are unicode strings."""
 183     if isinstance(obj, dict):
 184         result = True
 185         for k, v in six.iteritems(obj):
 186             if not check_unicode(k):
 187                 print("%s key is not unicode." % k, file=sys.stderr)
 188                 result = False
 189             elif not check_unicode(v):
 190                 print("%s value is not unicode." % k, file=sys.stderr)
 191                 result = False
 192         return result
 193     if isinstance(obj, bytes):
 194         return False
 195     if isinstance(obj, six.text_type):
 196         return True
 197     if isinstance(obj, Iterable):
 198         return all(check_unicode(o) for o in obj)
 199     return True
 200
 201
 202 def config_status(config, execute=True):
 203     # Sanitize config data to feed config.status
 204     # Ideally, all the backend and frontend code would handle the booleans, but
 205     # there are so many things involved, that it's easier to keep config.status
 206     # untouched for now.
 207     def sanitize_config(v):
 208         if v is True:
 209             return "1"
 210         if v is False:
 211             return ""
 212         # Serialize types that look like lists and tuples as lists.
 213         if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(v, Iterable):
 214             return list(v)
 215         return v
 216
 217     sanitized_config = {}
 218     sanitized_config["substs"] = {
 219         k: sanitize_config(v)
 220         for k, v in six.iteritems(config)
 221         if k
 222         not in (
 223             "DEFINES",
 224             "TOPSRCDIR",
 225             "TOPOBJDIR",
 226             "CONFIG_STATUS_DEPS",
 227             "OLD_CONFIGURE_SUBSTS",
 228             "OLD_CONFIGURE_DEFINES",
 229         )
 230     }
 231     for k, v in config["OLD_CONFIGURE_SUBSTS"]:
 232         sanitized_config["substs"][k] = sanitize_config(v)
 233     sanitized_config["defines"] = {
 234         k: sanitize_config(v) for k, v in six.iteritems(config["DEFINES"])
 235     }
 236     for k, v in config["OLD_CONFIGURE_DEFINES"]:
 237         sanitized_config["defines"][k] = sanitize_config(v)
 238     sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
 239     sanitized_config["topobjdir"] = config["TOPOBJDIR"]
 240     sanitized_config["mozconfig"] = config.get("MOZCONFIG")
 241
 242     if not check_unicode(sanitized_config):
 243         print("Configuration should be all unicode.", file=sys.stderr)
 244         print("Please file a bug for the above.", file=sys.stderr)
 245         return 1
 246
 247     # Create config.status. Eventually, we'll want to just do the work it does
 248     # here, when we're able to skip configure tests/use cached results/not rely
 249     # on autoconf.
 250     with codecs.open("config.status", "w", "utf-8") as fh:
 251         fh.write(
 252             textwrap.dedent(
 253                 """\
 254             #!%(python)s
 255             # coding=utf-8
 256             from mozbuild.configure.constants import *
 257         """
 258             )
 259             % {"python": config["PYTHON3"]}
 260         )
 261         for k, v in sorted(six.iteritems(sanitized_config)):
 262             fh.write("%s = " % k)
 263             pprint.pprint(v, stream=fh, indent=4)
 264         fh.write(
 265             "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']"
 266         )
 267
 268         if execute:
 269             fh.write(
 270                 textwrap.dedent(
 271                     """
 272                 if __name__ == '__main__':
 273                     from mozbuild.config_status import config_status
 274                     args = dict([(name, globals()[name]) for name in __all__])
 275                     config_status(**args)
 276             """
 277                 )
 278             )
 279
 280     partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
 281     partial_config.write_vars(sanitized_config)
 282
 283     # Write out a file so the build backend knows to re-run configure when
 284     # relevant Python changes.
 285     with io.open("config_status_deps.in", "w", encoding="utf-8", newline="\n") as fh:
 286         for f in sorted(
 287             itertools.chain(
 288                 config["CONFIG_STATUS_DEPS"],
 289                 iter_modules_in_path(config["TOPOBJDIR"], config["TOPSRCDIR"]),
 290             )
 291         ):
 292             fh.write("%s\n" % mozpath.normpath(f))
 293
 294     # Other things than us are going to run this file, so we need to give it
 295     # executable permissions.
 296     os.chmod("config.status", 0o755)
 297     if execute:
 298         from mozbuild.config_status import config_status
 299
 300         return config_status(args=[], **sanitized_config)
 301     return 0
 302
 303
 304 def _activate_build_virtualenv():
 305     """Ensure that the build virtualenv is activated
 306
 307     configure.py may be executed through Mach, or via "./configure, make".
 308     In the first case, the build virtualenv should already be activated.
 309     In the second case, we're likely being executed with the system Python, and must
 310     prepare the virtualenv and activate it ourselves.
 311     """
 312
 313     version = ".".join(str(i) for i in sys.version_info[0:3])
 314     print(f"Using Python {version} from {sys.executable}")
 315
 316     active_site = MozSiteMetadata.from_runtime()
 317     if active_site and active_site.site_name == "build":
 318         # We're already running within the "build" virtualenv, no additional work is
 319         # needed.
 320         return
 321
 322     # We're using the system python (or are nested within a non-build mach-managed
 323     # virtualenv), so we should activate the build virtualenv as expected by the rest of
 324     # configure.
 325
 326     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 327
 328     mach_site = MachSiteManager(
 329         topsrcdir,
 330         None,
 331         MachEnvRequirements(),
 332         ExternalPythonSite(sys.executable),
 333         SitePackagesSource.NONE,
 334     )
 335     mach_site.activate()
 336
 337     from mach.util import get_virtualenv_base_dir
 338
 339     build_site = CommandSiteManager.from_environment(
 340         topsrcdir,
 341         None,
 342         "build",
 343         get_virtualenv_base_dir(topsrcdir),
 344     )
 345     if not build_site.ensure():
 346         print("Created Python 3 virtualenv")
 347     build_site.activate()
 348
 349
 350 if __name__ == "__main__":
 351     sys.exit(main(sys.argv))