python/mozbuild/mozbuild/mozconfig.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 import re
   7 import subprocess
   8 import sys
   9 import traceback
  10 from pathlib import Path
  11 from textwrap import dedent
  12
  13 import six
  14 from mozboot.mozconfig import find_mozconfig
  15 from mozpack import path as mozpath
  16
  17 MOZCONFIG_BAD_EXIT_CODE = """
  18 Evaluation of your mozconfig exited with an error. This could be triggered
  19 by a command inside your mozconfig failing. Please change your mozconfig
  20 to not error and/or to catch errors in executed commands.
  21 """.strip()
  22
  23 MOZCONFIG_BAD_OUTPUT = """
  24 Evaluation of your mozconfig produced unexpected output.  This could be
  25 triggered by a command inside your mozconfig failing or producing some warnings
  26 or error messages. Please change your mozconfig to not error and/or to catch
  27 errors in executed commands.
  28 """.strip()
  29
  30
  31 class MozconfigLoadException(Exception):
  32     """Raised when a mozconfig could not be loaded properly.
  33
  34     This typically indicates a malformed or misbehaving mozconfig file.
  35     """
  36
  37     def __init__(self, path, message, output=None):
  38         self.path = path
  39         self.output = output
  40
  41         message = (
  42             dedent(
  43                 """
  44         Error loading mozconfig: {path}
  45
  46         {message}
  47         """
  48             )
  49             .format(path=self.path, message=message)
  50             .lstrip()
  51         )
  52
  53         if self.output:
  54             message += dedent(
  55                 """
  56             mozconfig output:
  57
  58             {output}
  59             """
  60             ).format(output="\n".join([six.ensure_text(s) for s in self.output]))
  61
  62         Exception.__init__(self, message)
  63
  64
  65 class MozconfigLoader(object):
  66     """Handles loading and parsing of mozconfig files."""
  67
  68     RE_MAKE_VARIABLE = re.compile(
  69         r"""
  70         ^\s*                    # Leading whitespace
  71         (?P<var>[a-zA-Z_0-9]+)  # Variable name
  72         \s* [?:]?= \s*          # Assignment operator surrounded by optional
  73                                 # spaces
  74         (?P<value>.*$)""",  # Everything else (likely the value)
  75         re.VERBOSE,
  76     )
  77
  78     IGNORE_SHELL_VARIABLES = {"_", "BASH_ARGV", "BASH_ARGV0", "BASH_ARGC"}
  79
  80     ENVIRONMENT_VARIABLES = {"CC", "CXX", "CFLAGS", "CXXFLAGS", "LDFLAGS", "MOZ_OBJDIR"}
  81
  82     AUTODETECT = object()
  83
  84     def __init__(self, topsrcdir):
  85         self.topsrcdir = topsrcdir
  86
  87     @property
  88     def _loader_script(self):
  89         our_dir = os.path.abspath(os.path.dirname(__file__))
  90
  91         return os.path.join(our_dir, "mozconfig_loader")
  92
  93     def read_mozconfig(self, path=None):
  94         """Read the contents of a mozconfig into a data structure.
  95
  96         This takes the path to a mozconfig to load. If the given path is
  97         AUTODETECT, will try to find a mozconfig from the environment using
  98         find_mozconfig().
  99
 100         mozconfig files are shell scripts. So, we can't just parse them.
 101         Instead, we run the shell script in a wrapper which allows us to record
 102         state from execution. Thus, the output from a mozconfig is a friendly
 103         static data structure.
 104         """
 105         if path is self.AUTODETECT:
 106             path = find_mozconfig(self.topsrcdir)
 107         if isinstance(path, Path):
 108             path = str(path)
 109
 110         result = {
 111             "path": path,
 112             "topobjdir": None,
 113             "configure_args": None,
 114             "make_flags": None,
 115             "make_extra": None,
 116             "env": None,
 117             "vars": None,
 118         }
 119
 120         if path is None:
 121             if "MOZ_OBJDIR" in os.environ:
 122                 result["topobjdir"] = os.environ["MOZ_OBJDIR"]
 123             return result
 124
 125         path = mozpath.normsep(path)
 126
 127         result["configure_args"] = []
 128         result["make_extra"] = []
 129         result["make_flags"] = []
 130
 131         # Since mozconfig_loader is a shell script, running it "normally"
 132         # actually leads to two shell executions on Windows. Avoid this by
 133         # directly calling sh mozconfig_loader.
 134         shell = "sh"
 135         env = dict(os.environ)
 136         env["PYTHONIOENCODING"] = "utf-8"
 137
 138         if "MOZILLABUILD" in os.environ:
 139             mozillabuild = os.environ["MOZILLABUILD"]
 140             if (Path(mozillabuild) / "msys2").exists():
 141                 shell = mozillabuild + "/msys2/usr/bin/sh"
 142             else:
 143                 shell = mozillabuild + "/msys/bin/sh"
 144             prefer_mozillabuild_path = [
 145                 os.path.dirname(shell),
 146                 str(Path(mozillabuild) / "bin"),
 147                 env["PATH"],
 148             ]
 149             env["PATH"] = os.pathsep.join(prefer_mozillabuild_path)
 150         if sys.platform == "win32":
 151             shell = shell + ".exe"
 152
 153         command = [
 154             mozpath.normsep(shell),
 155             mozpath.normsep(self._loader_script),
 156             mozpath.normsep(self.topsrcdir),
 157             mozpath.normsep(path),
 158             mozpath.normsep(sys.executable),
 159             mozpath.join(mozpath.dirname(self._loader_script), "action", "dump_env.py"),
 160         ]
 161
 162         try:
 163             # We need to capture stderr because that's where the shell sends
 164             # errors if execution fails.
 165             output = six.ensure_text(
 166                 subprocess.check_output(
 167                     command,
 168                     stderr=subprocess.STDOUT,
 169                     cwd=self.topsrcdir,
 170                     env=env,
 171                     universal_newlines=True,
 172                     encoding="utf-8",
 173                 )
 174             )
 175         except subprocess.CalledProcessError as e:
 176             lines = e.output.splitlines()
 177
 178             # Output before actual execution shouldn't be relevant.
 179             try:
 180                 index = lines.index("------END_BEFORE_SOURCE")
 181                 lines = lines[index + 1 :]
 182             except ValueError:
 183                 pass
 184
 185             raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines)
 186
 187         try:
 188             parsed = self._parse_loader_output(output)
 189         except AssertionError:
 190             # _parse_loader_output uses assertions to verify the
 191             # well-formedness of the shell output; when these fail, it
 192             # generally means there was a problem with the output, but we
 193             # include the assertion traceback just to be sure.
 194             print("Assertion failed in _parse_loader_output:")
 195             traceback.print_exc()
 196             raise MozconfigLoadException(
 197                 path, MOZCONFIG_BAD_OUTPUT, output.splitlines()
 198             )
 199
 200         def diff_vars(vars_before, vars_after):
 201             set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES
 202             set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES
 203             added = set2 - set1
 204             removed = set1 - set2
 205             maybe_modified = set1 & set2
 206             changed = {"added": {}, "removed": {}, "modified": {}, "unmodified": {}}
 207
 208             for key in added:
 209                 changed["added"][key] = vars_after[key]
 210
 211             for key in removed:
 212                 changed["removed"][key] = vars_before[key]
 213
 214             for key in maybe_modified:
 215                 if vars_before[key] != vars_after[key]:
 216                     changed["modified"][key] = (vars_before[key], vars_after[key])
 217                 elif key in self.ENVIRONMENT_VARIABLES:
 218                     # In order for irrelevant environment variable changes not
 219                     # to incur in re-running configure, only a set of
 220                     # environment variables are stored when they are
 221                     # unmodified. Otherwise, changes such as using a different
 222                     # terminal window, or even rebooting, would trigger
 223                     # reconfigures.
 224                     changed["unmodified"][key] = vars_after[key]
 225
 226             return changed
 227
 228         result["env"] = diff_vars(parsed["env_before"], parsed["env_after"])
 229
 230         # Environment variables also appear as shell variables, but that's
 231         # uninteresting duplication of information. Filter them out.
 232         def filt(x, y):
 233             return {k: v for k, v in x.items() if k not in y}
 234
 235         result["vars"] = diff_vars(
 236             filt(parsed["vars_before"], parsed["env_before"]),
 237             filt(parsed["vars_after"], parsed["env_after"]),
 238         )
 239
 240         result["configure_args"] = [self._expand(o) for o in parsed["ac"]]
 241
 242         if "MOZ_OBJDIR" in parsed["env_before"]:
 243             result["topobjdir"] = parsed["env_before"]["MOZ_OBJDIR"]
 244
 245         mk = [self._expand(o) for o in parsed["mk"]]
 246
 247         for o in mk:
 248             match = self.RE_MAKE_VARIABLE.match(o)
 249
 250             if match is None:
 251                 result["make_extra"].append(o)
 252                 continue
 253
 254             name, value = match.group("var"), match.group("value")
 255
 256             if name == "MOZ_MAKE_FLAGS":
 257                 result["make_flags"] = value.split()
 258                 continue
 259
 260             if name == "MOZ_OBJDIR":
 261                 result["topobjdir"] = value
 262                 if parsed["env_before"].get("MOZ_PROFILE_GENERATE") == "1":
 263                     # If MOZ_OBJDIR is specified in the mozconfig, we need to
 264                     # make sure that the '/instrumented' directory gets appended
 265                     # for the first build to avoid an objdir mismatch when
 266                     # running 'mach package' on Windows.
 267                     result["topobjdir"] = mozpath.join(
 268                         result["topobjdir"], "instrumented"
 269                     )
 270                 continue
 271
 272             result["make_extra"].append(o)
 273
 274         return result
 275
 276     def _parse_loader_output(self, output):
 277         mk_options = []
 278         ac_options = []
 279         before_source = {}
 280         after_source = {}
 281         env_before_source = {}
 282         env_after_source = {}
 283
 284         current = None
 285         current_type = None
 286         in_variable = None
 287
 288         for line in output.splitlines():
 289             if not line:
 290                 continue
 291
 292             if line.startswith("------BEGIN_"):
 293                 assert current_type is None
 294                 assert current is None
 295                 assert not in_variable
 296                 current_type = line[len("------BEGIN_") :]
 297                 current = []
 298                 continue
 299
 300             if line.startswith("------END_"):
 301                 assert not in_variable
 302                 section = line[len("------END_") :]
 303                 assert current_type == section
 304
 305                 if current_type == "AC_OPTION":
 306                     ac_options.append("\n".join(current))
 307                 elif current_type == "MK_OPTION":
 308                     mk_options.append("\n".join(current))
 309
 310                 current = None
 311                 current_type = None
 312                 continue
 313
 314             assert current_type is not None
 315
 316             vars_mapping = {
 317                 "BEFORE_SOURCE": before_source,
 318                 "AFTER_SOURCE": after_source,
 319                 "ENV_BEFORE_SOURCE": env_before_source,
 320                 "ENV_AFTER_SOURCE": env_after_source,
 321             }
 322
 323             if current_type in vars_mapping:
 324                 # mozconfigs are sourced using the Bourne shell (or at least
 325                 # in Bourne shell mode). This means |set| simply lists
 326                 # variables from the current shell (not functions). (Note that
 327                 # if Bash is installed in /bin/sh it acts like regular Bourne
 328                 # and doesn't print functions.) So, lines should have the
 329                 # form:
 330                 #
 331                 #  key='value'
 332                 #  key=value
 333                 #
 334                 # The only complication is multi-line variables. Those have the
 335                 # form:
 336                 #
 337                 #  key='first
 338                 #  second'
 339
 340                 # TODO Bug 818377 Properly handle multi-line variables of form:
 341                 # $ foo="a='b'
 342                 # c='d'"
 343                 # $ set
 344                 # foo='a='"'"'b'"'"'
 345                 # c='"'"'d'"'"
 346
 347                 name = in_variable
 348                 value = None
 349                 if in_variable:
 350                     # Reached the end of a multi-line variable.
 351                     if line.endswith("'") and not line.endswith("\\'"):
 352                         current.append(line[:-1])
 353                         value = "\n".join(current)
 354                         in_variable = None
 355                     else:
 356                         current.append(line)
 357                         continue
 358                 else:
 359                     equal_pos = line.find("=")
 360
 361                     if equal_pos < 1:
 362                         # TODO log warning?
 363                         continue
 364
 365                     name = line[0:equal_pos]
 366                     value = line[equal_pos + 1 :]
 367
 368                     if len(value):
 369                         has_quote = value[0] == "'"
 370
 371                         if has_quote:
 372                             value = value[1:]
 373
 374                         # Lines with a quote not ending in a quote are multi-line.
 375                         if has_quote and not value.endswith("'"):
 376                             in_variable = name
 377                             current.append(value)
 378                             continue
 379                         else:
 380                             value = value[:-1] if has_quote else value
 381
 382                 assert name is not None
 383
 384                 vars_mapping[current_type][name] = value
 385
 386                 current = []
 387
 388                 continue
 389
 390             current.append(line)
 391
 392         return {
 393             "mk": mk_options,
 394             "ac": ac_options,
 395             "vars_before": before_source,
 396             "vars_after": after_source,
 397             "env_before": env_before_source,
 398             "env_after": env_after_source,
 399         }
 400
 401     def _expand(self, s):
 402         return s.replace("@TOPSRCDIR@", self.topsrcdir)