python/mozbuild/mozbuild/backend/base.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import errno
   6 import io
   7 import itertools
   8 import os
   9 import time
  10 from abc import ABCMeta, abstractmethod
  11 from contextlib import contextmanager
  12
  13 import mozpack.path as mozpath
  14 import six
  15 from mach.mixin.logging import LoggingMixin
  16
  17 from mozbuild.base import ExecutionSummary
  18
  19 from ..frontend.data import ContextDerived
  20 from ..frontend.reader import EmptyConfig
  21 from ..preprocessor import Preprocessor
  22 from ..pythonutil import iter_modules_in_path
  23 from ..util import FileAvoidWrite, simple_diff
  24 from .configenvironment import ConfigEnvironment
  25
  26
  27 class BuildBackend(LoggingMixin):
  28     """Abstract base class for build backends.
  29
  30     A build backend is merely a consumer of the build configuration (the output
  31     of the frontend processing). It does something with said data. What exactly
  32     is the discretion of the specific implementation.
  33     """
  34
  35     __metaclass__ = ABCMeta
  36
  37     def __init__(self, environment):
  38         assert isinstance(environment, (ConfigEnvironment, EmptyConfig))
  39         self.populate_logger()
  40
  41         self.environment = environment
  42
  43         # Files whose modification should cause a new read and backend
  44         # generation.
  45         self.backend_input_files = set()
  46
  47         # Files generated by the backend.
  48         self._backend_output_files = set()
  49
  50         self._environments = {}
  51         self._environments[environment.topobjdir] = environment
  52
  53         # The number of backend files created.
  54         self._created_count = 0
  55
  56         # The number of backend files updated.
  57         self._updated_count = 0
  58
  59         # The number of unchanged backend files.
  60         self._unchanged_count = 0
  61
  62         # The number of deleted backend files.
  63         self._deleted_count = 0
  64
  65         # The total wall time spent in the backend. This counts the time the
  66         # backend writes out files, etc.
  67         self._execution_time = 0.0
  68
  69         # Mapping of changed file paths to diffs of the changes.
  70         self.file_diffs = {}
  71
  72         self.dry_run = False
  73
  74         self._init()
  75
  76     def summary(self):
  77         return ExecutionSummary(
  78             self.__class__.__name__.replace("Backend", "")
  79             + " backend executed in {execution_time:.2f}s\n  "
  80             "{total:d} total backend files; "
  81             "{created:d} created; "
  82             "{updated:d} updated; "
  83             "{unchanged:d} unchanged; "
  84             "{deleted:d} deleted",
  85             execution_time=self._execution_time,
  86             total=self._created_count + self._updated_count + self._unchanged_count,
  87             created=self._created_count,
  88             updated=self._updated_count,
  89             unchanged=self._unchanged_count,
  90             deleted=self._deleted_count,
  91         )
  92
  93     def _init(self):
  94         """Hook point for child classes to perform actions during __init__.
  95
  96         This exists so child classes don't need to implement __init__.
  97         """
  98
  99     def consume(self, objs):
 100         """Consume a stream of TreeMetadata instances.
 101
 102         This is the main method of the interface. This is what takes the
 103         frontend output and does something with it.
 104
 105         Child classes are not expected to implement this method. Instead, the
 106         base class consumes objects and calls methods (possibly) implemented by
 107         child classes.
 108         """
 109
 110         # Previously generated files.
 111         list_file = mozpath.join(
 112             self.environment.topobjdir, "backend.%s" % self.__class__.__name__
 113         )
 114         backend_output_list = set()
 115         if os.path.exists(list_file):
 116             with open(list_file) as fh:
 117                 backend_output_list.update(
 118                     mozpath.normsep(p) for p in fh.read().splitlines()
 119                 )
 120
 121         for obj in objs:
 122             obj_start = time.monotonic()
 123             if not self.consume_object(obj) and not isinstance(self, PartialBackend):
 124                 raise Exception("Unhandled object of type %s" % type(obj))
 125             self._execution_time += time.monotonic() - obj_start
 126
 127             if isinstance(obj, ContextDerived) and not isinstance(self, PartialBackend):
 128                 self.backend_input_files |= obj.context_all_paths
 129
 130         # Pull in all loaded Python as dependencies so any Python changes that
 131         # could influence our output result in a rescan.
 132         self.backend_input_files |= set(
 133             iter_modules_in_path(self.environment.topsrcdir, self.environment.topobjdir)
 134         )
 135
 136         finished_start = time.monotonic()
 137         self.consume_finished()
 138         self._execution_time += time.monotonic() - finished_start
 139
 140         # Purge backend files created in previous run, but not created anymore
 141         delete_files = backend_output_list - self._backend_output_files
 142         for path in delete_files:
 143             full_path = mozpath.join(self.environment.topobjdir, path)
 144             try:
 145                 with io.open(full_path, mode="r", encoding="utf-8") as existing:
 146                     old_content = existing.read()
 147                     if old_content:
 148                         self.file_diffs[full_path] = simple_diff(
 149                             full_path, old_content.splitlines(), None
 150                         )
 151             except IOError:
 152                 pass
 153             try:
 154                 if not self.dry_run:
 155                     os.unlink(full_path)
 156                 self._deleted_count += 1
 157             except OSError:
 158                 pass
 159         # Remove now empty directories
 160         for dir in set(mozpath.dirname(d) for d in delete_files):
 161             try:
 162                 os.removedirs(dir)
 163             except OSError:
 164                 pass
 165
 166         # Write out the list of backend files generated, if it changed.
 167         if backend_output_list != self._backend_output_files:
 168             with self._write_file(list_file) as fh:
 169                 fh.write("\n".join(sorted(self._backend_output_files)))
 170         else:
 171             # Always update its mtime if we're not in dry-run mode.
 172             if not self.dry_run:
 173                 with open(list_file, "a"):
 174                     os.utime(list_file, None)
 175
 176         # Write out the list of input files for the backend
 177         with self._write_file("%s.in" % list_file) as fh:
 178             fh.write(
 179                 "\n".join(sorted(mozpath.normsep(f) for f in self.backend_input_files))
 180             )
 181
 182     @abstractmethod
 183     def consume_object(self, obj):
 184         """Consumes an individual TreeMetadata instance.
 185
 186         This is the main method used by child classes to react to build
 187         metadata.
 188         """
 189
 190     def consume_finished(self):
 191         """Called when consume() has completed handling all objects."""
 192
 193     def build(self, config, output, jobs, verbose, what=None):
 194         """Called when 'mach build' is executed.
 195
 196         This should return the status value of a subprocess, where 0 denotes
 197         success and any other value is an error code. A return value of None
 198         indicates that the default 'make -f client.mk' should run.
 199         """
 200         return None
 201
 202     def _write_purgecaches(self, config):
 203         """Write .purgecaches sentinels.
 204
 205         The purgecaches mechanism exists to allow the platform to
 206         invalidate the XUL cache (which includes some JS) at application
 207         startup-time.  The application checks for .purgecaches in the
 208         application directory, which varies according to
 209         --enable-application/--enable-project.  There's a further wrinkle on
 210         macOS, where the real application directory is part of a Cocoa bundle
 211         produced from the regular application directory by the build
 212         system.  In this case, we write to both locations, since the
 213         build system recreates the Cocoa bundle from the contents of the
 214         regular application directory and might remove a sentinel
 215         created here.
 216         """
 217
 218         app = config.substs["MOZ_BUILD_APP"]
 219         if app == "mobile/android":
 220             # In order to take effect, .purgecaches sentinels would need to be
 221             # written to the Android device file system.
 222             return
 223
 224         root = mozpath.join(config.topobjdir, "dist", "bin")
 225
 226         if app == "browser":
 227             root = mozpath.join(config.topobjdir, "dist", "bin", "browser")
 228
 229         purgecaches_dirs = [root]
 230         if app == "browser" and "cocoa" == config.substs["MOZ_WIDGET_TOOLKIT"]:
 231             bundledir = mozpath.join(
 232                 config.topobjdir,
 233                 "dist",
 234                 config.substs["MOZ_MACBUNDLE_NAME"],
 235                 "Contents",
 236                 "Resources",
 237                 "browser",
 238             )
 239             purgecaches_dirs.append(bundledir)
 240
 241         for dir in purgecaches_dirs:
 242             with open(mozpath.join(dir, ".purgecaches"), "wt") as f:
 243                 f.write("\n")
 244
 245     def post_build(self, config, output, jobs, verbose, status):
 246         """Called late during 'mach build' execution, after `build(...)` has finished.
 247
 248         `status` is the status value returned from `build(...)`.
 249
 250         In the case where `build` returns `None`, this is called after
 251         the default `make` command has completed, with the status of
 252         that command.
 253
 254         This should return the status value from `build(...)`, or the
 255         status value of a subprocess, where 0 denotes success and any
 256         other value is an error code.
 257
 258         If an exception is raised, ``mach build`` will fail with a
 259         non-zero exit code.
 260         """
 261         self._write_purgecaches(config)
 262
 263         return status
 264
 265     @contextmanager
 266     def _write_file(self, path=None, fh=None, readmode="r"):
 267         """Context manager to write a file.
 268
 269         This is a glorified wrapper around FileAvoidWrite with integration to
 270         update the summary data on this instance.
 271
 272         Example usage:
 273
 274             with self._write_file('foo.txt') as fh:
 275                 fh.write('hello world')
 276         """
 277
 278         if path is not None:
 279             assert fh is None
 280             fh = FileAvoidWrite(
 281                 path, capture_diff=True, dry_run=self.dry_run, readmode=readmode
 282             )
 283         else:
 284             assert fh is not None
 285
 286         dirname = mozpath.dirname(fh.name)
 287         try:
 288             os.makedirs(dirname)
 289         except OSError as error:
 290             if error.errno != errno.EEXIST:
 291                 raise
 292
 293         yield fh
 294
 295         self._backend_output_files.add(
 296             mozpath.relpath(fh.name, self.environment.topobjdir)
 297         )
 298         existed, updated = fh.close()
 299         if fh.diff:
 300             self.file_diffs[fh.name] = fh.diff
 301         if not existed:
 302             self._created_count += 1
 303         elif updated:
 304             self._updated_count += 1
 305         else:
 306             self._unchanged_count += 1
 307
 308     @contextmanager
 309     def _get_preprocessor(self, obj):
 310         """Returns a preprocessor with a few predefined values depending on
 311         the given BaseConfigSubstitution(-like) object, and all the substs
 312         in the current environment."""
 313         pp = Preprocessor()
 314         srcdir = mozpath.dirname(obj.input_path)
 315         pp.context.update(
 316             {
 317                 k: " ".join(v) if isinstance(v, list) else v
 318                 for k, v in six.iteritems(obj.config.substs)
 319             }
 320         )
 321         pp.context.update(
 322             top_srcdir=obj.topsrcdir,
 323             topobjdir=obj.topobjdir,
 324             srcdir=srcdir,
 325             srcdir_rel=mozpath.relpath(srcdir, mozpath.dirname(obj.output_path)),
 326             relativesrcdir=mozpath.relpath(srcdir, obj.topsrcdir) or ".",
 327             DEPTH=mozpath.relpath(obj.topobjdir, mozpath.dirname(obj.output_path))
 328             or ".",
 329         )
 330         pp.do_filter("attemptSubstitution")
 331         pp.setMarker(None)
 332         with self._write_file(obj.output_path) as fh:
 333             pp.out = fh
 334             yield pp
 335
 336
 337 class PartialBackend(BuildBackend):
 338     """A PartialBackend is a BuildBackend declaring that its consume_object
 339     method may not handle all build configuration objects it's passed, and
 340     that it's fine."""
 341
 342
 343 def HybridBackend(*backends):
 344     """A HybridBackend is the combination of one or more PartialBackends
 345     with a non-partial BuildBackend.
 346
 347     Build configuration objects are passed to each backend, stopping at the
 348     first of them that declares having handled them.
 349     """
 350     assert len(backends) >= 2
 351     assert all(issubclass(b, PartialBackend) for b in backends[:-1])
 352     assert not (issubclass(backends[-1], PartialBackend))
 353     assert all(issubclass(b, BuildBackend) for b in backends)
 354
 355     class TheHybridBackend(BuildBackend):
 356         def __init__(self, environment):
 357             self._backends = [b(environment) for b in backends]
 358             super(TheHybridBackend, self).__init__(environment)
 359
 360         def consume_object(self, obj):
 361             return any(b.consume_object(obj) for b in self._backends)
 362
 363         def consume_finished(self):
 364             for backend in self._backends:
 365                 backend.consume_finished()
 366
 367             for attr in (
 368                 "_execution_time",
 369                 "_created_count",
 370                 "_updated_count",
 371                 "_unchanged_count",
 372                 "_deleted_count",
 373             ):
 374                 setattr(self, attr, sum(getattr(b, attr) for b in self._backends))
 375
 376             for b in self._backends:
 377                 self.file_diffs.update(b.file_diffs)
 378                 for attr in ("backend_input_files", "_backend_output_files"):
 379                     files = getattr(self, attr)
 380                     files |= getattr(b, attr)
 381
 382     name = "+".join(
 383         itertools.chain(
 384             (b.__name__.replace("Backend", "") for b in backends[:-1]),
 385             (b.__name__ for b in backends[-1:]),
 386         )
 387     )
 388
 389     return type(str(name), (TheHybridBackend,), {})