taskcluster/gecko_taskgraph/files_changed.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 """
   6 Support for optimizing tasks based on the set of files that have changed.
   7 """
   8
   9 import logging
  10 import os
  11
  12 from mozpack.path import match as mozpackmatch, join as join_path
  13 from mozversioncontrol import get_repository_object, InvalidRepoPath
  14 from subprocess import CalledProcessError
  15 from mozbuild.util import memoize
  16
  17 from gecko_taskgraph import GECKO
  18 from gecko_taskgraph.util.hg import get_json_automationrelevance
  19
  20 logger = logging.getLogger(__name__)
  21
  22
  23 @memoize
  24 def get_changed_files(repository, revision):
  25     """
  26     Get the set of files changed in the push headed by the given revision.
  27     Responses are cached, so multiple calls with the same arguments are OK.
  28     """
  29     contents = get_json_automationrelevance(repository, revision)
  30     try:
  31         changesets = contents["changesets"]
  32     except KeyError:
  33         # We shouldn't hit this error in CI.
  34         if os.environ.get("MOZ_AUTOMATION"):
  35             raise
  36
  37         # We're likely on an unpublished commit, grab changed files from
  38         # version control.
  39         return get_locally_changed_files(GECKO)
  40
  41     logger.debug("{} commits influencing task scheduling:".format(len(changesets)))
  42     changed_files = set()
  43     for c in changesets:
  44         desc = ""  # Support empty desc
  45         if c["desc"]:
  46             desc = c["desc"].splitlines()[0].encode("ascii", "ignore")
  47         logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc))
  48         changed_files |= set(c["files"])
  49
  50     return changed_files
  51
  52
  53 def check(params, file_patterns):
  54     """Determine whether any of the files changed in the indicated push to
  55     https://hg.mozilla.org match any of the given file patterns."""
  56     repository = params.get("head_repository")
  57     revision = params.get("head_rev")
  58     if not repository or not revision:
  59         logger.warning(
  60             "Missing `head_repository` or `head_rev` parameters; "
  61             "assuming all files have changed"
  62         )
  63         return True
  64
  65     changed_files = get_changed_files(repository, revision)
  66
  67     if "comm_head_repository" in params:
  68         repository = params.get("comm_head_repository")
  69         revision = params.get("comm_head_rev")
  70         if not revision:
  71             logger.warning(
  72                 "Missing `comm_head_rev` parameters; " "assuming all files have changed"
  73             )
  74             return True
  75
  76         changed_files |= {
  77             join_path("comm", file) for file in get_changed_files(repository, revision)
  78         }
  79
  80     for pattern in file_patterns:
  81         for path in changed_files:
  82             if mozpackmatch(path, pattern):
  83                 return True
  84
  85     return False
  86
  87
  88 @memoize
  89 def get_locally_changed_files(repo):
  90     try:
  91         vcs = get_repository_object(repo)
  92         return set(vcs.get_outgoing_files("AM"))
  93     except (InvalidRepoPath, CalledProcessError):
  94         return set()