testing/web-platform/manifestdownload.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 import tarfile
   7 from datetime import datetime, timedelta
   8
   9 import mozversioncontrol
  10 import requests
  11 import six
  12
  13 try:
  14     from cStringIO import StringIO as BytesIO
  15 except ImportError:
  16     from io import BytesIO
  17
  18 HEADERS = {"User-Agent": "wpt manifest download"}
  19
  20
  21 def get(logger, url, **kwargs):
  22     logger.debug(url)
  23     if "headers" not in kwargs:
  24         kwargs["headers"] = HEADERS
  25     return requests.get(url, **kwargs)
  26
  27
  28 def abs_path(path):
  29     return os.path.abspath(os.path.expanduser(path))
  30
  31
  32 def get_commits(logger, repo_root):
  33     try:
  34         repo = mozversioncontrol.get_repository_object(repo_root)
  35     except mozversioncontrol.InvalidRepoPath:
  36         logger.warning("No VCS found for path %s" % repo_root)
  37         return []
  38
  39     # The base_ref doesn't actually return a ref, sadly
  40     base_rev = repo.base_ref
  41     if repo.name == "git":
  42         logger.debug("Found git repo")
  43         logger.debug("Base rev is %s" % base_rev)
  44         if not repo.has_git_cinnabar:
  45             logger.error("git cinnabar not found")
  46             return []
  47         changeset_iter = (
  48             repo._run("cinnabar", "git2hg", rev).strip()
  49             for rev in repo._run(
  50                 "log",
  51                 "--format=%H",
  52                 "-n50",
  53                 base_rev,
  54                 "testing/web-platform/tests",
  55                 "testing/web-platform/mozilla/tests",
  56             ).splitlines()
  57         )
  58     else:
  59         logger.debug("Found hg repo")
  60         logger.debug("Base rev is %s" % base_rev)
  61         changeset_iter = repo._run(
  62             "log",
  63             "-fl50",
  64             "--template={node}\n",
  65             "-r",
  66             base_rev,
  67             "testing/web-platform/tests",
  68             "testing/web-platform/mozilla/tests",
  69         ).splitlines()
  70     return changeset_iter
  71
  72
  73 def should_download(logger, manifest_paths, rebuild_time=timedelta(days=5)):
  74     # TODO: Improve logic for when to download. Maybe if x revisions behind?
  75     for manifest_path in manifest_paths:
  76         if not os.path.exists(manifest_path):
  77             return True
  78         mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
  79         if mtime < datetime.now() - rebuild_time:
  80             return True
  81         if os.path.getsize(manifest_path) == 0:
  82             return True
  83
  84     logger.info("Skipping manifest download because existing file is recent")
  85     return False
  86
  87
  88 def taskcluster_url(logger, commits):
  89     artifact_path = "/artifacts/public/manifests.tar.gz"
  90
  91     repos = {
  92         "mozilla-central": "mozilla-central",
  93         "integration/autoland": "autoland",
  94         "releases/mozilla-esr102": "mozilla-esr102",
  95         "releases/mozilla-esr115": "mozilla-esr115",
  96     }
  97     cset_url = (
  98         "https://hg.mozilla.org/{repo}/json-pushes?"
  99         "changeset={changeset}&version=2&tipsonly=1"
 100     )
 101
 102     tc_url = (
 103         "https://firefox-ci-tc.services.mozilla.com/api/index/v1/"
 104         "task/gecko.v2.{name}."
 105         "revision.{changeset}.source.manifest-upload"
 106     )
 107
 108     default = (
 109         "https://firefox-ci-tc.services.mozilla.com/api/index/v1/"
 110         "task/gecko.v2.mozilla-central.latest.source.manifest-upload" + artifact_path
 111     )
 112
 113     for revision in commits:
 114         req = None
 115
 116         if revision == 40 * "0":
 117             continue
 118
 119         for repo_path, index_name in six.iteritems(repos):
 120             try:
 121                 req_headers = HEADERS.copy()
 122                 req_headers.update({"Accept": "application/json"})
 123                 req = get(
 124                     logger,
 125                     cset_url.format(changeset=revision, repo=repo_path),
 126                     headers=req_headers,
 127                 )
 128                 req.raise_for_status()
 129             except requests.exceptions.RequestException:
 130                 if req is not None and req.status_code == 404:
 131                     # The API returns a 404 if it can't find a changeset for the revision.
 132                     logger.debug("%s not found in %s" % (revision, repo_path))
 133                     continue
 134                 else:
 135                     return default
 136
 137             result = req.json()
 138
 139             pushes = result["pushes"]
 140             if not pushes:
 141                 logger.debug("Error reading response; 'pushes' key not found")
 142                 continue
 143             [cset] = next(iter(pushes.values()))["changesets"]
 144
 145             tc_index_url = tc_url.format(changeset=cset, name=index_name)
 146             try:
 147                 req = get(logger, tc_index_url)
 148             except requests.exceptions.RequestException:
 149                 return default
 150
 151             if req.status_code == 200:
 152                 return tc_index_url + artifact_path
 153
 154     logger.info(
 155         "Can't find a commit-specific manifest so just using the most " "recent one"
 156     )
 157
 158     return default
 159
 160
 161 def download_manifest(logger, test_paths, commits_func, url_func, force=False):
 162     manifest_paths = [item["manifest_path"] for item in six.itervalues(test_paths)]
 163
 164     if not force and not should_download(logger, manifest_paths):
 165         return True
 166
 167     commits = commits_func()
 168
 169     url = url_func(logger, commits)
 170     if not url:
 171         logger.warning("No generated manifest found")
 172         return False
 173
 174     logger.info("Downloading manifest from %s" % url)
 175     try:
 176         req = get(logger, url)
 177     except Exception:
 178         logger.warning("Downloading pregenerated manifest failed")
 179         return False
 180
 181     if req.status_code != 200:
 182         logger.warning(
 183             "Downloading pregenerated manifest failed; got "
 184             "HTTP status %d" % req.status_code
 185         )
 186         return False
 187
 188     tar = tarfile.open(mode="r:gz", fileobj=BytesIO(req.content))
 189     for paths in six.itervalues(test_paths):
 190         try:
 191             member = tar.getmember(paths["manifest_rel_path"].replace(os.path.sep, "/"))
 192         except KeyError:
 193             logger.warning(
 194                 "Failed to find downloaded manifest %s" % paths["manifest_rel_path"]
 195             )
 196         else:
 197             try:
 198                 logger.debug(
 199                     "Unpacking %s to %s" % (member.name, paths["manifest_path"])
 200                 )
 201                 src = tar.extractfile(member)
 202                 with open(paths["manifest_path"], "wb") as dest:
 203                     dest.write(src.read())
 204                 src.close()
 205             except IOError:
 206                 import traceback
 207
 208                 logger.warning(
 209                     "Failed to decompress %s:\n%s"
 210                     % (paths["manifest_rel_path"], traceback.format_exc())
 211                 )
 212                 return False
 213
 214         os.utime(paths["manifest_path"], None)
 215
 216     return True
 217
 218
 219 def download_from_taskcluster(logger, repo_root, test_paths, force=False):
 220     return download_manifest(
 221         logger,
 222         test_paths,
 223         lambda: get_commits(logger, repo_root),
 224         taskcluster_url,
 225         force,
 226     )