testing/web-platform/manifestdownload.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 import tarfile
   7 from datetime import datetime, timedelta
   8
   9 import mozversioncontrol
  10 import requests
  11 import six
  12
  13 try:
  14     from cStringIO import StringIO as BytesIO
  15 except ImportError:
  16     from io import BytesIO
  17
  18 HEADERS = {"User-Agent": "wpt manifest download"}
  19
  20
  21 def get(logger, url, **kwargs):
  22     logger.debug(url)
  23     if "headers" not in kwargs:
  24         kwargs["headers"] = HEADERS
  25     return requests.get(url, **kwargs)
  26
  27
  28 def abs_path(path):
  29     return os.path.abspath(os.path.expanduser(path))
  30
  31
  32 def get_commits(logger, repo_root):
  33     try:
  34         repo = mozversioncontrol.get_repository_object(repo_root)
  35     except mozversioncontrol.InvalidRepoPath:
  36         logger.warning("No VCS found for path %s" % repo_root)
  37         return []
  38
  39     # The base_ref doesn't actually return a ref, sadly
  40     base_rev = repo.base_ref
  41     if repo.name == "git":
  42         logger.debug("Found git repo")
  43         logger.debug("Base rev is %s" % base_rev)
  44         if not repo.has_git_cinnabar:
  45             logger.error("git cinnabar not found")
  46             return []
  47         changeset_iter = (
  48             repo._run("cinnabar", "git2hg", rev).strip()
  49             for rev in repo._run(
  50                 "log",
  51                 "--format=%H",
  52                 "-n50",
  53                 base_rev,
  54                 "testing/web-platform/tests",
  55                 "testing/web-platform/mozilla/tests",
  56             ).splitlines()
  57         )
  58     else:
  59         logger.debug("Found hg repo")
  60         logger.debug("Base rev is %s" % base_rev)
  61         changeset_iter = repo._run(
  62             "log",
  63             "-fl50",
  64             "--template={node}\n",
  65             "-r",
  66             base_rev,
  67             "testing/web-platform/tests",
  68             "testing/web-platform/mozilla/tests",
  69         ).splitlines()
  70     return changeset_iter
  71
  72
  73 def should_download(logger, manifest_paths, rebuild_time=timedelta(days=5)):
  74     # TODO: Improve logic for when to download. Maybe if x revisions behind?
  75     for manifest_path in manifest_paths:
  76         if not os.path.exists(manifest_path):
  77             return True
  78         mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
  79         if mtime < datetime.now() - rebuild_time:
  80             return True
  81         if os.path.getsize(manifest_path) == 0:
  82             return True
  83
  84     logger.info("Skipping manifest download because existing file is recent")
  85     return False
  86
  87
  88 def taskcluster_url(logger, commits):
  89     artifact_path = "/artifacts/public/manifests.tar.gz"
  90
  91     repos = {
  92         "mozilla-central": "mozilla-central",
  93         "integration/autoland": "autoland",
  94         "releases/mozilla-esr115": "mozilla-esr115",
  95     }
  96     cset_url = (
  97         "https://hg.mozilla.org/{repo}/json-pushes?"
  98         "changeset={changeset}&version=2&tipsonly=1"
  99     )
 100
 101     tc_url = (
 102         "https://firefox-ci-tc.services.mozilla.com/api/index/v1/"
 103         "task/gecko.v2.{name}."
 104         "revision.{changeset}.source.manifest-upload"
 105     )
 106
 107     default = (
 108         "https://firefox-ci-tc.services.mozilla.com/api/index/v1/"
 109         "task/gecko.v2.mozilla-central.latest.source.manifest-upload" + artifact_path
 110     )
 111
 112     for revision in commits:
 113         req = None
 114
 115         if revision == 40 * "0":
 116             continue
 117
 118         for repo_path, index_name in six.iteritems(repos):
 119             try:
 120                 req_headers = HEADERS.copy()
 121                 req_headers.update({"Accept": "application/json"})
 122                 req = get(
 123                     logger,
 124                     cset_url.format(changeset=revision, repo=repo_path),
 125                     headers=req_headers,
 126                 )
 127                 req.raise_for_status()
 128             except requests.exceptions.RequestException:
 129                 if req is not None and req.status_code == 404:
 130                     # The API returns a 404 if it can't find a changeset for the revision.
 131                     logger.debug("%s not found in %s" % (revision, repo_path))
 132                     continue
 133                 else:
 134                     return default
 135
 136             result = req.json()
 137
 138             pushes = result["pushes"]
 139             if not pushes:
 140                 logger.debug("Error reading response; 'pushes' key not found")
 141                 continue
 142             [cset] = next(iter(pushes.values()))["changesets"]
 143
 144             tc_index_url = tc_url.format(changeset=cset, name=index_name)
 145             try:
 146                 req = get(logger, tc_index_url)
 147             except requests.exceptions.RequestException:
 148                 return default
 149
 150             if req.status_code == 200:
 151                 return tc_index_url + artifact_path
 152
 153     logger.info(
 154         "Can't find a commit-specific manifest so just using the most " "recent one"
 155     )
 156
 157     return default
 158
 159
 160 def download_manifest(logger, test_paths, commits_func, url_func, force=False):
 161     manifest_paths = [
 162         (item["manifest_path"] if isinstance(item, dict) else item.manifest_path)
 163         for item in test_paths.values()
 164     ]
 165
 166     if not force and not should_download(logger, manifest_paths):
 167         return True
 168
 169     commits = commits_func()
 170
 171     url = url_func(logger, commits)
 172     if not url:
 173         logger.warning("No generated manifest found")
 174         return False
 175
 176     logger.info("Downloading manifest from %s" % url)
 177     try:
 178         req = get(logger, url)
 179     except Exception:
 180         logger.warning("Downloading pregenerated manifest failed")
 181         return False
 182
 183     if req.status_code != 200:
 184         logger.warning(
 185             "Downloading pregenerated manifest failed; got "
 186             "HTTP status %d" % req.status_code
 187         )
 188         return False
 189
 190     tar = tarfile.open(mode="r:gz", fileobj=BytesIO(req.content))
 191     for paths in six.itervalues(test_paths):
 192         manifest_rel_path = (
 193             paths["manifest_rel_path"]
 194             if isinstance(paths, dict)
 195             else paths.manifest_rel_path
 196         )
 197         manifest_path = (
 198             paths["manifest_path"] if isinstance(paths, dict) else paths.manifest_path
 199         )
 200
 201         try:
 202             member = tar.getmember(manifest_rel_path.replace(os.path.sep, "/"))
 203         except KeyError:
 204             logger.warning("Failed to find downloaded manifest %s" % manifest_rel_path)
 205         else:
 206             try:
 207                 logger.debug("Unpacking %s to %s" % (member.name, manifest_path))
 208                 src = tar.extractfile(member)
 209                 with open(manifest_path, "wb") as dest:
 210                     dest.write(src.read())
 211                 src.close()
 212             except IOError:
 213                 import traceback
 214
 215                 logger.warning(
 216                     "Failed to decompress %s:\n%s"
 217                     % (manifest_rel_path, traceback.format_exc())
 218                 )
 219                 return False
 220
 221         os.utime(manifest_path, None)
 222
 223     return True
 224
 225
 226 def download_from_taskcluster(logger, repo_root, test_paths, force=False):
 227     return download_manifest(
 228         logger,
 229         test_paths,
 230         lambda: get_commits(logger, repo_root),
 231         taskcluster_url,
 232         force,
 233     )