testing/web-platform/manifestdownload.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import
   6
   7 import os
   8 from datetime import datetime, timedelta
   9 import tarfile
  10 import requests
  11 import six
  12 import mozversioncontrol
  13
  14 try:
  15     from cStringIO import StringIO as BytesIO
  16 except ImportError:
  17     from io import BytesIO
  18
  19 HEADERS = {"User-Agent": "wpt manifest download"}
  20
  21
  22 def get(logger, url, **kwargs):
  23     logger.debug(url)
  24     if "headers" not in kwargs:
  25         kwargs["headers"] = HEADERS
  26     return requests.get(url, **kwargs)
  27
  28
  29 def abs_path(path):
  30     return os.path.abspath(os.path.expanduser(path))
  31
  32
  33 def get_commits(logger, repo_root):
  34     try:
  35         repo = mozversioncontrol.get_repository_object(repo_root)
  36     except mozversioncontrol.InvalidRepoPath:
  37         logger.warning("No VCS found for path %s" % repo_root)
  38         return []
  39
  40     # The base_ref doesn't actually return a ref, sadly
  41     base_rev = repo.base_ref
  42     if repo.name == "git":
  43         logger.debug("Found git repo")
  44         logger.debug("Base rev is %s" % base_rev)
  45         if not repo.has_git_cinnabar:
  46             logger.error("git cinnabar not found")
  47             return []
  48         changeset_iter = (
  49             repo._run("cinnabar", "git2hg", rev).strip()
  50             for rev in repo._run(
  51                 "log",
  52                 "--format=%H",
  53                 "-n50",
  54                 base_rev,
  55                 "testing/web-platform/tests",
  56                 "testing/web-platform/mozilla/tests",
  57             ).splitlines()
  58         )
  59     else:
  60         logger.debug("Found hg repo")
  61         logger.debug("Base rev is %s" % base_rev)
  62         changeset_iter = repo._run(
  63             "log",
  64             "-fl50",
  65             "--template={node}\n",
  66             "-r",
  67             base_rev,
  68             "testing/web-platform/tests",
  69             "testing/web-platform/mozilla/tests",
  70         ).splitlines()
  71     return changeset_iter
  72
  73
  74 def should_download(logger, manifest_paths, rebuild_time=timedelta(days=5)):
  75     # TODO: Improve logic for when to download. Maybe if x revisions behind?
  76     for manifest_path in manifest_paths:
  77         if not os.path.exists(manifest_path):
  78             return True
  79         mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
  80         if mtime < datetime.now() - rebuild_time:
  81             return True
  82         if os.path.getsize(manifest_path) == 0:
  83             return True
  84
  85     logger.info("Skipping manifest download because existing file is recent")
  86     return False
  87
  88
  89 def taskcluster_url(logger, commits):
  90     artifact_path = "/artifacts/public/manifests.tar.gz"
  91
  92     repos = {
  93         "mozilla-central": "mozilla-central",
  94         "integration/autoland": "autoland",
  95         "releases/mozilla-esr102": "mozilla-esr102",
  96     }
  97     cset_url = (
  98         "https://hg.mozilla.org/{repo}/json-pushes?"
  99         "changeset={changeset}&version=2&tipsonly=1"
 100     )
 101
 102     tc_url = (
 103         "https://firefox-ci-tc.services.mozilla.com/api/index/v1/"
 104         "task/gecko.v2.{name}."
 105         "revision.{changeset}.source.manifest-upload"
 106     )
 107
 108     default = (
 109         "https://firefox-ci-tc.services.mozilla.com/api/index/v1/"
 110         "task/gecko.v2.mozilla-central.latest.source.manifest-upload" + artifact_path
 111     )
 112
 113     for revision in commits:
 114         req = None
 115
 116         if revision == 40 * "0":
 117             continue
 118
 119         for repo_path, index_name in six.iteritems(repos):
 120             try:
 121                 req_headers = HEADERS.copy()
 122                 req_headers.update({"Accept": "application/json"})
 123                 req = get(
 124                     logger,
 125                     cset_url.format(changeset=revision, repo=repo_path),
 126                     headers=req_headers,
 127                 )
 128                 req.raise_for_status()
 129             except requests.exceptions.RequestException:
 130                 if req is not None and req.status_code == 404:
 131                     # The API returns a 404 if it can't find a changeset for the revision.
 132                     logger.debug("%s not found in %s" % (revision, repo_path))
 133                     continue
 134                 else:
 135                     return default
 136
 137             result = req.json()
 138
 139             pushes = result["pushes"]
 140             if not pushes:
 141                 logger.debug("Error reading response; 'pushes' key not found")
 142                 continue
 143             [cset] = next(iter(pushes.values()))["changesets"]
 144
 145             tc_index_url = tc_url.format(changeset=cset, name=index_name)
 146             try:
 147                 req = get(logger, tc_index_url)
 148             except requests.exceptions.RequestException:
 149                 return default
 150
 151             if req.status_code == 200:
 152                 return tc_index_url + artifact_path
 153
 154     logger.info(
 155         "Can't find a commit-specific manifest so just using the most " "recent one"
 156     )
 157
 158     return default
 159
 160
 161 def download_manifest(logger, test_paths, commits_func, url_func, force=False):
 162     manifest_paths = [item["manifest_path"] for item in six.itervalues(test_paths)]
 163
 164     if not force and not should_download(logger, manifest_paths):
 165         return True
 166
 167     commits = commits_func()
 168
 169     url = url_func(logger, commits)
 170     if not url:
 171         logger.warning("No generated manifest found")
 172         return False
 173
 174     logger.info("Downloading manifest from %s" % url)
 175     try:
 176         req = get(logger, url)
 177     except Exception:
 178         logger.warning("Downloading pregenerated manifest failed")
 179         return False
 180
 181     if req.status_code != 200:
 182         logger.warning(
 183             "Downloading pregenerated manifest failed; got "
 184             "HTTP status %d" % req.status_code
 185         )
 186         return False
 187
 188     tar = tarfile.open(mode="r:gz", fileobj=BytesIO(req.content))
 189     for paths in six.itervalues(test_paths):
 190         try:
 191             member = tar.getmember(paths["manifest_rel_path"].replace(os.path.sep, "/"))
 192         except KeyError:
 193             logger.warning(
 194                 "Failed to find downloaded manifest %s" % paths["manifest_rel_path"]
 195             )
 196         else:
 197             try:
 198                 logger.debug(
 199                     "Unpacking %s to %s" % (member.name, paths["manifest_path"])
 200                 )
 201                 src = tar.extractfile(member)
 202                 with open(paths["manifest_path"], "wb") as dest:
 203                     dest.write(src.read())
 204                 src.close()
 205             except IOError:
 206                 import traceback
 207
 208                 logger.warning(
 209                     "Failed to decompress %s:\n%s"
 210                     % (paths["manifest_rel_path"], traceback.format_exc())
 211                 )
 212                 return False
 213
 214         os.utime(paths["manifest_path"], None)
 215
 216     return True
 217
 218
 219 def download_from_taskcluster(logger, repo_root, test_paths, force=False):
 220     return download_manifest(
 221         logger,
 222         test_paths,
 223         lambda: get_commits(logger, repo_root),
 224         taskcluster_url,
 225         force,
 226     )