testing/web-platform/manifestupdate.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import errno
   7 import hashlib
   8 import imp
   9 import os
  10 import six
  11 import sys
  12
  13 from six.moves import configparser
  14
  15 from mozboot.util import get_state_dir
  16
  17 from mozlog.structured import commandline
  18
  19 import manifestdownload
  20 from wptrunner import wptcommandline
  21
  22 manifest = None
  23
  24
  25 def do_delayed_imports(wpt_dir):
  26     global manifest
  27     imp.load_source(
  28         "localpaths", os.path.join(wpt_dir, "tests", "tools", "localpaths.py")
  29     )
  30     sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
  31     import manifest
  32
  33
  34 def create_parser():
  35     p = argparse.ArgumentParser()
  36     p.add_argument(
  37         "--rebuild", action="store_true", help="Rebuild manifest from scratch"
  38     )
  39     download_group = p.add_mutually_exclusive_group()
  40     download_group.add_argument(
  41         "--download",
  42         dest="download",
  43         action="store_true",
  44         default=None,
  45         help="Always download even if the local manifest is recent",
  46     )
  47     download_group.add_argument(
  48         "--no-download",
  49         dest="download",
  50         action="store_false",
  51         help="Don't try to download the manifest",
  52     )
  53     p.add_argument(
  54         "--no-update",
  55         action="store_false",
  56         dest="update",
  57         default=True,
  58         help="Just download the manifest, don't update",
  59     )
  60     p.add_argument(
  61         "--config",
  62         action="store",
  63         dest="config_path",
  64         default=None,
  65         help="Path to wptrunner config file",
  66     )
  67     p.add_argument(
  68         "--rewrite-config",
  69         action="store_true",
  70         default=False,
  71         help="Force the local configuration to be regenerated",
  72     )
  73     p.add_argument(
  74         "--cache-root",
  75         action="store",
  76         default=os.path.join(get_state_dir(), "cache", "wpt"),
  77         help="Path to use for the metadata cache",
  78     )
  79     commandline.add_logging_group(p)
  80
  81     return p
  82
  83
  84 def ensure_kwargs(kwargs):
  85     _kwargs = vars(create_parser().parse_args([]))
  86     _kwargs.update(kwargs)
  87     return _kwargs
  88
  89
  90 def run(src_root, obj_root, logger=None, **kwargs):
  91     kwargs = ensure_kwargs(kwargs)
  92
  93     if logger is None:
  94         from wptrunner import wptlogging
  95
  96         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
  97
  98     src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
  99
 100     do_delayed_imports(src_wpt_dir)
 101
 102     if not kwargs["config_path"]:
 103         config_path = generate_config(
 104             logger,
 105             src_root,
 106             src_wpt_dir,
 107             os.path.join(obj_root, "_tests", "web-platform"),
 108             kwargs["rewrite_config"],
 109         )
 110     else:
 111         config_path = kwargs["config_path"]
 112
 113     if not os.path.exists(config_path):
 114         logger.critical("Config file %s does not exist" % config_path)
 115         return None
 116
 117     logger.debug("Using config path %s" % config_path)
 118
 119     test_paths = wptcommandline.get_test_paths(wptcommandline.config.read(config_path))
 120
 121     for paths in six.itervalues(test_paths):
 122         if "manifest_path" not in paths:
 123             paths["manifest_path"] = os.path.join(
 124                 paths["metadata_path"], "MANIFEST.json"
 125             )
 126
 127     ensure_manifest_directories(logger, test_paths)
 128
 129     local_config = read_local_config(src_wpt_dir)
 130     for section in ["manifest:upstream", "manifest:mozilla"]:
 131         url_base = local_config.get(section, "url_base")
 132         manifest_rel_path = os.path.join(
 133             local_config.get(section, "metadata"), "MANIFEST.json"
 134         )
 135         test_paths[url_base]["manifest_rel_path"] = manifest_rel_path
 136
 137     if not kwargs["rebuild"] and kwargs["download"] is not False:
 138         force_download = False if kwargs["download"] is None else True
 139         manifestdownload.download_from_taskcluster(
 140             logger, src_root, test_paths, force=force_download
 141         )
 142     else:
 143         logger.debug("Skipping manifest download")
 144
 145     update = kwargs["update"] or kwargs["rebuild"]
 146     manifests = load_and_update(
 147         logger,
 148         src_wpt_dir,
 149         test_paths,
 150         update=update,
 151         rebuild=kwargs["rebuild"],
 152         cache_root=kwargs["cache_root"],
 153     )
 154
 155     return manifests
 156
 157
 158 def ensure_manifest_directories(logger, test_paths):
 159     for paths in six.itervalues(test_paths):
 160         manifest_dir = os.path.dirname(paths["manifest_path"])
 161         if not os.path.exists(manifest_dir):
 162             logger.info("Creating directory %s" % manifest_dir)
 163             # Even though we just checked the path doesn't exist, there's a chance
 164             # of race condition with another process or thread having created it in
 165             # between. This happens during tests.
 166             try:
 167                 os.makedirs(manifest_dir)
 168             except OSError as e:
 169                 if e.errno != errno.EEXIST:
 170                     raise
 171         elif not os.path.isdir(manifest_dir):
 172             raise IOError("Manifest directory is a file")
 173
 174
 175 def read_local_config(wpt_dir):
 176     src_config_path = os.path.join(wpt_dir, "wptrunner.ini")
 177
 178     parser = configparser.SafeConfigParser()
 179     success = parser.read(src_config_path)
 180     assert src_config_path in success
 181     return parser
 182
 183
 184 def generate_config(logger, repo_root, wpt_dir, dest_path, force_rewrite=False):
 185     """Generate the local wptrunner.ini file to use locally"""
 186     if not os.path.exists(dest_path):
 187         # Even though we just checked the path doesn't exist, there's a chance
 188         # of race condition with another process or thread having created it in
 189         # between. This happens during tests.
 190         try:
 191             os.makedirs(dest_path)
 192         except OSError as e:
 193             if e.errno != errno.EEXIST:
 194                 raise
 195
 196     dest_config_path = os.path.join(dest_path, "wptrunner.local.ini")
 197
 198     if not force_rewrite and os.path.exists(dest_config_path):
 199         logger.debug("Config is up to date, not regenerating")
 200         return dest_config_path
 201
 202     logger.info("Creating config file %s" % dest_config_path)
 203
 204     parser = read_local_config(wpt_dir)
 205
 206     for section in ["manifest:upstream", "manifest:mozilla"]:
 207         meta_rel_path = parser.get(section, "metadata")
 208         tests_rel_path = parser.get(section, "tests")
 209
 210         parser.set(
 211             section, "manifest", os.path.join(dest_path, meta_rel_path, "MANIFEST.json")
 212         )
 213         parser.set(section, "metadata", os.path.join(wpt_dir, meta_rel_path))
 214         parser.set(section, "tests", os.path.join(wpt_dir, tests_rel_path))
 215
 216     parser.set(
 217         "paths",
 218         "prefs",
 219         os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))),
 220     )
 221
 222     with open(dest_config_path, "wt") as config_file:
 223         parser.write(config_file)
 224
 225     return dest_config_path
 226
 227
 228 def load_and_update(
 229     logger,
 230     wpt_dir,
 231     test_paths,
 232     rebuild=False,
 233     config_dir=None,
 234     cache_root=None,
 235     update=True,
 236 ):
 237     rv = {}
 238     wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir).encode()).hexdigest()
 239     for url_base, paths in six.iteritems(test_paths):
 240         manifest_path = paths["manifest_path"]
 241         this_cache_root = os.path.join(
 242             cache_root, wptdir_hash, os.path.dirname(paths["manifest_rel_path"])
 243         )
 244         m = manifest.manifest.load_and_update(
 245             paths["tests_path"],
 246             manifest_path,
 247             url_base,
 248             update=update,
 249             rebuild=rebuild,
 250             working_copy=True,
 251             cache_root=this_cache_root,
 252         )
 253         path_data = {"url_base": url_base}
 254         path_data.update(paths)
 255         rv[m] = path_data
 256
 257     return rv
 258
 259
 260 def log_error(logger, manifest_path, msg):
 261     logger.lint_error(
 262         path=manifest_path, message=msg, lineno=0, source="", linter="wpt-manifest"
 263     )