testing/web-platform/manifestupdate.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import configparser
   7 import errno
   8 import hashlib
   9 import os
  10 import sys
  11
  12 import manifestdownload
  13 from mach.util import get_state_dir
  14 from mozfile import load_source
  15 from mozlog.structured import commandline
  16 from wptrunner import wptcommandline
  17
  18 manifest = None
  19
  20
  21 def do_delayed_imports(wpt_dir):
  22     global manifest
  23     load_source("localpaths", os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
  24     sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
  25     import manifest
  26
  27
  28 def create_parser():
  29     p = argparse.ArgumentParser()
  30     p.add_argument(
  31         "--rebuild", action="store_true", help="Rebuild manifest from scratch"
  32     )
  33     download_group = p.add_mutually_exclusive_group()
  34     download_group.add_argument(
  35         "--download",
  36         dest="download",
  37         action="store_true",
  38         default=None,
  39         help="Always download even if the local manifest is recent",
  40     )
  41     download_group.add_argument(
  42         "--no-download",
  43         dest="download",
  44         action="store_false",
  45         help="Don't try to download the manifest",
  46     )
  47     p.add_argument(
  48         "--no-update",
  49         action="store_false",
  50         dest="update",
  51         default=True,
  52         help="Just download the manifest, don't update",
  53     )
  54     p.add_argument(
  55         "--config",
  56         action="store",
  57         dest="config_path",
  58         default=None,
  59         help="Path to wptrunner config file",
  60     )
  61     p.add_argument(
  62         "--rewrite-config",
  63         action="store_true",
  64         default=False,
  65         help="Force the local configuration to be regenerated",
  66     )
  67     p.add_argument(
  68         "--cache-root",
  69         action="store",
  70         default=os.path.join(get_state_dir(), "cache", "wpt"),
  71         help="Path to use for the metadata cache",
  72     )
  73     commandline.add_logging_group(p)
  74
  75     return p
  76
  77
  78 def ensure_kwargs(kwargs):
  79     _kwargs = vars(create_parser().parse_args([]))
  80     _kwargs.update(kwargs)
  81     return _kwargs
  82
  83
  84 def run(src_root, obj_root, logger=None, **kwargs):
  85     kwargs = ensure_kwargs(kwargs)
  86
  87     if logger is None:
  88         from wptrunner import wptlogging
  89
  90         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
  91
  92     src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
  93
  94     do_delayed_imports(src_wpt_dir)
  95
  96     if not kwargs["config_path"]:
  97         config_path = generate_config(
  98             logger,
  99             src_root,
 100             src_wpt_dir,
 101             os.path.join(obj_root, "_tests", "web-platform"),
 102             kwargs["rewrite_config"],
 103         )
 104     else:
 105         config_path = kwargs["config_path"]
 106
 107     if not os.path.exists(config_path):
 108         logger.critical("Config file %s does not exist" % config_path)
 109         return None
 110
 111     logger.debug("Using config path %s" % config_path)
 112
 113     test_paths = wptcommandline.get_test_paths(wptcommandline.config.read(config_path))
 114
 115     for paths in test_paths.values():
 116         if isinstance(paths, dict) and "manifest_path" not in paths:
 117             paths["manifest_path"] = os.path.join(
 118                 paths["metadata_path"], "MANIFEST.json"
 119             )
 120
 121     ensure_manifest_directories(logger, test_paths)
 122
 123     local_config = read_local_config(src_wpt_dir)
 124     for section in ["manifest:upstream", "manifest:mozilla"]:
 125         url_base = local_config.get(section, "url_base")
 126         manifest_rel_path = os.path.join(
 127             local_config.get(section, "metadata"), "MANIFEST.json"
 128         )
 129         if isinstance(test_paths[url_base], dict):
 130             test_paths[url_base]["manifest_rel_path"] = manifest_rel_path
 131         else:
 132             test_paths[url_base].manifest_rel_path = manifest_rel_path
 133
 134     if not kwargs["rebuild"] and kwargs["download"] is not False:
 135         force_download = False if kwargs["download"] is None else True
 136         manifestdownload.download_from_taskcluster(
 137             logger, src_root, test_paths, force=force_download
 138         )
 139     else:
 140         logger.debug("Skipping manifest download")
 141
 142     update = kwargs["update"] or kwargs["rebuild"]
 143     manifests = load_and_update(
 144         logger,
 145         src_wpt_dir,
 146         test_paths,
 147         update=update,
 148         rebuild=kwargs["rebuild"],
 149         cache_root=kwargs["cache_root"],
 150     )
 151
 152     return manifests
 153
 154
 155 def ensure_manifest_directories(logger, test_paths):
 156     for paths in test_paths.values():
 157         manifest_path = (
 158             paths["manifest_path"] if isinstance(paths, dict) else paths.manifest_path
 159         )
 160         manifest_dir = os.path.dirname(manifest_path)
 161         if not os.path.exists(manifest_dir):
 162             logger.info("Creating directory %s" % manifest_dir)
 163             # Even though we just checked the path doesn't exist, there's a chance
 164             # of race condition with another process or thread having created it in
 165             # between. This happens during tests.
 166             try:
 167                 os.makedirs(manifest_dir)
 168             except OSError as e:
 169                 if e.errno != errno.EEXIST:
 170                     raise
 171         elif not os.path.isdir(manifest_dir):
 172             raise IOError("Manifest directory is a file")
 173
 174
 175 def read_local_config(wpt_dir):
 176     src_config_path = os.path.join(wpt_dir, "wptrunner.ini")
 177
 178     parser = configparser.ConfigParser()
 179     success = parser.read(src_config_path)
 180     assert src_config_path in success
 181     return parser
 182
 183
 184 def generate_config(logger, repo_root, wpt_dir, dest_path, force_rewrite=False):
 185     """Generate the local wptrunner.ini file to use locally"""
 186     if not os.path.exists(dest_path):
 187         # Even though we just checked the path doesn't exist, there's a chance
 188         # of race condition with another process or thread having created it in
 189         # between. This happens during tests.
 190         try:
 191             os.makedirs(dest_path)
 192         except OSError as e:
 193             if e.errno != errno.EEXIST:
 194                 raise
 195
 196     dest_config_path = os.path.join(dest_path, "wptrunner.local.ini")
 197
 198     if not force_rewrite and os.path.exists(dest_config_path):
 199         logger.debug("Config is up to date, not regenerating")
 200         return dest_config_path
 201
 202     logger.info("Creating config file %s" % dest_config_path)
 203
 204     parser = read_local_config(wpt_dir)
 205
 206     for section in ["manifest:upstream", "manifest:mozilla"]:
 207         meta_rel_path = parser.get(section, "metadata")
 208         tests_rel_path = parser.get(section, "tests")
 209
 210         parser.set(
 211             section, "manifest", os.path.join(dest_path, meta_rel_path, "MANIFEST.json")
 212         )
 213         parser.set(section, "metadata", os.path.join(wpt_dir, meta_rel_path))
 214         parser.set(section, "tests", os.path.join(wpt_dir, tests_rel_path))
 215
 216     parser.set(
 217         "paths",
 218         "prefs",
 219         os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))),
 220     )
 221
 222     with open(dest_config_path, "wt") as config_file:
 223         parser.write(config_file)
 224
 225     return dest_config_path
 226
 227
 228 def load_and_update(
 229     logger,
 230     wpt_dir,
 231     test_paths,
 232     rebuild=False,
 233     config_dir=None,
 234     cache_root=None,
 235     update=True,
 236 ):
 237     rv = {}
 238     wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir).encode()).hexdigest()
 239     for url_base, paths in test_paths.items():
 240         manifest_path = (
 241             paths["manifest_path"] if isinstance(paths, dict) else paths.manifest_path
 242         )
 243         manifest_rel_path = (
 244             paths["manifest_rel_path"]
 245             if isinstance(paths, dict)
 246             else paths.manifest_rel_path
 247         )
 248         tests_path = (
 249             paths["tests_path"] if isinstance(paths, dict) else paths.tests_path
 250         )
 251         this_cache_root = os.path.join(
 252             cache_root, wptdir_hash, os.path.dirname(manifest_rel_path)
 253         )
 254         m = manifest.manifest.load_and_update(
 255             tests_path,
 256             manifest_path,
 257             url_base,
 258             update=update,
 259             rebuild=rebuild,
 260             working_copy=True,
 261             cache_root=this_cache_root,
 262         )
 263         path_data = {"url_base": url_base}
 264         if isinstance(paths, dict):
 265             path_data.update(paths)
 266         else:
 267             for key, value in paths.__dict__.items():
 268                 path_data[key] = value
 269
 270         rv[m] = path_data
 271
 272     return rv
 273
 274
 275 def log_error(logger, manifest_path, msg):
 276     logger.lint_error(
 277         path=manifest_path, message=msg, lineno=0, source="", linter="wpt-manifest"
 278     )