testing/web-platform/manifestupdate.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import errno
   7 import hashlib
   8 import imp
   9 import os
  10 import sys
  11
  12 import manifestdownload
  13 import six
  14 from mach.util import get_state_dir
  15 from mozlog.structured import commandline
  16 from six.moves import configparser
  17 from wptrunner import wptcommandline
  18
  19 manifest = None
  20
  21
  22 def do_delayed_imports(wpt_dir):
  23     global manifest
  24     imp.load_source(
  25         "localpaths", os.path.join(wpt_dir, "tests", "tools", "localpaths.py")
  26     )
  27     sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
  28     import manifest
  29
  30
  31 def create_parser():
  32     p = argparse.ArgumentParser()
  33     p.add_argument(
  34         "--rebuild", action="store_true", help="Rebuild manifest from scratch"
  35     )
  36     download_group = p.add_mutually_exclusive_group()
  37     download_group.add_argument(
  38         "--download",
  39         dest="download",
  40         action="store_true",
  41         default=None,
  42         help="Always download even if the local manifest is recent",
  43     )
  44     download_group.add_argument(
  45         "--no-download",
  46         dest="download",
  47         action="store_false",
  48         help="Don't try to download the manifest",
  49     )
  50     p.add_argument(
  51         "--no-update",
  52         action="store_false",
  53         dest="update",
  54         default=True,
  55         help="Just download the manifest, don't update",
  56     )
  57     p.add_argument(
  58         "--config",
  59         action="store",
  60         dest="config_path",
  61         default=None,
  62         help="Path to wptrunner config file",
  63     )
  64     p.add_argument(
  65         "--rewrite-config",
  66         action="store_true",
  67         default=False,
  68         help="Force the local configuration to be regenerated",
  69     )
  70     p.add_argument(
  71         "--cache-root",
  72         action="store",
  73         default=os.path.join(get_state_dir(), "cache", "wpt"),
  74         help="Path to use for the metadata cache",
  75     )
  76     commandline.add_logging_group(p)
  77
  78     return p
  79
  80
  81 def ensure_kwargs(kwargs):
  82     _kwargs = vars(create_parser().parse_args([]))
  83     _kwargs.update(kwargs)
  84     return _kwargs
  85
  86
  87 def run(src_root, obj_root, logger=None, **kwargs):
  88     kwargs = ensure_kwargs(kwargs)
  89
  90     if logger is None:
  91         from wptrunner import wptlogging
  92
  93         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
  94
  95     src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
  96
  97     do_delayed_imports(src_wpt_dir)
  98
  99     if not kwargs["config_path"]:
 100         config_path = generate_config(
 101             logger,
 102             src_root,
 103             src_wpt_dir,
 104             os.path.join(obj_root, "_tests", "web-platform"),
 105             kwargs["rewrite_config"],
 106         )
 107     else:
 108         config_path = kwargs["config_path"]
 109
 110     if not os.path.exists(config_path):
 111         logger.critical("Config file %s does not exist" % config_path)
 112         return None
 113
 114     logger.debug("Using config path %s" % config_path)
 115
 116     test_paths = wptcommandline.get_test_paths(wptcommandline.config.read(config_path))
 117
 118     for paths in six.itervalues(test_paths):
 119         if "manifest_path" not in paths:
 120             paths["manifest_path"] = os.path.join(
 121                 paths["metadata_path"], "MANIFEST.json"
 122             )
 123
 124     ensure_manifest_directories(logger, test_paths)
 125
 126     local_config = read_local_config(src_wpt_dir)
 127     for section in ["manifest:upstream", "manifest:mozilla"]:
 128         url_base = local_config.get(section, "url_base")
 129         manifest_rel_path = os.path.join(
 130             local_config.get(section, "metadata"), "MANIFEST.json"
 131         )
 132         test_paths[url_base]["manifest_rel_path"] = manifest_rel_path
 133
 134     if not kwargs["rebuild"] and kwargs["download"] is not False:
 135         force_download = False if kwargs["download"] is None else True
 136         manifestdownload.download_from_taskcluster(
 137             logger, src_root, test_paths, force=force_download
 138         )
 139     else:
 140         logger.debug("Skipping manifest download")
 141
 142     update = kwargs["update"] or kwargs["rebuild"]
 143     manifests = load_and_update(
 144         logger,
 145         src_wpt_dir,
 146         test_paths,
 147         update=update,
 148         rebuild=kwargs["rebuild"],
 149         cache_root=kwargs["cache_root"],
 150     )
 151
 152     return manifests
 153
 154
 155 def ensure_manifest_directories(logger, test_paths):
 156     for paths in six.itervalues(test_paths):
 157         manifest_dir = os.path.dirname(paths["manifest_path"])
 158         if not os.path.exists(manifest_dir):
 159             logger.info("Creating directory %s" % manifest_dir)
 160             # Even though we just checked the path doesn't exist, there's a chance
 161             # of race condition with another process or thread having created it in
 162             # between. This happens during tests.
 163             try:
 164                 os.makedirs(manifest_dir)
 165             except OSError as e:
 166                 if e.errno != errno.EEXIST:
 167                     raise
 168         elif not os.path.isdir(manifest_dir):
 169             raise IOError("Manifest directory is a file")
 170
 171
 172 def read_local_config(wpt_dir):
 173     src_config_path = os.path.join(wpt_dir, "wptrunner.ini")
 174
 175     parser = configparser.SafeConfigParser()
 176     success = parser.read(src_config_path)
 177     assert src_config_path in success
 178     return parser
 179
 180
 181 def generate_config(logger, repo_root, wpt_dir, dest_path, force_rewrite=False):
 182     """Generate the local wptrunner.ini file to use locally"""
 183     if not os.path.exists(dest_path):
 184         # Even though we just checked the path doesn't exist, there's a chance
 185         # of race condition with another process or thread having created it in
 186         # between. This happens during tests.
 187         try:
 188             os.makedirs(dest_path)
 189         except OSError as e:
 190             if e.errno != errno.EEXIST:
 191                 raise
 192
 193     dest_config_path = os.path.join(dest_path, "wptrunner.local.ini")
 194
 195     if not force_rewrite and os.path.exists(dest_config_path):
 196         logger.debug("Config is up to date, not regenerating")
 197         return dest_config_path
 198
 199     logger.info("Creating config file %s" % dest_config_path)
 200
 201     parser = read_local_config(wpt_dir)
 202
 203     for section in ["manifest:upstream", "manifest:mozilla"]:
 204         meta_rel_path = parser.get(section, "metadata")
 205         tests_rel_path = parser.get(section, "tests")
 206
 207         parser.set(
 208             section, "manifest", os.path.join(dest_path, meta_rel_path, "MANIFEST.json")
 209         )
 210         parser.set(section, "metadata", os.path.join(wpt_dir, meta_rel_path))
 211         parser.set(section, "tests", os.path.join(wpt_dir, tests_rel_path))
 212
 213     parser.set(
 214         "paths",
 215         "prefs",
 216         os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))),
 217     )
 218
 219     with open(dest_config_path, "wt") as config_file:
 220         parser.write(config_file)
 221
 222     return dest_config_path
 223
 224
 225 def load_and_update(
 226     logger,
 227     wpt_dir,
 228     test_paths,
 229     rebuild=False,
 230     config_dir=None,
 231     cache_root=None,
 232     update=True,
 233 ):
 234     rv = {}
 235     wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir).encode()).hexdigest()
 236     for url_base, paths in six.iteritems(test_paths):
 237         manifest_path = paths["manifest_path"]
 238         this_cache_root = os.path.join(
 239             cache_root, wptdir_hash, os.path.dirname(paths["manifest_rel_path"])
 240         )
 241         m = manifest.manifest.load_and_update(
 242             paths["tests_path"],
 243             manifest_path,
 244             url_base,
 245             update=update,
 246             rebuild=rebuild,
 247             working_copy=True,
 248             cache_root=this_cache_root,
 249         )
 250         path_data = {"url_base": url_base}
 251         path_data.update(paths)
 252         rv[m] = path_data
 253
 254     return rv
 255
 256
 257 def log_error(logger, manifest_path, msg):
 258     logger.lint_error(
 259         path=manifest_path, message=msg, lineno=0, source="", linter="wpt-manifest"
 260     )