testing/web-platform/manifestupdate.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import ConfigParser
   6 import argparse
   7 import hashlib
   8 import imp
   9 import os
  10 import sys
  11
  12 from mozboot.util import get_state_dir
  13
  14 from mozlog.structured import commandline
  15 from wptrunner.wptcommandline import set_from_config
  16
  17 import manifestdownload
  18 from wptrunner import wptcommandline
  19
  20 manifest = None
  21
  22
  23 def do_delayed_imports(wpt_dir):
  24     global manifest
  25     imp.load_source("localpaths",
  26                     os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
  27     sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
  28     import manifest
  29
  30
  31 def create_parser():
  32     p = argparse.ArgumentParser()
  33     p.add_argument("--rebuild", action="store_true",
  34                    help="Rebuild manifest from scratch")
  35     download_group = p.add_mutually_exclusive_group()
  36     download_group.add_argument(
  37         "--download", dest="download", action="store_true", default=None,
  38         help="Always download even if the local manifest is recent")
  39     download_group.add_argument(
  40         "--no-download", dest="download", action="store_false",
  41         help="Don't try to download the manifest")
  42     p.add_argument(
  43         "--no-update", action="store_false", dest="update",
  44         default=True, help="Just download the manifest, don't update")
  45     p.add_argument(
  46         "--config", action="store", dest="config_path", default=None,
  47         help="Path to wptrunner config file")
  48     p.add_argument(
  49         "--rewrite-config", action="store_true", default=False,
  50         help="Force the local configuration to be regenerated")
  51     p.add_argument(
  52         "--cache-root", action="store", default=os.path.join(get_state_dir(), "cache", "wpt"),
  53         help="Path to use for the metadata cache")
  54     commandline.add_logging_group(p)
  55
  56     return p
  57
  58
  59 def ensure_kwargs(kwargs):
  60     _kwargs = vars(create_parser().parse_args([]))
  61     _kwargs.update(kwargs)
  62     return _kwargs
  63
  64
  65 def run(src_root, obj_root, logger=None, **kwargs):
  66     kwargs = ensure_kwargs(kwargs)
  67
  68     if logger is None:
  69         from wptrunner import wptlogging
  70         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
  71
  72     src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
  73
  74     do_delayed_imports(src_wpt_dir)
  75
  76     if not kwargs["config_path"]:
  77         config_path = generate_config(logger,
  78                                       src_root,
  79                                       src_wpt_dir,
  80                                       os.path.join(obj_root, "_tests", "web-platform"),
  81                                       kwargs["rewrite_config"])
  82     else:
  83         config_path = kwargs["config_path"]
  84
  85     if not os.path.exists(config_path):
  86         logger.critical("Config file %s does not exist" % config_path)
  87         return None
  88
  89     logger.debug("Using config path %s" % config_path)
  90
  91     test_paths = wptcommandline.get_test_paths(
  92         wptcommandline.config.read(config_path))
  93
  94     for paths in test_paths.itervalues():
  95         if "manifest_path" not in paths:
  96             paths["manifest_path"] = os.path.join(paths["metadata_path"],
  97                                                   "MANIFEST.json")
  98
  99     ensure_manifest_directories(logger, test_paths)
 100
 101     local_config = read_local_config(src_wpt_dir)
 102     for section in ["manifest:upstream", "manifest:mozilla"]:
 103         url_base = local_config.get(section, "url_base")
 104         manifest_rel_path = os.path.join(local_config.get(section, "metadata"),
 105                                          "MANIFEST.json")
 106         test_paths[url_base]["manifest_rel_path"] = manifest_rel_path
 107
 108     if not kwargs["rebuild"] and kwargs["download"] is not False:
 109         force_download = False if kwargs["download"] is None else True
 110         manifestdownload.download_from_taskcluster(logger,
 111                                                    src_root,
 112                                                    test_paths,
 113                                                    force=force_download)
 114     else:
 115         logger.debug("Skipping manifest download")
 116
 117     update = kwargs["update"] or kwargs["rebuild"]
 118     manifests = load_and_update(logger, src_wpt_dir, test_paths,
 119                                 update=update,
 120                                 rebuild=kwargs["rebuild"],
 121                                 cache_root=kwargs["cache_root"])
 122
 123     return manifests
 124
 125
 126 def ensure_manifest_directories(logger, test_paths):
 127     for paths in test_paths.itervalues():
 128         manifest_dir = os.path.dirname(paths["manifest_path"])
 129         if not os.path.exists(manifest_dir):
 130             logger.info("Creating directory %s" % manifest_dir)
 131             os.makedirs(manifest_dir)
 132         elif not os.path.isdir(manifest_dir):
 133             raise IOError("Manifest directory is a file")
 134
 135
 136 def read_local_config(wpt_dir):
 137     src_config_path = os.path.join(wpt_dir, "wptrunner.ini")
 138
 139     parser = ConfigParser.SafeConfigParser()
 140     success = parser.read(src_config_path)
 141     assert src_config_path in success
 142     return parser
 143
 144
 145 def generate_config(logger, repo_root, wpt_dir, dest_path, force_rewrite=False):
 146     """Generate the local wptrunner.ini file to use locally"""
 147     if not os.path.exists(dest_path):
 148         os.makedirs(dest_path)
 149
 150     dest_config_path = os.path.join(dest_path, 'wptrunner.local.ini')
 151
 152     if not force_rewrite and os.path.exists(dest_config_path):
 153         logger.debug("Config is up to date, not regenerating")
 154         return dest_config_path
 155
 156     logger.info("Creating config file %s" % dest_config_path)
 157
 158     parser = read_local_config(wpt_dir)
 159
 160     for section in ["manifest:upstream", "manifest:mozilla"]:
 161         meta_rel_path = parser.get(section, "metadata")
 162         tests_rel_path = parser.get(section, "tests")
 163
 164         parser.set(section, "manifest",
 165                    os.path.join(dest_path, meta_rel_path, 'MANIFEST.json'))
 166         parser.set(section, "metadata", os.path.join(wpt_dir, meta_rel_path))
 167         parser.set(section, "tests", os.path.join(wpt_dir, tests_rel_path))
 168
 169     parser.set('paths', 'prefs', os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))))
 170
 171     with open(dest_config_path, 'wb') as config_file:
 172         parser.write(config_file)
 173
 174     return dest_config_path
 175
 176
 177 def load_and_update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None, cache_root=None,
 178                     update=True):
 179     rv = {}
 180     wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir)).hexdigest()
 181     for url_base, paths in test_paths.iteritems():
 182         manifest_path = paths["manifest_path"]
 183         this_cache_root = os.path.join(cache_root, wptdir_hash, os.path.dirname(paths["manifest_rel_path"]))
 184         m = manifest.manifest.load_and_update(paths["tests_path"],
 185                                               manifest_path,
 186                                               url_base,
 187                                               update=update,
 188                                               rebuild=rebuild,
 189                                               working_copy=True,
 190                                               cache_root=this_cache_root)
 191         path_data = {"url_base": url_base}
 192         path_data.update(paths)
 193         rv[m] = path_data
 194
 195     return rv
 196
 197
 198 def log_error(logger, manifest_path, msg):
 199     logger.lint_error(path=manifest_path,
 200                       message=msg,
 201                       lineno=0,
 202                       source="",
 203                       linter="wpt-manifest")