1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
12 from mozboot
.util
import get_state_dir
14 from mozlog
.structured
import commandline
15 from wptrunner
.wptcommandline
import set_from_config
17 import manifestdownload
18 from wptrunner
import wptcommandline
23 def do_delayed_imports(wpt_dir
):
25 imp
.load_source("localpaths",
26 os
.path
.join(wpt_dir
, "tests", "tools", "localpaths.py"))
27 sys
.path
.insert(0, os
.path
.join(wpt_dir
, "tools", "manifest"))
32 p
= argparse
.ArgumentParser()
33 p
.add_argument("--rebuild", action
="store_true",
34 help="Rebuild manifest from scratch")
35 download_group
= p
.add_mutually_exclusive_group()
36 download_group
.add_argument(
37 "--download", dest
="download", action
="store_true", default
=None,
38 help="Always download even if the local manifest is recent")
39 download_group
.add_argument(
40 "--no-download", dest
="download", action
="store_false",
41 help="Don't try to download the manifest")
43 "--no-update", action
="store_false", dest
="update",
44 default
=True, help="Just download the manifest, don't update")
46 "--config", action
="store", dest
="config_path", default
=None,
47 help="Path to wptrunner config file")
49 "--rewrite-config", action
="store_true", default
=False,
50 help="Force the local configuration to be regenerated")
52 "--cache-root", action
="store", default
=os
.path
.join(get_state_dir(), "cache", "wpt"),
53 help="Path to use for the metadata cache")
54 commandline
.add_logging_group(p
)
59 def ensure_kwargs(kwargs
):
60 _kwargs
= vars(create_parser().parse_args([]))
61 _kwargs
.update(kwargs
)
65 def run(src_root
, obj_root
, logger
=None, **kwargs
):
66 kwargs
= ensure_kwargs(kwargs
)
69 from wptrunner
import wptlogging
70 logger
= wptlogging
.setup(kwargs
, {"mach": sys
.stdout
})
72 src_wpt_dir
= os
.path
.join(src_root
, "testing", "web-platform")
74 do_delayed_imports(src_wpt_dir
)
76 if not kwargs
["config_path"]:
77 config_path
= generate_config(logger
,
80 os
.path
.join(obj_root
, "_tests", "web-platform"),
81 kwargs
["rewrite_config"])
83 config_path
= kwargs
["config_path"]
85 if not os
.path
.exists(config_path
):
86 logger
.critical("Config file %s does not exist" % config_path
)
89 logger
.debug("Using config path %s" % config_path
)
91 test_paths
= wptcommandline
.get_test_paths(
92 wptcommandline
.config
.read(config_path
))
94 for paths
in test_paths
.itervalues():
95 if "manifest_path" not in paths
:
96 paths
["manifest_path"] = os
.path
.join(paths
["metadata_path"],
99 ensure_manifest_directories(logger
, test_paths
)
101 local_config
= read_local_config(src_wpt_dir
)
102 for section
in ["manifest:upstream", "manifest:mozilla"]:
103 url_base
= local_config
.get(section
, "url_base")
104 manifest_rel_path
= os
.path
.join(local_config
.get(section
, "metadata"),
106 test_paths
[url_base
]["manifest_rel_path"] = manifest_rel_path
108 if not kwargs
["rebuild"] and kwargs
["download"] is not False:
109 force_download
= False if kwargs
["download"] is None else True
110 manifestdownload
.download_from_taskcluster(logger
,
113 force
=force_download
)
115 logger
.debug("Skipping manifest download")
117 update
= kwargs
["update"] or kwargs
["rebuild"]
118 manifests
= load_and_update(logger
, src_wpt_dir
, test_paths
,
120 rebuild
=kwargs
["rebuild"],
121 cache_root
=kwargs
["cache_root"])
126 def ensure_manifest_directories(logger
, test_paths
):
127 for paths
in test_paths
.itervalues():
128 manifest_dir
= os
.path
.dirname(paths
["manifest_path"])
129 if not os
.path
.exists(manifest_dir
):
130 logger
.info("Creating directory %s" % manifest_dir
)
131 os
.makedirs(manifest_dir
)
132 elif not os
.path
.isdir(manifest_dir
):
133 raise IOError("Manifest directory is a file")
136 def read_local_config(wpt_dir
):
137 src_config_path
= os
.path
.join(wpt_dir
, "wptrunner.ini")
139 parser
= ConfigParser
.SafeConfigParser()
140 success
= parser
.read(src_config_path
)
141 assert src_config_path
in success
145 def generate_config(logger
, repo_root
, wpt_dir
, dest_path
, force_rewrite
=False):
146 """Generate the local wptrunner.ini file to use locally"""
147 if not os
.path
.exists(dest_path
):
148 os
.makedirs(dest_path
)
150 dest_config_path
= os
.path
.join(dest_path
, 'wptrunner.local.ini')
152 if not force_rewrite
and os
.path
.exists(dest_config_path
):
153 logger
.debug("Config is up to date, not regenerating")
154 return dest_config_path
156 logger
.info("Creating config file %s" % dest_config_path
)
158 parser
= read_local_config(wpt_dir
)
160 for section
in ["manifest:upstream", "manifest:mozilla"]:
161 meta_rel_path
= parser
.get(section
, "metadata")
162 tests_rel_path
= parser
.get(section
, "tests")
164 parser
.set(section
, "manifest",
165 os
.path
.join(dest_path
, meta_rel_path
, 'MANIFEST.json'))
166 parser
.set(section
, "metadata", os
.path
.join(wpt_dir
, meta_rel_path
))
167 parser
.set(section
, "tests", os
.path
.join(wpt_dir
, tests_rel_path
))
169 parser
.set('paths', 'prefs', os
.path
.abspath(os
.path
.join(wpt_dir
, parser
.get("paths", "prefs"))))
171 with
open(dest_config_path
, 'wb') as config_file
:
172 parser
.write(config_file
)
174 return dest_config_path
177 def load_and_update(logger
, wpt_dir
, test_paths
, rebuild
=False, config_dir
=None, cache_root
=None,
180 wptdir_hash
= hashlib
.sha256(os
.path
.abspath(wpt_dir
)).hexdigest()
181 for url_base
, paths
in test_paths
.iteritems():
182 manifest_path
= paths
["manifest_path"]
183 this_cache_root
= os
.path
.join(cache_root
, wptdir_hash
, os
.path
.dirname(paths
["manifest_rel_path"]))
184 m
= manifest
.manifest
.load_and_update(paths
["tests_path"],
190 cache_root
=this_cache_root
)
191 path_data
= {"url_base": url_base
}
192 path_data
.update(paths
)
198 def log_error(logger
, manifest_path
, msg
):
199 logger
.lint_error(path
=manifest_path
,
203 linter
="wpt-manifest")