testing/web-platform/tests/tools/manifest/sourcefile.py

   1 import hashlib
   2 import re
   3 import os
   4 from collections import deque
   5 from six import binary_type, PY3
   6 from six.moves.urllib.parse import urljoin
   7 from fnmatch import fnmatch
   8
   9 MYPY = False
  10 if MYPY:
  11     # MYPY is set to True when run under Mypy.
  12     from typing import Any
  13     from typing import AnyStr
  14     from typing import BinaryIO
  15     from typing import Callable
  16     from typing import Deque
  17     from typing import Dict
  18     from typing import Iterable
  19     from typing import List
  20     from typing import Optional
  21     from typing import Pattern
  22     from typing import Set
  23     from typing import Text
  24     from typing import Tuple
  25     from typing import Union
  26     from typing import cast
  27
  28 try:
  29     from xml.etree import cElementTree as ElementTree
  30 except ImportError:
  31     from xml.etree import ElementTree as ElementTree  # type: ignore
  32
  33 import html5lib
  34
  35 from . import XMLParser
  36 from .item import (ManifestItem, ManualTest, WebDriverSpecTest, RefTest, TestharnessTest,
  37                    SupportFile, CrashTest, ConformanceCheckerTest, VisualTest)
  38 from .utils import ContextManagerBytesIO, cached_property
  39
  40 wd_pattern = "*.py"
  41 js_meta_re = re.compile(br"//\s*META:\s*(\w*)=(.*)$")
  42 python_meta_re = re.compile(br"#\s*META:\s*(\w*)=(.*)$")
  43
  44 reference_file_re = re.compile(r'(^|[\-_])(not)?ref[0-9]*([\-_]|$)')
  45
  46 space_chars = u"".join(html5lib.constants.spaceCharacters)  # type: Text
  47
  48 def replace_end(s, old, new):
  49     # type: (Text, Text, Text) -> Text
  50     """
  51     Given a string `s` that ends with `old`, replace that occurrence of `old`
  52     with `new`.
  53     """
  54     assert s.endswith(old)
  55     return s[:-len(old)] + new
  56
  57
  58 def read_script_metadata(f, regexp):
  59     # type: (BinaryIO, Pattern[bytes]) -> Iterable[Tuple[bytes, bytes]]
  60     """
  61     Yields any metadata (pairs of bytestrings) from the file-like object `f`,
  62     as specified according to a supplied regexp.
  63
  64     `regexp` - Regexp containing two groups containing the metadata name and
  65                value.
  66     """
  67     for line in f:
  68         assert isinstance(line, binary_type), line
  69         m = regexp.match(line)
  70         if not m:
  71             break
  72
  73         yield (m.groups()[0], m.groups()[1])
  74
  75
  76 _any_variants = {
  77     b"default": {"longhand": {b"window", b"dedicatedworker"}},
  78     b"window": {"suffix": ".any.html"},
  79     b"serviceworker": {"force_https": True},
  80     b"sharedworker": {},
  81     b"dedicatedworker": {"suffix": ".any.worker.html"},
  82     b"worker": {"longhand": {b"dedicatedworker", b"sharedworker", b"serviceworker"}},
  83     b"jsshell": {"suffix": ".any.js"},
  84 }  # type: Dict[bytes, Dict[str, Any]]
  85
  86
  87 def get_any_variants(item):
  88     # type: (bytes) -> Set[bytes]
  89     """
  90     Returns a set of variants (bytestrings) defined by the given keyword.
  91     """
  92     assert isinstance(item, binary_type), item
  93     assert not item.startswith(b"!"), item
  94
  95     variant = _any_variants.get(item, None)
  96     if variant is None:
  97         return set()
  98
  99     return variant.get("longhand", {item})
 100
 101
 102 def get_default_any_variants():
 103     # type: () -> Set[bytes]
 104     """
 105     Returns a set of variants (bytestrings) that will be used by default.
 106     """
 107     return set(_any_variants[b"default"]["longhand"])
 108
 109
 110 def parse_variants(value):
 111     # type: (bytes) -> Set[bytes]
 112     """
 113     Returns a set of variants (bytestrings) defined by a comma-separated value.
 114     """
 115     assert isinstance(value, binary_type), value
 116
 117     globals = get_default_any_variants()
 118
 119     for item in value.split(b","):
 120         item = item.strip()
 121         if item.startswith(b"!"):
 122             globals -= get_any_variants(item[1:])
 123         else:
 124             globals |= get_any_variants(item)
 125
 126     return globals
 127
 128
 129 def global_suffixes(value):
 130     # type: (bytes) -> Set[Tuple[bytes, bool]]
 131     """
 132     Yields tuples of the relevant filename suffix (a string) and whether the
 133     variant is intended to run in a JS shell, for the variants defined by the
 134     given comma-separated value.
 135     """
 136     assert isinstance(value, binary_type), value
 137
 138     rv = set()
 139
 140     global_types = parse_variants(value)
 141     for global_type in global_types:
 142         variant = _any_variants[global_type]
 143         suffix = variant.get("suffix", ".any.%s.html" % global_type.decode("utf-8"))
 144         rv.add((suffix, global_type == b"jsshell"))
 145
 146     return rv
 147
 148
 149 def global_variant_url(url, suffix):
 150     # type: (Text, Text) -> Text
 151     """
 152     Returns a url created from the given url and suffix (all strings).
 153     """
 154     url = url.replace(".any.", ".")
 155     # If the url must be loaded over https, ensure that it will have
 156     # the form .https.any.js
 157     if ".https." in url and suffix.startswith(".https."):
 158         url = url.replace(".https.", ".")
 159     elif ".h2." in url and suffix.startswith(".h2."):
 160         url = url.replace(".h2.", ".")
 161     return replace_end(url, ".js", suffix)
 162
 163
 164 def _parse_html(f):
 165     # type: (BinaryIO) -> ElementTree.ElementTree
 166     doc = html5lib.parse(f, treebuilder="etree", useChardet=False)
 167     if MYPY:
 168         return cast(ElementTree.ElementTree, doc)
 169     return doc
 170
 171 def _parse_xml(f):
 172     # type: (BinaryIO) -> ElementTree.ElementTree
 173     try:
 174         # raises ValueError with an unsupported encoding,
 175         # ParseError when there's an undefined entity
 176         return ElementTree.parse(f)
 177     except (ValueError, ElementTree.ParseError):
 178         f.seek(0)
 179         return ElementTree.parse(f, XMLParser.XMLParser())  # type: ignore
 180
 181
 182 class SourceFile(object):
 183     parsers = {"html":_parse_html,
 184                "xhtml":_parse_xml,
 185                "svg":_parse_xml}  # type: Dict[Text, Callable[[BinaryIO], ElementTree.ElementTree]]
 186
 187     root_dir_non_test = {"common"}
 188
 189     dir_non_test = {"resources",
 190                     "support",
 191                     "tools"}
 192
 193     dir_path_non_test = {("css21", "archive"),
 194                          ("css", "CSS2", "archive"),
 195                          ("css", "common")}  # type: Set[Tuple[bytes, ...]]
 196
 197     def __init__(self, tests_root, rel_path, url_base, hash=None, contents=None):
 198         # type: (AnyStr, AnyStr, Text, Optional[bytes], Optional[bytes]) -> None
 199         """Object representing a file in a source tree.
 200
 201         :param tests_root: Path to the root of the source tree
 202         :param rel_path: File path relative to tests_root
 203         :param url_base: Base URL used when converting file paths to urls
 204         :param contents: Byte array of the contents of the file or ``None``.
 205         """
 206
 207         assert not os.path.isabs(rel_path), rel_path
 208
 209         if os.name == "nt":
 210             # do slash normalization on Windows
 211             if isinstance(rel_path, binary_type):
 212                 rel_path = rel_path.replace(b"/", b"\\")
 213             else:
 214                 rel_path = rel_path.replace(u"/", u"\\")
 215
 216         dir_path, filename = os.path.split(rel_path)
 217         name, ext = os.path.splitext(filename)
 218
 219         type_flag = None
 220         if "-" in name:
 221             type_flag = name.rsplit("-", 1)[1].split(".")[0]
 222
 223         meta_flags = name.split(".")[1:]
 224
 225         self.tests_root = tests_root  # type: Union[bytes, Text]
 226         self.rel_path = rel_path  # type: Union[bytes, Text]
 227         self.dir_path = dir_path  # type: Union[bytes, Text]
 228         self.filename = filename  # type: Union[bytes, Text]
 229         self.name = name  # type: Union[bytes, Text]
 230         self.ext = ext  # type: Union[bytes, Text]
 231         self.type_flag = type_flag  # type: Optional[Union[bytes, Text]]
 232         self.meta_flags = meta_flags  # type: Union[List[bytes], List[Text]]
 233         self.url_base = url_base
 234         self.contents = contents
 235         self.items_cache = None  # type: Optional[Tuple[Text, List[ManifestItem]]]
 236         self._hash = hash
 237
 238     def __getstate__(self):
 239         # type: () -> Dict[str, Any]
 240         # Remove computed properties if we pickle this class
 241         rv = self.__dict__.copy()
 242
 243         if "__cached_properties__" in rv:
 244             cached_properties = rv["__cached_properties__"]
 245             for key in rv.keys():
 246                 if key in cached_properties:
 247                     del rv[key]
 248             del rv["__cached_properties__"]
 249         return rv
 250
 251     def name_prefix(self, prefix):
 252         # type: (bytes) -> bool
 253         """Check if the filename starts with a given prefix
 254
 255         :param prefix: The prefix to check"""
 256         return self.name.startswith(prefix)
 257
 258     def is_dir(self):
 259         # type: () -> bool
 260         """Return whether this file represents a directory."""
 261         if self.contents is not None:
 262             return False
 263
 264         return os.path.isdir(self.rel_path)
 265
 266     def open(self):
 267         # type: () -> BinaryIO
 268         """
 269         Return either
 270         * the contents specified in the constructor, if any;
 271         * a File object opened for reading the file contents.
 272         """
 273
 274         if self.contents is not None:
 275             wrapped = ContextManagerBytesIO(self.contents)
 276             if MYPY:
 277                 file_obj = cast(BinaryIO, wrapped)
 278             else:
 279                 file_obj = wrapped
 280         else:
 281             file_obj = open(self.path, 'rb')
 282         return file_obj
 283
 284     @cached_property
 285     def path(self):
 286         # type: () -> Union[bytes, Text]
 287         return os.path.join(self.tests_root, self.rel_path)
 288
 289     @cached_property
 290     def rel_url(self):
 291         # type: () -> Text
 292         assert not os.path.isabs(self.rel_path), self.rel_path
 293         return self.rel_path.replace(os.sep, "/")
 294
 295     @cached_property
 296     def url(self):
 297         # type: () -> Text
 298         return urljoin(self.url_base, self.rel_url)
 299
 300     @cached_property
 301     def hash(self):
 302         # type: () -> bytes
 303         if not self._hash:
 304             with self.open() as f:
 305                 content = f.read()
 306
 307             data = b"".join((b"blob ", b"%d" % len(content), b"\0", content))
 308             hash_str = hashlib.sha1(data).hexdigest()  # type: str
 309             if PY3:
 310                 self._hash = hash_str.encode("ascii")
 311             else:
 312                 self._hash = hash_str
 313
 314         return self._hash
 315
 316     def in_non_test_dir(self):
 317         # type: () -> bool
 318         if self.dir_path == "":
 319             return True
 320
 321         parts = self.dir_path.split(os.path.sep)
 322
 323         if (parts[0] in self.root_dir_non_test or
 324             any(item in self.dir_non_test for item in parts) or
 325             any(parts[:len(path)] == list(path) for path in self.dir_path_non_test)):
 326             return True
 327         return False
 328
 329     def in_conformance_checker_dir(self):
 330         # type: () -> bool
 331         return (self.dir_path == "conformance-checkers" or
 332                 self.dir_path.startswith("conformance-checkers" + os.path.sep))
 333
 334     @property
 335     def name_is_non_test(self):
 336         # type: () -> bool
 337         """Check if the file name matches the conditions for the file to
 338         be a non-test file"""
 339         return (self.is_dir() or
 340                 self.name_prefix("MANIFEST") or
 341                 self.filename == "META.yml" or
 342                 self.filename.startswith(".") or
 343                 self.filename.endswith(".headers") or
 344                 self.filename.endswith(".ini") or
 345                 self.in_non_test_dir())
 346
 347     @property
 348     def name_is_conformance(self):
 349         # type: () -> bool
 350         return (self.in_conformance_checker_dir() and
 351                 self.type_flag in ("is-valid", "no-valid"))
 352
 353     @property
 354     def name_is_conformance_support(self):
 355         # type: () -> bool
 356         return self.in_conformance_checker_dir()
 357
 358     @property
 359     def name_is_manual(self):
 360         # type: () -> bool
 361         """Check if the file name matches the conditions for the file to
 362         be a manual test file"""
 363         return self.type_flag == "manual"
 364
 365     @property
 366     def name_is_visual(self):
 367         # type: () -> bool
 368         """Check if the file name matches the conditions for the file to
 369         be a visual test file"""
 370         return self.type_flag == "visual"
 371
 372     @property
 373     def name_is_multi_global(self):
 374         # type: () -> bool
 375         """Check if the file name matches the conditions for the file to
 376         be a multi-global js test file"""
 377         return "any" in self.meta_flags and self.ext == ".js"
 378
 379     @property
 380     def name_is_worker(self):
 381         # type: () -> bool
 382         """Check if the file name matches the conditions for the file to
 383         be a worker js test file"""
 384         return "worker" in self.meta_flags and self.ext == ".js"
 385
 386     @property
 387     def name_is_window(self):
 388         # type: () -> bool
 389         """Check if the file name matches the conditions for the file to
 390         be a window js test file"""
 391         return "window" in self.meta_flags and self.ext == ".js"
 392
 393     @property
 394     def name_is_webdriver(self):
 395         # type: () -> bool
 396         """Check if the file name matches the conditions for the file to
 397         be a webdriver spec test file"""
 398         # wdspec tests are in subdirectories of /webdriver excluding __init__.py
 399         # files.
 400         rel_dir_tree = self.rel_path.split(os.path.sep)
 401         return (((rel_dir_tree[0] == "webdriver" and len(rel_dir_tree) > 1) or
 402                  (rel_dir_tree[:2] == ["infrastructure", "webdriver"] and
 403                   len(rel_dir_tree) > 2)) and
 404                 self.filename not in ("__init__.py", "conftest.py") and
 405                 fnmatch(self.filename, wd_pattern))
 406
 407     @property
 408     def name_is_reference(self):
 409         # type: () -> bool
 410         """Check if the file name matches the conditions for the file to
 411         be a reference file (not a reftest)"""
 412         return "/reference/" in self.url or bool(reference_file_re.search(self.name))
 413
 414     @property
 415     def name_is_crashtest(self):
 416         # type: () -> bool
 417         return self.type_flag == "crash" or "crashtests" in self.dir_path.split(os.path.sep)
 418
 419     @property
 420     def markup_type(self):
 421         # type: () -> Optional[Text]
 422         """Return the type of markup contained in a file, based on its extension,
 423         or None if it doesn't contain markup"""
 424         ext = self.ext
 425
 426         if not ext:
 427             return None
 428         if ext[0] == ".":
 429             ext = ext[1:]
 430         if ext in ["html", "htm"]:
 431             return "html"
 432         if ext in ["xhtml", "xht", "xml"]:
 433             return "xhtml"
 434         if ext == "svg":
 435             return "svg"
 436         return None
 437
 438     @cached_property
 439     def root(self):
 440         # type: () -> Optional[Union[ElementTree.Element, ElementTree.ElementTree]]
 441         """Return an ElementTree Element for the root node of the file if it contains
 442         markup, or None if it does not"""
 443         if not self.markup_type:
 444             return None
 445
 446         parser = self.parsers[self.markup_type]
 447
 448         with self.open() as f:
 449             try:
 450                 tree = parser(f)
 451             except Exception:
 452                 return None
 453
 454         if hasattr(tree, "getroot"):
 455             root = tree.getroot()  # type: Union[ElementTree.Element, ElementTree.ElementTree]
 456         else:
 457             root = tree
 458
 459         return root
 460
 461     @cached_property
 462     def timeout_nodes(self):
 463         # type: () -> List[ElementTree.Element]
 464         """List of ElementTree Elements corresponding to nodes in a test that
 465         specify timeouts"""
 466         assert self.root is not None
 467         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='timeout']")
 468
 469     @cached_property
 470     def script_metadata(self):
 471         # type: () -> Optional[List[Tuple[bytes, bytes]]]
 472         if self.name_is_worker or self.name_is_multi_global or self.name_is_window:
 473             regexp = js_meta_re
 474         elif self.name_is_webdriver:
 475             regexp = python_meta_re
 476         else:
 477             return None
 478
 479         with self.open() as f:
 480             return list(read_script_metadata(f, regexp))
 481
 482     @cached_property
 483     def timeout(self):
 484         # type: () -> Optional[Text]
 485         """The timeout of a test or reference file. "long" if the file has an extended timeout
 486         or None otherwise"""
 487         if self.script_metadata:
 488             if any(m == (b"timeout", b"long") for m in self.script_metadata):
 489                 return "long"
 490
 491         if self.root is None:
 492             return None
 493
 494         if self.timeout_nodes:
 495             timeout_str = self.timeout_nodes[0].attrib.get("content", None)  # type: Optional[Text]
 496             if timeout_str and timeout_str.lower() == "long":
 497                 return "long"
 498
 499         return None
 500
 501     @cached_property
 502     def viewport_nodes(self):
 503         # type: () -> List[ElementTree.Element]
 504         """List of ElementTree Elements corresponding to nodes in a test that
 505         specify viewport sizes"""
 506         assert self.root is not None
 507         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='viewport-size']")
 508
 509     @cached_property
 510     def viewport_size(self):
 511         # type: () -> Optional[Text]
 512         """The viewport size of a test or reference file"""
 513         if self.root is None:
 514             return None
 515
 516         if not self.viewport_nodes:
 517             return None
 518
 519         return self.viewport_nodes[0].attrib.get("content", None)
 520
 521     @cached_property
 522     def dpi_nodes(self):
 523         # type: () -> List[ElementTree.Element]
 524         """List of ElementTree Elements corresponding to nodes in a test that
 525         specify device pixel ratios"""
 526         assert self.root is not None
 527         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='device-pixel-ratio']")
 528
 529     @cached_property
 530     def dpi(self):
 531         # type: () -> Optional[Text]
 532         """The device pixel ratio of a test or reference file"""
 533         if self.root is None:
 534             return None
 535
 536         if not self.dpi_nodes:
 537             return None
 538
 539         return self.dpi_nodes[0].attrib.get("content", None)
 540
 541     @cached_property
 542     def fuzzy_nodes(self):
 543         # type: () -> List[ElementTree.Element]
 544         """List of ElementTree Elements corresponding to nodes in a test that
 545         specify reftest fuzziness"""
 546         assert self.root is not None
 547         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='fuzzy']")
 548
 549     @cached_property
 550     def fuzzy(self):
 551         # type: () -> Dict[Optional[Tuple[Text, Text, Text]], List[List[int]]]
 552         rv = {}  # type: Dict[Optional[Tuple[Text, Text, Text]], List[List[int]]]
 553         if self.root is None:
 554             return rv
 555
 556         if not self.fuzzy_nodes:
 557             return rv
 558
 559         args = [u"maxDifference", u"totalPixels"]
 560
 561         for node in self.fuzzy_nodes:
 562             item = node.attrib.get(u"content", u"")  # type: Text
 563
 564             parts = item.rsplit(u":", 1)
 565             if len(parts) == 1:
 566                 key = None  # type: Optional[Tuple[Text, Text, Text]]
 567                 value = parts[0]
 568             else:
 569                 key_part = urljoin(self.url, parts[0])
 570                 reftype = None
 571                 for ref in self.references:  # type: Tuple[Text, Text]
 572                     if ref[0] == key_part:
 573                         reftype = ref[1]
 574                         break
 575                 if reftype not in (u"==", u"!="):
 576                     raise ValueError("Fuzzy key %s doesn't correspond to a references" % key_part)
 577                 key = (self.url, key_part, reftype)
 578                 value = parts[1]
 579             ranges = value.split(u";")
 580             if len(ranges) != 2:
 581                 raise ValueError("Malformed fuzzy value %s" % item)
 582             arg_values = {}  # type: Dict[Text, List[int]]
 583             positional_args = deque()  # type: Deque[List[int]]
 584             for range_str_value in ranges:  # type: Text
 585                 name = None  # type: Optional[Text]
 586                 if u"=" in range_str_value:
 587                     name, range_str_value = [part.strip()
 588                                              for part in range_str_value.split(u"=", 1)]
 589                     if name not in args:
 590                         raise ValueError("%s is not a valid fuzzy property" % name)
 591                     if arg_values.get(name):
 592                         raise ValueError("Got multiple values for argument %s" % name)
 593                 if u"-" in range_str_value:
 594                     range_min, range_max = range_str_value.split(u"-")
 595                 else:
 596                     range_min = range_str_value
 597                     range_max = range_str_value
 598                 try:
 599                     range_value = [int(x.strip()) for x in (range_min, range_max)]
 600                 except ValueError:
 601                     raise ValueError("Fuzzy value %s must be a range of integers" %
 602                                      range_str_value)
 603                 if name is None:
 604                     positional_args.append(range_value)
 605                 else:
 606                     arg_values[name] = range_value
 607             rv[key] = []
 608             for arg_name in args:
 609                 if arg_values.get(arg_name):
 610                     arg_value = arg_values.pop(arg_name)
 611                 else:
 612                     arg_value = positional_args.popleft()
 613                 rv[key].append(arg_value)
 614             assert len(arg_values) == 0 and len(positional_args) == 0
 615         return rv
 616
 617     @cached_property
 618     def testharness_nodes(self):
 619         # type: () -> List[ElementTree.Element]
 620         """List of ElementTree Elements corresponding to nodes representing a
 621         testharness.js script"""
 622         assert self.root is not None
 623         return self.root.findall(".//{http://www.w3.org/1999/xhtml}script[@src='/resources/testharness.js']")
 624
 625     @cached_property
 626     def content_is_testharness(self):
 627         # type: () -> Optional[bool]
 628         """Boolean indicating whether the file content represents a
 629         testharness.js test"""
 630         if self.root is None:
 631             return None
 632         return bool(self.testharness_nodes)
 633
 634     @cached_property
 635     def variant_nodes(self):
 636         # type: () -> List[ElementTree.Element]
 637         """List of ElementTree Elements corresponding to nodes representing a
 638         test variant"""
 639         assert self.root is not None
 640         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='variant']")
 641
 642     @cached_property
 643     def test_variants(self):
 644         # type: () -> List[Text]
 645         rv = []  # type: List[Text]
 646         if self.ext == ".js":
 647             script_metadata = self.script_metadata
 648             assert script_metadata is not None
 649             for (key, value) in script_metadata:
 650                 if key == b"variant":
 651                     rv.append(value.decode("utf-8"))
 652         else:
 653             for element in self.variant_nodes:
 654                 if "content" in element.attrib:
 655                     variant = element.attrib["content"]  # type: Text
 656                     rv.append(variant)
 657
 658         for variant in rv:
 659             assert variant == "" or variant[0] in ["#", "?"], variant
 660
 661         if not rv:
 662             rv = [""]
 663
 664         return rv
 665
 666     @cached_property
 667     def testdriver_nodes(self):
 668         # type: () -> List[ElementTree.Element]
 669         """List of ElementTree Elements corresponding to nodes representing a
 670         testdriver.js script"""
 671         assert self.root is not None
 672         return self.root.findall(".//{http://www.w3.org/1999/xhtml}script[@src='/resources/testdriver.js']")
 673
 674     @cached_property
 675     def has_testdriver(self):
 676         # type: () -> Optional[bool]
 677         """Boolean indicating whether the file content represents a
 678         testharness.js test"""
 679         if self.root is None:
 680             return None
 681         return bool(self.testdriver_nodes)
 682
 683     @cached_property
 684     def reftest_nodes(self):
 685         # type: () -> List[ElementTree.Element]
 686         """List of ElementTree Elements corresponding to nodes representing a
 687         to a reftest <link>"""
 688         if self.root is None:
 689             return []
 690
 691         match_links = self.root.findall(".//{http://www.w3.org/1999/xhtml}link[@rel='match']")
 692         mismatch_links = self.root.findall(".//{http://www.w3.org/1999/xhtml}link[@rel='mismatch']")
 693         return match_links + mismatch_links
 694
 695     @cached_property
 696     def references(self):
 697         # type: () -> List[Tuple[Text, Text]]
 698         """List of (ref_url, relation) tuples for any reftest references specified in
 699         the file"""
 700         rv = []  # type: List[Tuple[Text, Text]]
 701         rel_map = {"match": "==", "mismatch": "!="}
 702         for item in self.reftest_nodes:
 703             if "href" in item.attrib:
 704                 ref_url = urljoin(self.url, item.attrib["href"].strip(space_chars))
 705                 ref_type = rel_map[item.attrib["rel"]]
 706                 rv.append((ref_url, ref_type))
 707         return rv
 708
 709     @cached_property
 710     def content_is_ref_node(self):
 711         # type: () -> bool
 712         """Boolean indicating whether the file is a non-leaf node in a reftest
 713         graph (i.e. if it contains any <link rel=[mis]match>"""
 714         return bool(self.references)
 715
 716     @cached_property
 717     def css_flag_nodes(self):
 718         # type: () -> List[ElementTree.Element]
 719         """List of ElementTree Elements corresponding to nodes representing a
 720         flag <meta>"""
 721         if self.root is None:
 722             return []
 723         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='flags']")
 724
 725     @cached_property
 726     def css_flags(self):
 727         # type: () -> Set[Text]
 728         """Set of flags specified in the file"""
 729         rv = set()  # type: Set[Text]
 730         for item in self.css_flag_nodes:
 731             if "content" in item.attrib:
 732                 for flag in item.attrib["content"].split():
 733                     rv.add(flag)
 734         return rv
 735
 736     @cached_property
 737     def content_is_css_manual(self):
 738         # type: () -> Optional[bool]
 739         """Boolean indicating whether the file content represents a
 740         CSS WG-style manual test"""
 741         if self.root is None:
 742             return None
 743         # return True if the intersection between the two sets is non-empty
 744         return bool(self.css_flags & {"animated", "font", "history", "interact", "paged", "speech", "userstyle"})
 745
 746     @cached_property
 747     def spec_link_nodes(self):
 748         # type: () -> List[ElementTree.Element]
 749         """List of ElementTree Elements corresponding to nodes representing a
 750         <link rel=help>, used to point to specs"""
 751         if self.root is None:
 752             return []
 753         return self.root.findall(".//{http://www.w3.org/1999/xhtml}link[@rel='help']")
 754
 755     @cached_property
 756     def spec_links(self):
 757         # type: () -> Set[Text]
 758         """Set of spec links specified in the file"""
 759         rv = set()  # type: Set[Text]
 760         for item in self.spec_link_nodes:
 761             if "href" in item.attrib:
 762                 rv.add(item.attrib["href"].strip(space_chars))
 763         return rv
 764
 765     @cached_property
 766     def content_is_css_visual(self):
 767         # type: () -> Optional[bool]
 768         """Boolean indicating whether the file content represents a
 769         CSS WG-style visual test"""
 770         if self.root is None:
 771             return None
 772         return bool(self.ext in {'.xht', '.html', '.xhtml', '.htm', '.xml', '.svg'} and
 773                     self.spec_links)
 774
 775     @property
 776     def type(self):
 777         # type: () -> Text
 778         rv, _ = self.manifest_items()
 779         return rv
 780
 781     def manifest_items(self):
 782         # type: () -> Tuple[Text, List[ManifestItem]]
 783         """List of manifest items corresponding to the file. There is typically one
 784         per test, but in the case of reftests a node may have corresponding manifest
 785         items without being a test itself."""
 786
 787         if self.items_cache:
 788             return self.items_cache
 789
 790         if self.name_is_non_test:
 791             rv = "support", [
 792                 SupportFile(
 793                     self.tests_root,
 794                     self.rel_path
 795                 )]  # type: Tuple[Text, List[ManifestItem]]
 796
 797         elif self.name_is_manual:
 798             rv = ManualTest.item_type, [
 799                 ManualTest(
 800                     self.tests_root,
 801                     self.rel_path,
 802                     self.url_base,
 803                     self.rel_url
 804                 )]
 805
 806         elif self.name_is_conformance:
 807             rv = ConformanceCheckerTest.item_type, [
 808                 ConformanceCheckerTest(
 809                     self.tests_root,
 810                     self.rel_path,
 811                     self.url_base,
 812                     self.rel_url
 813                 )]
 814
 815         elif self.name_is_conformance_support:
 816             rv = "support", [
 817                 SupportFile(
 818                     self.tests_root,
 819                     self.rel_path
 820                 )]
 821
 822         elif self.name_is_visual:
 823             rv = VisualTest.item_type, [
 824                 VisualTest(
 825                     self.tests_root,
 826                     self.rel_path,
 827                     self.url_base,
 828                     self.rel_url
 829                 )]
 830
 831         elif self.name_is_crashtest:
 832             rv = CrashTest.item_type, [
 833                 CrashTest(
 834                     self.tests_root,
 835                     self.rel_path,
 836                     self.url_base,
 837                     self.rel_url
 838                 )]
 839
 840         elif self.name_is_multi_global:
 841             globals = b""
 842             script_metadata = self.script_metadata
 843             assert script_metadata is not None
 844             for (key, value) in script_metadata:
 845                 if key == b"global":
 846                     globals = value
 847                     break
 848
 849             tests = [
 850                 TestharnessTest(
 851                     self.tests_root,
 852                     self.rel_path,
 853                     self.url_base,
 854                     global_variant_url(self.rel_url, suffix) + variant,
 855                     timeout=self.timeout,
 856                     jsshell=jsshell,
 857                     script_metadata=self.script_metadata
 858                 )
 859                 for (suffix, jsshell) in sorted(global_suffixes(globals))
 860                 for variant in self.test_variants
 861             ]   # type: List[ManifestItem]
 862             rv = TestharnessTest.item_type, tests
 863
 864         elif self.name_is_worker:
 865             test_url = replace_end(self.rel_url, ".worker.js", ".worker.html")
 866             tests = [
 867                 TestharnessTest(
 868                     self.tests_root,
 869                     self.rel_path,
 870                     self.url_base,
 871                     test_url + variant,
 872                     timeout=self.timeout,
 873                     script_metadata=self.script_metadata
 874                 )
 875                 for variant in self.test_variants
 876             ]
 877             rv = TestharnessTest.item_type, tests
 878
 879         elif self.name_is_window:
 880             test_url = replace_end(self.rel_url, ".window.js", ".window.html")
 881             tests = [
 882                 TestharnessTest(
 883                     self.tests_root,
 884                     self.rel_path,
 885                     self.url_base,
 886                     test_url + variant,
 887                     timeout=self.timeout,
 888                     script_metadata=self.script_metadata
 889                 )
 890                 for variant in self.test_variants
 891             ]
 892             rv = TestharnessTest.item_type, tests
 893
 894         elif self.name_is_webdriver:
 895             rv = WebDriverSpecTest.item_type, [
 896                 WebDriverSpecTest(
 897                     self.tests_root,
 898                     self.rel_path,
 899                     self.url_base,
 900                     self.rel_url,
 901                     timeout=self.timeout
 902                 )]
 903
 904         elif self.content_is_css_manual and not self.name_is_reference:
 905             rv = ManualTest.item_type, [
 906                 ManualTest(
 907                     self.tests_root,
 908                     self.rel_path,
 909                     self.url_base,
 910                     self.rel_url
 911                 )]
 912
 913         elif self.content_is_testharness:
 914             rv = TestharnessTest.item_type, []
 915             testdriver = self.has_testdriver
 916             for variant in self.test_variants:
 917                 url = self.rel_url + variant
 918                 rv[1].append(TestharnessTest(
 919                     self.tests_root,
 920                     self.rel_path,
 921                     self.url_base,
 922                     url,
 923                     timeout=self.timeout,
 924                     testdriver=testdriver,
 925                     script_metadata=self.script_metadata
 926                 ))
 927
 928         elif self.content_is_ref_node:
 929             rv = RefTest.item_type, [
 930                 RefTest(
 931                     self.tests_root,
 932                     self.rel_path,
 933                     self.url_base,
 934                     self.rel_url,
 935                     references=self.references,
 936                     timeout=self.timeout,
 937                     viewport_size=self.viewport_size,
 938                     dpi=self.dpi,
 939                     fuzzy=self.fuzzy
 940                 )]
 941
 942         elif self.content_is_css_visual and not self.name_is_reference:
 943             rv = VisualTest.item_type, [
 944                 VisualTest(
 945                     self.tests_root,
 946                     self.rel_path,
 947                     self.url_base,
 948                     self.rel_url
 949                 )]
 950
 951         else:
 952             rv = "support", [
 953                 SupportFile(
 954                     self.tests_root,
 955                     self.rel_path
 956                 )]
 957
 958         assert len(rv[1]) == len(set(rv[1]))
 959
 960         self.items_cache = rv
 961
 962         return rv