testing/test/test_skipfails.py

   1 # -*- coding: utf-8 -*-
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import json
   7 from pathlib import Path
   8
   9 import pytest
  10 from mozunit import main
  11 from skipfails import MockTask, Skipfails
  12
  13 DATA_PATH = Path(__file__).with_name("data")
  14
  15
  16 def test_get_revision():
  17     """Test get_revision"""
  18
  19     sf = Skipfails()
  20     with pytest.raises(ValueError) as e_info:
  21         sf.get_revision("")
  22     assert str(e_info.value) == "try_url scheme not https"
  23
  24     with pytest.raises(ValueError) as e_info:
  25         sf.get_revision("https://foo.bar")
  26     assert str(e_info.value) == "try_url server not treeherder.mozilla.org"
  27
  28     with pytest.raises(ValueError) as e_info:
  29         sf.get_revision("https://treeherder.mozilla.org")
  30     assert str(e_info.value) == "try_url query missing"
  31
  32     with pytest.raises(ValueError) as e_info:
  33         sf.get_revision("https://treeherder.mozilla.org?a=1")
  34     assert str(e_info.value) == "try_url query missing revision"
  35
  36     revision, repo = sf.get_revision(
  37         "https://treeherder.mozilla.org/jobs?repo=try&revision=5b1738d0af571777199ff3c694b1590ff574946b"
  38     )
  39     assert revision == "5b1738d0af571777199ff3c694b1590ff574946b"
  40     assert repo == "try"
  41
  42
  43 def test_get_tasks():
  44     """Test get_tasks import of mozci"""
  45
  46     from mozci.push import Push
  47
  48     revision = "5b1738d0af571777199ff3c694b1590ff574946b"
  49     repo = "try"
  50     push = Push(revision, repo)
  51     assert push is not None
  52
  53
  54 def get_failures(tasks_name, exp_f_name):
  55     """Runs Skipfails.get_failures on tasks to compare with failures"""
  56     sf = Skipfails()
  57     tasks_fp = DATA_PATH.joinpath(tasks_name).open("r", encoding="utf-8")
  58     tasks = json.load(tasks_fp)
  59     tasks = [MockTask(task) for task in tasks]
  60     exp_f_fp = DATA_PATH.joinpath(exp_f_name).open("r", encoding="utf-8")
  61     expected_failures = exp_f_fp.read().strip()
  62     failures = sf.get_failures(tasks)
  63     actual_failures = json.dumps(failures, indent=2, sort_keys=True).strip()
  64     assert actual_failures == expected_failures
  65
  66
  67 def test_get_failures_1():
  68     """Test get_failures 1"""
  69     get_failures("wayland-tasks-1.json", "wayland-failures-1.json")
  70
  71
  72 def test_get_failures_2():
  73     """Test get_failures 2"""
  74     get_failures("wayland-tasks-2.json", "wayland-failures-2.json")
  75
  76
  77 def test_get_failures_3():
  78     """Test get_failures 3"""
  79     get_failures("wayland-tasks-3.json", "wayland-failures-3.json")
  80
  81
  82 def test_get_failures_4():
  83     """Test get_failures 4"""
  84     get_failures("wayland-tasks-4.json", "wayland-failures-4.json")
  85
  86
  87 def test_get_bug_by_id():
  88     """Test get_bug_by_id"""
  89
  90     sf = Skipfails()
  91     id = 1682371
  92     bug = sf.get_bug_by_id(id)
  93     assert bug.id == id
  94     assert bug.product == "Testing"
  95     assert bug.component == "General"
  96     assert (
  97         bug.summary
  98         == "create tool to quickly parse and identify all failures from a try push and ideally annotate manifests"
  99     )
 100
 101
 102 def test_get_variants():
 103     """Test get_variants"""
 104
 105     sf = Skipfails()
 106     variants = sf.get_variants()
 107     assert "1proc" in variants
 108     assert variants["1proc"] == "e10s"
 109     assert "webrender-sw" in variants
 110     assert variants["webrender-sw"] == "swgl"
 111     assert "aab" in variants
 112     assert variants["aab"] == "aab"
 113
 114
 115 def test_task_to_skip_if():
 116     """Test task_to_skip_if"""
 117
 118     # preload task cache
 119     task_id = "UP-t3xrGSDWvUNjFGIt_aQ"
 120     task = {
 121         "expires": "2024-01-09T16:05:56.825Z",
 122         "extra": {
 123             "suite": "mochitest-plain",
 124             "test-setting": {
 125                 "build": {"type": "debug"},
 126                 "platform": {
 127                     "arch": "32",
 128                     "os": {"build": "2009", "name": "windows", "version": "11"},
 129                 },
 130                 "runtime": {},
 131             },
 132         },
 133     }
 134     sf = Skipfails()
 135     sf.tasks[task_id] = task
 136     # function under test
 137     skip_if = sf.task_to_skip_if(task_id)
 138     assert skip_if == "os == 'win' && os_version == '11' && bits == '32' && debug"
 139
 140
 141 def test_get_filename_in_manifest():
 142     """Test get_filename_in_manifest"""
 143
 144     sf = Skipfails()
 145
 146     assert (
 147         sf.get_filename_in_manifest(
 148             "browser/components/sessionstore/test/browser.toml",
 149             "browser/components/sessionstore/test/browser_closed_tabs_windows.js",
 150         )
 151         == "browser_closed_tabs_windows.js"
 152     )
 153     assert (
 154         sf.get_filename_in_manifest(
 155             "browser/base/content/test/webrtc/gracePeriod/browser.toml",
 156             "browser/base/content/test/webrtc/browser_devices_get_user_media_grace.js",
 157         )
 158         == "../browser_devices_get_user_media_grace.js"
 159     )
 160     assert (
 161         sf.get_filename_in_manifest(
 162             "dom/animation/test/mochitest.toml",
 163             "dom/animation/test/document-timeline/test_document-timeline.html",
 164         )
 165         == "document-timeline/test_document-timeline.html"
 166     )
 167
 168
 169 def test_label_to_platform_testname():
 170     """Test label_to_platform_testname"""
 171
 172     sf = Skipfails()
 173     label = "test-linux2204-64-wayland/opt-mochitest-browser-chrome-swr-13"
 174     platform, testname = sf.label_to_platform_testname(label)
 175     assert platform == "test-linux2204-64-wayland/opt"
 176     assert testname == "mochitest-browser-chrome"
 177
 178
 179 if __name__ == "__main__":
 180     main()