tools/tryselect/test/test_perfcomparators.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import tempfile
   6 from unittest import mock
   7
   8 import mozunit
   9 import pytest
  10 from tryselect.selectors.perfselector.perfcomparators import (
  11     BadComparatorArgs,
  12     BenchmarkComparator,
  13     ComparatorNotFound,
  14     get_comparator,
  15 )
  16
  17
  18 @pytest.mark.parametrize(
  19     "test_link",
  20     [
  21         "https://github.com/mozilla-mobile/firefox-android/pull/1627",
  22         "https://github.com/mozilla-mobile/firefox-android/pull/1876/"
  23         "commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365",
  24     ],
  25 )
  26 def test_benchmark_comparator(test_link):
  27     def _verify_extra_args(extra_args):
  28         assert len(extra_args) == 3
  29         if "commit" in test_link:
  30             assert (
  31                 "benchmark-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365"
  32                 in extra_args
  33             )
  34         else:
  35             assert "benchmark-revision=sha-for-link" in extra_args
  36         assert "benchmark-repository=url-for-link" in extra_args
  37         assert "benchmark-branch=ref-for-link" in extra_args
  38
  39     comparator = BenchmarkComparator(
  40         None, None, None, [f"base-link={test_link}", f"new-link={test_link}"]
  41     )
  42
  43     with mock.patch("requests.get") as mocked_get:
  44         magic_get = mock.MagicMock()
  45         magic_get.json.return_value = {
  46             "head": {
  47                 "repo": {
  48                     "html_url": "url-for-link",
  49                 },
  50                 "sha": "sha-for-link",
  51                 "ref": "ref-for-link",
  52             }
  53         }
  54         magic_get.status_code = 200
  55         mocked_get.return_value = magic_get
  56
  57         extra_args = []
  58         comparator.setup_base_revision(extra_args)
  59         _verify_extra_args(extra_args)
  60
  61         extra_args = []
  62         comparator.setup_new_revision(extra_args)
  63         _verify_extra_args(extra_args)
  64
  65
  66 def test_benchmark_comparator_no_pr_links():
  67     def _verify_extra_args(extra_args):
  68         assert len(extra_args) == 3
  69         assert "benchmark-revision=rev" in extra_args
  70         assert "benchmark-repository=link" in extra_args
  71         assert "benchmark-branch=fake" in extra_args
  72
  73     comparator = BenchmarkComparator(
  74         None,
  75         None,
  76         None,
  77         [
  78             "base-repo=link",
  79             "base-branch=fake",
  80             "base-revision=rev",
  81             "new-repo=link",
  82             "new-branch=fake",
  83             "new-revision=rev",
  84         ],
  85     )
  86
  87     with mock.patch("requests.get") as mocked_get:
  88         magic_get = mock.MagicMock()
  89         magic_get.json.return_value = {
  90             "head": {
  91                 "repo": {
  92                     "html_url": "url-for-link",
  93                 },
  94                 "sha": "sha-for-link",
  95                 "ref": "ref-for-link",
  96             }
  97         }
  98         magic_get.status_code = 200
  99         mocked_get.return_value = magic_get
 100
 101         extra_args = []
 102         comparator.setup_base_revision(extra_args)
 103         _verify_extra_args(extra_args)
 104
 105         extra_args = []
 106         comparator.setup_new_revision(extra_args)
 107         _verify_extra_args(extra_args)
 108
 109
 110 def test_benchmark_comparator_bad_args():
 111     comparator = BenchmarkComparator(
 112         None,
 113         None,
 114         None,
 115         [
 116             "base-bad-args=val",
 117         ],
 118     )
 119
 120     with pytest.raises(BadComparatorArgs):
 121         comparator.setup_base_revision([])
 122
 123
 124 def test_get_comparator_bad_name():
 125     with pytest.raises(ComparatorNotFound):
 126         get_comparator("BadName")
 127
 128
 129 def test_get_comparator_bad_script():
 130     with pytest.raises(ComparatorNotFound):
 131         with tempfile.NamedTemporaryFile() as tmpf:
 132             tmpf.close()
 133             get_comparator(tmpf.name)
 134
 135
 136 def test_get_comparator_benchmark_name():
 137     comparator_klass = get_comparator("BenchmarkComparator")
 138     assert comparator_klass.__name__ == "BenchmarkComparator"
 139
 140
 141 def test_get_comparator_benchmark_script():
 142     # If the get_comparator method is working for scripts, then
 143     # it should find the first defined class in this file, or the
 144     # first imported class that matches it
 145     comparator_klass = get_comparator(__file__)
 146     assert comparator_klass.__name__ == "BenchmarkComparator"
 147
 148
 149 if __name__ == "__main__":
 150     mozunit.main()