tools/lint/test/test_perfdocs_generation.py

   1 import os
   2 import pathlib
   3 from unittest import mock
   4
   5 import mozunit
   6
   7 LINTER = "perfdocs"
   8
   9
  10 def setup_sample_logger(logger, structured_logger, top_dir):
  11     from perfdocs.logger import PerfDocLogger
  12
  13     PerfDocLogger.LOGGER = structured_logger
  14     PerfDocLogger.PATHS = ["perfdocs"]
  15     PerfDocLogger.TOP_DIR = top_dir
  16
  17     import perfdocs.gatherer as gt
  18     import perfdocs.generator as gn
  19     import perfdocs.utils as utls
  20     import perfdocs.verifier as vf
  21
  22     gt.logger = logger
  23     vf.logger = logger
  24     gn.logger = logger
  25     utls.logger = logger
  26
  27
  28 @mock.patch("perfdocs.logger.PerfDocLogger")
  29 def test_perfdocs_generator_generate_perfdocs_pass(
  30     logger, structured_logger, perfdocs_sample
  31 ):
  32     from test_perfdocs import temp_file
  33
  34     top_dir = perfdocs_sample["top_dir"]
  35     setup_sample_logger(logger, structured_logger, top_dir)
  36
  37     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
  38     templates_dir.mkdir(parents=True, exist_ok=True)
  39
  40     from perfdocs.generator import Generator
  41     from perfdocs.verifier import Verifier
  42
  43     verifier = Verifier(top_dir)
  44     verifier.validate_tree()
  45
  46     generator = Generator(verifier, generate=True, workspace=top_dir)
  47     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
  48         generator.generate_perfdocs()
  49
  50     assert logger.warning.call_count == 0
  51
  52
  53 @mock.patch("perfdocs.logger.PerfDocLogger")
  54 def test_perfdocs_generator_needed_regeneration(
  55     logger, structured_logger, perfdocs_sample
  56 ):
  57     top_dir = perfdocs_sample["top_dir"]
  58     setup_sample_logger(logger, structured_logger, top_dir)
  59
  60     from perfdocs.generator import Generator
  61     from perfdocs.verifier import Verifier
  62
  63     verifier = Verifier(top_dir)
  64     verifier.validate_tree()
  65
  66     generator = Generator(verifier, generate=False, workspace=top_dir)
  67     generator.generate_perfdocs()
  68
  69     expected = "PerfDocs need to be regenerated."
  70     args, _ = logger.warning.call_args
  71
  72     assert logger.warning.call_count == 1
  73     assert args[0] == expected
  74
  75
  76 @mock.patch("perfdocs.generator.get_changed_files", new=lambda x: [])
  77 @mock.patch("perfdocs.generator.ON_TRY", new=True)
  78 @mock.patch("perfdocs.logger.PerfDocLogger")
  79 def test_perfdocs_generator_needed_update(logger, structured_logger, perfdocs_sample):
  80     from test_perfdocs import temp_file
  81
  82     top_dir = perfdocs_sample["top_dir"]
  83     setup_sample_logger(logger, structured_logger, top_dir)
  84
  85     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
  86     templates_dir.mkdir(parents=True, exist_ok=True)
  87
  88     from perfdocs.generator import Generator
  89     from perfdocs.verifier import Verifier
  90
  91     # Initializing perfdocs
  92     verifier = Verifier(top_dir)
  93     verifier.validate_tree()
  94
  95     generator = Generator(verifier, generate=True, workspace=top_dir)
  96     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
  97         generator.generate_perfdocs()
  98
  99         # Removed file for testing and run again
 100         generator._generate = False
 101         files = [f for f in os.listdir(generator.perfdocs_path)]
 102         for f in files:
 103             os.remove(str(pathlib.Path(generator.perfdocs_path, f)))
 104
 105         generator.generate_perfdocs()
 106
 107     expected = (
 108         "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. "
 109         "You can also apply the perfdocs.diff patch file produced from this "
 110         "reviewbot test to fix the issue."
 111     )
 112     args, _ = logger.warning.call_args
 113
 114     assert logger.warning.call_count == 1
 115     assert args[0] == expected
 116
 117     # Check to ensure a diff was produced
 118     assert logger.log.call_count == 6
 119
 120     logs = [v[0][0] for v in logger.log.call_args_list]
 121     for failure_log in (
 122         "Some files are missing or are funny.",
 123         "Missing in existing docs: index.rst",
 124         "Missing in existing docs: mozperftest.rst",
 125     ):
 126         assert failure_log in logs
 127
 128
 129 @mock.patch("perfdocs.generator.get_changed_files", new=lambda x: [])
 130 @mock.patch("perfdocs.generator.ON_TRY", new=True)
 131 def test_perfdocs_generator_update_with_no_changes(structured_logger, perfdocs_sample):
 132     """This test ensures that when no changed files exist, we'll still trigger a failure."""
 133     from perfdocs.logger import PerfDocLogger
 134     from test_perfdocs import temp_file
 135
 136     top_dir = perfdocs_sample["top_dir"]
 137
 138     logger_mock = mock.MagicMock()
 139     PerfDocLogger.LOGGER = logger_mock
 140     PerfDocLogger.PATHS = ["perfdocs"]
 141     PerfDocLogger.TOP_DIR = top_dir
 142     logger = PerfDocLogger()
 143
 144     setup_sample_logger(logger, logger_mock, top_dir)
 145
 146     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 147     templates_dir.mkdir(parents=True, exist_ok=True)
 148
 149     from perfdocs.generator import Generator
 150     from perfdocs.verifier import Verifier
 151
 152     # Initializing perfdocs
 153     verifier = Verifier(top_dir)
 154     verifier.validate_tree()
 155
 156     generator = Generator(verifier, generate=True, workspace=top_dir)
 157     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 158         generator.generate_perfdocs()
 159
 160         # Removed file for testing and run again
 161         generator._generate = False
 162         files = [f for f in os.listdir(generator.perfdocs_path)]
 163         for f in files:
 164             os.remove(str(pathlib.Path(generator.perfdocs_path, f)))
 165
 166         generator.generate_perfdocs()
 167
 168     expected = (
 169         "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. "
 170         "You can also apply the perfdocs.diff patch file produced from this "
 171         "reviewbot test to fix the issue."
 172     )
 173     assert logger.LOGGER.lint_error.call_args is not None
 174     _, msg = logger.LOGGER.lint_error.call_args
 175
 176     assert logger.FAILED
 177     assert logger.LOGGER.lint_error.call_count == 1
 178     assert msg["message"] == expected
 179     assert msg["rule"] == "Flawless performance docs (unknown file)"
 180
 181
 182 @mock.patch("perfdocs.logger.PerfDocLogger")
 183 def test_perfdocs_generator_created_perfdocs(
 184     logger, structured_logger, perfdocs_sample
 185 ):
 186     from test_perfdocs import temp_file
 187
 188     top_dir = perfdocs_sample["top_dir"]
 189     setup_sample_logger(logger, structured_logger, top_dir)
 190
 191     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 192     templates_dir.mkdir(parents=True, exist_ok=True)
 193
 194     from perfdocs.generator import Generator
 195     from perfdocs.verifier import Verifier
 196
 197     verifier = Verifier(top_dir)
 198     verifier.validate_tree()
 199
 200     generator = Generator(verifier, generate=True, workspace=top_dir)
 201     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 202         perfdocs_tmpdir = generator._create_perfdocs()
 203
 204     files = [f for f in os.listdir(perfdocs_tmpdir)]
 205     files.sort()
 206     expected_files = ["index.rst", "mozperftest.rst"]
 207
 208     for i, file in enumerate(files):
 209         assert file == expected_files[i]
 210
 211     with pathlib.Path(perfdocs_tmpdir, expected_files[0]).open() as f:
 212         filedata = f.readlines()
 213     assert "".join(filedata) == "  * :doc:`mozperftest`"
 214
 215
 216 @mock.patch("perfdocs.logger.PerfDocLogger")
 217 def test_perfdocs_generator_build_perfdocs(logger, structured_logger, perfdocs_sample):
 218     top_dir = perfdocs_sample["top_dir"]
 219     setup_sample_logger(logger, structured_logger, top_dir)
 220
 221     from perfdocs.generator import Generator
 222     from perfdocs.verifier import Verifier
 223
 224     verifier = Verifier(top_dir)
 225     verifier.validate_tree()
 226
 227     generator = Generator(verifier, generate=True, workspace=top_dir)
 228     frameworks_info = generator.build_perfdocs_from_tree()
 229
 230     expected = ["dynamic", "static"]
 231
 232     for framework in sorted(frameworks_info.keys()):
 233         for i, framework_info in enumerate(frameworks_info[framework]):
 234             assert framework_info == expected[i]
 235
 236
 237 @mock.patch("perfdocs.logger.PerfDocLogger")
 238 def test_perfdocs_generator_create_temp_dir(logger, structured_logger, perfdocs_sample):
 239     top_dir = perfdocs_sample["top_dir"]
 240     setup_sample_logger(logger, structured_logger, top_dir)
 241
 242     from perfdocs.generator import Generator
 243     from perfdocs.verifier import Verifier
 244
 245     verifier = Verifier(top_dir)
 246     verifier.validate_tree()
 247
 248     generator = Generator(verifier, generate=True, workspace=top_dir)
 249     tmpdir = generator._create_temp_dir()
 250
 251     assert pathlib.Path(tmpdir).is_dir()
 252
 253
 254 @mock.patch("perfdocs.logger.PerfDocLogger")
 255 def test_perfdocs_generator_create_temp_dir_fail(
 256     logger, structured_logger, perfdocs_sample
 257 ):
 258     top_dir = perfdocs_sample["top_dir"]
 259     setup_sample_logger(logger, structured_logger, top_dir)
 260
 261     from perfdocs.generator import Generator
 262     from perfdocs.verifier import Verifier
 263
 264     verifier = Verifier(top_dir)
 265     verifier.validate_tree()
 266
 267     generator = Generator(verifier, generate=True, workspace=top_dir)
 268     with mock.patch("perfdocs.generator.pathlib") as path_mock:
 269         path_mock.Path().mkdir.side_effect = OSError()
 270         path_mock.Path().is_dir.return_value = False
 271         tmpdir = generator._create_temp_dir()
 272
 273     expected = "Error creating temp file: "
 274     args, _ = logger.critical.call_args
 275
 276     assert not tmpdir
 277     assert logger.critical.call_count == 1
 278     assert args[0] == expected
 279
 280
 281 @mock.patch("perfdocs.logger.PerfDocLogger")
 282 def test_perfdocs_generator_save_perfdocs_pass(
 283     logger, structured_logger, perfdocs_sample
 284 ):
 285     from test_perfdocs import temp_file
 286
 287     top_dir = perfdocs_sample["top_dir"]
 288     setup_sample_logger(logger, structured_logger, top_dir)
 289
 290     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 291     templates_dir.mkdir(parents=True, exist_ok=True)
 292
 293     from perfdocs.generator import Generator
 294     from perfdocs.verifier import Verifier
 295
 296     verifier = Verifier(top_dir)
 297     verifier.validate_tree()
 298
 299     generator = Generator(verifier, generate=True, workspace=top_dir)
 300
 301     assert not generator.perfdocs_path.is_dir()
 302
 303     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 304         perfdocs_tmpdir = generator._create_perfdocs()
 305
 306     generator._save_perfdocs(perfdocs_tmpdir)
 307
 308     expected = ["index.rst", "mozperftest.rst"]
 309     files = [f for f in os.listdir(generator.perfdocs_path)]
 310     files.sort()
 311
 312     for i, file in enumerate(files):
 313         assert file == expected[i]
 314
 315
 316 @mock.patch("perfdocs.generator.shutil")
 317 @mock.patch("perfdocs.logger.PerfDocLogger")
 318 def test_perfdocs_generator_save_perfdocs_fail(
 319     logger, shutil, structured_logger, perfdocs_sample
 320 ):
 321     from test_perfdocs import temp_file
 322
 323     top_dir = perfdocs_sample["top_dir"]
 324     setup_sample_logger(logger, structured_logger, top_dir)
 325
 326     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 327     templates_dir.mkdir(parents=True, exist_ok=True)
 328
 329     from perfdocs.generator import Generator
 330     from perfdocs.verifier import Verifier
 331
 332     verifier = Verifier(top_dir)
 333     verifier.validate_tree()
 334
 335     generator = Generator(verifier, generate=True, workspace=top_dir)
 336     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 337         perfdocs_tmpdir = generator._create_perfdocs()
 338
 339     shutil.copytree = mock.Mock(side_effect=Exception())
 340     generator._save_perfdocs(perfdocs_tmpdir)
 341
 342     expected = "There was an error while saving the documentation: "
 343     args, _ = logger.critical.call_args
 344
 345     assert logger.critical.call_count == 1
 346     assert args[0] == expected
 347
 348
 349 if __name__ == "__main__":
 350     mozunit.main()