test/lisp/progmodes/flymake-tests.el

   1 ;;; flymake-tests.el --- Test suite for flymake -*- lexical-binding: t -*-
   2
   3 ;; Copyright (C) 2011-2018 Free Software Foundation, Inc.
   4
   5 ;; Author: Eduard Wiebe <usenet@pusto.de>
   6
   7 ;; This file is part of GNU Emacs.
   8
   9 ;; GNU Emacs is free software: you can redistribute it and/or modify
  10 ;; it under the terms of the GNU General Public License as published by
  11 ;; the Free Software Foundation, either version 3 of the License, or
  12 ;; (at your option) any later version.
  13
  14 ;; GNU Emacs is distributed in the hope that it will be useful,
  15 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ;; GNU General Public License for more details.
  18
  19 ;; You should have received a copy of the GNU General Public License
  20 ;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
  21
  22 ;;; Commentary:
  23
  24 ;;; Code:
  25 (require 'ert)
  26 (require 'flymake)
  27 (eval-when-compile (require 'subr-x)) ; string-trim
  28
  29 (defvar flymake-tests-data-directory
  30   (expand-file-name "lisp/progmodes/flymake-resources"
  31                     (or (getenv "EMACS_TEST_DIRECTORY")
  32                         (expand-file-name "../../.."
  33                                           (or load-file-name
  34                                               buffer-file-name))))
  35   "Directory containing flymake test data.")
  36
  37 \f
  38 ;;
  39 ;;
  40 (defun flymake-tests--wait-for-backends ()
  41   ;; Weirdness here...  https://debbugs.gnu.org/17647#25
  42   ;; ... meaning `sleep-for', and even
  43   ;; `accept-process-output', won't suffice as ways to get
  44   ;; process filters and sentinels to run, though they do work
  45   ;; fine in a non-interactive batch session. The only thing
  46   ;; that will indeed unblock pending process output is
  47   ;; reading an input event, so, as a workaround, use a dummy
  48   ;; `read-event' with a very short timeout.
  49   (unless noninteractive (read-event "" nil 0.1))
  50   (cl-loop repeat 5
  51            for notdone = (cl-set-difference (flymake-running-backends)
  52                                             (flymake-reporting-backends))
  53            while notdone
  54            unless noninteractive do (read-event "" nil 0.1)
  55            do (sleep-for (+ 0.5 flymake-no-changes-timeout))
  56            finally (when notdone (ert-fail
  57                                   (format "Some backends not reporting yet %s"
  58                                           notdone)))))
  59
  60 (cl-defun flymake-tests--call-with-fixture (fn file
  61                                                &key (severity-predicate
  62                                                      nil sev-pred-supplied-p))
  63   "Call FN after flymake setup in FILE, using `flymake-proc`.
  64 SEVERITY-PREDICATE is used to setup
  65 `flymake-proc-diagnostic-type-pred'"
  66   (let* ((file (expand-file-name file flymake-tests-data-directory))
  67          (visiting (find-buffer-visiting file))
  68          (buffer (or visiting (find-file-noselect file)))
  69          (process-environment (cons "LC_ALL=C" process-environment))
  70          (warning-minimum-log-level :error))
  71     (unwind-protect
  72         (with-current-buffer buffer
  73           (save-excursion
  74             (when sev-pred-supplied-p
  75               (setq-local flymake-proc-diagnostic-type-pred severity-predicate))
  76             (goto-char (point-min))
  77             (let ((flymake-start-on-flymake-mode nil))
  78               (unless flymake-mode (flymake-mode 1)))
  79             (flymake-start)
  80             (flymake-tests--wait-for-backends)
  81             (funcall fn)))
  82       (and buffer
  83            (not visiting)
  84            (let (kill-buffer-query-functions) (kill-buffer buffer))))))
  85
  86 (cl-defmacro flymake-tests--with-flymake ((file &rest args)
  87                                           &body body)
  88   (declare (indent 1)
  89            (debug (sexp &rest form)))
  90   `(flymake-tests--call-with-fixture (lambda () ,@body) ,file ,@args))
  91
  92 (ert-deftest warning-predicate-rx-gcc ()
  93   "Test GCC warning via regexp predicate."
  94   (skip-unless (and (executable-find "gcc") (executable-find "make")))
  95   (flymake-tests--with-flymake
  96       ("test.c" :severity-predicate "^[Ww]arning")
  97     (flymake-goto-next-error)
  98     (should (eq 'flymake-warning
  99                 (face-at-point)))))
 100
 101 (ert-deftest warning-predicate-function-gcc ()
 102   "Test GCC warning via function predicate."
 103   (skip-unless (and (executable-find "gcc") (executable-find "make")))
 104   (flymake-tests--with-flymake
 105       ("test.c" :severity-predicate
 106        (lambda (msg) (string-match "^[Ww]arning" msg)))
 107     (flymake-goto-next-error)
 108     (should (eq 'flymake-warning
 109                 (face-at-point)))))
 110
 111 (ert-deftest perl-backend ()
 112   "Test the perl backend"
 113   (skip-unless (executable-find "perl"))
 114   (flymake-tests--with-flymake ("test.pl")
 115     (flymake-goto-next-error)
 116     (should (eq 'flymake-warning (face-at-point)))
 117     (goto-char (point-max))
 118     (flymake-goto-prev-error)
 119     (should (eq 'flymake-error (face-at-point)))))
 120
 121 (ert-deftest ruby-backend ()
 122   "Test the ruby backend"
 123   (skip-unless (executable-find "ruby"))
 124   ;; Some versions of ruby fail if HOME doesn't exist (bug#29187).
 125   (let* ((tempdir (make-temp-file "flymake-tests-ruby" t))
 126          (process-environment (cons (format "HOME=%s" tempdir)
 127                                     process-environment))
 128          ;; And see https://debbugs.gnu.org/cgi/bugreport.cgi?bug=19657#20
 129          ;; for this particular yuckiness
 130          (abbreviated-home-dir nil))
 131     (unwind-protect
 132         (flymake-tests--with-flymake ("test.rb")
 133           (flymake-goto-next-error)
 134           (should (eq 'flymake-warning (face-at-point)))
 135           (flymake-goto-next-error)
 136           (should (eq 'flymake-error (face-at-point))))
 137       (delete-directory tempdir t))))
 138
 139 (ert-deftest different-diagnostic-types ()
 140   "Test GCC warning via function predicate."
 141   (skip-unless (and (executable-find "gcc")
 142                     (version<=
 143                      "5" (string-trim
 144                           (shell-command-to-string "gcc -dumpversion")))
 145                     (executable-find "make")))
 146   (let ((flymake-wrap-around nil))
 147     (flymake-tests--with-flymake
 148         ("errors-and-warnings.c")
 149       (flymake-goto-next-error)
 150       (should (eq 'flymake-error (face-at-point)))
 151       (flymake-goto-next-error)
 152       (should (eq 'flymake-note (face-at-point)))
 153       (flymake-goto-next-error)
 154       (should (eq 'flymake-warning (face-at-point)))
 155       (flymake-goto-next-error)
 156       (should (eq 'flymake-error (face-at-point)))
 157       (flymake-goto-next-error)
 158       (should (eq 'flymake-warning (face-at-point)))
 159       (flymake-goto-next-error)
 160       (should (eq 'flymake-warning (face-at-point)))
 161       (should-error (flymake-goto-next-error nil nil t)))))
 162
 163 (ert-deftest included-c-header-files ()
 164   "Test inclusion of .h header files."
 165   (skip-unless (and (executable-find "gcc") (executable-find "make")))
 166   (let ((flymake-wrap-around nil))
 167     (flymake-tests--with-flymake
 168         ("some-problems.h")
 169       (flymake-goto-next-error)
 170       (should (eq 'flymake-warning (face-at-point)))
 171       (flymake-goto-next-error)
 172       (should (eq 'flymake-error (face-at-point)))
 173       (should-error (flymake-goto-next-error nil nil t)))
 174     (flymake-tests--with-flymake
 175         ("no-problems.h")
 176       (should-error (flymake-goto-next-error nil nil t)))))
 177
 178 (defmacro flymake-tests--assert-set (set
 179                                      should
 180                                      should-not)
 181   (declare (indent 1))
 182   `(progn
 183      ,@(cl-loop
 184         for s in should
 185         collect `(should (memq (quote ,s) ,set)))
 186      ,@(cl-loop
 187         for s in should-not
 188         collect `(should-not (memq (quote ,s) ,set)))))
 189
 190 (defun flymake-tests--diagnose-words
 191     (report-fn type words)
 192   "Helper. Call REPORT-FN with diagnostics for WORDS in buffer."
 193   (funcall report-fn
 194            (cl-loop
 195             for word in words
 196             append
 197             (save-excursion
 198               (goto-char (point-min))
 199               (cl-loop while (word-search-forward word nil t)
 200                        collect (flymake-make-diagnostic
 201                                 (current-buffer)
 202                                 (match-beginning 0)
 203                                 (match-end 0)
 204                                 type
 205                                 (concat word " is wrong")))))))
 206
 207 (ert-deftest dummy-backends ()
 208   "Test many different kinds of backends."
 209   (with-temp-buffer
 210     (cl-letf
 211         (((symbol-function 'error-backend)
 212           (lambda (report-fn)
 213             (run-with-timer
 214              0.5 nil
 215              #'flymake-tests--diagnose-words report-fn :error '("manha" "prognata"))))
 216          ((symbol-function 'warning-backend)
 217           (lambda (report-fn)
 218             (run-with-timer
 219              0.5 nil
 220              #'flymake-tests--diagnose-words report-fn :warning '("ut" "dolor"))))
 221          ((symbol-function 'sync-backend)
 222           (lambda (report-fn)
 223             (flymake-tests--diagnose-words report-fn :note '("quis" "commodo"))))
 224          ((symbol-function 'panicking-backend)
 225           (lambda (report-fn)
 226             (run-with-timer
 227              0.5 nil
 228              report-fn :panic :explanation "The spanish inquisition!")))
 229          ((symbol-function 'crashing-backend)
 230           (lambda (_report-fn)
 231             ;; HACK: Shoosh log during tests
 232             (setq-local warning-minimum-log-level :emergency)
 233             (error "crashed"))))
 234       (insert "Lorem ipsum dolor sit amet, consectetur adipiscing
 235     elit, sed do eiusmod tempor incididunt ut labore et dolore
 236     manha aliqua. Ut enim ad minim veniam, quis nostrud
 237     exercitation ullamco laboris nisi ut aliquip ex ea commodo
 238     consequat. Duis aute irure dolor in reprehenderit in
 239     voluptate velit esse cillum dolore eu fugiat nulla
 240     pariatur. Excepteur sint occaecat cupidatat non prognata
 241     sunt in culpa qui officia deserunt mollit anim id est
 242     laborum.")
 243       (let ((flymake-diagnostic-functions
 244              (list 'error-backend 'warning-backend 'sync-backend
 245                    'panicking-backend
 246                    'crashing-backend
 247                    ))
 248             (flymake-wrap-around nil))
 249         (let ((flymake-start-on-flymake-mode nil))
 250           (flymake-mode))
 251         (flymake-start)
 252
 253         (flymake-tests--assert-set (flymake-running-backends)
 254           (error-backend warning-backend panicking-backend)
 255           (crashing-backend))
 256
 257         (flymake-tests--assert-set (flymake-disabled-backends)
 258           (crashing-backend)
 259           (error-backend warning-backend sync-backend
 260                          panicking-backend))
 261
 262         (flymake-tests--wait-for-backends)
 263
 264         (flymake-tests--assert-set (flymake-disabled-backends)
 265           (crashing-backend panicking-backend)
 266           (error-backend warning-backend sync-backend))
 267
 268         (goto-char (point-min))
 269         (flymake-goto-next-error)
 270         (should (eq 'flymake-warning (face-at-point))) ; dolor
 271         (flymake-goto-next-error)
 272         (should (eq 'flymake-warning (face-at-point))) ; ut
 273         (flymake-goto-next-error)
 274         (should (eq 'flymake-error (face-at-point))) ; manha
 275         (flymake-goto-next-error)
 276         (should (eq 'flymake-warning (face-at-point))) ; Ut
 277         (flymake-goto-next-error)
 278         (should (eq 'flymake-note (face-at-point))) ; quis
 279         (flymake-goto-next-error)
 280         (should (eq 'flymake-warning (face-at-point))) ; ut
 281         (flymake-goto-next-error)
 282         (should (eq 'flymake-note (face-at-point))) ; commodo
 283         (flymake-goto-next-error)
 284         (should (eq 'flymake-warning (face-at-point))) ; dolor
 285         (flymake-goto-next-error)
 286         (should (eq 'flymake-error (face-at-point))) ; prognata
 287         (should-error (flymake-goto-next-error nil nil t))))))
 288
 289 (ert-deftest recurrent-backend ()
 290   "Test a backend that calls REPORT-FN multiple times"
 291   (with-temp-buffer
 292     (let (tick)
 293       (cl-letf
 294           (((symbol-function 'eager-backend)
 295             (lambda (report-fn)
 296               (funcall report-fn nil :explanation "very eager but no diagnostics")
 297               (display-buffer (current-buffer))
 298               (run-with-timer
 299                0.5 nil
 300                (lambda ()
 301                  (flymake-tests--diagnose-words report-fn :warning '("consectetur"))
 302                  (setq tick t)
 303                  (run-with-timer
 304                   0.5 nil
 305                   (lambda ()
 306                     (flymake-tests--diagnose-words report-fn :error '("fugiat"))
 307                     (setq tick t))))))))
 308         (insert "Lorem ipsum dolor sit amet, consectetur adipiscing
 309     elit, sed do eiusmod tempor incididunt ut labore et dolore
 310     manha aliqua. Ut enim ad minim veniam, quis nostrud
 311     exercitation ullamco laboris nisi ut aliquip ex ea commodo
 312     consequat. Duis aute irure dolor in reprehenderit in
 313     voluptate velit esse cillum dolore eu fugiat nulla
 314     pariatur. Excepteur sint occaecat cupidatat non prognata
 315     sunt in culpa qui officia deserunt mollit anim id est
 316     laborum.")
 317         (let ((flymake-diagnostic-functions
 318                (list 'eager-backend))
 319               (flymake-wrap-around nil))
 320           (let ((flymake-start-on-flymake-mode nil))
 321             (flymake-mode))
 322           (flymake-start)
 323           (flymake-tests--assert-set (flymake-running-backends)
 324             (eager-backend) ())
 325           (cl-loop until tick repeat 4 do (sleep-for 0.2))
 326           (setq tick nil)
 327           (goto-char (point-max))
 328           (flymake-goto-prev-error)
 329           (should (eq 'flymake-warning (face-at-point))) ; consectetur
 330           (should-error (flymake-goto-prev-error nil nil t))
 331           (cl-loop until tick repeat 4 do (sleep-for 0.2))
 332           (flymake-goto-next-error)
 333           (should (eq 'flymake-error (face-at-point))) ; fugiat
 334           (flymake-goto-prev-error)
 335           (should (eq 'flymake-warning (face-at-point))) ; back at consectetur
 336           (should-error (flymake-goto-prev-error nil nil t))
 337           )))))
 338
 339 (ert-deftest eob-region-and-trailing-newline ()
 340   "`flymake-diag-region' at eob with varying trailing newlines."
 341   (cl-flet ((diag-region-substring
 342              (line col)
 343              (pcase-let
 344                   ((`(,a . ,b) (flymake-diag-region (current-buffer) line col)))
 345                 (buffer-substring a b))))
 346     (with-temp-buffer
 347       (insert "beg\nmmm\nend")
 348       (should (equal
 349                (diag-region-substring 3 3)
 350                "d"))
 351       (should (equal
 352                (diag-region-substring 3 nil)
 353                "end"))
 354       (insert "\n")
 355       (should (equal
 356                (diag-region-substring 4 1)
 357                "end"))
 358       (should (equal
 359                (diag-region-substring 4 nil)
 360                "end"))
 361       (insert "\n")
 362       (should (equal
 363                (diag-region-substring 5 1)
 364                "\n"))
 365       (should (equal
 366                (diag-region-substring 5 nil)
 367                "\n")))))
 368
 369
 370
 371 (provide 'flymake-tests)
 372
 373 ;;; flymake.el ends here