test/automated/ert-tests.el

   1 ;;; ert-tests.el --- ERT's self-tests  -*- lexical-binding: t -*-
   2
   3 ;; Copyright (C) 2007-2008, 2010-2012 Free Software Foundation, Inc.
   4
   5 ;; Author: Christian Ohler <ohler@gnu.org>
   6
   7 ;; This file is part of GNU Emacs.
   8
   9 ;; This program is free software: you can redistribute it and/or
  10 ;; modify it under the terms of the GNU General Public License as
  11 ;; published by the Free Software Foundation, either version 3 of the
  12 ;; License, or (at your option) any later version.
  13 ;;
  14 ;; This program is distributed in the hope that it will be useful, but
  15 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 ;; General Public License for more details.
  18 ;;
  19 ;; You should have received a copy of the GNU General Public License
  20 ;; along with this program.  If not, see `http://www.gnu.org/licenses/'.
  21
  22 ;;; Commentary:
  23
  24 ;; This file is part of ERT, the Emacs Lisp Regression Testing tool.
  25 ;; See ert.el or the texinfo manual for more details.
  26
  27 ;;; Code:
  28
  29 (eval-when-compile
  30   (require 'cl-lib))
  31 (require 'ert)
  32
  33
  34 ;;; Self-test that doesn't rely on ERT, for bootstrapping.
  35
  36 ;; This is used to test that bodies actually run.
  37 (defvar ert--test-body-was-run)
  38 (ert-deftest ert-test-body-runs ()
  39   (setq ert--test-body-was-run t))
  40
  41 (defun ert-self-test ()
  42   "Run ERT's self-tests and make sure they actually ran."
  43   (let ((window-configuration (current-window-configuration)))
  44     (let ((ert--test-body-was-run nil))
  45       ;; The buffer name chosen here should not compete with the default
  46       ;; results buffer name for completion in `switch-to-buffer'.
  47       (let ((stats (ert-run-tests-interactively "^ert-" " *ert self-tests*")))
  48         (cl-assert ert--test-body-was-run)
  49         (if (zerop (ert-stats-completed-unexpected stats))
  50             ;; Hide results window only when everything went well.
  51             (set-window-configuration window-configuration)
  52           (error "ERT self-test failed"))))))
  53
  54 (defun ert-self-test-and-exit ()
  55   "Run ERT's self-tests and exit Emacs.
  56
  57 The exit code will be zero if the tests passed, nonzero if they
  58 failed or if there was a problem."
  59   (unwind-protect
  60       (progn
  61         (ert-self-test)
  62         (kill-emacs 0))
  63     (unwind-protect
  64         (progn
  65           (message "Error running tests")
  66           (backtrace))
  67       (kill-emacs 1))))
  68
  69
  70 ;;; Further tests are defined using ERT.
  71
  72 (ert-deftest ert-test-nested-test-body-runs ()
  73   "Test that nested test bodies run."
  74   (let ((was-run nil))
  75     (let ((test (make-ert-test :body (lambda ()
  76                                        (setq was-run t)))))
  77       (cl-assert (not was-run))
  78       (ert-run-test test)
  79       (cl-assert was-run))))
  80
  81
  82 ;;; Test that pass/fail works.
  83 (ert-deftest ert-test-pass ()
  84   (let ((test (make-ert-test :body (lambda ()))))
  85     (let ((result (ert-run-test test)))
  86       (cl-assert (ert-test-passed-p result)))))
  87
  88 (ert-deftest ert-test-fail ()
  89   (let ((test (make-ert-test :body (lambda () (ert-fail "failure message")))))
  90     (let ((result (let ((ert-debug-on-error nil))
  91                     (ert-run-test test))))
  92       (cl-assert (ert-test-failed-p result) t)
  93       (cl-assert (equal (ert-test-result-with-condition-condition result)
  94                      '(ert-test-failed "failure message"))
  95               t))))
  96
  97 (ert-deftest ert-test-fail-debug-with-condition-case ()
  98   (let ((test (make-ert-test :body (lambda () (ert-fail "failure message")))))
  99     (condition-case condition
 100         (progn
 101           (let ((ert-debug-on-error t))
 102             (ert-run-test test))
 103           (cl-assert nil))
 104       ((error)
 105        (cl-assert (equal condition '(ert-test-failed "failure message")) t)))))
 106
 107 (ert-deftest ert-test-fail-debug-with-debugger-1 ()
 108   (let ((test (make-ert-test :body (lambda () (ert-fail "failure message")))))
 109     (let ((debugger (lambda (&rest _args)
 110                       (cl-assert nil))))
 111       (let ((ert-debug-on-error nil))
 112         (ert-run-test test)))))
 113
 114 (ert-deftest ert-test-fail-debug-with-debugger-2 ()
 115   (let ((test (make-ert-test :body (lambda () (ert-fail "failure message")))))
 116     (cl-block nil
 117       (let ((debugger (lambda (&rest _args)
 118                         (cl-return-from nil nil))))
 119         (let ((ert-debug-on-error t))
 120           (ert-run-test test))
 121         (cl-assert nil)))))
 122
 123 (ert-deftest ert-test-fail-debug-nested-with-debugger ()
 124   (let ((test (make-ert-test :body (lambda ()
 125                                      (let ((ert-debug-on-error t))
 126                                        (ert-fail "failure message"))))))
 127     (let ((debugger (lambda (&rest _args)
 128                       (cl-assert nil nil "Assertion a"))))
 129       (let ((ert-debug-on-error nil))
 130         (ert-run-test test))))
 131   (let ((test (make-ert-test :body (lambda ()
 132                                      (let ((ert-debug-on-error nil))
 133                                        (ert-fail "failure message"))))))
 134     (cl-block nil
 135       (let ((debugger (lambda (&rest _args)
 136                         (cl-return-from nil nil))))
 137         (let ((ert-debug-on-error t))
 138           (ert-run-test test))
 139         (cl-assert nil nil "Assertion b")))))
 140
 141 (ert-deftest ert-test-error ()
 142   (let ((test (make-ert-test :body (lambda () (error "Error message")))))
 143     (let ((result (let ((ert-debug-on-error nil))
 144                     (ert-run-test test))))
 145       (cl-assert (ert-test-failed-p result) t)
 146       (cl-assert (equal (ert-test-result-with-condition-condition result)
 147                      '(error "Error message"))
 148               t))))
 149
 150 (ert-deftest ert-test-error-debug ()
 151   (let ((test (make-ert-test :body (lambda () (error "Error message")))))
 152     (condition-case condition
 153         (progn
 154           (let ((ert-debug-on-error t))
 155             (ert-run-test test))
 156           (cl-assert nil))
 157       ((error)
 158        (cl-assert (equal condition '(error "Error message")) t)))))
 159
 160
 161 ;;; Test that `should' works.
 162 (ert-deftest ert-test-should ()
 163   (let ((test (make-ert-test :body (lambda () (should nil)))))
 164     (let ((result (let ((ert-debug-on-error nil))
 165                     (ert-run-test test))))
 166       (cl-assert (ert-test-failed-p result) t)
 167       (cl-assert (equal (ert-test-result-with-condition-condition result)
 168                      '(ert-test-failed ((should nil) :form nil :value nil)))
 169               t)))
 170   (let ((test (make-ert-test :body (lambda () (should t)))))
 171     (let ((result (ert-run-test test)))
 172       (cl-assert (ert-test-passed-p result) t))))
 173
 174 (ert-deftest ert-test-should-value ()
 175   (should (eql (should 'foo) 'foo))
 176   (should (eql (should 'bar) 'bar)))
 177
 178 (ert-deftest ert-test-should-not ()
 179   (let ((test (make-ert-test :body (lambda () (should-not t)))))
 180     (let ((result (let ((ert-debug-on-error nil))
 181                     (ert-run-test test))))
 182       (cl-assert (ert-test-failed-p result) t)
 183       (cl-assert (equal (ert-test-result-with-condition-condition result)
 184                      '(ert-test-failed ((should-not t) :form t :value t)))
 185               t)))
 186   (let ((test (make-ert-test :body (lambda () (should-not nil)))))
 187     (let ((result (ert-run-test test)))
 188       (cl-assert (ert-test-passed-p result)))))
 189
 190
 191 (ert-deftest ert-test-should-with-macrolet ()
 192   (let ((test (make-ert-test :body (lambda ()
 193                                      (cl-macrolet ((foo () `(progn t nil)))
 194                                        (should (foo)))))))
 195     (let ((result (let ((ert-debug-on-error nil))
 196                     (ert-run-test test))))
 197       (should (ert-test-failed-p result))
 198       (should (equal
 199                (ert-test-result-with-condition-condition result)
 200                '(ert-test-failed ((should (foo))
 201                                   :form (progn t nil)
 202                                   :value nil)))))))
 203
 204 (ert-deftest ert-test-should-error ()
 205   ;; No error.
 206   (let ((test (make-ert-test :body (lambda () (should-error (progn))))))
 207     (let ((result (let ((ert-debug-on-error nil))
 208                     (ert-run-test test))))
 209       (should (ert-test-failed-p result))
 210       (should (equal (ert-test-result-with-condition-condition result)
 211                      '(ert-test-failed
 212                        ((should-error (progn))
 213                         :form (progn)
 214                         :value nil
 215                         :fail-reason "did not signal an error"))))))
 216   ;; A simple error.
 217   (should (equal (should-error (error "Foo"))
 218                  '(error "Foo")))
 219   ;; Error of unexpected type.
 220   (let ((test (make-ert-test :body (lambda ()
 221                                      (should-error (error "Foo")
 222                                                    :type 'singularity-error)))))
 223     (let ((result (ert-run-test test)))
 224       (should (ert-test-failed-p result))
 225       (should (equal
 226                (ert-test-result-with-condition-condition result)
 227                '(ert-test-failed
 228                  ((should-error (error "Foo") :type 'singularity-error)
 229                   :form (error "Foo")
 230                   :condition (error "Foo")
 231                   :fail-reason
 232                   "the error signaled did not have the expected type"))))))
 233   ;; Error of the expected type.
 234   (let* ((error nil)
 235          (test (make-ert-test
 236                 :body (lambda ()
 237                         (setq error
 238                               (should-error (signal 'singularity-error nil)
 239                                             :type 'singularity-error))))))
 240     (let ((result (ert-run-test test)))
 241       (should (ert-test-passed-p result))
 242       (should (equal error '(singularity-error))))))
 243
 244 (ert-deftest ert-test-should-error-subtypes ()
 245   (should-error (signal 'singularity-error nil)
 246                 :type 'singularity-error
 247                 :exclude-subtypes t)
 248   (let ((test (make-ert-test
 249                :body (lambda ()
 250                        (should-error (signal 'arith-error nil)
 251                                      :type 'singularity-error)))))
 252     (let ((result (ert-run-test test)))
 253       (should (ert-test-failed-p result))
 254       (should (equal
 255                (ert-test-result-with-condition-condition result)
 256                '(ert-test-failed
 257                  ((should-error (signal 'arith-error nil)
 258                                 :type 'singularity-error)
 259                   :form (signal arith-error nil)
 260                   :condition (arith-error)
 261                   :fail-reason
 262                   "the error signaled did not have the expected type"))))))
 263   (let ((test (make-ert-test
 264                :body (lambda ()
 265                        (should-error (signal 'arith-error nil)
 266                                      :type 'singularity-error
 267                                      :exclude-subtypes t)))))
 268     (let ((result (ert-run-test test)))
 269       (should (ert-test-failed-p result))
 270       (should (equal
 271                (ert-test-result-with-condition-condition result)
 272                '(ert-test-failed
 273                  ((should-error (signal 'arith-error nil)
 274                                 :type 'singularity-error
 275                                 :exclude-subtypes t)
 276                   :form (signal arith-error nil)
 277                   :condition (arith-error)
 278                   :fail-reason
 279                   "the error signaled did not have the expected type"))))))
 280   (let ((test (make-ert-test
 281                :body (lambda ()
 282                        (should-error (signal 'singularity-error nil)
 283                                      :type 'arith-error
 284                                      :exclude-subtypes t)))))
 285     (let ((result (ert-run-test test)))
 286       (should (ert-test-failed-p result))
 287       (should (equal
 288                (ert-test-result-with-condition-condition result)
 289                '(ert-test-failed
 290                  ((should-error (signal 'singularity-error nil)
 291                                 :type 'arith-error
 292                                 :exclude-subtypes t)
 293                   :form (signal singularity-error nil)
 294                   :condition (singularity-error)
 295                   :fail-reason
 296                   "the error signaled was a subtype of the expected type")))))
 297     ))
 298
 299 (defmacro ert--test-my-list (&rest args)
 300   "Don't use this.  Instead, call `list' with ARGS, it does the same thing.
 301
 302 This macro is used to test if macroexpansion in `should' works."
 303   `(list ,@args))
 304
 305 (ert-deftest ert-test-should-failure-debugging ()
 306   "Test that `should' errors contain the information we expect them to."
 307   (cl-loop
 308    for (body expected-condition) in
 309    `((,(lambda () (let ((x nil)) (should x)))
 310       (ert-test-failed ((should x) :form x :value nil)))
 311      (,(lambda () (let ((x t)) (should-not x)))
 312       (ert-test-failed ((should-not x) :form x :value t)))
 313      (,(lambda () (let ((x t)) (should (not x))))
 314       (ert-test-failed ((should (not x)) :form (not t) :value nil)))
 315      (,(lambda () (let ((x nil)) (should-not (not x))))
 316       (ert-test-failed ((should-not (not x)) :form (not nil) :value t)))
 317      (,(lambda () (let ((x t) (y nil)) (should-not
 318                                    (ert--test-my-list x y))))
 319       (ert-test-failed
 320        ((should-not (ert--test-my-list x y))
 321         :form (list t nil)
 322         :value (t nil))))
 323      (,(lambda () (let ((_x t)) (should (error "Foo"))))
 324       (error "Foo")))
 325    do
 326    (let ((test (make-ert-test :body body)))
 327      (condition-case actual-condition
 328          (progn
 329            (let ((ert-debug-on-error t))
 330              (ert-run-test test))
 331            (cl-assert nil))
 332        ((error)
 333         (should (equal actual-condition expected-condition)))))))
 334
 335 (ert-deftest ert-test-deftest ()
 336   (should (equal (macroexpand '(ert-deftest abc () "foo" :tags '(bar)))
 337                  '(progn
 338                     (ert-set-test 'abc
 339                                   (make-ert-test :name 'abc
 340                                                  :documentation "foo"
 341                                                  :tags '(bar)
 342                                                  :body (lambda ())))
 343                     (push '(ert-deftest . abc) current-load-list)
 344                     'abc)))
 345   (should (equal (macroexpand '(ert-deftest def ()
 346                                  :expected-result ':passed))
 347                  '(progn
 348                     (ert-set-test 'def
 349                                   (make-ert-test :name 'def
 350                                                  :expected-result-type ':passed
 351                                                  :body (lambda ())))
 352                     (push '(ert-deftest . def) current-load-list)
 353                     'def)))
 354   ;; :documentation keyword is forbidden
 355   (should-error (macroexpand '(ert-deftest ghi ()
 356                                 :documentation "foo"))))
 357
 358 (ert-deftest ert-test-record-backtrace ()
 359   (let ((test (make-ert-test :body (lambda () (ert-fail "foo")))))
 360     (let ((result (ert-run-test test)))
 361       (should (ert-test-failed-p result))
 362       (with-temp-buffer
 363         (ert--print-backtrace (ert-test-failed-backtrace result))
 364         (goto-char (point-min))
 365         (end-of-line)
 366         (let ((first-line (buffer-substring-no-properties (point-min) (point))))
 367           (should (equal first-line "  signal(ert-test-failed (\"foo\"))")))))))
 368
 369 (ert-deftest ert-test-messages ()
 370   :tags '(:causes-redisplay)
 371   (let* ((message-string "Test message")
 372          (messages-buffer (get-buffer-create "*Messages*"))
 373          (test (make-ert-test :body (lambda () (message "%s" message-string)))))
 374     (with-current-buffer messages-buffer
 375       (let ((result (ert-run-test test)))
 376         (should (equal (concat message-string "\n")
 377                        (ert-test-result-messages result)))))))
 378
 379 (ert-deftest ert-test-running-tests ()
 380   (let ((outer-test (ert-get-test 'ert-test-running-tests)))
 381     (should (equal (ert-running-test) outer-test))
 382     (let (test1 test2 test3)
 383       (setq test1 (make-ert-test
 384                    :name "1"
 385                    :body (lambda ()
 386                            (should (equal (ert-running-test) outer-test))
 387                            (should (equal ert--running-tests
 388                                           (list test1 test2 test3
 389                                                 outer-test)))))
 390             test2 (make-ert-test
 391                    :name "2"
 392                    :body (lambda ()
 393                            (should (equal (ert-running-test) outer-test))
 394                            (should (equal ert--running-tests
 395                                           (list test3 test2 outer-test)))
 396                            (ert-run-test test1)))
 397             test3 (make-ert-test
 398                    :name "3"
 399                    :body (lambda ()
 400                            (should (equal (ert-running-test) outer-test))
 401                            (should (equal ert--running-tests
 402                                           (list test3 outer-test)))
 403                            (ert-run-test test2))))
 404       (should (ert-test-passed-p (ert-run-test test3))))))
 405
 406 (ert-deftest ert-test-test-result-expected-p ()
 407   "Test `ert-test-result-expected-p' and (implicitly) `ert-test-result-type-p'."
 408   ;; passing test
 409   (let ((test (make-ert-test :body (lambda ()))))
 410     (should (ert-test-result-expected-p test (ert-run-test test))))
 411   ;; unexpected failure
 412   (let ((test (make-ert-test :body (lambda () (ert-fail "failed")))))
 413     (should-not (ert-test-result-expected-p test (ert-run-test test))))
 414   ;; expected failure
 415   (let ((test (make-ert-test :body (lambda () (ert-fail "failed"))
 416                              :expected-result-type ':failed)))
 417     (should (ert-test-result-expected-p test (ert-run-test test))))
 418   ;; `not' expected type
 419   (let ((test (make-ert-test :body (lambda ())
 420                              :expected-result-type '(not :failed))))
 421     (should (ert-test-result-expected-p test (ert-run-test test))))
 422   (let ((test (make-ert-test :body (lambda ())
 423                              :expected-result-type '(not :passed))))
 424     (should-not (ert-test-result-expected-p test (ert-run-test test))))
 425   ;; `and' expected type
 426   (let ((test (make-ert-test :body (lambda ())
 427                              :expected-result-type '(and :passed :failed))))
 428     (should-not (ert-test-result-expected-p test (ert-run-test test))))
 429   (let ((test (make-ert-test :body (lambda ())
 430                              :expected-result-type '(and :passed
 431                                                          (not :failed)))))
 432     (should (ert-test-result-expected-p test (ert-run-test test))))
 433   ;; `or' expected type
 434   (let ((test (make-ert-test :body (lambda ())
 435                              :expected-result-type '(or (and :passed :failed)
 436                                                         :passed))))
 437     (should (ert-test-result-expected-p test (ert-run-test test))))
 438   (let ((test (make-ert-test :body (lambda ())
 439                              :expected-result-type '(or (and :passed :failed)
 440                                                         nil (not t)))))
 441     (should-not (ert-test-result-expected-p test (ert-run-test test)))))
 442
 443 ;;; Test `ert-select-tests'.
 444 (ert-deftest ert-test-select-regexp ()
 445   (should (equal (ert-select-tests "^ert-test-select-regexp$" t)
 446                  (list (ert-get-test 'ert-test-select-regexp)))))
 447
 448 (ert-deftest ert-test-test-boundp ()
 449   (should (ert-test-boundp 'ert-test-test-boundp))
 450   (should-not (ert-test-boundp (make-symbol "ert-not-a-test"))))
 451
 452 (ert-deftest ert-test-select-member ()
 453   (should (equal (ert-select-tests '(member ert-test-select-member) t)
 454                  (list (ert-get-test 'ert-test-select-member)))))
 455
 456 (ert-deftest ert-test-select-test ()
 457   (should (equal (ert-select-tests (ert-get-test 'ert-test-select-test) t)
 458                  (list (ert-get-test 'ert-test-select-test)))))
 459
 460 (ert-deftest ert-test-select-symbol ()
 461   (should (equal (ert-select-tests 'ert-test-select-symbol t)
 462                  (list (ert-get-test 'ert-test-select-symbol)))))
 463
 464 (ert-deftest ert-test-select-and ()
 465   (let ((test (make-ert-test
 466                :name nil
 467                :body nil
 468                :most-recent-result (make-ert-test-failed
 469                                     :condition nil
 470                                     :backtrace nil
 471                                     :infos nil))))
 472     (should (equal (ert-select-tests `(and (member ,test) :failed) t)
 473                    (list test)))))
 474
 475 (ert-deftest ert-test-select-tag ()
 476   (let ((test (make-ert-test
 477                :name nil
 478                :body nil
 479                :tags '(a b))))
 480     (should (equal (ert-select-tests `(tag a) (list test)) (list test)))
 481     (should (equal (ert-select-tests `(tag b) (list test)) (list test)))
 482     (should (equal (ert-select-tests `(tag c) (list test)) '()))))
 483
 484
 485 ;;; Tests for utility functions.
 486 (ert-deftest ert-test-proper-list-p ()
 487   (should (ert--proper-list-p '()))
 488   (should (ert--proper-list-p '(1)))
 489   (should (ert--proper-list-p '(1 2)))
 490   (should (ert--proper-list-p '(1 2 3)))
 491   (should (ert--proper-list-p '(1 2 3 4)))
 492   (should (not (ert--proper-list-p 'a)))
 493   (should (not (ert--proper-list-p '(1 . a))))
 494   (should (not (ert--proper-list-p '(1 2 . a))))
 495   (should (not (ert--proper-list-p '(1 2 3 . a))))
 496   (should (not (ert--proper-list-p '(1 2 3 4 . a))))
 497   (let ((a (list 1)))
 498     (setf (cdr (last a)) a)
 499     (should (not (ert--proper-list-p a))))
 500   (let ((a (list 1 2)))
 501     (setf (cdr (last a)) a)
 502     (should (not (ert--proper-list-p a))))
 503   (let ((a (list 1 2 3)))
 504     (setf (cdr (last a)) a)
 505     (should (not (ert--proper-list-p a))))
 506   (let ((a (list 1 2 3 4)))
 507     (setf (cdr (last a)) a)
 508     (should (not (ert--proper-list-p a))))
 509   (let ((a (list 1 2)))
 510     (setf (cdr (last a)) (cdr a))
 511     (should (not (ert--proper-list-p a))))
 512   (let ((a (list 1 2 3)))
 513     (setf (cdr (last a)) (cdr a))
 514     (should (not (ert--proper-list-p a))))
 515   (let ((a (list 1 2 3 4)))
 516     (setf (cdr (last a)) (cdr a))
 517     (should (not (ert--proper-list-p a))))
 518   (let ((a (list 1 2 3)))
 519     (setf (cdr (last a)) (cddr a))
 520     (should (not (ert--proper-list-p a))))
 521   (let ((a (list 1 2 3 4)))
 522     (setf (cdr (last a)) (cddr a))
 523     (should (not (ert--proper-list-p a))))
 524   (let ((a (list 1 2 3 4)))
 525     (setf (cdr (last a)) (cl-cdddr a))
 526     (should (not (ert--proper-list-p a)))))
 527
 528 (ert-deftest ert-test-parse-keys-and-body ()
 529   (should (equal (ert--parse-keys-and-body '(foo)) '(nil (foo))))
 530   (should (equal (ert--parse-keys-and-body '(:bar foo)) '((:bar foo) nil)))
 531   (should (equal (ert--parse-keys-and-body '(:bar foo a (b)))
 532                  '((:bar foo) (a (b)))))
 533   (should (equal (ert--parse-keys-and-body '(:bar foo :a (b)))
 534                  '((:bar foo :a (b)) nil)))
 535   (should (equal (ert--parse-keys-and-body '(bar foo :a (b)))
 536                  '(nil (bar foo :a (b)))))
 537   (should-error (ert--parse-keys-and-body '(:bar foo :a))))
 538
 539
 540 (ert-deftest ert-test-run-tests-interactively ()
 541   :tags '(:causes-redisplay)
 542   (let ((passing-test (make-ert-test :name 'passing-test
 543                                      :body (lambda () (ert-pass))))
 544         (failing-test (make-ert-test :name 'failing-test
 545                                      :body (lambda () (ert-fail
 546                                                        "failure message")))))
 547     (let ((ert-debug-on-error nil))
 548       (let* ((buffer-name (generate-new-buffer-name " *ert-test-run-tests*"))
 549              (messages nil)
 550              (mock-message-fn
 551               (lambda (format-string &rest args)
 552                 (push (apply #'format format-string args) messages))))
 553         (save-window-excursion
 554           (unwind-protect
 555               (let ((case-fold-search nil))
 556                 (ert-run-tests-interactively
 557                  `(member ,passing-test ,failing-test) buffer-name
 558                  mock-message-fn)
 559                 (should (equal messages `(,(concat
 560                                             "Ran 2 tests, 1 results were "
 561                                             "as expected, 1 unexpected"))))
 562                 (with-current-buffer buffer-name
 563                   (goto-char (point-min))
 564                   (should (equal
 565                            (buffer-substring (point-min)
 566                                              (save-excursion
 567                                                (forward-line 4)
 568                                                (point)))
 569                            (concat
 570                             "Selector: (member <passing-test> <failing-test>)\n"
 571                             "Passed: 1\n"
 572                             "Failed: 1 (1 unexpected)\n"
 573                             "Total:  2/2\n")))))
 574             (when (get-buffer buffer-name)
 575               (kill-buffer buffer-name))))))))
 576
 577 (ert-deftest ert-test-special-operator-p ()
 578   (should (ert--special-operator-p 'if))
 579   (should-not (ert--special-operator-p 'car))
 580   (should-not (ert--special-operator-p 'ert--special-operator-p))
 581   (let ((b (ert--gensym)))
 582     (should-not (ert--special-operator-p b))
 583     (fset b 'if)
 584     (should (ert--special-operator-p b))))
 585
 586 (ert-deftest ert-test-list-of-should-forms ()
 587   (let ((test (make-ert-test :body (lambda ()
 588                                      (should t)
 589                                      (should (null '()))
 590                                      (should nil)
 591                                      (should t)))))
 592     (let ((result (let ((ert-debug-on-error nil))
 593                     (ert-run-test test))))
 594       (should (equal (ert-test-result-should-forms result)
 595                      '(((should t) :form t :value t)
 596                        ((should (null '())) :form (null nil) :value t)
 597                        ((should nil) :form nil :value nil)))))))
 598
 599 (ert-deftest ert-test-list-of-should-forms-observers-should-not-stack ()
 600   (let ((test (make-ert-test
 601                :body (lambda ()
 602                        (let ((test2 (make-ert-test
 603                                      :body (lambda ()
 604                                              (should t)))))
 605                          (let ((result (ert-run-test test2)))
 606                            (should (ert-test-passed-p result))))))))
 607     (let ((result (let ((ert-debug-on-error nil))
 608                     (ert-run-test test))))
 609       (should (ert-test-passed-p result))
 610       (should (eql (length (ert-test-result-should-forms result))
 611                    1)))))
 612
 613 (ert-deftest ert-test-list-of-should-forms-no-deep-copy ()
 614   (let ((test (make-ert-test :body (lambda ()
 615                                      (let ((obj (list 'a)))
 616                                        (should (equal obj '(a)))
 617                                        (setf (car obj) 'b)
 618                                        (should (equal obj '(b))))))))
 619     (let ((result (let ((ert-debug-on-error nil))
 620                     (ert-run-test test))))
 621       (should (ert-test-passed-p result))
 622       (should (equal (ert-test-result-should-forms result)
 623                      '(((should (equal obj '(a))) :form (equal (b) (a)) :value t
 624                         :explanation nil)
 625                        ((should (equal obj '(b))) :form (equal (b) (b)) :value t
 626                         :explanation nil)
 627                        ))))))
 628
 629 (ert-deftest ert-test-remprop ()
 630   (let ((x (ert--gensym)))
 631     (should (equal (symbol-plist x) '()))
 632     ;; Remove nonexistent property on empty plist.
 633     (ert--remprop x 'b)
 634     (should (equal (symbol-plist x) '()))
 635     (put x 'a 1)
 636     (should (equal (symbol-plist x) '(a 1)))
 637     ;; Remove nonexistent property on nonempty plist.
 638     (ert--remprop x 'b)
 639     (should (equal (symbol-plist x) '(a 1)))
 640     (put x 'b 2)
 641     (put x 'c 3)
 642     (put x 'd 4)
 643     (should (equal (symbol-plist x) '(a 1 b 2 c 3 d 4)))
 644     ;; Remove property that is neither first nor last.
 645     (ert--remprop x 'c)
 646     (should (equal (symbol-plist x) '(a 1 b 2 d 4)))
 647     ;; Remove last property from a plist of length >1.
 648     (ert--remprop x 'd)
 649     (should (equal (symbol-plist x) '(a 1 b 2)))
 650     ;; Remove first property from a plist of length >1.
 651     (ert--remprop x 'a)
 652     (should (equal (symbol-plist x) '(b 2)))
 653     ;; Remove property when there is only one.
 654     (ert--remprop x 'b)
 655     (should (equal (symbol-plist x) '()))))
 656
 657 (ert-deftest ert-test-remove-if-not ()
 658   (let ((list (list 'a 'b 'c 'd))
 659         (i 0))
 660     (let ((result (ert--remove-if-not (lambda (x)
 661                                         (should (eql x (nth i list)))
 662                                         (cl-incf i)
 663                                         (member i '(2 3)))
 664                                       list)))
 665       (should (equal i 4))
 666       (should (equal result '(b c)))
 667       (should (equal list '(a b c d)))))
 668   (should (equal '()
 669                  (ert--remove-if-not (lambda (_x) (should nil)) '()))))
 670
 671 (ert-deftest ert-test-remove* ()
 672   (let ((list (list 'a 'b 'c 'd))
 673         (key-index 0)
 674         (test-index 0))
 675     (let ((result
 676            (ert--remove* 'foo list
 677                          :key (lambda (x)
 678                                 (should (eql x (nth key-index list)))
 679                                 (prog1
 680                                     (list key-index x)
 681                                   (cl-incf key-index)))
 682                          :test
 683                          (lambda (a b)
 684                            (should (eql a 'foo))
 685                            (should (equal b (list test-index
 686                                                   (nth test-index list))))
 687                            (cl-incf test-index)
 688                            (member test-index '(2 3))))))
 689       (should (equal key-index 4))
 690       (should (equal test-index 4))
 691       (should (equal result '(a d)))
 692       (should (equal list '(a b c d)))))
 693   (let ((x (cons nil nil))
 694         (y (cons nil nil)))
 695     (should (equal (ert--remove* x (list x y))
 696                    ;; or (list x), since we use `equal' -- the
 697                    ;; important thing is that only one element got
 698                    ;; removed, this proves that the default test is
 699                    ;; `eql', not `equal'
 700                    (list y)))))
 701
 702
 703 (ert-deftest ert-test-set-functions ()
 704   (let ((c1 (cons nil nil))
 705         (c2 (cons nil nil))
 706         (sym (make-symbol "a")))
 707     (let ((e '())
 708           (a (list 'a 'b sym nil "" "x" c1 c2))
 709           (b (list c1 'y 'b sym 'x)))
 710       (should (equal (ert--set-difference e e) e))
 711       (should (equal (ert--set-difference a e) a))
 712       (should (equal (ert--set-difference e a) e))
 713       (should (equal (ert--set-difference a a) e))
 714       (should (equal (ert--set-difference b e) b))
 715       (should (equal (ert--set-difference e b) e))
 716       (should (equal (ert--set-difference b b) e))
 717       (should (equal (ert--set-difference a b) (list 'a nil "" "x" c2)))
 718       (should (equal (ert--set-difference b a) (list 'y 'x)))
 719
 720       ;; We aren't testing whether this is really using `eq' rather than `eql'.
 721       (should (equal (ert--set-difference-eq e e) e))
 722       (should (equal (ert--set-difference-eq a e) a))
 723       (should (equal (ert--set-difference-eq e a) e))
 724       (should (equal (ert--set-difference-eq a a) e))
 725       (should (equal (ert--set-difference-eq b e) b))
 726       (should (equal (ert--set-difference-eq e b) e))
 727       (should (equal (ert--set-difference-eq b b) e))
 728       (should (equal (ert--set-difference-eq a b) (list 'a nil "" "x" c2)))
 729       (should (equal (ert--set-difference-eq b a) (list 'y 'x)))
 730
 731       (should (equal (ert--union e e) e))
 732       (should (equal (ert--union a e) a))
 733       (should (equal (ert--union e a) a))
 734       (should (equal (ert--union a a) a))
 735       (should (equal (ert--union b e) b))
 736       (should (equal (ert--union e b) b))
 737       (should (equal (ert--union b b) b))
 738       (should (equal (ert--union a b) (list 'a 'b sym nil "" "x" c1 c2 'y 'x)))
 739       (should (equal (ert--union b a) (list c1 'y 'b sym 'x 'a nil "" "x" c2)))
 740
 741       (should (equal (ert--intersection e e) e))
 742       (should (equal (ert--intersection a e) e))
 743       (should (equal (ert--intersection e a) e))
 744       (should (equal (ert--intersection a a) a))
 745       (should (equal (ert--intersection b e) e))
 746       (should (equal (ert--intersection e b) e))
 747       (should (equal (ert--intersection b b) b))
 748       (should (equal (ert--intersection a b) (list 'b sym c1)))
 749       (should (equal (ert--intersection b a) (list c1 'b sym))))))
 750
 751 (ert-deftest ert-test-gensym ()
 752   ;; Since the expansion of `should' calls `ert--gensym' and thus has a
 753   ;; side-effect on `ert--gensym-counter', we have to make sure all
 754   ;; macros in our test body are expanded before we rebind
 755   ;; `ert--gensym-counter' and run the body.  Otherwise, the test would
 756   ;; fail if run interpreted.
 757   (let ((body (byte-compile
 758                '(lambda ()
 759                   (should (equal (symbol-name (ert--gensym)) "G0"))
 760                   (should (equal (symbol-name (ert--gensym)) "G1"))
 761                   (should (equal (symbol-name (ert--gensym)) "G2"))
 762                   (should (equal (symbol-name (ert--gensym "foo")) "foo3"))
 763                   (should (equal (symbol-name (ert--gensym "bar")) "bar4"))
 764                   (should (equal ert--gensym-counter 5))))))
 765     (let ((ert--gensym-counter 0))
 766       (funcall body))))
 767
 768 (ert-deftest ert-test-coerce-to-vector ()
 769   (let* ((a (vector))
 770          (b (vector 1 a 3))
 771          (c (list))
 772          (d (list b a)))
 773     (should (eql (ert--coerce-to-vector a) a))
 774     (should (eql (ert--coerce-to-vector b) b))
 775     (should (equal (ert--coerce-to-vector c) (vector)))
 776     (should (equal (ert--coerce-to-vector d) (vector b a)))))
 777
 778 (ert-deftest ert-test-string-position ()
 779   (should (eql (ert--string-position ?x "") nil))
 780   (should (eql (ert--string-position ?a "abc") 0))
 781   (should (eql (ert--string-position ?b "abc") 1))
 782   (should (eql (ert--string-position ?c "abc") 2))
 783   (should (eql (ert--string-position ?d "abc") nil))
 784   (should (eql (ert--string-position ?A "abc") nil)))
 785
 786 (ert-deftest ert-test-mismatch ()
 787   (should (eql (ert--mismatch "" "") nil))
 788   (should (eql (ert--mismatch "" "a") 0))
 789   (should (eql (ert--mismatch "a" "a") nil))
 790   (should (eql (ert--mismatch "ab" "a") 1))
 791   (should (eql (ert--mismatch "Aa" "aA") 0))
 792   (should (eql (ert--mismatch '(a b c) '(a b d)) 2)))
 793
 794 (ert-deftest ert-test-string-first-line ()
 795   (should (equal (ert--string-first-line "") ""))
 796   (should (equal (ert--string-first-line "abc") "abc"))
 797   (should (equal (ert--string-first-line "abc\n") "abc"))
 798   (should (equal (ert--string-first-line "foo\nbar") "foo"))
 799   (should (equal (ert--string-first-line " foo\nbar\nbaz\n") " foo")))
 800
 801 (ert-deftest ert-test-explain-equal ()
 802   (should (equal (ert--explain-equal nil 'foo)
 803                  '(different-atoms nil foo)))
 804   (should (equal (ert--explain-equal '(a a) '(a b))
 805                  '(list-elt 1 (different-atoms a b))))
 806   (should (equal (ert--explain-equal '(1 48) '(1 49))
 807                  '(list-elt 1 (different-atoms (48 "#x30" "?0")
 808                                                (49 "#x31" "?1")))))
 809   (should (equal (ert--explain-equal 'nil '(a))
 810                  '(different-types nil (a))))
 811   (should (equal (ert--explain-equal '(a b c) '(a b c d))
 812                  '(proper-lists-of-different-length 3 4 (a b c) (a b c d)
 813                                                     first-mismatch-at 3)))
 814   (let ((sym (make-symbol "a")))
 815     (should (equal (ert--explain-equal 'a sym)
 816                    `(different-symbols-with-the-same-name a ,sym)))))
 817
 818 (ert-deftest ert-test-explain-equal-improper-list ()
 819   (should (equal (ert--explain-equal '(a . b) '(a . c))
 820                  '(cdr (different-atoms b c)))))
 821
 822 (ert-deftest ert-test-explain-equal-keymaps ()
 823   ;; This used to be very slow.
 824   (should (equal (make-keymap) (make-keymap)))
 825   (should (equal (make-sparse-keymap) (make-sparse-keymap))))
 826
 827 (ert-deftest ert-test-significant-plist-keys ()
 828   (should (equal (ert--significant-plist-keys '()) '()))
 829   (should (equal (ert--significant-plist-keys '(a b c d e f c g p q r nil s t))
 830                  '(a c e p s))))
 831
 832 (ert-deftest ert-test-plist-difference-explanation ()
 833   (should (equal (ert--plist-difference-explanation
 834                   '(a b c nil) '(a b))
 835                  nil))
 836   (should (equal (ert--plist-difference-explanation
 837                   '(a b c t) '(a b))
 838                  '(different-properties-for-key c (different-atoms t nil))))
 839   (should (equal (ert--plist-difference-explanation
 840                   '(a b c t) '(c nil a b))
 841                  '(different-properties-for-key c (different-atoms t nil))))
 842   (should (equal (ert--plist-difference-explanation
 843                   '(a b c (foo . bar)) '(c (foo . baz) a b))
 844                  '(different-properties-for-key c
 845                                                 (cdr
 846                                                  (different-atoms bar baz))))))
 847
 848 (ert-deftest ert-test-abbreviate-string ()
 849   (should (equal (ert--abbreviate-string "foo" 4 nil) "foo"))
 850   (should (equal (ert--abbreviate-string "foo" 3 nil) "foo"))
 851   (should (equal (ert--abbreviate-string "foo" 3 nil) "foo"))
 852   (should (equal (ert--abbreviate-string "foo" 2 nil) "fo"))
 853   (should (equal (ert--abbreviate-string "foo" 1 nil) "f"))
 854   (should (equal (ert--abbreviate-string "foo" 0 nil) ""))
 855   (should (equal (ert--abbreviate-string "bar" 4 t) "bar"))
 856   (should (equal (ert--abbreviate-string "bar" 3 t) "bar"))
 857   (should (equal (ert--abbreviate-string "bar" 3 t) "bar"))
 858   (should (equal (ert--abbreviate-string "bar" 2 t) "ar"))
 859   (should (equal (ert--abbreviate-string "bar" 1 t) "r"))
 860   (should (equal (ert--abbreviate-string "bar" 0 t) "")))
 861
 862 (ert-deftest ert-test-explain-equal-string-properties ()
 863   (should
 864    (equal (ert--explain-equal-including-properties #("foo" 0 1 (a b))
 865                                                    "foo")
 866           '(char 0 "f"
 867                  (different-properties-for-key a (different-atoms b nil))
 868                  context-before ""
 869                  context-after "oo")))
 870   (should (equal (ert--explain-equal-including-properties
 871                   #("foo" 1 3 (a b))
 872                   #("goo" 0 1 (c d)))
 873                  '(array-elt 0 (different-atoms (?f "#x66" "?f")
 874                                                 (?g "#x67" "?g")))))
 875   (should
 876    (equal (ert--explain-equal-including-properties
 877            #("foo" 0 1 (a b c d) 1 3 (a b))
 878            #("foo" 0 1 (c d a b) 1 2 (a foo)))
 879           '(char 1 "o" (different-properties-for-key a (different-atoms b foo))
 880                  context-before "f" context-after "o"))))
 881
 882 (ert-deftest ert-test-equal-including-properties ()
 883   (should (equal-including-properties "foo" "foo"))
 884   (should (ert-equal-including-properties "foo" "foo"))
 885
 886   (should (equal-including-properties #("foo" 0 3 (a b))
 887                                       (propertize "foo" 'a 'b)))
 888   (should (ert-equal-including-properties #("foo" 0 3 (a b))
 889                                           (propertize "foo" 'a 'b)))
 890
 891   (should (equal-including-properties #("foo" 0 3 (a b c d))
 892                                       (propertize "foo" 'a 'b 'c 'd)))
 893   (should (ert-equal-including-properties #("foo" 0 3 (a b c d))
 894                                           (propertize "foo" 'a 'b 'c 'd)))
 895
 896   (should-not (equal-including-properties #("foo" 0 3 (a b c e))
 897                                           (propertize "foo" 'a 'b 'c 'd)))
 898   (should-not (ert-equal-including-properties #("foo" 0 3 (a b c e))
 899                                               (propertize "foo" 'a 'b 'c 'd)))
 900
 901   ;; This is bug 6581.
 902   (should-not (equal-including-properties #("foo" 0 3 (a (t)))
 903                                           (propertize "foo" 'a (list t))))
 904   (should (ert-equal-including-properties #("foo" 0 3 (a (t)))
 905                                           (propertize "foo" 'a (list t)))))
 906
 907 (ert-deftest ert-test-stats-set-test-and-result ()
 908   (let* ((test-1 (make-ert-test :name 'test-1
 909                                 :body (lambda () nil)))
 910          (test-2 (make-ert-test :name 'test-2
 911                                 :body (lambda () nil)))
 912          (test-3 (make-ert-test :name 'test-2
 913                                 :body (lambda () nil)))
 914          (stats (ert--make-stats (list test-1 test-2) 't))
 915          (failed (make-ert-test-failed :condition nil
 916                                        :backtrace nil
 917                                        :infos nil)))
 918     (should (eql 2 (ert-stats-total stats)))
 919     (should (eql 0 (ert-stats-completed stats)))
 920     (should (eql 0 (ert-stats-completed-expected stats)))
 921     (should (eql 0 (ert-stats-completed-unexpected stats)))
 922     (ert--stats-set-test-and-result stats 0 test-1 (make-ert-test-passed))
 923     (should (eql 2 (ert-stats-total stats)))
 924     (should (eql 1 (ert-stats-completed stats)))
 925     (should (eql 1 (ert-stats-completed-expected stats)))
 926     (should (eql 0 (ert-stats-completed-unexpected stats)))
 927     (ert--stats-set-test-and-result stats 0 test-1 failed)
 928     (should (eql 2 (ert-stats-total stats)))
 929     (should (eql 1 (ert-stats-completed stats)))
 930     (should (eql 0 (ert-stats-completed-expected stats)))
 931     (should (eql 1 (ert-stats-completed-unexpected stats)))
 932     (ert--stats-set-test-and-result stats 0 test-1 nil)
 933     (should (eql 2 (ert-stats-total stats)))
 934     (should (eql 0 (ert-stats-completed stats)))
 935     (should (eql 0 (ert-stats-completed-expected stats)))
 936     (should (eql 0 (ert-stats-completed-unexpected stats)))
 937     (ert--stats-set-test-and-result stats 0 test-3 failed)
 938     (should (eql 2 (ert-stats-total stats)))
 939     (should (eql 1 (ert-stats-completed stats)))
 940     (should (eql 0 (ert-stats-completed-expected stats)))
 941     (should (eql 1 (ert-stats-completed-unexpected stats)))
 942     (ert--stats-set-test-and-result stats 1 test-2 (make-ert-test-passed))
 943     (should (eql 2 (ert-stats-total stats)))
 944     (should (eql 2 (ert-stats-completed stats)))
 945     (should (eql 1 (ert-stats-completed-expected stats)))
 946     (should (eql 1 (ert-stats-completed-unexpected stats)))
 947     (ert--stats-set-test-and-result stats 0 test-1 (make-ert-test-passed))
 948     (should (eql 2 (ert-stats-total stats)))
 949     (should (eql 2 (ert-stats-completed stats)))
 950     (should (eql 2 (ert-stats-completed-expected stats)))
 951     (should (eql 0 (ert-stats-completed-unexpected stats)))))
 952
 953
 954 (provide 'ert-tests)
 955
 956 ;;; ert-tests.el ends here