1 ;;;-*- Mode: Lisp; Syntax: ANSI-Common-Lisp -*-
4 Copyright
(c) 2004-
2005 Christopher K. Riesbeck
6 Permission is hereby granted
, free of charge
, to any person obtaining
7 a copy of this software and associated documentation files
(the "Software"),
8 to deal in the Software without restriction
, including without limitation
9 the rights to use
, copy
, modify
, merge
, publish
, distribute
, sublicense
,
10 and
/or sell copies of the Software
, and to permit persons to whom the
11 Software is furnished to do so
, subject to the following conditions
:
13 The above copyright notice and this permission notice shall be included
14 in all copies or substantial portions of the Software.
16 THE SOFTWARE IS PROVIDED
"AS IS", WITHOUT WARRANTY OF ANY KIND
, EXPRESS
17 OR IMPLIED
, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY
,
18 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM
, DAMAGES OR
20 OTHER LIABILITY
, WHETHER IN AN ACTION OF CONTRACT
, TORT OR OTHERWISE
,
21 ARISING FROM
, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 OTHER DEALINGS IN THE SOFTWARE.
28 1. Read the documentation at
:
29 https
://github.com
/OdonataResearchLLC
/lisp-unit
/wiki
31 2. Make a file of DEFINE-TEST
's. See exercise-tests.lisp for many
32 examples. If you want
, start your test file with
(REMOVE-TESTS :ALL
)
33 to clear any previously defined tests.
37 4.
(use-package :lisp-unit
)
39 5. Load your code file and your file of tests.
41 6. Test your code with
(RUN-TESTS '(test-name1 test-name2 ...
)) or
42 simply
(RUN-TESTS :ALL
) to run all defined tests.
44 A summary of how many tests passed and failed will be printed.
46 NOTE
: Nothing is compiled until RUN-TESTS is expanded. Redefining
47 functions or even macros does not require reloading any tests.
55 (defpackage :lisp-unit
58 (:export
:*print-summary
*
61 ;; Forms for assertions
72 ;; Functions for managing tests
80 ;; Functions for managing tags
85 ;; Functions for reporting test results
93 ;; Functions for extensibility via signals
94 (:export
:signal-results
98 (:export
:logically-equal
:set-equal
))
100 (in-package :lisp-unit
)
104 (defparameter *pass
* 0
105 "The passed assertion results.")
107 (defparameter *fail
* ()
108 "The failed assertion results.")
110 (defun reset-counters ()
111 "Reset the counters to empty lists."
112 (setf *pass
* 0 *fail
* ()))
116 (defparameter *print-summary
* nil
117 "Print a summary of the pass, fail, and error count if non-nil.")
119 (defparameter *print-failures
* nil
120 "Print failure messages if non-NIL.")
122 (defparameter *print-errors
* nil
123 "Print error messages if non-NIL.")
125 (defparameter *use-debugger
* nil
126 "If not NIL, enter the debugger when an error is encountered in an
129 (defparameter *signal-results
* nil
130 "Signal the result if non NIL.")
132 (defun use-debugger-p (condition)
133 "Debug or ignore errors."
135 ((eq :ask
*use-debugger
*)
136 (y-or-n-p "~A -- debug?" condition
))
139 (defun use-debugger (&optional
(flag t
))
140 "Use the debugger when testing, or not."
141 (setq *use-debugger
* flag
))
143 (defun signal-results (&optional
(flag t
))
144 "Signal the results for extensibility."
145 (setq *signal-results
* flag
))
147 (define-condition test-run-complete
()
152 "Signaled when a test run is finished."))
154 ;;; Global unit test database
156 (defparameter *test-db
* (make-hash-table :test
#'eq
)
157 "The unit test database is simply a hash table.")
159 (defun package-table (package &optional create
)
161 ((gethash (find-package package
) *test-db
*))
163 (setf (gethash package
*test-db
*) (make-hash-table)))
164 (t (warn "No tests defined for package: ~S" package
))))
166 ;;; Global tags database
168 (defparameter *tag-db
* (make-hash-table :test
#'eq
)
169 "The tag database is simply a hash table.")
171 (defun package-tags (package &optional create
)
172 "Return the tags DB for the package."
174 ((gethash (find-package package
) *tag-db
*))
176 (setf (gethash package
*tag-db
*) (make-hash-table)))
177 (t (warn "No tags defined for package: ~S" package
))))
179 ;;; Unit test definition
181 (defclass unit-test
()
190 (:default-initargs
:doc
"" :code
())
192 "Organize the unit test documentation and code."))
194 ;;; NOTE: Shamelessly taken from PG's analyze-body
195 (defun parse-body (body &optional doc tag
)
196 "Separate the components of the body."
197 (let ((item (first body
)))
199 ((and (listp item
) (eq :tag
(first item
)))
200 (parse-body (rest body
) doc
(nconc (rest item
) tag
)))
201 ((and (stringp item
) (not doc
) (rest body
))
203 (values doc tag
(rest body
))
204 (parse-body (rest body
) doc tag
)))
205 (t (values doc tag body
)))))
207 (defmacro define-test
(name &body body
)
208 "Store the test in the test database."
209 (let ((qname (gensym "NAME-")))
210 (multiple-value-bind (doc tag code
) (parse-body body
)
211 `(let* ((,qname
',name
)
212 (doc (or ,doc
(string ,qname
))))
215 (gethash ,qname
(package-table *package
* t
))
216 (make-instance 'unit-test
:doc doc
:code
',code
))
218 (loop for tag in
',tag do
220 ,qname
(gethash tag
(package-tags *package
* t
))))
221 ;; Return the name of the test
226 (defun list-tests (&optional
(package *package
*))
227 "Return a list of the tests in package."
228 (let ((table (package-table package
)))
230 (loop for test-name being each hash-key in table
231 collect test-name
))))
233 (defun test-documentation (name &optional
(package *package
*))
234 "Return the documentation for the test."
235 (let ((unit-test (gethash name
(package-table package
))))
237 (warn "No code defined for test ~A in package ~S."
241 (defun test-code (name &optional
(package *package
*))
242 "Returns the code stored for the test name."
243 (let ((unit-test (gethash name
(package-table package
))))
245 (warn "No code defined for test ~A in package ~S."
249 (defun remove-tests (names &optional
(package *package
*))
250 "Remove individual tests or entire sets."
255 (remhash (find-package package
) *test-db
*)
256 (remhash (find-package package
) *tag-db
*)))
257 (let ((table (package-table package
)))
260 (loop for name in names
261 always
(remhash name table
)
262 collect name into removed
263 finally
(return removed
))
264 ;; Remove tests from tags
265 (loop with tags
= (package-tags package
)
266 for tag being each hash-key in tags
267 using
(hash-value tagged-tests
)
271 (set-difference tagged-tests names
)))))))
275 (defun %tests-from-all-tags
(&optional
(package *package
*))
276 "Return all of the tests that have been tagged."
277 (loop for tests being each hash-value in
(package-tags package
)
278 nconc
(copy-list tests
) into all-tests
279 finally
(return (delete-duplicates all-tests
))))
281 (defun %tests-from-tags
(tags &optional
(package *package
*))
282 "Return the tests associated with the tags."
283 (loop with table
= (package-tags package
)
285 as tests
= (gethash tag table
)
286 if
(null tests
) do
(warn "No tests tagged with ~S." tag
)
287 else nconc
(copy-list tests
) into all-tests
288 finally
(return (delete-duplicates all-tests
))))
290 (defun list-tags (&optional
(package *package
*))
291 "Return a list of the tags in package."
292 (let ((tags (package-tags package
)))
294 (loop for tag being each hash-key in tags collect tag
))))
296 (defun tagged-tests (tags &optional
(package *package
*))
297 "Return a list of the tests associated with the tags."
299 (%tests-from-all-tags package
)
300 (%tests-from-tags tags package
)))
302 (defun remove-tags (tags &optional
(package *package
*))
303 "Remove individual tags or entire sets."
307 (remhash (find-package package
) *tag-db
*))
308 (let ((table (package-tags package
)))
310 (loop for tag in tags
311 always
(remhash tag table
)
312 collect tag into removed
313 finally
(return removed
))))))
317 (defmacro assert-eq
(expected form
&rest extras
)
318 "Assert whether expected and form are EQ."
319 `(expand-assert :equal
,form
,form
,expected
,extras
:test
#'eq
))
321 (defmacro assert-eql
(expected form
&rest extras
)
322 "Assert whether expected and form are EQL."
323 `(expand-assert :equal
,form
,form
,expected
,extras
:test
#'eql
))
325 (defmacro assert-equal
(expected form
&rest extras
)
326 "Assert whether expected and form are EQUAL."
327 `(expand-assert :equal
,form
,form
,expected
,extras
:test
#'equal
))
329 (defmacro assert-equalp
(expected form
&rest extras
)
330 "Assert whether expected and form are EQUALP."
331 `(expand-assert :equal
,form
,form
,expected
,extras
:test
#'equalp
))
333 (defmacro assert-error
(condition form
&rest extras
)
334 "Assert whether form signals condition."
335 `(expand-assert :error
,form
(expand-error-form ,form
)
338 (defmacro assert-expands
(expansion form
&rest extras
)
339 "Assert whether form expands to expansion."
340 `(expand-assert :macro
,form
341 (expand-macro-form ,form nil
)
344 (defmacro assert-false
(form &rest extras
)
345 "Assert whether the form is false."
346 `(expand-assert :result
,form
,form nil
,extras
))
348 (defmacro assert-equality
(test expected form
&rest extras
)
349 "Assert whether expected and form are equal according to test."
350 `(expand-assert :equal
,form
,form
,expected
,extras
:test
,test
))
352 (defmacro assert-prints
(output form
&rest extras
)
353 "Assert whether printing the form generates the output."
354 `(expand-assert :output
,form
(expand-output-form ,form
)
357 (defmacro assert-true
(form &rest extras
)
358 "Assert whether the form is true."
359 `(expand-assert :result
,form
,form t
,extras
))
361 (defmacro expand-assert
(type form body expected extras
&key
(test '#'eql
))
362 "Expand the assertion to the internal format."
363 `(internal-assert ,type
',form
365 (lambda () ,expected
)
366 (expand-extras ,extras
)
369 (defmacro expand-error-form
(form)
370 "Wrap the error assertion in HANDLER-CASE."
372 (condition (error) error
)))
374 (defmacro expand-output-form
(form)
375 "Capture the output of the form in a string."
376 (let ((out (gensym)))
377 `(let* ((,out
(make-string-output-stream))
379 (make-broadcast-stream *standard-output
* ,out
)))
381 (get-output-stream-string ,out
))))
383 (defmacro expand-macro-form
(form env
)
384 "Expand the macro form once."
385 `(macroexpand-1 ',form
,env
))
387 (defmacro expand-extras
(extras)
388 "Expand extra forms."
390 (list ,@(mapcan (lambda (form) (list `',form form
)) extras
))))
392 (defclass failure-result
()
413 "Failure details of the assertion."))
415 (defclass equal-result
(failure-result)
418 "Result of a failed equal assertion."))
420 (defun equal-result (test expected actual
)
421 "Return the result of an equal assertion."
423 (<= (length expected
) (length actual
))
424 (every test expected actual
)))
426 (defclass error-result
(failure-result)
429 "Result of a failed error assertion."))
431 (defun error-result (test expected actual
)
432 "Return the result of an error assertion."
433 (declare (ignore test
))
435 (eql (car actual
) (car expected
))
436 (typep (car actual
) (car expected
))))
438 (defclass macro-result
(failure-result)
441 "Result of a failed macro expansion assertion."))
443 ;;; FIXME: Review the internal tests for macros.
444 (defun macro-result (test expected actual
)
445 "Return the result of a macro assertion."
446 (declare (ignore test
))
447 (equal (car actual
) (car expected
)))
449 (defclass boolean-result
(failure-result)
452 "Result of a failed boolean assertion."))
454 (defun boolean-result (test expected actual
)
455 "Return the result of a result assertion."
456 (declare (ignore test
))
457 (logically-equal (car actual
) (car expected
)))
459 (defclass output-result
(failure-result)
462 "Result of a failed output assertion."))
464 (defun output-result (test expected actual
)
465 "Return the result of an output assertion."
466 (declare (ignore test
))
468 (string-trim '(#\newline
#\return
#\space
) (car actual
))
471 (defun assert-function (type)
472 "Return the function for the assertion type."
474 (:equal
#'equal-result
)
475 (:error
#'error-result
)
476 (:macro
#'macro-result
)
477 (:result
#'boolean-result
)
478 (:output
#'output-result
)))
480 (defun assert-class (type)
481 "Return the class for the assertion type."
483 (:equal
'equal-result
)
484 (:error
'error-result
)
485 (:macro
'macro-result
)
486 (:result
'boolean-result
)
487 (:output
'output-result
)))
489 (defun internal-assert
490 (type form code-thunk expected-thunk extras test
)
491 "Perform the assertion and record the results."
492 (let* ((actual (multiple-value-list (funcall code-thunk
)))
493 (expected (multiple-value-list (funcall expected-thunk
)))
495 (funcall (assert-function type
) test expected actual
)))
503 :extras
(when extras
(funcall extras
))
509 ;;; Unit test results
511 (defclass test-result
()
527 (:default-initargs
:exerr nil
)
529 "Store the results of the unit test."))
531 (defun print-summary (test-result)
532 "Print a summary of the test result."
533 (format t
"~&~A: ~S assertions passed, ~S failed"
536 (length (fail test-result
)))
537 (if (exerr test-result
)
538 (format t
", and an execution error.")
543 (defun run-code (code)
544 "Run the code to test the assertions."
545 (funcall (coerce `(lambda () ,@code
) 'function
)))
547 (defun run-test-thunk (name code
)
553 (if (use-debugger-p condition
)
555 (return-from run-test-thunk
561 :exerr condition
))))))
563 ;; Return the result count
570 ;;; Test results database
572 (defclass test-results-db
()
575 :initform
(make-hash-table :test
#'eq
)
592 :accessor failed-tests
)
596 :accessor error-tests
)
600 :accessor missing-tests
))
602 "Store the results of the tests for further evaluation."))
604 (defmethod print-object ((object test-results-db
) stream
)
605 "Print the summary counts with the object."
606 (let ((pass (pass object
))
608 (exerr (exerr object
)))
610 stream
"#<~A Total(~D) Passed(~D) Failed(~D) Errors(~D)>~%"
611 (class-name (class-of object
))
612 (+ pass fail
) pass fail exerr
)))
614 (defun test-names (test-results-db)
615 "Return a list of the test names in the database."
616 (loop for name being each hash-key in
(database test-results-db
)
619 (defun record-result (test-name code results
)
620 "Run the test code and record the result."
621 (let ((result (run-test-thunk test-name code
)))
623 (setf (gethash test-name
(database results
)) result
)
624 ;; Count passed tests
625 (when (plusp (pass result
))
626 (incf (pass results
) (pass result
)))
627 ;; Count failed tests and record the name
629 (incf (fail results
) (length (fail result
)))
630 (push test-name
(failed-tests results
)))
631 ;; Count errors and record the name
633 (incf (exerr results
))
634 (push test-name
(error-tests results
)))
636 (when *print-failures
* (print-failures result
))
637 (when *print-errors
* (print-errors result
))
638 (when (or *print-summary
* *print-failures
* *print-errors
*)
639 (print-summary result
))))
641 (defun summarize-results (results)
642 "Print a summary of all results."
643 (let ((pass (pass results
))
644 (fail (fail results
)))
645 (format t
"~&Unit Test Summary~%")
646 (format t
" | ~D assertions total~%" (+ pass fail
))
647 (format t
" | ~D passed~%" pass
)
648 (format t
" | ~D failed~%" fail
)
649 (format t
" | ~D execution errors~%" (exerr results
))
650 (format t
" | ~D missing tests~2%"
651 (length (missing-tests results
)))))
655 (defun %run-all-thunks
(&optional
(package *package
*))
656 "Run all of the test thunks in the package."
658 with results
= (make-instance 'test-results-db
)
659 for test-name being each hash-key in
(package-table package
)
660 using
(hash-value unit-test
)
662 (record-result test-name
(code unit-test
) results
)
664 (push test-name
(missing-tests results
))
665 ;; Summarize and return the test results
667 (when *signal-results
*
668 (signal 'test-run-complete
:results results
))
669 (summarize-results results
)
672 (defun %run-thunks
(test-names &optional
(package *package
*))
673 "Run the list of test thunks in the package."
675 with table
= (package-table package
)
676 and results
= (make-instance 'test-results-db
)
677 for test-name in test-names
678 as unit-test
= (gethash test-name table
)
680 (record-result test-name
(code unit-test
) results
)
682 (push test-name
(missing-tests results
))
684 (when *signal-results
*
685 (signal 'test-run-complete
:results results
))
686 (summarize-results results
)
689 (defun run-tests (test-names &optional
(package *package
*))
690 "Run the specified tests in package."
692 (if (eq :all test-names
)
693 (%run-all-thunks package
)
694 (%run-thunks test-names package
)))
696 (defun run-tags (tags &optional
(package *package
*))
697 "Run the tests associated with the specified tags in package."
699 (%run-thunks
(tagged-tests tags package
) package
))
703 (defgeneric print-failures
(result)
705 "Report the results of the failed assertion."))
707 (defmethod print-failures :around
((result failure-result
))
708 "Failure header and footer output."
709 (format t
"~& | Failed Form: ~S" (form result
))
711 (when (extras result
)
712 (format t
"~{~& | ~S => ~S~}~%" (extras result
)))
714 (class-name (class-of result
)))
716 (defmethod print-failures ((result failure-result
))
717 (format t
"~& | Expected ~{~S~^; ~} " (expected result
))
718 (format t
"~<~% | ~:;but saw ~{~S~^; ~}~>" (actual result
)))
720 (defmethod print-failures ((result error-result
))
721 (format t
"~& | ~@[Should have signalled ~{~S~^; ~} but saw~]"
723 (format t
" ~{~S~^; ~}" (actual result
)))
725 (defmethod print-failures ((result macro-result
))
726 (format t
"~& | Should have expanded to ~{~S~^; ~} "
728 (format t
"~<~%~:;but saw ~{~S~^; ~}~>" (actual result
)))
730 (defmethod print-failures ((result output-result
))
731 (format t
"~& | Should have printed ~{~S~^; ~} "
733 (format t
"~<~%~:;but saw ~{~S~^; ~}~>"
736 (defmethod print-failures ((result test-result
))
737 "Print the failed assertions in the unit test."
738 (loop for fail in
(fail result
) do
739 (print-failures fail
)))
741 (defmethod print-failures ((results test-results-db
))
742 "Print all of the failure tests."
743 (loop with db
= (database results
)
744 for test in
(failed-tests results
)
745 as result
= (gethash test db
)
747 (print-failures result
)
748 (print-summary result
)))
752 (defgeneric print-errors
(result)
754 "Print the error condition."))
756 (defmethod print-errors ((result test-result
))
757 "Print the error condition."
758 (let ((exerr (exerr result
))
759 (*print-escape
* nil
))
761 (format t
"~& | Execution error:~% | ~W" exerr
)
762 (format t
"~& |~%"))))
764 (defmethod print-errors ((results test-results-db
))
765 "Print all of the error tests."
766 (loop with db
= (database results
)
767 for test in
(error-tests results
)
768 as result
= (gethash test db
)
770 (print-errors result
)
771 (print-summary result
)))
773 ;;; Useful equality predicates for tests
775 (defun logically-equal (x y
)
776 "Return true if x and y are both false or both true."
777 (eql (not x
) (not y
)))
779 (defun set-equal (list1 list2
&rest initargs
&key key
(test #'equal
))
780 "Return true if every element of list1 is an element of list2 and
782 (declare (ignore key test
))
786 (apply #'subsetp list1 list2 initargs
)
787 (apply #'subsetp list2 list1 initargs
)))
789 (pushnew :lisp-unit common-lisp
:*features
*)