tests/test-util.lisp

   1 (defpackage :test-util
   2   (:use :cl :sb-ext)
   3   (:export #:with-test #:report-test-status #:*failures*
   4            #:really-invoke-debugger
   5            #:*break-on-failure* #:*break-on-expected-failure*
   6            #:make-kill-thread #:make-join-thread
   7            #:checked-compile
   8            #:runtime))
   9
  10 (in-package :test-util)
  11
  12 (defvar *test-count* 0)
  13 (defvar *test-file* nil)
  14 (defvar *failures* nil)
  15 (defvar *break-on-failure* nil)
  16 (defvar *break-on-expected-failure* nil)
  17
  18 (defvar *threads-to-kill*)
  19 (defvar *threads-to-join*)
  20
  21 (eval-when (:compile-toplevel :load-toplevel :execute)
  22   (require :sb-posix))
  23
  24 (sb-posix:putenv (format nil "SBCL_MACHINE_TYPE=~A" (machine-type)))
  25 (sb-posix:putenv (format nil "SBCL_SOFTWARE_TYPE=~A" (software-type)))
  26
  27 #+sb-thread
  28 (defun make-kill-thread (&rest args)
  29   (let ((thread (apply #'sb-thread:make-thread args)))
  30     (when (boundp '*threads-to-kill*)
  31       (push thread *threads-to-kill*))
  32     thread))
  33
  34 #+sb-thread
  35 (defun make-join-thread (&rest args)
  36   (let ((thread (apply #'sb-thread:make-thread args)))
  37     (when (boundp '*threads-to-join*)
  38       (push thread *threads-to-join*))
  39     thread))
  40
  41 (defun log-msg (&rest args)
  42   (apply #'format *trace-output* "~&::: ~@?~%" args)
  43   (force-output *trace-output*))
  44
  45 (defun log-msg/non-pretty (&rest args)
  46   (let ((*print-pretty* nil))
  47     (apply #'log-msg args)))
  48
  49 (defmacro with-test ((&key fails-on broken-on skipped-on name)
  50                      &body body)
  51   (let ((block-name (gensym))
  52         #+sb-thread (threads (gensym "THREADS")))
  53     (flet ((name-ok (x y)
  54              (declare (ignore y))
  55              (typecase x
  56                (symbol (let ((package (symbol-package x)))
  57                          (or (null package)
  58                              (eql package (find-package "CL"))
  59                              (eql package (find-package "KEYWORD"))
  60                              (eql (mismatch "SB-" (package-name package)) 3))))
  61                (integer t))))
  62       (unless (tree-equal name name :test #'name-ok)
  63         (error "test name must be all-keywords: ~S" name)))
  64     `(progn
  65        (start-test)
  66        (cond
  67          ((broken-p ,broken-on)
  68           (fail-test :skipped-broken ',name "Test broken on this platform"))
  69          ((skipped-p ,skipped-on)
  70           (fail-test :skipped-disabled ',name "Test disabled for this combination of platform and features"))
  71          (t
  72           (let (#+sb-thread (,threads (sb-thread:list-all-threads))
  73                 (*threads-to-join* nil)
  74                 (*threads-to-kill* nil))
  75             (block ,block-name
  76               (handler-bind ((error (lambda (error)
  77                                       (if (expected-failure-p ,fails-on)
  78                                           (fail-test :expected-failure ',name error)
  79                                           (fail-test :unexpected-failure ',name error))
  80                                       (return-from ,block-name))))
  81                 (progn
  82                   ;; Non-pretty is for cases like (with-test (:name (let ...)) ...
  83                   (log-msg/non-pretty "Running ~S" ',name)
  84                   ,@body
  85                   #+sb-thread
  86                   (let ((any-leftover nil))
  87                     (dolist (thread *threads-to-join*)
  88                       (ignore-errors (sb-thread:join-thread thread)))
  89                     (dolist (thread *threads-to-kill*)
  90                       (ignore-errors (sb-thread:terminate-thread thread)))
  91                     (setf ,threads (union (union *threads-to-kill*
  92                                                  *threads-to-join*)
  93                                           ,threads))
  94                     #+(and sb-safepoint-strictly (not win32))
  95                     (dolist (thread (sb-thread:list-all-threads))
  96                       (when (typep thread 'sb-thread:signal-handling-thread)
  97                         (ignore-errors (sb-thread:join-thread thread))))
  98                     (dolist (thread (sb-thread:list-all-threads))
  99                       (unless (or (not (sb-thread:thread-alive-p thread))
 100                                   (eql (the sb-thread:thread thread)
 101                                        sb-thread:*current-thread*)
 102                                   (member thread ,threads)
 103                                   (sb-thread:thread-ephemeral-p thread))
 104                         (setf any-leftover thread)
 105                         (ignore-errors (sb-thread:terminate-thread thread))))
 106                     (when any-leftover
 107                       (fail-test :leftover-thread ',name any-leftover)
 108                       (return-from ,block-name)))
 109                   (if (expected-failure-p ,fails-on)
 110                       (fail-test :unexpected-success ',name nil)
 111                       ;; Non-pretty is for cases like (with-test (:name (let ...)) ...
 112                       (log-msg/non-pretty "Success ~S" ',name)))))))))))
 113
 114 (defun report-test-status ()
 115   (with-standard-io-syntax
 116       (with-open-file (stream "test-status.lisp-expr"
 117                               :direction :output
 118                               :if-exists :supersede)
 119         (format stream "~s~%" *failures*))))
 120
 121 (defun start-test ()
 122   (unless (eq *test-file* *load-pathname*)
 123     (setf *test-file* *load-pathname*)
 124     (setf *test-count* 0))
 125   (incf *test-count*))
 126
 127 (defun really-invoke-debugger (condition)
 128   (with-simple-restart (continue "Continue")
 129     (let ((*invoke-debugger-hook* *invoke-debugger-hook*))
 130       (enable-debugger)
 131       (invoke-debugger condition))))
 132
 133 (defun fail-test (type test-name condition)
 134   (if (stringp condition)
 135       (log-msg "~@<~A ~S ~:_~A~:>"
 136                type test-name condition)
 137       (log-msg "~@<~A ~S ~:_due to ~S: ~4I~:_\"~A\"~:>"
 138                type test-name condition condition))
 139   (push (list type *test-file* (or test-name *test-count*))
 140         *failures*)
 141   (unless (stringp condition)
 142     (when (or (and *break-on-failure*
 143                    (not (eq type :expected-failure)))
 144               *break-on-expected-failure*)
 145       (really-invoke-debugger condition))))
 146
 147 (defun expected-failure-p (fails-on)
 148   (sb-impl::featurep fails-on))
 149
 150 (defun broken-p (broken-on)
 151   (sb-impl::featurep broken-on))
 152
 153 (defun skipped-p (skipped-on)
 154   (sb-impl::featurep skipped-on))
 155
 156 ;;; Compile FORM capturing and muffling all [style-]warnings and notes
 157 ;;; and return five values: 1) the compiled function 2) a Boolean
 158 ;;; indicating whether compilation failed 3) a list of warnings 4) a
 159 ;;; list of style-warnigns 5) a list of notes.
 160 ;;;
 161 ;;; An error can be signaled when COMPILE indicates failure as well as
 162 ;;; in case [style-]warning or note conditions are signaled. The
 163 ;;; keyword parameters ALLOW-{FAILURE,[STYLE-]WARNINGS,NOTES} control
 164 ;;; this behavior. All but ALLOW-NOTES default to NIL.
 165 ;;;
 166 ;;; Arguments to the ALLOW-{FAILURE,[STYLE-]WARNINGS,NOTES} keyword
 167 ;;; parameters are interpreted as type specifiers restricting the
 168 ;;; allowed conditions of the respective kind.
 169 (defun checked-compile (form
 170                         &key
 171                           allow-failure
 172                           allow-warnings
 173                           allow-style-warnings
 174                           (allow-notes t))
 175   (let ((warnings '())
 176         (style-warnings '())
 177         (notes '())
 178         (error-output (make-string-output-stream)))
 179     (handler-bind ((sb-ext:compiler-note
 180                     (lambda (condition)
 181                       (push condition notes)
 182                       (muffle-warning condition)))
 183                    (style-warning
 184                     (lambda (condition)
 185                       (push condition style-warnings)
 186                       (muffle-warning condition)))
 187                    (warning
 188                     (lambda (condition)
 189                       (push condition warnings)
 190                       (muffle-warning condition))))
 191       (multiple-value-bind (function warnings-p failure-p)
 192           (let ((*error-output* error-output))
 193             (compile nil form))
 194         (declare (ignore warnings-p))
 195         (labels ((fail (kind conditions &optional allowed-type)
 196                    (error "~@<Compilation of ~S signaled ~A~P:~
 197                            ~{~@:_~@:_~{~/sb-impl:print-symbol-with-prefix/: ~A~}~}~
 198                            ~@[~@:_~@:_Allowed type is ~S.~]~@:>"
 199                           form kind (length conditions)
 200                           (mapcar (lambda (condition)
 201                                     (list (type-of condition) condition))
 202                                   conditions)
 203                           allowed-type))
 204                  (check-conditions (kind conditions allow)
 205                    (cond
 206                      (allow
 207                       (let ((offenders (remove-if (lambda (condition)
 208                                                     (typep condition allow))
 209                                                   conditions)))
 210                         (when offenders
 211                           (fail kind offenders allow))))
 212                      (conditions
 213                       (fail kind conditions)))))
 214
 215           (when (and (not allow-failure) failure-p)
 216             (let ((output (get-output-stream-string error-output)))
 217               (error "~@<Compilation of ~S failed~@[ with ~
 218                       output~@:_~@:_~A~@:_~@:_~].~@:>"
 219                      form (when (plusp (length output)) output))))
 220
 221           (check-conditions "warning"       warnings       allow-warnings)
 222           (check-conditions "style-warning" style-warnings allow-style-warnings)
 223           (check-conditions "note"          notes          allow-notes)
 224
 225           ;; Since we may have prevented warnings from being taken
 226           ;; into account for FAILURE-P by muffling them, adjust the
 227           ;; second return value accordingly.
 228           (values function (when (or failure-p warnings) t)
 229                   warnings style-warnings notes))))))
 230
 231 ;;; Repeat calling THUNK until its cumulated runtime, measured using
 232 ;;; GET-INTERNAL-RUN-TIME, is larger than PRECISION. Repeat this
 233 ;;; REPETITIONS many times and return the time one call to THUNK took
 234 ;;; in seconds as a float, according to the minimum of the cumulated
 235 ;;; runtimes over the repetitions.
 236 ;;; This allows to easily measure the runtime of expressions that take
 237 ;;; much less time than one internal time unit. Also, the results are
 238 ;;; unaffected, modulo quantization effects, by changes to
 239 ;;; INTERNAL-TIME-UNITS-PER-SECOND.
 240 ;;; Taking the minimum is intended to reduce the error introduced by
 241 ;;; garbage collections occurring at unpredictable times. The inner
 242 ;;; loop doubles the number of calls to THUNK each time before again
 243 ;;; measuring the time spent, so that the time measurement overhead
 244 ;;; doesn't distort the result if calling THUNK takes very little time.
 245 (defun runtime* (thunk repetitions precision)
 246   (loop repeat repetitions
 247         minimize
 248         (loop with start = (get-internal-run-time)
 249               with duration = 0
 250               for n = 1 then (* n 2)
 251               for total-runs = n then (+ total-runs n)
 252               do (dotimes (i n)
 253                    (funcall thunk))
 254                  (setf duration (- (get-internal-run-time) start))
 255               when (> duration precision)
 256               return (/ (float duration) (float total-runs)))
 257         into min-internal-time-units-per-call
 258         finally (return (/ min-internal-time-units-per-call
 259                            (float internal-time-units-per-second)))))
 260
 261 (defmacro runtime (form &key (repetitions 3) (precision 10))
 262   `(runtime* (lambda () ,form) ,repetitions ,precision))