1 ;;;; This software is part of the SBCL system. See the README file for
4 ;;;; While most of SBCL is derived from the CMU CL system, the test
5 ;;;; files (like this one) were written from scratch after the fork
8 ;;;; This software is in the public domain and is provided with
9 ;;;; absolutely no warranty. See the COPYING and CREDITS files for
10 ;;;; more information.
16 (defun make-things-for-sxhash-test (n)
17 (setf *things
* (make-array n
:fill-pointer
0))
19 (vector-push-extend (list (sb-kernel:get-lisp-obj-address obj
)
23 #-
(or x86 x86-64
) ; precise GC
24 (return-from make-things-for-sxhash-test
(dotimes (i n t
) (store (make-foo))))
26 (sb-thread:join-thread
27 (sb-thread:make-thread
28 (lambda () (dotimes (i n t
) (store (make-foo))))))))
29 (compile 'make-things-for-sxhash-test
)
31 ;;; Assert that the C code which computes a perturbation of the object
32 ;;; address for lazy stable address-based hashing is the same as lisp.
33 ;;; Further, assert that each bit of the resulting positive-fixnum
34 ;;; can be in a 0 and 1 state (don't want any bits stuck at 0).
35 (with-test (:name
:address-based-sxhash-gcing
)
37 (let ((tracker (make-array 4 :element-type
'sb-ext
:word
39 ;; with this many sxhashes we should see a 1 bit in
40 ;; in each bit position.
41 (when (make-things-for-sxhash-test (+ 20 #+arm64
10))
43 (sb-int:dovector
(thing *things
*)
44 (destructuring-bind (old-addr old-hash object
) thing
45 (let* ((new-addr (sb-kernel:get-lisp-obj-address object
))
46 (new-hash (sxhash object
)))
47 (setf (aref tracker
0) (logior (aref tracker
0) old-hash
)
48 (aref tracker
1) (logior (aref tracker
1)
49 (logxor old-hash most-positive-fixnum
)))
50 (let* ((count-1s (logcount new-hash
))
51 (count-0s (- sb-vm
:n-positive-fixnum-bits count-1s
)))
52 (incf (aref tracker
2) count-1s
)
53 (incf (aref tracker
3) count-0s
))
54 (cond ((= new-addr old-addr
)
55 (warn "Can't test SXHASH after movement: didn't move"))
56 ((not (eql new-hash old-hash
))
57 (error "SXHASH failure"))))))
58 ;; show the mask where we saw 1 bits (respectively 0),
59 ;; and total number of 1 (respectively 0) bits.)
61 (format t
"~@{[~64,'0b] ~d~%~}"
62 (aref tracker
0) (aref tracker
2)
63 (aref tracker
1) (aref tracker
3))
64 (assert (= (aref tracker
0) most-positive-fixnum
))
65 (assert (= (aref tracker
1) most-positive-fixnum
))))))
67 ;;; SXHASH and PSXHASH should distribute hash values well over the
68 ;;; space of possible values, so that collisions between the hash
69 ;;; values of unequal objects should be very uncommon. (Except of
70 ;;; course the hash values must collide when the objects are EQUAL or
71 ;;; EQUALP respectively!)
73 ;; In order to better test not-EQ-but-EQUAL and not-EQ-but-EQUALP,
74 ;; we'd like to suppress some optimizations.
75 (declare (notinline complex float coerce
+ - expt
))
76 (flet ((make-sxhash-subtests ()
80 (cons (cons 1 0) (cons 0 0))
81 (cons (list 1 0) (list 0 0))
82 (list (cons 1 0) (list 0 0))
83 (list (cons 0 1) (list 0 0))
84 (list (cons 0 0) (cons 1 0))
85 (list (cons 0 0) (cons 0 1))
87 44 (float 44) (coerce 44 'double-float
)
88 -
44 (float -
44) (coerce -
44 'double-float
)
89 0 (float 0) (coerce 0 'double-float
)
90 -
0 (- (float 0)) (- (coerce 0 'double-float
))
91 -
121 (float -
121) (coerce -
121 'double-float
)
92 3/4 (float 3/4) (coerce 3/4 'double-float
)
93 -
3/4 (float -
3/4) (coerce -
3/4 'double-float
)
94 45 (float 45) (coerce 45 'double-float
)
95 441/10 (float 441/10) (coerce (float 441/10) 'double-float
)
97 (expt 2 33) (expt 2.0 33) (expt 2.0d0
33)
98 (- (expt 1/2 50)) (- (expt 0.5 50)) (- (expt 0.5d0
50))
99 (+ (expt 1/2 50)) (+ (expt 0.5 50)) (+ (expt 0.5d0
50))
101 (complex 1.0 2.0) (complex 1.0d0
2.0)
102 (complex 1.5 -
3/2) (complex 1.5 -
1.5d0
)
106 (copy-seq "foo") (copy-seq "foobar") (copy-seq "foobarbaz")
109 (copy-seq #*0) (copy-seq #*1)
110 (copy-seq #*00) (copy-seq #*10)
111 (copy-seq #*01) (copy-seq #*11)
112 (copy-seq #*10010) (copy-seq #*100101) (bit-not #*01101)
113 (make-array 6 :fill-pointer
6
114 :element-type
'bit
:initial-contents
#*100101)
116 #'allocate-instance
#'no-applicable-method
))
117 (make-psxhash-extra-subtests ()
125 (make-array 3 :fill-pointer
0)
126 (make-array 7 :fill-pointer
0 :element-type
'bit
)
127 (make-array 8 :fill-pointer
0 :element-type
'character
)
128 (vector (cons 1 0) (cons 0 0))
129 (vector (cons 0 1) (cons 0 0))
130 (vector (cons 0 0) (cons 1 0))
131 (vector (cons 0 0) (cons 0 1))
132 (vector (cons 1 0) (cons 0 0))
133 (vector (cons 0 1) (cons 0 0))
134 (vector (list 0 0) (cons 1 0))
135 (vector (list 0 0) (list 0 1))
136 (vector (vector 1 0) (list 0 0))
137 (vector (vector 0 1) (list 0 0))
138 (vector (vector 0 0) (list 1 0))
139 (vector (vector 0 0) (list 0 1))
141 (vector (vector 0 0) (list 0 1.0d0
))
142 (vector (vector -
0.0d0
0) (list 1.0 0))
147 (replace (make-array 101
151 (replace (make-array 14
152 :element-type
'(unsigned-byte 8)
155 (replace (make-array 14
168 (replace (make-array 14
169 :element-type
'character
172 (replace (make-array 11
173 :element-type
'character
176 (replace (make-array 12
180 (replace (make-array 13
184 (replace (make-array 13
188 ;; FIXME: What about multi-dimensional arrays, hmm?
191 (make-hash-table :test
'equal
)
195 (make-bar :x
(list 1))
196 (make-bar :y
(list 1))))
197 (t->boolean
(x) (if x t nil
)))
199 ;; * The APPEND noise here is to help more strenuously test
200 ;; not-EQ-but-EQUAL and not-EQ-but-EQUALP cases.
201 ;; * It seems not to be worth the hassle testing SXHASH on
202 ;; values whose structure isn't understood by EQUAL, since
203 ;; we get too many false positives "SXHASHes are equal even
204 ;; though values aren't EQUAL, what a crummy hash function!"
205 ;; FIXME: Or am I misunderstanding the intent of the
206 ;; the SXHASH specification? Perhaps SXHASH is supposed to
207 ;; descend into the structure of objects even when EQUAL
208 ;; doesn't, in order to avoid hashing together things which
209 ;; are guaranteed not to be EQUAL? The definition of SXHASH
210 ;; seems to leave this completely unspecified: should
211 ;; "well-distributed" depend on substructure that EQUAL
212 ;; ignores? For our internal hash tables, the stricter
213 ;; descend-into-the-structure behavior might improve
214 ;; performance even though it's not specified by ANSI. But
215 ;; is it reasonable for users to expect it? Hmm..
216 (sxhash-tests (append (make-sxhash-subtests)
217 (make-sxhash-subtests)))
218 (psxhash-tests (append sxhash-tests
219 (make-psxhash-extra-subtests)
220 (make-psxhash-extra-subtests))))
221 ;; Check that SXHASH compiler transforms give the same results
222 ;; as the out-of-line version of SXHASH.
223 (let* ((fundef `(lambda ()
224 (list ,@(mapcar (lambda (value)
227 (fun (compile nil fundef
)))
228 (assert (equal (funcall fun
)
229 (mapcar #'sxhash sxhash-tests
))))
230 ;; Note: The tests for SXHASH-equality iff EQUAL and
231 ;; PSXHASH-equality iff EQUALP could fail because of an unlucky
232 ;; random collision. That's not very likely (since there are
233 ;; (EXPT 2 29) possible hash values and only on the order of 100
234 ;; test cases, so even with the birthday paradox a collision has
235 ;; probability only (/ (EXPT 100 2) (EXPT 2 29)), but it's
236 ;; probably worth checking if you are getting a mystifying error
237 ;; from this test. (SXHASH values and PSXHASH values don't
238 ;; change from run to run, so the random chance of bogus failure
239 ;; happens once every time the code is changed in such a way
240 ;; that the SXHASH distribution changes, not once every time the
242 (dolist (i sxhash-tests
)
243 (declare (notinline funcall
))
244 (unless (typep (funcall #'sxhash i
) '(and fixnum unsigned-byte
))
245 (error "bad SXHASH behavior for ~S" i
))
246 (dolist (j sxhash-tests
)
247 (unless (or (eq (t->boolean
(equal i j
))
248 (t->boolean
(= (sxhash i
) (sxhash j
))))
249 (and (typep i
'number
)
252 (subtypep (type-of i
) (type-of j
))
253 (subtypep (type-of j
) (type-of i
))))
254 ;; (If you get a surprising failure here, maybe you were
255 ;; just very unlucky; see the notes above.)
256 (error "bad SXHASH behavior for ~S ~S" i j
))))
257 (dolist (i psxhash-tests
)
258 (unless (typep (sb-int:psxhash i
) '(and fixnum unsigned-byte
))
259 (error "bad PSXHASH behavior for ~S" i
))
260 (dolist (j psxhash-tests
)
261 (unless (eq (t->boolean
(equalp i j
))
262 (t->boolean
(= (sb-int:psxhash i
) (sb-int:psxhash j
))))
263 ;; (If you get a surprising failure here, maybe you were
264 ;; just very unlucky; see the notes above.)
265 (error "bad PSXHASH behavior for ~S ~S" i j
))))
268 ;;; As of sbcl-0.6.12.10, writing hash tables readably should work.
269 ;;; This isn't required by the ANSI standard, but it should be, since
270 ;;; it's well-defined useful behavior which ANSI prohibits the users
271 ;;; from implementing themselves. (ANSI says the users can't define
272 ;;; their own their own PRINT-OBJECT (HASH-TABLE T) methods, and they
273 ;;; can't even wiggle out of it by subclassing HASH-TABLE or STREAM.)
274 (let ((original-ht (make-hash-table :test
'equal
:size
111))
275 (original-keys '(1 10 11 400030002 -
100000000)))
276 (dolist (key original-keys
)
277 (setf (gethash key original-ht
)
279 (let* ((written-ht (with-output-to-string (s)
280 (write original-ht
:stream s
:readably t
)))
281 (read-ht (with-input-from-string (s written-ht
)
283 (assert (= (hash-table-count read-ht
)
284 (hash-table-count original-ht
)
285 (length original-keys
)))
286 (assert (eql (hash-table-test original-ht
) (hash-table-test read-ht
)))
287 (assert (eql (hash-table-size original-ht
) (hash-table-size read-ht
)))
288 (dolist (key original-keys
)
289 (assert (eql (gethash key read-ht
)
290 (gethash key original-ht
))))))
292 ;;; NIL is both SYMBOL and LIST
293 (dolist (fun '(sxhash sb-impl
::psxhash
))
294 (assert (= (eval `(,fun nil
))
296 (funcall (compile nil
`(lambda (x)
300 (funcall (compile nil
`(lambda (x)
304 (funcall (compile nil
`(lambda (x)
309 ;;; This test works reliably on non-conservative platforms and
310 ;;; somewhat reliably on conservative platforms with threads.
311 (defun call-and-scrub-stack (thunk &aux results
)
312 ;; Start by giving ourselves some headroom via a
313 ;; DX allocation around the actual allocation...
314 (let ((*s
* (make-array 25)))
315 (declare (special *s
*)
316 (dynamic-extent *s
*))
317 (assert (stack-allocated-p *s
*))
318 (setq results
(multiple-value-list (funcall thunk
))))
319 ;; ... and then arrange to have the no-longer-used parts of the
320 ;; control stack cleared.
321 (sb-sys:scrub-control-stack
)
322 (values-list results
))
323 (compile 'call-and-scrub-stack
)
325 (defmacro alloc
(&body body
)
326 "Execute BODY and try to reduce the chance of leaking a conservative root."
327 #+sb-thread
`(sb-thread:join-thread
328 (sb-thread:make-thread
330 (call-and-scrub-stack (lambda () ,@body
)))))
331 #-sb-thread
`(call-and-scrub-stack (lambda () ,@body
)))
333 (with-test (:name
(:hash-table
:weakness
:eql
:numbers
))
334 (flet ((random-number ()
336 (loop for weakness in
'(nil :key
:value
:key-and-value
:key-or-value
) do
337 (let* ((ht (make-hash-table :weakness weakness
))
338 (n (alloc (loop repeat
1000
339 count
(let ((key (random-number)))
341 (setf (gethash key ht
)
342 (random-number))))))))
345 (assert (= n
(hash-table-count ht
)))))))
347 (defun add-removable-stuff (ht &key
(n 100) (size 10))
348 (flet ((unique-object ()
349 (make-array size
:fill-pointer
0)))
350 (loop for i below n do
351 (multiple-value-bind (key value
)
352 (ecase (hash-table-weakness ht
)
353 ((:key
) (values (unique-object) i
))
354 ((:value
) (values i
(unique-object)))
356 (if (zerop (random 2))
357 (values (unique-object) i
)
358 (values i
(unique-object))))
360 (values (unique-object) (unique-object))))
361 (setf (gethash key ht
) value
)))
364 (defun print-ht (ht &optional
(stream t
))
365 (format stream
"Weakness: ~S~%" (hash-table-weakness ht
))
366 (format stream
"Table: ~S~%" (sb-impl::hash-table-pairs ht
))
367 (format stream
"Next: ~S~%" (sb-impl::hash-table-next-vector ht
))
368 (format stream
"Index: ~S~%" (sb-impl::hash-table-index-vector ht
))
369 (format stream
"Hash: ~S~%" (sb-impl::hash-table-hash-vector ht
))
370 (force-output stream
))
373 ((test-weakness (weakness)
374 `(with-test (:name
(:hash-table
:weakness
,weakness
:removal
))
375 (loop for test in
'(eq eql equal equalp
) do
376 (let ((ht (make-hash-table :test
'equal
:weakness
,weakness
)))
377 (alloc (add-removable-stuff ht
:n
117 :size
1))
379 do
; (format t "~A. count: ~A~%" i (hash-table-count ht))
381 until
(zerop (hash-table-count ht
))
387 ;; With conservative gc the test may not be
388 ;; bullet-proof so it's not an outright
389 ;; failure but a warning.
391 (if (eq *evaluator-mode
* :compile
)
394 (warn "Weak hash removal test failed for weakness ~A"
398 ;; I separated these into 4 named tests to see if I could figure something out-
399 ;; If the interpreted lambda itself it kept alive by call-and-scrub,
400 ;; as it must be, the most recent values of local variables could linger.
401 ;; So with x86 using the interpreter, :KEY weakness generates a failure warning
402 ;; but how does :KEY-OR-VALUE _not_ generate a warning?
404 (test-weakness :value
)
405 (test-weakness :key-and-value
)
406 (test-weakness :key-or-value
))
408 (with-test (:name
(:hash-table
:weakness
:string-interning
))
409 (let ((ht (make-hash-table :test
'equal
:weakness
:key
))
411 (setf (gethash s ht
) s
)
412 (assert (eq (gethash s ht
) s
))
413 (assert (eq (gethash (copy-seq s
) ht
) s
))))
415 ;;; see if hash_vector is not written when there is none ...
416 (with-test (:name
(:hash-table
:weakness
:eq
))
418 (let ((index (random 2000)))
419 (let ((first (+ most-positive-fixnum
(mod (* index
31) 9)))
421 (let ((hash-table (make-hash-table :weakness
:key
:test
'eq
)))
423 (setf (gethash (+ first i
) hash-table
) i
))
426 ;; used to crash in gc
427 (with-test (:name
(:hash-table
:weakness
:keep
))
429 (let ((h1 (make-hash-table :weakness
:key
:test
#'equal
))
431 (loop for i from
0 to
1000
433 for value
= (make-array 10000 :fill-pointer
0)
436 (setf (gethash key h1
) value
))
437 (sb-ext:gc
:full t
))))
439 ;;; DEFINE-HASH-TABLE-TEST
441 (defstruct custom-hash-key name
)
442 (defun custom-hash-test (x y
)
443 (equal (custom-hash-key-name x
)
444 (custom-hash-key-name y
)))
445 (defun custom-hash-hash (x)
446 (sxhash (custom-hash-key-name x
)))
447 (define-hash-table-test custom-hash-test custom-hash-hash
)
448 (with-test (:name
:define-hash-table-test
.1)
449 (let ((table (make-hash-table :test
'custom-hash-test
)))
450 (setf (gethash (make-custom-hash-key :name
"foo") table
) :foo
)
451 (setf (gethash (make-custom-hash-key :name
"bar") table
) :bar
)
452 (assert (eq :foo
(gethash (make-custom-hash-key :name
"foo") table
)))
453 (assert (eq :bar
(gethash (make-custom-hash-key :name
"bar") table
)))
454 (assert (eq 'custom-hash-test
(hash-table-test table
))))
455 (let ((table (make-hash-table :test
#'custom-hash-test
)))
456 (setf (gethash (make-custom-hash-key :name
"foo") table
) :foo
)
457 (setf (gethash (make-custom-hash-key :name
"bar") table
) :bar
)
458 (assert (eq :foo
(gethash (make-custom-hash-key :name
"foo") table
)))
459 (assert (eq :bar
(gethash (make-custom-hash-key :name
"bar") table
)))
460 (assert (eq 'custom-hash-test
(hash-table-test table
)))))
463 (defun head-eql (x y
)
464 (every #'eql
(subseq x
0 3) (subseq y
0 3)))
465 (define-hash-table-test head-eql
467 (logand most-positive-fixnum
468 (reduce #'+ (map 'list
#'sxhash
(subseq x
0 3))))))
469 (with-test (:name
:define-hash-table-test
.2)
470 (let ((table (make-hash-table :test
'head-eql
)))
471 (setf (gethash #(1 2 3 4) table
) :|
123|
)
472 (setf (gethash '(2 3 4 7) table
) :|
234|
)
473 (setf (gethash "foobar" table
) :foo
)
474 (assert (eq :|
123|
(gethash '(1 2 3 ! 6) table
)))
475 (assert (eq :|
234|
(gethash #(2 3 4 0 2 1 a
) table
)))
476 (assert (eq :foo
(gethash '(#\f #\o
#\o
1 2 3) table
)))
477 (assert (eq 'head-eql
(hash-table-test table
))))
478 (let ((table (make-hash-table :test
#'head-eql
)))
479 (setf (gethash #(1 2 3 4) table
) :|
123|
)
480 (setf (gethash '(2 3 4 7) table
) :|
234|
)
481 (setf (gethash "foobar" table
) :foo
)
482 (assert (eq :|
123|
(gethash '(1 2 3 ! 6) table
)))
483 (assert (eq :|
234|
(gethash #(2 3 4 0 2 1 a
) table
)))
484 (assert (eq :foo
(gethash '(#\f #\o
#\o
1 2 3) table
)))
485 (assert (eq 'head-eql
(hash-table-test table
)))))
487 (with-test (:name
:make-hash-table
/hash-fun
)
488 (let ((table (make-hash-table
490 :hash-function
(lambda (x)
491 (sxhash (coerce (abs x
) 'double-float
))))))
492 (incf (gethash 1 table
0))
493 (incf (gethash 1.0f0 table
))
494 (incf (gethash 1.0d0 table
))
495 (incf (gethash (complex 1.0f0
0.0f0
) table
))
496 (incf (gethash (complex 1.0d0
0.0d0
) table
))
497 (assert (= 5 (gethash 1 table
)))
498 (assert (eq '= (hash-table-test table
)))))
502 (sword 0 :type sb-vm
:signed-word
)
503 (sf 0s0
:type single-float
)
504 (df 0d0
:type double-float
)
505 (csf #c
(0s0 0s0
) :type
(complex single-float
))
506 (cdf #c
(0d0 0d0
) :type
(complex double-float
)))
508 (import 'sb-impl
::psxhash
)
509 (with-test (:name
:psxhash-raw-slots
)
510 (let ((empty (psxhash (make-rslotty))))
511 ;; unequalp values produce unequal hashes
512 (assert (/= empty
(psxhash (make-rslotty :uword
32))))
513 (assert (/= empty
(psxhash (make-rslotty :sword -
1800))))
514 (assert (/= empty
(psxhash (make-rslotty :sf
1s0
))))
515 (assert (/= empty
(psxhash (make-rslotty :df
1d0
))))
516 (assert (/= empty
(psxhash (make-rslotty :csf
#c
(1s0 1s0
)))))
517 (assert (/= empty
(psxhash (make-rslotty :cdf
#c
(1d0 1d0
)))))
518 ;; equalp values produce equal hashes
519 (assert (= empty
(psxhash (make-rslotty :sf -
0s0
:df -
0d0
))))
520 (assert (= empty
(psxhash (make-rslotty :csf
#c
(-0s0 -
0s0
)))))
521 (assert (= empty
(psxhash (make-rslotty :csf
#c
(0s0 -
0s0
)))))
522 (assert (= empty
(psxhash (make-rslotty :csf
#c
(-0s0 0s0
)))))
523 (assert (= empty
(psxhash (make-rslotty :cdf
#c
(-0d0 -
0d0
)))))
524 (assert (= empty
(psxhash (make-rslotty :cdf
#c
(0d0 -
0d0
)))))
525 (assert (= empty
(psxhash (make-rslotty :cdf
#c
(-0d0 0d0
)))))))
527 (defun my= (a b
) (= a b
))
528 (defun fixnum-hash (x) (sxhash (the fixnum x
)))
529 (defun fixnum-hash-worse (x) (logand (sxhash (the fixnum x
)) 7))
530 (define-hash-table-test my
= fixnum-hash
)
532 (with-test (:name
:hash-fun-is-function-designator
)
533 ;; Users shouldn't write this baroque expression to make an EQL table.
534 (assert-error (make-hash-table :hash-function nil
))
536 (assert-error (make-hash-table :test
#'eql
:hash-function nil
))
537 ;; :TEST, if unknown, does not imply a hash function
538 ;; even when it looks like it could.
539 (assert-error (make-hash-table :test
#'=))
540 ;; and of course this doesn't work either because the preceding doesn't
541 (assert-error (make-hash-table :test
#'= :hash-function nil
))
542 ;; Try user functions
543 (let ((h (make-hash-table :test
'my
=)))
544 (assert (eq (sb-impl::hash-table-hash-fun h
) #'fixnum-hash
)))
545 (let ((h (make-hash-table :test
'my
= :hash-function
'fixnum-hash-worse
)))
546 (assert (eq (sb-impl::hash-table-hash-fun h
) #'fixnum-hash-worse
)))
547 (let ((h (make-hash-table :test
'my
= :hash-function
#'fixnum-hash-worse
)))
548 (assert (eq (sb-impl::hash-table-hash-fun h
) #'fixnum-hash-worse
)))
549 (assert-error (make-hash-table :test
'my
= :hash-function nil
))) ; no good
552 (with-test (:name
:psxhash-large-floats
)
553 (flet ((test (integer)
554 (assert (= (sb-int:psxhash
(float integer
1d0
))
555 (sb-int:psxhash
(truncate (float integer
1d0
)))))
556 (assert (= (sb-int:psxhash
(float integer
))
557 (sb-int:psxhash
(truncate (float integer
)))))))
558 (loop for i from
80 to
100 by
4
560 (test (1+ (expt 2 i
)))
561 (test (1- (expt 2 i
)))
562 (test (- (expt 2 i
) (random (expt 2 i
))))
563 (test (+ (expt 2 i
) (random (expt 2 i
)))))))