2 ;;; Copyright (c) 2005--2007, by A.J. Rossini <blindglobe@gmail.com>
3 ;;; See COPYRIGHT file for any additional restrictions (BSD license).
4 ;;; Since 1991, ANSI was finally finished. Edited for ANSI Common Lisp.
6 ;;;; lsbasics -- Low level Lisp-Stat functions
8 ;;;; Copyright (c) 1991, by Luke Tierney. Permission is granted for
15 (defpackage :lisp-stat-basics
17 :lisp-stat-object-system
22 :lisp-stat-compound-data
25 (:shadowing-import-from
:lisp-stat-object-system
26 slot-value call-method call-next-method
)
29 copy-vector copy-array which repeat
30 permute-array sum prod count-elements mean if-else sample sort-data
34 ;; matrixp num-rows num-cols matmult identity-matrix diagonal row-list
35 ;; column-list inner-product outer-product cross-product transpose
36 ;; bind-columns bind-rows
39 chol-decomp lu-decomp lu-solve determinant inverse sv-decomp
40 qr-decomp rcondest make-rotation
41 fft make-sweep-matrix sweep-operator ax
+y numgrad numhess
43 ;; in linalg.lisp, possibly not supported by matlisp
44 spline kernel-dens kernel-smooth
46 make-rv-function make-rv-function-1
48 log-gamma uniform-rand normal-cdf normal-quant normal-dens
49 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
50 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
51 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
52 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
53 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
54 binomial-cdf binomial-quant binomial-pmf binomial-rand
59 ;; (defpackage :lisp-stat-basics
60 ;; (:nicknames :ls-basics)
61 ;; (:use ;; :common-lisp
62 ;; :lisp-stat-object-system
65 ;; ;;(:shadowing-import-from (package-shadowing-symbols #:lisp-stat-object-system))
69 ;; sequencep copy-vector copy-array iseq which repeat select
70 ;; permute-array sum prod count-elements mean if-else
71 ;; sample sort-data order rank
74 ;; ;; #+ kcl (export '(function-lambda-expression realp fixnump))
78 ;; compound-data-p map-elements compound-data-seq
79 ;; compound-data-length element-seq compound-data-proto
82 ;; log-gamma uniform-rand normal-cdf normal-quant normal-dens
83 ;; normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
84 ;; cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
85 ;; chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
86 ;; beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
87 ;; f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
88 ;; binomial-cdf binomial-quant binomial-pmf binomial-rand
92 ;; chol-decomp lu-decomp lu-solve determinant inverse sv-decomp
93 ;; qr-decomp rcondest make-rotation spline kernel-dens kernel-smooth
94 ;; fft make-sweep-matrix sweep-operator ax+y numgrad numhess
98 ;; matrixp num-rows num-cols matmult identity-matrix diagonal
99 ;; row-list column-list inner-product outer-product cross-product
100 ;; transpose bind-columns bind-rows
104 ;; +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
109 ;; ;; (import '(ccl:def-logical-directory ccl:ff-load ccl:deffcfun ccl:defccallable))
113 (in-package #:lisp-stat-basics
)
116 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
118 ;;;; Sequence Element Access
120 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
122 (defun get-next-element (x i
)
123 "Get element i from seq x. FIXME: not really??"
124 (let ((myseq (first x
)))
126 (let ((elem (first myseq
)))
127 (setf (first x
) (rest myseq
))
131 (defun set-next-element (x i v
)
132 (let ((seq (first x
)))
135 (setf (first x
) (rest seq
)))
136 (t (setf (aref seq i
) v
)))))
138 (defun make-next-element (x) (list x
))
141 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
143 ;;;; Copying Functions
145 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
148 ;;; COPY-VECTOR function
151 (defun copy-vector (x)
153 Returns a copy of the vector X"
157 ;;; COPY-ARRAY function
160 (defun copy-array (a)
162 Returns a copy of the array A"
163 (vector-to-array (copy-seq (array-data-vector a
))
164 (array-dimensions a
)))
166 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
168 ;;;; Sequence Functions
170 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
179 Returns a list of the indices where elements of sequence X are not NIL."
180 (let ((x (list (compound-data-seq x
)))
183 (flet ((add-result (x)
184 (if result
(setf (rest tail
) (list x
)) (setf result
(list x
)))
185 (setf tail
(if tail
(rest tail
) result
)))
186 (get-next-element (seq-list i
)
187 (cond ((consp (first seq-list
))
188 (let ((elem (first (first seq-list
))))
189 (setf (first seq-list
) (rest (first seq-list
)))
191 (t (aref (first seq-list
) i
)))))
192 (let ((n (length (first x
))))
193 (dotimes (i n result
)
194 (if (get-next-element x i
) (add-result i
)))))))
202 Repeats VALS. If TIMES is a number and VALS is a non-null, non-array atom,
203 a list of length TIMES with all elements eq to VALS is returned. If VALS
204 is a list and TIMES is a number then VALS is appended TIMES times. If
205 TIMES is a list of numbers then VALS must be a list of equal length and
206 the simpler version of repeat is mapped down the two lists.
207 Examples: (repeat 2 5) returns (2 2 2 2 2)
208 (repeat '(1 2) 3) returns (1 2 1 2 1 2)
209 (repeat '(4 5 6) '(1 2 3)) returns (4 5 5 6 6 6)
210 (repeat '((4) (5 6)) '(2 3)) returns (4 4 5 6 5 6 5 6)"
211 (cond ((compound-data-p b
)
212 (let* ((reps (coerce (compound-data-seq (map-elements #'repeat a b
))
214 (result (first reps
))
215 (tail (last (first reps
))))
216 (dolist (next (rest reps
) result
)
218 (setf (rest tail
) next
)
219 (setf tail
(last next
))))))
220 (t (let* ((a (if (compound-data-p a
)
221 (coerce (compound-data-seq a
) 'list
)
224 (dotimes (i b result
)
225 (let ((next (copy-list a
)))
226 (if result
(setf (rest (last next
)) result
))
227 (setf result next
)))))))
229 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
231 ;;;; Subset Selection and Mutation Functions
233 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
235 ;;;; is x an ordered sequence of nonnegative positive integers?
236 (defun ordered-nneg-seq(x)
239 (cx (make-next-element x
))
242 (let ((elem (check-nonneg-fixnum (get-next-element cx i
))))
243 (if (> m elem
) (return nil
) (setf m elem
)))))))
245 ;;;; select or set the subsequence corresponding to the specified indices
246 (defun sequence-select(x indices
&optional
(values nil set-values
))
252 (declare (fixnum rlen dlen vlen
))
254 ;; Check the input data
256 (check-sequence indices
)
257 (if set-values
(check-sequence values
))
259 ;; Find the data sizes
260 (setf data
(if (ordered-nneg-seq indices
) x
(coerce x
'vector
)))
261 (setf dlen
(length data
))
262 (setf rlen
(length indices
))
264 (setf vlen
(length values
))
265 (if (/= vlen rlen
) (error "value and index sequences do not match")))
267 ;; set up the result/value sequence
271 (make-sequence (if (listp x
) 'list
'vector
) rlen
)))
273 ;; get or set the sequence elements
276 (cr (make-next-element result
))
277 (ci (make-next-element indices
))
282 (declare (fixnum i j index
))
283 (setf index
(get-next-element ci i
))
284 (if (<= dlen index
) (error "index out of range - ~a" index
))
285 (let ((elem (get-next-element cr i
)))
292 ((not (and (< j index
) (consp nextx
))))
294 (setf nextx
(rest nextx
)))
295 (setf (first nextx
) elem
))
296 (t (setf (aref x index
) elem
)))))
298 (cr (make-next-element result
))
299 (ci (make-next-element indices
))
305 (declare (fixnum i j index
))
306 (setf index
(get-next-element ci i
))
307 (if (<= dlen index
) (error "index out of range - ~a" index
))
309 ((listp data
) ;; indices must be ordered
311 ((not (and (< j index
) (consp nextx
))))
313 (setf nextx
(rest nextx
)))
314 (setf elem
(first nextx
)))
315 (t (setf elem
(aref data index
))))
316 (set-next-element cr i elem
)))
320 (defun old-rowmajor-index (index indices dim olddim
)
321 "translate row major index in resulting subarray to row major index
322 in the original array."
323 (declare (fixnum index
))
324 (let ((rank (length dim
))
328 (declare (fixnum rank face oldface
))
332 (setf face
(* face
(aref dim i
)))
333 (setf oldface
(* oldface
(aref olddim i
))))
337 (setf face
(/ face
(aref dim i
)))
338 (setf oldface
(/ oldface
(aref olddim i
)))
340 (* oldface
(aref (aref indices i
) (floor (/ index face
))))) ;;*** is this floor really needed???
341 (setf index
(rem index face
)))
344 (defun subarray-select (a indexlist
&optional
(values nil set_values
))
345 "extract or set subarray for the indices from a displaced array."
357 (declare (fixnum rank n
))
359 (if (or (sequencep a
) (not (arrayp a
))) (error "not an array - ~a" a
))
360 (if (not (listp indexlist
)) (error "bad index list - ~a" indices
))
361 (if (/= (length indexlist
) (array-rank a
))
362 (error "wrong number of indices"))
364 (setf indices
(coerce indexlist
'vector
))
366 (setf olddim
(coerce (array-dimensions a
) 'vector
))
368 ;; compute the result dimension vector and fix up the indices
369 (setf rank
(array-rank a
))
370 (setf dim
(make-array rank
))
373 (setf index
(aref indices i
))
374 (setf n
(aref olddim i
))
375 (setf index
(if (fixnump index
) (vector index
) (coerce index
'vector
)))
376 (setf k
(length index
))
379 (if (<= n
(check-nonneg-fixnum (aref index j
)))
380 (error "index out of bounds - ~a" (aref index j
)))
381 (setf (aref indices i
) index
))
382 (setf (aref dim i
) (length index
)))
384 ;; set up the result or check the values
385 (let ((dim-list (coerce dim
'list
)))
389 ((compound-data-p values
)
390 (if (or (not (arrayp values
)) (/= rank
(array-rank values
)))
391 (error "bad values array - ~a" values
))
392 (setf vdim
(coerce (array-dimensions values
) 'vector
))
395 (if (/= (aref vdim i
) (aref dim i
))
396 (error "bad value array dimensions - ~a" values
)))
397 (setf result values
))
398 (t (setf result
(make-array dim-list
:initial-element values
)))))
399 (t (setf result
(make-array dim-list
)))))
401 ;; compute the result or set the values
402 (setf data
(compound-data-seq a
))
403 (setf result_data
(compound-data-seq result
))
404 (setf n
(length result_data
))
407 (setf k
(old-rowmajor-index i indices dim olddim
))
408 (if (or (> 0 k
) (>= k
(length data
))) (error "index out of range"))
410 (setf (aref data k
) (aref result_data i
))
411 (setf (aref result_data i
) (aref data k
))))
419 (defun select (x &rest args
)
420 "Args: (a &rest indices)
421 A can be a list or an array. If A is a list and INDICES is a single number
422 then the appropriate element of A is returned. If is a list and INDICES is
423 a list of numbers then the sublist of the corresponding elements is returned.
424 If A in an array then the number of INDICES must match the ARRAY-RANK of A.
425 If each index is a number then the appropriate array element is returned.
426 Otherwise the INDICES must all be lists of numbers and the corresponding
427 submatrix of A is returned. SELECT can be used in setf."
429 ((every #'fixnump args
)
430 (if (listp x
) (nth (first args
) x
) (apply #'aref x args
)))
431 ((sequencep x
) (sequence-select x
(first args
)))
432 (t (subarray-select x args
))))
435 ;; Built in SET-SELECT (SETF method for SELECT)
436 (defun set-select (x &rest args
)
437 (let ((indices (butlast args
))
438 (values (first (last args
))))
441 (if (not (consp indices
)) (error "bad indices - ~a" indices
))
442 (let* ((indices (first indices
))
443 (i-list (if (fixnump indices
) (list indices
) indices
))
444 (v-list (if (fixnump indices
) (list values
) values
)))
445 (sequence-select x i-list v-list
)))
447 (subarray-select x indices values
))
448 (t (error "bad argument type - ~a" x
)))
451 (defsetf select set-select
)
454 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
456 ;;;; Array Permutation Functions
458 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
460 (defun permute-indices (x y perm check
)
461 "Args: (x y perm check).
462 permute x into y using perm; all should be vectors; If check is TRUE
463 the routine will check to make sure no indices are reused, but x
465 (let ((rank (length x
)))
466 (declare (fixnum rank
))
469 (let ((k (aref perm i
)))
470 (if (not (fixnump k
)) (error "bad permutation sequence - ~a" perm
))
471 (if (or (< k
0) (>= k rank
))
472 (error "bad permutation sequence - ~a" perm
))
473 (setf (aref y i
) (aref x k
))
474 ;; to insure dimensions are not re-used
475 (if check
(setf (aref x k
) NIL
))))))
477 (defun indices-from-rowmajor (a k result
)
479 Compute indices in a from rowmajor index k, put in vector result."
482 (if (not (arrayp a
)) (error "not an array - ~a" a
))
483 (if (or (> 0 k
) (>= k
(array-total-size a
))) (error "index out of range"))
486 (rank (array-rank a
))
487 (dim (array-dimensions a
)))
488 (declare (fixnum face rank
))
489 (let ((cdim (make-next-element dim
)))
492 (setf face
(* face
(get-next-element cdim i
)))))
493 (let ((cdim (make-next-element dim
)))
495 (setf face
(/ face
(get-next-element cdim i
)))
496 (setf (aref result i
) (floor (/ k face
)))
497 (setf k
(rem k face
))))))
499 (defun translate-index (i result x perm indices oldindices ilist
)
500 "Args: (i result x perm indices oldindices ilist).
501 Translate row major index in original array to row major index in new
502 array. Use indices vectors and ilist for temporary storage."
504 (let ((rank (array-rank x
)))
505 (declare (fixnum rank
))
506 (indices-from-rowmajor x i oldindices
)
507 (permute-indices oldindices indices perm nil
)
508 (do ((next ilist
(rest next
))
510 ((not (and (< k rank
) (consp next
))))
511 (setf (first next
) (aref indices k
)))
512 (apply #'array-row-major-index result ilist
)))
514 (defun permute-array (x perm
)
516 Returns a copy of the array A permuted according to the permutation P."
517 (if (not (arrayp x
)) (error "not an array - ~a" x
))
518 (check-sequence perm
)
519 (if (/= (length perm
) (array-rank x
))
520 (error "bad permutation sequence - ~a" perm
))
521 (let* ((perm (coerce perm
'vector
))
522 (rank (array-rank x
))
523 (dim (make-array rank
))
524 (olddim (coerce (array-dimensions x
) 'vector
)))
525 (declare (fixnum rank
))
526 ;; construct new dimension vector
527 (permute-indices olddim dim perm t
)
528 ;; make result array and the index vectors and lists */
529 (let* ((result (make-array (coerce dim
'list
)))
530 (indices (make-array rank
))
531 (oldindices (make-array rank
))
532 (ilist (make-list rank
))
533 (data (compound-data-seq x
))
534 (result_data (compound-data-seq result
))
539 (setf (aref oldindices i
) (list nil
)))
540 ;; fill in the result
541 (if (/= n
(length result_data
)) (error "bad data"))
542 (dotimes (i n result
)
544 (let ((k (translate-index i result x perm indices oldindices ilist
)))
546 (setf (aref result_data k
) (aref data i
)))))))
548 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
550 ;;;; SUM, PROD, COUNT-ELEMENTS, and MEAN Functions
552 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
557 (let ((seq (compound-data-seq x
))
561 (setf sum
(+ sum
(if (numberp x
) x
(sum-1 x
)))))
562 (let ((n (length seq
)))
566 (let ((x (aref seq i
)))
567 (setf sum
(+ sum
(if (numberp x
) x
(sum-1 x
)))))))))))
569 (defun sum (&rest args
)
570 "Args: (&rest number-data)
571 Returns the sum of all the elements of its arguments. Returns 0 if there
572 are no arguments. Vector reducing."
574 (sum-1 (if (rest args
) args
(first args
)))
580 (let ((seq (compound-data-seq x
))
584 (setf prod
(* prod
(if (numberp x
) x
(prod-1 x
)))))
585 (let ((n (length seq
)))
589 (let ((x (aref seq i
)))
590 (setf prod
(* prod
(if (numberp x
) x
(prod-1 x
)))))))))))
592 (defun prod (&rest args
)
593 "Args: (&rest number-data)
594 Returns the product of all the elements of its arguments. Returns 1 if there
595 are no arguments. Vector reducing."
597 (prod-1 (if (rest args
) args
(first args
)))
600 (defun count-elements (x)
601 "Args: (number &rest more-numbers)
602 Returns the number of its arguments. Vector reducing"
603 (if (compound-data-p x
)
604 (let ((seq (compound-data-seq x
))
607 (dolist (x seq count
)
608 (incf count
(if (compound-data-p x
) (count-elements x
) 1)))
609 (let ((n (length seq
)))
613 (let ((x (aref seq i
)))
614 (incf count
(if (compound-data-p x
) (count-elements x
) 1)))))))
619 Returns the mean of the elements x. Vector reducing."
622 (labels ((add-to-mean (x)
623 (let ((count+1 (+ count
1.0)))
624 (setf mean
(+ (* (/ count count
+1) mean
) (* (/ count
+1) x
)))
625 (setf count count
+1)))
629 (let ((seq (compound-data-seq x
)))
632 (if (numberp x
) (add-to-mean x
) (find-mean x
)))
633 (let ((n (length seq
)))
636 (let ((x (aref seq i
)))
639 (find-mean x
))))))))))
643 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
645 ;;;; Sorting Functions
647 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
651 Returns a sequence with the numbers or strings in the sequence X in order."
652 (flet ((less (x y
) (if (numberp x
) (< x y
) (string-lessp x y
))))
653 (stable-sort (copy-seq (compound-data-seq x
)) #'less
)))
657 Returns a sequence of the indices of elements in the sequence of numbers
658 or strings X in order."
659 (let* ((seq (compound-data-seq x
))
660 (type (if (consp seq
) 'list
'vector
))
662 (flet ((entry (x) (setf i
(+ i
1)) (list x i
))
666 (if (numberp x
) (< x y
) (string-lessp x y
)))))
667 (let ((sorted-seq (stable-sort (map type
#'entry seq
) #'less
)))
668 (map type
#'second sorted-seq
)))))
670 ;; this isn't destructive -- do we document destructive only, or any
674 Returns a sequence with the elements of the list or array of numbers or
675 strings X replaced by their ranks."
676 (let ((ranked-seq (order (order x
))))
677 (make-compound-data (compound-data-shape x
) ranked-seq
)))
679 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
681 ;;;; IF-ELSE and SAMPLE Functions
683 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
685 (defun if-else (a x y
)
687 Takes simple or compound data items FIRST, X and Y and returns result of
688 elementswise selecting from X if FIRST is not NIL and from Y otherwise."
689 (flet ((base-if-else (a x y
) (if a x y
)))
690 (recursive-map-elements #'base-if-else
#'if-else a x y
)))
692 (defun sample (x ssize
&optional replace
)
693 "Args: (x n &optional (replace nil))
694 Returns a list of a random sample of size N from sequence X drawn with or
695 without replacement."
698 (x (if (consp x
) (coerce x
'vector
) (copy-vector x
)))
701 (dotimes (i ssize result
)
702 (let ((j (if replace
(random n
) (+ i
(random (- n i
))))))
703 (setf result
(cons (aref x j
) result
))
704 (unless replace
;; swap elements i and j
705 (let ((temp (aref x i
)))
706 (setf (aref x i
) (aref x j
))
707 (setf (aref x j
) temp
))))))))