3 ;;; Time-stamp: <2010-11-10 14:03:10 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2010, AJ Rossini. MIT License. See
8 ;;; LICENSE.mit in top level directory for details.
9 ;;; Purpose: packages for Common Lisp Statistics
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
17 ;;; Current structure, dependencies:
19 '((:ls-user
:depends-on lisp-stat
)
20 (:lisp-stat
:depends-on
'(cls-dataframe
24 (:cls-data
:depends-on
)
25 (:cls-dataframe
:depends-on cls-data
)
35 (defpackage :lisp-stat-config
36 (:documentation
"holds Common Lisp Statistics configuration/loading variables and related functions.")
37 (:nicknames
:cls-config
)
39 (:export
*common-lisp-stat-version
*
41 *lsos-files
* *basic-files
* *ls-files
*
42 *cls-installation-home-dir
* *cls-data-dir
* *cls-examples-dir
*))
44 (defpackage :lisp-stat-object-system
45 (:nicknames
:ls-objects
:lsos
)
47 (:shadow
:call-method
:call-next-method
)
48 (:export ls-object objectp
*object
* kind-of-p make-object
50 *set-slot-hook
* proto-slot-value self
51 send call-next-method call-method
52 defmeth defproto instance-slots proto-name
))
54 ;;; -types and -float probably ought to be moved into a -numerics
57 (defpackage :lisp-stat-types
58 (:documentation
"Provides some typeing for LispStat, but is clearly
62 check-nonneg-fixnum check-one-nonneg-fixnum
63 check-one-fixnum check-one-real check-one-number
))
65 (defpackage :lisp-stat-float
67 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
68 machine-epsilon base-float makedouble
70 make-base-trans-fun-2 make-base-trans-fun
72 base-log base-exp base-expt base-sqrt base-sin base-cos
73 base-tan base-asin base-acos base-atan base-sinh
74 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
75 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
76 BASE-FROUND BASE-SIGNUM BASE-CIS
))
78 ;;; Probably should move into cls-data package.
80 (defpackage :lisp-stat-compound-data
82 :lisp-stat-object-system
84 (:shadowing-import-from
:lisp-stat-object-system
85 call-next-method call-method
)
86 (:export compound-data-p
*compound-data-proto
*
88 compound-data-seq compound-data-length
89 element-list element-seq
91 recursive-map-elements map-elements repeat
93 get-next-element make-next-element set-next-element
96 select split-list which
100 (defpackage :lisp-stat-macros
102 :lisp-stat-compound-data
)
103 (:export make-rv-function make-rv-function-1
))
105 (defpackage :lisp-stat-basics
107 :lisp-stat-object-system
111 :lisp-stat-compound-data
)
112 (:shadowing-import-from
:lisp-stat-object-system
113 call-method call-next-method
)
114 (:export permute-array sum prod count-elements mean
117 (defpackage :lisp-stat-float
119 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
120 machine-epsilon base-float makedouble
122 make-base-trans-fun-2 make-base-trans-fun
124 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
125 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
126 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
127 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
128 BASE-FROUND BASE-SIGNUM BASE-CIS
))
130 (defpackage :lisp-stat-macros
132 :lisp-stat-compound-data
)
133 (:export make-rv-function make-rv-function-1
))
136 (defpackage :cls-matrix
137 (:documentation
"basic utilities for using lisp arrays as numerical
138 matrices. Not optimized, and must consider this slow. Routines
139 should be optimized, it is only that we need them first, optimize
142 (:export matrixp num-rows num-cols matmult identity-matrix diagonal
143 row-list column-list inner-product outer-product
144 cross-product transpose bind-columns bind-rows
145 array-data-vector vector-to-array
))
148 ;;; NEW CLOS STRUCTURE
150 ;; CLS-DATA contains the basic variable structure, classes, support/indexing, and mixins.
151 ;; CLS-DATAFRAME leverages CLS-DATA, XARRAY, and LISP-MATRIX
152 ;; CLS-DATAIO stores/saves structures
153 ;; CLS-DATATRANS converts between structures, and from DATAFRAMES to MODEL-MATRIXES
155 (defpackage :cls-data
157 (:shadowing-import-from
:xarray slice
)
159 ;; must work out what is needed for stat variable metadata, support
163 ;;; cls-data... in dataframe, though.
164 (defpackage :cls-dataframe
169 (:shadowing-import-from
:xarray slice
)
171 ;; generic container class for data -- if small enough
172 ;; could be value, otherwise might be reference.
178 list-of-columns
;; list-of-variables
179 list-of-rows
;; list-of-observations
182 varlabels caselabels nrows ncols
183 dataframe-dimension dataframe-dimensons
184 xref xtype xdims xdim xrank slice take carray
190 (defpackage :cls-dataio
191 (:documentation
"Data I/O and similar import technologies.")
194 :lisp-stat-object-system
197 (:shadowing-import-from
:lisp-stat-object-system
198 call-method call-next-method
)
199 (:export dsvstream-
>dataframe
201 dsvstream-
>listoflist
203 filename.dsv-
>dataframe
206 (defpackage :cls-datatrans
207 (:documentation
"Data I/O and similar import technologies.")
210 :lisp-stat-object-system
213 (:shadowing-import-from
:lisp-stat-object-system
214 call-method call-next-method
)
215 (:export listoflist-
>dataframe dataframe-
>listoflist
216 listoflist-
>array array-
>listoflist
217 listoflist-
>matrix-like matrix-like-
>listoflist
223 (defpackage :lisp-stat-model
224 (:documentation
"Model management for data analysis.")
228 ;; data structures for model and model/data combination
229 model statistical-model analysis
))
233 (defpackage :cls-visualize
237 (:shadowing-import-from
:xarray slice
)
241 (defpackage :cls-visualize-plplot
248 plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex
3D-plot-ex
))
253 (defpackage :lisp-stat-ffi-int
256 (:export ccl-store-integer ccl-store-double ccl-store-ptr
259 (defpackage :lisp-stat-probability
264 (:export log-gamma set-seed
266 normal-cdf normal-quant normal-dens normal-rand
268 cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
269 gamma-cdf gamma-quant gamma-dens gamma-rand
270 chisq-cdf chisq-quant chisq-dens chisq-rand
271 beta-cdf beta-quant beta-dens beta-rand
272 t-cdf t-quant t-dens t-rand
273 f-cdf f-quant f-dens f-rand
274 poisson-cdf poisson-quant poisson-pmf poisson-rand
275 binomial-cdf binomial-quant binomial-pmf binomial-rand
))
277 (defpackage :cls-probability
281 (:export probability-law distribution
282 density-fcn quantile-fcn survivorship-fcn
))
284 (defpackage :lisp-stat-math
286 :lisp-stat-object-system
288 :lisp-stat-compound-data
290 (:shadowing-import-from
:lisp-stat-object-system
291 call-method call-next-method
)
292 (:shadow expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
293 asin acos atan sinh cosh tanh asinh acosh atanh float random
294 truncate floor ceiling round minusp zerop plusp evenp oddp
295 < <= = /= >= > ;; complex
296 conjugate realpart imagpart phase
297 min max logand logior logxor lognot ffloor fceiling
298 ftruncate fround signum cis
)
299 (:export ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
300 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
301 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
303 conjugate realpart imagpart phase min max
304 logand logior logxor lognot ffloor fceiling ftruncate fround
306 (:documentation
"Vectorization of numerical functions"))
309 #|
;; some of this goes back in, but not all of it?
310 (defpackage :lisp-stat-linalg
317 :lisp-stat-compound-data
)
318 (:shadowing-import-from
:lisp-stat-math
319 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
320 asin acos atan sinh cosh tanh asinh acosh atanh float random
321 truncate floor ceiling round minusp zerop plusp evenp oddp
322 < <= = /= >= > complex conjugate realpart imagpart phase
323 min max logand logior logxor lognot ffloor fceiling
324 ftruncate fround signum cis
)
325 (:export chol-decomp lu-decomp lu-solve determinant inverse
326 sv-decomp qr-decomp rcondest make-rotation spline
327 kernel-dens kernel-smooth
328 fft make-sweep-matrix sweep-operator ax
+y eigen
330 check-real
;; for optimize
332 covariance-matrix matrix print-matrix solve
333 backsolve eigenvalues eigenvectors accumulate cumsum combine
341 (defpackage :lisp-stat-data
342 (:documentation
"Data management, integration, I/O, and other data technologies.")
343 (:nicknames
:ls-data
)
345 :lisp-stat-object-system
348 :lisp-stat-compound-data
)
349 (:shadowing-import-from
:lisp-stat-object-system
350 call-method call-next-method
)
352 ;; generic structures
354 empirical-statistical-variable
355 modelbased-statistical-variable
356 categorical-statistical-variable
357 nominal-statistical-variable
358 ordinal-statistical-variable
359 continuous-statistical-variable
361 ordering factor-levels nobs support pdmf draw
365 statistical-observation
366 measurement-types record
368 open-file-dialog read-data-file read-data-columns load-data
369 load-example
*variables
* *ask-on-redefine
*
370 def variables savevar undef
))
372 (defpackage :lisp-stat-descriptive-statistics
377 :lisp-stat-compound-data
379 (:shadowing-import-from
:lisp-stat-math
;; life is a vector!
380 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
381 asin acos atan sinh cosh tanh asinh acosh atanh float random
382 truncate floor ceiling round minusp zerop plusp evenp oddp
383 < <= = /= >= > ;; complex
384 conjugate realpart imagpart phase
385 min max logand logior logxor lognot ffloor fceiling
386 ftruncate fround signum cis
)
387 (:export mean standard-deviation variance
388 quantile median interquartile-range
391 (defpackage :lisp-stat-regression-linear
395 :lisp-stat-compound-data
396 :lisp-stat-descriptive-statistics
)
397 (:shadowing-import-from
:lisp-stat-object-system
398 call-method call-next-method
)
399 (:export regression-model fit-model
401 estimates covariance-matrix
402 ;; functions for helpers
404 print-object
;; for method dispatch
407 (defpackage :common-lisp-statistics
408 (:documentation
"Experimentation package for LispStat. Serious work
409 should be packaged up as a separate but similar package to help
410 drive reproducibility. By this I mean, creating a
411 data/analytics/analysis package with the minimal set of
412 objects/packages required.")
413 (:nicknames
:cls
:common-lisp-statistics
:lisp-stat
)
417 :xarray
;; generic reference -- internally supporting array, lol structs
419 :lisp-matrix
;; conversion to a more robust linalg approach
423 :lisp-stat-object-system
424 :lisp-stat-compound-data
425 :lisp-stat-probability
437 :lisp-stat-descriptive-statistics
438 :lisp-stat-regression-linear
440 ;; :cls-visualize-plplot
441 ;; :cls-visualize-cl2d
443 (:shadowing-import-from
:xarray slice
)
444 (:shadowing-import-from
:lisp-stat-object-system
445 call-method call-next-method
)
446 (:shadowing-import-from
:lisp-stat-math
447 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
448 asin acos atan sinh cosh tanh asinh acosh atanh float random
449 truncate floor ceiling round minusp zerop plusp evenp oddp
452 conjugate realpart imagpart phase
455 logand logior logxor lognot
456 ffloor fceiling ftruncate fround
460 *default-path
* *lsos-files
* *basic-files
* *ls-files
*
461 *cls-installation-home-dir
* *cls-data-dir
* *cls-examples-dir
*
465 defmeth send proto-slot-value
468 fixnump check-nonneg-fixnum check-one-fixnum
469 check-one-nonneg-fixnum
470 check-one-real check-one-number
478 compound-data-p
*compound-data-proto
* compound-object-p
479 compound-data-seq compound-data-length
480 element-list element-seq
482 recursive-map-elements map-elements
485 get-next-element make-next-element set-next-element
493 ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
494 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
495 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
497 conjugate realpart imagpart phase min max
498 logand logior logxor lognot ffloor fceiling ftruncate fround
501 #|
;; The following need to be re-found in lisp-matrix...
504 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
505 column-list inner-product outer-product cross-product transpose
506 bind-columns bind-rows
509 chol-decomp lu-decomp lu-solve determinant inverse
510 sv-decomp qr-decomp rcondest make-rotation spline
511 kernel-dens kernel-smooth
512 fft make-sweep-matrix sweep-operator ax
+y eigen
514 covariance-matrix matrix print-matrix solve
515 backsolve eigenvalues eigenvectors accumulate cumsum combine
518 ;; in linalg.lisp, possibly not supported by matlisp
519 spline kernel-dens kernel-smooth
526 ;; package LISPSTAT-MACROS
527 make-rv-function make-rv-function-1
530 xref xtype xdims xdim xdims
*
532 ;; package LISTOFLIST
533 sublists-of-same-size-p equal-listoflist transpose-listoflist
536 ;; need to take this list and make it symbols... specs could mean
537 ;; that we process the strings in different ways? Create such a
538 ;; macro to enable export within this package?
542 (mapc #'symbol-for-symbol-to-string-or-symbol
543 (do-external-symbols (s (find-package 'lisp-stat-data
) lst
) (push s lst
))))
546 open-file-dialog read-data-file read-data-columns load-data
547 load-example
*variables
* *ask-on-redefine
*
548 def variables savevar undef
550 dataframe-like dataframe-array make-dataframe
551 varlabels caselabels nrows ncols
552 dataframe-dimension dataframe-dimensons
553 dfref dfref-case dfref-var
554 consistent-dataframe-p
555 dataset list-of-columns list-of-rows
559 listoflist-
>dataframe listoflist-
>array listoflist-
>matrix-like
560 sublists-of-same-size-p
563 dsvstream-
>dataframe dsvstream-
>matrix dsvstream-
>listoflist
564 filename.dsv-
>dataframe
567 listoflist-
>dataframe dataframe-
>listoflist
568 listoflist-
>array array-
>listoflist
569 listoflist-
>matrix-like matrix-like-
>listoflist
572 ;; statistics.lsp (descriptions, should probably be moved
574 standard-deviation quantile median interquartile-range
577 ;; probability (dists.lisp)
579 uniform-rand normal-cdf normal-quant normal-dens
580 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
581 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
582 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
583 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
584 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
585 binomial-cdf binomial-quant binomial-pmf binomial-rand
587 ;; Here is where we have a problem -- lispstat core should be core
588 ;; data management and config problems, with packages providing
589 ;; specialized extensions to LispStat, i.e. regression, nonlin
590 ;; regression, bayesian regression via laplace approximation, etc.
592 ;; The following could be considered "recommended packages",
593 ;; similar to the idea of the recommended packages in R. Probably
594 ;; we want them to do the exporting within that package, therefore
595 ;; NOT being able to lock the "data-ish" package, but only the
596 ;; subpackages prior to export.
599 ;; -- linear regressin models.
600 regression-model fit-model
601 estimates covariance-matrix
603 regression-model-proto x y intercept sweep-matrix
604 basis weights included total-sum-of-squares residual-sum-of-squares
605 predictor-names response-name case-labels
609 ;; -- nonlinear regression models
610 nreg-model nreg-model-proto mean-function theta-hat epsilon
612 ;; we might need something like xtxinv here? But should be
613 ;; encapsulated, so we use the one in regression.lisp
616 bayes-model bayes-model-proto bayes-internals
628 ;;;; PACKAGES FOR USEABILITY
630 (defpackage :lisp-stat-data-examples
631 (:documentation
"Example data for unittests, examples, illustrations,")
633 :common-lisp-statistics
)
634 (:shadowing-import-from
:lisp-stat
635 call-method call-next-method
637 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
638 asin acos atan sinh cosh tanh asinh acosh atanh float random
639 truncate floor ceiling round minusp zerop plusp evenp oddp
640 < <= = /= >= > > ;; complex
641 conjugate realpart imagpart phase
642 min max logand logior logxor lognot ffloor fceiling
643 ftruncate fround signum cis
646 (:export iron aluminum absorbtion
650 (defpackage :lisp-stat-user
651 (:documentation
"Experimentation package for LispStat. Serious work
652 should be placed in a similar package elsewhere for
653 reproducibility. But this should hint as to what needs to be done
654 for a user- or analysis-package.")
655 (:nicknames
:ls-user
:cls-user
)
656 (:use
:common-lisp
; always needed for user playgrounds!
658 :common-lisp-statistics
659 :lisp-stat-data-examples
) ;; this last is to have 'things to play with'
660 (:shadowing-import-from
:lisp-stat
661 call-method call-next-method
663 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
664 asin acos atan sinh cosh tanh asinh acosh atanh float random
665 truncate floor ceiling round minusp zerop plusp evenp oddp
666 < <= = /= >= > > ;; complex
667 conjugate realpart imagpart phase
668 min max logand logior logxor lognot ffloor fceiling
669 ftruncate fround signum cis
673 (defpackage :lisp-stat-unittests
676 :lisp-stat
:lisp-stat-data-examples
)
677 (:shadowing-import-from
:lisp-stat
678 call-method call-next-method
;; objects
679 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
;; lsmath
680 asin acos atan sinh cosh tanh asinh acosh atanh float random
681 truncate floor ceiling round minusp zerop plusp evenp oddp
682 < <= = /= >= > ;; complex
683 conjugate realpart imagpart phase
684 min max logand logior logxor lognot ffloor fceiling
685 ftruncate fround signum cis
)
686 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard
; exec
687 almost
= almost
=lists numerical
=)) ; compare
689 (defpackage :cls-dataframe-example
691 :lift
:lisp-stat-unittests
692 :lisp-stat-data-examples
694 (:export absorbtion aluminum iron
))
697 (defpackage :lisp-stat-optimize
702 :lisp-stat-object-system
704 :lisp-stat-compound-data
710 :lisp-stat-linalg-data
714 (:shadowing-import-from
:lisp-stat-object-system
715 call-method call-next-method
)
716 (:shadowing-import-from
:lisp-stat-math
717 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
718 asin acos atan sinh cosh tanh asinh acosh atanh float random
719 truncate floor ceiling round minusp zerop plusp evenp oddp
720 < <= = /= >= > complex conjugate realpart imagpart phase
721 min max logand logior logxor lognot ffloor fceiling
722 ftruncate fround signum cis
)
728 newtonmax nelmeadmax
))