3 ;;; Time-stamp: <2012-10-05 04:08:40 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2010, AJ Rossini. MIT License. See
8 ;;; LICENSE.mit in top level directory for details.
9 ;;; Purpose: packages for Common Lisp Statistics
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
17 ;;; Current structure, dependencies:
19 '((:ls-user
:depends-on lisp-stat
)
20 (:lisp-stat
:depends-on
'(cls-dataframe
24 (:cls-data
:depends-on
)
25 (:cls-dataframe
:depends-on cls-data
)
32 (defpackage :lisp-stat-config
33 (:documentation
"holds Common Lisp Statistics configuration/loading variables and related functions.")
34 (:nicknames
:cls-config
)
36 (:export
*common-lisp-stat-version
*
38 *lsos-files
* *basic-files
* *ls-files
*
39 *cls-installation-home-dir
* *cls-data-dir
* *cls-examples-dir
*))
41 (defpackage :lisp-stat-object-system
42 (:nicknames
:ls-objects
:lsos
)
44 (:shadow
:call-method
:call-next-method
)
45 (:export ls-object objectp
*object
* kind-of-p make-object
47 *set-slot-hook
* proto-slot-value self
48 send call-next-method call-method
49 defmeth defproto instance-slots proto-name
))
51 ;;; -types and -float probably ought to be moved into a -numerics
54 (defpackage :lisp-stat-types
55 (:documentation
"Provides some typeing for LispStat, but is clearly
59 check-nonneg-fixnum check-one-nonneg-fixnum
60 check-one-fixnum check-one-real check-one-number
))
62 (defpackage :lisp-stat-float
64 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
65 machine-epsilon base-float makedouble
67 make-base-trans-fun-2 make-base-trans-fun
69 base-log base-exp base-expt base-sqrt base-sin base-cos
70 base-tan base-asin base-acos base-atan base-sinh
71 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
72 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
73 BASE-FROUND BASE-SIGNUM BASE-CIS
))
75 ;;; Probably should move into cls-data package.
77 (defpackage :lisp-stat-compound-data
79 :lisp-stat-object-system
81 (:shadowing-import-from
:lisp-stat-object-system
82 call-next-method call-method
)
83 (:export compound-data-p
*compound-data-proto
*
85 compound-data-seq compound-data-length
86 element-list element-seq
88 recursive-map-elements map-elements repeat
90 get-next-element make-next-element set-next-element
93 select split-list which
97 (defpackage :lisp-stat-macros
99 :lisp-stat-compound-data
)
100 (:export make-rv-function make-rv-function-1
))
102 (defpackage :lisp-stat-basics
104 :lisp-stat-object-system
108 :lisp-stat-compound-data
)
109 (:shadowing-import-from
:lisp-stat-object-system
110 call-method call-next-method
)
111 (:export permute-array sum prod count-elements mean
114 (defpackage :lisp-stat-float
116 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
117 machine-epsilon base-float makedouble
119 make-base-trans-fun-2 make-base-trans-fun
121 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
122 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
123 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
124 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
125 BASE-FROUND BASE-SIGNUM BASE-CIS
))
127 (defpackage :lisp-stat-macros
129 :lisp-stat-compound-data
)
130 (:export make-rv-function make-rv-function-1
))
132 ;;;;;;;; NEw CLS structure
134 (defpackage :cls-matrix
135 (:documentation
"basic utilities for using lisp arrays as numerical
136 matrices. Not optimized, and must consider this slow. Routines
137 should be optimized, it is only that we need them first, optimize
140 (:export matrixp num-rows num-cols matmult identity-matrix diagonal
141 row-list column-list inner-product outer-product
142 cross-product transpose bind-columns bind-rows
143 array-data-vector vector-to-array
))
145 ;;; NEW CLOS STRUCTURE
147 ;; CLS-DATA contains the basic variable structure, classes, support/indexing, and mixins.
148 ;; CLS-DATAFRAME leverages CLS-DATA, XARRAY, and LISP-MATRIX
149 ;; CLS-DATAIO stores/saves structures
150 ;; CLS-DATATRANS converts between structures, and from DATAFRAMES to MODEL-MATRIXES
152 (defpackage :cls-data
154 (:shadowing-import-from
:xarray slice
)
156 ;; must work out what is needed for stat variable metadata, support
160 ;;; cls-data... in dataframe, though.
161 (defpackage :cls-dataframe
166 (:shadowing-import-from
:xarray slice
)
168 ;; generic container class for data -- if small enough
169 ;; could be value, otherwise might be reference.
175 list-of-columns
;; list-of-variables
176 list-of-rows
;; list-of-observations
179 varlabels caselabels nrows ncols
180 dataframe-dimension dataframe-dimensons
181 xref xtype xdims xdim xrank slice take carray
186 (defpackage :cls-dataio
187 (:documentation
"Data I/O and similar import technologies.")
190 :lisp-stat-object-system
193 (:shadowing-import-from
:lisp-stat-object-system
194 call-method call-next-method
)
195 (:export dsvstream-
>dataframe
197 dsvstream-
>listoflist
199 filename.dsv-
>dataframe
))
201 (defpackage :cls-datatrans
202 (:documentation
"Data I/O and similar import technologies.")
205 :lisp-stat-object-system
208 (:shadowing-import-from
:lisp-stat-object-system
209 call-method call-next-method
)
210 (:export listoflist-
>dataframe dataframe-
>listoflist
211 listoflist-
>array array-
>listoflist
212 listoflist-
>matrix-like matrix-like-
>listoflist
))
216 (defpackage :lisp-stat-model
217 (:documentation
"Model management for data analysis.")
221 ;; data structures for model and model/data combination
222 model statistical-model analysis
))
226 (defpackage :cls-visualize
230 (:shadowing-import-from
:xarray slice
)
234 (defpackage :cls-visualize-plplot
241 plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex
3D-plot-ex
))
246 ;; (defpackage :lisp-stat-ffi-int
247 ;; (:use :common-lisp
249 ;; (:export ccl-store-integer ccl-store-double ccl-store-ptr
252 ;; (defpackage :lisp-stat-probability
253 ;; (:use :common-lisp
255 ;; :lisp-stat-ffi-int
256 ;; :lisp-stat-macros)
257 ;; (:export log-gamma set-seed
259 ;; normal-cdf normal-quant normal-dens normal-rand
261 ;; cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
262 ;; gamma-cdf gamma-quant gamma-dens gamma-rand
263 ;; chisq-cdf chisq-quant chisq-dens chisq-rand
264 ;; beta-cdf beta-quant beta-dens beta-rand
265 ;; t-cdf t-quant t-dens t-rand
266 ;; f-cdf f-quant f-dens f-rand
267 ;; poisson-cdf poisson-quant poisson-pmf poisson-rand
268 ;; binomial-cdf binomial-quant binomial-pmf binomial-rand))
270 (defpackage :cls-probability
274 (:export probability-law distribution
275 density-fcn quantile-fcn survivorship-fcn
))
277 (defpackage :lisp-stat-math
279 :lisp-stat-object-system
281 :lisp-stat-compound-data
283 (:shadowing-import-from
:lisp-stat-object-system
284 call-method call-next-method
)
285 (:shadow expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
286 asin acos atan sinh cosh tanh asinh acosh atanh float random
287 truncate floor ceiling round minusp zerop plusp evenp oddp
288 < <= = /= >= > ;; complex
289 conjugate realpart imagpart phase
290 min max logand logior logxor lognot ffloor fceiling
291 ftruncate fround signum cis
)
292 (:export ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
293 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
294 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
296 conjugate realpart imagpart phase min max
297 logand logior logxor lognot ffloor fceiling ftruncate fround
299 (:documentation
"Vectorization of numerical functions"))
302 #|
;; some of this goes back in, but not all of it?
303 (defpackage :lisp-stat-linalg
310 :lisp-stat-compound-data
)
311 (:shadowing-import-from
:lisp-stat-math
312 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
313 asin acos atan sinh cosh tanh asinh acosh atanh float random
314 truncate floor ceiling round minusp zerop plusp evenp oddp
315 < <= = /= >= > complex conjugate realpart imagpart phase
316 min max logand logior logxor lognot ffloor fceiling
317 ftruncate fround signum cis
)
318 (:export chol-decomp lu-decomp lu-solve determinant inverse
319 sv-decomp qr-decomp rcondest make-rotation spline
320 kernel-dens kernel-smooth
321 fft make-sweep-matrix sweep-operator ax
+y eigen
323 check-real
;; for optimize
325 covariance-matrix matrix print-matrix solve
326 backsolve eigenvalues eigenvectors accumulate cumsum combine
334 (defpackage :lisp-stat-data
335 (:documentation
"Data management, integration, I/O, and other data technologies.")
336 (:nicknames
:ls-data
)
338 :lisp-stat-object-system
341 :lisp-stat-compound-data
)
342 (:shadowing-import-from
:lisp-stat-object-system
343 call-method call-next-method
)
345 ;; generic structures
347 empirical-statistical-variable
348 modelbased-statistical-variable
349 categorical-statistical-variable
350 nominal-statistical-variable
351 ordinal-statistical-variable
352 continuous-statistical-variable
354 ordering factor-levels nobs support pdmf draw
358 statistical-observation
359 measurement-types record
361 open-file-dialog read-data-file read-data-columns load-data
362 load-example
*variables
* *ask-on-redefine
*
363 def variables savevar undef
))
365 (defpackage :lisp-stat-descriptive-statistics
370 :lisp-stat-compound-data
372 (:shadowing-import-from
:lisp-stat-math
;; life is a vector!
373 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
374 asin acos atan sinh cosh tanh asinh acosh atanh float random
375 truncate floor ceiling round minusp zerop plusp evenp oddp
376 < <= = /= >= > ;; complex
377 conjugate realpart imagpart phase
378 min max logand logior logxor lognot ffloor fceiling
379 ftruncate fround signum cis
)
380 (:export mean standard-deviation variance
381 quantile median interquartile-range
384 (defpackage :lisp-stat-regression-linear
388 :lisp-stat-compound-data
389 :lisp-stat-descriptive-statistics
)
390 (:shadowing-import-from
:lisp-stat-object-system
391 call-method call-next-method
)
392 (:export regression-model fit-model
394 estimates covariance-matrix
395 ;; functions for helpers
397 print-object
;; for method dispatch
400 (defpackage :common-lisp-statistics
401 (:documentation
"Experimentation package for LispStat. Serious work
402 should be packaged up as a separate but similar package to help
403 drive reproducibility. By this I mean, creating a
404 data/analytics/analysis package with the minimal set of
405 objects/packages required.")
406 (:nicknames
:cls
:common-lisp-statistics
:lisp-stat
)
410 :xarray
;; generic reference -- internally supporting array, lol structs
412 :lisp-matrix
;; conversion to a more robust linalg approach
416 :lisp-stat-object-system
417 :lisp-stat-compound-data
418 ;; :lisp-stat-probability
430 :lisp-stat-descriptive-statistics
431 :lisp-stat-regression-linear
433 ;; :cls-visualize-plplot
434 ;; :cls-visualize-cl2d
436 (:shadowing-import-from
:xarray slice
)
437 (:shadowing-import-from
:lisp-stat-object-system
438 call-method call-next-method
)
439 (:shadowing-import-from
:lisp-stat-math
440 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
441 asin acos atan sinh cosh tanh asinh acosh atanh float random
442 truncate floor ceiling round minusp zerop plusp evenp oddp
445 conjugate realpart imagpart phase
448 logand logior logxor lognot
449 ffloor fceiling ftruncate fround
453 *default-path
* *lsos-files
* *basic-files
* *ls-files
*
454 *cls-installation-home-dir
* *cls-data-dir
* *cls-examples-dir
*
458 defmeth send proto-slot-value
461 fixnump check-nonneg-fixnum check-one-fixnum
462 check-one-nonneg-fixnum
463 check-one-real check-one-number
471 compound-data-p
*compound-data-proto
* compound-object-p
472 compound-data-seq compound-data-length
473 element-list element-seq
475 recursive-map-elements map-elements
478 get-next-element make-next-element set-next-element
486 ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
487 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
488 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
490 conjugate realpart imagpart phase min max
491 logand logior logxor lognot ffloor fceiling ftruncate fround
494 #|
;; The following need to be re-found in lisp-matrix...
497 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
498 column-list inner-product outer-product cross-product transpose
499 bind-columns bind-rows
502 chol-decomp lu-decomp lu-solve determinant inverse
503 sv-decomp qr-decomp rcondest make-rotation spline
504 kernel-dens kernel-smooth
505 fft make-sweep-matrix sweep-operator ax
+y eigen
507 covariance-matrix matrix print-matrix solve
508 backsolve eigenvalues eigenvectors accumulate cumsum combine
511 ;; in linalg.lisp, possibly not supported by matlisp
512 spline kernel-dens kernel-smooth
517 ;; newtonmax nelmeadmax
519 ;; package LISPSTAT-MACROS
520 make-rv-function make-rv-function-1
523 xref xtype xdims xdim xdims
*
525 ;; package LISTOFLIST
526 sublists-of-same-size-p equal-listoflist transpose-listoflist
529 ;; need to take this list and make it symbols... specs could mean
530 ;; that we process the strings in different ways? Create such a
531 ;; macro to enable export within this package?
535 (mapc #'symbol-for-symbol-to-string-or-symbol
536 (do-external-symbols (s (find-package 'lisp-stat-data
) lst
) (push s lst
))))
539 open-file-dialog read-data-file read-data-columns load-data
540 load-example
*variables
* *ask-on-redefine
*
541 def variables savevar undef
543 dataframe-like dataframe-array make-dataframe
544 varlabels caselabels nrows ncols
545 dataframe-dimension dataframe-dimensons
546 dfref dfref-case dfref-var
547 consistent-dataframe-p
548 dataset list-of-columns list-of-rows
552 listoflist-
>dataframe listoflist-
>array listoflist-
>matrix-like
553 sublists-of-same-size-p
556 dsvstream-
>dataframe dsvstream-
>matrix dsvstream-
>listoflist
557 filename.dsv-
>dataframe
560 listoflist-
>dataframe dataframe-
>listoflist
561 listoflist-
>array array-
>listoflist
562 listoflist-
>matrix-like matrix-like-
>listoflist
565 ;; statistics.lsp (descriptions, should probably be moved
567 standard-deviation quantile median interquartile-range
570 ;; probability (dists.lisp)
571 ;; log-gamma set-seed
572 ;; uniform-rand normal-cdf normal-quant normal-dens
573 ;; normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
574 ;; cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
575 ;; chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
576 ;; beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
577 ;; f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
578 ;; binomial-cdf binomial-quant binomial-pmf binomial-rand
580 ;; Here is where we have a problem -- lispstat core should be core
581 ;; data management and config problems, with packages providing
582 ;; specialized extensions to LispStat, i.e. regression, nonlin
583 ;; regression, bayesian regression via laplace approximation, etc.
585 ;; The following could be considered "recommended packages",
586 ;; similar to the idea of the recommended packages in R. Probably
587 ;; we want them to do the exporting within that package, therefore
588 ;; NOT being able to lock the "data-ish" package, but only the
589 ;; subpackages prior to export.
592 ;; -- linear regressin models.
593 regression-model fit-model
594 estimates covariance-matrix
596 regression-model-proto x y intercept sweep-matrix
597 basis weights included total-sum-of-squares residual-sum-of-squares
598 predictor-names response-name case-labels
602 ;; -- nonlinear regression models
603 nreg-model nreg-model-proto mean-function theta-hat epsilon
605 ;; we might need something like xtxinv here? But should be
606 ;; encapsulated, so we use the one in regression.lisp
609 bayes-model bayes-model-proto bayes-internals
621 ;;;; PACKAGES FOR USEABILITY
623 (defpackage :lisp-stat-data-examples
624 (:documentation
"Example data for unittests, examples, illustrations,")
626 :common-lisp-statistics
)
627 (:shadowing-import-from
:lisp-stat
628 call-method call-next-method
630 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
631 asin acos atan sinh cosh tanh asinh acosh atanh float random
632 truncate floor ceiling round minusp zerop plusp evenp oddp
633 < <= = /= >= > > ;; complex
634 conjugate realpart imagpart phase
635 min max logand logior logxor lognot ffloor fceiling
636 ftruncate fround signum cis
639 (:export iron aluminum absorbtion
643 (defpackage :lisp-stat-user
644 (:documentation
"Experimentation package for LispStat. Serious work
645 should be placed in a similar package elsewhere for
646 reproducibility. But this should hint as to what needs to be done
647 for a user- or analysis-package.")
648 (:nicknames
:ls-user
:cls-user
)
649 (:use
:common-lisp
; always needed for user playgrounds!
651 :common-lisp-statistics
652 :lisp-stat-data-examples
) ;; this last is to have 'things to play with'
653 (:shadowing-import-from
:lisp-stat
654 call-method call-next-method
656 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
657 asin acos atan sinh cosh tanh asinh acosh atanh float random
658 truncate floor ceiling round minusp zerop plusp evenp oddp
659 < <= = /= >= > > ;; complex
660 conjugate realpart imagpart phase
661 min max logand logior logxor lognot ffloor fceiling
662 ftruncate fround signum cis
666 (defpackage :lisp-stat-unittests
669 :lisp-stat
:lisp-stat-data-examples
)
670 (:shadowing-import-from
:lisp-stat
671 call-method call-next-method
;; objects
672 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
;; lsmath
673 asin acos atan sinh cosh tanh asinh acosh atanh float random
674 truncate floor ceiling round minusp zerop plusp evenp oddp
675 < <= = /= >= > ;; complex
676 conjugate realpart imagpart phase
677 min max logand logior logxor lognot ffloor fceiling
678 ftruncate fround signum cis
)
679 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard
; exec
680 almost
= almost
=lists numerical
=)) ; compare
682 (defpackage :cls-dataframe-example
684 :lift
:lisp-stat-unittests
685 :lisp-stat-data-examples
687 (:export absorbtion aluminum iron
))
690 ;; (defpackage :lisp-stat-optimize
691 ;; (:use :common-lisp
694 ;; :lisp-stat-ffi-int
695 ;; :lisp-stat-object-system
697 ;; :lisp-stat-compound-data
703 ;; :lisp-stat-linalg-data
707 ;; (:shadowing-import-from :lisp-stat-object-system
708 ;; call-method call-next-method)
709 ;; (:shadowing-import-from :lisp-stat-math
710 ;; expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
711 ;; asin acos atan sinh cosh tanh asinh acosh atanh float random
712 ;; truncate floor ceiling round minusp zerop plusp evenp oddp
713 ;; < <= = /= >= > complex conjugate realpart imagpart phase
714 ;; min max logand logior logxor lognot ffloor fceiling
715 ;; ftruncate fround signum cis)
721 ;; newtonmax nelmeadmax))