3 ;;; Time-stamp: <2010-11-04 16:45:44 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2010, AJ Rossini. MIT License. See
8 ;;; LICENSE.mit in top level directory for details.
9 ;;; Purpose: packages for Common Lisp Statistics
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
17 ;;; Current structure, dependencies:
19 '((:ls-user
:depends-on lisp-stat
)
20 (:lisp-stat
:depends-on
'(cls-dataframe
24 (:cls-data
:depends-on
)
25 (:cls-dataframe
:depends-on cls-data
)
35 (defpackage :lisp-stat-config
36 (:documentation
"holds Common Lisp Statistics configuration/loading variables and related functions.")
37 (:nicknames
:cls-config
)
39 (:export
*common-lisp-stat-version
*
41 *lsos-files
* *basic-files
* *ls-files
*
42 *cls-installation-home-dir
* *cls-data-dir
* *cls-examples-dir
*))
44 (defpackage :lisp-stat-object-system
45 (:nicknames
:ls-objects
:lsos
)
47 (:shadow
:call-method
:call-next-method
)
48 (:export ls-object objectp
*object
* kind-of-p make-object
50 *set-slot-hook
* proto-slot-value self
51 send call-next-method call-method
52 defmeth defproto instance-slots proto-name
))
54 ;;; -types and -float probably ought to be moved into a -numerics
57 (defpackage :lisp-stat-types
58 (:documentation
"Provides some typeing for LispStat, but is clearly
62 check-nonneg-fixnum check-one-nonneg-fixnum
63 check-one-fixnum check-one-real check-one-number
))
65 (defpackage :lisp-stat-float
67 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
68 machine-epsilon base-float makedouble
70 make-base-trans-fun-2 make-base-trans-fun
72 base-log base-exp base-expt base-sqrt base-sin base-cos
73 base-tan base-asin base-acos base-atan base-sinh
74 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
75 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
76 BASE-FROUND BASE-SIGNUM BASE-CIS
))
78 ;;; Probably should move into cls-data package.
80 (defpackage :lisp-stat-compound-data
82 :lisp-stat-object-system
84 (:shadowing-import-from
:lisp-stat-object-system
85 call-next-method call-method
)
86 (:export compound-data-p
*compound-data-proto
*
88 compound-data-seq compound-data-length
89 element-list element-seq
91 recursive-map-elements map-elements repeat
93 get-next-element make-next-element set-next-element
96 select split-list which
100 (defpackage :lisp-stat-macros
102 :lisp-stat-compound-data
)
103 (:export make-rv-function make-rv-function-1
))
105 (defpackage :lisp-stat-basics
107 :lisp-stat-object-system
111 :lisp-stat-compound-data
)
112 (:shadowing-import-from
:lisp-stat-object-system
113 call-method call-next-method
)
114 (:export permute-array sum prod count-elements mean
117 (defpackage :lisp-stat-float
119 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
120 machine-epsilon base-float makedouble
122 make-base-trans-fun-2 make-base-trans-fun
124 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
125 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
126 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
127 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
128 BASE-FROUND BASE-SIGNUM BASE-CIS
))
130 (defpackage :lisp-stat-macros
132 :lisp-stat-compound-data
)
133 (:export make-rv-function make-rv-function-1
))
136 (defpackage :cls-matrix
137 (:documentation
"basic utilities for using lisp arrays as numerical
138 matrices. Not optimized, and must consider this slow. Routines
139 should be optimized, it is only that we need them first, optimize
142 (:export matrixp num-rows num-cols matmult identity-matrix diagonal
143 row-list column-list inner-product outer-product
144 cross-product transpose bind-columns bind-rows
145 array-data-vector vector-to-array
))
148 ;;; NEW CLOS STRUCTURE
150 ;; CLS-DATA contains the basic variable structure, classes, support/indexing, and mixins.
151 ;; CLS-DATAFRAME leverages CLS-DATA, XARRAY, and LISP-MATRIX
152 ;; CLS-DATAIO stores/saves structures
153 ;; CLS-DATATRANS converts between structures, and from DATAFRAMES to MODEL-MATRIXES
155 (defpackage :cls-data
157 (:shadowing-import-from
:xarray slice
)
159 ;; must work out what is needed for stat variable metadata, support
163 ;;; cls-data... in dataframe, though.
164 (defpackage :cls-dataframe
169 (:shadowing-import-from
:xarray slice
)
171 ;; generic container class for data -- if small enough
172 ;; could be value, otherwise might be reference.
178 list-of-columns
;; list-of-variables
179 list-of-rows
;; list-of-observations
182 varlabels caselabels nrows ncols
183 dataframe-dimension dataframe-dimensons
184 xref xtype xdims xdim xrank slice take carray
190 (defpackage :cls-dataio
191 (:documentation
"Data I/O and similar import technologies.")
194 :lisp-stat-object-system
197 (:shadowing-import-from
:lisp-stat-object-system
198 call-method call-next-method
)
199 (:export dsvstream-
>dataframe
201 dsvstream-
>listoflist
203 filename.dsv-
>dataframe
206 (defpackage :cls-datatrans
207 (:documentation
"Data I/O and similar import technologies.")
210 :lisp-stat-object-system
213 (:shadowing-import-from
:lisp-stat-object-system
214 call-method call-next-method
)
215 (:export listoflist-
>dataframe dataframe-
>listoflist
216 listoflist-
>array array-
>listoflist
217 listoflist-
>matrix-like matrix-like-
>listoflist
223 (defpackage :lisp-stat-model
224 (:documentation
"Model management for data analysis.")
228 ;; data structures for model and model/data combination
229 model statistical-model analysis
))
233 (defpackage :cls-visualize
237 (:shadowing-import-from
:xarray slice
)
241 (defpackage :cls-visualize-plplot
248 plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex
3D-plot-ex
))
253 (defpackage :lisp-stat-ffi-int
256 (:export ccl-store-integer ccl-store-double ccl-store-ptr
259 (defpackage :lisp-stat-probability
264 (:export log-gamma set-seed
266 normal-cdf normal-quant normal-dens normal-rand
268 cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
269 gamma-cdf gamma-quant gamma-dens gamma-rand
270 chisq-cdf chisq-quant chisq-dens chisq-rand
271 beta-cdf beta-quant beta-dens beta-rand
272 t-cdf t-quant t-dens t-rand
273 f-cdf f-quant f-dens f-rand
274 poisson-cdf poisson-quant poisson-pmf poisson-rand
275 binomial-cdf binomial-quant binomial-pmf binomial-rand
))
279 (defpackage :lisp-stat-math
281 :lisp-stat-object-system
283 :lisp-stat-compound-data
285 (:shadowing-import-from
:lisp-stat-object-system
286 call-method call-next-method
)
287 (:shadow expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
288 asin acos atan sinh cosh tanh asinh acosh atanh float random
289 truncate floor ceiling round minusp zerop plusp evenp oddp
290 < <= = /= >= > ;; complex
291 conjugate realpart imagpart phase
292 min max logand logior logxor lognot ffloor fceiling
293 ftruncate fround signum cis
)
294 (:export ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
295 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
296 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
298 conjugate realpart imagpart phase min max
299 logand logior logxor lognot ffloor fceiling ftruncate fround
301 (:documentation
"Vectorization of numerical functions"))
304 #|
;; some of this goes back in, but not all of it?
305 (defpackage :lisp-stat-linalg
312 :lisp-stat-compound-data
)
313 (:shadowing-import-from
:lisp-stat-math
314 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
315 asin acos atan sinh cosh tanh asinh acosh atanh float random
316 truncate floor ceiling round minusp zerop plusp evenp oddp
317 < <= = /= >= > complex conjugate realpart imagpart phase
318 min max logand logior logxor lognot ffloor fceiling
319 ftruncate fround signum cis
)
320 (:export chol-decomp lu-decomp lu-solve determinant inverse
321 sv-decomp qr-decomp rcondest make-rotation spline
322 kernel-dens kernel-smooth
323 fft make-sweep-matrix sweep-operator ax
+y eigen
325 check-real
;; for optimize
327 covariance-matrix matrix print-matrix solve
328 backsolve eigenvalues eigenvectors accumulate cumsum combine
336 (defpackage :lisp-stat-data
337 (:documentation
"Data management, integration, I/O, and other data technologies.")
338 (:nicknames
:ls-data
)
340 :lisp-stat-object-system
343 :lisp-stat-compound-data
)
344 (:shadowing-import-from
:lisp-stat-object-system
345 call-method call-next-method
)
347 ;; generic structures
349 empirical-statistical-variable
350 modelbased-statistical-variable
351 categorical-statistical-variable
352 nominal-statistical-variable
353 ordinal-statistical-variable
354 continuous-statistical-variable
356 ordering factor-levels nobs support pdmf draw
360 statistical-observation
361 measurement-types record
363 open-file-dialog read-data-file read-data-columns load-data
364 load-example
*variables
* *ask-on-redefine
*
365 def variables savevar undef
))
367 (defpackage :lisp-stat-descriptive-statistics
372 :lisp-stat-compound-data
374 (:shadowing-import-from
:lisp-stat-math
;; life is a vector!
375 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
376 asin acos atan sinh cosh tanh asinh acosh atanh float random
377 truncate floor ceiling round minusp zerop plusp evenp oddp
378 < <= = /= >= > ;; complex
379 conjugate realpart imagpart phase
380 min max logand logior logxor lognot ffloor fceiling
381 ftruncate fround signum cis
)
382 (:export mean standard-deviation variance
383 quantile median interquartile-range
386 (defpackage :lisp-stat-regression-linear
390 :lisp-stat-compound-data
391 :lisp-stat-descriptive-statistics
)
392 (:shadowing-import-from
:lisp-stat-object-system
393 call-method call-next-method
)
394 (:export regression-model fit-model
396 estimates covariance-matrix
397 ;; functions for helpers
399 print-object
;; for method dispatch
402 (defpackage :common-lisp-statistics
403 (:documentation
"Experimentation package for LispStat. Serious work
404 should be packaged up as a separate but similar package to help
405 drive reproducibility. By this I mean, creating a
406 data/analytics/analysis package with the minimal set of
407 objects/packages required.")
408 (:nicknames
:cls
:common-lisp-statistics
:lisp-stat
)
412 :xarray
;; generic reference -- internally supporting array, lol structs
414 :lisp-matrix
;; conversion to a more robust linalg approach
418 :lisp-stat-object-system
419 :lisp-stat-compound-data
420 :lisp-stat-probability
432 :lisp-stat-descriptive-statistics
433 :lisp-stat-regression-linear
435 ;; :cls-visualize-plplot
436 ;; :cls-visualize-cl2d
438 (:shadowing-import-from
:xarray slice
)
439 (:shadowing-import-from
:lisp-stat-object-system
440 call-method call-next-method
)
441 (:shadowing-import-from
:lisp-stat-math
442 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
443 asin acos atan sinh cosh tanh asinh acosh atanh float random
444 truncate floor ceiling round minusp zerop plusp evenp oddp
447 conjugate realpart imagpart phase
450 logand logior logxor lognot
451 ffloor fceiling ftruncate fround
455 *default-path
* *lsos-files
* *basic-files
* *ls-files
*
456 *cls-installation-home-dir
* *cls-data-dir
* *cls-examples-dir
*
460 defmeth send proto-slot-value
463 fixnump check-nonneg-fixnum check-one-fixnum
464 check-one-nonneg-fixnum
465 check-one-real check-one-number
473 compound-data-p
*compound-data-proto
* compound-object-p
474 compound-data-seq compound-data-length
475 element-list element-seq
477 recursive-map-elements map-elements
480 get-next-element make-next-element set-next-element
488 ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
489 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
490 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
492 conjugate realpart imagpart phase min max
493 logand logior logxor lognot ffloor fceiling ftruncate fround
496 #|
;; The following need to be re-found in lisp-matrix...
499 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
500 column-list inner-product outer-product cross-product transpose
501 bind-columns bind-rows
504 chol-decomp lu-decomp lu-solve determinant inverse
505 sv-decomp qr-decomp rcondest make-rotation spline
506 kernel-dens kernel-smooth
507 fft make-sweep-matrix sweep-operator ax
+y eigen
509 covariance-matrix matrix print-matrix solve
510 backsolve eigenvalues eigenvectors accumulate cumsum combine
513 ;; in linalg.lisp, possibly not supported by matlisp
514 spline kernel-dens kernel-smooth
521 ;; package LISPSTAT-MACROS
522 make-rv-function make-rv-function-1
525 xref xtype xdims xdim xdims
*
527 ;; package LISTOFLIST
528 sublists-of-same-size-p equal-listoflist transpose-listoflist
531 ;; need to take this list and make it symbols... specs could mean
532 ;; that we process the strings in different ways? Create such a
533 ;; macro to enable export within this package?
537 (mapc #'symbol-for-symbol-to-string-or-symbol
538 (do-external-symbols (s (find-package 'lisp-stat-data
) lst
) (push s lst
))))
541 open-file-dialog read-data-file read-data-columns load-data
542 load-example
*variables
* *ask-on-redefine
*
543 def variables savevar undef
545 dataframe-like dataframe-array make-dataframe
546 varlabels caselabels nrows ncols
547 dataframe-dimension dataframe-dimensons
548 dfref dfref-case dfref-var
549 consistent-dataframe-p
550 dataset list-of-columns list-of-rows
554 listoflist-
>dataframe listoflist-
>array listoflist-
>matrix-like
555 sublists-of-same-size-p
558 dsvstream-
>dataframe dsvstream-
>matrix dsvstream-
>listoflist
559 filename.dsv-
>dataframe
562 listoflist-
>dataframe dataframe-
>listoflist
563 listoflist-
>array array-
>listoflist
564 listoflist-
>matrix-like matrix-like-
>listoflist
567 ;; statistics.lsp (descriptions, should probably be moved
569 standard-deviation quantile median interquartile-range
572 ;; probability (dists.lisp)
574 uniform-rand normal-cdf normal-quant normal-dens
575 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
576 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
577 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
578 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
579 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
580 binomial-cdf binomial-quant binomial-pmf binomial-rand
582 ;; Here is where we have a problem -- lispstat core should be core
583 ;; data management and config problems, with packages providing
584 ;; specialized extensions to LispStat, i.e. regression, nonlin
585 ;; regression, bayesian regression via laplace approximation, etc.
587 ;; The following could be considered "recommended packages",
588 ;; similar to the idea of the recommended packages in R. Probably
589 ;; we want them to do the exporting within that package, therefore
590 ;; NOT being able to lock the "data-ish" package, but only the
591 ;; subpackages prior to export.
594 ;; -- linear regressin models.
595 regression-model fit-model
596 estimates covariance-matrix
598 regression-model-proto x y intercept sweep-matrix
599 basis weights included total-sum-of-squares residual-sum-of-squares
600 predictor-names response-name case-labels
604 ;; -- nonlinear regression models
605 nreg-model nreg-model-proto mean-function theta-hat epsilon
607 ;; we might need something like xtxinv here? But should be
608 ;; encapsulated, so we use the one in regression.lisp
611 bayes-model bayes-model-proto bayes-internals
623 ;;;; PACKAGES FOR USEABILITY
625 (defpackage :lisp-stat-data-examples
626 (:documentation
"Example data for unittests, examples, illustrations,")
628 :common-lisp-statistics
)
629 (:shadowing-import-from
:lisp-stat
630 call-method call-next-method
632 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
633 asin acos atan sinh cosh tanh asinh acosh atanh float random
634 truncate floor ceiling round minusp zerop plusp evenp oddp
635 < <= = /= >= > > ;; complex
636 conjugate realpart imagpart phase
637 min max logand logior logxor lognot ffloor fceiling
638 ftruncate fround signum cis
641 (:export iron aluminum absorbtion
645 (defpackage :lisp-stat-user
646 (:documentation
"Experimentation package for LispStat. Serious work
647 should be placed in a similar package elsewhere for
648 reproducibility. But this should hint as to what needs to be done
649 for a user- or analysis-package.")
650 (:nicknames
:ls-user
:cls-user
)
651 (:use
:common-lisp
; always needed for user playgrounds!
653 :common-lisp-statistics
654 :lisp-stat-data-examples
) ;; this last is to have 'things to play with'
655 (:shadowing-import-from
:lisp-stat
656 call-method call-next-method
658 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
659 asin acos atan sinh cosh tanh asinh acosh atanh float random
660 truncate floor ceiling round minusp zerop plusp evenp oddp
661 < <= = /= >= > > ;; complex
662 conjugate realpart imagpart phase
663 min max logand logior logxor lognot ffloor fceiling
664 ftruncate fround signum cis
668 (defpackage :lisp-stat-unittests
671 :lisp-stat
:lisp-stat-data-examples
)
672 (:shadowing-import-from
:lisp-stat
673 call-method call-next-method
;; objects
674 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
;; lsmath
675 asin acos atan sinh cosh tanh asinh acosh atanh float random
676 truncate floor ceiling round minusp zerop plusp evenp oddp
677 < <= = /= >= > ;; complex
678 conjugate realpart imagpart phase
679 min max logand logior logxor lognot ffloor fceiling
680 ftruncate fround signum cis
)
681 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard
; exec
682 almost
= almost
=lists numerical
=)) ; compare
684 (defpackage :cls-dataframe-example
686 :lift
:lisp-stat-unittests
687 :lisp-stat-data-examples
689 (:export absorbtion aluminum iron
))
692 (defpackage :lisp-stat-optimize
697 :lisp-stat-object-system
699 :lisp-stat-compound-data
705 :lisp-stat-linalg-data
709 (:shadowing-import-from
:lisp-stat-object-system
710 call-method call-next-method
)
711 (:shadowing-import-from
:lisp-stat-math
712 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
713 asin acos atan sinh cosh tanh asinh acosh atanh float random
714 truncate floor ceiling round minusp zerop plusp evenp oddp
715 < <= = /= >= > complex conjugate realpart imagpart phase
716 min max logand logior logxor lognot ffloor fceiling
717 ftruncate fround signum cis
)
723 newtonmax nelmeadmax
))