3 ;;; Time-stamp: <2010-01-03 16:21:25 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2010, AJ Rossini. MIT License. See
8 ;;; LICENSE.mit in top level directory for details.
9 ;;; Purpose: packages for Common Lisp Statistics
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
17 ;;; Current structure, dependencies:
19 '((:ls-user
:depends-on lisp-stat
)
20 (:lisp-stat
:depends-on
'(cls-dataframe
24 (:cls-data
:depends-on
)
25 (:cls-dataframe
:depends-on cls-data
)
34 (defpackage :lisp-stat-object-system
35 (:nicknames
:ls-objects
:lsos
)
37 (:shadow
:call-method
:call-next-method
)
38 (:export ls-object objectp
*object
* kind-of-p make-object
40 *set-slot-hook
* proto-slot-value self
41 send call-next-method call-method
42 defmeth defproto instance-slots proto-name
))
44 ;;; -types and -float probably ought to be moved into a -numerics
47 (defpackage :lisp-stat-types
48 (:documentation
"Provides some typeing for LispStat, but is clearly
52 check-nonneg-fixnum check-one-nonneg-fixnum
53 check-one-fixnum check-one-real check-one-number
))
55 (defpackage :lisp-stat-float
57 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
58 machine-epsilon base-float makedouble
60 make-base-trans-fun-2 make-base-trans-fun
62 base-log base-exp base-expt base-sqrt base-sin base-cos
63 base-tan base-asin base-acos base-atan base-sinh
64 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
65 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
66 BASE-FROUND BASE-SIGNUM BASE-CIS
))
68 ;;; Probably should move into cls-data package.
70 (defpackage :lisp-stat-compound-data
72 :lisp-stat-object-system
74 (:shadowing-import-from
:lisp-stat-object-system
75 call-next-method call-method
)
76 (:export compound-data-p
*compound-data-proto
*
78 compound-data-seq compound-data-length
79 element-list element-seq
81 recursive-map-elements map-elements repeat
83 get-next-element make-next-element set-next-element
86 select split-list which
90 (defpackage :lisp-stat-macros
92 :lisp-stat-compound-data
)
93 (:export make-rv-function make-rv-function-1
))
95 (defpackage :lisp-stat-basics
97 :lisp-stat-object-system
101 :lisp-stat-compound-data
)
102 (:shadowing-import-from
:lisp-stat-object-system
103 call-method call-next-method
)
104 (:export permute-array sum prod count-elements mean
107 (defpackage :lisp-stat-float
109 (:export
+stat-float-typing
+ +stat-cfloat-typing
+ +stat-float-template
+
110 machine-epsilon base-float makedouble
112 make-base-trans-fun-2 make-base-trans-fun
114 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
115 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
116 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
117 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
118 BASE-FROUND BASE-SIGNUM BASE-CIS
))
120 (defpackage :lisp-stat-macros
122 :lisp-stat-compound-data
)
123 (:export make-rv-function make-rv-function-1
))
126 (defpackage :cls-matrix
127 (:documentation
"basic utilities for using lisp arrays as numerical
128 matrices. Not optimized, and must consider this slow. Routines
129 should be optimized, it is only that we need them first, optimize
132 (:export matrixp num-rows num-cols matmult identity-matrix diagonal
133 row-list column-list inner-product outer-product
134 cross-product transpose bind-columns bind-rows
135 array-data-vector vector-to-array
))
138 ;;; NEW CLOS STRUCTURE
140 ;; CLS-DATA contains the basic variable structure, classes, support/indexing, and mixins.
141 ;; CLS-DATAFRAME leverages CLS-DATA, XARRAY, and LISP-MATRIX
142 ;; CLS-DATAIO stores/saves structures
143 ;; CLS-DATATRANS converts between structures, and from DATAFRAMES to MODEL-MATRIXES
145 (defpackage :cls-data
147 (:shadowing-import-from
:xarray slice
)
149 ;; must work out what is needed for stat variable metadata, support
153 ;;; cls-data... in dataframe, though.
154 (defpackage :cls-dataframe
159 (:shadowing-import-from
:xarray slice
)
161 ;; generic container class for data -- if small enough
162 ;; could be value, otherwise might be reference.
168 list-of-columns
;; list-of-variables
169 list-of-rows
;; list-of-observations
172 varlabels caselabels nrows ncols
173 dataframe-dimension dataframe-dimensons
174 xref xtype xdims xdim xrank slice take carray
180 (defpackage :cls-dataio
181 (:documentation
"Data I/O and similar import technologies.")
184 :lisp-stat-object-system
187 (:shadowing-import-from
:lisp-stat-object-system
188 call-method call-next-method
)
189 (:export dsvstream-
>dataframe
191 dsvstream-
>listoflist
193 filename.dsv-
>dataframe
196 (defpackage :cls-datatrans
197 (:documentation
"Data I/O and similar import technologies.")
200 :lisp-stat-object-system
203 (:shadowing-import-from
:lisp-stat-object-system
204 call-method call-next-method
)
205 (:export listoflist-
>dataframe dataframe-
>listoflist
206 listoflist-
>array array-
>listoflist
207 listoflist-
>matrix-like matrix-like-
>listoflist
213 (defpackage :lisp-stat-model
214 (:documentation
"Model management for data analysis.")
218 ;; data structures for model and model/data combination
219 model statistical-model analysis
))
223 (defpackage :cls-visualize
227 (:shadowing-import-from
:xarray slice
)
231 (defpackage :cls-visualize-plplot
238 plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex
3D-plot-ex
))
243 (defpackage :lisp-stat-ffi-int
246 (:export ccl-store-integer ccl-store-double ccl-store-ptr
249 (defpackage :lisp-stat-probability
254 (:export log-gamma set-seed
256 normal-cdf normal-quant normal-dens normal-rand
258 cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
259 gamma-cdf gamma-quant gamma-dens gamma-rand
260 chisq-cdf chisq-quant chisq-dens chisq-rand
261 beta-cdf beta-quant beta-dens beta-rand
262 t-cdf t-quant t-dens t-rand
263 f-cdf f-quant f-dens f-rand
264 poisson-cdf poisson-quant poisson-pmf poisson-rand
265 binomial-cdf binomial-quant binomial-pmf binomial-rand
))
269 (defpackage :lisp-stat-math
271 :lisp-stat-object-system
273 :lisp-stat-compound-data
275 (:shadowing-import-from
:lisp-stat-object-system
276 call-method call-next-method
)
277 (:shadow expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
278 asin acos atan sinh cosh tanh asinh acosh atanh float random
279 truncate floor ceiling round minusp zerop plusp evenp oddp
280 < <= = /= >= > ;; complex
281 conjugate realpart imagpart phase
282 min max logand logior logxor lognot ffloor fceiling
283 ftruncate fround signum cis
)
284 (:export ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
285 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
286 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
288 conjugate realpart imagpart phase min max
289 logand logior logxor lognot ffloor fceiling ftruncate fround
291 (:documentation
"Vectorization of numerical functions"))
294 #|
;; some of this goes back in, but not all of it?
295 (defpackage :lisp-stat-linalg
302 :lisp-stat-compound-data
)
303 (:shadowing-import-from
:lisp-stat-math
304 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
305 asin acos atan sinh cosh tanh asinh acosh atanh float random
306 truncate floor ceiling round minusp zerop plusp evenp oddp
307 < <= = /= >= > complex conjugate realpart imagpart phase
308 min max logand logior logxor lognot ffloor fceiling
309 ftruncate fround signum cis
)
310 (:export chol-decomp lu-decomp lu-solve determinant inverse
311 sv-decomp qr-decomp rcondest make-rotation spline
312 kernel-dens kernel-smooth
313 fft make-sweep-matrix sweep-operator ax
+y eigen
315 check-real
;; for optimize
317 covariance-matrix matrix print-matrix solve
318 backsolve eigenvalues eigenvectors accumulate cumsum combine
326 (defpackage :lisp-stat-data
327 (:documentation
"Data management, integration, I/O, and other data technologies.")
328 (:nicknames
:ls-data
)
330 :lisp-stat-object-system
333 :lisp-stat-compound-data
)
334 (:shadowing-import-from
:lisp-stat-object-system
335 call-method call-next-method
)
337 ;; generic structures
339 empirical-statistical-variable
340 modelbased-statistical-variable
341 categorical-statistical-variable
342 nominal-statistical-variable
343 ordinal-statistical-variable
344 continuous-statistical-variable
346 ordering factor-levels nobs support pdmf draw
350 statistical-observation
351 measurement-types record
353 open-file-dialog read-data-file read-data-columns load-data
354 load-example
*variables
* *ask-on-redefine
*
355 def variables savevar undef
))
357 (defpackage :lisp-stat-descriptive-statistics
362 :lisp-stat-compound-data
364 (:shadowing-import-from
:lisp-stat-math
;; life is a vector!
365 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
366 asin acos atan sinh cosh tanh asinh acosh atanh float random
367 truncate floor ceiling round minusp zerop plusp evenp oddp
368 < <= = /= >= > ;; complex
369 conjugate realpart imagpart phase
370 min max logand logior logxor lognot ffloor fceiling
371 ftruncate fround signum cis
)
372 (:export mean standard-deviation variance
373 quantile median interquartile-range
376 (defpackage :lisp-stat-regression-linear
380 :lisp-stat-compound-data
381 :lisp-stat-descriptive-statistics
)
382 (:shadowing-import-from
:lisp-stat-object-system
383 call-method call-next-method
)
384 (:export regression-model fit-model
386 estimates covariance-matrix
387 ;; functions for helpers
389 print-object
;; for method dispatch
392 (defpackage :common-lisp-statistics
393 (:documentation
"Experimentation package for LispStat. Serious work
394 should be packaged up as a separate but similar package to help
395 drive reproducibility. By this I mean, creating a
396 data/analytics/analysis package with the minimal set of
397 objects/packages required.")
398 (:nicknames
:cls
:common-lisp-statistics
:lisp-stat
)
402 :xarray
;; generic reference -- internally supporting array, lol structs
404 :lisp-matrix
;; conversion to a more robust linalg approach
408 :lisp-stat-object-system
409 :lisp-stat-compound-data
410 :lisp-stat-probability
422 :lisp-stat-descriptive-statistics
423 :lisp-stat-regression-linear
425 ;; :cls-visualize-plplot
426 ;; :cls-visualize-cl2d
428 (:shadowing-import-from
:xarray slice
)
429 (:shadowing-import-from
:lisp-stat-object-system
430 call-method call-next-method
)
431 (:shadowing-import-from
:lisp-stat-math
432 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
433 asin acos atan sinh cosh tanh asinh acosh atanh float random
434 truncate floor ceiling round minusp zerop plusp evenp oddp
437 conjugate realpart imagpart phase
440 logand logior logxor lognot
441 ffloor fceiling ftruncate fround
445 *default-path
* *lsos-files
* *basic-files
* *ls-files
*
446 *cls-home-dir
* *cls-data-dir
* *cls-examples-dir
*
450 defmeth send proto-slot-value
453 fixnump check-nonneg-fixnum check-one-fixnum
454 check-one-nonneg-fixnum
455 check-one-real check-one-number
463 compound-data-p
*compound-data-proto
* compound-object-p
464 compound-data-seq compound-data-length
465 element-list element-seq
467 recursive-map-elements map-elements
470 get-next-element make-next-element set-next-element
478 ^
** expt
+ -
* / mod rem pmin pmax abs
1+ 1- log exp sqrt sin cos
479 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
480 truncate floor ceiling round minusp zerop plusp evenp oddp
< <= =
482 conjugate realpart imagpart phase min max
483 logand logior logxor lognot ffloor fceiling ftruncate fround
486 #|
;; The following need to be re-found in lisp-matrix...
489 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
490 column-list inner-product outer-product cross-product transpose
491 bind-columns bind-rows
494 chol-decomp lu-decomp lu-solve determinant inverse
495 sv-decomp qr-decomp rcondest make-rotation spline
496 kernel-dens kernel-smooth
497 fft make-sweep-matrix sweep-operator ax
+y eigen
499 covariance-matrix matrix print-matrix solve
500 backsolve eigenvalues eigenvectors accumulate cumsum combine
503 ;; in linalg.lisp, possibly not supported by matlisp
504 spline kernel-dens kernel-smooth
511 ;; package LISPSTAT-MACROS
512 make-rv-function make-rv-function-1
515 xref xtype xdims xdim xdims
*
517 ;; package LISTOFLIST
518 sublists-of-same-size-p equal-listoflist transpose-listoflist
521 ;; need to take this list and make it symbols... specs could mean
522 ;; that we process the strings in different ways? Create such a
523 ;; macro to enable export within this package?
527 (mapc #'symbol-for-symbol-to-string-or-symbol
528 (do-external-symbols (s (find-package 'lisp-stat-data
) lst
) (push s lst
))))
531 open-file-dialog read-data-file read-data-columns load-data
532 load-example
*variables
* *ask-on-redefine
*
533 def variables savevar undef
535 dataframe-like dataframe-array make-dataframe
536 varlabels caselabels nrows ncols
537 dataframe-dimension dataframe-dimensons
538 dfref dfref-case dfref-var
539 consistent-dataframe-p
540 dataset list-of-columns list-of-rows
544 listoflist-
>dataframe listoflist-
>array listoflist-
>matrix-like
545 sublists-of-same-size-p
548 dsvstream-
>dataframe dsvstream-
>matrix dsvstream-
>listoflist
549 filename.dsv-
>dataframe
552 listoflist-
>dataframe dataframe-
>listoflist
553 listoflist-
>array array-
>listoflist
554 listoflist-
>matrix-like matrix-like-
>listoflist
557 ;; statistics.lsp (descriptions, should probably be moved
559 standard-deviation quantile median interquartile-range
562 ;; probability (dists.lisp)
564 uniform-rand normal-cdf normal-quant normal-dens
565 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
566 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
567 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
568 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
569 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
570 binomial-cdf binomial-quant binomial-pmf binomial-rand
572 ;; Here is where we have a problem -- lispstat core should be core
573 ;; data management and config problems, with packages providing
574 ;; specialized extensions to LispStat, i.e. regression, nonlin
575 ;; regression, bayesian regression via laplace approximation, etc.
577 ;; The following could be considered "recommended packages",
578 ;; similar to the idea of the recommended packages in R. Probably
579 ;; we want them to do the exporting within that package, therefore
580 ;; NOT being able to lock the "data-ish" package, but only the
581 ;; subpackages prior to export.
584 ;; -- linear regressin models.
585 regression-model fit-model
586 estimates covariance-matrix
588 regression-model-proto x y intercept sweep-matrix
589 basis weights included total-sum-of-squares residual-sum-of-squares
590 predictor-names response-name case-labels
594 ;; -- nonlinear regression models
595 nreg-model nreg-model-proto mean-function theta-hat epsilon
597 ;; we might need something like xtxinv here? But should be
598 ;; encapsulated, so we use the one in regression.lisp
601 bayes-model bayes-model-proto bayes-internals
613 ;;;; PACKAGES FOR USEABILITY
615 (defpackage :lisp-stat-data-examples
616 (:documentation
"Example data for unittests, examples, illustrations,")
618 :common-lisp-statistics
)
619 (:shadowing-import-from
:lisp-stat
620 call-method call-next-method
622 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
623 asin acos atan sinh cosh tanh asinh acosh atanh float random
624 truncate floor ceiling round minusp zerop plusp evenp oddp
625 < <= = /= >= > > ;; complex
626 conjugate realpart imagpart phase
627 min max logand logior logxor lognot ffloor fceiling
628 ftruncate fround signum cis
631 (:export iron aluminum absorbtion
635 (defpackage :lisp-stat-user
636 (:documentation
"Experimentation package for LispStat. Serious work
637 should be placed in a similar package elsewhere for
638 reproducibility. But this should hint as to what needs to be done
639 for a user- or analysis-package.")
640 (:nicknames
:ls-user
:cls-user
)
641 (:use
:common-lisp
; always needed for user playgrounds!
643 :common-lisp-statistics
644 :lisp-stat-data-examples
) ;; this last is to have 'things to play with'
645 (:shadowing-import-from
:lisp-stat
646 call-method call-next-method
648 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
649 asin acos atan sinh cosh tanh asinh acosh atanh float random
650 truncate floor ceiling round minusp zerop plusp evenp oddp
651 < <= = /= >= > > ;; complex
652 conjugate realpart imagpart phase
653 min max logand logior logxor lognot ffloor fceiling
654 ftruncate fround signum cis
658 (defpackage :lisp-stat-unittests
661 :lisp-stat
:lisp-stat-data-examples
)
662 (:shadowing-import-from
:lisp-stat
663 call-method call-next-method
;; objects
664 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
;; lsmath
665 asin acos atan sinh cosh tanh asinh acosh atanh float random
666 truncate floor ceiling round minusp zerop plusp evenp oddp
667 < <= = /= >= > ;; complex
668 conjugate realpart imagpart phase
669 min max logand logior logxor lognot ffloor fceiling
670 ftruncate fround signum cis
)
671 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard
; exec
672 almost
= almost
=lists numerical
=)) ; compare
674 (defpackage :cls-dataframe-example
676 :lift
:lisp-stat-unittests
677 :lisp-stat-data-examples
679 (:export absorbtion aluminum iron
))
682 (defpackage :lisp-stat-optimize
687 :lisp-stat-object-system
689 :lisp-stat-compound-data
695 :lisp-stat-linalg-data
699 (:shadowing-import-from
:lisp-stat-object-system
700 call-method call-next-method
)
701 (:shadowing-import-from
:lisp-stat-math
702 expt
+ -
* / ** mod rem abs
1+ 1- log exp sqrt sin cos tan
703 asin acos atan sinh cosh tanh asinh acosh atanh float random
704 truncate floor ceiling round minusp zerop plusp evenp oddp
705 < <= = /= >= > complex conjugate realpart imagpart phase
706 min max logand logior logxor lognot ffloor fceiling
707 ftruncate fround signum cis
)
713 newtonmax nelmeadmax
))