cleaning up work supporting xarray changes.
[CommonLispStat.git] / src / packages.lisp
blob2826a9b4b66c4c5b82c70a95dc42261445287c34
1 ;;; -*- mode: lisp -*-
3 ;;; Time-stamp: <2009-09-02 08:00:25 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2008, AJ Rossini. BSD, LLGPL, or GPLv2, depending
8 ;;; on how it arrives.
9 ;;; Purpose: package structure description for lispstat
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
15 (in-package :cl-user)
17 ;;; Basics
20 (defpackage :lisp-stat-object-system
21 (:nicknames :ls-objects :lsos)
22 (:use :common-lisp)
23 (:shadow :call-method :call-next-method)
24 (:export ls-object objectp *object* kind-of-p make-object
25 *message-hook*
26 *set-slot-hook* proto-slot-value self
27 send call-next-method call-method
28 defmeth defproto instance-slots proto-name))
30 ;;; -types and -float probably ought to be moved into a -numerics
31 ;;; package.
33 (defpackage :lisp-stat-types
34 (:documentation "Provides some typeing for LispStat, but is clearly
35 a bit incomplete.")
36 (:use :common-lisp)
37 (:export fixnump
38 check-nonneg-fixnum check-one-nonneg-fixnum
39 check-one-fixnum check-one-real check-one-number))
41 (defpackage :lisp-stat-float
42 (:use :common-lisp)
43 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
44 machine-epsilon base-float makedouble
46 make-base-trans-fun-2 make-base-trans-fun
48 base-log base-exp base-expt base-sqrt base-sin base-cos
49 base-tan base-asin base-acos base-atan base-sinh
50 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
51 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
52 BASE-FROUND BASE-SIGNUM BASE-CIS))
54 ;;; Probably should move into cls-data package.
56 (defpackage :lisp-stat-compound-data
57 (:use :common-lisp
58 :lisp-stat-object-system
59 :lisp-stat-types)
60 (:shadowing-import-from :lisp-stat-object-system
61 call-next-method call-method)
62 (:export compound-data-p *compound-data-proto*
63 compound-object-p
64 compound-data-seq compound-data-length
65 element-list element-seq
66 sort-data order rank
67 recursive-map-elements map-elements repeat
68 check-sequence
69 get-next-element make-next-element set-next-element
70 ;; sequencep
71 iseq ordered-nneg-seq
72 select split-list which
73 difference rseq
74 flatten-list))
76 (defpackage :lisp-stat-macros
77 (:use :common-lisp
78 :lisp-stat-compound-data)
79 (:export make-rv-function make-rv-function-1))
81 (defpackage :lisp-stat-basics
82 (:use :common-lisp
83 :lisp-stat-object-system
84 :lisp-stat-types
85 :lisp-stat-float
86 :lisp-stat-macros
87 :lisp-stat-compound-data)
88 (:shadowing-import-from :lisp-stat-object-system
89 call-method call-next-method)
90 (:export permute-array sum prod count-elements mean
91 if-else sample))
93 (defpackage :lisp-stat-float
94 (:use :common-lisp)
95 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
96 machine-epsilon base-float makedouble
98 make-base-trans-fun-2 make-base-trans-fun
100 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
101 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
102 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
103 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
104 BASE-FROUND BASE-SIGNUM BASE-CIS))
106 (defpackage :lisp-stat-macros
107 (:use :common-lisp
108 :lisp-stat-compound-data)
109 (:export make-rv-function make-rv-function-1))
112 (defpackage :cls-matrix
113 (:documentation "basic utilities for using lisp arrays as numerical
114 matrices. Not optimized, and must consider this slow. Routines
115 should be optimized, it is only that we need them first, optimize
116 them later.")
117 (:use :common-lisp)
118 (:export matrixp num-rows num-cols matmult identity-matrix diagonal
119 row-list column-list inner-product outer-product
120 cross-product transpose bind-columns bind-rows
121 array-data-vector vector-to-array))
124 ;;; NEW CLOS STRUCTURE
126 ;;; cls-data... in dataframe, though.
127 (defpackage :cls-dataframe
128 (:use :common-lisp
129 :xarray
130 :lisp-matrix)
131 (:shadowing-import-from :xarray slice)
132 (:export
133 ;; generic container class for data -- if small enough
134 ;; could be value, otherwise might be reference.
135 dataframe-like
136 dataframe-array
137 make-dataframe
139 ;; accessors
140 varlabels caselabels nrows ncols
141 dataframe-dimension dataframe-dimensons
142 xref xtype xdims xdim xrank slice take carray
144 dfref dfref-case dfref-var
145 consistent-dataframe-p
148 dataset
149 list-of-columns ;; list-of-variables
150 list-of-rows ;; list-of-observations
153 (defpackage :cls-data
154 (:use :common-lisp
155 :xarray
156 :lisp-matrix
157 :cls-dataframe) ; for dataframe
158 (:shadowing-import-from :xarray slice)
159 (:export listoflist->dataframe
160 listoflist->array
161 listoflist->matrix-like))
163 lists-of-same-size
164 equal-listoflist
165 transpose-listoflist
168 (defpackage :cls-dataimport
169 (:documentation "Data I/O and similar import technologies.")
170 (:use :common-lisp
171 :lisp-stat-object-system
172 :cls-dataframe
173 :cls-data
174 :rsm.string)
175 (:shadowing-import-from :lisp-stat-object-system
176 call-method call-next-method)
177 (:export dsvstream->dataframe dsvstream->matrix dsvstream->listoflist))
180 (defpackage :lisp-stat-model
181 (:documentation "Model management for data analysis.")
182 (:use :common-lisp
183 :lisp-matrix)
184 (:export
185 ;; data structures for model and model/data combination
186 model statistical-model analysis))
188 ;;; visualization
190 (defpackage :cls-visualize
191 (:use :common-lisp
192 :lisp-matrix
193 :cls-dataframe)
194 (:shadowing-import-from :xarray slice)
198 (defpackage :cls-visualize-plplot
199 (:use :common-lisp
200 :lisp-matrix
201 :cls-dataframe
202 :cl-plplot-system)
203 (:export
204 ;; examples
205 plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex 3D-plot-ex))
208 ;;; USER PACKAGES
210 (defpackage :lisp-stat-ffi-int
211 (:use :common-lisp
212 :cffi)
213 (:export ccl-store-integer ccl-store-double ccl-store-ptr
214 get-buf ))
216 (defpackage :lisp-stat-probability
217 (:use :common-lisp
218 :cffi
219 :lisp-stat-ffi-int
220 :lisp-stat-macros)
221 (:export log-gamma set-seed
222 uniform-rand
223 normal-cdf normal-quant normal-dens normal-rand
224 bivnorm-cdf
225 cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
226 gamma-cdf gamma-quant gamma-dens gamma-rand
227 chisq-cdf chisq-quant chisq-dens chisq-rand
228 beta-cdf beta-quant beta-dens beta-rand
229 t-cdf t-quant t-dens t-rand
230 f-cdf f-quant f-dens f-rand
231 poisson-cdf poisson-quant poisson-pmf poisson-rand
232 binomial-cdf binomial-quant binomial-pmf binomial-rand))
236 (defpackage :lisp-stat-math
237 (:use :common-lisp
238 :lisp-stat-object-system
239 :lisp-stat-macros
240 :lisp-stat-compound-data
241 :lisp-stat-float)
242 (:shadowing-import-from :lisp-stat-object-system
243 call-method call-next-method)
244 (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
245 asin acos atan sinh cosh tanh asinh acosh atanh float random
246 truncate floor ceiling round minusp zerop plusp evenp oddp
247 < <= = /= >= > ;; complex
248 conjugate realpart imagpart phase
249 min max logand logior logxor lognot ffloor fceiling
250 ftruncate fround signum cis)
251 (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
252 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
253 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
254 /= >= > ;; complex
255 conjugate realpart imagpart phase min max
256 logand logior logxor lognot ffloor fceiling ftruncate fround
257 signum cis)
258 (:documentation "Vectorization of numerical functions"))
261 #| ;; some of this goes back in, but not all of it?
262 (defpackage :lisp-stat-linalg
263 (:use :common-lisp
264 :cffi
265 :lisp-matrix
266 :lisp-stat-math
267 :lisp-stat-types
268 :lisp-stat-float
269 :lisp-stat-compound-data)
270 (:shadowing-import-from :lisp-stat-math
271 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
272 asin acos atan sinh cosh tanh asinh acosh atanh float random
273 truncate floor ceiling round minusp zerop plusp evenp oddp
274 < <= = /= >= > complex conjugate realpart imagpart phase
275 min max logand logior logxor lognot ffloor fceiling
276 ftruncate fround signum cis)
277 (:export chol-decomp lu-decomp lu-solve determinant inverse
278 sv-decomp qr-decomp rcondest make-rotation spline
279 kernel-dens kernel-smooth
280 fft make-sweep-matrix sweep-operator ax+y eigen
282 check-real ;; for optimize
284 covariance-matrix matrix print-matrix solve
285 backsolve eigenvalues eigenvectors accumulate cumsum combine
286 lowess))
293 (defpackage :lisp-stat-data
294 (:documentation "Data management, integration, I/O, and other data technologies.")
295 (:nicknames :ls-data)
296 (:use :common-lisp
297 :lisp-stat-object-system
298 :lisp-stat-config
299 :lisp-stat-types
300 :lisp-stat-compound-data)
301 (:shadowing-import-from :lisp-stat-object-system
302 call-method call-next-method)
303 (:export open-file-dialog read-data-file read-data-columns load-data
304 load-example *variables* *ask-on-redefine*
305 def variables savevar undef))
307 (defpackage :lisp-stat-descriptive-statistics
308 (:use :common-lisp
309 :lisp-matrix
310 :lisp-stat-data
311 :lisp-stat-math
312 :lisp-stat-compound-data
313 :lisp-stat-basics)
314 (:shadowing-import-from :lisp-stat-math ;; life is a vector!
315 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
316 asin acos atan sinh cosh tanh asinh acosh atanh float random
317 truncate floor ceiling round minusp zerop plusp evenp oddp
318 < <= = /= >= > ;; complex
319 conjugate realpart imagpart phase
320 min max logand logior logxor lognot ffloor fceiling
321 ftruncate fround signum cis)
322 (:export mean standard-deviation variance
323 quantile median interquartile-range
324 fivnum sample))
326 (defpackage :lisp-stat-regression-linear
327 (:use :common-lisp
328 :lisp-matrix
329 :lisp-stat-basics
330 :lisp-stat-compound-data
331 :lisp-stat-descriptive-statistics )
332 (:shadowing-import-from :lisp-stat-object-system
333 call-method call-next-method)
334 (:export regression-model fit-model
336 estimates covariance-matrix
337 ;; functions for helpers
338 lm xtxinv
339 print-object ;; for method dispatch
342 (defpackage :common-lisp-statistics
343 (:documentation "Experimentation package for LispStat. Serious work
344 should be packaged up as a separate but similar package to help
345 drive reproducibility. By this I mean, creating a
346 data/analytics/analysis package with the minimal set of
347 objects/packages required.")
348 (:nicknames :cls :common-lisp-statistics :lisp-stat)
349 (:use :common-lisp
350 :xarray ;; generic reference -- internally supporting array, lol structs
351 :lisp-matrix ;; conversion to a more robust linalg approach
352 :lisp-stat-config
353 :lisp-stat-object-system
354 :lisp-stat-compound-data
355 :lisp-stat-probability
356 :lisp-stat-types
357 :lisp-stat-float
358 :lisp-stat-basics
359 :lisp-stat-data
360 :cls-dataframe
361 :cls-data
362 :lisp-stat-math
363 :lisp-stat-descriptive-statistics
364 :lisp-stat-regression-linear
365 :cls-visualize
366 ;; :cybertiggyr-dsv
367 ;; :cls-visualize-plplot
369 (:shadowing-import-from :xarray slice)
370 (:shadowing-import-from :lisp-stat-object-system
371 call-method call-next-method)
372 (:shadowing-import-from :lisp-stat-math
373 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
374 asin acos atan sinh cosh tanh asinh acosh atanh float random
375 truncate floor ceiling round minusp zerop plusp evenp oddp
376 < <= = /= >= >
377 ;;complex
378 conjugate realpart imagpart phase
380 min max
381 logand logior logxor lognot
382 ffloor fceiling ftruncate fround
383 signum cis)
384 (:export
385 ;; lisp-stat-config:
386 *default-path* *lsos-files* *basic-files* *ls-files*
387 *lispstat-home-dir* *lispstat-data-dir* *lispstat-examples-dir*
389 ;; lsobjects :
390 defproto defproto2
391 defmeth send proto-slot-value
393 ;; lstypes :
394 fixnump check-nonneg-fixnum check-one-fixnum
395 check-one-nonneg-fixnum
396 check-one-real check-one-number
398 ;; lsmacros:
400 ;; lsfloat :
401 machine-epsilon
403 ;; compound :
404 compound-data-p *compound-data-proto* compound-object-p
405 compound-data-seq compound-data-length
406 element-list element-seq
407 sort-data order rank
408 recursive-map-elements map-elements
409 repeat
410 check-sequence
411 get-next-element make-next-element set-next-element
412 ;; sequencep
413 iseq
414 ordered-nneg-seq
415 select which
416 difference rseq
418 ;; lsmath.lsp
419 ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
420 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
421 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
422 /= >= > ;; complex
423 conjugate realpart imagpart phase min max
424 logand logior logxor lognot ffloor fceiling ftruncate fround
425 signum cis
427 #| ;; The following need to be re-found in lisp-matrix...
429 ;; matrices.lisp
430 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
431 column-list inner-product outer-product cross-product transpose
432 bind-columns bind-rows
434 ;; linalg.lisp
435 chol-decomp lu-decomp lu-solve determinant inverse
436 sv-decomp qr-decomp rcondest make-rotation spline
437 kernel-dens kernel-smooth
438 fft make-sweep-matrix sweep-operator ax+y eigen
439 check-real
440 covariance-matrix matrix print-matrix solve
441 backsolve eigenvalues eigenvectors accumulate cumsum combine
442 lowess
444 ;; in linalg.lisp, possibly not supported by matlisp
445 spline kernel-dens kernel-smooth
449 ;; optimize.lsp
450 newtonmax nelmeadmax
452 ;; lispstat-macros
453 make-rv-function make-rv-function-1
455 ;; xarray
456 xref xtype xdims xdim xdims*
457 lists-of-same-size equal-listoflist transpose-listoflist
459 ;; data
460 open-file-dialog read-data-file read-data-columns load-data
461 load-example *variables* *ask-on-redefine*
462 def variables savevar undef
464 ;; dataframe
465 dataframe-like dataframe-array make-dataframe
466 varlabels caselabels nrows ncols
467 dataframe-dimension dataframe-dimensons
468 dfref dfref-case dfref-var
469 consistent-dataframe-p
470 dataset list-of-columns list-of-rows
472 ;; listoflist
473 listoflist->dataframe listoflist->array listoflist->matrix-like
475 ;; statistics.lsp (descriptions, should probably be moved
476 ;; later...?
477 standard-deviation quantile median interquartile-range
478 fivnum sample
480 ;; probability (dists.lisp)
481 log-gamma set-seed
482 uniform-rand normal-cdf normal-quant normal-dens
483 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
484 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
485 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
486 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
487 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
488 binomial-cdf binomial-quant binomial-pmf binomial-rand
490 ;; Here is where we have a problem -- lispstat core should be core
491 ;; data management and config problems, with packages providing
492 ;; specialized extensions to LispStat, i.e. regression, nonlin
493 ;; regression, bayesian regression via laplace approximation, etc.
495 ;; The following could be considered "recommended packages",
496 ;; similar to the idea of the recommended packages in R. Probably
497 ;; we want them to do the exporting within that package, therefore
498 ;; NOT being able to lock the "data-ish" package, but only the
499 ;; subpackages prior to export.
501 ;; regression.lsp
502 ;; -- linear regressin models.
503 regression-model fit-model
504 estimates covariance-matrix
506 regression-model-proto x y intercept sweep-matrix
507 basis weights included total-sum-of-squares residual-sum-of-squares
508 predictor-names response-name case-labels
509 lm xtxinv
511 ;; nonlin.lsp
512 ;; -- nonlinear regression models
513 nreg-model nreg-model-proto mean-function theta-hat epsilon
514 count-limit verbose
515 ;; we might need something like xtxinv here? But should be
516 ;; encapsulated, so we use the one in regression.lisp
518 ;; bayes.lsp
519 bayes-model bayes-model-proto bayes-internals
521 ;; plots.lisp
522 plot-ex
523 contour-plot-ex
524 fn-contour-plot-ex
525 shade-plot-ex
526 3D-plot-ex
531 ;;;; PACKAGES FOR USEABILITY
533 (defpackage :lisp-stat-data-examples
534 (:documentation "Example data for unittests, examples, illustrations,")
535 (:use :common-lisp
536 :common-lisp-statistics)
537 (:shadowing-import-from :lisp-stat
538 call-method call-next-method
540 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
541 asin acos atan sinh cosh tanh asinh acosh atanh float random
542 truncate floor ceiling round minusp zerop plusp evenp oddp
543 < <= = /= >= > > ;; complex
544 conjugate realpart imagpart phase
545 min max logand logior logxor lognot ffloor fceiling
546 ftruncate fround signum cis
548 <= float imagpart)
549 (:export iron aluminum absorbtion
550 diabetes dlabs))
553 (defpackage :lisp-stat-user
554 (:documentation "Experimentation package for LispStat. Serious work
555 should be placed in a similar package elsewhere for
556 reproducibility. But this should hint as to what needs to be done
557 for a user- or analysis-package.")
558 (:nicknames :ls-user)
559 (:use :common-lisp ; always needed for user playgrounds!
560 :lisp-matrix
561 :common-lisp-statistics
562 :lisp-stat-data-examples) ;; this last is to have 'things to play with'
563 (:shadowing-import-from :lisp-stat
564 call-method call-next-method
566 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
567 asin acos atan sinh cosh tanh asinh acosh atanh float random
568 truncate floor ceiling round minusp zerop plusp evenp oddp
569 < <= = /= >= > > ;; complex
570 conjugate realpart imagpart phase
571 min max logand logior logxor lognot ffloor fceiling
572 ftruncate fround signum cis
574 <= float imagpart))
576 (defpackage :lisp-stat-unittests
577 (:use :common-lisp
578 :lift :lisp-matrix
579 :lisp-stat :lisp-stat-data-examples)
580 (:shadowing-import-from :lisp-stat
581 call-method call-next-method ;; objects
582 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
583 asin acos atan sinh cosh tanh asinh acosh atanh float random
584 truncate floor ceiling round minusp zerop plusp evenp oddp
585 < <= = /= >= > ;; complex
586 conjugate realpart imagpart phase
587 min max logand logior logxor lognot ffloor fceiling
588 ftruncate fround signum cis)
589 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
590 almost= almost=lists numerical=)) ; compare
592 (defpackage :cls-dataframe-example
593 (:use :common-lisp
594 :lift :lisp-stat-unittests
595 :lisp-stat-data-examples
596 :cls-dataframe)
597 (:export absorbtion aluminum iron))
600 (defpackage :lisp-stat-optimize
601 (:use :common-lisp
602 :cffi
603 :lisp-matrix
604 :lisp-stat-ffi-int
605 :lisp-stat-object-system
606 :lisp-stat-types
607 :lisp-stat-compound-data
608 :lisp-stat-math
609 :lisp-stat-float
610 :lisp-stat-basics
612 :lisp-stat-matrix
613 :lisp-stat-linalg-data
614 :lisp-stat-linalg
617 (:shadowing-import-from :lisp-stat-object-system
618 call-method call-next-method)
619 (:shadowing-import-from :lisp-stat-math
620 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
621 asin acos atan sinh cosh tanh asinh acosh atanh float random
622 truncate floor ceiling round minusp zerop plusp evenp oddp
623 < <= = /= >= > complex conjugate realpart imagpart phase
624 min max logand logior logxor lognot ffloor fceiling
625 ftruncate fround signum cis)
626 (:export
627 ;; derivatives
628 numgrad numhess
630 ;; optimization
631 newtonmax nelmeadmax))