data import, using RSM strings.
[CommonLispStat.git] / src / packages.lisp
blobc5169ed7b9a18961a956080da141cff6be2c9568
1 ;;; -*- mode: lisp -*-
3 ;;; Time-stamp: <2009-04-17 18:23:10 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2008, AJ Rossini. BSD, LLGPL, or GPLv2, depending
8 ;;; on how it arrives.
9 ;;; Purpose: package structure description for lispstat
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
15 ;;; This organization and structure is new to the 21st Century
16 ;;; version.
18 (in-package :cl-user)
20 ;;; Basics
23 (defpackage :lisp-stat-object-system
24 (:nicknames :ls-objects :lsos)
25 (:use :common-lisp)
26 (:shadow :call-method :call-next-method)
27 (:export ls-object objectp *object* kind-of-p make-object
28 *message-hook*
29 *set-slot-hook* proto-slot-value self
30 send call-next-method call-method
31 defmeth defproto instance-slots proto-name))
33 (defpackage :lisp-stat-types
34 (:documentation "Provides some typeing for LispStat, but is clearly
35 a bit incomplete.")
36 (:use :common-lisp)
37 (:export fixnump
38 check-nonneg-fixnum check-one-nonneg-fixnum
39 check-one-fixnum check-one-real check-one-number))
41 (defpackage :lisp-stat-float
42 (:use :common-lisp)
43 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
44 machine-epsilon base-float makedouble
46 make-base-trans-fun-2 make-base-trans-fun
48 base-log base-exp base-expt base-sqrt base-sin base-cos
49 base-tan base-asin base-acos base-atan base-sinh
50 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
51 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
52 BASE-FROUND BASE-SIGNUM BASE-CIS))
54 (defpackage :lisp-stat-compound-data
55 (:use :common-lisp
56 :lisp-stat-object-system
57 :lisp-stat-types)
58 (:shadowing-import-from :lisp-stat-object-system
59 call-next-method call-method)
60 (:export compound-data-p *compound-data-proto*
61 compound-object-p
62 compound-data-seq compound-data-length
63 element-list element-seq
64 sort-data order rank
65 recursive-map-elements map-elements repeat
66 check-sequence
67 get-next-element make-next-element set-next-element
68 ;; sequencep
69 iseq ordered-nneg-seq
70 select split-list which
71 difference rseq
72 flatten-list))
74 (defpackage :lisp-stat-macros
75 (:use :common-lisp
76 :lisp-stat-compound-data)
77 (:export make-rv-function make-rv-function-1))
79 (defpackage :lisp-stat-basics
80 (:use :common-lisp
81 :lisp-stat-object-system
82 :lisp-stat-types
83 :lisp-stat-float
84 :lisp-stat-macros
85 :lisp-stat-compound-data)
86 (:shadowing-import-from :lisp-stat-object-system
87 call-method call-next-method)
88 (:export permute-array sum prod count-elements mean
89 if-else sample))
91 (defpackage :lisp-stat-float
92 (:use :common-lisp)
93 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
94 machine-epsilon base-float makedouble
96 make-base-trans-fun-2 make-base-trans-fun
98 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
99 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
100 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
101 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
102 BASE-FROUND BASE-SIGNUM BASE-CIS))
104 (defpackage :lisp-stat-macros
105 (:use :common-lisp
106 :lisp-stat-compound-data)
107 (:export make-rv-function make-rv-function-1))
109 ;;; NEW CLOS STRUCTURE
113 (defpackage :lisp-stat-dataframe
114 (:use :common-lisp
115 :lisp-matrix)
116 (:export
117 ;; generic container class for data -- if small enough
118 ;; could be value, otherwise might be reference.
119 dataframe-like
120 dataframe-array
121 make-dataframe
123 ;; accessors
124 varlabels caselabels nrows ncols
125 dataframe-dimension dataframe-dimensons
126 dfref dfref-case dfref-var
127 consistent-dataframe-p
129 dataset
130 list-of-columns ;; list-of-variables
131 list-of-rows ;; list-of-observations
135 (defpackage :cls-data-listoflist
136 (:use :common-lisp
137 :lisp-matrix
138 :lisp-stat-dataframe) ; for dataframe
139 (:export lists-of-same-size
140 equal-listoflist
141 transpose-listoflist
142 listoflist->dataframe
143 listoflist->array
144 listoflist->matrix-like))
147 (defpackage :cls-dataimport
148 (:documentation "Data I/O and similar import technologies.")
149 (:use :common-lisp
150 :lisp-stat-object-system
151 :cls-dataframe
152 :cls-data-listoflist
153 :rsm.string)
154 (:shadowing-import-from :lisp-stat-object-system
155 call-method call-next-method)
156 (:export dsvstream->dataframe dsvstream->matrix dsvstream->listoflist))
159 (defpackage :lisp-stat-model
160 (:documentation "Model management for data analysis.")
161 (:use :common-lisp
162 :lisp-matrix)
163 (:export
164 ;; data structures for model and model/data combination
165 model statistical-model analysis))
168 (defpackage :lisp-stat-regression-linear-clos
169 (:use :common-lisp
170 :lisp-matrix
171 :lisp-stat-dataframe)
172 (:export regression-model))
175 (defpackage :cls-visualize
176 (:use :common-lisp
177 :lisp-matrix
178 :lisp-stat-dataframe))
180 (defpackage :cls-visualize-plplot
181 (:use :common-lisp
182 :lisp-matrix
183 :lisp-stat-dataframe
184 :cl-plplot-system)
185 (:export
186 plot-ex
187 contour-plot-ex
188 fn-contour-plot-ex
189 shade-plot-ex
190 3D-plot-ex ))
193 ;;; USER PACKAGES
195 (defpackage :lisp-stat-ffi-int
196 (:use :common-lisp
197 :cffi)
198 (:export ccl-store-integer ccl-store-double ccl-store-ptr
199 get-buf ))
201 (defpackage :lisp-stat-probability
202 (:use :common-lisp
203 :cffi
204 :lisp-stat-ffi-int
205 :lisp-stat-macros)
206 (:export log-gamma set-seed
207 uniform-rand
208 normal-cdf normal-quant normal-dens normal-rand
209 bivnorm-cdf
210 cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
211 gamma-cdf gamma-quant gamma-dens gamma-rand
212 chisq-cdf chisq-quant chisq-dens chisq-rand
213 beta-cdf beta-quant beta-dens beta-rand
214 t-cdf t-quant t-dens t-rand
215 f-cdf f-quant f-dens f-rand
216 poisson-cdf poisson-quant poisson-pmf poisson-rand
217 binomial-cdf binomial-quant binomial-pmf binomial-rand))
221 (defpackage :lisp-stat-math
222 (:use :common-lisp
223 :lisp-stat-object-system
224 :lisp-stat-macros
225 :lisp-stat-compound-data
226 :lisp-stat-float)
227 (:shadowing-import-from :lisp-stat-object-system
228 call-method call-next-method)
229 (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
230 asin acos atan sinh cosh tanh asinh acosh atanh float random
231 truncate floor ceiling round minusp zerop plusp evenp oddp
232 < <= = /= >= > ;; complex
233 conjugate realpart imagpart phase
234 min max logand logior logxor lognot ffloor fceiling
235 ftruncate fround signum cis)
236 (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
237 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
238 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
239 /= >= > ;; complex
240 conjugate realpart imagpart phase min max
241 logand logior logxor lognot ffloor fceiling ftruncate fround
242 signum cis)
243 (:documentation "Vectorization of numerical functions"))
246 #| ;; some of this goes back in, but not all of it?
247 (defpackage :lisp-stat-linalg
248 (:use :common-lisp
249 :cffi
250 :lisp-matrix
251 :lisp-stat-math
252 :lisp-stat-types
253 :lisp-stat-float
254 :lisp-stat-compound-data)
255 (:shadowing-import-from :lisp-stat-math
256 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
257 asin acos atan sinh cosh tanh asinh acosh atanh float random
258 truncate floor ceiling round minusp zerop plusp evenp oddp
259 < <= = /= >= > complex conjugate realpart imagpart phase
260 min max logand logior logxor lognot ffloor fceiling
261 ftruncate fround signum cis)
262 (:export chol-decomp lu-decomp lu-solve determinant inverse
263 sv-decomp qr-decomp rcondest make-rotation spline
264 kernel-dens kernel-smooth
265 fft make-sweep-matrix sweep-operator ax+y eigen
267 check-real ;; for optimize
269 covariance-matrix matrix print-matrix solve
270 backsolve eigenvalues eigenvectors accumulate cumsum combine
271 lowess))
278 (defpackage :lisp-stat-data
279 (:documentation "Data management, integration, I/O, and other data technologies.")
280 (:nicknames :ls-data)
281 (:use :common-lisp
282 :lisp-stat-object-system
283 :lisp-stat-config
284 :lisp-stat-types
285 :lisp-stat-compound-data)
286 (:shadowing-import-from :lisp-stat-object-system
287 call-method call-next-method)
288 (:export open-file-dialog read-data-file read-data-columns load-data
289 load-example *variables* *ask-on-redefine*
290 def variables savevar undef))
292 (defpackage :lisp-stat-descriptive-statistics
293 (:use :common-lisp
294 :lisp-stat-data
295 :lisp-stat-math
296 :lisp-stat-compound-data
297 :lisp-matrix
298 :lisp-stat-basics)
299 (:shadowing-import-from :lisp-stat-math ;; life is a vector!
300 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
301 asin acos atan sinh cosh tanh asinh acosh atanh float random
302 truncate floor ceiling round minusp zerop plusp evenp oddp
303 < <= = /= >= > ;; complex
304 conjugate realpart imagpart phase
305 min max logand logior logxor lognot ffloor fceiling
306 ftruncate fround signum cis)
307 (:export standard-deviation
308 quantile median interquartile-range
309 fivnum sample))
312 (defpackage :lisp-stat-regression-linear
313 (:use :common-lisp
314 :lisp-matrix
315 :lisp-stat-object-system
316 :lisp-stat-basics
317 :lisp-stat-compound-data
318 :lisp-stat-descriptive-statistics )
319 (:shadowing-import-from :lisp-stat-object-system
320 call-method call-next-method)
321 (:export regression-model fit-model
323 estimates covariance-matrix
324 ;; functions for helpers
325 lm xtxinv
326 print-object ;; for method dispatch
328 ;; OLD to remove
330 regression-model-proto x y intercept
331 ;; sweep-matrix
332 basis weights included
333 total-sum-of-squares residual-sum-of-squares
334 predictor-names response-name case-labels
337 (defpackage :lisp-stat
338 (:documentation "Experimentation package for LispStat. Serious
339 work should be packaged up elsewhere for reproducibility. By this
340 I mean, creating a data/analytics/analysis package with the
341 minimal set of objects required.")
342 (:use :common-lisp
343 :lisp-stat-object-system
344 :lisp-stat-compound-data
345 :lisp-stat-probability
346 :lisp-stat-types
347 :lisp-stat-float
348 :lisp-stat-basics
349 :lisp-stat-data
350 :lisp-stat-dataframe
351 :cls-data-listoflist
352 :lisp-stat-math
353 :lisp-matrix ;; conversion to a more robust linalg approach
354 :lisp-stat-descriptive-statistics
355 :lisp-stat-regression-linear
356 :cybertiggyr-dsv
357 :cls-visualize
358 :cls-visualize-plplot)
359 (:shadowing-import-from :lisp-stat-object-system
360 call-method call-next-method)
361 (:shadowing-import-from :lisp-stat-math
362 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
363 asin acos atan sinh cosh tanh asinh acosh atanh float random
364 truncate floor ceiling round minusp zerop plusp evenp oddp
365 < <= = /= >= >
366 ;;complex
367 conjugate realpart imagpart phase
369 min max
370 logand logior logxor lognot
371 ffloor fceiling ftruncate fround
372 signum cis)
373 (:export
374 ;; lsobjects :
375 defproto defproto2
376 defmeth send
377 proto-slot-value
379 ;; lstypes :
380 fixnump check-nonneg-fixnum check-one-fixnum
381 check-one-nonneg-fixnum
382 check-one-real check-one-number
384 ;; lsmacros:
386 ;; lsfloat :
387 machine-epsilon
389 ;; compound :
390 compound-data-p *compound-data-proto* compound-object-p
391 compound-data-seq compound-data-length
392 element-list element-seq
393 sort-data order rank
394 recursive-map-elements map-elements
395 repeat
396 check-sequence
397 get-next-element make-next-element set-next-element
398 ;; sequencep
399 iseq
400 ordered-nneg-seq
401 select which
402 difference rseq
404 ;; lsmath.lsp
405 ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
406 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
407 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
408 /= >= > ;; complex
409 conjugate realpart imagpart phase min max
410 logand logior logxor lognot ffloor fceiling ftruncate fround
411 signum cis
413 #| ;; The following need to be re-found in lisp-matrix...
415 ;; matrices.lisp
416 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
417 column-list inner-product outer-product cross-product transpose
418 bind-columns bind-rows
420 ;; linalg.lisp
421 chol-decomp lu-decomp lu-solve determinant inverse
422 sv-decomp qr-decomp rcondest make-rotation spline
423 kernel-dens kernel-smooth
424 fft make-sweep-matrix sweep-operator ax+y eigen
425 check-real
426 covariance-matrix matrix print-matrix solve
427 backsolve eigenvalues eigenvectors accumulate cumsum combine
428 lowess
430 ;; in linalg.lisp, possibly not supported by matlisp
431 spline kernel-dens kernel-smooth
435 ;; optimize.lsp
436 newtonmax nelmeadmax
438 ;; lispstat-macros
439 make-rv-function make-rv-function-1
441 ;; data
442 open-file-dialog read-data-file read-data-columns load-data
443 load-example *variables* *ask-on-redefine*
444 def variables savevar undef
446 ;; dataframe
447 dataframe-like dataframe-array make-dataframe
448 varlabels caselabels nrows ncols
449 dataframe-dimension dataframe-dimensons
450 dfref dfref-case dfref-var
451 consistent-dataframe-p
452 dataset
453 list-of-columns list-of-rows
455 ;; listoflist
456 lists-of-same-size equal-listoflist
457 transpose-listoflist
458 listoflist->dataframe listoflist->array listoflist->matrix-like
460 ;; statistics.lsp (descriptions, should probably be moved
461 ;; later...?
462 standard-deviation quantile median interquartile-range
463 fivnum sample
465 ;; probability (dists.lisp)
466 log-gamma set-seed
467 uniform-rand normal-cdf normal-quant normal-dens
468 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
469 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
470 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
471 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
472 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
473 binomial-cdf binomial-quant binomial-pmf binomial-rand
475 ;; Here is where we have a problem -- lispstat core should be core
476 ;; data management and config problems, with packages providing
477 ;; specialized extensions to LispStat, i.e. regression, nonlin
478 ;; regression, bayesian regression via laplace approximation, etc.
480 ;; The following could be considered "recommended packages",
481 ;; similar to the idea of the recommended packages in R. Probably
482 ;; we want them to do the exporting within that package, therefore
483 ;; NOT being able to lock the "data-ish" package, but only the
484 ;; subpackages prior to export.
486 ;; regression.lsp
487 ;; -- linear regressin models.
488 regression-model fit-model
489 estimates covariance-matrix
491 regression-model-proto x y intercept sweep-matrix
492 basis weights included total-sum-of-squares residual-sum-of-squares
493 predictor-names response-name case-labels
494 lm xtxinv
496 ;; nonlin.lsp
497 ;; -- nonlinear regression models
498 nreg-model nreg-model-proto mean-function theta-hat epsilon
499 count-limit verbose
500 ;; we might need something like xtxinv here? But should be
501 ;; encapsulated, so we use the one in regression.lisp
503 ;; bayes.lsp
504 bayes-model bayes-model-proto bayes-internals
506 ;; plots.lisp
507 plot-ex
508 contour-plot-ex
509 fn-contour-plot-ex
510 shade-plot-ex
511 3D-plot-ex
516 ;;;; PACKAGES FOR USEABILITY
518 (defpackage :lisp-stat-data-examples
519 (:documentation "Example data for unittests, examples, illustrations,")
520 (:use :common-lisp
521 :lisp-stat)
522 (:shadowing-import-from :lisp-stat
523 call-method call-next-method
525 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
526 asin acos atan sinh cosh tanh asinh acosh atanh float random
527 truncate floor ceiling round minusp zerop plusp evenp oddp
528 < <= = /= >= > > ;; complex
529 conjugate realpart imagpart phase
530 min max logand logior logxor lognot ffloor fceiling
531 ftruncate fround signum cis
533 <= float imagpart)
534 (:export iron aluminum absorbtion
535 diabetes dlabs))
538 (defpackage :lisp-stat-user
539 (:documentation "Experimentation package for LispStat.
540 Serious work should be placed in a similar package elsewhere for
541 reproducibility. But this should hint as to what needs to be
542 done for a user- or analysis-package.")
543 (:nicknames :ls-user)
544 (:use :common-lisp
545 :lisp-matrix
546 :lisp-stat
547 :lisp-stat-data-examples) ;; this last is to have 'things to play with'
548 (:shadowing-import-from :lisp-stat
549 call-method call-next-method
551 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
552 asin acos atan sinh cosh tanh asinh acosh atanh float random
553 truncate floor ceiling round minusp zerop plusp evenp oddp
554 < <= = /= >= > > ;; complex
555 conjugate realpart imagpart phase
556 min max logand logior logxor lognot ffloor fceiling
557 ftruncate fround signum cis
559 <= float imagpart))
561 (defpackage :lisp-stat-unittests
562 (:use :common-lisp
563 :lift :lisp-matrix
564 :lisp-stat :lisp-stat-data-examples)
565 (:shadowing-import-from :lisp-stat
566 call-method call-next-method ;; objects
567 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
568 asin acos atan sinh cosh tanh asinh acosh atanh float random
569 truncate floor ceiling round minusp zerop plusp evenp oddp
570 < <= = /= >= > ;; complex
571 conjugate realpart imagpart phase
572 min max logand logior logxor lognot ffloor fceiling
573 ftruncate fround signum cis)
574 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
575 almost= almost=lists numerical=)) ; compare
577 (defpackage :lisp-stat-dataframe-example
578 (:use :common-lisp
579 :lift :lisp-stat-unittests
580 :lisp-stat-data-examples
581 :lisp-stat-dataframe)
582 (:export absorbtion aluminum iron))
585 (defpackage :lisp-stat-optimize
586 (:use :common-lisp
587 :cffi
588 :lisp-matrix
589 :lisp-stat-ffi-int
590 :lisp-stat-object-system
591 :lisp-stat-types
592 :lisp-stat-compound-data
593 :lisp-stat-math
594 :lisp-stat-float
595 :lisp-stat-basics
597 :lisp-stat-matrix
598 :lisp-stat-linalg-data
599 :lisp-stat-linalg
602 (:shadowing-import-from :lisp-stat-object-system
603 call-method call-next-method)
604 (:shadowing-import-from :lisp-stat-math
605 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
606 asin acos atan sinh cosh tanh asinh acosh atanh float random
607 truncate floor ceiling round minusp zerop plusp evenp oddp
608 < <= = /= >= > complex conjugate realpart imagpart phase
609 min max logand logior logxor lognot ffloor fceiling
610 ftruncate fround signum cis)
611 (:export
612 ;; derivatives
613 numgrad numhess
615 ;; optimization
616 newtonmax nelmeadmax))