cleaned up code to load dataframes from DSV files, added cls-user as a test package.
[CommonLispStat.git] / src / packages.lisp
blobe264bc6f6c2c1e18377a0ac15f4714d4bb9b5455
1 ;;; -*- mode: lisp -*-
3 ;;; Time-stamp: <2009-12-18 18:35:06 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2008, AJ Rossini. BSD, LLGPL, or GPLv2, depending
8 ;;; on how it arrives.
9 ;;; Purpose: package structure description for lispstat
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
15 (in-package :cl-user)
17 ;;; Current structure, dependencies:
19 (progn
20 (ls-user :depends-on lisp-stat)
21 (lisp-stat :depends-on '(cls-dataframe
22 cls-data
23 lisp-matrix
25 (cls-dataframe :depends-on cls-data)
26 |#
30 ;;; Basics
32 (defpackage :lisp-stat-config
33 (:documentation "global settings and variables. Probably need a
34 localization tool as well.")
35 (:use :common-lisp)
36 (:export *common-lisp-stat-version*
37 *default-path* *lsos-files* *basic-files* *ls-files*
39 *cls-data-dir* *cls-home-dir* *cls-examples-dir*))
41 (defpackage :lisp-stat-object-system
42 (:nicknames :ls-objects :lsos)
43 (:use :common-lisp)
44 (:shadow :call-method :call-next-method)
45 (:export ls-object objectp *object* kind-of-p make-object
46 *message-hook*
47 *set-slot-hook* proto-slot-value self
48 send call-next-method call-method
49 defmeth defproto instance-slots proto-name))
51 ;;; -types and -float probably ought to be moved into a -numerics
52 ;;; package.
54 (defpackage :lisp-stat-types
55 (:documentation "Provides some typeing for LispStat, but is clearly
56 a bit incomplete.")
57 (:use :common-lisp)
58 (:export fixnump
59 check-nonneg-fixnum check-one-nonneg-fixnum
60 check-one-fixnum check-one-real check-one-number))
62 (defpackage :lisp-stat-float
63 (:use :common-lisp)
64 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
65 machine-epsilon base-float makedouble
67 make-base-trans-fun-2 make-base-trans-fun
69 base-log base-exp base-expt base-sqrt base-sin base-cos
70 base-tan base-asin base-acos base-atan base-sinh
71 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
72 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
73 BASE-FROUND BASE-SIGNUM BASE-CIS))
75 ;;; Probably should move into cls-data package.
77 (defpackage :lisp-stat-compound-data
78 (:use :common-lisp
79 :lisp-stat-object-system
80 :lisp-stat-types)
81 (:shadowing-import-from :lisp-stat-object-system
82 call-next-method call-method)
83 (:export compound-data-p *compound-data-proto*
84 compound-object-p
85 compound-data-seq compound-data-length
86 element-list element-seq
87 sort-data order rank
88 recursive-map-elements map-elements repeat
89 check-sequence
90 get-next-element make-next-element set-next-element
91 ;; sequencep
92 iseq ordered-nneg-seq
93 select split-list which
94 difference rseq
95 flatten-list))
97 (defpackage :lisp-stat-macros
98 (:use :common-lisp
99 :lisp-stat-compound-data)
100 (:export make-rv-function make-rv-function-1))
102 (defpackage :lisp-stat-basics
103 (:use :common-lisp
104 :lisp-stat-object-system
105 :lisp-stat-types
106 :lisp-stat-float
107 :lisp-stat-macros
108 :lisp-stat-compound-data)
109 (:shadowing-import-from :lisp-stat-object-system
110 call-method call-next-method)
111 (:export permute-array sum prod count-elements mean
112 if-else sample))
114 (defpackage :lisp-stat-float
115 (:use :common-lisp)
116 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
117 machine-epsilon base-float makedouble
119 make-base-trans-fun-2 make-base-trans-fun
121 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
122 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
123 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
124 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
125 BASE-FROUND BASE-SIGNUM BASE-CIS))
127 (defpackage :lisp-stat-macros
128 (:use :common-lisp
129 :lisp-stat-compound-data)
130 (:export make-rv-function make-rv-function-1))
133 (defpackage :cls-matrix
134 (:documentation "basic utilities for using lisp arrays as numerical
135 matrices. Not optimized, and must consider this slow. Routines
136 should be optimized, it is only that we need them first, optimize
137 them later.")
138 (:use :common-lisp)
139 (:export matrixp num-rows num-cols matmult identity-matrix diagonal
140 row-list column-list inner-product outer-product
141 cross-product transpose bind-columns bind-rows
142 array-data-vector vector-to-array))
145 ;;; NEW CLOS STRUCTURE
147 ;;; cls-data... in dataframe, though.
148 (defpackage :cls-dataframe
149 (:use :common-lisp
150 :xarray
151 :lisp-matrix)
152 (:shadowing-import-from :xarray slice)
153 (:export
154 ;; generic container class for data -- if small enough
155 ;; could be value, otherwise might be reference.
156 dataframe-like
157 dataframe-array
158 make-dataframe
160 ;; accessors
161 varlabels caselabels nrows ncols
162 dataframe-dimension dataframe-dimensons
163 xref xtype xdims xdim xrank slice take carray
165 dfref dfref-case dfref-var
166 consistent-dataframe-p
169 dataset
170 list-of-columns ;; list-of-variables
171 list-of-rows ;; list-of-observations
174 (defpackage :cls-data
175 (:use :common-lisp
176 :xarray
177 :lisp-matrix
178 :cls-dataframe) ; for dataframe
179 (:shadowing-import-from :xarray slice)
180 (:export listoflist->dataframe
181 listoflist->array
182 listoflist->matrix-like))
184 lists-of-same-size
185 equal-listoflist
186 transpose-listoflist
189 (defpackage :cls-dataimport
190 (:documentation "Data I/O and similar import technologies.")
191 (:use :common-lisp
192 :lisp-stat-object-system
193 :cls-dataframe
194 :cls-data
195 :rsm.string)
196 (:shadowing-import-from :lisp-stat-object-system
197 call-method call-next-method)
198 (:export dsvstream->dataframe dsvstream->matrix dsvstream->listoflist))
201 (defpackage :lisp-stat-model
202 (:documentation "Model management for data analysis.")
203 (:use :common-lisp
204 :lisp-matrix)
205 (:export
206 ;; data structures for model and model/data combination
207 model statistical-model analysis))
209 ;;; visualization
211 (defpackage :cls-visualize
212 (:use :common-lisp
213 :lisp-matrix
214 :cls-dataframe)
215 (:shadowing-import-from :xarray slice)
219 (defpackage :cls-visualize-plplot
220 (:use :common-lisp
221 :lisp-matrix
222 :cls-dataframe
223 :cl-plplot-system)
224 (:export
225 ;; examples
226 plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex 3D-plot-ex))
229 ;;; USER PACKAGES
231 (defpackage :lisp-stat-ffi-int
232 (:use :common-lisp
233 :cffi)
234 (:export ccl-store-integer ccl-store-double ccl-store-ptr
235 get-buf ))
237 (defpackage :lisp-stat-probability
238 (:use :common-lisp
239 :cffi
240 :lisp-stat-ffi-int
241 :lisp-stat-macros)
242 (:export log-gamma set-seed
243 uniform-rand
244 normal-cdf normal-quant normal-dens normal-rand
245 bivnorm-cdf
246 cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
247 gamma-cdf gamma-quant gamma-dens gamma-rand
248 chisq-cdf chisq-quant chisq-dens chisq-rand
249 beta-cdf beta-quant beta-dens beta-rand
250 t-cdf t-quant t-dens t-rand
251 f-cdf f-quant f-dens f-rand
252 poisson-cdf poisson-quant poisson-pmf poisson-rand
253 binomial-cdf binomial-quant binomial-pmf binomial-rand))
257 (defpackage :lisp-stat-math
258 (:use :common-lisp
259 :lisp-stat-object-system
260 :lisp-stat-macros
261 :lisp-stat-compound-data
262 :lisp-stat-float)
263 (:shadowing-import-from :lisp-stat-object-system
264 call-method call-next-method)
265 (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
266 asin acos atan sinh cosh tanh asinh acosh atanh float random
267 truncate floor ceiling round minusp zerop plusp evenp oddp
268 < <= = /= >= > ;; complex
269 conjugate realpart imagpart phase
270 min max logand logior logxor lognot ffloor fceiling
271 ftruncate fround signum cis)
272 (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
273 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
274 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
275 /= >= > ;; complex
276 conjugate realpart imagpart phase min max
277 logand logior logxor lognot ffloor fceiling ftruncate fround
278 signum cis)
279 (:documentation "Vectorization of numerical functions"))
282 #| ;; some of this goes back in, but not all of it?
283 (defpackage :lisp-stat-linalg
284 (:use :common-lisp
285 :cffi
286 :lisp-matrix
287 :lisp-stat-math
288 :lisp-stat-types
289 :lisp-stat-float
290 :lisp-stat-compound-data)
291 (:shadowing-import-from :lisp-stat-math
292 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
293 asin acos atan sinh cosh tanh asinh acosh atanh float random
294 truncate floor ceiling round minusp zerop plusp evenp oddp
295 < <= = /= >= > complex conjugate realpart imagpart phase
296 min max logand logior logxor lognot ffloor fceiling
297 ftruncate fround signum cis)
298 (:export chol-decomp lu-decomp lu-solve determinant inverse
299 sv-decomp qr-decomp rcondest make-rotation spline
300 kernel-dens kernel-smooth
301 fft make-sweep-matrix sweep-operator ax+y eigen
303 check-real ;; for optimize
305 covariance-matrix matrix print-matrix solve
306 backsolve eigenvalues eigenvectors accumulate cumsum combine
307 lowess))
314 (defpackage :lisp-stat-data
315 (:documentation "Data management, integration, I/O, and other data technologies.")
316 (:nicknames :ls-data)
317 (:use :common-lisp
318 :lisp-stat-object-system
319 :lisp-stat-config
320 :lisp-stat-types
321 :lisp-stat-compound-data)
322 (:shadowing-import-from :lisp-stat-object-system
323 call-method call-next-method)
324 (:export
325 ;; generic structures
326 ;; Variables
327 empirical-statistical-variable
328 modelbased-statistical-variable
329 categorical-statistical-variable
330 nominal-statistical-variable
331 ordinal-statistical-variable
332 continuous-statistical-variable
334 ordering factor-levels nobs support pdmf draw
335 print-object
337 ;; Observations
338 statistical-observation
339 measurement-types record
340 ;; XLS compat tools
341 open-file-dialog read-data-file read-data-columns load-data
342 load-example *variables* *ask-on-redefine*
343 def variables savevar undef))
345 (defpackage :lisp-stat-descriptive-statistics
346 (:use :common-lisp
347 :lisp-matrix
348 :lisp-stat-data
349 :lisp-stat-math
350 :lisp-stat-compound-data
351 :lisp-stat-basics)
352 (:shadowing-import-from :lisp-stat-math ;; life is a vector!
353 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
354 asin acos atan sinh cosh tanh asinh acosh atanh float random
355 truncate floor ceiling round minusp zerop plusp evenp oddp
356 < <= = /= >= > ;; complex
357 conjugate realpart imagpart phase
358 min max logand logior logxor lognot ffloor fceiling
359 ftruncate fround signum cis)
360 (:export mean standard-deviation variance
361 quantile median interquartile-range
362 fivnum sample))
364 (defpackage :lisp-stat-regression-linear
365 (:use :common-lisp
366 :lisp-matrix
367 :lisp-stat-basics
368 :lisp-stat-compound-data
369 :lisp-stat-descriptive-statistics )
370 (:shadowing-import-from :lisp-stat-object-system
371 call-method call-next-method)
372 (:export regression-model fit-model
374 estimates covariance-matrix
375 ;; functions for helpers
376 lm xtxinv
377 print-object ;; for method dispatch
380 (defpackage :common-lisp-statistics
381 (:documentation "Experimentation package for LispStat. Serious work
382 should be packaged up as a separate but similar package to help
383 drive reproducibility. By this I mean, creating a
384 data/analytics/analysis package with the minimal set of
385 objects/packages required.")
386 (:nicknames :cls :common-lisp-statistics :lisp-stat)
387 (:use :common-lisp
388 :xarray ;; generic reference -- internally supporting array, lol structs
389 :lisp-matrix ;; conversion to a more robust linalg approach
390 :lisp-stat-config
391 :lisp-stat-object-system
392 :lisp-stat-compound-data
393 :lisp-stat-probability
394 :lisp-stat-types
395 :lisp-stat-float
396 :lisp-stat-basics
397 :lisp-stat-data
398 :cls-dataframe
399 :cls-data
400 :lisp-stat-math
401 :lisp-stat-descriptive-statistics
402 :lisp-stat-regression-linear
403 :cls-visualize
404 ;; :cybertiggyr-dsv
405 ;; :cls-visualize-plplot
407 (:shadowing-import-from :xarray slice)
408 (:shadowing-import-from :lisp-stat-object-system
409 call-method call-next-method)
410 (:shadowing-import-from :lisp-stat-math
411 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
412 asin acos atan sinh cosh tanh asinh acosh atanh float random
413 truncate floor ceiling round minusp zerop plusp evenp oddp
414 < <= = /= >= >
415 ;;complex
416 conjugate realpart imagpart phase
418 min max
419 logand logior logxor lognot
420 ffloor fceiling ftruncate fround
421 signum cis)
422 (:export
423 ;; lisp-stat-config:
424 *default-path* *lsos-files* *basic-files* *ls-files*
425 l *cls-home-dir* *cls-data-dir* *cls-examples-dir*
427 ;; lsobjects :
428 defproto defproto2
429 defmeth send proto-slot-value
431 ;; lstypes :
432 fixnump check-nonneg-fixnum check-one-fixnum
433 check-one-nonneg-fixnum
434 check-one-real check-one-number
436 ;; lsmacros:
438 ;; lsfloat :
439 machine-epsilon
441 ;; compound :
442 compound-data-p *compound-data-proto* compound-object-p
443 compound-data-seq compound-data-length
444 element-list element-seq
445 sort-data order rank
446 recursive-map-elements map-elements
447 repeat
448 check-sequence
449 get-next-element make-next-element set-next-element
450 ;; sequencep
451 iseq
452 ordered-nneg-seq
453 select which
454 difference rseq
456 ;; lsmath.lsp
457 ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
458 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
459 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
460 /= >= > ;; complex
461 conjugate realpart imagpart phase min max
462 logand logior logxor lognot ffloor fceiling ftruncate fround
463 signum cis
465 #| ;; The following need to be re-found in lisp-matrix...
467 ;; matrices.lisp
468 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
469 column-list inner-product outer-product cross-product transpose
470 bind-columns bind-rows
472 ;; linalg.lisp
473 chol-decomp lu-decomp lu-solve determinant inverse
474 sv-decomp qr-decomp rcondest make-rotation spline
475 kernel-dens kernel-smooth
476 fft make-sweep-matrix sweep-operator ax+y eigen
477 check-real
478 covariance-matrix matrix print-matrix solve
479 backsolve eigenvalues eigenvectors accumulate cumsum combine
480 lowess
482 ;; in linalg.lisp, possibly not supported by matlisp
483 spline kernel-dens kernel-smooth
487 ;; optimize.lsp
488 newtonmax nelmeadmax
490 ;; lispstat-macros
491 make-rv-function make-rv-function-1
493 ;; xarray
494 xref xtype xdims xdim xdims*
495 lists-of-same-size equal-listoflist transpose-listoflist
497 ;; data
498 ;; need to take this list and make it strings... specs could mean
499 ;; that we process the strings in different ways?
501 (let ((lst ()))
502 (unlist
503 (mapc #'symbol-for-symbol-to-string-or-symbol
504 (do-external-symbols (s (find-package 'lisp-stat-data) lst) (push s lst))))
505 lst)
507 open-file-dialog read-data-file read-data-columns load-data
508 load-example *variables* *ask-on-redefine*
509 def variables savevar undef
510 ;; dataframe
511 dataframe-like dataframe-array make-dataframe
512 varlabels caselabels nrows ncols
513 dataframe-dimension dataframe-dimensons
514 dfref dfref-case dfref-var
515 consistent-dataframe-p
516 dataset list-of-columns list-of-rows
518 ;; listoflist
519 listoflist->dataframe listoflist->array listoflist->matrix-like
521 ;; statistics.lsp (descriptions, should probably be moved
522 ;; later...?
523 standard-deviation quantile median interquartile-range
524 fivnum sample
526 ;; probability (dists.lisp)
527 log-gamma set-seed
528 uniform-rand normal-cdf normal-quant normal-dens
529 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
530 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
531 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
532 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
533 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
534 binomial-cdf binomial-quant binomial-pmf binomial-rand
536 ;; Here is where we have a problem -- lispstat core should be core
537 ;; data management and config problems, with packages providing
538 ;; specialized extensions to LispStat, i.e. regression, nonlin
539 ;; regression, bayesian regression via laplace approximation, etc.
541 ;; The following could be considered "recommended packages",
542 ;; similar to the idea of the recommended packages in R. Probably
543 ;; we want them to do the exporting within that package, therefore
544 ;; NOT being able to lock the "data-ish" package, but only the
545 ;; subpackages prior to export.
547 ;; regression.lsp
548 ;; -- linear regressin models.
549 regression-model fit-model
550 estimates covariance-matrix
552 regression-model-proto x y intercept sweep-matrix
553 basis weights included total-sum-of-squares residual-sum-of-squares
554 predictor-names response-name case-labels
555 lm xtxinv
557 ;; nonlin.lsp
558 ;; -- nonlinear regression models
559 nreg-model nreg-model-proto mean-function theta-hat epsilon
560 count-limit verbose
561 ;; we might need something like xtxinv here? But should be
562 ;; encapsulated, so we use the one in regression.lisp
564 ;; bayes.lsp
565 bayes-model bayes-model-proto bayes-internals
567 ;; plots.lisp
568 plot-ex
569 contour-plot-ex
570 fn-contour-plot-ex
571 shade-plot-ex
572 3D-plot-ex
577 ;;;; PACKAGES FOR USEABILITY
579 (defpackage :lisp-stat-data-examples
580 (:documentation "Example data for unittests, examples, illustrations,")
581 (:use :common-lisp
582 :common-lisp-statistics)
583 (:shadowing-import-from :lisp-stat
584 call-method call-next-method
586 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
587 asin acos atan sinh cosh tanh asinh acosh atanh float random
588 truncate floor ceiling round minusp zerop plusp evenp oddp
589 < <= = /= >= > > ;; complex
590 conjugate realpart imagpart phase
591 min max logand logior logxor lognot ffloor fceiling
592 ftruncate fround signum cis
594 <= float imagpart)
595 (:export iron aluminum absorbtion
596 diabetes dlabs))
599 (defpackage :lisp-stat-user
600 (:documentation "Experimentation package for LispStat. Serious work
601 should be placed in a similar package elsewhere for
602 reproducibility. But this should hint as to what needs to be done
603 for a user- or analysis-package.")
604 (:nicknames :ls-user :cls-user)
605 (:use :common-lisp ; always needed for user playgrounds!
606 :lisp-matrix
607 :common-lisp-statistics
608 :lisp-stat-data-examples) ;; this last is to have 'things to play with'
609 (:shadowing-import-from :lisp-stat
610 call-method call-next-method
612 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
613 asin acos atan sinh cosh tanh asinh acosh atanh float random
614 truncate floor ceiling round minusp zerop plusp evenp oddp
615 < <= = /= >= > > ;; complex
616 conjugate realpart imagpart phase
617 min max logand logior logxor lognot ffloor fceiling
618 ftruncate fround signum cis
620 <= float imagpart))
622 (defpackage :lisp-stat-unittests
623 (:use :common-lisp
624 :lift :lisp-matrix
625 :lisp-stat :lisp-stat-data-examples)
626 (:shadowing-import-from :lisp-stat
627 call-method call-next-method ;; objects
628 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
629 asin acos atan sinh cosh tanh asinh acosh atanh float random
630 truncate floor ceiling round minusp zerop plusp evenp oddp
631 < <= = /= >= > ;; complex
632 conjugate realpart imagpart phase
633 min max logand logior logxor lognot ffloor fceiling
634 ftruncate fround signum cis)
635 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
636 almost= almost=lists numerical=)) ; compare
638 (defpackage :cls-dataframe-example
639 (:use :common-lisp
640 :lift :lisp-stat-unittests
641 :lisp-stat-data-examples
642 :cls-dataframe)
643 (:export absorbtion aluminum iron))
646 (defpackage :lisp-stat-optimize
647 (:use :common-lisp
648 :cffi
649 :lisp-matrix
650 :lisp-stat-ffi-int
651 :lisp-stat-object-system
652 :lisp-stat-types
653 :lisp-stat-compound-data
654 :lisp-stat-math
655 :lisp-stat-float
656 :lisp-stat-basics
658 :lisp-stat-matrix
659 :lisp-stat-linalg-data
660 :lisp-stat-linalg
663 (:shadowing-import-from :lisp-stat-object-system
664 call-method call-next-method)
665 (:shadowing-import-from :lisp-stat-math
666 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
667 asin acos atan sinh cosh tanh asinh acosh atanh float random
668 truncate floor ceiling round minusp zerop plusp evenp oddp
669 < <= = /= >= > complex conjugate realpart imagpart phase
670 min max logand logior logxor lognot ffloor fceiling
671 ftruncate fround signum cis)
672 (:export
673 ;; derivatives
674 numgrad numhess
676 ;; optimization
677 newtonmax nelmeadmax))