export filename.dsv->dataframe in the right place.
[CommonLispStat.git] / src / packages.lisp
blob36310475f8b2294e7acbe8b2827db997ed965324
1 ;;; -*- mode: lisp -*-
3 ;;; Time-stamp: <2009-12-21 12:41:03 tony>
4 ;;; Creation: <2008-03-11 19:18:34 user>
5 ;;; File: packages.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2007--2008, AJ Rossini. BSD, LLGPL, or GPLv2, depending
8 ;;; on how it arrives.
9 ;;; Purpose: package structure description for lispstat
11 ;;; What is this talk of 'release'? Klingons do not make software
12 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
13 ;;; designers and quality assurance people in its wake.
15 (in-package :cl-user)
17 ;;; Current structure, dependencies:
19 (progn
20 (ls-user :depends-on lisp-stat)
21 (lisp-stat :depends-on '(cls-dataframe
22 cls-data
23 lisp-matrix
25 (cls-dataframe :depends-on cls-data)
26 |#
30 ;;; Basics
32 (defpackage :lisp-stat-config
33 (:documentation "global settings and variables. Probably need a
34 localization tool as well.")
35 (:use :common-lisp)
36 (:export *common-lisp-stat-version*
37 *default-path* *lsos-files* *basic-files* *ls-files*
39 *cls-data-dir* *cls-home-dir* *cls-examples-dir*))
41 (defpackage :lisp-stat-object-system
42 (:nicknames :ls-objects :lsos)
43 (:use :common-lisp)
44 (:shadow :call-method :call-next-method)
45 (:export ls-object objectp *object* kind-of-p make-object
46 *message-hook*
47 *set-slot-hook* proto-slot-value self
48 send call-next-method call-method
49 defmeth defproto instance-slots proto-name))
51 ;;; -types and -float probably ought to be moved into a -numerics
52 ;;; package.
54 (defpackage :lisp-stat-types
55 (:documentation "Provides some typeing for LispStat, but is clearly
56 a bit incomplete.")
57 (:use :common-lisp)
58 (:export fixnump
59 check-nonneg-fixnum check-one-nonneg-fixnum
60 check-one-fixnum check-one-real check-one-number))
62 (defpackage :lisp-stat-float
63 (:use :common-lisp)
64 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
65 machine-epsilon base-float makedouble
67 make-base-trans-fun-2 make-base-trans-fun
69 base-log base-exp base-expt base-sqrt base-sin base-cos
70 base-tan base-asin base-acos base-atan base-sinh
71 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
72 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
73 BASE-FROUND BASE-SIGNUM BASE-CIS))
75 ;;; Probably should move into cls-data package.
77 (defpackage :lisp-stat-compound-data
78 (:use :common-lisp
79 :lisp-stat-object-system
80 :lisp-stat-types)
81 (:shadowing-import-from :lisp-stat-object-system
82 call-next-method call-method)
83 (:export compound-data-p *compound-data-proto*
84 compound-object-p
85 compound-data-seq compound-data-length
86 element-list element-seq
87 sort-data order rank
88 recursive-map-elements map-elements repeat
89 check-sequence
90 get-next-element make-next-element set-next-element
91 ;; sequencep
92 iseq ordered-nneg-seq
93 select split-list which
94 difference rseq
95 flatten-list))
97 (defpackage :lisp-stat-macros
98 (:use :common-lisp
99 :lisp-stat-compound-data)
100 (:export make-rv-function make-rv-function-1))
102 (defpackage :lisp-stat-basics
103 (:use :common-lisp
104 :lisp-stat-object-system
105 :lisp-stat-types
106 :lisp-stat-float
107 :lisp-stat-macros
108 :lisp-stat-compound-data)
109 (:shadowing-import-from :lisp-stat-object-system
110 call-method call-next-method)
111 (:export permute-array sum prod count-elements mean
112 if-else sample))
114 (defpackage :lisp-stat-float
115 (:use :common-lisp)
116 (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
117 machine-epsilon base-float makedouble
119 make-base-trans-fun-2 make-base-trans-fun
121 BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
122 BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
123 BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
124 BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
125 BASE-FROUND BASE-SIGNUM BASE-CIS))
127 (defpackage :lisp-stat-macros
128 (:use :common-lisp
129 :lisp-stat-compound-data)
130 (:export make-rv-function make-rv-function-1))
133 (defpackage :cls-matrix
134 (:documentation "basic utilities for using lisp arrays as numerical
135 matrices. Not optimized, and must consider this slow. Routines
136 should be optimized, it is only that we need them first, optimize
137 them later.")
138 (:use :common-lisp)
139 (:export matrixp num-rows num-cols matmult identity-matrix diagonal
140 row-list column-list inner-product outer-product
141 cross-product transpose bind-columns bind-rows
142 array-data-vector vector-to-array))
145 ;;; NEW CLOS STRUCTURE
147 ;;; cls-data... in dataframe, though.
148 (defpackage :cls-dataframe
149 (:use :common-lisp
150 :xarray
151 :lisp-matrix
152 :listoflist)
153 (:shadowing-import-from :xarray slice)
154 (:export
155 ;; generic container class for data -- if small enough
156 ;; could be value, otherwise might be reference.
157 dataframe-like
158 dataframe-array
159 make-dataframe
161 ;; accessors
162 varlabels caselabels nrows ncols
163 dataframe-dimension dataframe-dimensons
164 xref xtype xdims xdim xrank slice take carray
166 dfref dfref-case dfref-var
167 consistent-dataframe-p
170 dataset
171 list-of-columns ;; list-of-variables
172 list-of-rows ;; list-of-observations
175 (defpackage :cls-data
176 (:use :common-lisp
177 :xarray
178 :listoflist
179 :lisp-matrix
180 :cls-dataframe) ; for dataframe
181 (:shadowing-import-from :xarray slice)
182 (:export listoflist->dataframe
183 listoflist->array
184 listoflist->matrix-like
185 filename.dsv->dataframe))
187 (defpackage :cls-dataimport
188 (:documentation "Data I/O and similar import technologies.")
189 (:use :common-lisp
190 :lisp-stat-object-system
191 :cls-dataframe
192 :cls-data
193 :rsm.string)
194 (:shadowing-import-from :lisp-stat-object-system
195 call-method call-next-method)
196 (:export dsvstream->dataframe dsvstream->matrix dsvstream->listoflist))
199 (defpackage :lisp-stat-model
200 (:documentation "Model management for data analysis.")
201 (:use :common-lisp
202 :lisp-matrix)
203 (:export
204 ;; data structures for model and model/data combination
205 model statistical-model analysis))
207 ;;; visualization
209 (defpackage :cls-visualize
210 (:use :common-lisp
211 :lisp-matrix
212 :cls-dataframe)
213 (:shadowing-import-from :xarray slice)
217 (defpackage :cls-visualize-plplot
218 (:use :common-lisp
219 :lisp-matrix
220 :cls-dataframe
221 :cl-plplot-system)
222 (:export
223 ;; examples
224 plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex 3D-plot-ex))
227 ;;; USER PACKAGES
229 (defpackage :lisp-stat-ffi-int
230 (:use :common-lisp
231 :cffi)
232 (:export ccl-store-integer ccl-store-double ccl-store-ptr
233 get-buf ))
235 (defpackage :lisp-stat-probability
236 (:use :common-lisp
237 :cffi
238 :lisp-stat-ffi-int
239 :lisp-stat-macros)
240 (:export log-gamma set-seed
241 uniform-rand
242 normal-cdf normal-quant normal-dens normal-rand
243 bivnorm-cdf
244 cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
245 gamma-cdf gamma-quant gamma-dens gamma-rand
246 chisq-cdf chisq-quant chisq-dens chisq-rand
247 beta-cdf beta-quant beta-dens beta-rand
248 t-cdf t-quant t-dens t-rand
249 f-cdf f-quant f-dens f-rand
250 poisson-cdf poisson-quant poisson-pmf poisson-rand
251 binomial-cdf binomial-quant binomial-pmf binomial-rand))
255 (defpackage :lisp-stat-math
256 (:use :common-lisp
257 :lisp-stat-object-system
258 :lisp-stat-macros
259 :lisp-stat-compound-data
260 :lisp-stat-float)
261 (:shadowing-import-from :lisp-stat-object-system
262 call-method call-next-method)
263 (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
264 asin acos atan sinh cosh tanh asinh acosh atanh float random
265 truncate floor ceiling round minusp zerop plusp evenp oddp
266 < <= = /= >= > ;; complex
267 conjugate realpart imagpart phase
268 min max logand logior logxor lognot ffloor fceiling
269 ftruncate fround signum cis)
270 (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
271 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
272 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
273 /= >= > ;; complex
274 conjugate realpart imagpart phase min max
275 logand logior logxor lognot ffloor fceiling ftruncate fround
276 signum cis)
277 (:documentation "Vectorization of numerical functions"))
280 #| ;; some of this goes back in, but not all of it?
281 (defpackage :lisp-stat-linalg
282 (:use :common-lisp
283 :cffi
284 :lisp-matrix
285 :lisp-stat-math
286 :lisp-stat-types
287 :lisp-stat-float
288 :lisp-stat-compound-data)
289 (:shadowing-import-from :lisp-stat-math
290 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
291 asin acos atan sinh cosh tanh asinh acosh atanh float random
292 truncate floor ceiling round minusp zerop plusp evenp oddp
293 < <= = /= >= > complex conjugate realpart imagpart phase
294 min max logand logior logxor lognot ffloor fceiling
295 ftruncate fround signum cis)
296 (:export chol-decomp lu-decomp lu-solve determinant inverse
297 sv-decomp qr-decomp rcondest make-rotation spline
298 kernel-dens kernel-smooth
299 fft make-sweep-matrix sweep-operator ax+y eigen
301 check-real ;; for optimize
303 covariance-matrix matrix print-matrix solve
304 backsolve eigenvalues eigenvectors accumulate cumsum combine
305 lowess))
312 (defpackage :lisp-stat-data
313 (:documentation "Data management, integration, I/O, and other data technologies.")
314 (:nicknames :ls-data)
315 (:use :common-lisp
316 :lisp-stat-object-system
317 :lisp-stat-config
318 :lisp-stat-types
319 :lisp-stat-compound-data)
320 (:shadowing-import-from :lisp-stat-object-system
321 call-method call-next-method)
322 (:export
323 ;; generic structures
324 ;; Variables
325 empirical-statistical-variable
326 modelbased-statistical-variable
327 categorical-statistical-variable
328 nominal-statistical-variable
329 ordinal-statistical-variable
330 continuous-statistical-variable
332 ordering factor-levels nobs support pdmf draw
333 print-object
335 ;; Observations
336 statistical-observation
337 measurement-types record
338 ;; XLS compat tools
339 open-file-dialog read-data-file read-data-columns load-data
340 load-example *variables* *ask-on-redefine*
341 def variables savevar undef))
343 (defpackage :lisp-stat-descriptive-statistics
344 (:use :common-lisp
345 :lisp-matrix
346 :lisp-stat-data
347 :lisp-stat-math
348 :lisp-stat-compound-data
349 :lisp-stat-basics)
350 (:shadowing-import-from :lisp-stat-math ;; life is a vector!
351 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
352 asin acos atan sinh cosh tanh asinh acosh atanh float random
353 truncate floor ceiling round minusp zerop plusp evenp oddp
354 < <= = /= >= > ;; complex
355 conjugate realpart imagpart phase
356 min max logand logior logxor lognot ffloor fceiling
357 ftruncate fround signum cis)
358 (:export mean standard-deviation variance
359 quantile median interquartile-range
360 fivnum sample))
362 (defpackage :lisp-stat-regression-linear
363 (:use :common-lisp
364 :lisp-matrix
365 :lisp-stat-basics
366 :lisp-stat-compound-data
367 :lisp-stat-descriptive-statistics )
368 (:shadowing-import-from :lisp-stat-object-system
369 call-method call-next-method)
370 (:export regression-model fit-model
372 estimates covariance-matrix
373 ;; functions for helpers
374 lm xtxinv
375 print-object ;; for method dispatch
378 (defpackage :common-lisp-statistics
379 (:documentation "Experimentation package for LispStat. Serious work
380 should be packaged up as a separate but similar package to help
381 drive reproducibility. By this I mean, creating a
382 data/analytics/analysis package with the minimal set of
383 objects/packages required.")
384 (:nicknames :cls :common-lisp-statistics :lisp-stat)
385 (:use :common-lisp
386 :xarray ;; generic reference -- internally supporting array, lol structs
387 :listoflist
388 :lisp-matrix ;; conversion to a more robust linalg approach
389 :lisp-stat-config
390 :lisp-stat-object-system
391 :lisp-stat-compound-data
392 :lisp-stat-probability
393 :lisp-stat-types
394 :lisp-stat-float
395 :lisp-stat-basics
396 :lisp-stat-data
397 :cls-dataframe
398 :cls-data
399 :lisp-stat-math
400 :lisp-stat-descriptive-statistics
401 :lisp-stat-regression-linear
402 :cls-visualize
403 ;; :cybertiggyr-dsv
404 ;; :cls-visualize-plplot
406 (:shadowing-import-from :xarray slice)
407 (:shadowing-import-from :lisp-stat-object-system
408 call-method call-next-method)
409 (:shadowing-import-from :lisp-stat-math
410 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
411 asin acos atan sinh cosh tanh asinh acosh atanh float random
412 truncate floor ceiling round minusp zerop plusp evenp oddp
413 < <= = /= >= >
414 ;;complex
415 conjugate realpart imagpart phase
417 min max
418 logand logior logxor lognot
419 ffloor fceiling ftruncate fround
420 signum cis)
421 (:export
422 ;; lisp-stat-config:
423 *default-path* *lsos-files* *basic-files* *ls-files*
424 l *cls-home-dir* *cls-data-dir* *cls-examples-dir*
426 ;; lsobjects :
427 defproto defproto2
428 defmeth send proto-slot-value
430 ;; lstypes :
431 fixnump check-nonneg-fixnum check-one-fixnum
432 check-one-nonneg-fixnum
433 check-one-real check-one-number
435 ;; lsmacros:
437 ;; lsfloat :
438 machine-epsilon
440 ;; compound :
441 compound-data-p *compound-data-proto* compound-object-p
442 compound-data-seq compound-data-length
443 element-list element-seq
444 sort-data order rank
445 recursive-map-elements map-elements
446 repeat
447 check-sequence
448 get-next-element make-next-element set-next-element
449 ;; sequencep
450 iseq
451 ordered-nneg-seq
452 select which
453 difference rseq
455 ;; lsmath.lsp
456 ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
457 tan asin acos atan sinh cosh tanh asinh acosh atanh float random
458 truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
459 /= >= > ;; complex
460 conjugate realpart imagpart phase min max
461 logand logior logxor lognot ffloor fceiling ftruncate fround
462 signum cis
464 #| ;; The following need to be re-found in lisp-matrix...
466 ;; matrices.lisp
467 matrixp num-rows num-cols matmult identity-matrix diagonal row-list
468 column-list inner-product outer-product cross-product transpose
469 bind-columns bind-rows
471 ;; linalg.lisp
472 chol-decomp lu-decomp lu-solve determinant inverse
473 sv-decomp qr-decomp rcondest make-rotation spline
474 kernel-dens kernel-smooth
475 fft make-sweep-matrix sweep-operator ax+y eigen
476 check-real
477 covariance-matrix matrix print-matrix solve
478 backsolve eigenvalues eigenvectors accumulate cumsum combine
479 lowess
481 ;; in linalg.lisp, possibly not supported by matlisp
482 spline kernel-dens kernel-smooth
486 ;; optimize.lsp
487 newtonmax nelmeadmax
489 ;; lispstat-macros
490 make-rv-function make-rv-function-1
492 ;; xarray
493 xref xtype xdims xdim xdims*
495 ;; listoflist
496 sublists-of-same-size-p equal-listoflist transpose-listoflist
498 ;; data
499 ;; need to take this list and make it strings... specs could mean
500 ;; that we process the strings in different ways?
502 (let ((lst ()))
503 (unlist
504 (mapc #'symbol-for-symbol-to-string-or-symbol
505 (do-external-symbols (s (find-package 'lisp-stat-data) lst) (push s lst))))
506 lst)
508 open-file-dialog read-data-file read-data-columns load-data
509 load-example *variables* *ask-on-redefine*
510 def variables savevar undef
511 ;; dataframe
512 dataframe-like dataframe-array make-dataframe
513 varlabels caselabels nrows ncols
514 dataframe-dimension dataframe-dimensons
515 dfref dfref-case dfref-var
516 consistent-dataframe-p
517 dataset list-of-columns list-of-rows
519 ;; listoflist
520 listoflist->dataframe listoflist->array listoflist->matrix-like
522 ;; statistics.lsp (descriptions, should probably be moved
523 ;; later...?
524 standard-deviation quantile median interquartile-range
525 fivnum sample
527 ;; probability (dists.lisp)
528 log-gamma set-seed
529 uniform-rand normal-cdf normal-quant normal-dens
530 normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
531 cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
532 chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
533 beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
534 f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
535 binomial-cdf binomial-quant binomial-pmf binomial-rand
537 ;; Here is where we have a problem -- lispstat core should be core
538 ;; data management and config problems, with packages providing
539 ;; specialized extensions to LispStat, i.e. regression, nonlin
540 ;; regression, bayesian regression via laplace approximation, etc.
542 ;; The following could be considered "recommended packages",
543 ;; similar to the idea of the recommended packages in R. Probably
544 ;; we want them to do the exporting within that package, therefore
545 ;; NOT being able to lock the "data-ish" package, but only the
546 ;; subpackages prior to export.
548 ;; regression.lsp
549 ;; -- linear regressin models.
550 regression-model fit-model
551 estimates covariance-matrix
553 regression-model-proto x y intercept sweep-matrix
554 basis weights included total-sum-of-squares residual-sum-of-squares
555 predictor-names response-name case-labels
556 lm xtxinv
558 ;; nonlin.lsp
559 ;; -- nonlinear regression models
560 nreg-model nreg-model-proto mean-function theta-hat epsilon
561 count-limit verbose
562 ;; we might need something like xtxinv here? But should be
563 ;; encapsulated, so we use the one in regression.lisp
565 ;; bayes.lsp
566 bayes-model bayes-model-proto bayes-internals
568 ;; plots.lisp
569 plot-ex
570 contour-plot-ex
571 fn-contour-plot-ex
572 shade-plot-ex
573 3D-plot-ex
578 ;;;; PACKAGES FOR USEABILITY
580 (defpackage :lisp-stat-data-examples
581 (:documentation "Example data for unittests, examples, illustrations,")
582 (:use :common-lisp
583 :common-lisp-statistics)
584 (:shadowing-import-from :lisp-stat
585 call-method call-next-method
587 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
588 asin acos atan sinh cosh tanh asinh acosh atanh float random
589 truncate floor ceiling round minusp zerop plusp evenp oddp
590 < <= = /= >= > > ;; complex
591 conjugate realpart imagpart phase
592 min max logand logior logxor lognot ffloor fceiling
593 ftruncate fround signum cis
595 <= float imagpart)
596 (:export iron aluminum absorbtion
597 diabetes dlabs))
600 (defpackage :lisp-stat-user
601 (:documentation "Experimentation package for LispStat. Serious work
602 should be placed in a similar package elsewhere for
603 reproducibility. But this should hint as to what needs to be done
604 for a user- or analysis-package.")
605 (:nicknames :ls-user :cls-user)
606 (:use :common-lisp ; always needed for user playgrounds!
607 :lisp-matrix
608 :common-lisp-statistics
609 :lisp-stat-data-examples) ;; this last is to have 'things to play with'
610 (:shadowing-import-from :lisp-stat
611 call-method call-next-method
613 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
614 asin acos atan sinh cosh tanh asinh acosh atanh float random
615 truncate floor ceiling round minusp zerop plusp evenp oddp
616 < <= = /= >= > > ;; complex
617 conjugate realpart imagpart phase
618 min max logand logior logxor lognot ffloor fceiling
619 ftruncate fround signum cis
621 <= float imagpart))
623 (defpackage :lisp-stat-unittests
624 (:use :common-lisp
625 :lift :lisp-matrix
626 :lisp-stat :lisp-stat-data-examples)
627 (:shadowing-import-from :lisp-stat
628 call-method call-next-method ;; objects
629 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
630 asin acos atan sinh cosh tanh asinh acosh atanh float random
631 truncate floor ceiling round minusp zerop plusp evenp oddp
632 < <= = /= >= > ;; complex
633 conjugate realpart imagpart phase
634 min max logand logior logxor lognot ffloor fceiling
635 ftruncate fround signum cis)
636 (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
637 almost= almost=lists numerical=)) ; compare
639 (defpackage :cls-dataframe-example
640 (:use :common-lisp
641 :lift :lisp-stat-unittests
642 :lisp-stat-data-examples
643 :cls-dataframe)
644 (:export absorbtion aluminum iron))
647 (defpackage :lisp-stat-optimize
648 (:use :common-lisp
649 :cffi
650 :lisp-matrix
651 :lisp-stat-ffi-int
652 :lisp-stat-object-system
653 :lisp-stat-types
654 :lisp-stat-compound-data
655 :lisp-stat-math
656 :lisp-stat-float
657 :lisp-stat-basics
659 :lisp-stat-matrix
660 :lisp-stat-linalg-data
661 :lisp-stat-linalg
664 (:shadowing-import-from :lisp-stat-object-system
665 call-method call-next-method)
666 (:shadowing-import-from :lisp-stat-math
667 expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
668 asin acos atan sinh cosh tanh asinh acosh atanh float random
669 truncate floor ceiling round minusp zerop plusp evenp oddp
670 < <= = /= >= > complex conjugate realpart imagpart phase
671 min max logand logior logxor lognot ffloor fceiling
672 ftruncate fround signum cis)
673 (:export
674 ;; derivatives
675 numgrad numhess
677 ;; optimization
678 newtonmax nelmeadmax))