src/packages.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2009-07-14 14:16:22 tony>
   4 ;;; Creation:   <2008-03-11 19:18:34 user>
   5 ;;; File:       packages.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c)2007--2008, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   8 ;;;             on how it arrives.
   9 ;;; Purpose:    package structure description for lispstat
  10
  11 ;;; What is this talk of 'release'? Klingons do not make software
  12 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  13 ;;; designers and quality assurance people in its wake.
  14
  15 (in-package :cl-user)
  16
  17 ;;; Basics
  18
  19
  20 (defpackage :lisp-stat-object-system
  21   (:nicknames :ls-objects :lsos)
  22   (:use :common-lisp)
  23   (:shadow :call-method :call-next-method)
  24   (:export ls-object objectp *object* kind-of-p make-object
  25            *message-hook*
  26            *set-slot-hook* proto-slot-value self
  27            send call-next-method call-method
  28            defmeth defproto instance-slots proto-name))
  29
  30 (defpackage :lisp-stat-types
  31   (:documentation "Provides some typeing for LispStat, but is clearly
  32   a bit incomplete.")
  33   (:use :common-lisp)
  34   (:export fixnump
  35            check-nonneg-fixnum check-one-nonneg-fixnum
  36            check-one-fixnum check-one-real check-one-number))
  37
  38 (defpackage :lisp-stat-float
  39   (:use :common-lisp)
  40   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  41            machine-epsilon base-float makedouble
  42
  43            make-base-trans-fun-2 make-base-trans-fun
  44
  45            base-log base-exp base-expt base-sqrt base-sin base-cos
  46            base-tan base-asin base-acos base-atan base-sinh
  47            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
  48            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
  49            BASE-FROUND BASE-SIGNUM BASE-CIS))
  50
  51 (defpackage :lisp-stat-compound-data
  52   (:use :common-lisp
  53         :lisp-stat-object-system
  54         :lisp-stat-types)
  55   (:shadowing-import-from :lisp-stat-object-system
  56                           call-next-method call-method)
  57   (:export compound-data-p *compound-data-proto*
  58            compound-object-p
  59            compound-data-seq compound-data-length
  60            element-list element-seq
  61            sort-data order rank
  62            recursive-map-elements map-elements repeat
  63            check-sequence
  64            get-next-element make-next-element set-next-element
  65            ;; sequencep
  66            iseq ordered-nneg-seq
  67            select split-list which
  68            difference rseq
  69            flatten-list))
  70
  71 (defpackage :lisp-stat-macros
  72   (:use :common-lisp
  73         :lisp-stat-compound-data)
  74   (:export make-rv-function make-rv-function-1))
  75
  76 (defpackage :lisp-stat-basics
  77     (:use :common-lisp
  78           :lisp-stat-object-system
  79           :lisp-stat-types
  80           :lisp-stat-float
  81           :lisp-stat-macros
  82           :lisp-stat-compound-data)
  83   (:shadowing-import-from :lisp-stat-object-system
  84                           call-method call-next-method)
  85   (:export permute-array sum prod count-elements mean
  86            if-else sample))
  87
  88 (defpackage :lisp-stat-float
  89   (:use :common-lisp)
  90   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  91            machine-epsilon base-float makedouble
  92
  93            make-base-trans-fun-2 make-base-trans-fun
  94
  95            BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
  96            BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
  97            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
  98            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
  99            BASE-FROUND BASE-SIGNUM BASE-CIS))
 100
 101 (defpackage :lisp-stat-macros
 102   (:use :common-lisp
 103         :lisp-stat-compound-data)
 104   (:export make-rv-function make-rv-function-1))
 105
 106 ;;; NEW CLOS STRUCTURE
 107
 108
 109
 110 (defpackage :cls-dataframe
 111   (:use :common-lisp
 112         :xarray
 113         :lisp-matrix)
 114   (:export
 115    ;; generic container class for data -- if small enough
 116    ;; could be value, otherwise might be reference.
 117    dataframe-like
 118    dataframe-array
 119    make-dataframe
 120
 121    ;; accessors
 122    varlabels caselabels  nrows ncols
 123    dataframe-dimension dataframe-dimensons
 124    xref xtype xdims xdim xrank slice take carray
 125 #|
 126    dfref dfref-case dfref-var
 127    consistent-dataframe-p
 128 |#
 129
 130    dataset
 131    list-of-columns ;; list-of-variables
 132    list-of-rows ;; list-of-observations
 133    ))
 134
 135
 136 (defpackage :cls-data-listoflist
 137   (:use :common-lisp
 138         :lisp-matrix
 139         :cls-dataframe) ; for dataframe
 140   (:export lists-of-same-size
 141            equal-listoflist
 142            transpose-listoflist
 143            listoflist->dataframe
 144            listoflist->array
 145            listoflist->matrix-like))
 146
 147
 148 (defpackage :cls-dataimport
 149   (:documentation "Data I/O and similar import technologies.")
 150   (:use :common-lisp
 151         :lisp-stat-object-system
 152         :cls-dataframe
 153         :cls-data-listoflist
 154         :rsm.string)
 155   (:shadowing-import-from :lisp-stat-object-system
 156                           call-method call-next-method)
 157   (:export dsvstream->dataframe dsvstream->matrix dsvstream->listoflist))
 158
 159
 160 (defpackage :lisp-stat-model
 161   (:documentation "Model management for data analysis.")
 162   (:use :common-lisp
 163         :lisp-matrix)
 164   (:export
 165    ;; data structures for model and model/data combination
 166    model statistical-model analysis))
 167
 168 ;;; visualization
 169
 170 (defpackage :cls-visualize
 171   (:use :common-lisp
 172         :lisp-matrix
 173         :cls-dataframe))
 174
 175 #|
 176 (defpackage :cls-visualize-plplot
 177   (:use :common-lisp
 178         :lisp-matrix
 179         :cls-dataframe
 180         :cl-plplot-system)
 181   (:export
 182    ;; examples
 183    plot-ex contour-plot-ex fn-contour-plot-ex shade-plot-ex 3D-plot-ex))
 184 |#
 185
 186 ;;; USER PACKAGES
 187
 188 (defpackage :lisp-stat-ffi-int
 189   (:use :common-lisp
 190         :cffi)
 191   (:export ccl-store-integer ccl-store-double ccl-store-ptr
 192            get-buf ))
 193
 194 (defpackage :lisp-stat-probability
 195   (:use :common-lisp
 196         :cffi
 197         :lisp-stat-ffi-int
 198         :lisp-stat-macros)
 199   (:export log-gamma set-seed
 200            uniform-rand
 201            normal-cdf normal-quant normal-dens normal-rand
 202            bivnorm-cdf
 203            cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
 204            gamma-cdf gamma-quant gamma-dens gamma-rand
 205            chisq-cdf chisq-quant chisq-dens chisq-rand
 206            beta-cdf beta-quant beta-dens beta-rand
 207            t-cdf t-quant t-dens t-rand
 208            f-cdf f-quant f-dens f-rand
 209            poisson-cdf poisson-quant poisson-pmf poisson-rand
 210            binomial-cdf binomial-quant binomial-pmf binomial-rand))
 211
 212
 213
 214 (defpackage :lisp-stat-math
 215    (:use :common-lisp
 216          :lisp-stat-object-system
 217          :lisp-stat-macros
 218          :lisp-stat-compound-data
 219          :lisp-stat-float)
 220    (:shadowing-import-from :lisp-stat-object-system
 221                            call-method call-next-method)
 222    (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 223            asin acos atan sinh cosh tanh asinh acosh atanh float random
 224            truncate floor ceiling round minusp zerop plusp evenp oddp
 225            < <= = /= >= > ;; complex
 226            conjugate realpart imagpart phase
 227            min max logand logior logxor lognot ffloor fceiling
 228            ftruncate fround signum cis)
 229    (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 230            tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 231            truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 232            /= >= > ;; complex
 233            conjugate realpart imagpart phase min max
 234            logand logior logxor lognot ffloor fceiling ftruncate fround
 235            signum cis)
 236    (:documentation "Vectorization of numerical functions"))
 237
 238
 239 #| ;; some of this goes back in, but not all of it?
 240 (defpackage :lisp-stat-linalg
 241   (:use :common-lisp
 242         :cffi
 243         :lisp-matrix
 244         :lisp-stat-math
 245         :lisp-stat-types
 246         :lisp-stat-float
 247         :lisp-stat-compound-data)
 248   (:shadowing-import-from :lisp-stat-math
 249           expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 250           asin acos atan sinh cosh tanh asinh acosh atanh float random
 251           truncate floor ceiling round minusp zerop plusp evenp oddp
 252           < <= = /= >= > complex conjugate realpart imagpart phase
 253           min max logand logior logxor lognot ffloor fceiling
 254           ftruncate fround signum cis)
 255   (:export chol-decomp lu-decomp lu-solve determinant inverse
 256            sv-decomp qr-decomp rcondest make-rotation spline
 257            kernel-dens kernel-smooth
 258            fft make-sweep-matrix sweep-operator ax+y eigen
 259
 260            check-real ;; for optimize
 261
 262            covariance-matrix matrix print-matrix solve
 263            backsolve eigenvalues eigenvectors accumulate cumsum combine
 264            lowess))
 265
 266 |#
 267
 268
 269
 270
 271 (defpackage :lisp-stat-data
 272   (:documentation "Data management, integration, I/O, and other data technologies.")
 273   (:nicknames :ls-data)
 274   (:use :common-lisp
 275         :lisp-stat-object-system
 276         :lisp-stat-config
 277         :lisp-stat-types
 278         :lisp-stat-compound-data)
 279   (:shadowing-import-from :lisp-stat-object-system
 280                           call-method call-next-method)
 281   (:export open-file-dialog read-data-file read-data-columns load-data
 282            load-example *variables* *ask-on-redefine*
 283            def variables savevar undef))
 284
 285 (defpackage :lisp-stat-descriptive-statistics
 286  (:use :common-lisp
 287        :lisp-stat-data
 288        :lisp-stat-math
 289        :lisp-stat-compound-data
 290        :lisp-matrix
 291        :lisp-stat-basics)
 292    (:shadowing-import-from :lisp-stat-math ;; life is a vector!
 293       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 294       asin acos atan sinh cosh tanh asinh acosh atanh float random
 295       truncate floor ceiling round minusp zerop plusp evenp oddp
 296       < <= = /= >= > ;; complex
 297       conjugate realpart imagpart phase
 298       min max logand logior logxor lognot ffloor fceiling
 299       ftruncate fround signum cis)
 300    (:export standard-deviation
 301             quantile median interquartile-range
 302             fivnum sample))
 303
 304
 305 (defpackage :lisp-stat-regression-linear
 306   (:use :common-lisp
 307         :lisp-matrix
 308         :lisp-stat-basics
 309         :lisp-stat-compound-data
 310         :lisp-stat-descriptive-statistics )
 311   (:shadowing-import-from :lisp-stat-object-system
 312                           call-method call-next-method)
 313   (:export regression-model fit-model
 314
 315            estimates covariance-matrix
 316            ;; functions for helpers
 317            lm xtxinv
 318            print-object ;; for method dispatch
 319            ))
 320
 321 (defpackage :common-lisp-statistics
 322   (:documentation "Experimentation package for LispStat.  Serious work
 323     should be packaged up elsewhere for reproducibility.  By this I
 324     mean, creating a data/analytics/analysis package with the minimal
 325     set of objects required.")
 326   (:nicknames :cls :common-lisp-statistics :lisp-stat)
 327   (:use :common-lisp
 328         :lisp-stat-config
 329         :lisp-stat-object-system
 330         :lisp-stat-compound-data
 331         :lisp-stat-probability
 332         :lisp-stat-types
 333         :lisp-stat-float
 334         :lisp-stat-basics
 335         :lisp-stat-data
 336         :cls-dataframe
 337         :cls-data-listoflist
 338         :lisp-stat-math
 339         :lisp-matrix ;; conversion to a more robust linalg approach
 340         :lisp-stat-descriptive-statistics
 341         :lisp-stat-regression-linear
 342         ;; :cybertiggyr-dsv
 343         :cls-visualize
 344         ;; :cls-visualize-plplot
 345         )
 346   (:shadowing-import-from :lisp-stat-object-system
 347                           call-method call-next-method)
 348   (:shadowing-import-from :lisp-stat-math
 349         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 350         asin acos atan sinh cosh tanh asinh acosh atanh float random
 351         truncate floor ceiling round minusp zerop plusp evenp oddp
 352         < <= = /= >= >
 353         ;;complex
 354         conjugate realpart imagpart phase
 355
 356         min max
 357         logand logior logxor lognot
 358         ffloor fceiling ftruncate fround
 359         signum cis)
 360   (:export
 361    ;; lisp-stat-config:
 362    *default-path* *lsos-files* *basic-files* *ls-files*
 363    *lispstat-home-dir* *lispstat-data-dir* *lispstat-examples-dir*
 364
 365    ;; lsobjects :
 366    defproto defproto2
 367    defmeth send proto-slot-value
 368
 369    ;; lstypes :
 370    fixnump check-nonneg-fixnum check-one-fixnum
 371    check-one-nonneg-fixnum
 372    check-one-real check-one-number
 373
 374    ;; lsmacros:
 375
 376    ;; lsfloat :
 377    machine-epsilon
 378
 379    ;; compound :
 380    compound-data-p *compound-data-proto* compound-object-p
 381    compound-data-seq compound-data-length
 382    element-list element-seq
 383    sort-data order rank
 384    recursive-map-elements map-elements
 385    repeat
 386    check-sequence
 387    get-next-element make-next-element set-next-element
 388    ;; sequencep
 389    iseq
 390    ordered-nneg-seq
 391    select which
 392    difference rseq
 393
 394    ;; lsmath.lsp
 395    ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 396    tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 397    truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 398    /= >= > ;; complex
 399    conjugate realpart imagpart phase min max
 400    logand logior logxor lognot ffloor fceiling ftruncate fround
 401    signum cis
 402
 403 #| ;; The following need to be re-found in lisp-matrix...
 404
 405    ;; matrices.lisp
 406    matrixp num-rows num-cols matmult identity-matrix diagonal row-list
 407    column-list inner-product outer-product cross-product transpose
 408    bind-columns bind-rows
 409
 410    ;; linalg.lisp
 411    chol-decomp lu-decomp lu-solve determinant inverse
 412    sv-decomp qr-decomp rcondest make-rotation spline
 413    kernel-dens kernel-smooth
 414    fft make-sweep-matrix sweep-operator ax+y eigen
 415    check-real
 416    covariance-matrix matrix print-matrix solve
 417    backsolve eigenvalues eigenvectors accumulate cumsum combine
 418    lowess
 419
 420    ;; in linalg.lisp, possibly not supported by matlisp
 421    spline kernel-dens kernel-smooth
 422
 423 |#
 424
 425    ;; optimize.lsp
 426    newtonmax nelmeadmax
 427
 428    ;; lispstat-macros
 429    make-rv-function make-rv-function-1
 430
 431    ;; xarray
 432    xref xtype xdims xdim xdims*
 433
 434    ;; data
 435    open-file-dialog read-data-file read-data-columns load-data
 436    load-example *variables* *ask-on-redefine*
 437    def variables savevar undef
 438
 439    ;; dataframe
 440    dataframe-like dataframe-array make-dataframe
 441    varlabels caselabels nrows ncols
 442    dataframe-dimension dataframe-dimensons
 443    dfref dfref-case dfref-var
 444    consistent-dataframe-p
 445    dataset list-of-columns list-of-rows
 446
 447    ;; listoflist
 448    lists-of-same-size equal-listoflist
 449    transpose-listoflist
 450    listoflist->dataframe listoflist->array listoflist->matrix-like
 451
 452    ;; statistics.lsp  (descriptions, should probably be moved
 453    ;; later...?
 454    standard-deviation quantile median interquartile-range
 455    fivnum sample
 456
 457    ;; probability (dists.lisp)
 458    log-gamma set-seed
 459    uniform-rand normal-cdf normal-quant normal-dens
 460    normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
 461    cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
 462    chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
 463    beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
 464    f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
 465    binomial-cdf binomial-quant binomial-pmf binomial-rand
 466
 467    ;; Here is where we have a problem -- lispstat core should be core
 468    ;; data management and config problems, with packages providing
 469    ;; specialized extensions to LispStat, i.e. regression, nonlin
 470    ;; regression, bayesian regression via laplace approximation, etc.
 471
 472    ;; The following could be considered "recommended packages",
 473    ;; similar to the idea of the recommended packages in R.  Probably
 474    ;; we want them to do the exporting within that package, therefore
 475    ;; NOT being able to lock the "data-ish" package, but only the
 476    ;; subpackages prior to export.
 477
 478    ;; regression.lsp
 479    ;; -- linear regressin models.
 480    regression-model fit-model
 481    estimates covariance-matrix
 482
 483    regression-model-proto x y intercept sweep-matrix
 484    basis weights included total-sum-of-squares residual-sum-of-squares
 485    predictor-names response-name case-labels
 486    lm xtxinv
 487
 488    ;; nonlin.lsp
 489    ;; -- nonlinear regression models
 490    nreg-model nreg-model-proto mean-function theta-hat epsilon
 491    count-limit verbose
 492    ;; we might need something like xtxinv here?  But should be
 493    ;; encapsulated, so we use the one in regression.lisp
 494
 495    ;; bayes.lsp
 496    bayes-model bayes-model-proto bayes-internals
 497
 498    ;; plots.lisp
 499    plot-ex
 500    contour-plot-ex
 501    fn-contour-plot-ex
 502    shade-plot-ex
 503    3D-plot-ex
 504
 505    ))
 506
 507
 508 ;;;; PACKAGES FOR USEABILITY
 509
 510 (defpackage :lisp-stat-data-examples
 511   (:documentation "Example data for unittests, examples, illustrations,")
 512   (:use :common-lisp
 513         :common-lisp-statistics)
 514   (:shadowing-import-from :lisp-stat
 515                           call-method call-next-method
 516
 517       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 518       asin acos atan sinh cosh tanh asinh acosh atanh float random
 519       truncate floor ceiling round minusp zerop plusp evenp oddp
 520       < <= = /= >= > > ;; complex
 521       conjugate realpart imagpart phase
 522       min max logand logior logxor lognot ffloor fceiling
 523       ftruncate fround signum cis
 524
 525       <= float imagpart)
 526   (:export iron aluminum absorbtion
 527            diabetes dlabs))
 528
 529
 530 (defpackage :lisp-stat-user
 531   (:documentation "Experimentation package for LispStat.  Serious work
 532     should be placed in a similar package elsewhere for
 533     reproducibility.  But this should hint as to what needs to be done
 534     for a user- or analysis-package.")
 535   (:nicknames :ls-user)
 536   (:use :common-lisp ; always needed for user playgrounds!
 537         :lisp-matrix
 538         :common-lisp-statistics
 539         :lisp-stat-data-examples) ;; this last is to have 'things to play with'
 540   (:shadowing-import-from :lisp-stat
 541                           call-method call-next-method
 542
 543       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 544       asin acos atan sinh cosh tanh asinh acosh atanh float random
 545       truncate floor ceiling round minusp zerop plusp evenp oddp
 546       < <= = /= >= > > ;; complex
 547       conjugate realpart imagpart phase
 548       min max logand logior logxor lognot ffloor fceiling
 549       ftruncate fround signum cis
 550
 551       <= float imagpart))
 552
 553 (defpackage :lisp-stat-unittests
 554   (:use :common-lisp
 555         :lift :lisp-matrix
 556         :lisp-stat :lisp-stat-data-examples)
 557   (:shadowing-import-from :lisp-stat
 558                           call-method call-next-method ;; objects
 559         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
 560         asin acos atan sinh cosh tanh asinh acosh atanh float random
 561         truncate floor ceiling round minusp zerop plusp evenp oddp
 562         < <= = /= >= > ;; complex
 563         conjugate realpart imagpart phase
 564         min max logand logior logxor lognot ffloor fceiling
 565         ftruncate fround signum cis)
 566   (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
 567            almost= almost=lists numerical=)) ; compare
 568
 569 (defpackage :cls-dataframe-example
 570   (:use :common-lisp
 571         :lift  :lisp-stat-unittests
 572         :lisp-stat-data-examples
 573         :cls-dataframe)
 574   (:export absorbtion aluminum iron))
 575
 576
 577 (defpackage :lisp-stat-optimize
 578  (:use :common-lisp
 579        :cffi
 580        :lisp-matrix
 581        :lisp-stat-ffi-int
 582        :lisp-stat-object-system
 583        :lisp-stat-types
 584        :lisp-stat-compound-data
 585        :lisp-stat-math
 586        :lisp-stat-float
 587        :lisp-stat-basics
 588 #|
 589        :lisp-stat-matrix
 590        :lisp-stat-linalg-data
 591        :lisp-stat-linalg
 592 |#
 593        )
 594  (:shadowing-import-from :lisp-stat-object-system
 595                          call-method call-next-method)
 596  (:shadowing-import-from :lisp-stat-math
 597            expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 598            asin acos atan sinh cosh tanh asinh acosh atanh float random
 599            truncate floor ceiling round minusp zerop plusp evenp oddp
 600            < <= = /= >= > complex conjugate realpart imagpart phase
 601            min max logand logior logxor lognot ffloor fceiling
 602            ftruncate fround signum cis)
 603  (:export
 604      ;; derivatives
 605      numgrad numhess
 606
 607      ;; optimization
 608      newtonmax nelmeadmax))