src/packages.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2009-04-15 08:47:26 tony>
   4 ;;; Creation:   <2008-03-11 19:18:34 user>
   5 ;;; File:       packages.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c)2007--2008, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   8 ;;;             on how it arrives.
   9 ;;; Purpose:    package structure description for lispstat
  10
  11 ;;; What is this talk of 'release'? Klingons do not make software
  12 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  13 ;;; designers and quality assurance people in its wake.
  14
  15 ;;; This organization and structure is new to the 21st Century
  16 ;;; version.
  17
  18 (in-package :cl-user)
  19
  20 ;;; Basics
  21
  22
  23 (defpackage :lisp-stat-object-system
  24   (:nicknames :ls-objects :lsos)
  25   (:use :common-lisp)
  26   (:shadow :call-method :call-next-method)
  27   (:export ls-object objectp *object* kind-of-p make-object
  28            *message-hook*
  29            *set-slot-hook* proto-slot-value self
  30            send call-next-method call-method
  31            defmeth defproto instance-slots proto-name))
  32
  33 (defpackage :lisp-stat-types
  34   (:documentation "Provides some typeing for LispStat, but is clearly
  35   a bit incomplete.")
  36   (:use :common-lisp)
  37   (:export fixnump
  38            check-nonneg-fixnum check-one-nonneg-fixnum
  39            check-one-fixnum check-one-real check-one-number))
  40
  41 (defpackage :lisp-stat-float
  42   (:use :common-lisp)
  43   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  44            machine-epsilon base-float makedouble
  45
  46            make-base-trans-fun-2 make-base-trans-fun
  47
  48            base-log base-exp base-expt base-sqrt base-sin base-cos
  49            base-tan base-asin base-acos base-atan base-sinh
  50            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
  51            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
  52            BASE-FROUND BASE-SIGNUM BASE-CIS))
  53
  54 (defpackage :lisp-stat-compound-data
  55   (:use :common-lisp
  56         :lisp-stat-object-system
  57         :lisp-stat-types)
  58   (:shadowing-import-from :lisp-stat-object-system
  59                           call-next-method call-method)
  60   (:export compound-data-p *compound-data-proto*
  61            compound-object-p
  62            compound-data-seq compound-data-length
  63            element-list element-seq
  64            sort-data order rank
  65            recursive-map-elements map-elements repeat
  66            check-sequence
  67            get-next-element make-next-element set-next-element
  68            ;; sequencep
  69            iseq ordered-nneg-seq
  70            select split-list which
  71            difference rseq
  72            flatten-list))
  73
  74 (defpackage :lisp-stat-macros
  75   (:use :common-lisp
  76         :lisp-stat-compound-data)
  77   (:export make-rv-function make-rv-function-1))
  78
  79 (defpackage :lisp-stat-basics
  80     (:use :common-lisp
  81           :lisp-stat-object-system
  82           :lisp-stat-types
  83           :lisp-stat-float
  84           :lisp-stat-macros
  85           :lisp-stat-compound-data)
  86   (:shadowing-import-from :lisp-stat-object-system
  87                           call-method call-next-method)
  88   (:export permute-array sum prod count-elements mean
  89            if-else sample))
  90
  91 (defpackage :lisp-stat-float
  92   (:use :common-lisp)
  93   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  94            machine-epsilon base-float makedouble
  95
  96            make-base-trans-fun-2 make-base-trans-fun
  97
  98            BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
  99            BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
 100            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
 101            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
 102            BASE-FROUND BASE-SIGNUM BASE-CIS))
 103
 104 (defpackage :lisp-stat-macros
 105   (:use :common-lisp
 106         :lisp-stat-compound-data)
 107   (:export make-rv-function make-rv-function-1))
 108
 109 ;;; NEW CLOS STRUCTURE
 110
 111
 112
 113 (defpackage :lisp-stat-dataframe
 114   (:use :common-lisp
 115         :lisp-matrix)
 116   (:export
 117    ;; generic container class for data -- if small enough
 118    ;; could be value, otherwise might be reference.
 119    dataframe-like
 120    dataframe-array
 121    make-dataframe
 122
 123    ;; accessors
 124    varlabels caselabels nrows ncols
 125    dataframe-dimension dataframe-dimensons
 126    dfref dfref-case dfref-var
 127    consistent-dataframe-p
 128
 129    dataset
 130    list-of-columns ;; list-of-variables
 131    list-of-rows ;; list-of-observations
 132    ))
 133
 134
 135 (defpackage :cls-data-listoflist
 136   (:use :common-lisp
 137         :lisp-matrix
 138         :lisp-stat-dataframe) ; for dataframe
 139   (:export lists-of-same-size
 140            equal-listoflist
 141            transpose-listoflist
 142            listoflist->dataframe
 143            listoflist->array
 144            listoflist->matrix-like))
 145
 146 (defpackage :lisp-stat-model
 147   (:documentation "Model management for data analysis.")
 148   (:use :common-lisp
 149         :lisp-matrix)
 150   (:export
 151    ;; data structures for model and model/data combination
 152    model statistical-model analysis))
 153
 154 #|
 155  (defpackage :lisp-stat-regression-linear-clos
 156   (:use :common-lisp
 157         :lisp-matrix
 158         :lisp-stat-dataframe)
 159   (:export regression-model))
 160 |#
 161
 162 (defpackage :lisp-stat-visualize
 163   (:use :common-lisp
 164         :lisp-matrix
 165         :lisp-stat-dataframe))
 166
 167 (defpackage :lisp-stat-visualize-plplot
 168   (:use :common-lisp
 169         :lisp-matrix
 170         :lisp-stat-dataframe
 171         :cl-plplot-system)
 172   (:export
 173    plot-ex
 174    contour-plot-ex
 175    fn-contour-plot-ex
 176    shade-plot-ex
 177    3D-plot-ex ))
 178
 179
 180 ;;; USER PACKAGES
 181
 182 (defpackage :lisp-stat-ffi-int
 183   (:use :common-lisp
 184         :cffi)
 185   (:export ccl-store-integer ccl-store-double ccl-store-ptr
 186            get-buf ))
 187
 188 (defpackage :lisp-stat-probability
 189   (:use :common-lisp
 190         :cffi
 191         :lisp-stat-ffi-int
 192         :lisp-stat-macros)
 193   (:export log-gamma set-seed
 194            uniform-rand
 195            normal-cdf normal-quant normal-dens normal-rand
 196            bivnorm-cdf
 197            cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
 198            gamma-cdf gamma-quant gamma-dens gamma-rand
 199            chisq-cdf chisq-quant chisq-dens chisq-rand
 200            beta-cdf beta-quant beta-dens beta-rand
 201            t-cdf t-quant t-dens t-rand
 202            f-cdf f-quant f-dens f-rand
 203            poisson-cdf poisson-quant poisson-pmf poisson-rand
 204            binomial-cdf binomial-quant binomial-pmf binomial-rand))
 205
 206
 207
 208 (defpackage :lisp-stat-math
 209    (:use :common-lisp
 210          :lisp-stat-object-system
 211          :lisp-stat-macros
 212          :lisp-stat-compound-data
 213          :lisp-stat-float)
 214    (:shadowing-import-from :lisp-stat-object-system
 215                            call-method call-next-method)
 216    (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 217            asin acos atan sinh cosh tanh asinh acosh atanh float random
 218            truncate floor ceiling round minusp zerop plusp evenp oddp
 219            < <= = /= >= > ;; complex
 220            conjugate realpart imagpart phase
 221            min max logand logior logxor lognot ffloor fceiling
 222            ftruncate fround signum cis)
 223    (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 224            tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 225            truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 226            /= >= > ;; complex
 227            conjugate realpart imagpart phase min max
 228            logand logior logxor lognot ffloor fceiling ftruncate fround
 229            signum cis)
 230    (:documentation "Vectorization of numerical functions"))
 231
 232
 233 #| ;; some of this goes back in, but not all of it?
 234 (defpackage :lisp-stat-linalg
 235   (:use :common-lisp
 236         :cffi
 237         :lisp-matrix
 238         :lisp-stat-math
 239         :lisp-stat-types
 240         :lisp-stat-float
 241         :lisp-stat-compound-data)
 242   (:shadowing-import-from :lisp-stat-math
 243           expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 244           asin acos atan sinh cosh tanh asinh acosh atanh float random
 245           truncate floor ceiling round minusp zerop plusp evenp oddp
 246           < <= = /= >= > complex conjugate realpart imagpart phase
 247           min max logand logior logxor lognot ffloor fceiling
 248           ftruncate fround signum cis)
 249   (:export chol-decomp lu-decomp lu-solve determinant inverse
 250            sv-decomp qr-decomp rcondest make-rotation spline
 251            kernel-dens kernel-smooth
 252            fft make-sweep-matrix sweep-operator ax+y eigen
 253
 254            check-real ;; for optimize
 255
 256            covariance-matrix matrix print-matrix solve
 257            backsolve eigenvalues eigenvectors accumulate cumsum combine
 258            lowess))
 259
 260 |#
 261
 262
 263
 264
 265 (defpackage :lisp-stat-data
 266   (:documentation "Data management, integration, I/O, and other data technologies.")
 267   (:nicknames :ls-data)
 268   (:use :common-lisp
 269         :lisp-stat-object-system
 270         :lisp-stat-config
 271         :lisp-stat-types
 272         :lisp-stat-compound-data)
 273   (:shadowing-import-from :lisp-stat-object-system
 274                           call-method call-next-method)
 275   (:export open-file-dialog read-data-file read-data-columns load-data
 276            load-example *variables* *ask-on-redefine*
 277            def variables savevar undef))
 278
 279 (defpackage :lisp-stat-descriptive-statistics
 280  (:use :common-lisp
 281        :lisp-stat-data
 282        :lisp-stat-math
 283        :lisp-stat-compound-data
 284        :lisp-matrix
 285        :lisp-stat-basics)
 286    (:shadowing-import-from :lisp-stat-math ;; life is a vector!
 287       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 288       asin acos atan sinh cosh tanh asinh acosh atanh float random
 289       truncate floor ceiling round minusp zerop plusp evenp oddp
 290       < <= = /= >= > ;; complex
 291       conjugate realpart imagpart phase
 292       min max logand logior logxor lognot ffloor fceiling
 293       ftruncate fround signum cis)
 294    (:export standard-deviation
 295             quantile median interquartile-range
 296             fivnum sample))
 297
 298
 299 (defpackage :lisp-stat-regression-linear
 300   (:use :common-lisp
 301         :lisp-matrix
 302         :lisp-stat-object-system
 303         :lisp-stat-basics
 304         :lisp-stat-compound-data
 305         :lisp-stat-descriptive-statistics )
 306   (:shadowing-import-from :lisp-stat-object-system
 307                           call-method call-next-method)
 308   (:export regression-model fit-model
 309
 310            estimates covariance-matrix
 311            ;; functions for helpers
 312            lm xtxinv
 313            print-object ;; for method dispatch
 314
 315            ;; OLD to remove
 316
 317            regression-model-proto x y intercept
 318            ;; sweep-matrix
 319            basis weights included
 320            total-sum-of-squares residual-sum-of-squares
 321            predictor-names response-name case-labels
 322            ))
 323
 324 (defpackage :lisp-stat
 325     (:documentation "Experimentation package for LispStat.  Serious
 326     work should be packaged up elsewhere for reproducibility.  By this
 327     I mean, creating a data/analytics/analysis package with the
 328     minimal set of objects required.")
 329   (:use :common-lisp
 330         :lisp-stat-object-system
 331         :lisp-stat-compound-data
 332         :lisp-stat-probability
 333         :lisp-stat-types
 334         :lisp-stat-float
 335         :lisp-stat-basics
 336         :lisp-stat-data
 337         :lisp-stat-dataframe
 338         :lisp-stat-math
 339         :lisp-matrix ;; conversion to a more robust linalg approach
 340         :lisp-stat-descriptive-statistics
 341         :lisp-stat-regression-linear
 342         :cybertiggyr-dsv)
 343   (:shadowing-import-from :lisp-stat-object-system
 344                           call-method call-next-method)
 345   (:shadowing-import-from :lisp-stat-math
 346         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 347         asin acos atan sinh cosh tanh asinh acosh atanh float random
 348         truncate floor ceiling round minusp zerop plusp evenp oddp
 349         < <= = /= >= >
 350         ;;complex
 351         conjugate realpart imagpart phase
 352
 353         min max
 354         logand logior logxor lognot
 355         ffloor fceiling ftruncate fround
 356         signum cis)
 357   (:export
 358    ;; lsobjects :
 359    defproto defproto2
 360    defmeth send
 361    proto-slot-value
 362
 363    ;; lstypes :
 364    fixnump check-nonneg-fixnum check-one-fixnum
 365    check-one-nonneg-fixnum
 366    check-one-real check-one-number
 367
 368    ;; lsmacros:
 369
 370    ;; lsfloat :
 371    machine-epsilon
 372
 373    ;; compound :
 374    compound-data-p *compound-data-proto* compound-object-p
 375    compound-data-seq compound-data-length
 376    element-list element-seq
 377    sort-data order rank
 378    recursive-map-elements map-elements
 379    repeat
 380    check-sequence
 381    get-next-element make-next-element set-next-element
 382    ;; sequencep
 383    iseq
 384    ordered-nneg-seq
 385    select which
 386    difference rseq
 387
 388    ;; lsmath.lsp
 389    ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 390    tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 391    truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 392    /= >= > ;; complex
 393    conjugate realpart imagpart phase min max
 394    logand logior logxor lognot ffloor fceiling ftruncate fround
 395    signum cis
 396
 397 #| ;; The following need to be re-found in lisp-matrix...
 398
 399    ;; matrices.lisp
 400    matrixp num-rows num-cols matmult identity-matrix diagonal row-list
 401    column-list inner-product outer-product cross-product transpose
 402    bind-columns bind-rows
 403
 404    ;; linalg.lisp
 405    chol-decomp lu-decomp lu-solve determinant inverse
 406    sv-decomp qr-decomp rcondest make-rotation spline
 407    kernel-dens kernel-smooth
 408    fft make-sweep-matrix sweep-operator ax+y eigen
 409    check-real
 410    covariance-matrix matrix print-matrix solve
 411    backsolve eigenvalues eigenvectors accumulate cumsum combine
 412    lowess
 413
 414    ;; in linalg.lisp, possibly not supported by matlisp
 415    spline kernel-dens kernel-smooth
 416
 417 |#
 418
 419    ;; optimize.lsp
 420    newtonmax nelmeadmax
 421
 422    ;; lispstat-macros
 423    make-rv-function make-rv-function-1
 424
 425    ;; data
 426    open-file-dialog read-data-file read-data-columns load-data
 427    load-example *variables* *ask-on-redefine*
 428    def variables savevar undef
 429
 430    ;; dataframe
 431    dataframe-like dataframe-array make-dataframe
 432    varlabels caselabels nrows ncols
 433    dataframe-dimension dataframe-dimensons
 434    dfref dfref-case dfref-var
 435    consistent-dataframe-p
 436    dataset
 437    list-of-columns list-of-rows
 438
 439    ;; listoflist
 440    lists-of-same-size equal-listoflist
 441    transpose-listoflist
 442    listoflist->dataframe listoflist->array listoflist->matrix-like
 443
 444    ;; statistics.lsp  (descriptions, should probably be moved
 445    ;; later...?
 446    standard-deviation quantile median interquartile-range
 447    fivnum sample
 448
 449    ;; probability (dists.lisp)
 450    log-gamma set-seed
 451    uniform-rand normal-cdf normal-quant normal-dens
 452    normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
 453    cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
 454    chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
 455    beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
 456    f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
 457    binomial-cdf binomial-quant binomial-pmf binomial-rand
 458
 459    ;; Here is where we have a problem -- lispstat core should be core
 460    ;; data management and config problems, with packages providing
 461    ;; specialized extensions to LispStat, i.e. regression, nonlin
 462    ;; regression, bayesian regression via laplace approximation, etc.
 463
 464    ;; The following could be considered "recommended packages",
 465    ;; similar to the idea of the recommended packages in R.  Probably
 466    ;; we want them to do the exporting within that package, therefore
 467    ;; NOT being able to lock the "data-ish" package, but only the
 468    ;; subpackages prior to export.
 469
 470    ;; regression.lsp
 471    ;; -- linear regressin models.
 472    regression-model fit-model
 473    estimates covariance-matrix
 474
 475    regression-model-proto x y intercept sweep-matrix
 476    basis weights included total-sum-of-squares residual-sum-of-squares
 477    predictor-names response-name case-labels
 478    lm xtxinv
 479
 480    ;; nonlin.lsp
 481    ;; -- nonlinear regression models
 482    nreg-model nreg-model-proto mean-function theta-hat epsilon
 483    count-limit verbose
 484    ;; we might need something like xtxinv here?  But should be
 485    ;; encapsulated, so we use the one in regression.lisp
 486
 487    ;; bayes.lsp
 488    bayes-model bayes-model-proto bayes-internals
 489
 490
 491    ;; plots.lisp
 492    plot-ex
 493    contour-plot-ex
 494    fn-contour-plot-ex
 495    shade-plot-ex
 496    3D-plot-ex
 497
 498    ))
 499
 500
 501 ;;;; PACKAGES FOR USEABILITY
 502
 503 (defpackage :lisp-stat-data-examples
 504   (:documentation "Example data for unittests, examples, illustrations,")
 505   (:use :common-lisp
 506         :lisp-stat)
 507   (:shadowing-import-from :lisp-stat
 508                           call-method call-next-method
 509
 510       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 511       asin acos atan sinh cosh tanh asinh acosh atanh float random
 512       truncate floor ceiling round minusp zerop plusp evenp oddp
 513       < <= = /= >= > > ;; complex
 514       conjugate realpart imagpart phase
 515       min max logand logior logxor lognot ffloor fceiling
 516       ftruncate fround signum cis
 517
 518       <= float imagpart)
 519   (:export iron aluminum absorbtion
 520            diabetes dlabs))
 521
 522
 523 (defpackage :lisp-stat-user
 524   (:documentation "Experimentation package for LispStat.
 525 Serious work should be placed in a similar package elsewhere for
 526 reproducibility.  But this should hint as to what needs to be
 527 done for a user- or analysis-package.")
 528   (:nicknames :ls-user)
 529   (:use :common-lisp
 530         :lisp-matrix
 531         :lisp-stat
 532         :lisp-stat-data-examples) ;; this last is to have 'things to play with'
 533   (:shadowing-import-from :lisp-stat
 534                           call-method call-next-method
 535
 536       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 537       asin acos atan sinh cosh tanh asinh acosh atanh float random
 538       truncate floor ceiling round minusp zerop plusp evenp oddp
 539       < <= = /= >= > > ;; complex
 540       conjugate realpart imagpart phase
 541       min max logand logior logxor lognot ffloor fceiling
 542       ftruncate fround signum cis
 543
 544       <= float imagpart))
 545
 546 (defpackage :lisp-stat-unittests
 547   (:use :common-lisp
 548         :lift :lisp-matrix
 549         :lisp-stat :lisp-stat-data-examples)
 550   (:shadowing-import-from :lisp-stat
 551                           call-method call-next-method ;; objects
 552         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
 553         asin acos atan sinh cosh tanh asinh acosh atanh float random
 554         truncate floor ceiling round minusp zerop plusp evenp oddp
 555         < <= = /= >= > ;; complex
 556         conjugate realpart imagpart phase
 557         min max logand logior logxor lognot ffloor fceiling
 558         ftruncate fround signum cis)
 559   (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
 560            almost= almost=lists numerical=)) ; compare
 561
 562 (defpackage :lisp-stat-dataframe-example
 563   (:use :common-lisp
 564         :lift  :lisp-stat-unittests
 565         :lisp-stat-data-examples
 566         :lisp-stat-dataframe)
 567   (:export absorbtion aluminum iron))
 568
 569
 570 (defpackage :lisp-stat-optimize
 571  (:use :common-lisp
 572        :cffi
 573        :lisp-matrix
 574        :lisp-stat-ffi-int
 575        :lisp-stat-object-system
 576        :lisp-stat-types
 577        :lisp-stat-compound-data
 578        :lisp-stat-math
 579        :lisp-stat-float
 580        :lisp-stat-basics
 581 #|
 582        :lisp-stat-matrix
 583        :lisp-stat-linalg-data
 584        :lisp-stat-linalg
 585 |#
 586        )
 587  (:shadowing-import-from :lisp-stat-object-system
 588                          call-method call-next-method)
 589  (:shadowing-import-from :lisp-stat-math
 590            expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 591            asin acos atan sinh cosh tanh asinh acosh atanh float random
 592            truncate floor ceiling round minusp zerop plusp evenp oddp
 593            < <= = /= >= > complex conjugate realpart imagpart phase
 594            min max logand logior logxor lognot ffloor fceiling
 595            ftruncate fround signum cis)
 596  (:export
 597      ;; derivatives
 598      numgrad numhess
 599
 600      ;; optimization
 601      newtonmax nelmeadmax))