src/packages.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2009-04-17 18:23:10 tony>
   4 ;;; Creation:   <2008-03-11 19:18:34 user>
   5 ;;; File:       packages.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c)2007--2008, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   8 ;;;             on how it arrives.
   9 ;;; Purpose:    package structure description for lispstat
  10
  11 ;;; What is this talk of 'release'? Klingons do not make software
  12 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  13 ;;; designers and quality assurance people in its wake.
  14
  15 ;;; This organization and structure is new to the 21st Century
  16 ;;; version.
  17
  18 (in-package :cl-user)
  19
  20 ;;; Basics
  21
  22
  23 (defpackage :lisp-stat-object-system
  24   (:nicknames :ls-objects :lsos)
  25   (:use :common-lisp)
  26   (:shadow :call-method :call-next-method)
  27   (:export ls-object objectp *object* kind-of-p make-object
  28            *message-hook*
  29            *set-slot-hook* proto-slot-value self
  30            send call-next-method call-method
  31            defmeth defproto instance-slots proto-name))
  32
  33 (defpackage :lisp-stat-types
  34   (:documentation "Provides some typeing for LispStat, but is clearly
  35   a bit incomplete.")
  36   (:use :common-lisp)
  37   (:export fixnump
  38            check-nonneg-fixnum check-one-nonneg-fixnum
  39            check-one-fixnum check-one-real check-one-number))
  40
  41 (defpackage :lisp-stat-float
  42   (:use :common-lisp)
  43   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  44            machine-epsilon base-float makedouble
  45
  46            make-base-trans-fun-2 make-base-trans-fun
  47
  48            base-log base-exp base-expt base-sqrt base-sin base-cos
  49            base-tan base-asin base-acos base-atan base-sinh
  50            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
  51            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
  52            BASE-FROUND BASE-SIGNUM BASE-CIS))
  53
  54 (defpackage :lisp-stat-compound-data
  55   (:use :common-lisp
  56         :lisp-stat-object-system
  57         :lisp-stat-types)
  58   (:shadowing-import-from :lisp-stat-object-system
  59                           call-next-method call-method)
  60   (:export compound-data-p *compound-data-proto*
  61            compound-object-p
  62            compound-data-seq compound-data-length
  63            element-list element-seq
  64            sort-data order rank
  65            recursive-map-elements map-elements repeat
  66            check-sequence
  67            get-next-element make-next-element set-next-element
  68            ;; sequencep
  69            iseq ordered-nneg-seq
  70            select split-list which
  71            difference rseq
  72            flatten-list))
  73
  74 (defpackage :lisp-stat-macros
  75   (:use :common-lisp
  76         :lisp-stat-compound-data)
  77   (:export make-rv-function make-rv-function-1))
  78
  79 (defpackage :lisp-stat-basics
  80     (:use :common-lisp
  81           :lisp-stat-object-system
  82           :lisp-stat-types
  83           :lisp-stat-float
  84           :lisp-stat-macros
  85           :lisp-stat-compound-data)
  86   (:shadowing-import-from :lisp-stat-object-system
  87                           call-method call-next-method)
  88   (:export permute-array sum prod count-elements mean
  89            if-else sample))
  90
  91 (defpackage :lisp-stat-float
  92   (:use :common-lisp)
  93   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  94            machine-epsilon base-float makedouble
  95
  96            make-base-trans-fun-2 make-base-trans-fun
  97
  98            BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
  99            BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
 100            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
 101            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
 102            BASE-FROUND BASE-SIGNUM BASE-CIS))
 103
 104 (defpackage :lisp-stat-macros
 105   (:use :common-lisp
 106         :lisp-stat-compound-data)
 107   (:export make-rv-function make-rv-function-1))
 108
 109 ;;; NEW CLOS STRUCTURE
 110
 111
 112
 113 (defpackage :lisp-stat-dataframe
 114   (:use :common-lisp
 115         :lisp-matrix)
 116   (:export
 117    ;; generic container class for data -- if small enough
 118    ;; could be value, otherwise might be reference.
 119    dataframe-like
 120    dataframe-array
 121    make-dataframe
 122
 123    ;; accessors
 124    varlabels caselabels nrows ncols
 125    dataframe-dimension dataframe-dimensons
 126    dfref dfref-case dfref-var
 127    consistent-dataframe-p
 128
 129    dataset
 130    list-of-columns ;; list-of-variables
 131    list-of-rows ;; list-of-observations
 132    ))
 133
 134
 135 (defpackage :cls-data-listoflist
 136   (:use :common-lisp
 137         :lisp-matrix
 138         :lisp-stat-dataframe) ; for dataframe
 139   (:export lists-of-same-size
 140            equal-listoflist
 141            transpose-listoflist
 142            listoflist->dataframe
 143            listoflist->array
 144            listoflist->matrix-like))
 145
 146
 147 (defpackage :cls-dataimport
 148   (:documentation "Data I/O and similar import technologies.")
 149   (:use :common-lisp
 150         :lisp-stat-object-system
 151         :cls-dataframe
 152         :cls-data-listoflist
 153         :rsm.string)
 154   (:shadowing-import-from :lisp-stat-object-system
 155                           call-method call-next-method)
 156   (:export dsvstream->dataframe dsvstream->matrix dsvstream->listoflist))
 157
 158
 159 (defpackage :lisp-stat-model
 160   (:documentation "Model management for data analysis.")
 161   (:use :common-lisp
 162         :lisp-matrix)
 163   (:export
 164    ;; data structures for model and model/data combination
 165    model statistical-model analysis))
 166
 167 #|
 168  (defpackage :lisp-stat-regression-linear-clos
 169   (:use :common-lisp
 170         :lisp-matrix
 171         :lisp-stat-dataframe)
 172   (:export regression-model))
 173 |#
 174
 175 (defpackage :cls-visualize
 176   (:use :common-lisp
 177         :lisp-matrix
 178         :lisp-stat-dataframe))
 179
 180 (defpackage :cls-visualize-plplot
 181   (:use :common-lisp
 182         :lisp-matrix
 183         :lisp-stat-dataframe
 184         :cl-plplot-system)
 185   (:export
 186    plot-ex
 187    contour-plot-ex
 188    fn-contour-plot-ex
 189    shade-plot-ex
 190    3D-plot-ex ))
 191
 192
 193 ;;; USER PACKAGES
 194
 195 (defpackage :lisp-stat-ffi-int
 196   (:use :common-lisp
 197         :cffi)
 198   (:export ccl-store-integer ccl-store-double ccl-store-ptr
 199            get-buf ))
 200
 201 (defpackage :lisp-stat-probability
 202   (:use :common-lisp
 203         :cffi
 204         :lisp-stat-ffi-int
 205         :lisp-stat-macros)
 206   (:export log-gamma set-seed
 207            uniform-rand
 208            normal-cdf normal-quant normal-dens normal-rand
 209            bivnorm-cdf
 210            cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
 211            gamma-cdf gamma-quant gamma-dens gamma-rand
 212            chisq-cdf chisq-quant chisq-dens chisq-rand
 213            beta-cdf beta-quant beta-dens beta-rand
 214            t-cdf t-quant t-dens t-rand
 215            f-cdf f-quant f-dens f-rand
 216            poisson-cdf poisson-quant poisson-pmf poisson-rand
 217            binomial-cdf binomial-quant binomial-pmf binomial-rand))
 218
 219
 220
 221 (defpackage :lisp-stat-math
 222    (:use :common-lisp
 223          :lisp-stat-object-system
 224          :lisp-stat-macros
 225          :lisp-stat-compound-data
 226          :lisp-stat-float)
 227    (:shadowing-import-from :lisp-stat-object-system
 228                            call-method call-next-method)
 229    (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 230            asin acos atan sinh cosh tanh asinh acosh atanh float random
 231            truncate floor ceiling round minusp zerop plusp evenp oddp
 232            < <= = /= >= > ;; complex
 233            conjugate realpart imagpart phase
 234            min max logand logior logxor lognot ffloor fceiling
 235            ftruncate fround signum cis)
 236    (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 237            tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 238            truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 239            /= >= > ;; complex
 240            conjugate realpart imagpart phase min max
 241            logand logior logxor lognot ffloor fceiling ftruncate fround
 242            signum cis)
 243    (:documentation "Vectorization of numerical functions"))
 244
 245
 246 #| ;; some of this goes back in, but not all of it?
 247 (defpackage :lisp-stat-linalg
 248   (:use :common-lisp
 249         :cffi
 250         :lisp-matrix
 251         :lisp-stat-math
 252         :lisp-stat-types
 253         :lisp-stat-float
 254         :lisp-stat-compound-data)
 255   (:shadowing-import-from :lisp-stat-math
 256           expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 257           asin acos atan sinh cosh tanh asinh acosh atanh float random
 258           truncate floor ceiling round minusp zerop plusp evenp oddp
 259           < <= = /= >= > complex conjugate realpart imagpart phase
 260           min max logand logior logxor lognot ffloor fceiling
 261           ftruncate fround signum cis)
 262   (:export chol-decomp lu-decomp lu-solve determinant inverse
 263            sv-decomp qr-decomp rcondest make-rotation spline
 264            kernel-dens kernel-smooth
 265            fft make-sweep-matrix sweep-operator ax+y eigen
 266
 267            check-real ;; for optimize
 268
 269            covariance-matrix matrix print-matrix solve
 270            backsolve eigenvalues eigenvectors accumulate cumsum combine
 271            lowess))
 272
 273 |#
 274
 275
 276
 277
 278 (defpackage :lisp-stat-data
 279   (:documentation "Data management, integration, I/O, and other data technologies.")
 280   (:nicknames :ls-data)
 281   (:use :common-lisp
 282         :lisp-stat-object-system
 283         :lisp-stat-config
 284         :lisp-stat-types
 285         :lisp-stat-compound-data)
 286   (:shadowing-import-from :lisp-stat-object-system
 287                           call-method call-next-method)
 288   (:export open-file-dialog read-data-file read-data-columns load-data
 289            load-example *variables* *ask-on-redefine*
 290            def variables savevar undef))
 291
 292 (defpackage :lisp-stat-descriptive-statistics
 293  (:use :common-lisp
 294        :lisp-stat-data
 295        :lisp-stat-math
 296        :lisp-stat-compound-data
 297        :lisp-matrix
 298        :lisp-stat-basics)
 299    (:shadowing-import-from :lisp-stat-math ;; life is a vector!
 300       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 301       asin acos atan sinh cosh tanh asinh acosh atanh float random
 302       truncate floor ceiling round minusp zerop plusp evenp oddp
 303       < <= = /= >= > ;; complex
 304       conjugate realpart imagpart phase
 305       min max logand logior logxor lognot ffloor fceiling
 306       ftruncate fround signum cis)
 307    (:export standard-deviation
 308             quantile median interquartile-range
 309             fivnum sample))
 310
 311
 312 (defpackage :lisp-stat-regression-linear
 313   (:use :common-lisp
 314         :lisp-matrix
 315         :lisp-stat-object-system
 316         :lisp-stat-basics
 317         :lisp-stat-compound-data
 318         :lisp-stat-descriptive-statistics )
 319   (:shadowing-import-from :lisp-stat-object-system
 320                           call-method call-next-method)
 321   (:export regression-model fit-model
 322
 323            estimates covariance-matrix
 324            ;; functions for helpers
 325            lm xtxinv
 326            print-object ;; for method dispatch
 327
 328            ;; OLD to remove
 329
 330            regression-model-proto x y intercept
 331            ;; sweep-matrix
 332            basis weights included
 333            total-sum-of-squares residual-sum-of-squares
 334            predictor-names response-name case-labels
 335            ))
 336
 337 (defpackage :lisp-stat
 338     (:documentation "Experimentation package for LispStat.  Serious
 339     work should be packaged up elsewhere for reproducibility.  By this
 340     I mean, creating a data/analytics/analysis package with the
 341     minimal set of objects required.")
 342   (:use :common-lisp
 343         :lisp-stat-object-system
 344         :lisp-stat-compound-data
 345         :lisp-stat-probability
 346         :lisp-stat-types
 347         :lisp-stat-float
 348         :lisp-stat-basics
 349         :lisp-stat-data
 350         :lisp-stat-dataframe
 351         :cls-data-listoflist
 352         :lisp-stat-math
 353         :lisp-matrix ;; conversion to a more robust linalg approach
 354         :lisp-stat-descriptive-statistics
 355         :lisp-stat-regression-linear
 356         :cybertiggyr-dsv
 357         :cls-visualize
 358         :cls-visualize-plplot)
 359   (:shadowing-import-from :lisp-stat-object-system
 360                           call-method call-next-method)
 361   (:shadowing-import-from :lisp-stat-math
 362         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 363         asin acos atan sinh cosh tanh asinh acosh atanh float random
 364         truncate floor ceiling round minusp zerop plusp evenp oddp
 365         < <= = /= >= >
 366         ;;complex
 367         conjugate realpart imagpart phase
 368
 369         min max
 370         logand logior logxor lognot
 371         ffloor fceiling ftruncate fround
 372         signum cis)
 373   (:export
 374    ;; lsobjects :
 375    defproto defproto2
 376    defmeth send
 377    proto-slot-value
 378
 379    ;; lstypes :
 380    fixnump check-nonneg-fixnum check-one-fixnum
 381    check-one-nonneg-fixnum
 382    check-one-real check-one-number
 383
 384    ;; lsmacros:
 385
 386    ;; lsfloat :
 387    machine-epsilon
 388
 389    ;; compound :
 390    compound-data-p *compound-data-proto* compound-object-p
 391    compound-data-seq compound-data-length
 392    element-list element-seq
 393    sort-data order rank
 394    recursive-map-elements map-elements
 395    repeat
 396    check-sequence
 397    get-next-element make-next-element set-next-element
 398    ;; sequencep
 399    iseq
 400    ordered-nneg-seq
 401    select which
 402    difference rseq
 403
 404    ;; lsmath.lsp
 405    ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 406    tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 407    truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 408    /= >= > ;; complex
 409    conjugate realpart imagpart phase min max
 410    logand logior logxor lognot ffloor fceiling ftruncate fround
 411    signum cis
 412
 413 #| ;; The following need to be re-found in lisp-matrix...
 414
 415    ;; matrices.lisp
 416    matrixp num-rows num-cols matmult identity-matrix diagonal row-list
 417    column-list inner-product outer-product cross-product transpose
 418    bind-columns bind-rows
 419
 420    ;; linalg.lisp
 421    chol-decomp lu-decomp lu-solve determinant inverse
 422    sv-decomp qr-decomp rcondest make-rotation spline
 423    kernel-dens kernel-smooth
 424    fft make-sweep-matrix sweep-operator ax+y eigen
 425    check-real
 426    covariance-matrix matrix print-matrix solve
 427    backsolve eigenvalues eigenvectors accumulate cumsum combine
 428    lowess
 429
 430    ;; in linalg.lisp, possibly not supported by matlisp
 431    spline kernel-dens kernel-smooth
 432
 433 |#
 434
 435    ;; optimize.lsp
 436    newtonmax nelmeadmax
 437
 438    ;; lispstat-macros
 439    make-rv-function make-rv-function-1
 440
 441    ;; data
 442    open-file-dialog read-data-file read-data-columns load-data
 443    load-example *variables* *ask-on-redefine*
 444    def variables savevar undef
 445
 446    ;; dataframe
 447    dataframe-like dataframe-array make-dataframe
 448    varlabels caselabels nrows ncols
 449    dataframe-dimension dataframe-dimensons
 450    dfref dfref-case dfref-var
 451    consistent-dataframe-p
 452    dataset
 453    list-of-columns list-of-rows
 454
 455    ;; listoflist
 456    lists-of-same-size equal-listoflist
 457    transpose-listoflist
 458    listoflist->dataframe listoflist->array listoflist->matrix-like
 459
 460    ;; statistics.lsp  (descriptions, should probably be moved
 461    ;; later...?
 462    standard-deviation quantile median interquartile-range
 463    fivnum sample
 464
 465    ;; probability (dists.lisp)
 466    log-gamma set-seed
 467    uniform-rand normal-cdf normal-quant normal-dens
 468    normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
 469    cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
 470    chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
 471    beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
 472    f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
 473    binomial-cdf binomial-quant binomial-pmf binomial-rand
 474
 475    ;; Here is where we have a problem -- lispstat core should be core
 476    ;; data management and config problems, with packages providing
 477    ;; specialized extensions to LispStat, i.e. regression, nonlin
 478    ;; regression, bayesian regression via laplace approximation, etc.
 479
 480    ;; The following could be considered "recommended packages",
 481    ;; similar to the idea of the recommended packages in R.  Probably
 482    ;; we want them to do the exporting within that package, therefore
 483    ;; NOT being able to lock the "data-ish" package, but only the
 484    ;; subpackages prior to export.
 485
 486    ;; regression.lsp
 487    ;; -- linear regressin models.
 488    regression-model fit-model
 489    estimates covariance-matrix
 490
 491    regression-model-proto x y intercept sweep-matrix
 492    basis weights included total-sum-of-squares residual-sum-of-squares
 493    predictor-names response-name case-labels
 494    lm xtxinv
 495
 496    ;; nonlin.lsp
 497    ;; -- nonlinear regression models
 498    nreg-model nreg-model-proto mean-function theta-hat epsilon
 499    count-limit verbose
 500    ;; we might need something like xtxinv here?  But should be
 501    ;; encapsulated, so we use the one in regression.lisp
 502
 503    ;; bayes.lsp
 504    bayes-model bayes-model-proto bayes-internals
 505
 506    ;; plots.lisp
 507    plot-ex
 508    contour-plot-ex
 509    fn-contour-plot-ex
 510    shade-plot-ex
 511    3D-plot-ex
 512
 513    ))
 514
 515
 516 ;;;; PACKAGES FOR USEABILITY
 517
 518 (defpackage :lisp-stat-data-examples
 519   (:documentation "Example data for unittests, examples, illustrations,")
 520   (:use :common-lisp
 521         :lisp-stat)
 522   (:shadowing-import-from :lisp-stat
 523                           call-method call-next-method
 524
 525       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 526       asin acos atan sinh cosh tanh asinh acosh atanh float random
 527       truncate floor ceiling round minusp zerop plusp evenp oddp
 528       < <= = /= >= > > ;; complex
 529       conjugate realpart imagpart phase
 530       min max logand logior logxor lognot ffloor fceiling
 531       ftruncate fround signum cis
 532
 533       <= float imagpart)
 534   (:export iron aluminum absorbtion
 535            diabetes dlabs))
 536
 537
 538 (defpackage :lisp-stat-user
 539   (:documentation "Experimentation package for LispStat.
 540 Serious work should be placed in a similar package elsewhere for
 541 reproducibility.  But this should hint as to what needs to be
 542 done for a user- or analysis-package.")
 543   (:nicknames :ls-user)
 544   (:use :common-lisp
 545         :lisp-matrix
 546         :lisp-stat
 547         :lisp-stat-data-examples) ;; this last is to have 'things to play with'
 548   (:shadowing-import-from :lisp-stat
 549                           call-method call-next-method
 550
 551       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 552       asin acos atan sinh cosh tanh asinh acosh atanh float random
 553       truncate floor ceiling round minusp zerop plusp evenp oddp
 554       < <= = /= >= > > ;; complex
 555       conjugate realpart imagpart phase
 556       min max logand logior logxor lognot ffloor fceiling
 557       ftruncate fround signum cis
 558
 559       <= float imagpart))
 560
 561 (defpackage :lisp-stat-unittests
 562   (:use :common-lisp
 563         :lift :lisp-matrix
 564         :lisp-stat :lisp-stat-data-examples)
 565   (:shadowing-import-from :lisp-stat
 566                           call-method call-next-method ;; objects
 567         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
 568         asin acos atan sinh cosh tanh asinh acosh atanh float random
 569         truncate floor ceiling round minusp zerop plusp evenp oddp
 570         < <= = /= >= > ;; complex
 571         conjugate realpart imagpart phase
 572         min max logand logior logxor lognot ffloor fceiling
 573         ftruncate fround signum cis)
 574   (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
 575            almost= almost=lists numerical=)) ; compare
 576
 577 (defpackage :lisp-stat-dataframe-example
 578   (:use :common-lisp
 579         :lift  :lisp-stat-unittests
 580         :lisp-stat-data-examples
 581         :lisp-stat-dataframe)
 582   (:export absorbtion aluminum iron))
 583
 584
 585 (defpackage :lisp-stat-optimize
 586  (:use :common-lisp
 587        :cffi
 588        :lisp-matrix
 589        :lisp-stat-ffi-int
 590        :lisp-stat-object-system
 591        :lisp-stat-types
 592        :lisp-stat-compound-data
 593        :lisp-stat-math
 594        :lisp-stat-float
 595        :lisp-stat-basics
 596 #|
 597        :lisp-stat-matrix
 598        :lisp-stat-linalg-data
 599        :lisp-stat-linalg
 600 |#
 601        )
 602  (:shadowing-import-from :lisp-stat-object-system
 603                          call-method call-next-method)
 604  (:shadowing-import-from :lisp-stat-math
 605            expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 606            asin acos atan sinh cosh tanh asinh acosh atanh float random
 607            truncate floor ceiling round minusp zerop plusp evenp oddp
 608            < <= = /= >= > complex conjugate realpart imagpart phase
 609            min max logand logior logxor lognot ffloor fceiling
 610            ftruncate fround signum cis)
 611  (:export
 612      ;; derivatives
 613      numgrad numhess
 614
 615      ;; optimization
 616      newtonmax nelmeadmax))