src/packages.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2009-04-13 12:03:08 tony>
   4 ;;; Creation:   <2008-03-11 19:18:34 user>
   5 ;;; File:       packages.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c)2007--2008, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   8 ;;;             on how it arrives.
   9 ;;; Purpose:    package structure description for lispstat
  10
  11 ;;; What is this talk of 'release'? Klingons do not make software
  12 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  13 ;;; designers and quality assurance people in its wake.
  14
  15 ;;; This organization and structure is new to the 21st Century
  16 ;;; version.
  17
  18 (in-package :cl-user)
  19
  20 ;;; Basics
  21
  22
  23 (defpackage :lisp-stat-object-system
  24   (:nicknames :ls-objects :lsos)
  25   (:use :common-lisp)
  26   (:shadow :call-method :call-next-method)
  27   (:export ls-object objectp *object* kind-of-p make-object
  28            *message-hook*
  29            *set-slot-hook* proto-slot-value self
  30            send call-next-method call-method
  31            defmeth defproto instance-slots proto-name))
  32
  33 (defpackage :lisp-stat-types
  34   (:documentation "Provides some typeing for LispStat, but is clearly
  35   a bit incomplete.")
  36   (:use :common-lisp)
  37   (:export fixnump
  38            check-nonneg-fixnum check-one-nonneg-fixnum
  39            check-one-fixnum check-one-real check-one-number))
  40
  41 (defpackage :lisp-stat-float
  42   (:use :common-lisp)
  43   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  44            machine-epsilon base-float makedouble
  45
  46            make-base-trans-fun-2 make-base-trans-fun
  47
  48            base-log base-exp base-expt base-sqrt base-sin base-cos
  49            base-tan base-asin base-acos base-atan base-sinh
  50            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
  51            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
  52            BASE-FROUND BASE-SIGNUM BASE-CIS))
  53
  54 (defpackage :lisp-stat-compound-data
  55   (:use :common-lisp
  56         :lisp-stat-object-system
  57         :lisp-stat-types)
  58   (:shadowing-import-from :lisp-stat-object-system
  59                           call-next-method call-method)
  60   (:export compound-data-p *compound-data-proto*
  61            compound-object-p
  62            compound-data-seq compound-data-length
  63            element-list element-seq
  64            sort-data order rank
  65            recursive-map-elements map-elements repeat
  66            check-sequence
  67            get-next-element make-next-element set-next-element
  68            ;; sequencep
  69            iseq ordered-nneg-seq
  70            select split-list which
  71            difference rseq
  72            flatten-list))
  73
  74 (defpackage :lisp-stat-macros
  75   (:use :common-lisp
  76         :lisp-stat-compound-data)
  77   (:export make-rv-function make-rv-function-1))
  78
  79 (defpackage :lisp-stat-basics
  80     (:use :common-lisp
  81           :lisp-stat-object-system
  82           :lisp-stat-types
  83           :lisp-stat-float
  84           :lisp-stat-macros
  85           :lisp-stat-compound-data)
  86   (:shadowing-import-from :lisp-stat-object-system
  87                           call-method call-next-method)
  88   (:export permute-array sum prod count-elements mean
  89            if-else sample))
  90
  91 (defpackage :lisp-stat-float
  92   (:use :common-lisp)
  93   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  94            machine-epsilon base-float makedouble
  95
  96            make-base-trans-fun-2 make-base-trans-fun
  97
  98            BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
  99            BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
 100            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
 101            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
 102            BASE-FROUND BASE-SIGNUM BASE-CIS))
 103
 104 (defpackage :lisp-stat-macros
 105   (:use :common-lisp
 106         :lisp-stat-compound-data)
 107   (:export make-rv-function make-rv-function-1))
 108
 109 ;;; NEW CLOS STRUCTURE
 110
 111
 112
 113 (defpackage :lisp-stat-dataframe
 114   (:use :common-lisp
 115         :lisp-matrix)
 116   (:export
 117    ;; generic container class for data -- if small enough
 118    ;; could be value, otherwise might be reference.
 119    dataframe-like
 120    dataframe-array
 121    make-dataframe
 122
 123    ;; accessors
 124    varlabels caselabels nrows ncols
 125    dataframe-dimension dataframe-dimensons
 126    dfref dfref-case dfref-var
 127    consistent-dataframe-p
 128
 129    dataset
 130    list-of-columns ;; list-of-variables
 131    list-of-rows ;; list-of-observations
 132    ))
 133
 134
 135 (defpackage :cls-data-listoflist
 136   (:use :common-lisp
 137         :lisp-stat-dataframe) ; for dataframe
 138   (:export lists-of-same-size
 139            equal-listoflist
 140            transpose-listoflist
 141            make-dataframe-from-listoflist
 142            make-array-from-listoflist))
 143
 144 (defpackage :lisp-stat-model
 145   (:documentation "Model management for data analysis.")
 146   (:use :common-lisp
 147         :lisp-matrix)
 148   (:export
 149    ;; data structures for model and model/data combination
 150    model statistical-model analysis))
 151
 152 #|
 153  (defpackage :lisp-stat-regression-linear-clos
 154   (:use :common-lisp
 155         :lisp-matrix
 156         :lisp-stat-dataframe)
 157   (:export regression-model))
 158 |#
 159
 160 (defpackage :lisp-stat-visualize
 161   (:use :common-lisp
 162         :lisp-matrix
 163         :lisp-stat-dataframe))
 164
 165 (defpackage :lisp-stat-visualize-plplot
 166   (:use :common-lisp
 167         :lisp-matrix
 168         :lisp-stat-dataframe
 169         :cl-plplot-system)
 170   (:export
 171    plot-ex
 172    contour-plot-ex
 173    fn-contour-plot-ex
 174    shade-plot-ex
 175    3D-plot-ex ))
 176
 177
 178 ;;; USER PACKAGES
 179
 180 (defpackage :lisp-stat-ffi-int
 181   (:use :common-lisp
 182         :cffi)
 183   (:export ccl-store-integer ccl-store-double ccl-store-ptr
 184            get-buf ))
 185
 186 (defpackage :lisp-stat-probability
 187   (:use :common-lisp
 188         :cffi
 189         :lisp-stat-ffi-int
 190         :lisp-stat-macros)
 191   (:export log-gamma set-seed
 192            uniform-rand
 193            normal-cdf normal-quant normal-dens normal-rand
 194            bivnorm-cdf
 195            cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
 196            gamma-cdf gamma-quant gamma-dens gamma-rand
 197            chisq-cdf chisq-quant chisq-dens chisq-rand
 198            beta-cdf beta-quant beta-dens beta-rand
 199            t-cdf t-quant t-dens t-rand
 200            f-cdf f-quant f-dens f-rand
 201            poisson-cdf poisson-quant poisson-pmf poisson-rand
 202            binomial-cdf binomial-quant binomial-pmf binomial-rand))
 203
 204
 205
 206 (defpackage :lisp-stat-math
 207    (:use :common-lisp
 208          :lisp-stat-object-system
 209          :lisp-stat-macros
 210          :lisp-stat-compound-data
 211          :lisp-stat-float)
 212    (:shadowing-import-from :lisp-stat-object-system
 213                            call-method call-next-method)
 214    (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 215            asin acos atan sinh cosh tanh asinh acosh atanh float random
 216            truncate floor ceiling round minusp zerop plusp evenp oddp
 217            < <= = /= >= > ;; complex
 218            conjugate realpart imagpart phase
 219            min max logand logior logxor lognot ffloor fceiling
 220            ftruncate fround signum cis)
 221    (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 222            tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 223            truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 224            /= >= > ;; complex
 225            conjugate realpart imagpart phase min max
 226            logand logior logxor lognot ffloor fceiling ftruncate fround
 227            signum cis)
 228    (:documentation "Vectorization of numerical functions"))
 229
 230
 231 #| ;; some of this goes back in, but not all of it?
 232 (defpackage :lisp-stat-linalg
 233   (:use :common-lisp
 234         :cffi
 235         :lisp-matrix
 236         :lisp-stat-math
 237         :lisp-stat-types
 238         :lisp-stat-float
 239         :lisp-stat-compound-data)
 240   (:shadowing-import-from :lisp-stat-math
 241           expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 242           asin acos atan sinh cosh tanh asinh acosh atanh float random
 243           truncate floor ceiling round minusp zerop plusp evenp oddp
 244           < <= = /= >= > complex conjugate realpart imagpart phase
 245           min max logand logior logxor lognot ffloor fceiling
 246           ftruncate fround signum cis)
 247   (:export chol-decomp lu-decomp lu-solve determinant inverse
 248            sv-decomp qr-decomp rcondest make-rotation spline
 249            kernel-dens kernel-smooth
 250            fft make-sweep-matrix sweep-operator ax+y eigen
 251
 252            check-real ;; for optimize
 253
 254            covariance-matrix matrix print-matrix solve
 255            backsolve eigenvalues eigenvectors accumulate cumsum combine
 256            lowess))
 257
 258 |#
 259
 260
 261
 262
 263 (defpackage :lisp-stat-data
 264   (:documentation "Data management, integration, I/O, and other data technologies.")
 265   (:nicknames :ls-data)
 266   (:use :common-lisp
 267         :lisp-stat-object-system
 268         :lisp-stat-config
 269         :lisp-stat-types
 270         :lisp-stat-compound-data)
 271   (:shadowing-import-from :lisp-stat-object-system
 272                           call-method call-next-method)
 273   (:export open-file-dialog read-data-file read-data-columns load-data
 274            load-example *variables* *ask-on-redefine*
 275            def variables savevar undef))
 276
 277 (defpackage :lisp-stat-descriptive-statistics
 278  (:use :common-lisp
 279        :lisp-stat-data
 280        :lisp-stat-math
 281        :lisp-stat-compound-data
 282        :lisp-matrix
 283        :lisp-stat-basics)
 284    (:shadowing-import-from :lisp-stat-math ;; life is a vector!
 285       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 286       asin acos atan sinh cosh tanh asinh acosh atanh float random
 287       truncate floor ceiling round minusp zerop plusp evenp oddp
 288       < <= = /= >= > ;; complex
 289       conjugate realpart imagpart phase
 290       min max logand logior logxor lognot ffloor fceiling
 291       ftruncate fround signum cis)
 292    (:export standard-deviation
 293             quantile median interquartile-range
 294             fivnum sample))
 295
 296
 297 (defpackage :lisp-stat-regression-linear
 298   (:use :common-lisp
 299         :lisp-matrix
 300         :lisp-stat-object-system
 301         :lisp-stat-basics
 302         :lisp-stat-compound-data
 303         :lisp-stat-descriptive-statistics )
 304   (:shadowing-import-from :lisp-stat-object-system
 305                           call-method call-next-method)
 306   (:export regression-model fit-model
 307
 308            estimates covariation-matrix
 309            ;; functions for helpers
 310            lm xtxinv
 311
 312            ;; OLD to remove
 313
 314            regression-model-proto x y intercept
 315            ;; sweep-matrix
 316            basis weights included
 317            total-sum-of-squares residual-sum-of-squares
 318            predictor-names response-name case-labels
 319            ))
 320
 321 (defpackage :lisp-stat
 322     (:documentation "Experimentation package for LispStat.  Serious
 323     work should be packaged up elsewhere for reproducibility.  By this
 324     I mean, creating a data/analytics/analysis package with the
 325     minimal set of objects required.")
 326   (:use :common-lisp
 327         :lisp-stat-object-system
 328         :lisp-stat-compound-data
 329         :lisp-stat-probability
 330         :lisp-stat-types
 331         :lisp-stat-float
 332         :lisp-stat-basics
 333         :lisp-stat-data
 334         :lisp-stat-dataframe
 335         :lisp-stat-math
 336         :lisp-matrix ;; conversion to a more robust linalg approach
 337         :lisp-stat-descriptive-statistics
 338         :lisp-stat-regression-linear
 339         :cybertiggyr-dsv)
 340   (:shadowing-import-from :lisp-stat-object-system
 341                           call-method call-next-method)
 342   (:shadowing-import-from :lisp-stat-math
 343         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 344         asin acos atan sinh cosh tanh asinh acosh atanh float random
 345         truncate floor ceiling round minusp zerop plusp evenp oddp
 346         < <= = /= >= >
 347         ;;complex
 348         conjugate realpart imagpart phase
 349
 350         min max
 351         logand logior logxor lognot
 352         ffloor fceiling ftruncate fround
 353         signum cis)
 354   (:export
 355    ;; lsobjects :
 356    defproto defproto2
 357    defmeth send
 358    proto-slot-value
 359
 360    ;; lstypes :
 361    fixnump check-nonneg-fixnum check-one-fixnum
 362    check-one-nonneg-fixnum
 363    check-one-real check-one-number
 364
 365    ;; lsmacros:
 366
 367    ;; lsfloat :
 368    machine-epsilon
 369
 370    ;; compound :
 371    compound-data-p *compound-data-proto* compound-object-p
 372    compound-data-seq compound-data-length
 373    element-list element-seq
 374    sort-data order rank
 375    recursive-map-elements map-elements
 376    repeat
 377    check-sequence
 378    get-next-element make-next-element set-next-element
 379    ;; sequencep
 380    iseq
 381    ordered-nneg-seq
 382    select which
 383    difference rseq
 384
 385    ;; lsmath.lsp
 386    ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 387    tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 388    truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 389    /= >= > ;; complex
 390    conjugate realpart imagpart phase min max
 391    logand logior logxor lognot ffloor fceiling ftruncate fround
 392    signum cis
 393
 394 #| ;; The following need to be re-found in lisp-matrix...
 395
 396    ;; matrices.lisp
 397    matrixp num-rows num-cols matmult identity-matrix diagonal row-list
 398    column-list inner-product outer-product cross-product transpose
 399    bind-columns bind-rows
 400
 401    ;; linalg.lisp
 402    chol-decomp lu-decomp lu-solve determinant inverse
 403    sv-decomp qr-decomp rcondest make-rotation spline
 404    kernel-dens kernel-smooth
 405    fft make-sweep-matrix sweep-operator ax+y eigen
 406    check-real
 407    covariance-matrix matrix print-matrix solve
 408    backsolve eigenvalues eigenvectors accumulate cumsum combine
 409    lowess
 410
 411    ;; in linalg.lisp, possibly not supported by matlisp
 412    spline kernel-dens kernel-smooth
 413
 414 |#
 415
 416    ;; optimize.lsp
 417    newtonmax nelmeadmax
 418
 419    ;; lispstat-macros
 420    make-rv-function make-rv-function-1
 421
 422    ;; data
 423    open-file-dialog read-data-file read-data-columns load-data
 424    load-example *variables* *ask-on-redefine*
 425    def variables savevar undef
 426
 427    ;; dataframe
 428    dataframe-like dataframe-array make-dataframe
 429    varlabels caselabels nrows ncols
 430    dataframe-dimension dataframe-dimensons
 431    dfref dfref-case dfref-var
 432    consistent-dataframe-p
 433    dataset
 434    list-of-columns list-of-rows
 435
 436    ;; listoflist
 437    lists-of-same-size equal-listoflist
 438    transpose-listoflist
 439    make-dataframe-from-listoflist  make-array-from-listoflist
 440
 441    ;; statistics.lsp  (descriptions, should probably be moved
 442    ;; later...?
 443    standard-deviation quantile median interquartile-range
 444    fivnum sample
 445
 446    ;; probability (dists.lisp)
 447    log-gamma set-seed
 448    uniform-rand normal-cdf normal-quant normal-dens
 449    normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
 450    cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
 451    chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
 452    beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
 453    f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
 454    binomial-cdf binomial-quant binomial-pmf binomial-rand
 455
 456    ;; Here is where we have a problem -- lispstat core should be core
 457    ;; data management and config problems, with packages providing
 458    ;; specialized extensions to LispStat, i.e. regression, nonlin
 459    ;; regression, bayesian regression via laplace approximation, etc.
 460
 461    ;; The following could be considered "recommended packages",
 462    ;; similar to the idea of the recommended packages in R.  Probably
 463    ;; we want them to do the exporting within that package, therefore
 464    ;; NOT being able to lock the "data-ish" package, but only the
 465    ;; subpackages prior to export.
 466
 467    ;; regression.lsp
 468    ;; -- linear regressin models.
 469    regression-model fit-model
 470    estimates covariation-matrix
 471
 472    regression-model-proto x y intercept sweep-matrix
 473    basis weights included total-sum-of-squares residual-sum-of-squares
 474    predictor-names response-name case-labels
 475    lm xtxinv
 476
 477    ;; nonlin.lsp
 478    ;; -- nonlinear regression models
 479    nreg-model nreg-model-proto mean-function theta-hat epsilon
 480    count-limit verbose
 481    ;; we might need something like xtxinv here?  But should be
 482    ;; encapsulated, so we use the one in regression.lisp
 483
 484    ;; bayes.lsp
 485    bayes-model bayes-model-proto bayes-internals
 486
 487
 488    ;; plots.lisp
 489    plot-ex
 490    contour-plot-ex
 491    fn-contour-plot-ex
 492    shade-plot-ex
 493    3D-plot-ex
 494
 495    ))
 496
 497
 498 ;;;; PACKAGES FOR USEABILITY
 499
 500 (defpackage :lisp-stat-data-examples
 501   (:documentation "Example data for unittests, examples, illustrations,")
 502   (:use :common-lisp
 503         :lisp-stat)
 504   (:shadowing-import-from :lisp-stat
 505                           call-method call-next-method
 506
 507       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 508       asin acos atan sinh cosh tanh asinh acosh atanh float random
 509       truncate floor ceiling round minusp zerop plusp evenp oddp
 510       < <= = /= >= > > ;; complex
 511       conjugate realpart imagpart phase
 512       min max logand logior logxor lognot ffloor fceiling
 513       ftruncate fround signum cis
 514
 515       <= float imagpart)
 516   (:export iron aluminum absorbtion
 517            diabetes dlabs))
 518
 519
 520 (defpackage :lisp-stat-user
 521   (:documentation "Experimentation package for LispStat.
 522 Serious work should be placed in a similar package elsewhere for
 523 reproducibility.  But this should hint as to what needs to be
 524 done for a user- or analysis-package.")
 525   (:nicknames :ls-user)
 526   (:use :common-lisp
 527         :lisp-matrix
 528         :lisp-stat
 529         :lisp-stat-data-examples) ;; this last is to have 'things to play with'
 530   (:shadowing-import-from :lisp-stat
 531                           call-method call-next-method
 532
 533       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 534       asin acos atan sinh cosh tanh asinh acosh atanh float random
 535       truncate floor ceiling round minusp zerop plusp evenp oddp
 536       < <= = /= >= > > ;; complex
 537       conjugate realpart imagpart phase
 538       min max logand logior logxor lognot ffloor fceiling
 539       ftruncate fround signum cis
 540
 541       <= float imagpart))
 542
 543 (defpackage :lisp-stat-unittests
 544   (:use :common-lisp :lift :lisp-stat :lisp-stat-data-examples)
 545   (:shadowing-import-from :lisp-stat
 546                           call-method call-next-method ;; objects
 547         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
 548         asin acos atan sinh cosh tanh asinh acosh atanh float random
 549         truncate floor ceiling round minusp zerop plusp evenp oddp
 550         < <= = /= >= > ;; complex
 551         conjugate realpart imagpart phase
 552         min max logand logior logxor lognot ffloor fceiling
 553         ftruncate fround signum cis)
 554   (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
 555            almost= almost=lists numerical=)) ; compare
 556
 557 (defpackage :lisp-stat-dataframe-example
 558   (:use :common-lisp
 559         :lift  :lisp-stat-unittests
 560         :lisp-stat-data-examples
 561         :lisp-stat-dataframe)
 562   (:export absorbtion aluminum iron))
 563
 564
 565 (defpackage :lisp-stat-optimize
 566  (:use :common-lisp
 567        :cffi
 568        :lisp-matrix
 569        :lisp-stat-ffi-int
 570        :lisp-stat-object-system
 571        :lisp-stat-types
 572        :lisp-stat-compound-data
 573        :lisp-stat-math
 574        :lisp-stat-float
 575        :lisp-stat-basics
 576 #|
 577        :lisp-stat-matrix
 578        :lisp-stat-linalg-data
 579        :lisp-stat-linalg
 580 |#
 581        )
 582  (:shadowing-import-from :lisp-stat-object-system
 583                          call-method call-next-method)
 584  (:shadowing-import-from :lisp-stat-math
 585            expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 586            asin acos atan sinh cosh tanh asinh acosh atanh float random
 587            truncate floor ceiling round minusp zerop plusp evenp oddp
 588            < <= = /= >= > complex conjugate realpart imagpart phase
 589            min max logand logior logxor lognot ffloor fceiling
 590            ftruncate fround signum cis)
 591  (:export
 592      ;; derivatives
 593      numgrad numhess
 594
 595      ;; optimization
 596      newtonmax nelmeadmax))