src/packages.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2009-03-21 09:28:46 tony>
   4 ;;; Creation:   <2008-03-11 19:18:34 user>
   5 ;;; File:       packages.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c)2007--2008, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   8 ;;;             on how it arrives.
   9 ;;; Purpose:    package structure description for lispstat
  10
  11 ;;; What is this talk of 'release'? Klingons do not make software
  12 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  13 ;;; designers and quality assurance people in its wake.
  14
  15 ;;; This organization and structure is new to the 21st Century
  16 ;;; version.
  17
  18 (in-package :cl-user)
  19
  20 ;;; LispStat Basics
  21
  22 (defpackage :lisp-stat-object-system
  23   (:nicknames :ls-objects :lsos)
  24   (:use :common-lisp)
  25   (:shadow :call-method :call-next-method :slot-value)
  26   (:export ls-object objectp *object* kind-of-p make-object
  27            *message-hook*
  28            *set-slot-hook* slot-value self
  29            send call-next-method call-method
  30            defmeth defproto instance-slots proto-name))
  31
  32 (defpackage :lisp-stat-types
  33   (:documentation "Provides some typeing for LispStat, but is clearly
  34   a bit incomplete.")
  35   (:use :common-lisp)
  36   (:export fixnump
  37            check-nonneg-fixnum check-one-nonneg-fixnum
  38            check-one-fixnum check-one-real check-one-number))
  39
  40 (defpackage :lisp-stat-float
  41   (:use :common-lisp)
  42   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  43            machine-epsilon base-float makedouble
  44
  45            make-base-trans-fun-2 make-base-trans-fun
  46
  47            base-log base-exp base-expt base-sqrt base-sin base-cos
  48            base-tan base-asin base-acos base-atan base-sinh
  49            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
  50            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
  51            BASE-FROUND BASE-SIGNUM BASE-CIS))
  52
  53 (defpackage :lisp-stat-compound-data
  54   (:use :common-lisp
  55         :lisp-stat-object-system
  56         :lisp-stat-types)
  57   (:shadowing-import-from :lisp-stat-object-system
  58                           slot-value
  59                           call-next-method call-method)
  60   (:export compound-data-p *compound-data-proto*
  61            compound-object-p
  62            compound-data-seq compound-data-length
  63            element-list element-seq
  64            sort-data order rank
  65            recursive-map-elements map-elements repeat
  66            check-sequence
  67            get-next-element make-next-element set-next-element
  68            ;; sequencep
  69            iseq ordered-nneg-seq
  70            select split-list which
  71            difference rseq
  72            flatten-list))
  73
  74 (defpackage :lisp-stat-macros
  75   (:use :common-lisp
  76         :lisp-stat-compound-data)
  77   (:export make-rv-function make-rv-function-1))
  78
  79 (defpackage :lisp-stat-basics
  80     (:use :common-lisp
  81           :lisp-stat-object-system
  82           :lisp-stat-types
  83           :lisp-stat-float
  84           :lisp-stat-macros
  85           :lisp-stat-compound-data)
  86   (:shadowing-import-from :lisp-stat-object-system
  87                           slot-value call-method call-next-method)
  88   (:export permute-array sum prod count-elements mean
  89            if-else sample))
  90
  91 (defpackage :lisp-stat-float
  92   (:use :common-lisp)
  93   (:export +stat-float-typing+ +stat-cfloat-typing+ +stat-float-template+
  94            machine-epsilon base-float makedouble
  95
  96            make-base-trans-fun-2 make-base-trans-fun
  97
  98            BASE-LOG BASE-EXP BASE-EXPT BASE-SQRT BASE-SIN BASE-COS
  99            BASE-TAN BASE-ASIN BASE-ACOS BASE-ATAN BASE-SINH
 100            BASE-COSH BASE-TANH BASE-ASINH BASE-ACOSH BASE-ATANH
 101            BASE-ABS BASE-PHASE BASE-FFLOOR BASE-FCEILING BASE-FTRUNCATE
 102            BASE-FROUND BASE-SIGNUM BASE-CIS))
 103
 104 (defpackage :lisp-stat-macros
 105   (:use :common-lisp
 106         :lisp-stat-compound-data)
 107   (:export make-rv-function make-rv-function-1))
 108
 109 ;;; NEW CLOS STRUCTURE
 110
 111
 112
 113 (defpackage :lisp-stat-data-clos
 114   (:use :common-lisp
 115         :lisp-matrix)
 116   (:export
 117    ;; generic container class for data -- if small enough
 118    ;; could be value, otherwise might be reference.
 119    dataframe-like
 120    dataframe-array
 121
 122    ;; accessors
 123    dataset
 124    nrows ncols
 125    list-of-columns ;; list-of-variables
 126    list-of-rows ;; list-of-observations
 127
 128    ;; ???
 129    get-variable-matrix get-variable-vector
 130    data-pointer
 131    ))
 132
 133 (defpackage :cls-data-listoflist
 134   (:use :common-lisp
 135         :lisp-stat-data-clos) ; for dataframe
 136   (:export lists-of-same-size
 137            equal-listoflist
 138            transpose-listoflist
 139            make-dataframe-from-listoflist
 140            make-array-from-listoflist))
 141
 142 #|
 143  (defpackage :lisp-stat-regression-linear-clos
 144   (:use :common-lisp
 145         :lisp-matrix
 146         :lisp-stat-data-clos)
 147   (:export regression-model))
 148 |#
 149
 150 (defpackage :lisp-stat-visualize
 151   (:use :common-lisp
 152         :lisp-matrix
 153         :lisp-stat-data-clos))
 154
 155 (defpackage :lisp-stat-visualize-plplot
 156   (:use :common-lisp
 157         :lisp-matrix
 158         :lisp-stat-data-clos
 159         :cl-plplot-system)
 160   (:export
 161    plot-ex
 162    contour-plot-ex
 163    fn-contour-plot-ex
 164    shade-plot-ex
 165    3D-plot-ex ))
 166
 167
 168 ;;; USER PACKAGES
 169
 170 (defpackage :lisp-stat-ffi-int
 171   (:use :common-lisp
 172         :cffi)
 173   (:export ccl-store-integer ccl-store-double ccl-store-ptr
 174            get-buf ))
 175
 176 (defpackage :lisp-stat-probability
 177   (:use :common-lisp
 178         :cffi
 179         :lisp-stat-ffi-int
 180         :lisp-stat-macros)
 181   (:export log-gamma set-seed
 182            uniform-rand
 183            normal-cdf normal-quant normal-dens normal-rand
 184            bivnorm-cdf
 185            cauchy-cdf cauchy-quant cauchy-dens cauchy-rand
 186            gamma-cdf gamma-quant gamma-dens gamma-rand
 187            chisq-cdf chisq-quant chisq-dens chisq-rand
 188            beta-cdf beta-quant beta-dens beta-rand
 189            t-cdf t-quant t-dens t-rand
 190            f-cdf f-quant f-dens f-rand
 191            poisson-cdf poisson-quant poisson-pmf poisson-rand
 192            binomial-cdf binomial-quant binomial-pmf binomial-rand))
 193
 194
 195 #| removed, replace by lisp-matrix
 196 (defpackage :lisp-stat-matrix
 197   (:use :common-lisp
 198         :cffi
 199         :lisp-stat-compound-data)
 200   (:export matrixp ;;  matrix -- conflicts!
 201            num-rows num-cols matmult identity-matrix diagonal
 202            row-list column-list inner-product outer-product
 203            cross-product
 204            ;; transpose bind-columns bind-rows
 205            array-data-vector vector-to-array
 206
 207            check-matrix check-square-matrix
 208
 209            copy-array copy-vector
 210            ))
 211 |#
 212
 213 #| replaced by lisp-matrix
 214 (defpackage :lisp-stat-linalg-data
 215   (:use :common-lisp
 216         :cffi
 217         :lisp-stat-ffi-int
 218         :lisp-stat-types
 219         :lisp-stat-compound-data
 220         :lisp-stat-matrix)
 221   (:export ;; more to add
 222       +mode-in+ +mode-re+ +mode-cx+ mode-of
 223
 224       la-data-mode la-allocate la-free
 225
 226       la-get-double la-put-double
 227       la-put-integer
 228
 229       la-matrix la-free-matrix la-matrix-to-data la-data-to-matrix
 230       la-vector la-free-vector la-vector-to-data la-data-to-vector ))
 231 |#
 232
 233 (defpackage :lisp-stat-math
 234    (:use :common-lisp
 235          :lisp-stat-object-system
 236          :lisp-stat-macros
 237          :lisp-stat-compound-data
 238          :lisp-stat-float)
 239    (:shadowing-import-from :lisp-stat-object-system
 240                            slot-value call-method call-next-method)
 241    (:shadow expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 242            asin acos atan sinh cosh tanh asinh acosh atanh float random
 243            truncate floor ceiling round minusp zerop plusp evenp oddp
 244            < <= = /= >= > ;; complex
 245            conjugate realpart imagpart phase
 246            min max logand logior logxor lognot ffloor fceiling
 247            ftruncate fround signum cis)
 248    (:export ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 249            tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 250            truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 251            /= >= > ;; complex
 252            conjugate realpart imagpart phase min max
 253            logand logior logxor lognot ffloor fceiling ftruncate fround
 254            signum cis)
 255    (:documentation "Vectorization of numerical functions"))
 256
 257
 258 #| ;; some of this goes back in, but not all of it?
 259 (defpackage :lisp-stat-linalg
 260   (:use :common-lisp
 261         :cffi
 262         :lisp-matrix
 263         :lisp-stat-math
 264         :lisp-stat-types
 265         :lisp-stat-float
 266         :lisp-stat-compound-data)
 267   (:shadowing-import-from :lisp-stat-math
 268           expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 269           asin acos atan sinh cosh tanh asinh acosh atanh float random
 270           truncate floor ceiling round minusp zerop plusp evenp oddp
 271           < <= = /= >= > complex conjugate realpart imagpart phase
 272           min max logand logior logxor lognot ffloor fceiling
 273           ftruncate fround signum cis)
 274   (:export chol-decomp lu-decomp lu-solve determinant inverse
 275            sv-decomp qr-decomp rcondest make-rotation spline
 276            kernel-dens kernel-smooth
 277            fft make-sweep-matrix sweep-operator ax+y eigen
 278
 279            check-real ;; for optimize
 280
 281            covariance-matrix matrix print-matrix solve
 282            backsolve eigenvalues eigenvectors accumulate cumsum combine
 283            lowess))
 284
 285 |#
 286
 287
 288
 289
 290 (defpackage :lisp-stat-data
 291   (:documentation "Data management, integration, I/O, and other data technologies.")
 292   (:nicknames :ls-data)
 293   (:use :common-lisp
 294         :lisp-stat-object-system
 295         :lisp-stat-config
 296         :lisp-stat-types
 297         :lisp-stat-compound-data)
 298   (:shadowing-import-from :lisp-stat-object-system
 299                           slot-value call-method call-next-method)
 300   (:export open-file-dialog read-data-file read-data-columns load-data
 301            load-example *variables* *ask-on-redefine*
 302            def variables savevar undef))
 303
 304 (defpackage :lisp-stat-descriptive-statistics
 305  (:use :common-lisp
 306        :lisp-stat-data
 307        :lisp-stat-math
 308        :lisp-stat-compound-data
 309        :lisp-matrix
 310        :lisp-stat-basics)
 311    (:shadowing-import-from :lisp-stat-math ;; life is a vector!
 312       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 313       asin acos atan sinh cosh tanh asinh acosh atanh float random
 314       truncate floor ceiling round minusp zerop plusp evenp oddp
 315       < <= = /= >= > ;; complex
 316       conjugate realpart imagpart phase
 317       min max logand logior logxor lognot ffloor fceiling
 318       ftruncate fround signum cis)
 319    (:export standard-deviation
 320             quantile median interquartile-range
 321             fivnum sample))
 322
 323
 324 (defpackage :lisp-stat-regression-linear
 325   (:use :common-lisp
 326         :lisp-matrix
 327         :lisp-stat-object-system
 328         :lisp-stat-basics
 329         :lisp-stat-compound-data
 330         :lisp-stat-descriptive-statistics
 331 #|
 332         :lisp-stat-math
 333 |#
 334         )
 335   (:shadowing-import-from :lisp-stat-object-system
 336                           slot-value call-method call-next-method)
 337 #|
 338   (:shadowing-import-from :lisp-stat-math
 339       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 340       asin acos atan sinh cosh tanh asinh acosh atanh float random
 341       truncate floor ceiling round minusp zerop plusp evenp oddp
 342       < <= = /= >= > ;; complex
 343       conjugate realpart imagpart phase
 344       min max logand logior logxor lognot ffloor fceiling
 345       ftruncate fround signum cis)
 346 |#
 347   (:export regression-model regression-model-proto x y intercept
 348            ;; sweep-matrix
 349            basis weights included
 350            total-sum-of-squares residual-sum-of-squares
 351            predictor-names response-name case-labels
 352            ;; functions for helpers
 353            lm xtxinv   ))
 354
 355 (defpackage :lisp-stat
 356     (:documentation "Experimentation package for LispStat.  Serious
 357     work should be packaged up elsewhere for reproducibility.  By this
 358     I mean, creating a data/analytics/analysis package with the
 359     minimal set of objects required.")
 360   (:use :common-lisp
 361         :lisp-stat-object-system
 362         :lisp-stat-compound-data
 363         :lisp-stat-probability
 364         :lisp-stat-types
 365         :lisp-stat-float
 366         :lisp-stat-basics
 367         :lisp-stat-data
 368         :lisp-stat-data-clos
 369         :lisp-stat-math
 370         :lisp-matrix ;; conversion to a more robust linalg approach
 371         :lisp-stat-descriptive-statistics
 372         :lisp-stat-regression-linear
 373         :cybertiggyr-dsv)
 374   (:shadowing-import-from :lisp-stat-object-system
 375         slot-value call-method call-next-method)
 376   (:shadowing-import-from :lisp-stat-math
 377         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 378         asin acos atan sinh cosh tanh asinh acosh atanh float random
 379         truncate floor ceiling round minusp zerop plusp evenp oddp
 380         < <= = /= >= >
 381         ;;complex
 382         conjugate realpart imagpart phase
 383
 384         min max
 385         logand logior logxor lognot
 386         ffloor fceiling ftruncate fround
 387         signum cis)
 388   (:export
 389    ;; lsobjects :
 390    defproto defproto2
 391    defmeth send
 392
 393    ;; lstypes :
 394    fixnump check-nonneg-fixnum check-one-fixnum
 395    check-one-real check-one-number
 396
 397    ;; lsmacros:
 398
 399    ;; lsfloat :
 400    machine-epsilon
 401
 402    ;; compound :
 403    compound-data-p *compound-data-proto* compound-object-p
 404    compound-data-seq compound-data-length
 405    element-list element-seq
 406    sort-data order rank
 407    recursive-map-elements map-elements
 408    repeat
 409    check-sequence
 410    get-next-element make-next-element set-next-element
 411    ;; sequencep
 412    iseq
 413    ordered-nneg-seq
 414    select which
 415    difference rseq
 416
 417    ;; lsmath.lsp
 418    ^ ** expt + - * / mod rem pmin pmax abs 1+ 1- log exp sqrt sin cos
 419    tan asin acos atan sinh cosh tanh asinh acosh atanh float random
 420    truncate floor ceiling round minusp zerop plusp evenp oddp < <= =
 421    /= >= > ;; complex
 422    conjugate realpart imagpart phase min max
 423    logand logior logxor lognot ffloor fceiling ftruncate fround
 424    signum cis
 425
 426 #| ;; The following need to be re-found in lisp-matrix...
 427
 428    ;; matrices.lisp
 429    matrixp num-rows num-cols matmult identity-matrix diagonal row-list
 430    column-list inner-product outer-product cross-product transpose
 431    bind-columns bind-rows
 432
 433    ;; linalg.lisp
 434    chol-decomp lu-decomp lu-solve determinant inverse
 435    sv-decomp qr-decomp rcondest make-rotation spline
 436    kernel-dens kernel-smooth
 437    fft make-sweep-matrix sweep-operator ax+y eigen
 438    check-real
 439    covariance-matrix matrix print-matrix solve
 440    backsolve eigenvalues eigenvectors accumulate cumsum combine
 441    lowess
 442
 443    ;; in linalg.lisp, possibly not supported by matlisp
 444    spline kernel-dens kernel-smooth
 445
 446 |#
 447
 448    ;; optimize.lsp
 449    newtonmax nelmeadmax
 450
 451    ;; lispstat-macros
 452    make-rv-function make-rv-function-1
 453
 454    ;; data
 455    open-file-dialog read-data-file read-data-columns load-data
 456    load-example *variables* *ask-on-redefine*
 457    def variables savevar undef
 458
 459    ;; data-clos
 460    dataframe-like
 461    dataframe-array
 462
 463    ;; listoflist
 464    lists-of-same-size equal-listoflist
 465    transpose-listoflist
 466    make-dataframe-from-listoflist  make-array-from-listoflist
 467
 468    ;; statistics.lsp  (descriptions, should probably be moved
 469    ;; later...?
 470    standard-deviation quantile median interquartile-range
 471    fivnum sample
 472
 473    ;; probability (dists.lisp)
 474    log-gamma set-seed
 475    uniform-rand normal-cdf normal-quant normal-dens
 476    normal-rand bivnorm-cdf cauchy-cdf cauchy-quant cauchy-dens
 477    cauchy-rand gamma-cdf gamma-quant gamma-dens gamma-rand
 478    chisq-cdf chisq-quant chisq-dens chisq-rand beta-cdf beta-quant
 479    beta-dens beta-rand t-cdf t-quant t-dens t-rand f-cdf f-quant
 480    f-dens f-rand poisson-cdf poisson-quant poisson-pmf poisson-rand
 481    binomial-cdf binomial-quant binomial-pmf binomial-rand
 482
 483    ;; Here is where we have a problem -- lispstat core should be core
 484    ;; data management and config problems, with packages providing
 485    ;; specialized extensions to LispStat, i.e. regression, nonlin
 486    ;; regression, bayesian regression via laplace approximation, etc.
 487
 488    ;; The following could be considered "recommended packages",
 489    ;; similar to the idea of the recommended packages in R.  Probably
 490    ;; we want them to do the exporting within that package, therefore
 491    ;; NOT being able to lock the "data-ish" package, but only the
 492    ;; subpackages prior to export.
 493
 494    ;; regression.lsp
 495    ;; -- linear regressin models.
 496    regression-model regression-model-proto x y intercept sweep-matrix
 497    basis weights included total-sum-of-squares residual-sum-of-squares
 498    predictor-names response-name case-labels
 499    lm xtxinv
 500
 501    ;; nonlin.lsp
 502    ;; -- nonlinear regression models
 503    nreg-model nreg-model-proto mean-function theta-hat epsilon
 504    count-limit verbose
 505    ;; we might need something like xtxinv here?  But should be
 506    ;; encapsulated, so we use the one in regression.lisp
 507
 508    ;; bayes.lsp
 509    bayes-model bayes-model-proto bayes-internals
 510
 511
 512    ;; plots.lisp
 513    plot-ex
 514    contour-plot-ex
 515    fn-contour-plot-ex
 516    shade-plot-ex
 517    3D-plot-ex
 518
 519    ))
 520
 521
 522 ;;;; PACKAGES FOR USEABILITY
 523
 524 (defpackage :lisp-stat-data-examples
 525   (:documentation "Example data for unittests, examples, illustrations,")
 526   (:use :common-lisp
 527         :lisp-stat)
 528   (:shadowing-import-from :lisp-stat
 529       slot-value call-method call-next-method
 530
 531       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 532       asin acos atan sinh cosh tanh asinh acosh atanh float random
 533       truncate floor ceiling round minusp zerop plusp evenp oddp
 534       < <= = /= >= > > ;; complex
 535       conjugate realpart imagpart phase
 536       min max logand logior logxor lognot ffloor fceiling
 537       ftruncate fround signum cis
 538
 539       <= float imagpart)
 540   (:export iron aluminum absorbtion
 541            diabetes dlabs))
 542
 543
 544 (defpackage :lisp-stat-user
 545   (:documentation "Experimentation package for LispStat.
 546 Serious work should be placed in a similar package elsewhere for
 547 reproducibility.  But this should hint as to what needs to be
 548 done for a user- or analysis-package.")
 549   (:nicknames :ls-user)
 550   (:use :common-lisp
 551         :lisp-matrix
 552         :lisp-stat
 553         :lisp-stat-data-examples) ;; this last is to have 'things to play with'
 554   (:shadowing-import-from :lisp-stat
 555       slot-value call-method call-next-method
 556
 557       expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 558       asin acos atan sinh cosh tanh asinh acosh atanh float random
 559       truncate floor ceiling round minusp zerop plusp evenp oddp
 560       < <= = /= >= > > ;; complex
 561       conjugate realpart imagpart phase
 562       min max logand logior logxor lognot ffloor fceiling
 563       ftruncate fround signum cis
 564
 565       <= float imagpart))
 566
 567 (defpackage :lisp-stat-unittests
 568   (:use :common-lisp :lift :lisp-stat :lisp-stat-data-examples)
 569   (:shadowing-import-from :lisp-stat
 570         slot-value call-method call-next-method ;; objects
 571         expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan ;; lsmath
 572         asin acos atan sinh cosh tanh asinh acosh atanh float random
 573         truncate floor ceiling round minusp zerop plusp evenp oddp
 574         < <= = /= >= > ;; complex
 575         conjugate realpart imagpart phase
 576         min max logand logior logxor lognot ffloor fceiling
 577         ftruncate fround signum cis)
 578   (:export run-lisp-stat-tests run-lisp-stat-test scoreboard ; exec
 579            almost= almost=lists numerical=)) ; compare
 580
 581 (defpackage :lisp-stat-data-clos-example
 582   (:use :common-lisp
 583         :lift  :lisp-stat-unittests
 584         :lisp-stat-data-examples
 585         :lisp-stat-data-clos)
 586   (:export absorbtion aluminum iron))
 587
 588
 589 (defpackage :lisp-stat-optimize
 590  (:use :common-lisp
 591        :cffi
 592        :lisp-matrix
 593        :lisp-stat-ffi-int
 594        :lisp-stat-object-system
 595        :lisp-stat-types
 596        :lisp-stat-compound-data
 597        :lisp-stat-math
 598        :lisp-stat-float
 599        :lisp-stat-basics
 600 #|
 601        :lisp-stat-matrix
 602        :lisp-stat-linalg-data
 603        :lisp-stat-linalg
 604 |#
 605        )
 606  (:shadowing-import-from :lisp-stat-object-system
 607                          slot-value call-method call-next-method)
 608  (:shadowing-import-from :lisp-stat-math
 609            expt + - * / ** mod rem abs 1+ 1- log exp sqrt sin cos tan
 610            asin acos atan sinh cosh tanh asinh acosh atanh float random
 611            truncate floor ceiling round minusp zerop plusp evenp oddp
 612            < <= = /= >= > complex conjugate realpart imagpart phase
 613            min max logand logior logxor lognot ffloor fceiling
 614            ftruncate fround signum cis)
 615  (:export
 616      ;; derivatives
 617      numgrad numhess
 618
 619      ;; optimization
 620      newtonmax nelmeadmax))