TODO.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2009-04-28 13:10:21 tony>
   4 ;;; Creation:   <2008-09-08 08:06:30 tony>
   5 ;;; File:       TODO.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c) 2007-2008, AJ Rossini <blindglobe@gmail.com>.  BSD.
   8 ;;; Purpose: Stuff that needs to be made working sits inside the
   9 ;;;          progns... This file contains the current challenges to
  10 ;;;          solve, including a description of the setup and the work
  11 ;;;          to solve....
  12
  13 ;;; What is this talk of 'release'? Klingons do not make software
  14 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  15 ;;; designers and quality assurance people in its wake.
  16
  17 ;;; SET UP
  18
  19 (in-package :cl-user)
  20 ;;(asdf:oos 'asdf:load-op 'lisp-matrix)
  21 ;;(asdf:oos 'asdf:compile-op 'lispstat :force t)
  22 ;;(asdf:oos 'asdf:load-op 'lispstat)
  23
  24 (in-package :lisp-stat-unittests)
  25
  26 ;; tests = 80, failures = 8, errors = 15
  27 (run-tests :suite 'lisp-stat-ut)
  28 (describe (run-tests :suite 'lisp-stat-ut))
  29
  30 ;; FIXME: Example: currently not relevant, yet
  31 ;;   (describe (lift::run-test :test-case  'lisp-stat-unittests::create-proto
  32 ;;                             :suite 'lisp-stat-unittests::lisp-stat-ut-proto))
  33
  34 (describe (lift::run-tests :suite 'lisp-stat-ut-dataframe))
  35 (lift::run-tests :suite 'lisp-stat-ut-dataframe)
  36
  37 (describe
  38  (lift::run-test
  39   :test-case  'lisp-stat-unittests::create-proto
  40   :suite 'lisp-stat-unittests::lisp-stat-ut-proto))
  41
  42 (describe 'lisp-stat-ut)
  43
  44 (in-package :ls-user)
  45
  46 (progn
  47   ;; Plotting -- need to figure out the core-dump, or change libraries.
  48   ;; (asdf:oos 'asdf:load-op 'cl-plplot)
  49
  50   (plot-ex)
  51   (defparameter *gdev* "xcairo")
  52   ;; (defparameter *gdev* "xwin")
  53   (cl-plplot::plsdev *gdev*)
  54
  55   ;; there is currently a loose pointer floating around that causes
  56   ;; errors the 3rd time that we create a plot (and crashes SBCL the
  57   ;; 4th time).  Order independent.
  58   (contour-plot-ex)
  59   (fn-contour-plot-ex)
  60   (shade-plot-ex)
  61   (3D-plot-ex))
  62
  63
  64 (progn
  65   ;; REVIEW: general Lisp use guidance
  66
  67   (fdefinition 'make-matrix)
  68   (documentation 'make-matrix 'function)
  69
  70 #| Examples from CLHS, a bit of guidance.
  71
  72   ;; This function assumes its callers have checked the types of the
  73   ;; arguments, and authorizes the compiler to build in that assumption.
  74   (defun discriminant (a b c)
  75    (declare (number a b c))
  76    "Compute the discriminant for a quadratic equation."
  77    (- (* b b) (* 4 a c))) =>  DISCRIMINANT
  78   (discriminant 1 2/3 -2) =>  76/9
  79
  80   ;; This function assumes its callers have not checked the types of the
  81   ;; arguments, and performs explicit type checks before making any assumptions.
  82  (defun careful-discriminant (a b c)
  83    "Compute the discriminant for a quadratic equation."
  84    (check-type a number)
  85    (check-type b number)
  86    (check-type c number)
  87    (locally (declare (number a b c))
  88      (- (* b b) (* 4 a c)))) =>  CAREFUL-DISCRIMINANT
  89  (careful-discriminant 1 2/3 -2) =>  76/9
  90 |#
  91   )
  92
  93
  94 #+nil
  95 (progn ;; experiments with GSL and the Lisp interface.
  96   (asdf:oos 'asdf:load-op 'gsll)
  97   (asdf:oos 'asdf:load-op 'gsll-tests)
  98
  99   ;; the following should be equivalent
 100   (defparameter *t1*  (LIST 6.18d0 6.647777777777779d0 6.18d0))
 101   (defparameter *t2*  (MULTIPLE-VALUE-LIST
 102                (LET ((VEC
 103                       (gsll:make-marray 'DOUBLE-FLOAT
 104                                         :INITIAL-CONTENTS '(-3.21d0 1.0d0 12.8d0)))
 105                      (WEIGHTS
 106                       (gsll:MAKE-MARRAY 'DOUBLE-FLOAT
 107                                         :INITIAL-CONTENTS '(3.0d0 1.0d0 2.0d0))))
 108                  (LET ((MEAN (gsll:MEAN VEC)))
 109                    (LIST (gsll:ABSOLUTE-DEVIATION VEC)
 110                          (gsll:WEIGHTED-ABSOLUTE-DEVIATION VEC WEIGHTS)
 111                          (gsll:ABSOLUTE-DEVIATION VEC MEAN))))))
 112   (eql *t1* *t2*)
 113
 114   ;; from (gsll:examples 'gsll::numerical-integration) ...
 115   (gsll:integration-qng gsll::one-sine 0.0d0 PI)
 116
 117   (gsll:defun-single axpb (x) (+ (* 2 x) 3)) ;; a<-2, b<-3
 118   (gsll:integration-qng axpb 1d0 2d0)
 119
 120   (let ((a 2)
 121         (b 3))
 122     (defun-single axpb2 (x) (+ (* a x) b)))
 123   (gsll:integration-qng axpb2 1d0 2d0)
 124
 125   ;;   BAD
 126   ;;   (gsll:integration-qng
 127   ;;    (let ((a 2)
 128   ;;     (b 3))
 129   ;;      (defun-single axpb2 (x) (+ (* a x) b)))
 130   ;;    1d0 2d0)
 131
 132   ;; right, but weird expansion...
 133   (gsll:integration-qng
 134    (let ((a 2)
 135          (b 3))
 136      (defun axpb2 (x) (+ (* a x) b))
 137      (gsll:def-single-function axpb2)
 138      axpb2)
 139    1d0 2d0)
 140
 141   ;; Linear least squares
 142
 143   (gsll:gsl-lookup "gsl_linalg_LU_decomp") ; => gsll:lu-decomposition
 144   (gsll:gsl-lookup "gsl_linalg_LU_solve") ; => gsll:lu-solve
 145   )
 146
 147
 148
 149 #+nil
 150 (progn ;; philosophy time
 151
 152   (setf my-model (model :name "ex1"
 153                         :data-slots (list w x y z)
 154                         :param-slots (list alpha beta gamma)
 155                         :math-form (regression-model :formula '(= w (+ (* beta x)
 156                                                                      (* alpha y)
 157                                                                      (* gamma z)
 158                                                                      normal-error))
 159                                                      :centrality 'median ; 'mean
 160                                                      )))
 161
 162 #| or:
 163   #R"W ~ x+  y + z "
 164 |#
 165
 166   (setf my-dataset (statistical-table :table data-frame-contents
 167                                       :metadata (list (:case-names (list ))
 168                                                       (:var-names (list ))
 169                                                       (:documentation "string of doc"))))
 170
 171   (setf my-analysis (analysis
 172                      :model my-model
 173                      :data my-dataset
 174                      :parameter-map (pairing (model-param-slots my-model)
 175                                              (data-var-names my-dataset))))
 176
 177   ;; ontological implications -- the analysis is an abstract class of
 178   ;; data, model, and mapping between the model and data.  The fit is
 179   ;; the instantiation of such.  This provides a statistical object
 180   ;; computation theory which can be realized as "executable
 181   ;; statistics" or "computable statistics".
 182   (setf my-analysis (analyze my-fit
 183                              :estimation-method 'linear-least-squares-regression))
 184
 185   ;; one of the tricks here is that one needs to provide the structure
 186   ;; from which to consider estimation, and more importantly, the
 187   ;; validity of the estimation.
 188
 189   ;;
 190   (setf linear-least-squares-regression
 191         (estimation-method-definition
 192          :variable-defintions ((list
 193                                 ;; from MachLearn: supervised,
 194                                 ;; unsupervised
 195                                 :data-response-vars list-drv ; nil if unsup
 196                                 ;;
 197                                 :param-vars list-pv
 198                                      :data-predictor-vars list-dpv
 199                                      ;; nil in this case.  these
 200                                      ;; describe "out-of-box" specs
 201                                      :hyper-vars list-hv))
 202          :form '(regression-additive-error
 203                  :central-form (linear-form drv pv dpv)
 204                  :error-form 'normal-error)
 205          :resulting-decision '(point-estimation interval-estimation)
 206          :philosophy 'frequentist
 207          :documentation "use least squares to fit a linear regression
 208                          model to data."))
 209
 210   (defparameter *statistical-philosophies*
 211     '(frequentist bayesian fiducial decision-analysis)
 212     "can be combined to build decision-making approaches and
 213     characterizations")
 214
 215   (defparameter *decisions*
 216     '(estimation selection testing)
 217     "possible results from a...")
 218   ;; is this really true?  One can embedded hypothesis testing within
 219   ;; estimation, as the hypothesis estimated to select.  And
 220   ;; categorical/continuous rear their ugly heads, but not really in
 221   ;; an essential way.
 222
 223   (defparameter *ontology-of-decision-procedures*
 224     (list :decisions
 225           (list :estimation
 226                 (list :point
 227                       (list :maximum-likelihood
 228                             :minimum-entropy
 229                             :least-squares
 230                             :method-of-moments)
 231                       :interval
 232                       (list :maximum-likelihood
 233                             :))
 234                 :testing
 235                 (list :fisherian
 236                       :neyman-pearson
 237                       (list :traditional
 238                             :bioequivalence-inversion)
 239                       :selection
 240                       (list :ranking
 241                             :top-k-of-n-select))
 242                 :parametric
 243                 :partially-parametric))
 244     "start of ontology"))
 245