src/data/data.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2008-11-16 20:24:10 tony>
   4 ;;; Creation:   <2005-08-xx 21:34:07 rossini>
   5 ;;; File:       data.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c)2005--2008, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   8 ;;;             on how it arrives.
   9 ;;; Purpose:    data package for lispstat
  10
  11 ;;; What is this talk of 'release'? Klingons do not make software
  12 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  13 ;;; designers and quality assurance people in its wake.
  14
  15 ;;; This organization and structure is new to the 21st Century
  16 ;;; version.
  17
  18 (in-package :lisp-stat-data)
  19
  20 ;;; conside that data has 3 genotypic chracteristrics.  The first
  21 ;;; would be form -- scalar, vector, array.  second would be datarep
  22 ;;; ("computer science simplistic data" type.  in particular integer,
  23 ;;; real, string, symbol.  The last would be statistical type
  24 ;;; ("usually handled by computer sicience approaches via metadata").
  25 ;;; augmenting datarep type with use in a statistical context,
  26 ;;; i.e. that would include nominal, ordinal, integer, continous,
  27 ;;; interval (orderable subtypes).  Clearly, the statistical type can
  28 ;;; be inherited, likewise the numerical type as well.  The form can
  29 ;;; be pushed up or simplified as necessary, but this can be
  30 ;;; challenging.
  31
  32 ;;; The first approach we will take for CLS is to handle this as
  33 ;;; lisp-only structures.  At the time of realization
  34 ;;; (?instantiation?) of an "abstract" model, the data should be
  35 ;;; pushed into an appropriate form (either "en masse", or
  36 ;;; "on-demand") into a linear algebra framework.
  37
  38 ;;; There is some excellent material on this by John Chambers in one
  39 ;;; of his earlier books.  Reference is being ignored to encourage
  40 ;;; people to read them all.  With all due respect to John, they've
  41 ;;; lasted quite well, but need to be updated.
  42
  43
  44 ;;; The purpose of this package is to manage data which will be
  45 ;;; processed by LispStat.  In particular, it will be important to
  46 ;;; register variables, datasets, relational structures, and other
  47 ;;; objects which could be the target for statistical modeling and
  48 ;;; inference.
  49
  50 (defvar *lisp-stat-data-table* (make-hash-table)
  51   "Marks up the data the could be used by.")
  52
  53 (defvar *lisp-stat-data-count* 0
  54   "number of items currently recorded.")
  55
  56 ;;; Data (storage) Types, dt-{.*}
  57 ;;;
  58 ;;; Data types are the representation of data from a computer-science
  59 ;;; perspective, i.e. what it is that they contain.  These types
  60 ;;; include particular forms of compound types (i.e. dataframe is
  61 ;;; array-like, but types differ, difference is row-wise, while array
  62 ;;; is a compound of elements of the same type.
  63 ;;;
  64 ;;; This is completely subject to change.
  65
  66 ;;Examples:
  67 ;;  (defun equidimensional (a)
  68 ;;    (or (< (array-rank a) 2)
  69 ;;        (apply #'= (array-dimensions a)))) =>  EQUIDIMENSIONAL
  70 ;;  (deftype square-matrix (&optional type size)
  71 ;;    `(and (array ,type (,size ,size))
  72 ;;          (satisfies equidimensional))) =>  SQUARE-MATRIX
  73
  74 (defun array-of-equal-dt-scalar-type (x)
  75   ;; return dt-scalar-type which fits (more precise that works)
  76   (if x
  77       'integer
  78       nil))
  79
  80 (defun array-of-equal-dt-scalar-type-within-column (x)
  81   ;; return dt-scalar-type which fits (more precise that works)
  82   (if x
  83       'integer
  84       nil))
  85
  86
  87
  88 (deftype dt-scalar (&optional type)
  89   `(or integer double complex symbol))
  90
  91 (deftype dt-array (&optional ndim dimlist)
  92   `(satisfies array-of-equal-dt-scalar-type))
  93
  94 (deftype dt-dataframe (&optional )
  95   `(satisfies array-of-equal-dt-scalar-type-within-column))
  96
  97 ;(deftype dt-relationaldata ()
  98 ;  `(satisfies (foreach unit in relationalUnit
  99 ;              (typep unit 'dt-dataframe))))
 100
 101
 102 ;;; Statistical Variable Types, sv-{.*}
 103 ;;;
 104 ;;; Statistical variable types work to represent the statistical
 105 ;;; category represented by the variable, i.e. nominal, ordinal,
 106 ;;; integral, continous, ratio.   This metadata can be used to hint at
 107 ;;; appropriate analysis methods -- or perhaps more critically, to
 108 ;;; define how these methods will fail in the final interrpretation.
 109
 110 (deftype sv-nominal (&optional n)
 111   ())
 112
 113 (deftype sv-ordinal (ordering &optional n)
 114   ())
 115
 116 (deftype sv-categorical ()
 117   `(satisfies (or sv-nominal sv-ordinal)))
 118 ;;(deftype sv-integer )
 119 ;;(deftype sv-real )  ;; precision could be a secondary component of real, rational, complex.
 120 ;;(deftype sv-rational )
 121 ;;(deftype sv-complex )
 122 ;;(deftype sv-continuous (or 'sv-integer 'sv-real 'sv-rational 'sv-complex)) ;; perhaps, call it "mostly contin..."
 123
 124
 125 ;;; Data I/O
 126
 127 ;; We can read 2 types of data -- those which are pure data, and those
 128 ;; which are impure (lisp-enabled, data as program as data thingy's).
 129
 130 (defparameter *lisp-stat-data-formats*
 131   '(csv tsv))
 132
 133 ;; (defgeneric data-read (srce frmt)
 134 ;;   "read data from stream srce, in format frmt.")
 135
 136 ;; (defgeneric data-write (srce frmt)
 137 ;;   "read data from stream srce, in format frmt.")
 138
 139 ;; (defmacro with-data (body)
 140 ;;   "Stream-handling, maintaining I/O through object typing.")
 141
 142 ;; design-wise should these be replaced with a "with-data" form?
 143
 144 ;;; These need to be elsewhere...!
 145 ;; DSV processing
 146 ;; XML processing
 147
 148 ;;; DM operations should be somewhere else as well.
 149 ;;; Data Management
 150
 151 ;; the goal is to have 2 operations which can be used to create new
 152 ;; data formats out of old ones.
 153
 154 ;; (defgeneric data-subset (ds description)
 155 ;;   "Take a dataset and make it smaller.")
 156
 157 ;; (defgeneric data-relate (ds description)
 158 ;;   "Take 2 or more datasets, and grow them into a bigger one through
 159 ;; relating them (i.e. merge is one example).")
 160
 161 ;;; What should we be able to do?
 162
 163 ;;; Actions on a single dataset
 164 ;;* subset-dataset original-set
 165 ;; :list-of-columns :list-of-rows :list-of-rows-and-columns
 166 ;; :list-of-indices
 167 ;;* resample-dataset original-set ;
 168 ;; :by-rows :by-columns :row-weights :column-weights
 169 ;; :new-number-of-columns :new-number-of-rows
 170 ;;
 171
 172 ;;; Actions based on 2 or more datasets
 173 ;;* concat-dataset set1 set2   ; no matching
 174 ;; :by-row :by-column :kronecker-product
 175 ;;* merge-dataset set1 set2
 176 ;; :match-on-column :match-on-row
 177
 178
 179 ;;; Data tools from "statistics.lsp"
 180
 181 ;;;;
 182 ;;;; Data File Reading
 183 ;;;;
 184
 185 (defun count-file-columns (fname)
 186 "Args: (fname)
 187 Returns the number of lisp items on the first nonblank line of file FNAME."
 188   (with-open-file (f fname)
 189     (if f
 190         (let ((line (do ((line (read-line f) (read-line f)))
 191                         ((or (null line) (< 0 (length line))) line))))
 192           (if line
 193               (with-input-from-string (s line)
 194                 (do ((n 0 (+ n 1)) (eof (gensym)))
 195                     ((eq eof (read s nil eof)) n))))))))
 196
 197 #+xlisp (defvar *xlisptable* *readtable*)
 198
 199 (if (not (fboundp 'open-file-dialog))
 200   #+dialogs
 201   (defun open-file-dialog () ;; why?(&optional set)
 202     (get-string-dialog "Enter a data file name:"))
 203   #-dialogs
 204   (defun open-file-dialog () ;; why? (&optional set)
 205     (error "You must provide a file name explicitly")))
 206
 207 (defun read-data-file (&optional (file (open-file-dialog t)))
 208 "Args:  (file)
 209 Returns a list of all lisp objects in FILE. FILE can be a string or a symbol,
 210 in which case the symbol'f print name is used."
 211   (if file
 212       (let ((eof (gensym)))
 213         (with-open-file (f file)
 214           (if f
 215               (do* ((r (read f nil eof) (read f nil eof))
 216                     (x (list nil))
 217                     (tail x (cdr tail)))
 218                    ((eq r eof) (cdr x))
 219                    (setf (cdr tail) (list r))))))))
 220
 221 ;;; New definition to avoid stack size limit in apply
 222 (defun read-data-columns (&optional (file (open-file-dialog t))
 223                                     (cols (if file
 224                                               (count-file-columns file))))
 225 "Args: (&optional file cols)
 226 Reads the data in FILE as COLS columns and returns a list of lists representing the columns."
 227   (if (and file cols)
 228       (transpose (split-list (read-data-file file) cols))))
 229
 230
 231 ;;; FIXME:AJR:  ALL THE FOLLOWING NEED TO BE SOLVED BY PLATFORM-INDEP PATHNAME WORK!
 232 ;;; FIXME:AJR: use either string or pathname.
 233
 234 (defun path-string-to-path (p s)
 235     (pathname (concatenate 'string (namestring p) s)))
 236
 237 (defun load-data (file)
 238 "Args: (file) as string
 239 Read in data file from the data examples library."
 240   (if (load (path-string-to-path *lispstat-data-dir* file))
 241       t
 242       (load (path-string-to-path *lispstat-data-dir* file))))
 243
 244 (defun load-example (file)
 245   "Args: (file) as string
 246 Read in lisp example file from the examples library."
 247   (if (load (path-string-to-path *lispstat-example-dir* file))
 248       t
 249       (load (path-string-to-path *lispstat-example-dir* file))))
 250
 251 ;;;;
 252 ;;;; Listing and Saving Variables and Functions
 253 ;;;;
 254
 255 (defvar *variables* nil)
 256 (defvar *ask-on-redefine* nil)
 257
 258 (defmacro def (symbol value)
 259 "Syntax: (def var form)
 260 VAR is not evaluated and must be a symbol.  Assigns the value of FORM to
 261 VAR and adds VAR to the list *VARIABLES* of def'ed variables. Returns VAR.
 262 If VAR is already bound and the global variable *ASK-ON-REDEFINE*
 263 is not nil then you are asked if you want to redefine the variable."
 264   `(unless (and *ask-on-redefine*
 265                 (boundp ',symbol)
 266                 (not (y-or-n-p "Variable has a value. Redefine?")))
 267     (if (boundp ',symbol)
 268         (setf ,symbol ,value)
 269         (defvar ,symbol ,value))
 270     (pushnew ',symbol *variables*)
 271     ',symbol))
 272
 273 (defun variables-list ()
 274     (mapcar #'intern (sort-data (mapcar #'string *variables*))))
 275
 276 (defun variables ()
 277 "Args:()
 278 Returns a list of the names of all def'ed variables to STREAM"
 279   (if *variables*
 280       (mapcar #'intern (sort-data (mapcar #'string *variables*)))))
 281
 282 (defun savevar (vars file)
 283 "Args: (vars file-name-root)
 284 VARS is a symbol or a list of symbols. FILE-NAME-ROOT is a string (or a symbol
 285 whose print name is used) not endinf in .lsp. The VARS and their current values
 286 are written to the file FILE-NAME-ROOT.lsp in a form suitable for use with the
 287 load command."
 288   (with-open-file (f (concatenate 'string (namestring file) ".lsp")
 289                      :direction :output)
 290     (let ((vars (if (consp vars) vars (list vars))))
 291       (flet ((save-one (x)
 292                (let ((v (symbol-value x)))
 293                  (if (objectp v)
 294                      (format f "(def ~s ~s)~%" x (send v :save))
 295                    (format f "(def ~s '~s)~%" x v)))))
 296             (mapcar #'save-one vars))
 297       vars)))
 298
 299 (defun undef (v)
 300 "Args: (v)
 301 If V is the symbol of a defined variable the variable it is unbound and
 302 removed from the list of defined variables. If V is a list of variable
 303 names each is unbound and removed. Returns V."
 304   (dolist (s (if (listp v) v (list v)))
 305           (when (member s *variables*)
 306                 (setq *variables* (delete s *variables*))
 307                 (makunbound s)))
 308   v)
 309