src/data/data-clos.lisp

   1 ;;; -*- mode: lisp -*-
   2
   3 ;;; Time-stamp: <2009-03-25 08:56:16 tony>
   4 ;;; Creation:   <2008-03-12 17:18:42 blindglobe@gmail.com>
   5 ;;; File:       data-clos.lisp
   6 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   7 ;;; Copyright:  (c)2008, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   8 ;;;             on how it arrives.
   9
  10 ;;; Purpose:    Data packaging and access for Common Lisp Statistics.
  11 ;;;             This redoes data storage structures in a CLOS based
  12 ;;;             framework.
  13 ;;;
  14
  15 ;;; What is this talk of 'release'? Klingons do not make software
  16 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  17 ;;; designers and quality assurance people in its wake.
  18
  19 (in-package :lisp-stat-data-clos)
  20
  21 ;;; No real basis for work, there is a bit of new-ness and R-ness to
  22 ;;; this work. In particular, the notion of relation is key and
  23 ;;; integral to the analysis.  Tables are related and matched vectors,
  24 ;;; for example.  "column" vectors are related observations (by
  25 ;;; measure/recording) while "row" vectors are related readings (by
  26 ;;; case)
  27
  28 ;;; Relational structure -- can we capture a completely unnormalized
  29 ;;; data strucutre to propose possible modeling approaches, and
  30 ;;; propose appropriate models and inferential strategies?
  31
  32 ;;; So we want a verb-driven API for data collection construction.  We
  33 ;;; should encode independence or lack of, as possible.
  34
  35 ;;; Need to figure out typed vectors.  We then map a series of typed
  36 ;;; vectors over to tables where columns are equal typed.  In a sense,
  37 ;;; this is a relation (1-1) of equal-typed arrays.  For the most
  38 ;;; part, this ends up making the R data.frame into a relational
  39 ;;; building block (considering 1-1 mappings using row ID as a
  40 ;;; relation).  Is this a worthwhile generalization?
  41
  42 ;;; verbs vs semantics for DS conversion -- consider the possibily of
  43 ;;; how adverbs and verbs relate, where to put which semantically to
  44 ;;; allow for general approach.
  45
  46 ;;; eg. Kasper's talk on the FUSION collection of parsers.
  47
  48 ;;;
  49 ;;; Need to consider modification APIs
  50 ;;; actions are:
  51 ;;; - import
  52 ;;; - get/set row names (case names)
  53 ;;; -         column names (variable names)
  54 ;;; -         dataset values
  55 ;;; -         annotation/metadata
  56 ;;; -    make sure that we do coherency checking in the exported
  57 ;;; -    functions.
  58 ;;; -    ...
  59 ;;; - reshapeData/reformat/reshapr a reformed version of the dataset (no
  60 ;;;           additional input).
  61 ;;; -         either overwriting or not, i.e. with or without copy.
  62 ;;; - check consistency of resulting data with metadata and related
  63 ;;;           data information.
  64 ;;; -
  65
  66
  67 ;;; Misc Functions (to move into a lisp data manipulation support package)
  68
  69 (defun gen-seq (n &optional (start 1))
  70   "There has to be a better way -- I'm sure of it!  default count from 1.
  71  (gen-seq 4) ; =>  (1 2 3 4)
  72  (gen-seq 0) ; => nil
  73  (gen-seq 5 3) ; => 3 4 5
  74 "
  75   (if (>= n start)
  76       (append (gen-seq (- n 1) start) (list n))))
  77
  78 (defun repeat-seq (n item)
  79   "FIXME: There has to be a better way -- I'm sure of it!
  80  (repeat-seq 3 \"d\") ; => (\"d\" \"d\" \"d\")
  81  (repeat-seq 3 'd) ; => ('d 'd 'd)
  82 "
  83   (if (>= n 1)
  84       (append (repeat-seq (1- n)  item) (list item))))
  85
  86
  87
  88 (defun strsym->indexnum (df strsym)
  89   "Returns a number indicating which column in DF has STRSYM labeling
  90 it.  Probably should be a method dispatching on the type of
  91 DATAFRAME-LIKE."
  92   (position strsym (varlabels df)))
  93
  94 #|
  95
  96   (equal 'testme 'testme)
  97   (defparameter *test-pos* 'testme)
  98   (position *test-pos* (list 'a 'b 'testme 'c))
  99   (position #'(lambda (x) (equal x "testme")) (list "a" "b" "testme" "c"))
 100   (position #'(lambda (x) (equal x 1)) (list 2 1 3 4))
 101
 102
 103 |#
 104 ;;; abstract dataframe class
 105
 106 (defclass dataframe-like (matrix-like)
 107   (
 108    ;; Matrix-like (from lisp-matrix) is basically a rectangular table
 109    ;; without storage.  We emulate that, and add storage, row/column
 110    ;; labels, and within-column-typing.
 111
 112    ;; STORE is the storage component.  We ignore this in the DATAFRAME-LIKE
 113    ;; class, as it is the primary differentiator, driving how access
 114    ;; (getting/setting) is done.   We create methods depending on the
 115    ;; storage component, which access data as appropriate.  See
 116    ;; DATAFRAME-ARRAY for an example implementation.
 117    ;; the rest of this is metadata.  In particular, we should find a
 118    ;; more flexible, compact way to store this.
 119    (case-labels :initform nil
 120                 :initarg :case-labels
 121                 :type list
 122                 :accessor case-labels
 123                 :documentation "labels used for describing cases (doc
 124                                 metadata), possibly used for merging.")
 125    (var-labels :initform nil
 126                :initarg :var-labels
 127                :type list
 128                :accessor var-labels
 129                :documentation "Variable names.")
 130    (var-types :initform nil
 131               :initarg :var-types
 132               :type list
 133               :accessor var-types
 134               :documentation "variable types to ensure fit")
 135    (documentation-string :initform nil
 136                          :initarg :doc
 137                          :accessor doc-string
 138                          :documentation "additional information,
 139   potentially uncomputable, possibly metadata, about dataframe-like
 140   instance."))
 141   (:documentation "Abstract class for standard statistical analysis
 142                    dataset for independent data.  Rows are considered
 143                    to be independent, matching observations.  Columns
 144                    are considered to be type-consistent, match a
 145                    variable with distribution.  inherits from
 146                    lisp-matrix base MATRIX-LIKE class.
 147
 148                    DATAFRAME-LIKE is the basic cases by variables
 149                    framework.  Need to embed this within other
 150                    structures which allow for generalized relations.
 151                    Goal is to ensure that relations imply and drive
 152                    the potential for statistical relativeness such as
 153                    correlation, interference, and similar concepts."))
 154
 155
 156 ;;; Generics specialized above matrix-like, particularly for
 157 ;;; dataframe-like objects.  Need methods for any storage
 158 ;;; implementation.
 159
 160 (defgeneric dataframe-dimensions (df)
 161   (:documentation "")
 162   (:method ((df dataframe-like))
 163     (error "dispatch on virtual class.")))
 164
 165 (defgeneric dataframe-dimension (df index)
 166   (:documentation "")
 167   (:method ((df dataframe-like) index)
 168     (elt (dataframe-dimensions df) index)))
 169
 170 (defgeneric dfref (df index1 index2 &key return-type)
 171   (:documentation "scalar access with selection of possible return
 172   object types.")
 173   (:method ((df dataframe-like) index1 index2 &key return-type)
 174     (error "need a real class with real storage to reference elements.")))
 175
 176 ;;; Specializing on superclasses...
 177 ;;; Access and Extraction: implementations needed for any storage
 178 ;;; type.  But here, just to point out that we've got a specializing
 179 ;;; virtual subclass (DATAFRAME-LIKE specializing MATRIX-LIKE).
 180
 181 (defmethod nrows ((df dataframe-like))
 182   "specializes on inheritance from matrix-like in lisp-matrix."
 183   (error "Need implementation; can't dispatch on virtual class DATAFRAME-LIKE."))
 184
 185 (defmethod ncols ((df dataframe-like))
 186   "specializes on inheritance from matrix-like in lisp-matrix."
 187   (error "Need implementation; can't dispatch on virtual class DATAFRAME-LIKE."))
 188
 189 ;; Testing consistency/coherency.
 190
 191 (defgeneric consistent-dataframe-p (df)
 192   (:documentation "methods to check for consistency.")
 193   (:method ((df dataframe-like))
 194     (error "Need implementation; can't dispatch on virtual class DATAFRAME-LIKE.")))
 195
 196 #|
 197
 198  (defun ensure-consistent-datatable-type (dt lot)
 199   "given a datatable and a listoftypes, ensure that the datatble
 200   variables are consistent."
 201   (destructuring-bind (n p) ;; why use let when we can be cool?  Sigh.
 202       (array-dimensions dt)
 203     (dotimes (i n)
 204       (dotimes (j p)
 205         (check-type  (aref dt i j) (elt lot j))))))
 206 |#
 207
 208
 209 ;;; GENERAL FUNCTIONS WHICH DISPATCH ON INTERNAL METHODS OR ARGS
 210 ;;;
 211 ;;; Q: change the following to generic functions and dispatch on
 212 ;;; array, matrix, and dataframe?  Others?
 213 (defun make-labels (initstr num &key (return-type 'string))
 214   "generate a list of strings (or symbols?) which can be used as
 215 labels, i.e. something like
 216    '(a1 a2 a3)
 217 or
 218    '(\"a1\" \"a2\" \"a3\")."
 219   (concatenate return-type (repeat-seq initstr num) (gen-seq num)))
 220
 221 (defun make-dataframe (newdata &key
 222                        (vartypes nil)
 223                        (caselabels nil) (varlabels nil)
 224                        (doc "no docs"))
 225   (check-type newdata array)
 226   (check-type caselabels sequence)
 227   (check-type varlabels sequence)
 228   (check-type doc string)
 229
 230   (let ((newcaselabels (if caselabels
 231                            caselabels
 232                            (make-labels "C" (array-dimension store 0))))
 233         (newvarlabels (if varlabels
 234                           varlabels
 235                           (make-labels "C" (array-dimension store 1)))))
 236     (make-instance 'dataframe-array
 237                    :storage newdata
 238                    :nrows (length newcaselabels)
 239                    :ncols (length newvarlabels)
 240                    :case-labels newcaselabels
 241                    :var-labels newvarlabels
 242                    :var-types)))
 243
 244 (defun row-order-as-list (ary)
 245   "Pull out data in row order into a list."
 246   (let ((result (list))
 247         (nrows (nth 0 (array-dimensions ary)))
 248         (ncols (nth 1 (array-dimensions ary))))
 249     (dotimes (i ncols)
 250       (dotimes (j nrows)
 251         (append result (aref ary i j))))))
 252
 253 (defun col-order-as-list (ary)
 254   "Pull out data in row order into a list."
 255   (let ((result (list))
 256         (nrows (nth 0 (array-dimensions ary)))
 257         (ncols (nth 1 (array-dimensions ary))))
 258     (dotimes (i nrows)
 259       (dotimes (j ncols)
 260         (append result (aref ary i j))))))
 261
 262 (defun transpose-array (ary)
 263   "map NxM to MxN."
 264   (make-array (reverse (array-dimensions ary))
 265       :initial-contents (col-order-as-list ary)))
 266
 267 ;;; THE FOLLOWING 2 dual-sets done to provide error checking
 268 ;;; possibilities on top of the generic function structure.  Not
 269 ;;; intended as make-work!
 270
 271 (defun varlabels (df)
 272   "Variable-name handling for DATAFRAME-LIKE.  Needs error checking."
 273   (var-labels df))
 274
 275 (defun set-varlabels (df vl)
 276   "Variable-name handling for DATAFRAME-LIKE.  Needs error checking."
 277   (if (= (length (var-labels df))
 278          (length vl))
 279       (setf (var-labels df) vl)
 280       (error "wrong size.")))
 281
 282 (defsetf varlabels set-varlabels)
 283
 284 ;;; Case-name handling for Tables.  Needs error checking.
 285 (defun caselabels (df)
 286   "Case-name handling for DATAFRAME-LIKE.  Needs error checking."
 287   (case-labels df))
 288
 289 (defun set-caselabels (df cl)
 290   "Case-name handling for DATAFRAME-LIKE.  Needs error checking."
 291   (if (= (length (case-labels df))
 292          (length cl))
 293       (setf (case-labels df) cl)
 294       (error "wrong size.")))
 295
 296 (defsetf caselabels set-caselabels)
 297
 298 ;;;;;;;;;;;; IMPLEMENTATIONS, with appropriate methods.
 299
 300 ;; See also:
 301 ;; (documentation 'dataframe-like  'type)
 302
 303 (defclass dataframe-array (dataframe-like)
 304   ((store :initform nil
 305           :initarg :storage
 306           :type (array * *)
 307           :accessor dataset
 308           :documentation "Data storage: typed as array."))
 309   (:documentation "example implementation of dataframe-like using storage
 310   based on lisp arrays.  An obvious alternative could be a
 311   dataframe-matrix-like which uses the lisp-matrix classes."))
 312
 313 (defmethod nrows ((df dataframe-array))
 314   "specializes on inheritance from matrix-like in lisp-matrix."
 315   (array-dimension (dataset df) 0))
 316
 317 (defmethod ncols ((df dataframe-array))
 318   "specializes on inheritance from matrix-like in lisp-matrix."
 319   (array-dimension (dataset df) 1))
 320
 321 (defmethod consistent-dataframe-p ((ds dataframe-array))
 322   "Test that dataframe-like is internally consistent with metadata.
 323 Ensure that dims of stored data are same as case and var labels.
 324
 325 Currently checks length of things, but needs to check type of things
 326 as well."
 327   (and
 328    ;; ensure dimensionality
 329    (equal (list (ncols ds) (nrows ds)) ; array-dimensions (dataset ds))
 330           (list (length (var-labels ds))
 331                 (length (case-labels ds))))
 332    ;; when dims sane, check-type for each variable
 333    (progn
 334      (dolist (i (ncols ds))
 335        (dotimes (j (nrows ds))
 336          (typep (aref (dataset ds) i j) (nth i (var-types ds)))))
 337      t)))
 338
 339
 340
 341 #|
 342
 343  (defun testecase (s)
 344    (ecase s
 345      ((scalar) 1)
 346      ((asd asdf) 2)))
 347
 348  (testecase 'scalar)
 349  (testecase 'asd)
 350  (testecase 'asdf)
 351  (testecase 'as)
 352 |#
 353
 354
 355 (defmethod dfref ((df dataframe-array) (index1 number) (index2 number) &key return-type)
 356   "Returns a scalar in array, in the same vein as aref, mref, vref, etc.
 357 idx1/2 is row/col or case/var.  Return-type could be 'scalar,
 358 'dataframe, ..."
 359   (ecase return-type
 360     ((scalar)
 361      (aref (dataset df) index1 index2))
 362     ((dataframe)
 363      (make-instance 'dataframe-array
 364                     :storage (make-array
 365                               (list 1 1)
 366                               :initial-contents (dfref df index1 index2))
 367                     ;; ensure copy for this and following
 368                     :doc (doc-string df)
 369                     :case-labels (nth index1 (caselabels df))
 370                     :var-labels (nth index2  (varlabels df))
 371                     ;; shound the type spec assume, as
 372                     ;; below, or should it inherit from the
 373                     ;; dataframe we are selecting from?
 374                     :var-types (nth index2 (var-types df))))))
 375
 376
 377
 378 (defmethod dfref ((df dataframe-array) (index1 string) (index2 string) &key return-type)
 379   "Returns a scalar in array, in the same vein as aref, mref, vref, etc.
 380 idx1/2 is row/col or case/var.  This method dispatches when using
 381 strings or symbols.  Merge with the index-as-number variant?"
 382   (let ((idx1 (strsym->indexnum df index1))
 383         (idx2 (strsym->indexnum df index2)))
 384     (ecase return-type
 385       ((scalar) (aref (dataset df) idx1 idx2))
 386       ((dataframe) (make-instance 'dataframe-array
 387                                   :storage (make-array
 388                                             (list 1 1)
 389                                             :initial-contents (dfref df idx1 idx2))
 390                                   ;; ensure copy for this and following
 391                                   :doc (doc-string df)
 392                                   :case-labels (elt (caselabels df) idx1)
 393                                   :var-labels  (elt (varlabels df) idx2)
 394                                   ;; shound the type spec assume, as
 395                                   ;; below, or should it inherit from the
 396                                   ;; dataframe we are selecting from?
 397                                   :var-types (nth idx2 (var-types df)))))))
 398
 399
 400 (defun dfref-var (df index return-type)
 401   "Returns the data in a single variable as type.
 402 type = sequence, vector, vector-like (if valid numeric type) or dataframe."
 403   (ecase return-type
 404     (('list)
 405      (map 'list
 406           #'(lambda (x) (dfref df index x))
 407           (gen-seq (nth 2 (array-dimensions (dataset df))))))
 408     (('vector) t)
 409     (:vector-like t)
 410     (:matrix-like t)
 411     (:dataframe t)))
 412
 413 (defun dfref-case (df index return-type)
 414   "Returns row as sequence."
 415   (ecase return-type
 416     (:list
 417      (map 'list
 418           #'(lambda (x) (dfref df x index))
 419           (gen-seq (nth 1 (array-dimensions (dataset df))))))
 420     (:vector t)
 421     (:vector-like t)
 422     (:matrix-like t)
 423     (:dataframe t)))
 424
 425 ;; FIXME
 426 (defun dfref-2indexlist (df indexlist1 indexlist2 &key (return-type :array))
 427   "return an array, row X col dims.  FIXME TESTME"
 428   (case return-type
 429     (:array
 430      (let ((my-pre-array (list)))
 431        (dolist (x indexlist1)
 432          (dolist (y indexlist2)
 433            (append my-pre-array (dfref df x y))))
 434        (make-array (list (length indexlist1)
 435                          (length indexlist2))
 436                    :initial-contents my-pre-array)))
 437     (:dataframe
 438      (make-instance 'dataframe-array
 439                     :storage (make-array
 440                               (list (length indexlist1)
 441                                     (length indexlist2))
 442                               :initial-contents (dataset df))
 443                     ;; ensure copy for this and following
 444                     :doc (doc-string df)
 445                     ;; the following 2 need to be subseted based on
 446                     ;; the values of indexlist1 and indexlist2
 447                     :case-labels (case-labels df)
 448                     :var-labels (var-labels df)))))
 449
 450 ;;; Do we establish methods for dataframe-like, which specialize to
 451 ;;; particular instances of storage?
 452
 453 (defmethod print-object ((object dataframe-array) stream)
 454   (print-unreadable-object (object stream :type t)
 455     (format stream " ~d x ~d" (nrows object) (ncols object))
 456     (terpri stream)
 457     ;; (format stream "~T ~{~S ~T~}" (var-labels object))
 458     (dotimes (j (ncols object))
 459       (write-char #\tab stream)
 460       (format stream "~A~T" (nth j (var-labels object))))
 461     (dotimes (i (nrows object))
 462       (terpri stream)
 463       (format stream "~A:~T" (nth i (case-labels object)))
 464       (dotimes (j (ncols object))
 465         ;; (write-char #\space stream)
 466         (write-char #\tab stream)
 467         (write (dfref object i j) :stream stream)))))
 468
 469 #|
 470  (defun print-structure-relational (ds)
 471   "example of what we want the methods to look like.  Should be sort
 472 of like a graph of spreadsheets if the storage is a relational
 473 structure."
 474   (dolist (k (relations ds))
 475     (let ((currentRelationSet (getRelation ds k)))
 476       (print-as-row (var-labels currentRelationSet))
 477       (let ((j -1))
 478         (dolist (i (case-labels currentRelationSet))
 479           (print-as-row
 480            (append (list i)
 481                    (dfref-obsn (dataset currentRelationSet)
 482                                (incf j)))))))))
 483 |#