blob f485c2d4e65a0cb780fee08fc429d40ce3b4d165
1 ;;; -*- mode: lisp -*-
3 ;;; Time-stamp: <2009-03-31 17:20:39 tony>
4 ;;; Creation: <2008-03-12 17:18:42 blindglobe@gmail.com>
5 ;;; File: data-clos.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2008, AJ Rossini. BSD, LLGPL, or GPLv2, depending
8 ;;; on how it arrives.
10 ;;; Purpose: Data packaging and access for Common Lisp Statistics.
11 ;;; This redoes data storage structures in a CLOS based
12 ;;; framework.
13 ;;;
15 ;;; What is this talk of 'release'? Klingons do not make software
16 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
17 ;;; designers and quality assurance people in its wake.
19 (in-package :lisp-stat-data-clos)
21 ;;; No real basis for work, there is a bit of new-ness and R-ness to
22 ;;; this work. In particular, the notion of relation is key and
23 ;;; integral to the analysis. Tables are related and matched vectors,
24 ;;; for example. "column" vectors are related observations (by
25 ;;; measure/recording) while "row" vectors are related readings (by
26 ;;; case)
28 ;;; Relational structure -- can we capture a completely unnormalized
29 ;;; data strucutre to propose possible modeling approaches, and
30 ;;; propose appropriate models and inferential strategies?
32 ;;; So we want a verb-driven API for data collection construction. We
33 ;;; should encode independence or lack of, as possible.
35 ;;; Need to figure out typed vectors. We then map a series of typed
36 ;;; vectors over to tables where columns are equal typed. In a sense,
37 ;;; this is a relation (1-1) of equal-typed arrays. For the most
38 ;;; part, this ends up making the R data.frame into a relational
39 ;;; building block (considering 1-1 mappings using row ID as a
40 ;;; relation). Is this a worthwhile generalization?
42 ;;; verbs vs semantics for DS conversion -- consider the possibily of
43 ;;; how adverbs and verbs relate, where to put which semantically to
44 ;;; allow for general approach.
46 ;;; eg. Kasper's talk on the FUSION collection of parsers.
48 ;;;
49 ;;; Need to consider modification APIs
50 ;;; actions are:
51 ;;; - import
52 ;;; - get/set row names (case names)
53 ;;; - column names (variable names)
54 ;;; - dataset values
56 ;;; - make sure that we do coherency checking in the exported
57 ;;; - functions.
58 ;;; - ...
59 ;;; - reshapeData/reformat/reshapr a reformed version of the dataset (no
61 ;;; - either overwriting or not, i.e. with or without copy.
62 ;;; - check consistency of resulting data with metadata and related
63 ;;; data information.
64 ;;; -
67 ;;; Misc Functions (to move into a lisp data manipulation support package)
69 ;; the next two should be merged into a general replicator pattern.
70 (defun gen-seq (n &optional (start 1))
71 "Generates an integer sequence of length N starting at START. Used
72 for indexing."
73 (if (>= n start)
74 (append (gen-seq (- n 1) start) (list n))))
76 (defun repeat-seq (n item)
77 "FIXME: There has to be a better way -- I'm sure of it!
78 (repeat-seq 3 \"d\") ; => (\"d\" \"d\" \"d\")
79 (repeat-seq 3 'd) ; => ('d 'd 'd)"
80 (if (>= n 1)
81 (append (repeat-seq (1- n) item) (list item))))
83 (defun strsym->indexnum (df strsym)
84 "Returns a number indicating the DF column labelled by STRSYM.
85 Probably should be a method dispatching on DATAFRAME-LIKE type."
86 (position strsym (varlabels df)))
88 (defun string->number (str)
89 "Convert a string <str> representing a number to a number. A second value is
90 returned indicating the success of the conversion.
91 Examples:
92 (string->number \"123\") ; => 123 t
93 (string->number \"1.23\") ; => 1.23 t"
96 (values num (numberp num)))))
101 (equal 'testme 'testme)
102 (defparameter *test-pos* 'testme)
103 (position *test-pos* (list 'a 'b 'testme 'c))
104 (position #'(lambda (x) (equal x "testme")) (list "a" "b" "testme" "c"))
105 (position #'(lambda (x) (equal x 1)) (list 2 1 3 4))
109 ;;; abstract dataframe class
111 (defclass dataframe-like (matrix-like)
113 ;; Matrix-like (from lisp-matrix) is basically a rectangular table
114 ;; without storage. We emulate that, and add storage, row/column
115 ;; labels, and within-column-typing.
117 ;; STORE is the storage component. We ignore this in the DATAFRAME-LIKE
118 ;; class, as it is the primary differentiator, driving how access
119 ;; (getting/setting) is done. We create methods depending on the
120 ;; storage component, which access data as appropriate. See
121 ;; DATAFRAME-ARRAY for an example implementation.
122 ;; the rest of this is metadata. In particular, we should find a
123 ;; more flexible, compact way to store this.
124 (case-labels :initform nil
125 :initarg :case-labels
126 :type list
127 :accessor case-labels
128 :documentation "labels used for describing cases (doc
129 metadata), possibly used for merging.")
130 (var-labels :initform nil
131 :initarg :var-labels
132 :type list
133 :accessor var-labels
134 :documentation "Variable names.")
135 (var-types :initform nil
136 :initarg :var-types
137 :type list
138 :accessor var-types
139 :documentation "variable types to ensure fit")
140 (documentation-string :initform nil
141 :initarg :doc
142 :accessor doc-string
145 instance."))
146 (:documentation "Abstract class for standard statistical analysis
147 dataset for independent data. Rows are considered
148 to be independent, matching observations. Columns
149 are considered to be type-consistent, match a
150 variable with distribution. inherits from
151 lisp-matrix base MATRIX-LIKE class.
153 DATAFRAME-LIKE is the basic cases by variables
154 framework. Need to embed this within other
155 structures which allow for generalized relations.
156 Goal is to ensure that relations imply and drive
157 the potential for statistical relativeness such as
158 correlation, interference, and similar concepts."))
161 ;;; Generics specialized above matrix-like, particularly for
162 ;;; dataframe-like objects. Need methods for any storage
163 ;;; implementation.
165 (defgeneric dataframe-dimensions (df)
166 (:documentation "")
167 (:method ((df dataframe-like))
168 (error "dispatch on virtual class.")))
170 (defgeneric dataframe-dimension (df index)
171 (:documentation "")
172 (:method ((df dataframe-like) index)
173 (elt (dataframe-dimensions df) index)))
175 (defgeneric dfref (df index1 index2)
176 (:documentation "scalar access with selection of possible return object types.")
177 (:method ((df dataframe-like) index1 index2)
178 (error "Need real class with real storage to reference elements.")))
180 ;;; Specializing on superclasses...
181 ;;; Access and Extraction: implementations needed for any storage
182 ;;; type. But here, just to point out that we've got a specializing
183 ;;; virtual subclass (DATAFRAME-LIKE specializing MATRIX-LIKE).
185 (defmethod nrows ((df dataframe-like))
186 "specializes on inheritance from matrix-like in lisp-matrix."
187 (error "Need implementation; can't dispatch on virtual class DATAFRAME-LIKE."))
189 (defmethod ncols ((df dataframe-like))
190 "specializes on inheritance from matrix-like in lisp-matrix."
191 (error "Need implementation; can't dispatch on virtual class DATAFRAME-LIKE."))
193 ;; Testing consistency/coherency.
195 (defgeneric consistent-dataframe-p (df)
196 (:documentation "methods to check for consistency.")
197 (:method ((df dataframe-like))
198 (and
199 ;; ensure dimensionality
200 (= (length (var-labels df)) (ncols df)) ; array-dimensions (dataset df))
201 (= (length (case-labels df)) (nrows df))
202 ;; when dims sane, check-type for each variable
203 (progn
204 (dotimes (i (nrows df))
205 (dotimes (j (ncols df))
206 ;; below, dfref bombs if not a df-like subclass...
207 (typep (dfref df i j) (nth j (var-types df)))))
208 t))))
212 (defun ensure-consistent-datatable-type (dt lot)
213 "given a datatable and a listoftypes, ensure that the datatble
214 variables are consistent."
215 (destructuring-bind (n p) ;; why use let when we can be cool? Sigh.
216 (array-dimensions dt)
217 (dotimes (i n)
218 (dotimes (j p)
219 (check-type (aref dt i j) (elt lot j))))))
223 ;;; GENERAL FUNCTIONS WHICH DISPATCH ON INTERNAL METHODS OR ARGS
225 ;;; Q: change the following to generic functions and dispatch on
226 ;;; array, matrix, and dataframe? Others?
227 (defun make-labels (initstr num)
228 "generate a list of strings which can be used as labels, i.e. something like
229 '(\"a1\" \"a2\" \"a3\")."
230 (check-type initstr string)
231 (mapcar #'(lambda (x y) (concatenate 'string x y))
232 (repeat-seq num initstr)
233 (mapcar #'(lambda (x) (format nil "~A" x)) (gen-seq num))))
236 (make-labels 'c 2)
237 (make-labels "c" 4)
240 (defun ncase-store (store)
241 (etypecase store
242 (array (array-dimension store 0))
243 (matrix-like (nrows store))))
245 (defun nvars-store (store)
246 (etypecase store
247 (array (array-dimension store 1))
248 (matrix-like (ncols store))))
251 (defun make-dataframe (newdata
252 &key (vartypes nil)
253 (caselabels nil) (varlabels nil)
254 (doc "no docs"))
255 "Helper function to use instead of make-instance to assure
256 construction of proper DF-array."
257 (check-type newdata (or matrix-like array))
258 (check-type caselabels sequence)
259 (check-type varlabels sequence)
260 (check-type doc string)
261 (let ((ncases (ncase-store newdata))
262 (nvars (nvars-store newdata)))
263 (if caselabels (assert (= ncases (length caselabels))))
264 (if varlabels (assert (= nvars (length varlabels))))
265 (let ((newcaselabels (if caselabels
266 caselabels
267 (make-labels "C" ncases)))
268 (newvarlabels (if varlabels
269 varlabels
270 (make-labels "V" nvars))))
271 (etypecase newdata
272 (array
273 (make-instance 'dataframe-array
274 :storage newdata
275 :nrows (length newcaselabels)
276 :ncols (length newvarlabels)
277 :case-labels newcaselabels
278 :var-labels newvarlabels
279 :var-types vartypes))
280 (matrix-like
281 (make-instance 'dataframe-matrixlike
282 :storage newdata
283 :nrows (length newcaselabels)
284 :ncols (length newvarlabels)
285 :case-labels newcaselabels
286 :var-labels newvarlabels
287 :var-types vartypes))))))
290 (make-dataframe #2A((1.2d0 1.3d0) (2.0d0 4.0d0)))
291 (make-dataframe #2A(('a 1) ('b 2)))
292 (dfref (make-dataframe #2A(('a 1) ('b 2))) 0 1)
293 (dfref (make-dataframe #2A(('a 1) ('b 2))) 1 0)
294 (make-dataframe 4) ; ERROR, should we allow?
295 (make-dataframe #2A((4)))
296 (make-dataframe (rand 10 5)) ;; ERROR, but should work!
300 (defun row-order-as-list (ary)
301 "Pull out data in row order into a list."
302 (let ((result (list))
303 (nrows (nth 0 (array-dimensions ary)))
304 (ncols (nth 1 (array-dimensions ary))))
305 (dotimes (i ncols)
306 (dotimes (j nrows)
307 (append result (aref ary i j))))))
309 (defun col-order-as-list (ary)
310 "Pull out data in row order into a list."
311 (let ((result (list))
312 (nrows (nth 0 (array-dimensions ary)))
313 (ncols (nth 1 (array-dimensions ary))))
314 (dotimes (i nrows)
315 (dotimes (j ncols)
316 (append result (aref ary i j))))))
318 (defun transpose-array (ary)
319 "map NxM to MxN."
320 (make-array (reverse (array-dimensions ary))
321 :initial-contents (col-order-as-list ary)))
323 ;;; THE FOLLOWING 2 dual-sets done to provide error checking
324 ;;; possibilities on top of the generic function structure. Not
325 ;;; intended as make-work!
327 (defun varlabels (df)
328 "Variable-name handling for DATAFRAME-LIKE. Needs error checking."
329 (var-labels df))
331 (defun set-varlabels (df vl)
332 "Variable-name handling for DATAFRAME-LIKE. Needs error checking."
333 (if (= (length (var-labels df))
334 (length vl))
335 (setf (var-labels df) vl)
336 (error "wrong size.")))
338 (defsetf varlabels set-varlabels)
340 ;;; Case-name handling for Tables. Needs error checking.
341 (defun caselabels (df)
342 "Case-name handling for DATAFRAME-LIKE. Needs error checking."
343 (case-labels df))
345 (defun set-caselabels (df cl)
346 "Case-name handling for DATAFRAME-LIKE. Needs error checking."
347 (if (= (length (case-labels df))
348 (length cl))
349 (setf (case-labels df) cl)
350 (error "wrong size.")))
352 (defsetf caselabels set-caselabels)
354 ;;;;;;;;;;;; IMPLEMENTATIONS, with appropriate methods.
356 ;; (documentation 'dataframe-like 'type)
358 ;;;;; DATAFRAME-ARRAY
360 (defclass dataframe-array (dataframe-like)
361 ((store :initform nil
362 :initarg :storage
363 :type (array * *)
364 :accessor dataset
365 :documentation "Data storage: typed as array."))
366 (:documentation "example implementation of dataframe-like using storage
367 based on lisp arrays. An obvious alternative could be a
368 dataframe-matrix-like which uses the lisp-matrix classes."))
370 (defmethod nrows ((df dataframe-array))
371 "specializes on inheritance from matrix-like in lisp-matrix."
372 (array-dimension (dataset df) 0))
374 (defmethod ncols ((df dataframe-array))
375 "specializes on inheritance from matrix-like in lisp-matrix."
376 (array-dimension (dataset df) 1))
378 (defmethod dfref ((df dataframe-array)
379 (index1 number) (index2 number))
380 "Returns a scalar in array, in the same vein as aref, mref, vref, etc.
381 idx1/2 is row/col or case/var."
382 (aref (dataset df) index1 index2))
385 ;;;;; DATAFRAME-MATRIXLIKE
387 (defclass dataframe-matrixlike (dataframe-like)
388 ((store :initform nil
389 :initarg :storage
390 :type matrix-like
391 :accessor dataset
392 :documentation "Data storage: typed as matrix-like
393 (numerical only)."))
394 (:documentation "example implementation of dataframe-like using storage
395 based on lisp-matrix structures."))
397 (defmethod nrows ((df dataframe-matrixlike))
398 "specializes on inheritance from matrix-like in lisp-matrix."
399 (matrix-dimension (dataset df) 0))
401 (defmethod ncols ((df dataframe-matrixlike))
402 "specializes on inheritance from matrix-like in lisp-matrix."
403 (matrix-dimension (dataset df) 1))
405 (defmethod dfref ((df dataframe-matrixlike)
406 (index1 number) (index2 number))
407 "Returns a scalar in array, in the same vein as aref, mref, vref, etc.
408 idx1/2 is row/col or case/var."
409 (mref (dataset df) index1 index2))
413 ;;;;;; IMPLEMENTATION INDEPENDENT FUNCTIONS AND METHODS
414 ;;;;;; (use only dfref, nrows, ncols and similar dataframe-like
415 ;;;;;; components as core).
417 (defun dfref-var (df index return-type)
418 "Returns the data in a single variable as type.
419 type = sequence, vector, vector-like (if valid numeric type) or dataframe."
420 (ecase return-type
421 (('list)
422 (map 'list
423 #'(lambda (x) (dfref df index x))
424 (gen-seq (nth 2 (array-dimensions (dataset df))))))
425 (('vector) t)
426 (:vector-like t)
427 (:matrix-like t)
428 (:dataframe t)))
430 (defun dfref-case (df index return-type)
431 "Returns row as sequence."
432 (ecase return-type
433 (:list
434 (map 'list
435 #'(lambda (x) (dfref df x index))
436 (gen-seq (nth 1 (array-dimensions (dataset df))))))
437 (:vector t)
438 (:vector-like t)
439 (:matrix-like t)
440 (:dataframe t)))
442 ;; FIXME
443 (defun dfref-2indexlist (df indexlist1 indexlist2 &key (return-type :array))
444 "return an array, row X col dims. FIXME TESTME"
445 (case return-type
446 (:array
447 (let ((my-pre-array (list)))
448 (dolist (x indexlist1)
449 (dolist (y indexlist2)
450 (append my-pre-array (dfref df x y))))
451 (make-array (list (length indexlist1)
452 (length indexlist2))
453 :initial-contents my-pre-array)))
454 (:dataframe
455 (make-instance 'dataframe-array
456 :storage (make-array
457 (list (length indexlist1)
458 (length indexlist2))
459 :initial-contents (dataset df))
460 ;; ensure copy for this and following
461 :doc (doc-string df)
462 ;; the following 2 need to be subseted based on
463 ;; the values of indexlist1 and indexlist2
464 :case-labels (case-labels df)
465 :var-labels (var-labels df)))))
467 ;;; Do we establish methods for dataframe-like, which specialize to
468 ;;; particular instances of storage?
470 (defmethod print-object ((object dataframe-like) stream)
471 (print-unreadable-object (object stream :type t)
472 (format stream " ~d x ~d" (nrows object) (ncols object))
473 (terpri stream)
474 ;; (format stream "~T ~{~S ~T~}" (var-labels object))
475 (dotimes (j (ncols object)) ; print labels
476 (write-char #\tab stream)
477 (write-char #\tab stream)
478 (format stream "~T~A~T" (nth j (var-labels object))))
479 (dotimes (i (nrows object)) ; print obs row
480 (terpri stream)
481 (format stream "~A:~T" (nth i (case-labels object)))
482 (dotimes (j (ncols object))
483 (write-char #\tab stream) ; (write-char #\space stream)
484 ;; (write (dfref object i j) :stream stream)
485 (format stream "~7,3E" (dfref object i j)) ; if works, need to include a general output mechanism control
486 ))))
489 (defun print-structure-relational (ds)
490 "example of what we want the methods to look like. Should be sort
491 of like a graph of spreadsheets if the storage is a relational
492 structure."
493 (dolist (k (relations ds))
494 (let ((currentRelationSet (getRelation ds k)))
495 (print-as-row (var-labels currentRelationSet))
496 (let ((j -1))
497 (dolist (i (case-labels currentRelationSet))
498 (print-as-row
499 (append (list i)
500 (dfref-obsn (dataset currentRelationSet)
501 (incf j)))))))))
503 (defun testecase (s)
504 (ecase s
505 ((scalar) 1)
506 ((asd asdf) 2)))
508 (testecase 'scalar)
509 (testecase 'asd)
510 (testecase 'asdf)
511 (testecase 'as)