3 ;;; Time-stamp: <2009-12-21 12:57:29 tony>
4 ;;; Creation: <2008-03-12 17:18:42 blindglobe@gmail.com>
5 ;;; File: dataframe.lisp
6 ;;; Author: AJ Rossini <blindglobe@gmail.com>
7 ;;; Copyright: (c)2008, AJ Rossini. BSD, LLGPL, or GPLv2, depending
10 ;;; Purpose: Data packaging and access for Common Lisp Statistics.
11 ;;; This redoes dataframe structures in a CLOS based
12 ;;; framework. Currently contains the virtual class
13 ;;; DATAFRAME-LIKE as well as the actual classes
14 ;;; DATAFRAME-ARRAY and DATAFRAME-MATRIXLIKE
16 ;;; What is this talk of 'release'? Klingons do not make software
17 ;;; 'releases'. Our software 'escapes', leaving a bloody trail of
18 ;;; designers and quality assurance people in its wake.
20 (in-package :cls-dataframe
)
22 ;;; No real basis for work, there is a bit of new-ness and R-ness to
23 ;;; this work. In particular, the notion of relation is key and
24 ;;; integral to the analysis. Tables are related and matched vectors,
25 ;;; for example. "column" vectors are related observations (by
26 ;;; measure/recording) while "row" vectors are related readings (by
27 ;;; case, independence). This does mean that we are placing
28 ;;; statistical semantics into the computational data object -- and
29 ;;; that it is a violation of use to consider rows which are not at
30 ;;; the least conditionally independent (though the conditioning
31 ;;; should be outside the data set, not internally specified).
33 ;;; So we want a verb-driven API for data collection construction. We
34 ;;; should encode independence or lack of, as a computable status.
36 ;;; Need to figure out statistically-typed vectors. We then map a
37 ;;; series of typed vectors over to tables where columns are equal
38 ;;; typed. In a sense, this is a relation (1-1) of equal-typed
39 ;;; arrays. For the most part, this ends up making the R data.frame
40 ;;; into a relational building block (considering 1-1 mappings using
41 ;;; row ID as a relation). Is this a worthwhile generalization or
42 ;;; communicable analogy?
44 ;;; verbs vs semantics for DF construction -- consider the possibily
45 ;;; of how adverbs and verbs relate, where to put which semantically
46 ;;; to allow for general approach.
48 ;;; Need to consider modification APIs
51 ;;; - get/set row names (case names)
52 ;;; - column names (variable names)
54 ;;; - annotation/metadata
55 ;;; - make sure that we do coherency checking in the exported
58 ;;; - reshapeData/reformat/reshapr a reformed version of the dataset (no
59 ;;; additional input).
60 ;;; - either overwriting or not, i.e. with or without copy.
61 ;;; - check consistency of resulting data with metadata and related
64 ;;; Is there any need for an N-way dataframe (N>2) ? Am currently
65 ;;; assuming not, that this is specializing only the
66 ;;; "independent cases"-by-variables format and that there would be
67 ;;; other tools for other structures.
69 ;;; Misc Functions (to move into a lisp data manipulation support package)
71 ;; the next two should be merged into a general replicator pattern.
72 (defun gen-seq (n &optional
(start 1))
73 "Generates an integer sequence of length N starting at START. Used
76 (append (gen-seq (- n
1) start
) (list n
))))
78 (defun repeat-seq (n item
)
79 "FIXME: There has to be a better way -- I'm sure of it!
80 (repeat-seq 3 \"d\") ; => (\"d\" \"d\" \"d\")
81 (repeat-seq 3 'd) ; => ('d 'd 'd)
82 (repeat-seq 3 (list 1 2))"
84 (append (repeat-seq (1- n
) item
) (list item
))))
86 (defun strsym->indexnum
(df strsym
)
87 "Returns a number indicating the DF column labelled by STRSYM.
88 Probably should be generic/methods dispatching on DATAFRAME-LIKE type."
89 (position strsym
(varlabels df
)))
91 (defun string->number
(str)
92 "Convert a string <str> representing a number to a number. A second
93 value is returned indicating the success of the conversion. Examples:
94 (string->number \"123\") ; => 123 t
95 (string->number \"1.23\") ; => 1.23 t"
96 (let ((*read-eval
* nil
))
97 (let ((num (read-from-string str
)))
98 (values num
(numberp num
)))))
101 (equal 'testme
'testme
)
102 (defparameter *test-pos
* 'testme
)
103 (position *test-pos
* (list 'a
'b
'testme
'c
))
104 (position #'(lambda (x) (equal x
"testme")) (list "a" "b" "testme" "c"))
105 (position #'(lambda (x) (equal x
1)) (list 2 1 3 4))
108 ;;; abstract dataframe class
110 (defclass dataframe-like
(matrix-like)
111 ((store :initform nil
113 :documentation
"not useful in the -like virtual class case.")
114 (case-labels :initform nil
115 :initarg
:case-labels
117 :accessor case-labels
118 :documentation
"labels used for describing cases (doc
119 metadata), possibly used for merging.")
120 (var-labels :initform nil
124 :documentation
"Variable names.")
125 (var-types :initform nil
129 :documentation
"variable types to ensure fit. Must be
130 list of symbols of valid types for check-type.")
131 (doc-string :initform nil
134 :documentation
"additional information, potentially
135 uncomputable, possibly metadata, about dataframe-like
137 (:documentation
"Abstract class for standard statistical analysis
138 dataset for independent data. Rows are considered to be
139 independent, matching observations. Columns are considered to be
140 type-consistent, match a variable with distribution. inherits
141 from lisp-matrix base MATRIX-LIKE class. MATRIX-LIKE (from
142 lisp-matrix) is basically a rectangular table without storage.
143 We emulate that, and add storage, row/column labels, and
144 within-column-typing.
146 DATAFRAME-LIKE is the basic cases by variables framework. Need
147 to embed this within other structures which allow for generalized
148 relations. Goal is to ensure that relations imply and drive the
149 potential for statistical relativeness such as correlation,
150 interference, and similar concepts.
152 STORE is the storage component. We ignore this in the
153 DATAFRAME-LIKE class, as it is the primary differentiator,
154 spec'ing the structure used for storing the actual data. We
155 create methods which depend on STORE for access. The only
156 critical component is that STORE be a class which is
157 xarray-compliant. Examples of such mixins are DATAFRAME-ARRAY
158 and DATAFRAME-MATRIXLIKE. The rest of this structure is
161 ;;; Specializing on superclasses...
163 ;;; Access and Extraction: implementations needed for any storage
164 ;;; type. But here, just to point out that we've got a specializing
165 ;;; virtual subclass (DATAFRAME-LIKE specializing MATRIX-LIKE).
167 (defgeneric nvars
(df)
168 (:documentation
"number of variables represented in storage type.")
169 (:method
((df dataframe-like
))
170 (xdim (store df
) 0)))
172 (defgeneric ncases
(df)
173 (:documentation
"number of cases (indep observantions) represented by storage.")
174 (:method
((df dataframe-like
))
175 (xdim (store df
) 1)))
177 ;; Testing consistency/coherency.
179 (defgeneric consistent-dataframe-p
(df)
180 (:documentation
"methods to check for consistency. Mostly of
181 internal interest, since ideally we'd have to use standard
182 constructs to ensure that we do not get the dataframe structure
184 (:method
(object) "General objects are not consistent dataframes!" nil
)
185 (:method
((df dataframe-like
))
186 "At minimum, must dispatch on virtual-class."
188 ;; ensure dimensionality
189 (= (length (var-labels df
)) (ncols df
)) ; array-dimensions (dataset df))
190 (= (length (case-labels df
)) (nrows df
))
191 ;; when dims are sane, ensure variable-typing is consistent
193 (dotimes (i (nrows df
))
194 (dotimes (j (ncols df
))
195 ;; xref bombs if not a df-like subclass so we don't worry
196 ;; about specialization.
197 ;; (check-type (aref dt i j) (elt lot j)))))) ???
198 (typep (xref df i j
) (nth j
(var-types df
)))))
202 ;;; FUNCTIONS WHICH DISPATCH ON INTERNAL METHODS OR ARGS
204 ;;; Q: change the following to generic functions and dispatch on
205 ;;; array, matrix, and dataframe? Others?
206 (defun make-labels (initstr num
)
207 "generate a list of strings which can be used as labels, i.e. something like
208 (make-labels \"a\" 3) => '(\"a1\" \"a2\" \"a3\")."
209 (check-type initstr string
)
210 (mapcar #'(lambda (x y
) (concatenate 'string x y
))
211 (repeat-seq num initstr
)
212 (mapcar #'(lambda (x) (format nil
"~A" x
)) (gen-seq num
))))
214 (defun ncase-store (store)
215 "Return number of cases (rows) in dataframe storage. Doesn't test
216 that that list is a valid listoflist dataframe structure."
218 (array (array-dimension store
0))
219 (matrix-like (nrows store
))
220 (list (length store
))))
222 (defun nvars-store (store)
223 "Return number of variables (columns) in dataframe storage. Doesn't
224 test that that list is a valid listoflist dataframe structure."
226 (array (array-dimension store
1))
227 (matrix-like (ncols store
))
228 (list (length (elt store
0)))))
231 (defun make-dataframe (newdata
233 (caselabels nil
) (varlabels nil
)
235 "Helper function to use instead of make-instance to assure
236 construction of proper DF-array."
237 (check-type newdata
(or matrix-like array list
))
238 (check-type caselabels sequence
)
239 (check-type varlabels sequence
)
240 (check-type doc string
)
241 (let ((ncases (ncase-store newdata
))
242 (nvars (nvars-store newdata
)))
243 (if caselabels
(assert (= ncases
(length caselabels
))))
244 (if varlabels
(assert (= nvars
(length varlabels
))))
245 (let ((newcaselabels (if caselabels
247 (make-labels "C" ncases
)))
248 (newvarlabels (if varlabels
250 (make-labels "V" nvars
))))
253 (make-instance 'dataframe-listoflist
255 :nrows
(length newcaselabels
)
256 :ncols
(length newvarlabels
)
257 :case-labels newcaselabels
258 :var-labels newvarlabels
259 :var-types vartypes
))
261 (make-instance 'dataframe-array
263 :nrows
(length newcaselabels
)
264 :ncols
(length newvarlabels
)
265 :case-labels newcaselabels
266 :var-labels newvarlabels
267 :var-types vartypes
))
269 (make-instance 'dataframe-matrixlike
271 :nrows
(length newcaselabels
)
272 :ncols
(length newvarlabels
)
273 :case-labels newcaselabels
274 :var-labels newvarlabels
275 :var-types vartypes
))
280 (make-dataframe #2A
((1.2d0
1.3d0
) (2.0d0
4.0d0
)))
281 (make-dataframe #2A
(('a
1) ('b
2)))
282 (xref (make-dataframe #2A
(('a
1) ('b
2))) 0 1)
283 (xref (make-dataframe #2A
(('a
1) ('b
2))) 1 0)
284 (make-dataframe 4) ; ERROR, should we allow?
285 (make-dataframe #2A
((4)))
286 (make-dataframe (rand 10 5)) ;; ERROR, but should work!
290 (defun row-order-as-list (ary)
291 "Pull out data in row order into a list."
292 (let ((result (list))
293 (nrows (nth 0 (array-dimensions ary
)))
294 (ncols (nth 1 (array-dimensions ary
))))
297 (append result
(aref ary i j
))))))
299 (defun col-order-as-list (ary)
300 "Pull out data in row order into a list."
301 (let ((result (list))
302 (nrows (nth 0 (array-dimensions ary
)))
303 (ncols (nth 1 (array-dimensions ary
))))
306 (append result
(aref ary i j
))))))
308 (defun transpose-array (ary)
310 (make-array (reverse (array-dimensions ary
))
311 :initial-contents
(col-order-as-list ary
)))
313 ;;; THE FOLLOWING 2 dual-sets done to provide error checking
314 ;;; possibilities on top of the generic function structure. Not
315 ;;; intended as make-work!
317 (defun varlabels (df)
318 "Variable-name handling for DATAFRAME-LIKE. Needs error checking."
321 (defun set-varlabels (df vl
)
322 "Variable-name handling for DATAFRAME-LIKE. Needs error checking."
323 (if (= (length (var-labels df
))
325 (setf (var-labels df
) vl
)
326 (error "wrong size.")))
328 (defsetf varlabels set-varlabels
)
330 ;;; Case-name handling for Tables. Needs error checking.
331 (defun caselabels (df)
332 "Case-name handling for DATAFRAME-LIKE. Needs error checking."
335 (defun set-caselabels (df cl
)
336 "Case-name handling for DATAFRAME-LIKE. Needs error checking."
337 (if (= (length (case-labels df
))
339 (setf (case-labels df
) cl
)
340 (error "wrong size.")))
342 (defsetf caselabels set-caselabels
)
344 ;;;;;;;;;;;; IMPLEMENTATIONS, with appropriate methods.
346 ;; (documentation 'dataframe-like 'type)
351 ;;; Do we establish methods for dataframe-like, which specialize to
352 ;;; particular instances of storage?
354 (defmethod print-object ((object dataframe-like
) stream
)
355 (print-unreadable-object (object stream
:type t
)
356 (format stream
" ~d x ~d" (nrows object
) (ncols object
))
358 ;; (format stream "~T ~{~S ~T~}" (var-labels object))
359 (dotimes (j (ncols object
)) ; print labels
360 (write-char #\tab stream
)
361 (write-char #\tab stream
)
362 (format stream
"~T~A~T" (nth j
(var-labels object
))))
363 (dotimes (i (nrows object
)) ; print obs row
365 (format stream
"~A:~T" (nth i
(case-labels object
)))
366 (dotimes (j (ncols object
))
367 (write-char #\tab stream
) ; (write-char #\space stream)
368 ;; (write (xref object i j) :stream stream)
369 (format stream
"~7,3E" (xref object i j
)) ; if works, need to include a general output mechanism control
373 (defun print-structure-relational (ds)
374 "example of what we want the methods to look like. Should be sort
375 of like a graph of spreadsheets if the storage is a relational
377 (dolist (k (relations ds
))
378 (let ((currentRelationSet (getRelation ds k
)))
379 (print-as-row (var-labels currentRelationSet
))
381 (dolist (i (case-labels currentRelationSet
))
384 (xref-obsn (dataset currentRelationSet
)
399 ;;; Vector-like generalizations: we consider observation-like and
400 ;;; variable-like to be abstract classes which provide row and column
401 ;;; access to dataframe structures. These will be specialized, in
402 ;;; that rows correspond to an observation (or case?) which are
403 ;;; multitype, while columns correspond to a variable, which must be
404 ;;; singularly typed.
406 (defclass observation-like
(dataframe-like)
408 (:documentation
"dataframe-like with only 1 row, is an observation-like."))
410 (defclass variable-like
(dataframe-like)
412 (:documentation
"dataframe-like with only 1 column is a variable-like."))
414 ;;; Need to implement views, i.e. dataframe-view-like,
415 ;;; observation-view-like, variable-view-like.
419 ;;; Need to consider read-only variants, leveraging the xref
424 ;;; Dataframe <-> Listoflist support
425 ;; the following will be handy to help out folks adjust. It should
426 ;; provide a means to write code faster and better.
428 ;; leverages listoflist support in our version of xarray
429 (defun listoflist->dataframe
(lol) ; &key (type :row-major))
430 "Create a cases-by-variables data frame consisting of numeric data,
431 from a ROW-MAJOR list-of-lists representation. A COLUMN-MAJOR
432 representation should be handled using the transpose-listoflists
434 (check-type lol list
) ; imperfect -- must verify all list elements are also lists.
435 (if (sublists-of-same-size-p lol
)
436 (make-dataframe (listoflist->array lol
))
437 (error "make-data-set-from-lists: no combining different length lists"))
438 (error "make-data-set-from-lists: proposed name exists"))
441 ;;;;;;;; from dataframe-xarray experiment
444 (defmethod xref ((obj dataframe-like
) &rest subscripts
)
445 "For data-frame-like, dispatch on storage object."
446 (xref (dataset obj
) subscripts
))
448 (defmethod (setf xref
) (value (obj dataframe-like
) &rest subscripts
)
449 (setf (xref (dataset obj
) subscripts
) value
))
451 (defmethod xref ((obj matrix-like
) &rest indices
))
453 (defmethod xtype ((obj dataframe-like
))
454 "Unlike the standard xtype, here we need to return a vector of the
455 types. Vectors can have single types, but arrays have single type.
456 Dataframe-like have multiple types, variable-like single type,
457 case-like has multiple types, and matrix-like has single type.")
459 (defmethod xdims ((obj dataframe-like
))
460 (dataframe-dimensions obj
))
462 ;; use default methods at this point, except for potentially weird DFs
463 (defmethod xdims* ())
465 (defmethod xdim ((obj dataframe-like
) index
)
466 (dataframe-dimension index
))
475 (defmethod carray ())
477 (defmacro with-dataframe
(env &rest progn
)
478 "Compute using variable names with with.data.frame type semantics.")
480 (defmacro with-data
(body)
481 "Stream-handling, maintaining I/O through object typing.")