data.lisp

   1 ;;; -*- mode: lisp -*-
   2 ;;; Copyright (c) 2005--2007, by A.J. Rossini <blindglobe@gmail.com>
   3 ;;; See COPYRIGHT file for any additional restrictions (BSD license).
   4 ;;; Since 1991, ANSI was finally finished.  Edited for ANSI Common Lisp.
   5
   6 ;;; File:       data.lisp
   7 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   8 ;;; Copyright:  (c)2007, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
   9 ;;;             on how it arrives.
  10 ;;; Purpose:    data package for lispstat
  11 ;;; Time-stamp: <2006-05-19 12:33:41 rossini>
  12 ;;; Creation:   <2006-05-17 21:34:07 rossini>
  13
  14 ;;; What is this talk of 'release'? Klingons do not make software
  15 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  16 ;;; designers and quality assurance people in its wake.
  17
  18 ;;; This organization and structure is new to the 21st Century
  19 ;;; version.
  20
  21 ;;; conside    that data has 3 genotypic chracteristrics.  The first
  22 ;;; would be form -- scalar, vector, array.   second would be
  23 ;;; datarep type.  in particular integer, real, string, symbol.  The last
  24 ;;; would be statistical type.  augmenting datarep type with use in a
  25 ;;; statistical context, i.e. that would include nominal, ordinal,
  26 ;;; integer, continous, interval (orderable subtypes).  Clearly, the
  27 ;;; statistical type can be inherited, likewise the numerical type as
  28 ;;; well.  The form can be pushed up or simplified as necessary, but
  29 ;;; this can be challenging.
  30
  31 (in-package :cl-user)
  32
  33 (defpackage :lisp-stat-data
  34   (:documentation "Data I/O, management, other data technologies.")
  35   (:nicknames :ls-data)
  36   (:use :common-lisp
  37         ;;:cxml
  38         :lisp-stat-config
  39         :lisp-stat-object-system
  40         :lisp-stat-types
  41         :lisp-stat-compound-data
  42         :lisp-stat-matrix
  43         :lisp-stat-linalg)
  44   (:shadowing-import-from :lisp-stat-object-system
  45                           slot-value call-method call-next-method)
  46   (:export open-file-dialog read-data-file read-data-columns load-data
  47            load-example *variables* *ask-on-redefine*
  48            def variables savevar undef))
  49
  50 (in-package :lisp-stat-data)
  51
  52 ;;; The purpose of this package is to manage data which will be
  53 ;;; processed by LispStat.  In particular, it willbe importnat to
  54 ;;; register variables, datasets, relational structures, and other
  55 ;;; objects which could be the target for statistical modeling and
  56 ;;; inference.
  57
  58 (defvar *lisp-stat-data-table* (make-hash-table)
  59   "Marks up the data the could be used by.")
  60
  61 (defvar *lisp-stat-data-count* 0
  62   "number of items currently recorded.")
  63
  64
  65 ;;; Data (storage) Types, dt-{.*}
  66 ;;;
  67 ;;; Data types are the representation of data from a computer-science
  68 ;;; perspective, i.e. what it is that they contain.  These types
  69 ;;; include particular forms of compound types (i.e. dataframe is
  70 ;;; array-like, but types differ, difference is row-wise, while array
  71 ;;; is a compound of elements of the same type.
  72 ;;;
  73
  74 ;;Examples:
  75 ;;  (defun equidimensional (a)
  76 ;;    (or (< (array-rank a) 2)
  77 ;;        (apply #'= (array-dimensions a)))) =>  EQUIDIMENSIONAL
  78 ;;  (deftype square-matrix (&optional type size)
  79 ;;    `(and (array ,type (,size ,size))
  80 ;;          (satisfies equidimensional))) =>  SQUARE-MATRIX
  81
  82 (defun array-of-equal-dt-scalar-type (x)
  83   ;; return dt-scalar-type which fits (more precise that works)
  84   (if x
  85       'integer
  86       nil))
  87
  88 (defun array-of-equal-dt-scalar-type-within-column (x)
  89   ;; return dt-scalar-type which fits (more precise that works)
  90   (if x
  91       'integer
  92       nil))
  93
  94
  95
  96 (deftype dt-scalar (&optional type)
  97   `(or integer double complex symbol))
  98
  99 (deftype dt-array (&optional ndim dimlist)
 100   `(satisfies array-of-equal-dt-scalar-type))
 101
 102 (deftype dt-dataframe (&optional )
 103   `(satisfies array-of-equal-dt-scalar-type-within-column))
 104
 105 ;(deftype dt-relationaldata ()
 106 ;  `(satisfies (foreach unit in relationalUnit
 107 ;              (typep unit 'dt-dataframe))))
 108
 109
 110 ;;; Statistical Variable Types, sv-{.*}
 111 ;;;
 112 ;;; Statistical variable types work to represent the statistical
 113 ;;; category represented by the variable, i.e. nominal, ordinal,
 114 ;;; integral, continous, ratio.   This metadata can be used to hint at
 115 ;;; appropriate analysis methods -- or perhaps more critically, to
 116 ;;; define how these methods will fail in the final interrpretation.
 117
 118 (deftype sv-nominal (&optional n)
 119   ())
 120
 121 (deftype sv-ordinal (ordering &optional n)
 122   ())
 123
 124 (deftype sv-categorical ()
 125   `(satisfies (or sv-nominal sv-ordinal)))
 126 ;;(deftype sv-integer )
 127 ;;(deftype sv-real )  ;; precision could be a secondary component of real, rational, complex.
 128 ;;(deftype sv-rational )
 129 ;;(deftype sv-complex )
 130 ;;(deftype sv-continuous (or 'sv-integer 'sv-real 'sv-rational 'sv-complex)) ;; perhaps, call it "mostly contin..."
 131
 132
 133 ;;; Data I/O
 134
 135 ;; We can read 2 types of data -- those which are pure data, and those
 136 ;; which are impure (lisp-enabled, data as program as data thingy's).
 137
 138 (defparameter *lisp-stat-data-formats*
 139   '(csv tsv))
 140
 141 ;; (defgeneric data-read (srce frmt)
 142 ;;   "read data from stream srce, in format frmt.")
 143
 144 ;; (defgeneric data-write (srce frmt)
 145 ;;   "read data from stream srce, in format frmt.")
 146
 147 ;; (defmacro with-data (body)
 148 ;;   "Stream-handling, maintaining I/O through object typing.")
 149
 150 ;; design-wise should these be replaced with a "with-data" form?
 151
 152 ;; DSV processing
 153
 154 ;; XML processing
 155
 156 ;;; Data Management
 157
 158 ;; the goal is to have 2 operations which can be used to create new
 159 ;; data formats out of old ones.
 160
 161 ;; (defgeneric data-subset (ds description)
 162 ;;   "Take a dataset and make it smaller.")
 163
 164 ;; (defgeneric data-relate (ds description)
 165 ;;   "Take 2 or more datasets, and grow them into a bigger one through
 166 ;; relating them (i.e. merge is one example).")
 167
 168 ;;; Data tools from "statistics.lsp"
 169
 170 ;;;;
 171 ;;;; Data File Reading
 172 ;;;;
 173
 174 (defun count-file-columns (fname)
 175 "Args: (fname)
 176 Returns the number of lisp items on the first nonblank line of file FNAME."
 177   (with-open-file (f fname)
 178     (if f
 179         (let ((line (do ((line (read-line f) (read-line f)))
 180                         ((or (null line) (< 0 (length line))) line))))
 181           (if line
 182               (with-input-from-string (s line)
 183                 (do ((n 0 (+ n 1)) (eof (gensym)))
 184                     ((eq eof (read s nil eof)) n))))))))
 185
 186 #+xlisp (defvar *xlisptable* *readtable*)
 187
 188 (if (not (fboundp 'open-file-dialog))
 189   #+dialogs
 190   (defun open-file-dialog () ;; why?(&optional set)
 191     (get-string-dialog "Enter a data file name:"))
 192   #-dialogs
 193   (defun open-file-dialog () ;; why? (&optional set)
 194     (error "You must provide a file name explicitly")))
 195
 196 (defun read-data-file (&optional (file (open-file-dialog t)))
 197 "Args:  (file)
 198 Returns a list of all lisp objects in FILE. FILE can be a string or a symbol,
 199 in which case the symbol'f print name is used."
 200   (if file
 201       (let ((eof (gensym)))
 202         (with-open-file (f file)
 203           (if f
 204               (do* ((r (read f nil eof) (read f nil eof))
 205                     (x (list nil))
 206                     (tail x (cdr tail)))
 207                    ((eq r eof) (cdr x))
 208                    (setf (cdr tail) (list r))))))))
 209
 210 ;;; New definition to avoid stack size limit in apply
 211 (defun read-data-columns (&optional (file (open-file-dialog t))
 212                                     (cols (if file
 213                                               (count-file-columns file))))
 214 "Args: (&optional file cols)
 215 Reads the data in FILE as COLS columns and returns a list of lists representing the columns."
 216   (if (and file cols)
 217       (transpose (split-list (read-data-file file) cols))))
 218
 219
 220 ;;; FIXME:AJR:  ALL THE FOLLOWING NEED TO BE SOLVED BY PLATFORM-INDEP PATHNAME WORK!
 221 ;;; FIXME:AJR: use either string or pathname.
 222
 223 (defun path-string-to-path (p s)
 224     (pathname (concatenate 'string (namestring p) s)))
 225
 226 (defun load-data (file)
 227 "Args: (file) as string
 228 Read in data file from the data examples library."
 229   (if (load (path-string-to-path *lispstat-data-dir* file))
 230       t
 231       (load (path-string-to-path *lispstat-examples-dir* file))))
 232
 233 (defun load-example (file)
 234   "Args: (file) as string
 235 Read in lisp example file from the examples library."
 236   (if (load (path-string-to-path *lispstat-examples-dir* file))
 237       t
 238       (load (path-string-to-path *lispstat-data-dir* file))))
 239
 240 ;;;;
 241 ;;;; Listing and Saving Variables and Functions
 242 ;;;;
 243
 244 (defvar *variables* nil)
 245 (defvar *ask-on-redefine* nil)
 246
 247 (defmacro def (symbol value)
 248 "Syntax: (def var form)
 249 VAR is not evaluated and must be a symbol.  Assigns the value of FORM to
 250 VAR and adds VAR to the list *VARIABLES* of def'ed variables. Returns VAR.
 251 If VAR is already bound and the global variable *ASK-ON-REDEFINE*
 252 is not nil then you are asked if you want to redefine the variable."
 253   `(unless (and *ask-on-redefine*
 254                 (boundp ',symbol)
 255                 (not (y-or-n-p "Variable has a value. Redefine?")))
 256     (if (boundp ',symbol)
 257         (setf ,symbol ,value)
 258         (defvar ,symbol ,value))
 259     (pushnew ',symbol *variables*)
 260     ',symbol))
 261
 262 (defun variables-list ()
 263     (mapcar #'intern (sort-data (mapcar #'string *variables*))))
 264
 265 (defun variables ()
 266 "Args:()
 267 Returns a list of the names of all def'ed variables to STREAM"
 268   (if *variables*
 269       (mapcar #'intern (sort-data (mapcar #'string *variables*)))))
 270
 271 (defun savevar (vars file)
 272 "Args: (vars file-name-root)
 273 VARS is a symbol or a list of symbols. FILE-NAME-ROOT is a string (or a symbol
 274 whose print name is used) not endinf in .lsp. The VARS and their current values
 275 are written to the file FILE-NAME-ROOT.lsp in a form suitable for use with the
 276 load command."
 277   (with-open-file (f (concatenate 'string (namestring file) ".lsp")
 278                      :direction :output)
 279     (let ((vars (if (consp vars) vars (list vars))))
 280       (flet ((save-one (x)
 281                (let ((v (symbol-value x)))
 282                  (if (objectp v)
 283                      (format f "(def ~s ~s)~%" x (send v :save))
 284                    (format f "(def ~s '~s)~%" x v)))))
 285             (mapcar #'save-one vars))
 286       vars)))
 287
 288 (defun undef (v)
 289 "Args: (v)
 290 If V is the symbol of a defined variable the variable it is unbound and
 291 removed from the list of defined variables. If V is a list of variable
 292 names each is unbound and removed. Returns V."
 293   (dolist (s (if (listp v) v (list v)))
 294           (when (member s *variables*)
 295                 (setq *variables* (delete s *variables*))
 296                 (makunbound s)))
 297   v)
 298