data.lisp

   1 ;;; -*- mode: lisp -*-
   2 ;;; Copyright (c) 2005--2007, by A.J. Rossini <blindglobe@gmail.com>
   3 ;;; See COPYRIGHT file for any additional restrictions (BSD license).
   4 ;;; Since 1991, ANSI was finally finished.  Edited for ANSI Common Lisp.
   5
   6 ;;; File:       data.lisp
   7 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
   8 ;;; Copyright:  (c)2007, AJ Rossini.  BSD, LLGPL, or GPLv2, depending on how it arrives.
   9 ;;; Purpose:    data package for lispstat
  10 ;;; Time-stamp: <2006-05-19 12:33:41 rossini>
  11 ;;; Creation:   <2006-05-17 21:34:07 rossini>
  12
  13 ;;; What is this talk of 'release'? Klingons do not make software
  14 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  15 ;;; designers and quality assurance people in its wake.
  16
  17 ;;; This organization and structure is new to the 21st Century
  18 ;;; version.
  19
  20
  21 (in-package :cl-user)
  22
  23 (defpackage :lisp-stat-data
  24   (:documentation "Data I/O, management, other data technologies.")
  25   (:nicknames :ls-data)
  26   (:use :common-lisp
  27         :cxml
  28         :lisp-stat-object-system
  29         :lisp-stat-compound
  30         :lisp-stat-matrices
  31         :lisp-stat-linalg
  32         :lisp-stat-sequence)
  33   (:shadowing-import-from :lisp-stat-object-system
  34                           slot-value call-method call-next-method)
  35   (:export
  36    ;;; from statistics.lsp
  37           open-file-dialog read-data-file read-data-columns load-data
  38           load-example *variables* *ask-on-redefine* def variables savevar
  39           undef  ))
  40
  41 (in-package :lisp-stat-data)
  42
  43 ;;; The purpose of this package is to manage data which will be
  44 ;;; processed by LispStat.  In particular, it willbe importnat to
  45 ;;; register variables, datasets, relational structures, and other
  46 ;;; objects which could be the target for statistical modeling and
  47 ;;; inference.
  48
  49 (defvar *lisp-stat-data-table* (make-hashtable)
  50   "Marks up the data the could be used by.")
  51
  52 (defvar *lisp-stat-data-count* 0
  53   "number of items currently recorded.")
  54
  55 ;;; Data I/O
  56
  57 ;; We can read 2 types of data -- those which are pure data, and those
  58 ;; which are imprue (lisp-enables).
  59
  60 (defparameter *lisp-stat-data-formats*
  61   '(csv tsv))
  62
  63 (defgeneric data-read (srce frmt)
  64   "read data from stream srce, in format frmt.")
  65
  66 (defgeneric data-write (srce frmt)
  67   "read data from stream srce, in format frmt.")
  68
  69 (defmacro with-data (body)
  70   "Stream-handling, maintaining I/O through object typing.")
  71
  72 ;; design-wise should these be replaced with a "with-data" form?
  73
  74
  75 ;; DSV processing
  76
  77 ;; XML processing
  78
  79 ;;; Data Management
  80
  81 ;; the goal is to have 2 operations which can be used to create new
  82 ;; data formats out of old ones.
  83
  84 (defgeneric data-subset (ds description)
  85   "Take a dataset and make it smaller.")
  86
  87 (defgeneric data-relate (ds description)
  88   "Take 2 or more datasets, and grow them into a bigger one through
  89 relating them (i.e. merge is one example).")
  90
  91 ;;; Data tools from "statistics.lsp"
  92
  93 ;;;;
  94 ;;;; Data File Reading
  95 ;;;;
  96
  97 (defun count-file-columns (fname)
  98 "Args: (fname)
  99 Returns the number of lisp items on the first nonblank line of file FNAME."
 100   (with-open-file (f fname)
 101     (if f
 102         (let ((line (do ((line (read-line f) (read-line f)))
 103                         ((or (null line) (< 0 (length line))) line))))
 104           (if line
 105               (with-input-from-string (s line)
 106                 (do ((n 0 (+ n 1)) (eof (gensym)))
 107                     ((eq eof (read s nil eof)) n))))))))
 108
 109 #+xlisp (defvar *xlisptable* *readtable*)
 110
 111 (if (not (fboundp 'open-file-dialog))
 112   #+dialogs
 113   (defun open-file-dialog (&optional set)
 114     (get-string-dialog "Enter a data file name:"))
 115   #-dialogs
 116   (defun open-file-dialog (&optional set)
 117     (error "You must provide a file name explicitly")))
 118
 119 (defun read-data-file (&optional (file (open-file-dialog t)))
 120 "Args:  (file)
 121 Returns a list of all lisp objects in FILE. FILE can be a string or a symbol,
 122 in which case the symbol'f print name is used."
 123   (if file
 124       (let ((eof (gensym)))
 125         (with-open-file (f file)
 126           (if f
 127               (do* ((r (read f nil eof) (read f nil eof))
 128                     (x (list nil))
 129                     (tail x (cdr tail)))
 130                    ((eq r eof) (cdr x))
 131                    (setf (cdr tail) (list r))))))))
 132
 133 ;;; New definition to avoid stack size limit in apply
 134 (defun read-data-columns (&optional (file (open-file-dialog t))
 135                                     (cols (if file
 136                                               (count-file-columns file))))
 137 "Args: (&optional file cols)
 138 Reads the data in FILE as COLS columns and returns a list of lists representing the columns."
 139   (if (and file cols)
 140       (transpose (split-list (read-data-file file) cols))))
 141
 142
 143 ;;; FIXME:AJR:  ALL THE FOLLOWING ARE SOLVED BY PLATFORM-INDEP PATHNAME WORK!
 144
 145 #+unix
 146 (defun load-data (file)
 147 "Args: (file)
 148 Read in data file from the data examples library."
 149   (if (load (format nil "~aData/~a" *default-path* file))
 150       t
 151       (load (format nil "~aExamples/~a" *default-path* file))))
 152
 153 #+unix
 154 (defun load-example (file)
 155 "Args: (file)
 156 Read in lisp example file from the examples library."
 157   (if (load (format nil "~aExamples/~a" *default-path* file))
 158       t
 159       (load (format nil "~aData/~a" *default-path* file))))
 160 #+macintosh
 161 (defun load-data (s) (require s (concatenate 'string ":Data:" s)))
 162 #+macintosh
 163 (defun load-example (s) (require s (concatenate 'string ":Examples:" s)))
 164
 165 #+msdos
 166 (defun load-data (file)
 167 "Args: (file)
 168 Read in data file from the data examples library."
 169   (load (format nil "~aData\\~a" *default-path* file)))
 170
 171 #+msdos
 172 (defun load-example (file)
 173 "Args: (file)
 174 Read in lisp example file from the examples library."
 175   (load (format nil "~aExamples\\~a" *default-path* file)))
 176
 177 ;;;;
 178 ;;;; Listing and Saving Variables and Functions
 179 ;;;;
 180
 181 (defvar *variables* nil)
 182 (defvar *ask-on-redefine* nil)
 183
 184 (defmacro def (symbol value)
 185 "Syntax: (def var form)
 186 VAR is not evaluated and must be a symbol.  Assigns the value of FORM to
 187 VAR and adds VAR to the list *VARIABLES* of def'ed variables. Returns VAR.
 188 If VAR is already bound and the global variable *ASK-ON-REDEFINE*
 189 is not nil then you are asked if you want to redefine the variable."
 190   `(unless (and *ask-on-redefine*
 191                 (boundp ',symbol)
 192                 (not (y-or-n-p "Variable has a value. Redefine?")))
 193            (pushnew ',symbol *variables*)
 194            (setf ,symbol ,value)
 195            ',symbol))
 196
 197 (defun variables-list ()
 198     (mapcar #'intern (sort-data (mapcar #'string *variables*))))
 199
 200 (defun variables ()
 201 "Args:()
 202 Returns a list of the names of all def'ed variables to STREAM"
 203   (if *variables*
 204       (mapcar #'intern (sort-data (mapcar #'string *variables*)))))
 205
 206 (defun savevar (vars file)
 207 "Args: (vars file-name-root)
 208 VARS is a symbol or a list of symbols. FILE-NAME-ROOT is a string (or a symbol
 209 whose print name is used) not endinf in .lsp. The VARS and their current values
 210 are written to the file FILE-NAME-ROOT.lsp in a form suitable for use with the
 211 load command."
 212   (with-open-file (f (strcat (string file) ".lsp") :direction :output)
 213     (let ((vars (if (consp vars) vars (list vars))))
 214       (flet ((save-one (x)
 215                (let ((v (symbol-value x)))
 216                  (if (objectp v)
 217                      (format f "(def ~s ~s)~%" x (send v :save))
 218                    (format f "(def ~s '~s)~%" x v)))))
 219             (mapcar #'save-one vars))
 220       vars)))
 221
 222 (defun undef (v)
 223 "Args: (v)
 224 If V is the symbol of a defined variable the variable it is unbound and
 225 removed from the list of defined variables. If V is a list of variable
 226 names each is unbound and removed. Returns V."
 227   (dolist (s (if (listp v) v (list v)))
 228           (when (member s *variables*)
 229                 (setq *variables* (delete s *variables*))
 230                 (makunbound s)))
 231   v)
 232