src/data/import.lisp

   1 ;;; -*- mode: lisp -*-
   2 ;;; Copyright (c) 2008, by A.J. Rossini <blindglobe@gmail.com>
   3 ;;; See COPYRIGHT file for any additional restrictions (BSD license).
   4 ;;; Since 1991, ANSI was finally finished.  Edited for ANSI Common Lisp.
   5
   6 ;;; Time-stamp: <2009-04-20 19:07:46 tony>
   7 ;;; Creation:   <2008-09-03 08:10:00 tony>
   8 ;;; File:       import.lisp
   9 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
  10 ;;; Copyright:  (c)2007, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
  11 ;;;             on how it arrives.
  12 ;;; Purpose:    base structures for importing data into CLS
  13
  14 ;;; What is this talk of 'release'? Klingons do not make software
  15 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  16 ;;; designers and quality assurance people in its wake.
  17
  18 (in-package :cls-dataimport)
  19
  20
  21
  22
  23
  24
  25 ;;; Data I/O
  26
  27 ;; We can read 2 types of data -- those which are non-lisp-native
  28 ;; data, and those which are lisp-native (lisp-enabled, an extension
  29 ;; of lisp-serialized, i.e. data as program as data thingy's).
  30
  31 ;; of the non-native, there could be raw sources (ascii file formats),
  32 ;; xml sources (xml -> lisp, possible with some preprocessing.
  33
  34 ;;; Reading from DSV files:
  35
  36 ;;; consider either the cybertyggyr-dsv package, or the rsm.string
  37 ;;; package.  The latter seems to actually work a bit at what we need
  38 ;;; to acccomplish, but the former is a challenge to get right when we
  39 ;;; need to think abut what it is that we need to get done.  The
  40 ;;; latter is also better licensed. i.e. BSD-style.
  41
  42
  43 (defparameter *lisp-stat-data-external-source-formats*
  44   '(csv tsv xml ;; ex of text-based (UTF, ASCII, or similar) formats
  45     sql ;; ex of RDBMS call
  46     fcs affy))  ;; ex of binary formats
  47
  48 (defparameter *lisp-stat-data-import-referencing-type*
  49   '(lisp-data-structure reference lisp-function))
  50
  51 (defgeneric data-import (source source-format referencing-type)
  52   (:documentation "read data from stream srce, in format srce-frmt;
  53                    return a reftype, which could be a
  54                    lisp-data-structure, a reference to such, or a lisp
  55                    function which can be evaluated to generate
  56                    either."))
  57
  58 (defgeneric data-export (data target-format target-referencing-type)
  59   (:documentation "write data from stream srce, in format srce-frmt;
  60                    return a reftype, which could be a
  61                    lisp-data-structure, a reference to such, or a lisp
  62                    function which can be evaluated to generate
  63                    either."))
  64
  65
  66
  67
  68 ;;; Potentially useful functions
  69
  70 ;; the following belongs here if we are working externally, but might
  71 ;; belong with data if we are working internlly
  72
  73 ;; (defmacro with-data (body)
  74 ;;   "Stream-handling, maintaining I/O through object typing.")
  75
  76
  77 ;;;
  78 ;;; Related to data file reading
  79 ;;;
  80
  81 (defun count-file-columns (fname)
  82 "Args: (fname)
  83 Returns the number of lisp items on the first nonblank line of file FNAME."
  84   (with-open-file (f fname)
  85     (if f
  86         (let ((line (do ((line (read-line f) (read-line f)))
  87                         ((or (null line) (< 0 (length line))) line))))
  88           (if line
  89               (with-input-from-string (s line)
  90                 (do ((n 0 (+ n 1)) (eof (gensym)))
  91                     ((eq eof (read s nil eof)) n))))))))
  92
  93 (if (not (fboundp 'open-file-dialog))
  94   #+dialogs
  95   (defun open-file-dialog () ;; why?(&optional set)
  96     (get-string-dialog "Enter a data file name:"))
  97   #-dialogs
  98   (defun open-file-dialog () ;; why? (&optional set)
  99     (error "You must provide a file name explicitly")))
 100
 101 (defun read-data-file (&optional (file (open-file-dialog t)))
 102 "Args:  (file)
 103 Returns a list of all lisp objects in FILE. FILE can be a string or a symbol,
 104 in which case the symbol'f print name is used."
 105   (if file
 106       (let ((eof (gensym)))
 107         (with-open-file (f file)
 108           (if f
 109               (do* ((r (read f nil eof) (read f nil eof))
 110                     (x (list nil))
 111                     (tail x (cdr tail)))
 112                    ((eq r eof) (cdr x))
 113                    (setf (cdr tail) (list r))))))))
 114
 115 ;;; New definition to avoid stack size limit in apply
 116 (defun read-data-columns (&optional (file (open-file-dialog t))
 117                                     (cols (if file
 118                                               (count-file-columns file))))
 119 "Args: (&optional file cols)
 120 Reads the data in FILE as COLS columns and returns a list of lists representing the columns."
 121   (if (and file cols)
 122       (transpose (split-list (read-data-file file) cols))))
 123
 124
 125 ;;; FIXME:AJR:  ALL THE FOLLOWING NEED TO BE SOLVED BY PLATFORM-INDEP PATHNAME WORK!
 126 ;;; FIXME:AJR: use either string or pathname.
 127
 128 (defun path-string-to-path (p s)
 129     (pathname (concatenate 'string (namestring p) s)))
 130
 131 (defun load-data (file)
 132 "Args: (file) as string
 133 Read in data file from the data examples library."
 134   (if (load (path-string-to-path *lispstat-data-dir* file))
 135       t
 136       (load (path-string-to-path *lispstat-data-dir* file))))
 137
 138 (defun load-example (file)
 139   "Args: (file) as string
 140 Read in lisp example file from the examples library."
 141   (if (load (path-string-to-path *lispstat-examples-dir* file))
 142       t
 143       (load (path-string-to-path *lispstat-examples-dir* file))))
 144
 145 ;;;
 146 ;;; Saving Variables and Functions
 147 ;;;
 148
 149 (defun savevar (vars file)
 150 "Args: (vars file-name-root)
 151 VARS is a symbol or a list of symbols. FILE-NAME-ROOT is a string (or a symbol
 152 whose print name is used) not endinf in .lsp. The VARS and their current values
 153 are written to the file FILE-NAME-ROOT.lsp in a form suitable for use with the
 154 load command."
 155   (with-open-file (f (concatenate 'string (namestring file) ".lsp")
 156                      :direction :output)
 157     (let ((vars (if (consp vars) vars (list vars))))
 158       (flet ((save-one (x)
 159                (let ((v (symbol-value x)))
 160                  (if (objectp v)
 161                      (format f "(def ~s ~s)~%" x (send v :save))
 162                    (format f "(def ~s '~s)~%" x v)))))
 163             (mapcar #'save-one vars))
 164       vars)))
 165
 166
 167
 168 ;;; General modification approaches.
 169
 170 (defgeneric importData (source featureList)
 171   (:documentation "command to get data into CLS.  Specific methods
 172   will need to handle pathnames, internal data structures, and
 173   external services such as DBMS's.  We would like to be able to do
 174   thinks like:
 175      (importData MyPathName '(:formattype 'csvString))
 176      (importData '(sqlConnection :server host.domain.net :port 666)
 177                  '(:formattype 'table
 178   and so on."))
 179
 180
 181 (defun pathname-example (name)
 182   (let ((my-path (parse-namestring name)))
 183     (values (pathname-name my-path :case :common)
 184             (pathname-name my-path :case :local))))
 185
 186 (defvar sourceTypes (list 'csv 'lisp 'tsv 'special)
 187   "list of possible symbols.
 188
 189 Thsees are used to specify source formats that might be supported for
 190 input.  CSV and TSV are standard, LISP refers to forms, and SPECIAL
 191 refers to a FUNCTION which parses as appropriately.")
 192
 193 ;;; WRONG LOGIC.
 194 (defmethod importData ((fileHandle pathname)
 195                        (fmt list)) ;sourceTypes))
 196   "File-based input for data.
 197 Usually used by:
 198  (importData (parse-namestring 'path/to/file')
 199              (list :format 'csv))
 200
 201  (importData myPathName (list :format 'lisp))
 202 ."
 203   (let* ((fmtType (getf fmt :format))
 204          (newData (getDataAsLists fileHandle fmtType)))
 205     (case fmtType
 206       ('csv (  ))
 207       ('tsv (  ))
 208       ('lisp ( ))
 209       ('special (let ((parserFcn (getf fmt :special-parser)))))
 210       (:default (error "no standard default importData format")))))
 211
 212 (defmethod importData ((ds array) (fmt list))
 213   "mapping arrays into CLS data.")
 214
 215 (defmethod importData ((dsSpec DBMSandSQLextract)
 216                        (fmt mappingTypes))
 217   "mapping DBMS into CLS data.")
 218
 219
 220
 221 ;;(defmacro with-dataframe (env &rest progn)
 222 ;;  "Compute using variable names with with.data.frame type semantics.")
 223
 224