src/data/import.lisp

   1 ;;; -*- mode: lisp -*-
   2 ;;; Copyright (c) 2008, by A.J. Rossini <blindglobe@gmail.com>
   3 ;;; See COPYRIGHT file for any additional restrictions (BSD license).
   4 ;;; Since 1991, ANSI was finally finished.  Edited for ANSI Common Lisp.
   5
   6 ;;; Time-stamp: <2008-09-03 08:37:34 tony>
   7 ;;; Creation:   <2008-09-03 08:10:00 tony>
   8 ;;; File:       import.lisp
   9 ;;; Author:     AJ Rossini <blindglobe@gmail.com>
  10 ;;; Copyright:  (c)2007, AJ Rossini.  BSD, LLGPL, or GPLv2, depending
  11 ;;;             on how it arrives.
  12 ;;; Purpose:    base structures for importing data into CLS
  13
  14 ;;; What is this talk of 'release'? Klingons do not make software
  15 ;;; 'releases'.  Our software 'escapes', leaving a bloody trail of
  16 ;;; designers and quality assurance people in its wake.
  17
  18
  19 (in-package :cl-user)
  20
  21 (defpackage :lisp-stat-data-import
  22   (:documentation "Data I/O and similar import technologies.")
  23   (:nicknames :ls-data-import)
  24   (:use :common-lisp
  25         :lisp-stat-object-system
  26         :lisp-stat-data)
  27   (:shadowing-import-from :lisp-stat-object-system
  28                           slot-value call-method call-next-method)
  29   (:export data-import data-export))
  30
  31 (in-package :lisp-stat-data-import)
  32
  33 ;;; The purpose of this package is to provide the basic structures for
  34 ;;; importing data, to be further processed by LispStat.
  35
  36
  37
  38 ;;; Data I/O
  39
  40 ;; We can read 2 types of data -- those which are non-lisp-native
  41 ;; data, and those which are lisp-native (lisp-enabled, an extension
  42 ;; of lisp-serialized, i.e. data as program as data thingy's).
  43
  44 ;; of the non-native, there could be raw sources (ascii file formats),
  45 ;; xml sources (xml -> lisp, possible with some preprocessing.
  46
  47 (defparameter *lisp-stat-data-external-source-formats*
  48   '(csv tsv xml ;; ex of text-based (UTF, ASCII, or similar) formats
  49     sql ;; ex of RDBMS call
  50     fcs affy))  ;; ex of binary formats
  51
  52 (defparameter *lisp-stat-data-import-referencing-type*
  53   '(lisp-data-structure reference lisp-function))
  54
  55
  56
  57 (defgeneric data-import (source source-format referencing-type)
  58   (:documentation "read data from stream srce, in format srce-frmt;
  59                    return a reftype, which could be a
  60                    lisp-data-structure, a reference to such, or a lisp
  61                    function which can be evaluated to generate
  62                    either."))
  63
  64 (defgeneric data-export (data target-format target-referencing-type)
  65   (:documentation "write data from stream srce, in format srce-frmt;
  66                    return a reftype, which could be a
  67                    lisp-data-structure, a reference to such, or a lisp
  68                    function which can be evaluated to generate
  69                    either."))
  70
  71
  72
  73
  74 ;;; Potentially useful functions
  75
  76 ;; the following belongs here if we are working externally, but might
  77 ;; belong with data if we are working internlly
  78
  79 ;; (defmacro with-data (body)
  80 ;;   "Stream-handling, maintaining I/O through object typing.")
  81
  82
  83 ;;;
  84 ;;; Related to data file reading
  85 ;;;
  86
  87 (defun count-file-columns (fname)
  88 "Args: (fname)
  89 Returns the number of lisp items on the first nonblank line of file FNAME."
  90   (with-open-file (f fname)
  91     (if f
  92         (let ((line (do ((line (read-line f) (read-line f)))
  93                         ((or (null line) (< 0 (length line))) line))))
  94           (if line
  95               (with-input-from-string (s line)
  96                 (do ((n 0 (+ n 1)) (eof (gensym)))
  97                     ((eq eof (read s nil eof)) n))))))))
  98
  99 (if (not (fboundp 'open-file-dialog))
 100   #+dialogs
 101   (defun open-file-dialog () ;; why?(&optional set)
 102     (get-string-dialog "Enter a data file name:"))
 103   #-dialogs
 104   (defun open-file-dialog () ;; why? (&optional set)
 105     (error "You must provide a file name explicitly")))
 106
 107 (defun read-data-file (&optional (file (open-file-dialog t)))
 108 "Args:  (file)
 109 Returns a list of all lisp objects in FILE. FILE can be a string or a symbol,
 110 in which case the symbol'f print name is used."
 111   (if file
 112       (let ((eof (gensym)))
 113         (with-open-file (f file)
 114           (if f
 115               (do* ((r (read f nil eof) (read f nil eof))
 116                     (x (list nil))
 117                     (tail x (cdr tail)))
 118                    ((eq r eof) (cdr x))
 119                    (setf (cdr tail) (list r))))))))
 120
 121 ;;; New definition to avoid stack size limit in apply
 122 (defun read-data-columns (&optional (file (open-file-dialog t))
 123                                     (cols (if file
 124                                               (count-file-columns file))))
 125 "Args: (&optional file cols)
 126 Reads the data in FILE as COLS columns and returns a list of lists representing the columns."
 127   (if (and file cols)
 128       (transpose (split-list (read-data-file file) cols))))
 129
 130
 131 ;;; FIXME:AJR:  ALL THE FOLLOWING NEED TO BE SOLVED BY PLATFORM-INDEP PATHNAME WORK!
 132 ;;; FIXME:AJR: use either string or pathname.
 133
 134 (defun path-string-to-path (p s)
 135     (pathname (concatenate 'string (namestring p) s)))
 136
 137 (defun load-data (file)
 138 "Args: (file) as string
 139 Read in data file from the data examples library."
 140   (if (load (path-string-to-path *lispstat-data-dir* file))
 141       t
 142       (load (path-string-to-path *lispstat-examples-dir* file))))
 143
 144 (defun load-example (file)
 145   "Args: (file) as string
 146 Read in lisp example file from the examples library."
 147   (if (load (path-string-to-path *lispstat-examples-dir* file))
 148       t
 149       (load (path-string-to-path *lispstat-data-dir* file))))
 150
 151 ;;;
 152 ;;; Saving Variables and Functions
 153 ;;;
 154
 155 (defun savevar (vars file)
 156 "Args: (vars file-name-root)
 157 VARS is a symbol or a list of symbols. FILE-NAME-ROOT is a string (or a symbol
 158 whose print name is used) not endinf in .lsp. The VARS and their current values
 159 are written to the file FILE-NAME-ROOT.lsp in a form suitable for use with the
 160 load command."
 161   (with-open-file (f (concatenate 'string (namestring file) ".lsp")
 162                      :direction :output)
 163     (let ((vars (if (consp vars) vars (list vars))))
 164       (flet ((save-one (x)
 165                (let ((v (symbol-value x)))
 166                  (if (objectp v)
 167                      (format f "(def ~s ~s)~%" x (send v :save))
 168                    (format f "(def ~s '~s)~%" x v)))))
 169             (mapcar #'save-one vars))
 170       vars)))
 171
 172