io.streams/gray-stream-methods.lisp

   1 ;;;; -*- Mode: Lisp; Syntax: ANSI-Common-Lisp; Indent-tabs-mode: NIL -*-
   2 ;;;
   3 ;;; gray-stream-methods.lisp --- Implementation using gray streams.
   4 ;;;
   5 ;;; Copyright (C) 2006-2008, Stelian Ionescu  <sionescu@common-lisp.net>
   6 ;;;
   7 ;;; This code is free software; you can redistribute it and/or
   8 ;;; modify it under the terms of the version 2.1 of
   9 ;;; the GNU Lesser General Public License as published by
  10 ;;; the Free Software Foundation, as clarified by the
  11 ;;; preamble found here:
  12 ;;;     http://opensource.franz.com/preamble.html
  13 ;;;
  14 ;;; This program is distributed in the hope that it will be useful,
  15 ;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ;;; GNU General Public License for more details.
  18 ;;;
  19 ;;; You should have received a copy of the GNU Lesser General
  20 ;;; Public License along with this library; if not, write to the
  21 ;;; Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  22 ;;; Boston, MA 02110-1301, USA
  23
  24 (in-package :io.streams)
  25
  26 ;;;; Instance Initialization
  27
  28 (defun free-stream-buffers (ib ob)
  29   (when ib (free-iobuf ib))
  30   (when ob (free-iobuf ob)))
  31
  32 ;;; TODO: use the buffer pool
  33 ;;; TODO: handle instance reinitialization
  34 (defmethod shared-initialize :after ((stream dual-channel-gray-stream) slot-names
  35                                      &key (input-buffer-size +bytes-per-iobuf+)
  36                                      (output-buffer-size +bytes-per-iobuf+)
  37                                      (external-format :default))
  38   (declare (ignore slot-names))
  39   (unless input-buffer-size (setf input-buffer-size +bytes-per-iobuf+))
  40   (unless output-buffer-size (setf output-buffer-size +bytes-per-iobuf+))
  41   (check-type input-buffer-size buffer-index)
  42   (check-type output-buffer-size buffer-index)
  43   (with-accessors ((ib input-buffer-of)
  44                    (ob output-buffer-of)
  45                    (ef external-format-of))
  46       stream
  47     (setf ib (allocate-iobuf input-buffer-size)
  48           ob (allocate-iobuf output-buffer-size)
  49           ef external-format)
  50     (trivial-garbage:finalize stream #'(lambda () (free-stream-buffers ib ob)))))
  51
  52 ;;;; Common Methods
  53
  54 (defmethod stream-element-type ((stream dual-channel-gray-stream))
  55   '(unsigned-byte 8))
  56
  57 ;; TODO: use the buffer pool
  58 (defmethod close :around ((stream dual-channel-gray-stream) &key abort)
  59   (with-accessors ((ib input-buffer-of)
  60                    (ob output-buffer-of))
  61       stream
  62     (trivial-garbage:cancel-finalization stream)
  63     (unless (or abort (null ib)) (finish-output stream))
  64     (free-stream-buffers ib ob)
  65     (setf ib nil ob nil))
  66   (call-next-method)
  67   (values stream))
  68
  69 (defmethod close ((stream dual-channel-gray-stream) &key abort)
  70   (declare (ignore stream abort)))
  71
  72 (defmethod (setf external-format-of)
  73     (external-format (stream dual-channel-gray-stream))
  74   (setf (slot-value stream 'external-format)
  75         (babel:ensure-external-format external-format)))
  76
  77 ;;;; Input Methods
  78
  79 (defun %to-octets (buff start end ef)
  80   (babel:string-to-octets buff :start start :end end
  81                           :encoding (babel:external-format-encoding ef)))
  82
  83 (defmethod stream-clear-input ((stream dual-channel-gray-stream))
  84   (with-accessors ((ib input-buffer-of))
  85       stream
  86     (iobuf-reset ib)
  87     nil))
  88
  89 (defun %fill-ibuf (read-fn fd buf &optional timeout)
  90   (when timeout
  91     (let ((readablep (iomux:wait-until-fd-ready fd :read timeout)))
  92       (unless readablep
  93         (return-from %fill-ibuf :timeout))))
  94   (let ((num (nix:repeat-upon-eintr
  95                (funcall read-fn fd (iobuf-end-pointer buf)
  96                         (iobuf-end-space-length buf)))))
  97     (if (zerop num)
  98         :eof
  99         (incf (iobuf-end buf) num))))
 100
 101 (defun %read-into-simple-array-ub8 (stream array start end)
 102   (declare (type dual-channel-gray-stream stream))
 103   (with-accessors ((ib input-buffer-of)
 104                    (fd input-fd-of)
 105                    (read-fn read-fn-of))
 106       stream
 107     (let ((octets-needed (- end start)))
 108       (loop :with array-offset := start
 109             :for octets-in-buffer := (iobuf-length ib)
 110             :for nbytes := (min octets-needed octets-in-buffer)
 111          :when (plusp nbytes) :do
 112            (iobuf-copy-into-lisp-array ib (iobuf-start ib)
 113                                        array array-offset nbytes)
 114            (incf array-offset nbytes)
 115            (decf octets-needed nbytes)
 116            (incf (iobuf-start ib) nbytes)
 117          :if (zerop octets-needed) :do (loop-finish)
 118          :else :do (iobuf-reset ib)
 119          :when (eq :eof (%fill-ibuf read-fn fd ib)) :do (loop-finish)
 120          :finally (return array-offset)))))
 121
 122 (defun %read-into-string (stream string start end)
 123   (declare (type dual-channel-gray-stream stream))
 124   (loop :for offset :from start :below end
 125         :for char := (stream-read-char stream)
 126      :if (eq char :eof) :do (loop-finish)
 127      :else :do (setf (char string offset) char)
 128      :finally (return offset)))
 129
 130 (defun %read-into-vector (stream vector start end)
 131   (declare (type dual-channel-gray-stream stream))
 132   (loop :for offset :from start :below end
 133         :for octet := (stream-read-byte stream)
 134      :if (eq octet :eof) :do (loop-finish)
 135      :else :do (setf (aref vector offset) octet)
 136      :finally (return offset)))
 137
 138 (defmacro check-bounds (sequence start end)
 139   (with-gensyms (length)
 140     `(let ((,length (length ,sequence)))
 141        (unless ,end
 142          (setq ,end ,length))
 143        (unless (<= ,start ,end ,length)
 144          (error "Wrong sequence bounds. start: ~S end: ~S" ,start ,end)))))
 145
 146 (declaim (inline %read-sequence))
 147 (defun %read-sequence (stream seq start end)
 148   (check-bounds seq start end)
 149   (when (< start end)
 150     (etypecase seq
 151       (ub8-sarray (%read-into-simple-array-ub8 stream seq start end))
 152       (string (%read-into-string stream seq start end))
 153       (ub8-vector (%read-into-vector stream seq start end)))))
 154
 155 (declaim (inline read-sequence*))
 156 (defun read-sequence* (stream sequence &key (start 0) end)
 157   (%read-sequence stream sequence start end))
 158
 159 (defmethod stream-read-sequence
 160     ((stream dual-channel-gray-stream) sequence start end &key)
 161   (%read-sequence stream sequence start end))
 162
 163 ;;;; Output Methods
 164
 165 (defun %write-n-bytes (write-fn fd buf nbytes &optional timeout)
 166   (declare (type stream-buffer buf))
 167   (let ((bytes-written 0))
 168     (labels ((write-once ()
 169                (let ((num (handler-case
 170                               (nix:repeat-upon-condition-decreasing-timeout
 171                                   ((nix:eintr) timeout-var timeout)
 172                                 (prog1
 173                                     (funcall write-fn fd (inc-pointer buf bytes-written)
 174                                              nbytes)
 175                                   (when (and timeout-var (zerop timeout-var))
 176                                     (return-from %write-n-bytes
 177                                       (values nil :timeout)))))
 178                             (nix:epipe ()
 179                               (return-from %write-n-bytes (values nil :eof))))))
 180                  (unless (zerop num) (incf bytes-written num))))
 181              (write-or-return ()
 182                (unless (write-once)
 183                  (when (errorp)
 184                    ;; FIXME signal something better -- maybe analyze the status
 185                    (return-from %write-n-bytes (values nil :fail)))))
 186              (buffer-emptyp () (= bytes-written nbytes))
 187              (errorp () (handler-case (iomux:wait-until-fd-ready fd :write)
 188                           (iomux:poll-error () t)
 189                           (:no-error (r w) (declare (ignore r w)) nil))))
 190       (loop :until (buffer-emptyp) :do (write-or-return)
 191          :finally (return (values t bytes-written))))))
 192
 193 (defun %flush-obuf (write-fn fd buf &optional timeout)
 194   (declare (type iobuf buf))
 195   (let ((bytes-written 0))
 196     (labels ((write-once ()
 197                (let ((num (handler-case
 198                               (nix:repeat-upon-condition-decreasing-timeout
 199                                   ((nix:eintr) timeout-var timeout)
 200                                 (prog1
 201                                     (funcall write-fn fd (iobuf-start-pointer buf)
 202                                              (iobuf-length buf))
 203                                   (when (and timeout-var (zerop timeout-var))
 204                                     (return-from %flush-obuf
 205                                       (values nil :timeout)))))
 206                             (nix:epipe ()
 207                               (return-from %flush-obuf (values nil :eof))))))
 208                  (unless (zerop num)
 209                    (incf (iobuf-start buf) num)
 210                    (incf bytes-written num))))
 211              (write-or-return ()
 212                (unless (write-once)
 213                  (when (errorp)
 214                    ;; FIXME signal something better -- maybe analyze the status
 215                    (return-from %flush-obuf (values nil :fail)))))
 216              (buffer-emptyp ()
 217                (when (iobuf-empty-p buf)
 218                  (iobuf-reset buf) t))
 219              (errorp () (handler-case (iomux:wait-until-fd-ready fd :write)
 220                           (iomux:poll-error () t)
 221                           (:no-error (r w) (declare (ignore r w)) nil))))
 222       (loop :until (buffer-emptyp) :do (write-or-return)
 223          :finally (return (values t bytes-written))))))
 224
 225 ;;; TODO: add timeout support
 226 (defun %flush-obuf-if-needed (stream)
 227   (declare (type dual-channel-gray-stream stream))
 228   (with-accessors ((fd output-fd-of)
 229                    (write-fn write-fn-of)
 230                    (ob output-buffer-of)
 231                    (dirtyp dirtyp))
 232       stream
 233     (when (or dirtyp (iobuf-full-p ob))
 234       (%flush-obuf write-fn fd ob)
 235       (setf dirtyp nil))))
 236
 237 (defmethod stream-clear-output ((stream dual-channel-gray-stream))
 238   (with-accessors ((ob output-buffer-of)
 239                    (dirtyp dirtyp))
 240       stream
 241     (iobuf-reset ob)
 242     (setf dirtyp nil)
 243     nil))
 244
 245 (defmethod stream-finish-output ((stream dual-channel-gray-stream))
 246   (with-accessors ((fd output-fd-of)
 247                    (write-fn write-fn-of)
 248                    (ob output-buffer-of)
 249                    (dirtyp dirtyp))
 250       stream
 251     (%flush-obuf write-fn fd ob)
 252     (setf dirtyp nil)
 253     nil))
 254
 255 (defmethod stream-force-output ((stream dual-channel-gray-stream))
 256   (setf (dirtyp stream) t))
 257
 258 (defun %write-simple-array-ub8 (stream array start end)
 259   (declare (type dual-channel-gray-stream stream))
 260   (with-accessors ((fd output-fd-of)
 261                    (write-fn write-fn-of)
 262                    (ob output-buffer-of))
 263       stream
 264     (let ((octets-needed (- end start)))
 265       (cond ((<= octets-needed (iobuf-end-space-length ob))
 266              (iobuf-copy-from-lisp-array array start ob
 267                                          (iobuf-end ob) octets-needed)
 268              (incf (iobuf-end ob) octets-needed)
 269              (%flush-obuf-if-needed stream))
 270             (t
 271              (with-pointer-to-vector-data (ptr array)
 272                (%flush-obuf write-fn fd ob)
 273                (let ((ret (%write-n-bytes write-fn fd (inc-pointer ptr start)
 274                                           octets-needed)))
 275                  (when (numberp ret)
 276                    (incf (iobuf-end ob) octets-needed))))))
 277       (values array))))
 278
 279 (defun %write-vector-ub8 (stream vector start end)
 280   (declare (type dual-channel-gray-stream stream))
 281   (%write-simple-array-ub8 stream (coerce vector 'ub8-sarray) start end))
 282
 283 (defun %write-vector (stream vector start end)
 284   (declare (type dual-channel-gray-stream stream))
 285   (loop :for offset :from start :below end
 286         :for octet := (aref vector offset)
 287      :do (stream-write-byte stream octet)
 288      :finally (return vector)))
 289
 290 (declaim (inline %write-sequence))
 291 (defun %write-sequence (stream seq start end)
 292   (check-bounds seq start end)
 293   (when (< start end)
 294     (etypecase seq
 295       (ub8-sarray (%write-simple-array-ub8 stream seq start end))
 296       (string (stream-write-string stream seq start end))
 297       (ub8-vector (%write-vector-ub8 stream seq start end))
 298       (vector (%write-vector stream seq start end)))))
 299
 300 (declaim (inline write-sequence*))
 301 (defun write-sequence* (stream sequence &key (start 0) end)
 302   (%write-sequence stream sequence start end))
 303
 304 (defmethod stream-write-sequence ((stream dual-channel-gray-stream)
 305                                   sequence start end &key)
 306   (%write-sequence stream sequence start end))
 307
 308 ;;;; Character Input
 309
 310 (defun maybe-find-line-ending (read-fn fd ib ef)
 311   (let* ((start-off (iobuf-start ib))
 312          (char-code (bref ib start-off)))
 313     (block nil
 314       (ecase (babel:external-format-eol-style ef)
 315         (:lf (when (= char-code (char-code #\Linefeed))
 316                (incf (iobuf-start ib))
 317                (return #\Newline)))
 318         (:cr (when (= char-code (char-code #\Return))
 319                (incf (iobuf-start ib))
 320                (return #\Newline)))
 321         (:crlf (when (= char-code (char-code #\Return))
 322                  (when (and (= (iobuf-length ib) 1)
 323                             (eq :eof (%fill-ibuf read-fn fd ib)))
 324                    (incf (iobuf-start ib))
 325                    (return #\Return))
 326                  (when (= (bref ib (1+ start-off))
 327                           (char-code #\Linefeed))
 328                    (incf (iobuf-start ib) 2)
 329                    (return #\Newline))))))))
 330
 331 (defconstant +max-octets-per-char+ 6)
 332
 333 ;;; FIXME: currently we return :EOF when read(2) returns 0
 334 ;;;        we should distinguish hard end-of-files (EOF and buffer empty)
 335 ;;;        from soft end-of-files (EOF and *some* bytes still in the buffer
 336 ;;;        but not enough to make a full character)
 337 (defmethod stream-read-char ((stream dual-channel-gray-stream))
 338   (with-accessors ((fd input-fd-of)
 339                    (ib input-buffer-of)
 340                    (read-fn read-fn-of)
 341                    (unread-index ibuf-unread-index-of)
 342                    (ef external-format-of))
 343       stream
 344     (setf unread-index (iobuf-start ib))
 345     (let ((str nil)
 346           (ret nil))
 347       (flet ((fill-buf-or-eof ()
 348                (setf ret (%fill-ibuf read-fn fd ib))
 349                (when (eq ret :eof)
 350                  (return-from stream-read-char :eof))))
 351         (cond ((zerop (iobuf-length ib))
 352                (iobuf-reset ib)
 353                (fill-buf-or-eof))
 354               ;; Some encodings such as CESU or Java's modified UTF-8 take
 355               ;; as much as 6 bytes per character. Make sure we have enough
 356               ;; space to collect read-ahead bytes if required.
 357               ((< (iobuf-length ib) +max-octets-per-char+)
 358                (iobuf-copy-data-to-start ib)
 359                (setf unread-index 0)))
 360         ;; line-end handling
 361         (when-let ((it (maybe-find-line-ending read-fn fd ib ef)))
 362           (return-from stream-read-char it))
 363         (tagbody :start
 364            (handler-case
 365                (setf (values str ret)
 366                      (foreign-string-to-lisp
 367                       (iobuf-data ib)
 368                       :offset (iobuf-start ib)
 369                       :count (iobuf-length ib)
 370                       :encoding (babel:external-format-encoding ef)
 371                       :max-chars 1))
 372              (babel:end-of-input-in-character ()
 373                (fill-buf-or-eof)
 374                (go :start)))
 375            (incf (iobuf-start ib) ret))
 376         (char str 0)))))
 377
 378 (defun maybe-find-line-ending-no-hang (fd ib ef)
 379   (declare (ignore fd))
 380   (let* ((start-off (iobuf-start ib))
 381          (char-code (bref ib start-off)))
 382     (block nil
 383       (ecase (babel:external-format-eol-style ef)
 384         (:lf (when (= char-code (char-code #\Linefeed))
 385                (incf (iobuf-start ib))
 386                (return #\Newline)))
 387         (:cr (when (= char-code (char-code #\Return))
 388                (incf (iobuf-start ib))
 389                (return #\Newline)))
 390         (:crlf (when (= char-code (char-code #\Return))
 391                  (when (= (iobuf-length ib) 1)
 392                    (incf (iobuf-start ib))
 393                    (return :starvation))
 394                  (when (= (bref ib (1+ start-off))
 395                           (char-code #\Linefeed))
 396                    (incf (iobuf-start ib) 2)
 397                    (return #\Newline))))))))
 398
 399 (defmethod stream-read-char-no-hang ((stream dual-channel-gray-stream))
 400   (with-accessors ((fd input-fd-of)
 401                    (read-fn read-fn-of)
 402                    (ib input-buffer-of)
 403                    (ef external-format-of))
 404       stream
 405     (let ((str nil)
 406           (ret nil)
 407           (eof nil))
 408       (block nil
 409         ;; BUG: this comparision is probably buggy, FIXME.  A similar
 410         ;; bug was fixed in STREAM-READ-CHAR.  Must write a test for
 411         ;; this one first.
 412         (when (< 0 (iobuf-end-space-length ib) 4)
 413           (iobuf-copy-data-to-start ib))
 414         (when (and (iomux:fd-ready-p fd :read)
 415                    (eq :eof (%fill-ibuf read-fn fd ib)))
 416           (setf eof t))
 417         (when (zerop (iobuf-length ib))
 418           (return (if eof :eof nil)))
 419         ;; line-end handling
 420         (let ((line-end (maybe-find-line-ending-no-hang fd ib ef)))
 421           (cond ((eq line-end :starvation)
 422                  (return (if eof #\Return nil)))
 423                 ((characterp line-end)
 424                  (return line-end))))
 425         ;; octet decoding
 426         (handler-case
 427             (setf (values str ret)
 428                   (foreign-string-to-lisp
 429                    (iobuf-data ib)
 430                    :offset (iobuf-start ib)
 431                    :count (iobuf-length ib)
 432                    :encoding (babel:external-format-encoding ef)
 433                    :max-chars 1))
 434           (babel:end-of-input-in-character ()
 435             (return nil)))
 436         (incf (iobuf-start ib) ret)
 437         (char str 0)))))
 438
 439 (defun %stream-unread-char (stream)
 440   (declare (type dual-channel-gray-stream stream))
 441   (with-accessors ((ib input-buffer-of)
 442                    (unread-index ibuf-unread-index-of))
 443       stream
 444     (symbol-macrolet ((start (iobuf-start ib)))
 445       (cond
 446         ((> start unread-index) (setf start unread-index))
 447         (t (error "No uncommitted character to unread")))))
 448   nil)
 449
 450 (defmethod stream-unread-char ((stream dual-channel-gray-stream) character)
 451   (declare (ignore character))
 452   (%stream-unread-char stream))
 453
 454 (defmethod stream-peek-char ((stream dual-channel-gray-stream))
 455   (let ((char (stream-read-char stream)))
 456     (cond ((eq char :eof) :eof)
 457           (t (%stream-unread-char stream)
 458              (values char)))))
 459
 460 ;; (defmethod stream-read-line ((stream dual-channel-gray-stream))
 461 ;;   )
 462
 463 (defmethod stream-listen ((stream dual-channel-gray-stream))
 464   (let ((char (stream-read-char-no-hang stream)))
 465     (cond ((characterp char) (stream-unread-char stream char) t)
 466           ((eq char :eof) nil)
 467           (t t))))
 468
 469 ;;;; Character Output
 470
 471 (defmethod stream-write-char ((stream dual-channel-gray-stream)
 472                               (character character))
 473   (%flush-obuf-if-needed stream)
 474   (if (char= character #\Newline)
 475       (%write-line-terminator
 476        stream (babel:external-format-eol-style (external-format-of stream)))
 477       ;; FIXME: avoid consing a string here. At worst, declare it dynamic-extent
 478       (stream-write-string stream (make-string 1 :initial-element character))))
 479
 480 (defmethod stream-line-column ((stream dual-channel-gray-stream))
 481   0)
 482
 483 (defmethod stream-start-line-p ((stream dual-channel-gray-stream))
 484   (values nil))
 485
 486 (defmethod stream-terpri ((stream dual-channel-gray-stream))
 487   (write-char #\Newline stream) nil)
 488
 489 (defmethod stream-fresh-line ((stream dual-channel-gray-stream))
 490   (write-char #\Newline stream) t)
 491
 492 (define-constant +unix-line-terminator+
 493     (make-array 1 :element-type 'ub8 :initial-contents '(10))
 494   :test 'equalp)
 495
 496 (define-constant +dos-line-terminator+
 497     (make-array 2 :element-type 'ub8 :initial-contents '(13 10))
 498   :test 'equalp)
 499
 500 (define-constant +mac-line-terminator+
 501     (make-array 1 :element-type 'ub8 :initial-contents '(13))
 502   :test 'equalp)
 503
 504 (defun %write-line-terminator (stream line-terminator)
 505   (case line-terminator
 506     (:lf (%write-simple-array-ub8 stream +unix-line-terminator+ 0 1))
 507     (:cr (%write-simple-array-ub8 stream +mac-line-terminator+ 0 1))
 508     (:crlf (%write-simple-array-ub8 stream +dos-line-terminator+ 0 2))))
 509
 510 (defmethod stream-write-string ((stream dual-channel-gray-stream)
 511                                 (string string) &optional (start 0) end)
 512   (check-bounds string start end)
 513   (when (< start end)
 514     (let* ((octets nil)
 515            (ef (external-format-of stream))
 516            (line-terminator (babel:external-format-eol-style ef)))
 517       (loop :for off1 := start :then (1+ off2)
 518             :for nl-off := (position #\Newline string :start off1)
 519             :for off2 := (or nl-off end)
 520          :when nl-off :do (%write-line-terminator stream line-terminator)
 521          :when (> off2 off1) :do
 522          ;; FIXME: should probably convert directly to a foreign buffer?
 523          (setf octets (%to-octets string off1 off2 ef))
 524          (%write-simple-array-ub8 stream octets 0 (length octets))
 525          :while (< off2 end))))
 526   (values string))
 527
 528 ;;;; Binary Input
 529
 530 (defmethod stream-read-byte ((stream dual-channel-gray-stream))
 531   (with-accessors ((fd input-fd-of)
 532                    (read-fn read-fn-of)
 533                    (ib input-buffer-of))
 534       stream
 535     (flet ((fill-buf-or-eof ()
 536              (iobuf-reset ib)
 537              (when (eq :eof (%fill-ibuf read-fn fd ib))
 538                (return-from stream-read-byte :eof))))
 539       (when (zerop (iobuf-length ib))
 540         (fill-buf-or-eof))
 541       (iobuf-pop-octet ib))))
 542
 543 ;;;; Binary Output
 544
 545 (defmethod stream-write-byte ((stream dual-channel-gray-stream) integer)
 546   (check-type integer ub8 "an unsigned 8-bit value")
 547   (with-accessors ((ob output-buffer-of))
 548       stream
 549     (%flush-obuf-if-needed stream)
 550     (iobuf-push-octet ob integer)))