src/code/load.lisp

   1 ;;;; parts of the loader which make sense in the cross-compilation
   2 ;;;; host (and which are useful in the host, because they're used by
   3 ;;;; GENESIS)
   4 ;;;;
   5 ;;;; based on the CMU CL load.lisp code, written by Skef Wholey and
   6 ;;;; Rob Maclachlan
   7
   8 ;;;; This software is part of the SBCL system. See the README file for
   9 ;;;; more information.
  10 ;;;;
  11 ;;;; This software is derived from the CMU CL system, which was
  12 ;;;; written at Carnegie Mellon University and released into the
  13 ;;;; public domain. The software is in the public domain and is
  14 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
  15 ;;;; files for more information.
  16
  17 (in-package "SB!FASL")
  18 \f
  19 ;;;; There looks to be an exciting amount of state being modified
  20 ;;;; here: certainly enough that I (dan, 2003.1.22) don't want to mess
  21 ;;;; around deciding how to thread-safetify it.  So we use a Big Lock.
  22 ;;;; Because this code is mutually recursive with the compiler, we use
  23 ;;;; the **WORLD-LOCK**.
  24
  25 ;;;; miscellaneous load utilities
  26
  27 ;;; Output the current number of semicolons after a fresh-line.
  28 ;;; FIXME: non-mnemonic name
  29 (defun load-fresh-line ()
  30   (fresh-line)
  31   (let ((semicolons ";;;;;;;;;;;;;;;;"))
  32     (do ((count *load-depth* (- count (length semicolons))))
  33         ((< count (length semicolons))
  34          (write-string semicolons *standard-output* :end count))
  35       (declare (fixnum count))
  36       (write-string semicolons))
  37     (write-char #\space)))
  38
  39 ;;; If VERBOSE, output (to *STANDARD-OUTPUT*) a message about how
  40 ;;; we're loading from STREAM-WE-ARE-LOADING-FROM.
  41 (defun maybe-announce-load (stream-we-are-loading-from verbose)
  42   (when verbose
  43     (load-fresh-line)
  44     (let ((name #-sb-xc-host (file-name stream-we-are-loading-from)
  45                 #+sb-xc-host nil))
  46       (if name
  47           (format t "loading ~S~%" name)
  48           (format t "loading stuff from ~S~%" stream-we-are-loading-from)))))
  49 \f
  50 ;;;; utilities for reading from fasl files
  51
  52 #!-sb-fluid (declaim (inline read-byte))
  53
  54 ;;; This expands into code to read an N-byte unsigned integer using
  55 ;;; FAST-READ-BYTE.
  56 (defmacro fast-read-u-integer (n)
  57   (let (bytes)
  58     `(let ,(loop for i from 0 below n
  59                  collect (let ((name (gensym "B")))
  60                            (push name bytes)
  61                            `(,name ,(if (zerop i)
  62                                         `(fast-read-byte)
  63                                         `(ash (fast-read-byte) ,(* i 8))))))
  64        (logior ,@bytes))))
  65
  66 ;;; like FAST-READ-U-INTEGER, but the size may be determined at run time
  67 (defmacro fast-read-var-u-integer (n)
  68   (let ((n-pos (gensym))
  69         (n-res (gensym))
  70         (n-cnt (gensym)))
  71     `(do ((,n-pos 8 (+ ,n-pos 8))
  72           (,n-cnt (1- ,n) (1- ,n-cnt))
  73           (,n-res
  74            (fast-read-byte)
  75            (dpb (fast-read-byte) (byte 8 ,n-pos) ,n-res)))
  76          ((zerop ,n-cnt) ,n-res)
  77        (declare (type index ,n-pos ,n-cnt)))))
  78
  79 ;;; FIXME: why do all of these reading functions and macros declare
  80 ;;; (SPEED 0)?  was there some bug in the compiler which has since
  81 ;;; been fixed?  --njf, 2004-09-08
  82 ;;; Afaict, the (SPEED 0) declarations in here avoid code bloat,
  83 ;;; by counteracting the INLINE declaration on the local definition
  84 ;;; of FAST-READ-BYTE. At least, that is the effect, and it seems
  85 ;;; reasonable. Pretty much the INLINE declaration is probably
  86 ;;; the thing that deserves to go away.
  87
  88 ;;; Read a signed integer.
  89 (defmacro fast-read-s-integer (n)
  90   (declare (optimize (speed 0)))
  91   (let ((n-last (gensym)))
  92     (do ((res `(let ((,n-last (fast-read-byte)))
  93                  (if (zerop (logand ,n-last #x80))
  94                      ,n-last
  95                      (logior ,n-last #x-100)))
  96               `(logior (fast-read-byte)
  97                        (ash (the (signed-byte ,(* cnt 8)) ,res) 8)))
  98          (cnt 1 (1+ cnt)))
  99         ((>= cnt n) res))))
 100
 101 ;;; Read an N-byte unsigned integer from FASL-INPUT-STREAM.
 102 (defmacro read-arg (n fasl-input-stream)
 103   (if (= n 1)
 104       `(the (unsigned-byte 8) (read-byte ,fasl-input-stream))
 105       `(with-fast-read-byte ((unsigned-byte 8) ,fasl-input-stream)
 106          (fast-read-u-integer ,n))))
 107
 108 ;; FIXME: on x86-64, these functions exceed 600, 900, and 1200 bytes of code
 109 ;; respectively. Either don't inline them, or make a "really" fast inline case
 110 ;; that punts if inapplicable. e.g. if the fast-read-byte buffer will not be
 111 ;; refilled, then SAP-REF-WORD could work to read 8 bytes.
 112 ;; But this would only be feasible on machines that are little-endian
 113 ;; and that allow unaligned reads. (like x86)
 114 (declaim (inline read-byte-arg read-halfword-arg read-word-arg))
 115 (defun read-byte-arg (stream)
 116   (declare (optimize (speed 0)))
 117   (read-arg 1 stream))
 118
 119 (defun read-halfword-arg (stream)
 120   (declare (optimize (speed 0)))
 121   (read-arg #.(/ sb!vm:n-word-bytes 2) stream)) ; READ-ARG doesn't eval N
 122
 123 (defun read-word-arg (stream)
 124   (declare (optimize (speed 0)))
 125   (read-arg #.sb!vm:n-word-bytes stream))
 126
 127 (defun read-unsigned-byte-32-arg (stream)
 128   (declare (optimize (speed 0)))
 129   (read-arg 4 stream))
 130
 131 \f
 132 ;;;; the fop table
 133
 134 ;;; The table is implemented as a simple-vector indexed by the table
 135 ;;; offset. The offset is kept in at index 0 of the vector.
 136 ;;;
 137 ;;; FOPs use the table to save stuff, other FOPs refer to the table by
 138 ;;; direct indexes via REF-FOP-TABLE.
 139
 140 (declaim (inline ref-fop-table))
 141 (defun ref-fop-table (fasl-input index)
 142   (svref (%fasl-input-table fasl-input) (1+ (the index index))))
 143
 144 (defun push-fop-table (thing fasl-input) ; and return THING
 145   (let* ((table (%fasl-input-table fasl-input))
 146          (index (+ (the index (aref table 0)) 1)))
 147     (declare (fixnum index)
 148              (simple-vector table))
 149     (when (eql index (length table))
 150       (setf table (grow-fop-vector table index)
 151             (%fasl-input-table fasl-input) table))
 152     (setf (aref table 0) index
 153           (aref table index) thing)))
 154
 155 ;;; These two routines are used for both the stack and the table.
 156 (defun grow-fop-vector (old-vector old-size)
 157   (declare (simple-vector old-vector)
 158            (type index old-size))
 159   (let* ((new-size (* old-size 2))
 160          (new-vector (make-array new-size)))
 161     (declare (fixnum new-size)
 162              (simple-vector new-vector old-vector))
 163     (replace new-vector old-vector)
 164     (nuke-fop-vector old-vector)
 165     new-vector))
 166
 167 (defun nuke-fop-vector (vector)
 168   (declare (simple-vector vector)
 169            #!-gencgc (ignore vector)
 170            (optimize speed))
 171   ;; Make sure we don't keep any garbage.
 172   #!+gencgc
 173   (fill vector 0))
 174
 175 \f
 176 ;;;; the fop stack
 177
 178 (declaim (inline fop-stack-empty-p))
 179 (defun fop-stack-empty-p (stack)
 180   (eql 0 (svref stack 0)))
 181
 182 ;; Ensure that N arguments can be popped from the FOP stack.
 183 ;; Return the stack and the pointer to the first argument.
 184 ;; Update the new top-of-stack to reflect that all N have been popped.
 185 (defun fop-stack-pop-n (stack n)
 186   (declare (type index n))
 187   (let* ((top (the index (svref stack 0)))
 188          (new-top (- top n)))
 189     (if (minusp new-top) ; 0 is ok at this point
 190         (error "FOP stack underflow")
 191         (progn (setf (svref stack 0) new-top)
 192                (1+ new-top)))))
 193
 194 (defun push-fop-stack (value fasl-input)
 195   (let* ((stack (%fasl-input-stack fasl-input))
 196          (next (1+ (the index (svref stack 0)))))
 197     (declare (type index next))
 198     (when (eql (length stack) next)
 199       (setf stack (grow-fop-vector stack next)
 200             (%fasl-input-stack fasl-input) stack))
 201     (setf (svref stack 0) next
 202           (svref stack next) value)))
 203
 204 \f
 205 ;;;; Conditions signalled on invalid fasls (wrong fasl version, etc),
 206 ;;;; so that user code (esp. ASDF) can reasonably handle attempts to
 207 ;;;; load such fasls by recompiling them, etc. For simplicity's sake
 208 ;;;; make only condition INVALID-FASL part of the public interface,
 209 ;;;; and keep the guts internal.
 210
 211 (define-condition invalid-fasl (error)
 212   ((stream :reader invalid-fasl-stream :initarg :stream)
 213    (expected :reader invalid-fasl-expected :initarg :expected))
 214   (:report
 215    (lambda (condition stream)
 216      (format stream "~S is an invalid fasl file."
 217              (invalid-fasl-stream condition)))))
 218
 219 (define-condition invalid-fasl-header (invalid-fasl)
 220   ((byte :reader invalid-fasl-byte :initarg :byte)
 221    (byte-nr :reader invalid-fasl-byte-nr :initarg :byte-nr))
 222   (:report
 223    (lambda (condition stream)
 224      (format stream "~@<~S contains an illegal byte in the FASL header at ~
 225                      position ~A: Expected ~A, got ~A.~:@>"
 226              (invalid-fasl-stream condition)
 227              (invalid-fasl-byte-nr condition)
 228              (invalid-fasl-expected condition)
 229              (invalid-fasl-byte condition)))))
 230
 231 (define-condition invalid-fasl-version (invalid-fasl)
 232   ((version :reader invalid-fasl-version :initarg :version))
 233   (:report
 234    (lambda (condition stream)
 235      (format stream "~@<~S is a fasl file compiled with SBCL ~W, and ~
 236                       can't be loaded into SBCL ~W.~:@>"
 237              (invalid-fasl-stream condition)
 238              (invalid-fasl-version condition)
 239              (invalid-fasl-expected condition)))))
 240
 241 (define-condition invalid-fasl-implementation (invalid-fasl)
 242   ((implementation :reader invalid-fasl-implementation
 243                    :initarg :implementation))
 244   (:report
 245    (lambda (condition stream)
 246      (format stream "~S was compiled for implementation ~A, but this is a ~A."
 247              (invalid-fasl-stream condition)
 248              (invalid-fasl-implementation condition)
 249              (invalid-fasl-expected condition)))))
 250
 251 (define-condition invalid-fasl-features (invalid-fasl)
 252   ((potential-features :reader invalid-fasl-potential-features
 253                        :initarg :potential-features)
 254    (features :reader invalid-fasl-features :initarg :features))
 255   (:report
 256    (lambda (condition stream)
 257      (format stream "~@<incompatible ~S in fasl file ~S: ~2I~_~
 258                      Of features affecting binary compatibility, ~4I~_~S~2I~_~
 259                      the fasl has ~4I~_~A,~2I~_~
 260                      while the runtime expects ~4I~_~A.~:>"
 261              '*features*
 262              (invalid-fasl-stream condition)
 263              (invalid-fasl-potential-features condition)
 264              (invalid-fasl-features condition)
 265              (invalid-fasl-expected condition)))))
 266
 267 ;;; Skips past the shebang line on stream, if any.
 268 (defun maybe-skip-shebang-line (stream)
 269   (let ((p (file-position stream)))
 270     (flet ((next () (read-byte stream nil)))
 271       (unwind-protect
 272            (when (and (eq (next) (char-code #\#))
 273                       (eq (next) (char-code #\!)))
 274              (setf p nil)
 275              (loop for x = (next)
 276                    until (or (not x) (eq x (char-code #\newline)))))
 277         (when p
 278           (file-position stream p))))
 279     t))
 280
 281 ;;; Returns T if the stream is a binary input stream with a FASL header.
 282 #-sb-xc-host ;; FIXME: function belongs in 'target-load'
 283 (defun fasl-header-p (stream &key errorp)
 284   (unless (and (member (stream-element-type stream) '(character base-char))
 285                ;; give up if it's not a file stream, or it's an
 286                ;; fd-stream but it's either not bivalent or not
 287                ;; seekable (doesn't really have a file)
 288                (or (not (typep stream 'file-stream))
 289                    (and (typep stream 'fd-stream)
 290                         (or (not (sb!impl::fd-stream-bivalent-p stream))
 291                             (not (sb!impl::fd-stream-file stream))))))
 292     (let ((p (file-position stream)))
 293       (unwind-protect
 294            (let* ((header *fasl-header-string-start-string*)
 295                   (buffer (make-array (length header) :element-type '(unsigned-byte 8)))
 296                   (n 0))
 297              (flet ((scan ()
 298                       (maybe-skip-shebang-line stream)
 299                       (setf n (read-sequence buffer stream))))
 300                (if errorp
 301                    (scan)
 302                    (or (ignore-errors (scan))
 303                        ;; no a binary input stream
 304                        (return-from fasl-header-p nil))))
 305              (if (mismatch buffer header
 306                            :test #'(lambda (code char) (= code (char-code char))))
 307                  ;; Immediate EOF is valid -- we want to match what
 308                  ;; CHECK-FASL-HEADER does...
 309                  (or (zerop n)
 310                      (when errorp
 311                        (error 'fasl-header-missing
 312                               :stream stream
 313                               :fhsss buffer
 314                               :expected header)))
 315                  t))
 316         (file-position stream p)))))
 317
 318
 319 ;;;; LOAD-AS-FASL
 320 ;;;;
 321 ;;;; Note: LOAD-AS-FASL is used not only by LOAD, but also (with
 322 ;;;; suitable modification of the fop table) in GENESIS. Therefore,
 323 ;;;; it's needed not only in the target Lisp, but also in the
 324 ;;;; cross-compilation host.
 325
 326 ;;; a helper function for LOAD-FASL-GROUP
 327 ;;;
 328 ;;; Return true if we successfully read a FASL header from the stream, or NIL
 329 ;;; if EOF was hit before anything except the optional shebang line was read.
 330 ;;; Signal an error if we encounter garbage.
 331 (defun check-fasl-header (stream)
 332   (maybe-skip-shebang-line stream)
 333   (let ((byte (read-byte stream nil)))
 334     (when byte
 335       ;; Read and validate constant string prefix in fasl header.
 336       (let* ((fhsss *fasl-header-string-start-string*)
 337              (fhsss-length (length fhsss)))
 338         (unless (= byte (char-code (schar fhsss 0)))
 339           (error 'invalid-fasl-header
 340                  :stream stream
 341                  :byte-nr 0
 342                  :byte byte
 343                  :expected (char-code (schar fhsss 0))))
 344         (do ((byte (read-byte stream) (read-byte stream))
 345              (count 1 (1+ count)))
 346             ((= byte +fasl-header-string-stop-char-code+)
 347              t)
 348           (declare (fixnum byte count))
 349           (when (and (< count fhsss-length)
 350                      (not (eql byte (char-code (schar fhsss count)))))
 351             (error 'invalid-fasl-header
 352                    :stream stream
 353                    :byte-nr count
 354                    :byte byte
 355                    :expected (char-code (schar fhsss count))))))
 356       ;; Read and validate version-specific compatibility stuff.
 357       (flet ((string-from-stream ()
 358                (let* ((length (read-unsigned-byte-32-arg stream))
 359                       (result (make-string length)))
 360                  (read-string-as-bytes stream result)
 361                  result)))
 362         ;; Read and validate implementation and version.
 363         (let ((implementation (keywordicate (string-from-stream)))
 364               (expected-implementation +backend-fasl-file-implementation+))
 365           (unless (string= expected-implementation implementation)
 366             (error 'invalid-fasl-implementation
 367                    :stream stream
 368                    :implementation implementation
 369                    :expected expected-implementation)))
 370         (let* ((fasl-version (read-word-arg stream))
 371                (sbcl-version (if (<= fasl-version 76)
 372                                  "1.0.11.18"
 373                                  (string-from-stream)))
 374                (expected-version (sb!xc:lisp-implementation-version)))
 375           (unless (string= expected-version sbcl-version)
 376             (restart-case
 377                 (error 'invalid-fasl-version
 378                        :stream stream
 379                        :version sbcl-version
 380                        :expected expected-version)
 381               (continue () :report "Load the fasl file anyway"))))
 382         ;; Read and validate *FEATURES* which affect binary compatibility.
 383         (let ((faff-in-this-file (string-from-stream)))
 384           (unless (string= faff-in-this-file *features-affecting-fasl-format*)
 385             (error 'invalid-fasl-features
 386                    :stream stream
 387                    :potential-features *features-potentially-affecting-fasl-format*
 388                    :expected *features-affecting-fasl-format*
 389                    :features faff-in-this-file)))
 390         ;; success
 391         t))))
 392
 393 ;; Setting this variable gives you a trace of fops as they are loaded and
 394 ;; executed.
 395 #!+sb-show
 396 (defvar *show-fops-p* nil)
 397
 398 ;;;
 399 ;;; a helper function for LOAD-AS-FASL
 400 ;;;
 401 ;;; Return true if we successfully load a group from the stream, or
 402 ;;; NIL if EOF was encountered while trying to read from the stream.
 403 ;;; Dispatch to the right function for each fop.
 404 (defconstant +2-operand-fops+ #xE0) ; start of the range
 405 (defun load-fasl-group (fasl-input print)
 406   ;;
 407   ;; PRINT causes most tlf-equivalent forms to print their primary value.
 408   ;; This differs from loading of Lisp source, which prints all values of
 409   ;; only truly-toplevel forms.  This is permissible per CLHS -
 410   ;;  "If print is true, load incrementally prints information to standard
 411   ;;   output showing the progress of the loading process. [...]
 412   ;;   For a compiled file, what is printed might not reflect precisely the
 413   ;;   contents of the source file, but some information is generally printed."
 414   ;;
 415   (declare (ignorable print))
 416   (let ((stream (%fasl-input-stream fasl-input))
 417         #!+sb-show (trace *show-fops-p*))
 418     (unless (check-fasl-header stream)
 419       (return-from load-fasl-group))
 420     (catch 'fasl-group-end
 421      (setf (svref (%fasl-input-table fasl-input) 0) 0)
 422      (macrolet ((tracing (&body forms)
 423                   #!+sb-show `(when trace ,@forms)
 424                   #!-sb-show (progn forms nil)))
 425       (loop
 426        (let ((byte (the (unsigned-byte 8) (read-byte stream))))
 427          ;; Do some debugging output.
 428          (tracing
 429            (format *trace-output* "~&~6x : [~D,~D] ~2,'0x(~A)"
 430                    (1- (file-position stream))
 431                    (svref (%fasl-input-stack fasl-input) 0) ; stack pointer
 432                    (svref (%fasl-input-table fasl-input) 0) ; table pointer
 433                    byte (aref **fop-names** byte)))
 434          ;; Actually execute the fop.
 435          (let ((result
 436                 (let ((function (svref **fop-funs** byte)))
 437                   (cond ((not (functionp function))
 438                          (error "corrupt fasl file: FOP code #x~x" byte))
 439                         ((zerop (sbit (car **fop-signatures**) (ash byte -2)))
 440                          ;; takes no arguments from the fasl file
 441                          (funcall function fasl-input))
 442                         (t
 443                          ;; one or two arguments come from the fasl file
 444                          (let (arg1 arg2) ; See !%DEFINE-FOP for encoding
 445                            (with-fast-read-byte ((unsigned-byte 8) stream)
 446                              (setq arg1 (fast-read-var-u-integer
 447                                          (ash 1 (logand byte 3))))
 448                              (when (>= byte +2-operand-fops+)
 449                                (setq arg2 (fast-read-var-u-integer
 450                                            (ash 1 (ldb (byte 2 2) byte))))))
 451                            (tracing
 452                              (format *trace-output* "{~D~@[,~D~]}" arg1 arg2))
 453                            (if arg2
 454                                (funcall function fasl-input arg1 arg2)
 455                                (funcall function fasl-input arg1))))))))
 456            (when (plusp (sbit (cdr **fop-signatures**) byte))
 457              (push-fop-stack result fasl-input))
 458            (let ((stack (%fasl-input-stack fasl-input)))
 459              (declare (ignorable stack)) ; not used in xc-host
 460              (tracing
 461               (let ((ptr (svref stack 0)))
 462                 (format *trace-output* " -- ~[<empty>,~D~:;[~:*~D,~D] ~S~]"
 463                         ptr (svref (%fasl-input-table fasl-input) 0)
 464                         (unless (eql ptr 0) (aref stack ptr)))
 465                 (terpri *trace-output*)))
 466              #-sb-xc-host
 467              (macrolet ((terminator-opcode ()
 468                           (or (get 'fop-funcall-for-effect 'opcode)
 469                               (error "Missing FOP definition?"))))
 470                (when (and (eq byte (terminator-opcode))
 471                           (fop-stack-empty-p stack)) ; (presumed) end of TLF
 472                  (awhen (%fasl-input-deprecated-stuff fasl-input)
 473                    ;; Delaying this message rather than printing it
 474                    ;; in fop-fdefn makes it more informative (usually).
 475                    (setf (%fasl-input-deprecated-stuff fasl-input) nil)
 476                    (loader-deprecation-warn
 477                     it
 478                     (and (eq (svref stack 1) 'sb!impl::%defun) (svref stack 2))))
 479                  (when print
 480                    (load-fresh-line)
 481                    (prin1 result))))))))))))
 482
 483 (defun load-as-fasl (stream verbose print)
 484   (when (zerop (file-length stream))
 485     (error "attempt to load an empty FASL file:~%  ~S" (namestring stream)))
 486   (maybe-announce-load stream verbose)
 487   (let ((fasl-input (make-fasl-input stream)))
 488     (unwind-protect
 489          (loop while (load-fasl-group fasl-input print))
 490       ;; Nuke the table and stack to avoid keeping garbage on
 491       ;; conservatively collected platforms.
 492       (nuke-fop-vector (%fasl-input-table fasl-input))
 493       (nuke-fop-vector (%fasl-input-stack fasl-input))))
 494   t)
 495
 496 (declaim (notinline read-byte)) ; Why is it even *declaimed* inline above?
 497 \f
 498 ;;;; stuff for debugging/tuning by collecting statistics on FOPs (?)
 499
 500 #|
 501 (defvar *fop-counts* (make-array 256 :initial-element 0))
 502 (defvar *fop-times* (make-array 256 :initial-element 0))
 503 (defvar *print-fops* nil)
 504
 505 (defun clear-counts ()
 506   (fill (the simple-vector *fop-counts*) 0)
 507   (fill (the simple-vector *fop-times*) 0)
 508   t)
 509
 510 (defun analyze-counts ()
 511   (let ((counts ())
 512         (total-count 0)
 513         (times ())
 514         (total-time 0))
 515     (macrolet ((breakdown (lvar tvar vec)
 516                  `(progn
 517                    (dotimes (i 255)
 518                      (declare (fixnum i))
 519                      (let ((n (svref ,vec i)))
 520                        (push (cons (svref *fop-names* i) n) ,lvar)
 521                        (incf ,tvar n)))
 522                    (setq ,lvar (subseq (sort ,lvar (lambda (x y)
 523                                                      (> (cdr x) (cdr y))))
 524                                        0 10)))))
 525
 526       (breakdown counts total-count *fop-counts*)
 527       (breakdown times total-time *fop-times*)
 528       (format t "Total fop count is ~D~%" total-count)
 529       (dolist (c counts)
 530         (format t "~30S: ~4D~%" (car c) (cdr c)))
 531       (format t "~%Total fop time is ~D~%" (/ (float total-time) 60.0))
 532       (dolist (m times)
 533         (format t "~30S: ~6,2F~%" (car m) (/ (float (cdr m)) 60.0))))))
 534 |#
 535