src/cold/defun-load-or-cload-xcompiler.lisp

   1 ;;;; This software is part of the SBCL system. See the README file for
   2 ;;;; more information.
   3 ;;;;
   4 ;;;; This software is derived from the CMU CL system, which was
   5 ;;;; written at Carnegie Mellon University and released into the
   6 ;;;; public domain. The software is in the public domain and is
   7 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
   8 ;;;; files for more information.
   9
  10 (in-package "SB-COLD")
  11
  12 (defparameter *full-calls-to-warn-about*
  13   '(;mask-signed-field ;; Too many to fix
  14     ))
  15
  16 ;;; Set of function names whose definition will never be seen in make-host-2,
  17 ;;; as they are deferred until warm load.
  18 ;;; The table is populated later in this file.
  19 (defparameter *undefined-fun-whitelist* (make-hash-table :test 'equal))
  20
  21 (export '*symbol-values-for-genesis*)
  22 (let ((pathname "output/init-symbol-values.lisp-expr"))
  23   (defvar *symbol-values-for-genesis*
  24     (and (probe-file pathname) (read-from-file pathname)))
  25   (defun save-initial-symbol-values ()
  26     (with-open-file (f pathname :direction :output :if-exists :supersede)
  27       (declare (special *symbol-values-for-genesis*)) ; non-toplevel DEFVAR
  28       (write *symbol-values-for-genesis* :stream f :readably t))))
  29
  30 (when (make-host-1-parallelism)
  31   (require :sb-posix))
  32 #+#.(cl:if (cl:find-package "SB-POSIX") '(and) '(or))
  33 (defun parallel-make-host-1 (max-jobs)
  34   (let ((subprocess-count 0)
  35         (subprocess-list nil))
  36     (flet ((wait ()
  37              (multiple-value-bind (pid status) (sb-posix:wait)
  38                (format t "~&; Subprocess ~D exit status ~D~%"  pid status)
  39                (setq subprocess-list (delete pid subprocess-list)))
  40              (decf subprocess-count)))
  41       (do-stems-and-flags (stem flags)
  42         (unless (position :not-host flags)
  43           (when (>= subprocess-count max-jobs)
  44             (wait))
  45           (let ((pid (sb-posix:fork)))
  46             (when (zerop pid)
  47               (in-host-compilation-mode
  48                (lambda () (compile-stem stem flags :host-compile)))
  49               ;; FIXME: convey exit code based on COMPILE result.
  50               (sb-sys:os-exit 0))
  51             (push pid subprocess-list)
  52             (incf subprocess-count)
  53             ;; Do not wait for the compile to finish. Just load as source.
  54             (let ((source (merge-pathnames (stem-remap-target stem)
  55                                            (make-pathname :type "lisp"))))
  56               (let ((sb-ext:*evaluator-mode* :interpret))
  57                 (in-host-compilation-mode
  58                  (lambda ()
  59                    (load source :verbose t :print nil))))))))
  60       (loop (if (plusp subprocess-count) (wait) (return)))))
  61
  62   ;; We want to load compiled files, because that's what this function promises.
  63   ;; Reloading is tricky because constructors for interned ctypes will construct
  64   ;; new objects via their LOAD-TIME-VALUE forms, but globaldb already stored
  65   ;; some objects from the interpreted pre-load.
  66   ;; So wipe everything out that causes problems down the line.
  67   ;; (Or perhaps we could make their effects idempotent)
  68   (format t "~&; Parallel build: Clearing globaldb~%")
  69   (do-all-symbols (s)
  70     (when (get s :sb-xc-globaldb-info)
  71       (remf (symbol-plist s) :sb-xc-globaldb-info)))
  72   (fill (symbol-value 'sb!c::*info-types*) nil)
  73   (clrhash (symbol-value 'sb!kernel::*def!struct-type-make-load-form-fun*))
  74   (clrhash (symbol-value 'sb!kernel::*def!struct-supertype*))
  75   (clrhash (symbol-value 'sb!kernel::*forward-referenced-layouts*))
  76   (setf (symbol-value 'sb!kernel:*type-system-initialized*) nil)
  77   (makunbound 'sb!c::*backend-primitive-type-names*)
  78   (makunbound 'sb!c::*backend-primitive-type-aliases*)
  79
  80   (format t "~&; Parallel build: Reloading compilation artifacts~%")
  81   ;; Now it works to load fasls.
  82   (in-host-compilation-mode
  83    (lambda ()
  84      (handler-bind ((sb-kernel:redefinition-warning #'muffle-warning))
  85        (do-stems-and-flags (stem flags)
  86          (unless (position :not-host flags)
  87            (load (stem-object-path stem flags :host-compile)
  88                  :verbose t :print nil))))))
  89   (format t "~&; Parallel build: Fasl loading complete~%"))
  90
  91 ;;; Either load or compile-then-load the cross-compiler into the
  92 ;;; cross-compilation host Common Lisp.
  93 (defun load-or-cload-xcompiler (load-or-cload-stem)
  94
  95   (declare (type function load-or-cload-stem))
  96
  97   ;; The running-in-the-host-Lisp Python cross-compiler defines its
  98   ;; own versions of a number of functions which should not overwrite
  99   ;; host-Lisp functions. Instead we put them in a special package.
 100   ;;
 101   ;; The common theme of the functions, macros, constants, and so
 102   ;; forth in this package is that they run in the host and affect the
 103   ;; compilation of the target.
 104   (let ((package-name "SB-XC"))
 105     (make-package package-name :use nil :nicknames nil)
 106     (dolist (name '(;; the constants (except for T and NIL which have
 107                     ;; a specially hacked correspondence between
 108                     ;; cross-compilation host Lisp and target Lisp)
 109                     "ARRAY-DIMENSION-LIMIT"
 110                     "ARRAY-RANK-LIMIT"
 111                     "ARRAY-TOTAL-SIZE-LIMIT"
 112                     "BOOLE-1"
 113                     "BOOLE-2"
 114                     "BOOLE-AND"
 115                     "BOOLE-ANDC1"
 116                     "BOOLE-ANDC2"
 117                     "BOOLE-C1"
 118                     "BOOLE-C2"
 119                     "BOOLE-CLR"
 120                     "BOOLE-EQV"
 121                     "BOOLE-IOR"
 122                     "BOOLE-NAND"
 123                     "BOOLE-NOR"
 124                     "BOOLE-ORC1"
 125                     "BOOLE-ORC2"
 126                     "BOOLE-SET"
 127                     "BOOLE-XOR"
 128                     "CALL-ARGUMENTS-LIMIT"
 129                     "CHAR-CODE-LIMIT"
 130                     "DEFMETHOD"
 131                     "DOUBLE-FLOAT-EPSILON"
 132                     "DOUBLE-FLOAT-NEGATIVE-EPSILON"
 133                     "INTERNAL-TIME-UNITS-PER-SECOND"
 134                     "LAMBDA-LIST-KEYWORDS"
 135                     "LAMBDA-PARAMETERS-LIMIT"
 136                     "LEAST-NEGATIVE-DOUBLE-FLOAT"
 137                     "LEAST-NEGATIVE-LONG-FLOAT"
 138                     "LEAST-NEGATIVE-NORMALIZED-DOUBLE-FLOAT"
 139                     "LEAST-NEGATIVE-NORMALIZED-LONG-FLOAT"
 140                     "LEAST-NEGATIVE-NORMALIZED-SHORT-FLOAT"
 141                     "LEAST-NEGATIVE-NORMALIZED-SINGLE-FLOAT"
 142                     "LEAST-NEGATIVE-SHORT-FLOAT"
 143                     "LEAST-NEGATIVE-SINGLE-FLOAT"
 144                     "LEAST-POSITIVE-DOUBLE-FLOAT"
 145                     "LEAST-POSITIVE-LONG-FLOAT"
 146                     "LEAST-POSITIVE-NORMALIZED-DOUBLE-FLOAT"
 147                     "LEAST-POSITIVE-NORMALIZED-LONG-FLOAT"
 148                     "LEAST-POSITIVE-NORMALIZED-SHORT-FLOAT"
 149                     "LEAST-POSITIVE-NORMALIZED-SINGLE-FLOAT"
 150                     "LEAST-POSITIVE-SHORT-FLOAT"
 151                     "LEAST-POSITIVE-SINGLE-FLOAT"
 152                     "LONG-FLOAT-EPSILON"
 153                     "LONG-FLOAT-NEGATIVE-EPSILON"
 154                     "MOST-NEGATIVE-DOUBLE-FLOAT"
 155                     "MOST-NEGATIVE-FIXNUM"
 156                     "MOST-NEGATIVE-LONG-FLOAT"
 157                     "MOST-NEGATIVE-SHORT-FLOAT"
 158                     "MOST-NEGATIVE-SINGLE-FLOAT"
 159                     "MOST-POSITIVE-DOUBLE-FLOAT"
 160                     "MOST-POSITIVE-FIXNUM"
 161                     "MOST-POSITIVE-LONG-FLOAT"
 162                     "MOST-POSITIVE-SHORT-FLOAT"
 163                     "MOST-POSITIVE-SINGLE-FLOAT"
 164                     "MULTIPLE-VALUES-LIMIT"
 165                     "PI"
 166                     "SHORT-FLOAT-EPSILON"
 167                     "SHORT-FLOAT-NEGATIVE-EPSILON"
 168                     "SINGLE-FLOAT-EPSILON"
 169                     "SINGLE-FLOAT-NEGATIVE-EPSILON"
 170
 171                     ;; everything else which needs a separate
 172                     ;; existence in xc and target
 173                     "BOOLE"
 174                     "BOOLE-CLR" "BOOLE-SET" "BOOLE-1" "BOOLE-2"
 175                     "BOOLE-C1" "BOOLE-C2" "BOOLE-AND" "BOOLE-IOR"
 176                     "BOOLE-XOR" "BOOLE-EQV" "BOOLE-NAND" "BOOLE-NOR"
 177                     "BOOLE-ANDC1" "BOOLE-ANDC2" "BOOLE-ORC1" "BOOLE-ORC2"
 178                     "BUILT-IN-CLASS"
 179                     "BYTE" "BYTE-POSITION" "BYTE-SIZE"
 180                     "CHAR-CODE"
 181                     "CLASS" "CLASS-NAME" "CLASS-OF"
 182                     "CODE-CHAR"
 183                     "COMPILE-FILE"
 184                     "COMPILE-FILE-PATHNAME"
 185                     "*COMPILE-FILE-PATHNAME*"
 186                     "*COMPILE-FILE-TRUENAME*"
 187                     "*COMPILE-PRINT*"
 188                     "*COMPILE-VERBOSE*"
 189                     "COMPILER-MACRO-FUNCTION"
 190                     "CONSTANTP"
 191                     "DEFCONSTANT"
 192                     "DEFINE-MODIFY-MACRO"
 193                     "DEFINE-SETF-EXPANDER"
 194                     "DEFMACRO" "DEFSETF" "DEFSTRUCT" "DEFTYPE"
 195                     "DEPOSIT-FIELD" "DPB"
 196                     "FBOUNDP" "FDEFINITION" "FMAKUNBOUND"
 197                     "FIND-CLASS"
 198                     "GENSYM" "*GENSYM-COUNTER*"
 199                     "GET-SETF-EXPANSION"
 200                     "LDB" "LDB-TEST"
 201                     "LISP-IMPLEMENTATION-TYPE" "LISP-IMPLEMENTATION-VERSION"
 202                     "MACRO-FUNCTION"
 203                     "MACROEXPAND" "MACROEXPAND-1" "*MACROEXPAND-HOOK*"
 204                     "MAKE-LOAD-FORM"
 205                     "MAKE-LOAD-FORM-SAVING-SLOTS"
 206                     "MASK-FIELD"
 207                     "PROCLAIM"
 208                     "SPECIAL-OPERATOR-P"
 209                     "STANDARD-CLASS"
 210                     "STRUCTURE-CLASS"
 211                     "SUBTYPEP"
 212                     "TYPE-OF" "TYPEP"
 213                     "UPGRADED-ARRAY-ELEMENT-TYPE"
 214                     "UPGRADED-COMPLEX-PART-TYPE"
 215                     "WITH-COMPILATION-UNIT"))
 216       (export (intern name package-name) package-name)))
 217   ;; Symbols that can't be entered into the whitelist
 218   ;; until this function executes.
 219   (setf (gethash (intern "MAKE-LOAD-FORM" "SB-XC")
 220                  *undefined-fun-whitelist*) t)
 221   ;; don't watch:
 222   (dolist (package (list-all-packages))
 223     (when (= (mismatch (package-name package) "SB!") 3)
 224       (shadowing-import
 225        (mapcar (lambda (name) (find-symbol name "SB-XC"))
 226                '("BYTE" "BYTE-POSITION" "BYTE-SIZE"
 227                  "DPB" "LDB" "LDB-TEST"
 228                  "DEPOSIT-FIELD" "MASK-FIELD"
 229
 230                  "BOOLE"
 231                  "BOOLE-CLR" "BOOLE-SET" "BOOLE-1" "BOOLE-2"
 232                  "BOOLE-C1" "BOOLE-C2" "BOOLE-AND" "BOOLE-IOR"
 233                  "BOOLE-XOR" "BOOLE-EQV" "BOOLE-NAND" "BOOLE-NOR"
 234                  "BOOLE-ANDC1" "BOOLE-ANDC2" "BOOLE-ORC1" "BOOLE-ORC2"))
 235        package)))
 236
 237   ;; Build a version of Python to run in the host Common Lisp, to be
 238   ;; used only in cross-compilation.
 239   ;;
 240   ;; Note that files which are marked :ASSEM, to cause them to be
 241   ;; processed with SB!C:ASSEMBLE-FILE when we're running under the
 242   ;; cross-compiler or the target lisp, are still processed here, just
 243   ;; with the ordinary Lisp compiler, and this is intentional, in
 244   ;; order to make the compiler aware of the definitions of assembly
 245   ;; routines.
 246   (if (and (make-host-1-parallelism)
 247            (eq load-or-cload-stem #'host-cload-stem))
 248       (funcall 'parallel-make-host-1 ; avoid potential style-warning
 249                (make-host-1-parallelism))
 250       (do-stems-and-flags (stem flags)
 251         (unless (find :not-host flags)
 252           (funcall load-or-cload-stem stem flags)
 253           #!+sb-show (warn-when-cl-snapshot-diff *cl-snapshot*))))
 254
 255   ;; If the cross-compilation host is SBCL itself, we can use the
 256   ;; PURIFY extension to freeze everything in place, reducing the
 257   ;; amount of work done on future GCs. In machines with limited
 258   ;; memory, this could help, by reducing the amount of memory which
 259   ;; needs to be juggled in a full GC. And it can hardly hurt, since
 260   ;; (in the ordinary build procedure anyway) essentially everything
 261   ;; which is reachable at this point will remain reachable for the
 262   ;; entire run.
 263   ;;
 264   ;; (Except that purifying actually slows down GENCGC). -- JES, 2006-05-30
 265   #+(and sbcl (not gencgc))
 266   (sb-ext:purify)
 267
 268   (values))
 269
 270 ;; Keep these in order by package, then symbol.
 271 (dolist (sym
 272          (append
 273           ;; CL, EXT, KERNEL
 274           '(allocate-instance
 275             compute-applicable-methods
 276             slot-makunbound
 277             sb!ext:run-program
 278             sb!kernel:profile-deinit)
 279           ;; CLOS implementation
 280           '(sb!mop:class-finalized-p
 281             sb!mop:class-prototype
 282             sb!mop:eql-specializer-object
 283             sb!mop:finalize-inheritance
 284             sb!mop:generic-function-name
 285             sb!pcl::%force-cache-flushes
 286             sb!pcl::check-wrapper-validity
 287             sb!pcl::class-has-a-forward-referenced-superclass-p
 288             sb!pcl::class-wrapper
 289             sb!pcl::compute-gf-ftype
 290             sb!pcl::definition-source
 291             sb!pcl:ensure-class-finalized
 292             sb!pcl::get-instance-hash-code)
 293           ;; CLOS-based packages
 294           '(sb!gray:stream-clear-input
 295             sb!gray:stream-clear-output
 296             sb!gray:stream-file-position
 297             sb!gray:stream-finish-output
 298             sb!gray:stream-force-output
 299             sb!gray:stream-fresh-line
 300             sb!gray:stream-line-column
 301             sb!gray:stream-line-length
 302             sb!gray:stream-listen
 303             sb!gray:stream-peek-char
 304             sb!gray:stream-read-byte
 305             sb!gray:stream-read-char
 306             sb!gray:stream-read-char-no-hang
 307             sb!gray:stream-read-line
 308             sb!gray:stream-read-sequence
 309             sb!gray:stream-terpri
 310             sb!gray:stream-unread-char
 311             sb!gray:stream-write-byte
 312             sb!gray:stream-write-char
 313             sb!gray:stream-write-sequence
 314             sb!gray:stream-write-string
 315             sb!sequence:concatenate
 316             sb!sequence:copy-seq
 317             sb!sequence:count
 318             sb!sequence:count-if
 319             sb!sequence:count-if-not
 320             sb!sequence:delete
 321             sb!sequence:delete-duplicates
 322             sb!sequence:delete-if
 323             sb!sequence:delete-if-not
 324             (setf sb!sequence:elt)
 325             sb!sequence:elt
 326             sb!sequence:emptyp
 327             sb!sequence:fill
 328             sb!sequence:find
 329             sb!sequence:find-if
 330             sb!sequence:find-if-not
 331             (setf sb!sequence:iterator-element)
 332             sb!sequence:iterator-endp
 333             sb!sequence:iterator-step
 334             sb!sequence:length
 335             sb!sequence:make-sequence-iterator
 336             sb!sequence:make-sequence-like
 337             sb!sequence:map
 338             sb!sequence:merge
 339             sb!sequence:mismatch
 340             sb!sequence:nreverse
 341             sb!sequence:nsubstitute
 342             sb!sequence:nsubstitute-if
 343             sb!sequence:nsubstitute-if-not
 344             sb!sequence:position
 345             sb!sequence:position-if
 346             sb!sequence:position-if-not
 347             sb!sequence:reduce
 348             sb!sequence:remove
 349             sb!sequence:remove-duplicates
 350             sb!sequence:remove-if
 351             sb!sequence:remove-if-not
 352             sb!sequence:replace
 353             sb!sequence:reverse
 354             sb!sequence:search
 355             sb!sequence:sort
 356             sb!sequence:stable-sort
 357             sb!sequence:subseq
 358             sb!sequence:substitute
 359             sb!sequence:substitute-if
 360             sb!sequence:substitute-if-not)
 361           ;; Fast interpreter
 362           #!+sb-fasteval
 363           '(sb!interpreter:%fun-type
 364             sb!interpreter:env-policy
 365             sb!interpreter:eval-in-environment
 366             sb!interpreter:find-lexical-fun
 367             sb!interpreter:find-lexical-var
 368             sb!interpreter::flush-everything
 369             sb!interpreter::fun-lexically-notinline-p
 370             sb!interpreter:lexenv-from-env
 371             sb!interpreter:list-locals
 372             sb!interpreter:prepare-for-compile
 373             sb!interpreter::reconstruct-syntactic-closure-env)
 374           ;; Other
 375           '(sb!debug::find-interrupted-name-and-frame
 376             sb!impl::encapsulate-generic-function
 377             sb!impl::encapsulated-generic-function-p
 378             sb!impl::get-processes-status-changes
 379             sb!impl::step-form
 380             sb!impl::step-values
 381             sb!impl::unencapsulate-generic-function)))
 382   (setf (gethash sym *undefined-fun-whitelist*) t))