1 ;;;; This file contains the implementation-independent code for Pack
2 ;;;; phase in the compiler. Pack is responsible for assigning TNs to
3 ;;;; storage allocations or "register allocation".
5 ;;;; This software is part of the SBCL system. See the README file for
8 ;;;; This software is derived from the CMU CL system, which was
9 ;;;; written at Carnegie Mellon University and released into the
10 ;;;; public domain. The software is in the public domain and is
11 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
12 ;;;; files for more information.
14 (in-package "SB!REGALLOC")
16 ;;; for debugging: some parameters controlling which optimizations we
18 (defvar *pack-assign-costs
* t
)
19 (defvar *pack-optimize-saves
* t
)
20 ;;; FIXME: Perhaps SB-FLUID should be renamed to SB-TWEAK and these
21 ;;; should be made conditional on SB-TWEAK.
23 (declaim (ftype (function (component) index
) ir2-block-count
))
25 ;;;; conflict determination
27 ;;; Return true if the element at the specified offset, or in any of
28 ;;; the [size-1] subsequent offsets, in SB has a conflict with TN:
29 ;;; -- If a component-live TN (:COMPONENT kind), then iterate over
30 ;;; all the blocks. If the element at OFFSET is used anywhere in
31 ;;; any of the component's blocks (always-live /= 0), then there
33 ;;; -- If TN is global (Confs true), then iterate over the blocks TN
34 ;;; is live in (using TN-GLOBAL-CONFLICTS). If the TN is live
35 ;;; everywhere in the block (:LIVE), then there is a conflict
36 ;;; if the element at offset is used anywhere in the block
37 ;;; (Always-Live /= 0). Otherwise, we use the local TN number for
38 ;;; TN in block to find whether TN has a conflict at Offset in
40 ;;; -- If TN is local, then we just check for a conflict in the block
43 ;;; If there is a conflict, returns the first such conflicting offset.
44 (defun offset-conflicts-in-sb (tn sb offset
&key
(size 1))
45 (declare (type tn tn
) (type finite-sb sb
) (type index offset size
))
46 (let ((confs (tn-global-conflicts tn
))
48 (sb-conflicts (finite-sb-conflicts sb
))
49 (sb-always-live (finite-sb-always-live sb
)))
50 (macrolet ((do-offsets ((var) &body body
)
51 `(loop for
,var upfrom offset
53 thereis
(progn ,@body
))))
56 (do-offsets (offset-iter)
57 (let ((loc-live (svref sb-always-live offset-iter
)))
58 (dotimes (i (ir2-block-count *component-being-compiled
*))
59 (when (/= (sbit loc-live i
) 0)
60 (return offset-iter
))))))
62 ;; TN is global, iterate over the blocks TN is live in.
63 (do ((conf confs
(global-conflicts-next-tnwise conf
)))
66 (let* ((block (global-conflicts-block conf
))
67 (num (ir2-block-number block
)))
68 (if (eq (global-conflicts-kind conf
) :live
)
69 (do-offsets (offset-iter)
70 (let ((loc-live (svref sb-always-live offset-iter
)))
71 (when (/= (sbit loc-live num
) 0)
72 (return-from offset-conflicts-in-sb offset-iter
))))
73 (do-offsets (offset-iter)
74 (let ((loc-confs (svref sb-conflicts offset-iter
)))
75 (when (/= (sbit (svref loc-confs num
)
76 (global-conflicts-number conf
))
78 (return-from offset-conflicts-in-sb offset-iter
))))))))
80 (do-offsets (offset-iter)
81 (and (/= (sbit (svref (svref sb-conflicts offset-iter
)
82 (ir2-block-number (tn-local tn
)))
87 ;;; Return true if TN has a conflict in SC at the specified offset.
88 (declaim (ftype (sfunction (tn sc index
) (or null index
)) conflicts-in-sc
))
89 (defun conflicts-in-sc (tn sc offset
)
90 (declare (type tn tn
) (type sc sc
) (type index offset
))
91 (offset-conflicts-in-sb tn
(sc-sb sc
) offset
92 :size
(sc-element-size sc
)))
94 ;;; Add TN's conflicts into the conflicts for the location at OFFSET
95 ;;; in SC. We iterate over each location in TN, adding to the
96 ;;; conflicts for that location:
97 ;;; -- If TN is a :COMPONENT TN, then iterate over all the blocks,
98 ;;; setting all of the local conflict bits and the always-live bit.
99 ;;; This records a conflict with any TN that has a LTN number in
100 ;;; the block, as well as with :ALWAYS-LIVE and :ENVIRONMENT TNs.
101 ;;; -- If TN is global, then iterate over the blocks TN is live in. In
102 ;;; addition to setting the always-live bit to represent the conflict
103 ;;; with TNs live throughout the block, we also set bits in the
104 ;;; local conflicts. If TN is :ALWAYS-LIVE in the block, we set all
105 ;;; the bits, otherwise we OR in the local conflict bits.
106 ;;; -- If the TN is local, then we just do the block it is local to,
107 ;;; setting always-live and OR'ing in the local conflicts.
108 (defun add-location-conflicts (tn sc offset optimize
)
109 (declare (type tn tn
) (type sc sc
) (type index offset
))
110 (let ((confs (tn-global-conflicts tn
))
113 (dotimes (i (sc-element-size sc
))
114 (declare (type index i
))
115 (let* ((this-offset (+ offset i
))
116 (loc-confs (svref (finite-sb-conflicts sb
) this-offset
))
117 (loc-live (svref (finite-sb-always-live sb
) this-offset
)))
119 ((eq kind
:component
)
120 (dotimes (num (ir2-block-count *component-being-compiled
*))
121 (declare (type index num
))
122 (setf (sbit loc-live num
) 1)
123 (set-bit-vector (svref loc-confs num
))))
125 (do ((conf confs
(global-conflicts-next-tnwise conf
)))
127 (let* ((block (global-conflicts-block conf
))
128 (num (ir2-block-number block
))
129 (local-confs (svref loc-confs num
)))
130 (declare (type local-tn-bit-vector local-confs
))
131 (setf (sbit loc-live num
) 1)
132 (if (eq (global-conflicts-kind conf
) :live
)
133 (set-bit-vector local-confs
)
134 (bit-ior local-confs
(global-conflicts-conflicts conf
) t
)))))
136 (let ((num (ir2-block-number (tn-local tn
))))
137 (setf (sbit loc-live num
) 1)
138 (bit-ior (the local-tn-bit-vector
(svref loc-confs num
))
139 (tn-local-conflicts tn
) t
))))
140 ;; Calculating ALWAYS-LIVE-COUNT is moderately expensive, and
141 ;; currently the information isn't used unless (> SPEED
144 (setf (svref (finite-sb-always-live-count sb
) this-offset
)
145 (find-location-usage sb this-offset
))))))
148 ;; A rought measure of how much a given OFFSET in SB is currently
149 ;; used. Current implementation counts the amount of blocks where the
150 ;; offset has been marked as ALWAYS-LIVE.
151 (defun find-location-usage (sb offset
)
152 (declare (optimize speed
))
153 (declare (type sb sb
) (type index offset
))
154 (let* ((always-live (svref (finite-sb-always-live sb
) offset
)))
155 (declare (simple-bit-vector always-live
))
156 (count 1 always-live
)))
158 ;;; Return the total number of IR2-BLOCKs in COMPONENT.
159 (defun ir2-block-count (component)
160 (declare (type component component
))
161 (do ((2block (block-info (block-next (component-head component
)))
162 (ir2-block-next 2block
)))
164 (error "What? No ir2 blocks have a non-nil number?"))
165 (when (ir2-block-number 2block
)
166 (return (1+ (ir2-block-number 2block
))))))
168 ;;; Ensure that the conflicts vectors for each :FINITE SB are large
169 ;;; enough for the number of blocks allocated. Also clear any old
170 ;;; conflicts and reset the current size to the initial size.
171 (defun init-sb-vectors (component)
172 (let ((nblocks (ir2-block-count component
)))
173 (dolist (sb *backend-sb-list
*)
174 (unless (eq (sb-kind sb
) :non-packed
)
175 (let* ((conflicts (finite-sb-conflicts sb
))
176 (always-live (finite-sb-always-live sb
))
177 (always-live-count (finite-sb-always-live-count sb
))
178 (max-locs (length conflicts
))
179 (last-count (finite-sb-last-block-count sb
)))
180 (unless (zerop max-locs
)
181 (let ((current-size (length (the simple-vector
182 (svref conflicts
0)))))
184 ((> nblocks current-size
)
185 (let ((new-size (max nblocks
(* current-size
2))))
186 (declare (type index new-size
))
187 (dotimes (i max-locs
)
188 (declare (type index i
))
189 (let ((new-vec (make-array new-size
)))
190 (let ((old (svref conflicts i
)))
191 (declare (simple-vector old
))
192 (dotimes (j current-size
)
193 (declare (type index j
))
194 (setf (svref new-vec j
)
195 (clear-bit-vector (svref old j
)))))
197 (do ((j current-size
(1+ j
)))
199 (declare (type index j
))
200 (setf (svref new-vec j
)
201 (make-array local-tn-limit
:element-type
'bit
202 :initial-element
0)))
203 (setf (svref conflicts i
) new-vec
))
204 (setf (svref always-live i
)
205 (make-array new-size
:element-type
'bit
207 (setf (svref always-live-count i
) 0))))
209 (dotimes (i (finite-sb-current-size sb
))
210 (declare (type index i
))
211 (let ((conf (svref conflicts i
)))
212 (declare (simple-vector conf
))
213 (dotimes (j last-count
)
214 (declare (type index j
))
215 (clear-bit-vector (svref conf j
))))
216 (clear-bit-vector (svref always-live i
))
217 (setf (svref always-live-count i
) 0))))))
219 (setf (finite-sb-last-block-count sb
) nblocks
)
220 (setf (finite-sb-current-size sb
) (sb-size sb
))
221 (setf (finite-sb-last-offset sb
) 0))))))
223 ;;; Expand the :UNBOUNDED SB backing SC by either the initial size or
224 ;;; the SC element size, whichever is larger. If NEEDED-SIZE is
225 ;;; larger, then use that size.
226 (defun grow-sc (sc &optional
(needed-size 0))
227 (declare (type sc sc
) (type index needed-size
))
228 (let* ((sb (sc-sb sc
))
229 (size (finite-sb-current-size sb
))
230 (align-mask (1- (sc-alignment sc
)))
231 (inc (max (finite-sb-size-increment sb
)
232 (+ (sc-element-size sc
)
233 (- (logandc2 (+ size align-mask
) align-mask
)
235 (- needed-size size
)))
236 (new-size (let ((align-mask (1- (finite-sb-size-alignment sb
))))
237 (logandc2 (+ size inc align-mask
) align-mask
)))
238 (conflicts (finite-sb-conflicts sb
))
239 (block-size (if (zerop (length conflicts
))
240 (ir2-block-count *component-being-compiled
*)
241 (length (the simple-vector
(svref conflicts
0)))))
242 (padded-size (ash 1 (integer-length (1- new-size
)))))
243 (declare (type index inc new-size padded-size
))
244 (aver (eq (sb-kind sb
) :unbounded
))
246 (when (> padded-size
(length conflicts
))
247 (let ((new-conf (make-array padded-size
)))
248 (replace new-conf conflicts
)
249 (do ((i size
(1+ i
)))
251 (declare (type index i
))
252 (let ((loc-confs (make-array block-size
)))
253 (dotimes (j block-size
)
254 (setf (svref loc-confs j
)
255 (make-array local-tn-limit
257 :element-type
'bit
)))
258 (setf (svref new-conf i
) loc-confs
)))
259 (setf (finite-sb-conflicts sb
) new-conf
))
261 (let ((new-live (make-array padded-size
)))
262 (replace new-live
(finite-sb-always-live sb
))
263 (do ((i size
(1+ i
)))
265 (setf (svref new-live i
)
266 (make-array block-size
268 :element-type
'bit
)))
269 (setf (finite-sb-always-live sb
) new-live
))
271 (let ((new-live-count (make-array padded-size
)))
272 (declare (optimize speed
)) ;; FILL deftransform
273 (replace new-live-count
(finite-sb-always-live-count sb
))
274 (fill new-live-count
0 :start size
)
275 (setf (finite-sb-always-live-count sb
) new-live-count
))
277 (let ((new-tns (make-array padded-size
:initial-element nil
)))
278 (replace new-tns
(finite-sb-live-tns sb
))
279 (fill (finite-sb-live-tns sb
) nil
)
280 (setf (finite-sb-live-tns sb
) new-tns
)))
282 (setf (finite-sb-current-size sb
) new-size
))
288 ;;; Give someone a hard time because there isn't any load function
289 ;;; defined to move from SRC to DEST.
290 (defun no-load-fun-error (src dest
)
291 (let* ((src-sc (tn-sc src
))
292 (src-name (sc-name src-sc
))
293 (dest-sc (tn-sc dest
))
294 (dest-name (sc-name dest-sc
)))
295 (cond ((eq (sb-kind (sc-sb src-sc
)) :non-packed
)
296 (unless (member src-sc
(sc-constant-scs dest-sc
))
297 (error "loading from an invalid constant SC?~@
298 VM definition inconsistent, try recompiling."))
299 (error "no load function defined to load SC ~S ~
300 from its constant SC ~S"
302 ((member src-sc
(sc-alternate-scs dest-sc
))
303 (error "no load function defined to load SC ~S from its ~
306 ((member dest-sc
(sc-alternate-scs src-sc
))
307 (error "no load function defined to save SC ~S in its ~
311 ;; FIXME: "VM definition is inconsistent" shouldn't be a
312 ;; possibility in SBCL.
313 (error "loading to/from SCs that aren't alternates?~@
314 VM definition is inconsistent, try recompiling.")))))
316 ;;; Called when we failed to pack TN. If RESTRICTED is true, then we
317 ;;; are restricted to pack TN in its SC.
318 (defun failed-to-pack-error (tn restricted
)
319 (declare (type tn tn
))
320 (let* ((sc (tn-sc tn
))
321 (scs (cons sc
(sc-alternate-scs sc
))))
324 (error "failed to pack restricted TN ~S in its SC ~S"
327 (aver (not (find :unbounded scs
328 :key
(lambda (x) (sb-kind (sc-sb x
))))))
329 (let ((ptype (tn-primitive-type tn
)))
332 (aver (member (sc-number sc
) (primitive-type-scs ptype
)))
333 (error "SC ~S doesn't have any :UNBOUNDED alternate SCs, but is~@
334 a SC for primitive-type ~S."
335 (sc-name sc
) (primitive-type-name ptype
)))
337 (error "SC ~S doesn't have any :UNBOUNDED alternate SCs."
340 ;;; Return a list of format arguments describing how TN is used in
342 (defun describe-tn-use (loc tn op
)
343 (let* ((vop (tn-ref-vop op
))
344 (args (vop-args vop
))
345 (results (vop-results vop
))
346 (name (with-simple-output-to-string (stream)
347 (print-tn-guts tn stream
)))
348 (2comp (component-info *component-being-compiled
*))
351 ((setq temp
(position-in #'tn-ref-across tn args
:key
#'tn-ref-tn
))
352 `("~2D: ~A (~:R argument)" ,loc
,name
,(1+ temp
)))
353 ((setq temp
(position-in #'tn-ref-across tn results
:key
#'tn-ref-tn
))
354 `("~2D: ~A (~:R result)" ,loc
,name
,(1+ temp
)))
355 ((setq temp
(position-in #'tn-ref-across tn args
:key
#'tn-ref-load-tn
))
356 `("~2D: ~A (~:R argument load TN)" ,loc
,name
,(1+ temp
)))
357 ((setq temp
(position-in #'tn-ref-across tn results
:key
359 `("~2D: ~A (~:R result load TN)" ,loc
,name
,(1+ temp
)))
360 ((setq temp
(position-in #'tn-ref-across tn
(vop-temps vop
)
362 `("~2D: ~A (temporary ~A)" ,loc
,name
363 ,(operand-parse-name (elt (vop-parse-temps
365 (vop-info-name (vop-info vop
))))
367 ((eq (tn-kind tn
) :component
)
368 `("~2D: ~A (component live)" ,loc
,name
))
369 ((position-in #'tn-next tn
(ir2-component-wired-tns 2comp
))
370 `("~2D: ~A (wired)" ,loc
,name
))
371 ((position-in #'tn-next tn
(ir2-component-restricted-tns 2comp
))
372 `("~2D: ~A (restricted)" ,loc
,name
))
374 `("~2D: not referenced?" ,loc
)))))
376 ;;; If load TN packing fails, try to give a helpful error message. We
377 ;;; find a TN in each location that conflicts, and print it.
378 (defun failed-to-pack-load-tn-error (scs op
)
379 (declare (list scs
) (type tn-ref op
))
383 (let* ((sb (sc-sb sc
))
384 (confs (finite-sb-live-tns sb
)))
385 (aver (eq (sb-kind sb
) :finite
))
386 (dolist (el (sc-locations sc
))
387 (declare (type index el
))
388 (let ((conf (load-tn-conflicts-in-sc op sc el t
)))
390 (used (describe-tn-use el conf op
))
392 (end (+ el
(sc-element-size sc
))))
395 (declare (type index i end
))
396 (let ((victim (svref confs i
)))
398 (used (describe-tn-use el victim op
))
401 (multiple-value-bind (arg-p n more-p costs load-scs incon
)
402 (get-operand-info op
)
403 (declare (ignore costs load-scs
))
405 (error "unable to pack a Load-TN in SC ~{~A~#[~^~;, or ~:;,~]~} ~
406 for the ~:R ~:[result~;argument~] to~@
408 ~:[since all SC elements are in use:~:{~%~@?~}~%~;~
409 ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~
411 Current cost info inconsistent with that in effect at compile ~
412 time. Recompile.~%Compilation order may be incorrect.~]"
413 (mapcar #'sc-name scs
)
415 (vop-info-name (vop-info (tn-ref-vop op
)))
419 ;;; This is called when none of the SCs that we can load OP into are
420 ;;; allowed by OP's primitive-type.
421 (defun no-load-scs-allowed-by-primitive-type-error (ref)
422 (declare (type tn-ref ref
))
423 (let* ((tn (tn-ref-tn ref
))
424 (ptype (tn-primitive-type tn
)))
425 (multiple-value-bind (arg-p pos more-p costs load-scs incon
)
426 (get-operand-info ref
)
427 (declare (ignore costs
))
429 (error "~S is not valid as the ~:R ~:[result~;argument~] to VOP:~
431 since the TN's primitive type ~S doesn't allow any of the SCs~@
432 allowed by the operand restriction:~% ~S~
434 Current cost info inconsistent with that in effect at compile ~
435 time. Recompile.~%Compilation order may be incorrect.~]"
437 (template-name (vop-info (tn-ref-vop ref
)))
438 (primitive-type-name ptype
)
439 (mapcar #'sc-name
(listify-restrictions load-scs
))
444 ;;; Do stuff to note that TN is spilled at VOP for the debugger's benefit.
445 (defun note-spilled-tn (tn vop
)
446 (when (and (tn-leaf tn
) (vop-save-set vop
))
447 (let ((2comp (component-info *component-being-compiled
*)))
448 (setf (gethash tn
(ir2-component-spilled-tns 2comp
)) t
)
449 (pushnew tn
(gethash vop
(ir2-component-spilled-vops 2comp
)))))
452 ;;; Make a save TN for TN, pack it, and return it. We copy various
453 ;;; conflict information from the TN so that pack does the right
455 (defun pack-save-tn (tn)
456 (declare (type tn tn
))
457 (let ((res (make-tn 0 :save nil nil
)))
458 (dolist (alt (sc-alternate-scs (tn-sc tn
))
459 (error "no unbounded alternate for SC ~S"
460 (sc-name (tn-sc tn
))))
461 (when (eq (sb-kind (sc-sb alt
)) :unbounded
)
462 (setf (tn-save-tn tn
) res
)
463 (setf (tn-save-tn res
) tn
)
464 (setf (tn-sc res
) alt
)
468 ;;; Find the load function for moving from SRC to DEST and emit a
469 ;;; MOVE-OPERAND VOP with that function as its info arg.
470 (defun emit-operand-load (node block src dest before
)
471 (declare (type node node
) (type ir2-block block
)
472 (type tn src dest
) (type (or vop null
) before
))
473 (emit-load-template node block
474 (template-or-lose 'move-operand
)
476 (list (or (svref (sc-move-funs (tn-sc dest
))
477 (sc-number (tn-sc src
)))
478 (no-load-fun-error src dest
)))
482 ;;; Find the preceding use of the VOP NAME in the emit order, starting
483 ;;; with VOP. We must find the VOP in the same IR1 block.
484 (defun reverse-find-vop (name vop
)
485 (do* ((block (vop-block vop
) (ir2-block-prev block
))
486 (last vop
(ir2-block-last-vop block
)))
488 (aver (eq (ir2-block-block block
) (ir2-block-block (vop-block vop
))))
489 (do ((current last
(vop-prev current
)))
491 (when (eq (vop-info-name (vop-info current
)) name
)
492 (return-from reverse-find-vop current
)))))
494 ;;; For TNs that have other than one writer, we save the TN before
495 ;;; each call. If a local call (MOVE-ARGS is :LOCAL-CALL), then we
496 ;;; scan back for the ALLOCATE-FRAME VOP, and emit the save there.
497 ;;; This is necessary because in a self-recursive local call, the
498 ;;; registers holding the current arguments may get trashed by setting
499 ;;; up the call arguments. The ALLOCATE-FRAME VOP marks a place at
500 ;;; which the values are known to be good.
501 (defun save-complex-writer-tn (tn vop
)
502 (let ((save (or (tn-save-tn tn
)
504 (node (vop-node vop
))
505 (block (vop-block vop
))
506 (next (vop-next vop
)))
507 (when (eq (tn-kind save
) :specified-save
)
508 (setf (tn-kind save
) :save
))
509 (aver (eq (tn-kind save
) :save
))
510 (emit-operand-load node block tn save
511 (if (eq (vop-info-move-args (vop-info vop
))
513 (reverse-find-vop 'allocate-frame vop
)
515 (emit-operand-load node block save tn next
)))
517 ;;; Return a VOP after which is an OK place to save the value of TN.
518 ;;; For correctness, it is only required that this location be after
519 ;;; any possible write and before any possible restore location.
521 ;;; In practice, we return the unique writer VOP, but give up if the
522 ;;; TN is ever read by a VOP with MOVE-ARGS :LOCAL-CALL. This prevents
523 ;;; us from being confused by non-tail local calls.
525 ;;; When looking for writes, we have to ignore uses of MOVE-OPERAND,
526 ;;; since they will correspond to restores that we have already done.
527 (defun find-single-writer (tn)
528 (declare (type tn tn
))
529 (do ((write (tn-writes tn
) (tn-ref-next write
))
533 (do ((read (tn-reads tn
) (tn-ref-next read
)))
535 (when (eq (vop-info-move-args
542 (unless (eq (vop-info-name (vop-info (tn-ref-vop write
)))
544 (when res
(return nil
))
547 ;;; Try to save TN at a single location. If we succeed, return T,
549 (defun save-single-writer-tn (tn)
550 (declare (type tn tn
))
551 (let* ((old-save (tn-save-tn tn
))
552 (save (or old-save
(pack-save-tn tn
)))
553 (writer (find-single-writer tn
)))
556 (eq (tn-kind old-save
) :specified-save
)))
557 (emit-operand-load (vop-node writer
) (vop-block writer
)
558 tn save
(vop-next writer
))
559 (setf (tn-kind save
) :save-once
)
562 ;;; Restore a TN with a :SAVE-ONCE save TN.
563 (defun restore-single-writer-tn (tn vop
)
564 (declare (type tn
) (type vop vop
))
565 (let ((save (tn-save-tn tn
)))
566 (aver (eq (tn-kind save
) :save-once
))
567 (emit-operand-load (vop-node vop
) (vop-block vop
) save tn
(vop-next vop
)))
570 ;;; Save a single TN that needs to be saved, choosing save-once if
571 ;;; appropriate. This is also called by SPILL-AND-PACK-LOAD-TN.
572 (defun basic-save-tn (tn vop
)
573 (declare (type tn tn
) (type vop vop
))
574 (let ((save (tn-save-tn tn
)))
575 (cond ((and save
(eq (tn-kind save
) :save-once
))
576 (restore-single-writer-tn tn vop
))
577 ((save-single-writer-tn tn
)
578 (restore-single-writer-tn tn vop
))
580 (save-complex-writer-tn tn vop
))))
583 ;;; Scan over the VOPs in BLOCK, emiting saving code for TNs noted in
584 ;;; the codegen info that are packed into saved SCs.
585 (defun emit-saves (block)
586 (declare (type ir2-block block
))
587 (do ((vop (ir2-block-start-vop block
) (vop-next vop
)))
589 (when (eq (vop-info-save-p (vop-info vop
)) t
)
590 (do-live-tns (tn (vop-save-set vop
) block
)
591 (when (and (sc-save-p (tn-sc tn
))
592 (not (eq (tn-kind tn
) :component
))
593 ;; Ignore closed over but not read values (due to
596 (basic-save-tn tn vop
)))))
600 ;;;; optimized saving
602 ;;; Save TN if it isn't a single-writer TN that has already been
603 ;;; saved. If multi-write, we insert the save BEFORE the specified
604 ;;; VOP. CONTEXT is a VOP used to tell which node/block to use for the
606 (defun save-if-necessary (tn before context
)
607 (declare (type tn tn
) (type (or vop null
) before
) (type vop context
))
608 (let ((save (tn-save-tn tn
)))
609 (when (eq (tn-kind save
) :specified-save
)
610 (setf (tn-kind save
) :save
))
611 (aver (member (tn-kind save
) '(:save
:save-once
)))
612 (unless (eq (tn-kind save
) :save-once
)
613 (or (save-single-writer-tn tn
)
614 (emit-operand-load (vop-node context
) (vop-block context
)
618 ;;; Load the TN from its save location, allocating one if necessary.
619 ;;; The load is inserted BEFORE the specified VOP. CONTEXT is a VOP
620 ;;; used to tell which node/block to use for the new VOP.
621 (defun restore-tn (tn before context
)
622 (declare (type tn tn
) (type (or vop null
) before
) (type vop context
))
623 (let ((save (or (tn-save-tn tn
) (pack-save-tn tn
))))
624 (emit-operand-load (vop-node context
) (vop-block context
)
628 ;;; Start scanning backward at the end of BLOCK, looking which TNs are
629 ;;; live and looking for places where we have to save. We manipulate
630 ;;; two sets: SAVES and RESTORES.
632 ;;; SAVES is a set of all the TNs that have to be saved because they
633 ;;; are restored after some call. We normally delay saving until the
634 ;;; beginning of the block, but we must save immediately if we see a
635 ;;; write of the saved TN. We also immediately save all TNs and exit
636 ;;; when we see a NOTE-ENVIRONMENT-START VOP, since saves can't be
637 ;;; done before the environment is properly initialized.
639 ;;; RESTORES is a set of all the TNs read (and not written) between
640 ;;; here and the next call, i.e. the set of TNs that must be restored
641 ;;; when we reach the next (earlier) call VOP. Unlike SAVES, this set
642 ;;; is cleared when we do the restoring after a call. Any TNs that
643 ;;; were in RESTORES are moved into SAVES to ensure that they are
644 ;;; saved at some point.
646 ;;; SAVES and RESTORES are represented using both a list and a
647 ;;; bit-vector so that we can quickly iterate and test for membership.
648 ;;; The incoming SAVES and RESTORES args are used for computing these
649 ;;; sets (the initial contents are ignored.)
651 ;;; When we hit a VOP with :COMPUTE-ONLY SAVE-P (an internal error
652 ;;; location), we pretend that all live TNs were read, unless (= speed
653 ;;; 3), in which case we mark all the TNs that are live but not
654 ;;; restored as spilled.
655 (defun optimized-emit-saves-block (block saves restores
)
656 (declare (type ir2-block block
) (type simple-bit-vector saves restores
))
657 (let ((1block (ir2-block-block block
))
661 (declare (list saves-list restores-list
))
662 (clear-bit-vector saves
)
663 (clear-bit-vector restores
)
664 (do-live-tns (tn (ir2-block-live-in block
) block
)
665 (when (and (sc-save-p (tn-sc tn
))
666 (not (eq (tn-kind tn
) :component
)))
667 (let ((num (tn-number tn
)))
668 (setf (sbit restores num
) 1)
669 (push tn restores-list
))))
671 (do ((block block
(ir2-block-prev block
))
673 ((not (eq (ir2-block-block block
) 1block
))
674 (aver (not skipping
))
675 (dolist (save saves-list
)
676 (let ((start (ir2-block-start-vop prev
)))
677 (save-if-necessary save start start
)))
679 (do ((vop (ir2-block-last-vop block
) (vop-prev vop
)))
681 (let ((info (vop-info vop
)))
682 (case (vop-info-name info
)
686 (note-environment-start
687 (aver (not skipping
))
688 (dolist (save saves-list
)
689 (save-if-necessary save
(vop-next vop
) vop
))
690 (return-from optimized-emit-saves-block block
)))
693 (do ((write (vop-results vop
) (tn-ref-across write
)))
695 (let* ((tn (tn-ref-tn write
))
696 (num (tn-number tn
)))
697 (unless (zerop (sbit restores num
))
698 (setf (sbit restores num
) 0)
700 (delete tn restores-list
:test
#'eq
)))
701 (unless (zerop (sbit saves num
))
702 (setf (sbit saves num
) 0)
703 (save-if-necessary tn
(vop-next vop
) vop
)
705 (delete tn saves-list
:test
#'eq
))))))
707 (macrolet ((save-note-read (tn)
709 (num (tn-number tn
)))
710 (when (and (sc-save-p (tn-sc tn
))
711 (zerop (sbit restores num
))
712 (not (eq (tn-kind tn
) :component
)))
713 (setf (sbit restores num
) 1)
714 (push tn restores-list
)))))
716 (case (vop-info-save-p info
)
718 (dolist (tn restores-list
)
719 ;; Ignore closed over but not read values (due to
722 (restore-tn tn
(vop-next vop
) vop
)
723 (let ((num (tn-number tn
)))
724 (when (zerop (sbit saves num
))
726 (setf (sbit saves num
) 1)))))
727 (setq restores-list nil
)
728 (clear-bit-vector restores
))
730 (cond ((policy (vop-node vop
) (= speed
3))
731 (do-live-tns (tn (vop-save-set vop
) block
)
732 (when (zerop (sbit restores
(tn-number tn
)))
733 (note-spilled-tn tn vop
))))
735 (do-live-tns (tn (vop-save-set vop
) block
)
736 (save-note-read tn
))))))
738 (if (eq (vop-info-move-args info
) :local-call
)
740 (do ((read (vop-args vop
) (tn-ref-across read
)))
742 (save-note-read (tn-ref-tn read
))))))))))
744 ;;; This is like EMIT-SAVES, only different. We avoid redundant saving
745 ;;; within the block, and don't restore values that aren't used before
746 ;;; the next call. This function is just the top level loop over the
747 ;;; blocks in the component, which locates blocks that need saving
749 (defun optimized-emit-saves (component)
750 (declare (type component component
))
751 (let* ((gtn-count (1+ (ir2-component-global-tn-counter
752 (component-info component
))))
753 (saves (make-array gtn-count
:element-type
'bit
))
754 (restores (make-array gtn-count
:element-type
'bit
))
755 (block (ir2-block-prev (block-info (component-tail component
))))
756 (head (block-info (component-head component
))))
758 (when (eq block head
) (return))
759 (when (do ((vop (ir2-block-start-vop block
) (vop-next vop
)))
761 (when (eq (vop-info-save-p (vop-info vop
)) t
)
763 (setq block
(optimized-emit-saves-block block saves restores
)))
764 (setq block
(ir2-block-prev block
)))))
766 ;;; Iterate over the normal TNs, finding the cost of packing on the
767 ;;; stack in units of the number of references. We count all read
768 ;;; references as +1, write references as + *tn-write-cost*, and
769 ;;; subtract out REGISTER-SAVE-PENALTY for each place where we would
770 ;;; have to save a register.
771 ;;; The subtraction reflects the fact that having a value in a
772 ;;; register around a call means that code to spill and unspill must
775 ;;; The costs also take into account the loop depth at which each
776 ;;; reference occurs: the penalty or cost is incremented by the depth
777 ;;; scaled by *tn-loop-depth-multiplier*. The default (NIL) is to let
778 ;;; this be one more than the max of the cost for reads (1), for write
779 ;;; references and for being live across a call.
780 (defvar *tn-write-cost
* 2)
781 (defvar *tn-loop-depth-multiplier
* nil
)
783 (defun assign-tn-costs (component)
784 (let* ((save-penalty *backend-register-save-penalty
*)
785 (write-cost *tn-write-cost
*)
786 (depth-scale (or *tn-loop-depth-multiplier
*
787 (1+ (max 1 write-cost save-penalty
)))))
788 (flet ((vop-depth-cost (vop)
789 (let ((loop (block-loop
793 (* depth-scale
(loop-depth loop
))
795 (do-ir2-blocks (block component
)
796 (do ((vop (ir2-block-start-vop block
) (vop-next vop
)))
798 (when (eq (vop-info-save-p (vop-info vop
)) t
)
799 (let ((penalty (+ save-penalty
(vop-depth-cost vop
))))
800 (do-live-tns (tn (vop-save-set vop
) block
)
801 (decf (tn-cost tn
) penalty
))))))
803 (do ((tn (ir2-component-normal-tns (component-info component
))
806 (let ((cost (tn-cost tn
)))
807 (declare (fixnum cost
))
808 (do ((ref (tn-reads tn
) (tn-ref-next ref
)))
810 (incf cost
(1+ (vop-depth-cost (tn-ref-vop ref
)))))
811 (do ((ref (tn-writes tn
) (tn-ref-next ref
)))
813 (incf cost
(+ write-cost
(vop-depth-cost (tn-ref-vop ref
)))))
814 (setf (tn-cost tn
) cost
))))))
816 ;;; Iterate over the normal TNs, folding over the depth of the looops
817 ;;; that the TN is used in and storing the result in TN-LOOP-DEPTH.
818 ;;: reducer is the function used to join depth values together. #'max
819 ;;; gives the maximum depth, #'+ the sum.
820 (defun assign-tn-depths (component &key
(reducer #'max
))
821 (declare (type function reducer
))
823 ;; We only use tn depth for normal TNs
824 (do ((tn (ir2-component-normal-tns (component-info component
))
828 (declare (type fixnum depth
))
830 (declare (type (or null tn-ref
) ref
))
831 (do ((ref ref
(tn-ref-next ref
)))
833 (let* ((vop (tn-ref-vop ref
))
834 (block (ir2-block-block (vop-block vop
)))
835 (loop (block-loop block
)))
836 (setf depth
(funcall reducer
842 (frob (tn-writes tn
))
843 (setf (tn-loop-depth tn
) depth
))))))
847 ;;; These variables indicate the last location at which we computed
848 ;;; the Live-TNs. They hold the BLOCK and VOP values that were passed
849 ;;; to COMPUTE-LIVE-TNS.
850 (defvar *live-block
*)
853 ;;; If we unpack some TNs, then we mark all affected blocks by
854 ;;; sticking them in this hash-table. This is initially null. We
855 ;;; create the hashtable if we do any unpacking.
856 (defvar *repack-blocks
*)
857 (declaim (type list
*repack-blocks
*))
859 ;;; Set the LIVE-TNS vectors in all :FINITE SBs to represent the TNs
860 ;;; live at the end of BLOCK.
861 (defun init-live-tns (block)
862 (dolist (sb *backend-sb-list
*)
863 (when (eq (sb-kind sb
) :finite
)
864 (fill (finite-sb-live-tns sb
) nil
)))
866 (do-live-tns (tn (ir2-block-live-in block
) block
)
867 (let* ((sc (tn-sc tn
))
869 (when (eq (sb-kind sb
) :finite
)
870 ;; KLUDGE: we can have "live" TNs that are neither read
871 ;; to nor written from, due to more aggressive (type-
872 ;; directed) constant propagation. Such TNs will never
873 ;; be assigned an offset nor be in conflict with anything.
875 ;; Ideally, it seems to me we could make sure these TNs
876 ;; are never allocated in the first place in
877 ;; ASSIGN-LAMBDA-VAR-TNS.
879 (do ((offset (tn-offset tn
) (1+ offset
))
880 (end (+ (tn-offset tn
) (sc-element-size sc
))))
882 (declare (type index offset end
))
883 (setf (svref (finite-sb-live-tns sb
) offset
) tn
))
884 (aver (and (null (tn-reads tn
)) (null (tn-writes tn
))))))))
886 (setq *live-block
* block
)
887 (setq *live-vop
* (ir2-block-last-vop block
))
891 ;;; Set the LIVE-TNs in :FINITE SBs to represent the TNs live
892 ;;; immediately after the evaluation of VOP in BLOCK, excluding
893 ;;; results of the VOP. If VOP is null, then compute the live TNs at
894 ;;; the beginning of the block. Sequential calls on the same block
895 ;;; must be in reverse VOP order.
896 (defun compute-live-tns (block vop
)
897 (declare (type ir2-block block
) (type vop vop
))
898 (unless (eq block
*live-block
*)
899 (init-live-tns block
))
901 (do ((current *live-vop
* (vop-prev current
)))
903 (do ((res (vop-results vop
) (tn-ref-across res
)))
905 (let* ((tn (tn-ref-tn res
))
908 (when (eq (sb-kind sb
) :finite
)
909 (do ((offset (tn-offset tn
) (1+ offset
))
910 (end (+ (tn-offset tn
) (sc-element-size sc
))))
912 (declare (type index offset end
))
913 (setf (svref (finite-sb-live-tns sb
) offset
) nil
))))))
914 (do ((ref (vop-refs current
) (tn-ref-next-ref ref
)))
916 (let ((ltn (tn-ref-load-tn ref
)))
918 (let* ((sc (tn-sc ltn
))
920 (when (eq (sb-kind sb
) :finite
)
921 (let ((tns (finite-sb-live-tns sb
)))
922 (do ((offset (tn-offset ltn
) (1+ offset
))
923 (end (+ (tn-offset ltn
) (sc-element-size sc
))))
925 (declare (type index offset end
))
926 (aver (null (svref tns offset
)))))))))
928 (let* ((tn (tn-ref-tn ref
))
931 (when (eq (sb-kind sb
) :finite
)
932 (let ((tns (finite-sb-live-tns sb
)))
933 (do ((offset (tn-offset tn
) (1+ offset
))
934 (end (+ (tn-offset tn
) (sc-element-size sc
))))
936 (declare (type index offset end
))
937 (if (tn-ref-write-p ref
)
938 (setf (svref tns offset
) nil
)
939 (let ((old (svref tns offset
)))
940 (aver (or (null old
) (eq old tn
)))
941 (setf (svref tns offset
) tn
)))))))))
943 (setq *live-vop
* vop
)
946 ;;; This is kind of like OFFSET-CONFLICTS-IN-SB, except that it uses
947 ;;; the VOP refs to determine whether a Load-TN for OP could be packed
948 ;;; in the specified location, disregarding conflicts with TNs not
949 ;;; referenced by this VOP. There is a conflict if either:
950 ;;; 1. The reference is a result, and the same location is either:
951 ;;; -- Used by some other result.
952 ;;; -- Used in any way after the reference (exclusive).
953 ;;; 2. The reference is an argument, and the same location is either:
954 ;;; -- Used by some other argument.
955 ;;; -- Used in any way before the reference (exclusive).
957 ;;; In 1 (and 2) above, the first bullet corresponds to result-result
958 ;;; (and argument-argument) conflicts. We need this case because there
959 ;;; aren't any TN-REFs to represent the implicit reading of results or
960 ;;; writing of arguments.
962 ;;; The second bullet corresponds to conflicts with temporaries or
963 ;;; between arguments and results.
965 ;;; We consider both the TN-REF-TN and the TN-REF-LOAD-TN (if any) to
966 ;;; be referenced simultaneously and in the same way. This causes
967 ;;; load-TNs to appear live to the beginning (or end) of the VOP, as
970 ;;; We return a conflicting TN if there is a conflict.
971 (defun load-tn-offset-conflicts-in-sb (op sb offset
)
972 (declare (type tn-ref op
) (type finite-sb sb
) (type index offset
))
973 (aver (eq (sb-kind sb
) :finite
))
974 (let ((vop (tn-ref-vop op
)))
975 (labels ((tn-overlaps (tn)
976 (let ((sc (tn-sc tn
))
977 (tn-offset (tn-offset tn
)))
978 (when (and (eq (sc-sb sc
) sb
)
979 (<= tn-offset offset
)
982 (+ tn-offset
(sc-element-size sc
)))))
985 (let ((tn (tn-ref-tn ref
))
986 (ltn (tn-ref-load-tn ref
)))
988 (and ltn
(tn-overlaps ltn
)))))
990 (do ((ops ops
(tn-ref-across ops
)))
992 (let ((found (same ops
)))
993 (when (and found
(not (eq ops op
)))
996 (do ((refs refs
(tn-ref-next-ref refs
)))
998 (let ((found (same refs
)))
999 (when found
(return found
))))))
1000 (declare (inline is-op is-ref tn-overlaps
))
1001 (if (tn-ref-write-p op
)
1002 (or (is-op (vop-results vop
))
1003 (is-ref (vop-refs vop
) op
))
1004 (or (is-op (vop-args vop
))
1005 (is-ref (tn-ref-next-ref op
) nil
))))))
1007 ;;; Iterate over all the elements in the SB that would be allocated by
1008 ;;; allocating a TN in SC at Offset, checking for conflict with
1009 ;;; load-TNs or other TNs (live in the LIVE-TNS, which must be set
1010 ;;; up.) We also return true if there aren't enough locations after
1011 ;;; Offset to hold a TN in SC. If Ignore-Live is true, then we ignore
1012 ;;; the live-TNs, considering only references within Op's VOP.
1014 ;;; We return a conflicting TN, or :OVERFLOW if the TN won't fit.
1015 (defun load-tn-conflicts-in-sc (op sc offset ignore-live
)
1016 (let* ((sb (sc-sb sc
))
1017 (size (finite-sb-current-size sb
)))
1018 (do ((i offset
(1+ i
))
1019 (end (+ offset
(sc-element-size sc
))))
1021 (declare (type index i end
))
1022 (let ((res (or (when (>= i size
) :overflow
)
1023 (and (not ignore-live
)
1024 (svref (finite-sb-live-tns sb
) i
))
1025 (load-tn-offset-conflicts-in-sb op sb i
))))
1026 (when res
(return res
))))))
1028 ;;; If a load-TN for OP is targeted to a legal location in SC, then
1029 ;;; return the offset, otherwise return NIL. We see whether the target
1030 ;;; of the operand is packed, and try that location. There isn't any
1031 ;;; need to chain down the target path, since everything is packed
1034 ;;; We require the target to be in SC (and not merely to overlap with
1035 ;;; SC). This prevents SC information from being lost in load TNs (we
1036 ;;; won't pack a load TN in ANY-REG when it is targeted to a
1037 ;;; DESCRIPTOR-REG.) This shouldn't hurt the code as long as all
1038 ;;; relevant overlapping SCs are allowed in the operand SC
1040 (defun find-load-tn-target (op sc
)
1041 (declare (inline member
))
1042 (let ((target (tn-ref-target op
)))
1044 (let* ((tn (tn-ref-tn target
))
1045 (loc (tn-offset tn
)))
1046 (if (and (eq (tn-sc tn
) sc
)
1047 (member (the index loc
) (sc-locations sc
))
1048 (not (load-tn-conflicts-in-sc op sc loc nil
)))
1052 ;;; Select a legal location for a load TN for Op in SC. We just
1053 ;;; iterate over the SC's locations. If we can't find a legal
1054 ;;; location, return NIL.
1055 (defun select-load-tn-location (op sc
)
1056 (declare (type tn-ref op
) (type sc sc
))
1058 ;; Check any target location first.
1059 (let ((target (tn-ref-target op
)))
1061 (let* ((tn (tn-ref-tn target
))
1062 (loc (tn-offset tn
)))
1063 (when (and (eq (sc-sb sc
) (sc-sb (tn-sc tn
)))
1064 (member (the index loc
) (sc-locations sc
))
1065 (not (load-tn-conflicts-in-sc op sc loc nil
)))
1066 (return-from select-load-tn-location loc
)))))
1068 (dolist (loc (sc-locations sc
) nil
)
1069 (unless (load-tn-conflicts-in-sc op sc loc nil
)
1072 (defevent unpack-tn
"Unpacked a TN to satisfy operand SC restriction.")
1074 ;;; Make TN's location the same as for its save TN (allocating a save
1075 ;;; TN if necessary.) Delete any save/restore code that has been
1076 ;;; emitted thus far. Mark all blocks containing references as needing
1078 (defun unpack-tn (tn)
1080 (let ((stn (or (tn-save-tn tn
)
1081 (pack-save-tn tn
))))
1082 (setf (tn-sc tn
) (tn-sc stn
))
1083 (setf (tn-offset tn
) (tn-offset stn
))
1085 (do ((ref refs
(tn-ref-next ref
)))
1087 (let ((vop (tn-ref-vop ref
)))
1088 (if (eq (vop-info-name (vop-info vop
)) 'move-operand
)
1090 (pushnew (vop-block vop
) *repack-blocks
*))))))
1092 (zot (tn-writes tn
))))
1096 (defevent unpack-fallback
"Unpacked some operand TN.")
1098 ;;; This is called by PACK-LOAD-TN where there isn't any location free
1099 ;;; that we can pack into. What we do is move some live TN in one of
1100 ;;; the specified SCs to memory, then mark all blocks that reference
1101 ;;; the TN as needing repacking. If we succeed, we throw to UNPACKED-TN.
1102 ;;; If we fail, we return NIL.
1104 ;;; We can unpack any live TN that appears in the NORMAL-TNs list
1105 ;;; (isn't wired or restricted.) We prefer to unpack TNs that are not
1106 ;;; used by the VOP. If we can't find any such TN, then we unpack some
1107 ;;; argument or result TN. The only way we can fail is if all
1108 ;;; locations in SC are used by load-TNs or temporaries in VOP.
1109 (defun unpack-for-load-tn (sc op
)
1110 (declare (type sc sc
) (type tn-ref op
))
1111 (let ((sb (sc-sb sc
))
1112 (normal-tns (ir2-component-normal-tns
1113 (component-info *component-being-compiled
*)))
1114 (node (vop-node (tn-ref-vop op
)))
1116 (flet ((unpack-em (victims)
1117 (pushnew (vop-block (tn-ref-vop op
)) *repack-blocks
*)
1118 (dolist (victim victims
)
1119 (event unpack-tn node
)
1121 (throw 'unpacked-tn nil
)))
1122 (dolist (loc (sc-locations sc
))
1123 (declare (type index loc
))
1125 (collect ((victims nil adjoin
))
1127 (end (+ loc
(sc-element-size sc
))))
1129 (declare (type index i end
))
1130 (let ((victim (svref (finite-sb-live-tns sb
) i
)))
1132 (unless (find-in #'tn-next victim normal-tns
)
1136 (let ((conf (load-tn-conflicts-in-sc op sc loc t
)))
1138 (unpack-em (victims)))
1139 ((eq conf
:overflow
))
1141 (cond ((find conf
(victims))
1142 (setq fallback
(victims)))
1143 ((find-in #'tn-next conf normal-tns
)
1144 (setq fallback
(list conf
))))))))))
1147 (event unpack-fallback node
)
1148 (unpack-em fallback
))))
1152 ;;; Try to pack a load TN in the SCs indicated by Load-SCs. If we run
1153 ;;; out of SCs, then we unpack some TN and try again. We return the
1156 ;;; Note: we allow a Load-TN to be packed in the target location even
1157 ;;; if that location is in a SC not allowed by the primitive type.
1158 ;;; (The SC must still be allowed by the operand restriction.) This
1159 ;;; makes move VOPs more efficient, since we won't do a move from the
1160 ;;; stack into a non-descriptor any-reg through a descriptor argument
1161 ;;; load-TN. This does give targeting some real semantics, making it
1162 ;;; not a pure advisory to pack. It allows pack to do some packing it
1163 ;;; wouldn't have done before.
1164 (defun pack-load-tn (load-scs op
)
1165 (declare (type sc-vector load-scs
) (type tn-ref op
))
1166 (let ((vop (tn-ref-vop op
)))
1167 (compute-live-tns (vop-block vop
) vop
))
1169 (let* ((tn (tn-ref-tn op
))
1170 (ptype (tn-primitive-type tn
))
1171 (scs (svref load-scs
(sc-number (tn-sc tn
)))))
1172 (let ((current-scs scs
)
1178 (no-load-scs-allowed-by-primitive-type-error op
))
1179 (dolist (sc allowed
)
1180 (unpack-for-load-tn sc op
))
1181 (failed-to-pack-load-tn-error allowed op
))
1183 (let* ((sc (svref *backend-sc-numbers
* (pop current-scs
)))
1184 (target (find-load-tn-target op sc
)))
1185 (when (or target
(sc-allowed-by-primitive-type sc ptype
))
1186 (let ((loc (or target
1187 (select-load-tn-location op sc
))))
1189 (let ((res (make-tn 0 :load nil sc
)))
1190 (setf (tn-offset res
) loc
)
1192 (push sc allowed
)))))))))
1194 ;;; Scan a list of load-SCs vectors and a list of TN-REFS threaded by
1195 ;;; TN-REF-ACROSS. When we find a reference whose TN doesn't satisfy
1196 ;;; the restriction, we pack a Load-TN and load the operand into it.
1197 ;;; If a load-tn has already been allocated, we can assume that the
1198 ;;; restriction is satisfied.
1199 #!-sb-fluid
(declaim (inline check-operand-restrictions
))
1200 (defun check-operand-restrictions (scs ops
)
1201 (declare (list scs
) (type (or tn-ref null
) ops
))
1203 ;; Check the targeted operands first.
1204 (do ((scs scs
(cdr scs
))
1205 (op ops
(tn-ref-across op
)))
1207 (let ((target (tn-ref-target op
)))
1209 (let* ((load-tn (tn-ref-load-tn op
))
1210 (load-scs (svref (car scs
)
1212 (tn-sc (or load-tn
(tn-ref-tn op
)))))))
1214 (aver (eq load-scs t
))
1215 (unless (eq load-scs t
)
1216 (setf (tn-ref-load-tn op
)
1217 (pack-load-tn (car scs
) op
))))))))
1219 (do ((scs scs
(cdr scs
))
1220 (op ops
(tn-ref-across op
)))
1222 (let ((target (tn-ref-target op
)))
1224 (let* ((load-tn (tn-ref-load-tn op
))
1225 (load-scs (svref (car scs
)
1227 (tn-sc (or load-tn
(tn-ref-tn op
)))))))
1229 (aver (eq load-scs t
))
1230 (unless (eq load-scs t
)
1231 (setf (tn-ref-load-tn op
)
1232 (pack-load-tn (car scs
) op
))))))))
1236 ;;; Scan the VOPs in BLOCK, looking for operands whose SC restrictions
1237 ;;; aren't satisfied. We do the results first, since they are
1238 ;;; evaluated later, and our conflict analysis is a backward scan.
1239 (defun pack-load-tns (block)
1241 (let ((*live-block
* nil
)
1243 (do ((vop (ir2-block-last-vop block
) (vop-prev vop
)))
1245 (let ((info (vop-info vop
)))
1246 (check-operand-restrictions (vop-info-result-load-scs info
)
1248 (check-operand-restrictions (vop-info-arg-load-scs info
)
1254 ;;; Link the TN-REFS READ and WRITE together using the TN-REF-TARGET
1255 ;;; when this seems like a good idea. Currently we always do, as this
1256 ;;; increases the success of load-TN targeting.
1257 (defun target-if-desirable (read write
)
1258 (declare (type tn-ref read write
))
1259 ;; As per the comments at the definition of TN-REF-TARGET, read and
1260 ;; write refs are always paired, with TARGET in the read pointing to
1261 ;; the write and vice versa.
1262 (aver (eq (tn-ref-write-p read
)
1263 (not (tn-ref-write-p write
))))
1264 (setf (tn-ref-target read
) write
)
1265 (setf (tn-ref-target write
) read
))
1267 ;;; If TN can be packed into SC so as to honor a preference to TARGET,
1268 ;;; then return the offset to pack at, otherwise return NIL. TARGET
1269 ;;; must be already packed.
1270 (defun check-ok-target (target tn sc
)
1271 (declare (type tn target tn
) (type sc sc
) (inline member
))
1272 (let* ((loc (tn-offset target
))
1273 (target-sc (tn-sc target
))
1274 (target-sb (sc-sb target-sc
)))
1275 (declare (type index loc
))
1276 ;; We can honor a preference if:
1277 ;; -- TARGET's location is in SC's locations.
1278 ;; -- The element sizes of the two SCs are the same.
1279 ;; -- TN doesn't conflict with target's location.
1280 (if (and (eq target-sb
(sc-sb sc
))
1281 (or (eq (sb-kind target-sb
) :unbounded
)
1282 (member loc
(sc-locations sc
)))
1283 (= (sc-element-size target-sc
) (sc-element-size sc
))
1284 (not (conflicts-in-sc tn sc loc
))
1285 (zerop (mod loc
(sc-alignment sc
))))
1289 ;;; Scan along the target path from TN, looking at readers or writers.
1290 ;;; When we find a TN, call CALLEE with that TN, and then resume
1291 ;;; walking down that TN's target. As soon as there is no target, or
1292 ;;; if the TN has multiple readers (writers), we stop walking the
1293 ;;; targetting chain. We also always stop after 10 iterations to get
1294 ;;; around potential circularity problems.
1296 ;;; Why the single-reader/writer constraint? As far as I can tell,
1297 ;;; this is concerned with straight pipeline of data, e.g. CASTs. In
1298 ;;; that case, limiting to chains of length 10 seems to be more than
1300 (declaim (inline %call-with-target-tns
))
1301 (defun %call-with-target-tns
(tn callee
1302 &key
(limit 10) (reads t
) (writes t
))
1303 (declare (type tn tn
) (type function callee
) (type index limit
))
1304 (flet ((frob-slot (slot-function)
1305 (declare (type function slot-function
))
1308 (declare (type index count
))
1310 (let ((refs (funcall slot-function current
)))
1311 (unless (and (plusp count
)
1313 (not (tn-ref-next refs
)))
1315 (let ((target (tn-ref-target refs
)))
1316 (unless target
(return nil
))
1317 (setq current
(tn-ref-tn target
))
1318 (funcall callee current
)
1321 (frob-slot #'tn-reads
))
1323 (frob-slot #'tn-writes
))
1326 (defmacro do-target-tns
((target-variable source-tn
1327 &rest keys
&key limit reads writes
)
1329 (declare (ignore limit reads writes
))
1330 (let ((callback (sb!xc
:gensym
"CALLBACK")))
1331 `(flet ((,callback
(,target-variable
)
1333 (declare (dynamic-extent #',callback
))
1334 (%call-with-target-tns
,source-tn
#',callback
,@keys
))))
1336 (defun find-ok-target-offset (tn sc
)
1337 (declare (type tn tn
) (type sc sc
))
1338 (do-target-tns (target tn
)
1339 (awhen (and (tn-offset target
)
1340 (check-ok-target target tn sc
))
1341 (return-from find-ok-target-offset it
))))
1343 ;;;; location selection
1345 ;;; Select some location for TN in SC, returning the offset if we
1346 ;;; succeed, and NIL if we fail.
1348 ;;; For :UNBOUNDED SCs just find the smallest correctly aligned offset
1349 ;;; where the TN doesn't conflict with the TNs that have already been
1350 ;;; packed. For :FINITE SCs try to pack the TN into the most heavily
1351 ;;; used locations first (as estimated in FIND-LOCATION-USAGE).
1353 ;;; Historically SELECT-LOCATION tried did the opposite and tried to
1354 ;;; distribute the TNs evenly across the available locations. At least
1355 ;;; on register-starved architectures (x86) this seems to be a bad
1356 ;;; strategy. -- JES 2004-09-11
1357 (defun select-location (tn sc
&key use-reserved-locs optimize
)
1358 (declare (type tn tn
) (type sc sc
) (inline member
))
1359 (let* ((sb (sc-sb sc
))
1360 (element-size (sc-element-size sc
))
1361 (alignment (sc-alignment sc
))
1362 (align-mask (1- alignment
))
1363 (size (finite-sb-current-size sb
)))
1364 (flet ((attempt-location (start-offset)
1365 (let ((conflict (conflicts-in-sc tn sc start-offset
)))
1367 (logandc2 (+ conflict align-mask
1)
1369 (return-from select-location start-offset
)))))
1370 (if (eq (sb-kind sb
) :unbounded
)
1371 (loop with offset
= 0
1372 until
(> (+ offset element-size
) size
) do
1373 (setf offset
(attempt-location offset
)))
1374 (let ((locations (sc-locations sc
)))
1377 (schwartzian-stable-sort-list
1379 :key
(lambda (location-offset)
1380 (loop for offset from location-offset
1383 (finite-sb-always-live-count sb
)
1385 (dolist (offset locations
)
1386 (when (or use-reserved-locs
1388 (sc-reserve-locations sc
))))
1389 (attempt-location offset
))))))))
1391 ;;; If a save TN, return the saved TN, otherwise return TN. This is
1392 ;;; useful for getting the conflicts of a TN that might be a save TN.
1393 (defun original-tn (tn)
1394 (declare (type tn tn
))
1395 (if (member (tn-kind tn
) '(:save
:save-once
:specified-save
))
1402 (declaim (inline unbounded-sc-p
))
1403 (defun unbounded-sc-p (sc)
1404 (eq (sb-kind (sc-sb sc
)) :unbounded
))
1406 (defun unbounded-tn-p (tn)
1407 (unbounded-sc-p (tn-sc tn
)))
1408 (declaim (notinline unbounded-sc-p
))
1410 ;;; Attempt to pack TN in all possible SCs, first in the SC chosen by
1411 ;;; representation selection, then in the alternate SCs in the order
1412 ;;; they were specified in the SC definition. If the TN-COST is
1413 ;;; negative, then we don't attempt to pack in SCs that must be saved.
1414 ;;; If Restricted, then we can only pack in TN-SC, not in any
1417 ;;; If we are attempting to pack in the SC of the save TN for a TN
1418 ;;; with a :SPECIFIED-SAVE TN, then we pack in that location, instead
1419 ;;; of allocating a new stack location.
1420 (defun pack-tn (tn restricted optimize
&key
(allow-unbounded-sc t
))
1421 (declare (type tn tn
))
1422 (aver (not (tn-offset tn
)))
1423 (let* ((original (original-tn tn
))
1425 (alternates (unless restricted
(sc-alternate-scs fsc
)))
1426 (save (tn-save-tn tn
))
1429 (eq (tn-kind save
) :specified-save
))
1431 (do ((sc fsc
(pop alternates
)))
1433 (failed-to-pack-error tn restricted
))
1434 (unless (or allow-unbounded-sc
1435 (not (unbounded-sc-p sc
)))
1437 (when (eq sc specified-save-sc
)
1438 (unless (tn-offset save
)
1439 (pack-tn save nil optimize
))
1440 (setf (tn-offset tn
) (tn-offset save
))
1441 (setf (tn-sc tn
) (tn-sc save
))
1443 (when (or restricted
1444 (not (and (minusp (tn-cost tn
)) (sc-save-p sc
))))
1445 (let ((loc (or (find-ok-target-offset original sc
)
1446 (select-location original sc
:optimize optimize
)
1448 (select-location original sc
:use-reserved-locs t
1449 :optimize optimize
))
1450 (when (unbounded-sc-p sc
)
1452 (or (select-location original sc
)
1453 (error "failed to pack after growing SC?"))))))
1455 (add-location-conflicts original sc loc optimize
)
1456 (setf (tn-sc tn
) sc
)
1457 (setf (tn-offset tn
) loc
)
1461 ;;; Pack a wired TN, checking that the offset is in bounds for the SB,
1462 ;;; and that the TN doesn't conflict with some other TN already packed
1463 ;;; in that location. If the TN is wired to a location beyond the end
1464 ;;; of a :UNBOUNDED SB, then grow the SB enough to hold the TN.
1466 ;;; ### Checking for conflicts is disabled for :SPECIFIED-SAVE TNs.
1467 ;;; This is kind of a hack to make specifying wired stack save
1468 ;;; locations for local call arguments (such as OLD-FP) work, since
1469 ;;; the caller and callee OLD-FP save locations may conflict when the
1470 ;;; save locations don't really (due to being in different frames.)
1471 (defun pack-wired-tn (tn optimize
)
1472 (declare (type tn tn
))
1473 (let* ((sc (tn-sc tn
))
1475 (offset (tn-offset tn
))
1476 (end (+ offset
(sc-element-size sc
)))
1477 (original (original-tn tn
)))
1478 (when (> end
(finite-sb-current-size sb
))
1479 (unless (eq (sb-kind sb
) :unbounded
)
1480 (error "~S is wired to a location that is out of bounds." tn
))
1483 ;; For non-x86 ports the presence of a save-tn associated with a
1484 ;; tn is used to identify the old-fp and return-pc tns. It depends
1485 ;; on the old-fp and return-pc being passed in registers.
1486 #!-
(or x86 x86-64 arm arm64
)
1487 (when (and (not (eq (tn-kind tn
) :specified-save
))
1488 (conflicts-in-sc original sc offset
))
1489 (error "~S is wired to a location that it conflicts with." tn
))
1491 ;; Use the above check, but only print a verbose warning. This can
1492 ;; be helpful for debugging the x86 port.
1494 (when (and (not (eq (tn-kind tn
) :specified-save
))
1495 (conflicts-in-sc original sc offset
))
1496 (format t
"~&* Pack-wired-tn possible conflict:~% ~
1497 tn: ~S; tn-kind: ~S~% ~
1499 sb: ~S; sb-name: ~S; sb-kind: ~S~% ~
1500 offset: ~S; end: ~S~% ~
1502 tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%"
1504 sb
(sb-name sb
) (sb-kind sb
)
1507 (tn-save-tn tn
) (tn-kind (tn-save-tn tn
))))
1509 ;; On the x86 ports the old-fp and return-pc are often passed on
1510 ;; the stack so the above hack for the other ports does not always
1511 ;; work. Here the old-fp and return-pc tns are identified by being
1512 ;; on the stack in their standard save locations.
1513 #!+(or x86 x86-64 arm arm64
)
1514 (when (and (not (eq (tn-kind tn
) :specified-save
))
1515 (not (and (string= (sb-name sb
)
1516 #!-
(or arm arm64
) "STACK"
1517 #!+(or arm arm64
) "CONTROL-STACK")
1520 (conflicts-in-sc original sc offset
))
1521 (error "~S is wired to location ~D in SC ~A of kind ~S that it conflicts with."
1522 tn offset sc
(tn-kind tn
)))
1524 (add-location-conflicts original sc offset optimize
)))
1526 (defevent repack-block
"Repacked a block due to TN unpacking.")
1528 ;;; KLUDGE: Prior to SBCL version 0.8.9.xx, this function was known as
1529 ;;; PACK-BEFORE-GC-HOOK, but was non-functional since approximately
1530 ;;; version 0.8.3.xx since the removal of GC hooks from the system.
1531 ;;; This currently (as of 2004-04-12) runs now after every call to
1532 ;;; PACK, rather than -- as was originally intended -- once per GC
1533 ;;; cycle; this is probably non-optimal, and might require tuning,
1534 ;;; maybe to be called when the data structures exceed a certain size,
1535 ;;; or maybe once every N times. The KLUDGE is that this rewrite has
1536 ;;; done nothing to improve the reentrance or threadsafety of the
1537 ;;; compiler; it still fails to be callable from several threads at
1540 ;;; Brief experiments indicate that during a compilation cycle this
1541 ;;; causes about 10% more consing, and takes about 1%-2% more time.
1543 ;;; -- CSR, 2004-04-12
1544 (defun clean-up-pack-structures ()
1545 (dolist (sb *backend-sb-list
*)
1546 (unless (eq (sb-kind sb
) :non-packed
)
1547 (let ((size (sb-size sb
)))
1548 (fill (finite-sb-always-live sb
) nil
)
1549 (setf (finite-sb-always-live sb
)
1550 (make-array size
:initial-element
#*))
1551 (setf (finite-sb-always-live-count sb
)
1552 (make-array size
:initial-element
0))
1554 (fill (finite-sb-conflicts sb
) nil
)
1555 (setf (finite-sb-conflicts sb
)
1556 (make-array size
:initial-element
'#()))
1558 (fill (finite-sb-live-tns sb
) nil
)
1559 (setf (finite-sb-live-tns sb
)
1560 (make-array size
:initial-element nil
))))))
1562 (defun tn-lexical-depth (tn)
1563 (let ((path t
)) ; dummy initial value
1564 (labels ((path (lambda)
1566 (lambda lambda
(lambda-parent lambda
)))
1569 (register-scope (lambda)
1570 (let ((new-path (path lambda
)))
1571 (setf path
(if (eql path t
)
1574 0 (mismatch path new-path
))))))
1576 (do ((ref ref
(tn-ref-next ref
)))
1579 (awhen (vop-node (tn-ref-vop ref
))
1580 (register-scope (lexenv-lambda (node-lexenv it
)))))))
1581 (walk-tn-refs (tn-reads tn
))
1582 (walk-tn-refs (tn-writes tn
))
1584 most-positive-fixnum
1587 (declaim (type (member :iterative
:greedy
:adaptive
)
1588 *register-allocation-method
*))
1589 (defvar *register-allocation-method
* :adaptive
)
1591 (declaim (ftype function pack-greedy pack-iterative
))
1593 (defun pack (component)
1595 (let ((optimize nil
)
1597 (2comp (component-info component
)))
1598 (init-sb-vectors component
)
1600 ;; Determine whether we want to do more expensive packing by
1601 ;; checking whether any blocks in the component have (> SPEED
1604 ;; Also, determine if any such block also declares (speed 3),
1605 ;; in which case :adaptive register allocation will switch to
1606 ;; the iterative Chaitin-Briggs spilling/coloring algorithm.
1608 ;; FIXME: This means that a declaration can have a minor
1609 ;; effect even outside its scope, and as the packing is done
1610 ;; component-globally it'd be tricky to use strict scoping. I
1611 ;; think this is still acceptable since it's just a tradeoff
1612 ;; between compilation speed and allocation quality and
1613 ;; doesn't affect the semantics of the generated code in any
1614 ;; way. -- JES 2004-10-06
1615 (do-ir2-blocks (block component
)
1616 (let ((block (block-last (ir2-block-block block
))))
1617 (when (policy block
(> speed compilation-speed
))
1619 (when (policy block
(= speed
3))
1623 ;; Assign costs to normal TNs so we know which ones should always
1624 ;; be packed on the stack, and which are important not to spill.
1625 (when *pack-assign-costs
*
1626 (assign-tn-costs component
))
1628 ;; Actually allocate registers for most TNs. After this, only
1629 ;; :normal tns may be left unallocated (or TNs :restricted to
1630 ;; an unbounded SC).
1631 (funcall (ecase *register-allocation-method
*
1632 (:greedy
#'pack-greedy
)
1633 (:iterative
#'pack-iterative
)
1634 (:adaptive
(if speed-3
#'pack-iterative
#'pack-greedy
)))
1635 component
2comp optimize
)
1637 ;; Pack any leftover normal/restricted TN that is not already
1638 ;; allocated to a finite SC, or TNs that do not appear in any
1639 ;; local TN map (e.g. :MORE TNs). Since we'll likely be
1640 ;; allocating on the stack, first allocate TNs that are
1641 ;; associated with code at shallow lexical depths: this will
1642 ;; allocate long live ranges (i.e. TNs with more conflicts)
1643 ;; first, and hopefully minimise stack fragmentation.
1644 ;; Component TNs are a degenerate case: they are always live.
1645 (let ((component-tns '())
1646 (contiguous-tns '())
1648 (flet ((register-tn (tn)
1649 (unless (tn-offset tn
)
1652 (push tn component-tns
))
1653 ((:environment
:debug-environment
)
1654 (push tn contiguous-tns
))
1657 (do ((tn (ir2-component-restricted-tns 2comp
) (tn-next tn
)))
1659 ;; by this time, restricted TNs must either be
1660 ;; allocated in the right SC or unbounded
1661 (aver (or (tn-offset tn
) (unbounded-tn-p tn
)))
1663 (do ((tn (ir2-component-normal-tns 2comp
) (tn-next tn
)))
1666 (flet ((pack-tns (tns &optional in-order
)
1667 (dolist (tn (if in-order
1669 (schwartzian-stable-sort-list
1670 tns
#'< :key
#'tn-lexical-depth
)))
1671 (unless (tn-offset tn
)
1672 (pack-tn tn nil optimize
)))))
1673 ;; first pack TNs that are known to have simple live
1674 ;; ranges (contiguous lexical scopes)
1675 (pack-tns component-tns t
)
1676 (pack-tns contiguous-tns
)
1679 ;; Do load TN packing and emit saves.
1680 (let ((*repack-blocks
* nil
))
1681 (cond ((and optimize
*pack-optimize-saves
*)
1682 (optimized-emit-saves component
)
1683 (do-ir2-blocks (block component
)
1684 (pack-load-tns block
)))
1686 (do-ir2-blocks (block component
)
1688 (pack-load-tns block
))))
1690 (unless *repack-blocks
* (return))
1691 (let ((orpb *repack-blocks
*))
1692 (setq *repack-blocks
* nil
)
1693 (dolist (block orpb
)
1694 (event repack-block
)
1695 (pack-load-tns block
)))))
1698 (clean-up-pack-structures)))
1700 (defun pack-greedy (component 2comp optimize
)
1701 (declare (type component component
)
1702 (type ir2-component
2comp
))
1703 ;; Pack wired TNs first.
1704 (do ((tn (ir2-component-wired-tns 2comp
) (tn-next tn
)))
1706 (pack-wired-tn tn optimize
))
1708 ;; Then, pack restricted TNs, ones that are live over the whole
1709 ;; component first (they cause no fragmentation). Sort by TN cost
1710 ;; to help important TNs get good targeting.
1711 (collect ((component)
1713 (do ((tn (ir2-component-restricted-tns 2comp
) (tn-next tn
)))
1715 (unless (or (tn-offset tn
) (unbounded-tn-p tn
))
1716 (if (eq :component
(tn-kind tn
))
1719 (flet ((pack-tns (tns)
1720 (dolist (tn (stable-sort tns
#'> :key
#'tn-cost
))
1721 (pack-tn tn t optimize
))))
1722 (pack-tns (component))
1723 (pack-tns (normal))))
1725 (cond ((and *loop-analyze
* *pack-assign-costs
*)
1726 ;; Allocate normal TNs, starting with the TNs that are
1727 ;; heavily used in deep loops (which is taken into account in
1728 ;; TN spill costs). Only allocate in finite SCs (i.e. not on
1731 (do ((tn (ir2-component-normal-tns 2comp
) (tn-next tn
)))
1733 (unless (or (tn-offset tn
)
1734 (eq (tn-kind tn
) :more
)
1736 (and (sc-save-p (tn-sc tn
)) ; SC caller-save, but TN
1737 (minusp (tn-cost tn
)))) ; lives over many calls
1739 (dolist (tn (stable-sort (tns) #'> :key
#'tn-cost
))
1740 (unless (tn-offset tn
)
1741 ;; if it can't fit in a bounded SC, the final pass will
1742 ;; take care of stack packing.
1743 (pack-tn tn nil optimize
:allow-unbounded-sc nil
)))))
1745 ;; If loop analysis has been disabled we might as well revert
1746 ;; to the old behaviour of just packing TNs linearly as they
1748 (do-ir2-blocks (block component
)
1749 (let ((ltns (ir2-block-local-tns block
)))
1750 (do ((i (1- (ir2-block-local-tn-count block
)) (1- i
)))
1752 (declare (fixnum i
))
1753 (let ((tn (svref ltns i
)))
1754 (unless (or (null tn
)
1757 (unbounded-tn-p tn
))
1758 (pack-tn tn nil optimize
:allow-unbounded-sc nil
)))))))))
