1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2024 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.cc,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.cc.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
60 2. choose insn with least contribution to register pressure,
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
127 #include "coretypes.h"
131 #include "cfghooks.h"
133 #include "memmodel.h"
135 #include "insn-config.h"
139 #include "insn-attr.h"
141 #include "cfgbuild.h"
142 #include "sched-int.h"
143 #include "common/common-target.h"
146 #include "dumpfile.h"
147 #include "print-rtl.h"
148 #include "function-abi.h"
150 #ifdef INSN_SCHEDULING
152 /* True if we do register pressure relief through live-range
154 static bool live_range_shrinkage_p
;
156 /* Switch on live range shrinkage. */
158 initialize_live_range_shrinkage (void)
160 live_range_shrinkage_p
= true;
163 /* Switch off live range shrinkage. */
165 finish_live_range_shrinkage (void)
167 live_range_shrinkage_p
= false;
170 /* issue_rate is the number of insns that can be scheduled in the same
171 machine cycle. It can be defined in the config/mach/mach.h file,
172 otherwise we set it to 1. */
176 /* This can be set to true by a backend if the scheduler should not
177 enable a DCE pass. */
180 /* The current initiation interval used when modulo scheduling. */
181 static int modulo_ii
;
183 /* The maximum number of stages we are prepared to handle. */
184 static int modulo_max_stages
;
186 /* The number of insns that exist in each iteration of the loop. We use this
187 to detect when we've scheduled all insns from the first iteration. */
188 static int modulo_n_insns
;
190 /* The current count of insns in the first iteration of the loop that have
191 already been scheduled. */
192 static int modulo_insns_scheduled
;
194 /* The maximum uid of insns from the first iteration of the loop. */
195 static int modulo_iter0_max_uid
;
197 /* The number of times we should attempt to backtrack when modulo scheduling.
198 Decreased each time we have to backtrack. */
199 static int modulo_backtracks_left
;
201 /* The stage in which the last insn from the original loop was
203 static int modulo_last_stage
;
205 /* sched-verbose controls the amount of debugging output the
206 scheduler prints. It is controlled by -fsched-verbose=N:
207 N=0: no debugging output.
209 N=2: bb's probabilities, detailed ready list info, unit/insn info.
210 N=3: rtl at abort point, control-flow, regions info.
211 N=5: dependences info. */
212 int sched_verbose
= 0;
214 /* Debugging file. All printouts are sent to dump. */
215 FILE *sched_dump
= 0;
217 /* This is a placeholder for the scheduler parameters common
218 to all schedulers. */
219 struct common_sched_info_def
*common_sched_info
;
221 #define INSN_TICK(INSN) (HID (INSN)->tick)
222 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
223 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
224 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
225 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
226 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
227 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
228 /* Cached cost of the instruction. Use insn_sched_cost to get cost of the
229 insn. -1 here means that the field is not initialized. */
230 #define INSN_COST(INSN) (HID (INSN)->cost)
232 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
233 then it should be recalculated from scratch. */
234 #define INVALID_TICK (-(max_insn_queue_index + 1))
235 /* The minimal value of the INSN_TICK of an instruction. */
236 #define MIN_TICK (-max_insn_queue_index)
238 /* Original order of insns in the ready list.
239 Used to keep order of normal insns while separating DEBUG_INSNs. */
240 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
242 /* The deciding reason for INSN's place in the ready list. */
243 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
245 /* List of important notes we must keep around. This is a pointer to the
246 last element in the list. */
249 static struct spec_info_def spec_info_var
;
250 /* Description of the speculative part of the scheduling.
251 If NULL - no speculation. */
252 spec_info_t spec_info
= NULL
;
254 /* True, if recovery block was added during scheduling of current block.
255 Used to determine, if we need to fix INSN_TICKs. */
256 static bool haifa_recovery_bb_recently_added_p
;
258 /* True, if recovery block was added during this scheduling pass.
259 Used to determine if we should have empty memory pools of dependencies
260 after finishing current region. */
261 bool haifa_recovery_bb_ever_added_p
;
263 /* Counters of different types of speculative instructions. */
264 static int nr_begin_data
, nr_be_in_data
, nr_begin_control
, nr_be_in_control
;
266 /* Array used in {unlink, restore}_bb_notes. */
267 static rtx_insn
**bb_header
= 0;
269 /* Basic block after which recovery blocks will be created. */
270 static basic_block before_recovery
;
272 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
274 basic_block after_recovery
;
276 /* FALSE if we add bb to another region, so we don't need to initialize it. */
277 bool adding_bb_to_current_region_p
= true;
281 /* An instruction is ready to be scheduled when all insns preceding it
282 have already been scheduled. It is important to ensure that all
283 insns which use its result will not be executed until its result
284 has been computed. An insn is maintained in one of four structures:
286 (P) the "Pending" set of insns which cannot be scheduled until
287 their dependencies have been satisfied.
288 (Q) the "Queued" set of insns that can be scheduled when sufficient
290 (R) the "Ready" list of unscheduled, uncommitted insns.
291 (S) the "Scheduled" list of insns.
293 Initially, all insns are either "Pending" or "Ready" depending on
294 whether their dependencies are satisfied.
296 Insns move from the "Ready" list to the "Scheduled" list as they
297 are committed to the schedule. As this occurs, the insns in the
298 "Pending" list have their dependencies satisfied and move to either
299 the "Ready" list or the "Queued" set depending on whether
300 sufficient time has passed to make them ready. As time passes,
301 insns move from the "Queued" set to the "Ready" list.
303 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
304 unscheduled insns, i.e., those that are ready, queued, and pending.
305 The "Queued" set (Q) is implemented by the variable `insn_queue'.
306 The "Ready" list (R) is implemented by the variables `ready' and
308 The "Scheduled" list (S) is the new insn chain built by this pass.
310 The transition (R->S) is implemented in the scheduling loop in
311 `schedule_block' when the best insn to schedule is chosen.
312 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
313 insns move from the ready list to the scheduled list.
314 The transition (Q->R) is implemented in 'queue_to_insn' as time
315 passes or stalls are introduced. */
317 /* Implement a circular buffer to delay instructions until sufficient
318 time has passed. For the new pipeline description interface,
319 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
320 than maximal time of instruction execution computed by genattr.cc on
321 the base maximal time of functional unit reservations and getting a
322 result. This is the longest time an insn may be queued. */
324 static rtx_insn_list
**insn_queue
;
325 static int q_ptr
= 0;
326 static int q_size
= 0;
327 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
328 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
330 #define QUEUE_SCHEDULED (-3)
331 #define QUEUE_NOWHERE (-2)
332 #define QUEUE_READY (-1)
333 /* QUEUE_SCHEDULED - INSN is scheduled.
334 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
336 QUEUE_READY - INSN is in ready list.
337 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
339 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
341 /* The following variable value refers for all current and future
342 reservations of the processor units. */
345 /* The following variable value is size of memory representing all
346 current and future reservations of the processor units. */
347 size_t dfa_state_size
;
349 /* The following array is used to find the best insn from ready when
350 the automaton pipeline interface is used. */
351 signed char *ready_try
= NULL
;
353 /* The ready list. */
354 struct ready_list ready
= {NULL
, 0, 0, 0, 0};
356 /* The pointer to the ready list (to be removed). */
357 static struct ready_list
*readyp
= &ready
;
359 /* Scheduling clock. */
360 static int clock_var
;
362 /* Clock at which the previous instruction was issued. */
363 static int last_clock_var
;
365 /* Set to true if, when queuing a shadow insn, we discover that it would be
366 scheduled too late. */
367 static bool must_backtrack
;
369 /* The following variable value is number of essential insns issued on
370 the current cycle. An insn is essential one if it changes the
372 int cycle_issued_insns
;
374 /* This records the actual schedule. It is built up during the main phase
375 of schedule_block, and afterwards used to reorder the insns in the RTL. */
376 static vec
<rtx_insn
*> scheduled_insns
;
378 static int may_trap_exp (const_rtx
, int);
380 /* Nonzero iff the address is comprised from at most 1 register. */
381 #define CONST_BASED_ADDRESS_P(x) \
383 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
384 || (GET_CODE (x) == LO_SUM)) \
385 && (CONSTANT_P (XEXP (x, 0)) \
386 || CONSTANT_P (XEXP (x, 1)))))
388 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
389 as found by analyzing insn's expression. */
392 static int haifa_luid_for_non_insn (rtx x
);
394 /* Haifa version of sched_info hooks common to all headers. */
395 const struct common_sched_info_def haifa_common_sched_info
=
397 NULL
, /* fix_recovery_cfg */
398 NULL
, /* add_block */
399 NULL
, /* estimate_number_of_insns */
400 haifa_luid_for_non_insn
, /* luid_for_non_insn */
401 SCHED_PASS_UNKNOWN
/* sched_pass_id */
404 /* Mapping from instruction UID to its Logical UID. */
405 vec
<int> sched_luids
;
407 /* Next LUID to assign to an instruction. */
408 int sched_max_luid
= 1;
410 /* Haifa Instruction Data. */
411 vec
<haifa_insn_data_def
> h_i_d
;
413 void (* sched_init_only_bb
) (basic_block
, basic_block
);
415 /* Split block function. Different schedulers might use different functions
416 to handle their internal data consistent. */
417 basic_block (* sched_split_block
) (basic_block
, rtx
);
419 /* Create empty basic block after the specified block. */
420 basic_block (* sched_create_empty_bb
) (basic_block
);
422 /* Return the number of cycles until INSN is expected to be ready.
423 Return zero if it already is. */
425 insn_delay (rtx_insn
*insn
)
427 return MAX (INSN_TICK (insn
) - clock_var
, 0);
431 may_trap_exp (const_rtx x
, int is_store
)
440 if (code
== MEM
&& may_trap_p (x
))
447 /* The insn uses memory: a volatile load. */
448 if (MEM_VOLATILE_P (x
))
450 /* An exception-free load. */
453 /* A load with 1 base register, to be further checked. */
454 if (CONST_BASED_ADDRESS_P (XEXP (x
, 0)))
455 return PFREE_CANDIDATE
;
456 /* No info on the load, to be further checked. */
457 return PRISKY_CANDIDATE
;
462 int i
, insn_class
= TRAP_FREE
;
464 /* Neither store nor load, check if it may cause a trap. */
467 /* Recursive step: walk the insn... */
468 fmt
= GET_RTX_FORMAT (code
);
469 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
473 int tmp_class
= may_trap_exp (XEXP (x
, i
), is_store
);
474 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
476 else if (fmt
[i
] == 'E')
479 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
481 int tmp_class
= may_trap_exp (XVECEXP (x
, i
, j
), is_store
);
482 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
483 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
487 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
494 /* Classifies rtx X of an insn for the purpose of verifying that X can be
495 executed speculatively (and consequently the insn can be moved
496 speculatively), by examining X, returning:
497 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
498 TRAP_FREE: non-load insn.
499 IFREE: load from a globally safe location.
500 IRISKY: volatile load.
501 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
502 being either PFREE or PRISKY. */
505 haifa_classify_rtx (const_rtx x
)
507 int tmp_class
= TRAP_FREE
;
508 int insn_class
= TRAP_FREE
;
511 if (GET_CODE (x
) == PARALLEL
)
513 int i
, len
= XVECLEN (x
, 0);
515 for (i
= len
- 1; i
>= 0; i
--)
517 tmp_class
= haifa_classify_rtx (XVECEXP (x
, 0, i
));
518 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
519 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
529 /* Test if it is a 'store'. */
530 tmp_class
= may_trap_exp (XEXP (x
, 0), 1);
533 /* Test if it is a store. */
534 tmp_class
= may_trap_exp (SET_DEST (x
), 1);
535 if (tmp_class
== TRAP_RISKY
)
537 /* Test if it is a load. */
539 WORST_CLASS (tmp_class
,
540 may_trap_exp (SET_SRC (x
), 0));
543 tmp_class
= haifa_classify_rtx (COND_EXEC_CODE (x
));
544 if (tmp_class
== TRAP_RISKY
)
546 tmp_class
= WORST_CLASS (tmp_class
,
547 may_trap_exp (COND_EXEC_TEST (x
), 0));
550 tmp_class
= TRAP_RISKY
;
554 insn_class
= tmp_class
;
561 haifa_classify_insn (const_rtx insn
)
563 return haifa_classify_rtx (PATTERN (insn
));
566 /* After the scheduler initialization function has been called, this function
567 can be called to enable modulo scheduling. II is the initiation interval
568 we should use, it affects the delays for delay_pairs that were recorded as
569 separated by a given number of stages.
571 MAX_STAGES provides us with a limit
572 after which we give up scheduling; the caller must have unrolled at least
573 as many copies of the loop body and recorded delay_pairs for them.
575 INSNS is the number of real (non-debug) insns in one iteration of
576 the loop. MAX_UID can be used to test whether an insn belongs to
577 the first iteration of the loop; all of them have a uid lower than
580 set_modulo_params (int ii
, int max_stages
, int insns
, int max_uid
)
583 modulo_max_stages
= max_stages
;
584 modulo_n_insns
= insns
;
585 modulo_iter0_max_uid
= max_uid
;
586 modulo_backtracks_left
= param_max_modulo_backtrack_attempts
;
589 /* A structure to record a pair of insns where the first one is a real
590 insn that has delay slots, and the second is its delayed shadow.
591 I1 is scheduled normally and will emit an assembly instruction,
592 while I2 describes the side effect that takes place at the
593 transition between cycles CYCLES and (CYCLES + 1) after I1. */
596 struct delay_pair
*next_same_i1
;
599 /* When doing modulo scheduling, we a delay_pair can also be used to
600 show that I1 and I2 are the same insn in a different stage. If that
601 is the case, STAGES will be nonzero. */
605 /* Helpers for delay hashing. */
607 struct delay_i1_hasher
: nofree_ptr_hash
<delay_pair
>
609 typedef void *compare_type
;
610 static inline hashval_t
hash (const delay_pair
*);
611 static inline bool equal (const delay_pair
*, const void *);
614 /* Returns a hash value for X, based on hashing just I1. */
617 delay_i1_hasher::hash (const delay_pair
*x
)
619 return htab_hash_pointer (x
->i1
);
622 /* Return true if I1 of pair X is the same as that of pair Y. */
625 delay_i1_hasher::equal (const delay_pair
*x
, const void *y
)
630 struct delay_i2_hasher
: free_ptr_hash
<delay_pair
>
632 typedef void *compare_type
;
633 static inline hashval_t
hash (const delay_pair
*);
634 static inline bool equal (const delay_pair
*, const void *);
637 /* Returns a hash value for X, based on hashing just I2. */
640 delay_i2_hasher::hash (const delay_pair
*x
)
642 return htab_hash_pointer (x
->i2
);
645 /* Return true if I2 of pair X is the same as that of pair Y. */
648 delay_i2_hasher::equal (const delay_pair
*x
, const void *y
)
653 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
655 static hash_table
<delay_i1_hasher
> *delay_htab
;
656 static hash_table
<delay_i2_hasher
> *delay_htab_i2
;
658 /* Called through htab_traverse. Walk the hashtable using I2 as
659 index, and delete all elements involving an UID higher than
660 that pointed to by *DATA. */
662 haifa_htab_i2_traverse (delay_pair
**slot
, int *data
)
665 struct delay_pair
*p
= *slot
;
666 if (INSN_UID (p
->i2
) >= maxuid
|| INSN_UID (p
->i1
) >= maxuid
)
668 delay_htab_i2
->clear_slot (slot
);
673 /* Called through htab_traverse. Walk the hashtable using I2 as
674 index, and delete all elements involving an UID higher than
675 that pointed to by *DATA. */
677 haifa_htab_i1_traverse (delay_pair
**pslot
, int *data
)
680 struct delay_pair
*p
, *first
, **pprev
;
682 if (INSN_UID ((*pslot
)->i1
) >= maxuid
)
684 delay_htab
->clear_slot (pslot
);
688 for (p
= *pslot
; p
; p
= p
->next_same_i1
)
690 if (INSN_UID (p
->i2
) < maxuid
)
693 pprev
= &p
->next_same_i1
;
698 delay_htab
->clear_slot (pslot
);
704 /* Discard all delay pairs which involve an insn with an UID higher
707 discard_delay_pairs_above (int max_uid
)
709 delay_htab
->traverse
<int *, haifa_htab_i1_traverse
> (&max_uid
);
710 delay_htab_i2
->traverse
<int *, haifa_htab_i2_traverse
> (&max_uid
);
713 /* This function can be called by a port just before it starts the final
714 scheduling pass. It records the fact that an instruction with delay
715 slots has been split into two insns, I1 and I2. The first one will be
716 scheduled normally and initiates the operation. The second one is a
717 shadow which must follow a specific number of cycles after I1; its only
718 purpose is to show the side effect that occurs at that cycle in the RTL.
719 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
720 while I2 retains the original insn type.
722 There are two ways in which the number of cycles can be specified,
723 involving the CYCLES and STAGES arguments to this function. If STAGES
724 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
725 which is multiplied by MODULO_II to give the number of cycles. This is
726 only useful if the caller also calls set_modulo_params to enable modulo
730 record_delay_slot_pair (rtx_insn
*i1
, rtx_insn
*i2
, int cycles
, int stages
)
732 struct delay_pair
*p
= XNEW (struct delay_pair
);
733 struct delay_pair
**slot
;
742 delay_htab
= new hash_table
<delay_i1_hasher
> (10);
743 delay_htab_i2
= new hash_table
<delay_i2_hasher
> (10);
745 slot
= delay_htab
->find_slot_with_hash (i1
, htab_hash_pointer (i1
), INSERT
);
746 p
->next_same_i1
= *slot
;
748 slot
= delay_htab_i2
->find_slot (p
, INSERT
);
752 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
753 and return the other insn if so. Return NULL otherwise. */
755 real_insn_for_shadow (rtx_insn
*insn
)
757 struct delay_pair
*pair
;
762 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
763 if (!pair
|| pair
->stages
> 0)
768 /* For a pair P of insns, return the fixed distance in cycles from the first
769 insn after which the second must be scheduled. */
771 pair_delay (struct delay_pair
*p
)
776 return p
->stages
* modulo_ii
;
779 /* Given an insn INSN, add a dependence on its delayed shadow if it
780 has one. Also try to find situations where shadows depend on each other
781 and add dependencies to the real insns to limit the amount of backtracking
784 add_delay_dependencies (rtx_insn
*insn
)
786 struct delay_pair
*pair
;
787 sd_iterator_def sd_it
;
793 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
796 add_dependence (insn
, pair
->i1
, REG_DEP_ANTI
);
800 FOR_EACH_DEP (pair
->i2
, SD_LIST_BACK
, sd_it
, dep
)
802 rtx_insn
*pro
= DEP_PRO (dep
);
803 struct delay_pair
*other_pair
804 = delay_htab_i2
->find_with_hash (pro
, htab_hash_pointer (pro
));
805 if (!other_pair
|| other_pair
->stages
)
807 if (pair_delay (other_pair
) >= pair_delay (pair
))
809 if (sched_verbose
>= 4)
811 fprintf (sched_dump
, ";;\tadding dependence %d <- %d\n",
812 INSN_UID (other_pair
->i1
),
813 INSN_UID (pair
->i1
));
814 fprintf (sched_dump
, ";;\tpair1 %d <- %d, cost %d\n",
818 fprintf (sched_dump
, ";;\tpair2 %d <- %d, cost %d\n",
819 INSN_UID (other_pair
->i1
),
820 INSN_UID (other_pair
->i2
),
821 pair_delay (other_pair
));
823 add_dependence (pair
->i1
, other_pair
->i1
, REG_DEP_ANTI
);
828 /* Forward declarations. */
830 static int priority (rtx_insn
*, bool force_recompute
= false);
831 static int autopref_rank_for_schedule (const rtx_insn
*, const rtx_insn
*);
832 static int rank_for_schedule (const void *, const void *);
833 static void swap_sort (rtx_insn
**, int);
834 static void queue_insn (rtx_insn
*, int, const char *);
835 static int schedule_insn (rtx_insn
*);
836 static void adjust_priority (rtx_insn
*);
837 static void advance_one_cycle (void);
838 static void extend_h_i_d (void);
841 /* Notes handling mechanism:
842 =========================
843 Generally, NOTES are saved before scheduling and restored after scheduling.
844 The scheduler distinguishes between two types of notes:
846 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
847 Before scheduling a region, a pointer to the note is added to the insn
848 that follows or precedes it. (This happens as part of the data dependence
849 computation). After scheduling an insn, the pointer contained in it is
850 used for regenerating the corresponding note (in reemit_notes).
852 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
853 these notes are put in a list (in rm_other_notes() and
854 unlink_other_notes ()). After scheduling the block, these notes are
855 inserted at the beginning of the block (in schedule_block()). */
857 static void ready_add (struct ready_list
*, rtx_insn
*, bool);
858 static rtx_insn
*ready_remove_first (struct ready_list
*);
859 static rtx_insn
*ready_remove_first_dispatch (struct ready_list
*ready
);
861 static void queue_to_ready (struct ready_list
*);
862 static int early_queue_to_ready (state_t
, struct ready_list
*);
864 /* The following functions are used to implement multi-pass scheduling
865 on the first cycle. */
866 static rtx_insn
*ready_remove (struct ready_list
*, int);
867 static void ready_remove_insn (rtx_insn
*);
869 static void fix_inter_tick (rtx_insn
*, rtx_insn
*);
870 static int fix_tick_ready (rtx_insn
*);
871 static void change_queue_index (rtx_insn
*, int);
873 /* The following functions are used to implement scheduling of data/control
874 speculative instructions. */
876 static void extend_h_i_d (void);
877 static void init_h_i_d (rtx_insn
*);
878 static int haifa_speculate_insn (rtx_insn
*, ds_t
, rtx
*);
879 static void generate_recovery_code (rtx_insn
*);
880 static void process_insn_forw_deps_be_in_spec (rtx_insn
*, rtx_insn
*, ds_t
);
881 static void begin_speculative_block (rtx_insn
*);
882 static void add_to_speculative_block (rtx_insn
*);
883 static void init_before_recovery (basic_block
*);
884 static void create_check_block_twin (rtx_insn
*, bool);
885 static void fix_recovery_deps (basic_block
);
886 static bool haifa_change_pattern (rtx_insn
*, rtx
);
887 static void dump_new_block_header (int, basic_block
, rtx_insn
*, rtx_insn
*);
888 static void restore_bb_notes (basic_block
);
889 static void fix_jump_move (rtx_insn
*);
890 static void move_block_after_check (rtx_insn
*);
891 static void move_succs (vec
<edge
, va_gc
> **, basic_block
);
892 static void sched_remove_insn (rtx_insn
*);
893 static void clear_priorities (rtx_insn
*, rtx_vec_t
*);
894 static void calc_priorities (const rtx_vec_t
&);
895 static void add_jump_dependencies (rtx_insn
*, rtx_insn
*);
897 #endif /* INSN_SCHEDULING */
899 /* Point to state used for the current scheduling pass. */
900 struct haifa_sched_info
*current_sched_info
;
902 #ifndef INSN_SCHEDULING
904 schedule_insns (void)
909 /* Do register pressure sensitive insn scheduling if the flag is set
911 enum sched_pressure_algorithm sched_pressure
;
913 /* Map regno -> its pressure class. The map defined only when
914 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
915 enum reg_class
*sched_regno_pressure_class
;
917 /* The current register pressure. Only elements corresponding pressure
918 classes are defined. */
919 static int curr_reg_pressure
[N_REG_CLASSES
];
921 /* Saved value of the previous array. */
922 static int saved_reg_pressure
[N_REG_CLASSES
];
924 /* Register living at given scheduling point. */
925 static bitmap curr_reg_live
;
927 /* Saved value of the previous array. */
928 static bitmap saved_reg_live
;
930 /* Registers mentioned in the current region. */
931 static bitmap region_ref_regs
;
933 /* Temporary bitmap used for SCHED_PRESSURE_MODEL. */
934 static bitmap tmp_bitmap
;
936 /* Effective number of available registers of a given class (see comment
937 in sched_pressure_start_bb). */
938 static int sched_class_regs_num
[N_REG_CLASSES
];
939 /* The number of registers that the function would need to save before it
940 uses them, and the number of fixed_regs. Helpers for calculating of
941 sched_class_regs_num. */
942 static int call_saved_regs_num
[N_REG_CLASSES
];
943 static int fixed_regs_num
[N_REG_CLASSES
];
945 /* Initiate register pressure relative info for scheduling the current
946 region. Currently it is only clearing register mentioned in the
949 sched_init_region_reg_pressure_info (void)
951 bitmap_clear (region_ref_regs
);
954 /* PRESSURE[CL] describes the pressure on register class CL. Update it
955 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
956 LIVE tracks the set of live registers; if it is null, assume that
957 every birth or death is genuine. */
959 mark_regno_birth_or_death (bitmap live
, int *pressure
, int regno
, bool birth_p
)
961 enum reg_class pressure_class
;
963 pressure_class
= sched_regno_pressure_class
[regno
];
964 if (regno
>= FIRST_PSEUDO_REGISTER
)
966 if (pressure_class
!= NO_REGS
)
970 if (!live
|| bitmap_set_bit (live
, regno
))
971 pressure
[pressure_class
]
972 += (ira_reg_class_max_nregs
973 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
977 if (!live
|| bitmap_clear_bit (live
, regno
))
978 pressure
[pressure_class
]
979 -= (ira_reg_class_max_nregs
980 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
984 else if (pressure_class
!= NO_REGS
985 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs
, regno
))
989 if (!live
|| bitmap_set_bit (live
, regno
))
990 pressure
[pressure_class
]++;
994 if (!live
|| bitmap_clear_bit (live
, regno
))
995 pressure
[pressure_class
]--;
1000 /* Initiate current register pressure related info from living
1001 registers given by LIVE. */
1003 initiate_reg_pressure_info (bitmap live
)
1009 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1010 curr_reg_pressure
[ira_pressure_classes
[i
]] = 0;
1011 bitmap_clear (curr_reg_live
);
1012 EXECUTE_IF_SET_IN_BITMAP (live
, 0, j
, bi
)
1013 if (sched_pressure
== SCHED_PRESSURE_MODEL
1014 || current_nr_blocks
== 1
1015 || bitmap_bit_p (region_ref_regs
, j
))
1016 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
, j
, true);
1019 /* Mark registers in X as mentioned in the current region. */
1021 setup_ref_regs (rtx x
)
1024 const RTX_CODE code
= GET_CODE (x
);
1029 bitmap_set_range (region_ref_regs
, REGNO (x
), REG_NREGS (x
));
1032 fmt
= GET_RTX_FORMAT (code
);
1033 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
1035 setup_ref_regs (XEXP (x
, i
));
1036 else if (fmt
[i
] == 'E')
1038 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
1039 setup_ref_regs (XVECEXP (x
, i
, j
));
1043 /* Initiate current register pressure related info at the start of
1046 initiate_bb_reg_pressure_info (basic_block bb
)
1048 unsigned int i ATTRIBUTE_UNUSED
;
1051 if (current_nr_blocks
> 1)
1052 FOR_BB_INSNS (bb
, insn
)
1053 if (NONDEBUG_INSN_P (insn
))
1054 setup_ref_regs (PATTERN (insn
));
1055 initiate_reg_pressure_info (df_get_live_in (bb
));
1056 if (bb_has_eh_pred (bb
))
1059 unsigned int regno
= EH_RETURN_DATA_REGNO (i
);
1061 if (regno
== INVALID_REGNUM
)
1063 if (! bitmap_bit_p (df_get_live_in (bb
), regno
))
1064 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
1069 /* Save current register pressure related info. */
1071 save_reg_pressure (void)
1075 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1076 saved_reg_pressure
[ira_pressure_classes
[i
]]
1077 = curr_reg_pressure
[ira_pressure_classes
[i
]];
1078 bitmap_copy (saved_reg_live
, curr_reg_live
);
1081 /* Restore saved register pressure related info. */
1083 restore_reg_pressure (void)
1087 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1088 curr_reg_pressure
[ira_pressure_classes
[i
]]
1089 = saved_reg_pressure
[ira_pressure_classes
[i
]];
1090 bitmap_copy (curr_reg_live
, saved_reg_live
);
1093 /* Return TRUE if the register is dying after its USE. */
1095 dying_use_p (struct reg_use_data
*use
)
1097 struct reg_use_data
*next
;
1099 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
1100 if (NONDEBUG_INSN_P (next
->insn
)
1101 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
1106 /* Print info about the current register pressure and its excess for
1107 each pressure class. */
1109 print_curr_reg_pressure (void)
1114 fprintf (sched_dump
, ";;\t");
1115 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1117 cl
= ira_pressure_classes
[i
];
1118 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1119 fprintf (sched_dump
, " %s:%d(%d)", reg_class_names
[cl
],
1120 curr_reg_pressure
[cl
],
1121 curr_reg_pressure
[cl
] - sched_class_regs_num
[cl
]);
1123 fprintf (sched_dump
, "\n");
1126 /* Determine if INSN has a condition that is clobbered if a register
1127 in SET_REGS is modified. */
1129 cond_clobbered_p (rtx_insn
*insn
, HARD_REG_SET set_regs
)
1131 rtx pat
= PATTERN (insn
);
1132 gcc_assert (GET_CODE (pat
) == COND_EXEC
);
1133 if (TEST_HARD_REG_BIT (set_regs
, REGNO (XEXP (COND_EXEC_TEST (pat
), 0))))
1135 sd_iterator_def sd_it
;
1137 haifa_change_pattern (insn
, ORIG_PAT (insn
));
1138 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
1139 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1140 TODO_SPEC (insn
) = HARD_DEP
;
1141 if (sched_verbose
>= 2)
1142 fprintf (sched_dump
,
1143 ";;\t\tdequeue insn %s because of clobbered condition\n",
1144 (*current_sched_info
->print_insn
) (insn
, 0));
1151 /* This function should be called after modifying the pattern of INSN,
1152 to update scheduler data structures as needed. */
1154 update_insn_after_change (rtx_insn
*insn
)
1156 sd_iterator_def sd_it
;
1159 dfa_clear_single_insn_cache (insn
);
1161 sd_it
= sd_iterator_start (insn
,
1162 SD_LIST_FORW
| SD_LIST_BACK
| SD_LIST_RES_BACK
);
1163 while (sd_iterator_cond (&sd_it
, &dep
))
1165 DEP_COST (dep
) = UNKNOWN_DEP_COST
;
1166 sd_iterator_next (&sd_it
);
1169 /* Invalidate INSN_COST, so it'll be recalculated. */
1170 INSN_COST (insn
) = -1;
1171 /* Invalidate INSN_TICK, so it'll be recalculated. */
1172 INSN_TICK (insn
) = INVALID_TICK
;
1174 /* Invalidate autoprefetch data entry. */
1175 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
1176 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1177 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
1178 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1182 /* Two VECs, one to hold dependencies for which pattern replacements
1183 need to be applied or restored at the start of the next cycle, and
1184 another to hold an integer that is either one, to apply the
1185 corresponding replacement, or zero to restore it. */
1186 static vec
<dep_t
> next_cycle_replace_deps
;
1187 static vec
<int> next_cycle_apply
;
1189 static void apply_replacement (dep_t
, bool);
1190 static void restore_pattern (dep_t
, bool);
1192 /* Look at the remaining dependencies for insn NEXT, and compute and return
1193 the TODO_SPEC value we should use for it. This is called after one of
1194 NEXT's dependencies has been resolved.
1195 We also perform pattern replacements for predication, and for broken
1196 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1200 recompute_todo_spec (rtx_insn
*next
, bool for_backtrack
)
1203 sd_iterator_def sd_it
;
1204 dep_t dep
, modify_dep
= NULL
;
1208 bool first_p
= true;
1210 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
1211 /* NEXT has all its dependencies resolved. */
1214 if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
1217 /* If NEXT is intended to sit adjacent to this instruction, we don't
1218 want to try to break any dependencies. Treat it as a HARD_DEP. */
1219 if (SCHED_GROUP_P (next
))
1222 /* Now we've got NEXT with speculative deps only.
1223 1. Look at the deps to see what we have to do.
1224 2. Check if we can do 'todo'. */
1227 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1229 rtx_insn
*pro
= DEP_PRO (dep
);
1230 ds_t ds
= DEP_STATUS (dep
) & SPECULATIVE
;
1232 if (DEBUG_INSN_P (pro
) && !DEBUG_INSN_P (next
))
1245 new_ds
= ds_merge (new_ds
, ds
);
1247 else if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
1249 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1254 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1256 else if (DEP_REPLACE (dep
) != NULL
)
1258 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1263 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1267 if (n_replace
> 0 && n_control
== 0 && n_spec
== 0)
1269 if (!dbg_cnt (sched_breakdep
))
1271 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1273 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
1276 if (desc
->insn
== next
&& !for_backtrack
)
1278 gcc_assert (n_replace
== 1);
1279 apply_replacement (dep
, true);
1281 DEP_STATUS (dep
) |= DEP_CANCELLED
;
1287 else if (n_control
== 1 && n_replace
== 0 && n_spec
== 0)
1289 rtx_insn
*pro
, *other
;
1291 rtx cond
= NULL_RTX
;
1293 rtx_insn
*prev
= NULL
;
1297 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0
1298 || (ORIG_PAT (next
) != NULL_RTX
1299 && PREDICATED_PAT (next
) == NULL_RTX
))
1302 pro
= DEP_PRO (modify_dep
);
1303 other
= real_insn_for_shadow (pro
);
1304 if (other
!= NULL_RTX
)
1307 cond
= sched_get_reverse_condition_uncached (pro
);
1308 regno
= REGNO (XEXP (cond
, 0));
1310 /* Find the last scheduled insn that modifies the condition register.
1311 We can stop looking once we find the insn we depend on through the
1312 REG_DEP_CONTROL; if the condition register isn't modified after it,
1313 we know that it still has the right value. */
1314 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
1315 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns
, i
, prev
)
1319 find_all_hard_reg_sets (prev
, &t
, true);
1320 if (TEST_HARD_REG_BIT (t
, regno
))
1325 if (ORIG_PAT (next
) == NULL_RTX
)
1327 ORIG_PAT (next
) = PATTERN (next
);
1329 new_pat
= gen_rtx_COND_EXEC (VOIDmode
, cond
, PATTERN (next
));
1330 success
= haifa_change_pattern (next
, new_pat
);
1333 PREDICATED_PAT (next
) = new_pat
;
1335 else if (PATTERN (next
) != PREDICATED_PAT (next
))
1337 bool success
= haifa_change_pattern (next
,
1338 PREDICATED_PAT (next
));
1339 gcc_assert (success
);
1341 DEP_STATUS (modify_dep
) |= DEP_CANCELLED
;
1345 if (PREDICATED_PAT (next
) != NULL_RTX
)
1347 int tick
= INSN_TICK (next
);
1348 bool success
= haifa_change_pattern (next
,
1350 INSN_TICK (next
) = tick
;
1351 gcc_assert (success
);
1354 /* We can't handle the case where there are both speculative and control
1355 dependencies, so we return HARD_DEP in such a case. Also fail if
1356 we have speculative dependencies with not enough points, or more than
1357 one control dependency. */
1358 if ((n_spec
> 0 && (n_control
> 0 || n_replace
> 0))
1360 /* Too few points? */
1361 && ds_weak (new_ds
) < spec_info
->data_weakness_cutoff
)
1369 /* Pointer to the last instruction scheduled. */
1370 static rtx_insn
*last_scheduled_insn
;
1372 /* Pointer to the last nondebug instruction scheduled within the
1373 block, or the prev_head of the scheduling block. Used by
1374 rank_for_schedule, so that insns independent of the last scheduled
1375 insn will be preferred over dependent instructions. */
1376 static rtx_insn
*last_nondebug_scheduled_insn
;
1378 /* Pointer that iterates through the list of unscheduled insns if we
1379 have a dbg_cnt enabled. It always points at an insn prior to the
1380 first unscheduled one. */
1381 static rtx_insn
*nonscheduled_insns_begin
;
1383 /* Compute cost of executing INSN.
1384 This is the number of cycles between instruction issue and
1385 instruction results. */
1387 insn_sched_cost (rtx_insn
*insn
)
1396 if (recog_memoized (insn
) < 0)
1399 cost
= insn_default_latency (insn
);
1406 cost
= INSN_COST (insn
);
1410 /* A USE insn, or something else we don't need to
1411 understand. We can't pass these directly to
1412 result_ready_cost or insn_default_latency because it will
1413 trigger a fatal error for unrecognizable insns. */
1414 if (recog_memoized (insn
) < 0)
1416 INSN_COST (insn
) = 0;
1421 cost
= insn_default_latency (insn
);
1425 INSN_COST (insn
) = cost
;
1432 /* Compute cost of dependence LINK.
1433 This is the number of cycles between instruction issue and
1434 instruction results.
1435 ??? We also use this function to call recog_memoized on all insns. */
1437 dep_cost_1 (dep_t link
, dw_t dw
)
1439 rtx_insn
*insn
= DEP_PRO (link
);
1440 rtx_insn
*used
= DEP_CON (link
);
1443 if (DEP_COST (link
) != UNKNOWN_DEP_COST
)
1444 return DEP_COST (link
);
1448 struct delay_pair
*delay_entry
;
1450 = delay_htab_i2
->find_with_hash (used
, htab_hash_pointer (used
));
1453 if (delay_entry
->i1
== insn
)
1455 DEP_COST (link
) = pair_delay (delay_entry
);
1456 return DEP_COST (link
);
1461 /* A USE insn should never require the value used to be computed.
1462 This allows the computation of a function's result and parameter
1463 values to overlap the return and call. We don't care about the
1464 dependence cost when only decreasing register pressure. */
1465 if (recog_memoized (used
) < 0)
1468 recog_memoized (insn
);
1472 enum reg_note dep_type
= DEP_TYPE (link
);
1474 cost
= insn_sched_cost (insn
);
1476 if (INSN_CODE (insn
) >= 0)
1478 if (dep_type
== REG_DEP_ANTI
)
1480 else if (dep_type
== REG_DEP_OUTPUT
)
1482 cost
= (insn_default_latency (insn
)
1483 - insn_default_latency (used
));
1487 else if (bypass_p (insn
))
1488 cost
= insn_latency (insn
, used
);
1492 if (targetm
.sched
.adjust_cost
)
1493 cost
= targetm
.sched
.adjust_cost (used
, (int) dep_type
, insn
, cost
,
1500 DEP_COST (link
) = cost
;
1504 /* Compute cost of dependence LINK.
1505 This is the number of cycles between instruction issue and
1506 instruction results. */
1508 dep_cost (dep_t link
)
1510 return dep_cost_1 (link
, 0);
1513 /* Use this sel-sched.cc friendly function in reorder2 instead of increasing
1514 INSN_PRIORITY explicitly. */
1516 increase_insn_priority (rtx_insn
*insn
, int amount
)
1518 if (!sel_sched_p ())
1520 /* We're dealing with haifa-sched.cc INSN_PRIORITY. */
1521 if (INSN_PRIORITY_KNOWN (insn
))
1522 INSN_PRIORITY (insn
) += amount
;
1526 /* In sel-sched.cc INSN_PRIORITY is not kept up to date.
1527 Use EXPR_PRIORITY instead. */
1528 sel_add_to_insn_priority (insn
, amount
);
1532 /* Return 'true' if DEP should be included in priority calculations. */
1534 contributes_to_priority_p (dep_t dep
)
1536 if (DEBUG_INSN_P (DEP_CON (dep
))
1537 || DEBUG_INSN_P (DEP_PRO (dep
)))
1540 /* Critical path is meaningful in block boundaries only. */
1541 if (!current_sched_info
->contributes_to_priority (DEP_CON (dep
),
1545 if (DEP_REPLACE (dep
) != NULL
)
1548 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1549 then speculative instructions will less likely be
1550 scheduled. That is because the priority of
1551 their producers will increase, and, thus, the
1552 producers will more likely be scheduled, thus,
1553 resolving the dependence. */
1554 if (sched_deps_info
->generate_spec_deps
1555 && !(spec_info
->flags
& COUNT_SPEC_IN_CRITICAL_PATH
)
1556 && (DEP_STATUS (dep
) & SPECULATIVE
))
1562 /* Compute the number of nondebug deps in list LIST for INSN. */
1565 dep_list_size (rtx_insn
*insn
, sd_list_types_def list
)
1567 sd_iterator_def sd_it
;
1569 int dbgcount
= 0, nodbgcount
= 0;
1571 if (!MAY_HAVE_DEBUG_INSNS
)
1572 return sd_lists_size (insn
, list
);
1574 FOR_EACH_DEP (insn
, list
, sd_it
, dep
)
1576 if (DEBUG_INSN_P (DEP_CON (dep
)))
1578 else if (!DEBUG_INSN_P (DEP_PRO (dep
)))
1582 gcc_assert (dbgcount
+ nodbgcount
== sd_lists_size (insn
, list
));
1589 /* Compute the priority number for INSN. */
1591 priority (rtx_insn
*insn
, bool force_recompute
)
1593 if (! INSN_P (insn
))
1596 /* We should not be interested in priority of an already scheduled insn. */
1597 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
1599 if (force_recompute
|| !INSN_PRIORITY_KNOWN (insn
))
1601 int this_priority
= -1;
1605 int this_fusion_priority
;
1607 targetm
.sched
.fusion_priority (insn
, FUSION_MAX_PRIORITY
,
1608 &this_fusion_priority
, &this_priority
);
1609 INSN_FUSION_PRIORITY (insn
) = this_fusion_priority
;
1611 else if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
1612 /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn
1613 has some forward deps but all of them are ignored by
1614 contributes_to_priority hook. At the moment we set priority of
1616 this_priority
= insn_sched_cost (insn
);
1619 rtx_insn
*prev_first
, *twin
;
1622 /* For recovery check instructions we calculate priority slightly
1623 different than that of normal instructions. Instead of walking
1624 through INSN_FORW_DEPS (check) list, we walk through
1625 INSN_FORW_DEPS list of each instruction in the corresponding
1628 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1629 rec
= sel_sched_p () ? NULL
: RECOVERY_BLOCK (insn
);
1630 if (!rec
|| rec
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1632 prev_first
= PREV_INSN (insn
);
1637 prev_first
= NEXT_INSN (BB_HEAD (rec
));
1638 twin
= PREV_INSN (BB_END (rec
));
1643 sd_iterator_def sd_it
;
1646 FOR_EACH_DEP (twin
, SD_LIST_FORW
, sd_it
, dep
)
1651 next
= DEP_CON (dep
);
1653 if (BLOCK_FOR_INSN (next
) != rec
)
1657 if (!contributes_to_priority_p (dep
))
1661 cost
= dep_cost (dep
);
1664 struct _dep _dep1
, *dep1
= &_dep1
;
1666 init_dep (dep1
, insn
, next
, REG_DEP_ANTI
);
1668 cost
= dep_cost (dep1
);
1671 next_priority
= cost
+ priority (next
);
1673 if (next_priority
> this_priority
)
1674 this_priority
= next_priority
;
1678 twin
= PREV_INSN (twin
);
1680 while (twin
!= prev_first
);
1683 if (this_priority
< 0)
1685 gcc_assert (this_priority
== -1);
1687 this_priority
= insn_sched_cost (insn
);
1690 INSN_PRIORITY (insn
) = this_priority
;
1691 INSN_PRIORITY_STATUS (insn
) = 1;
1694 return INSN_PRIORITY (insn
);
1697 /* Macros and functions for keeping the priority queue sorted, and
1698 dealing with queuing and dequeuing of instructions. */
1700 /* For each pressure class CL, set DEATH[CL] to the number of registers
1701 in that class that die in INSN. */
1704 calculate_reg_deaths (rtx_insn
*insn
, int *death
)
1707 struct reg_use_data
*use
;
1709 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1710 death
[ira_pressure_classes
[i
]] = 0;
1711 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
1712 if (dying_use_p (use
))
1713 mark_regno_birth_or_death (0, death
, use
->regno
, true);
1716 /* Setup info about the current register pressure impact of scheduling
1717 INSN at the current scheduling point. */
1719 setup_insn_reg_pressure_info (rtx_insn
*insn
)
1721 int i
, change
, before
, after
, hard_regno
;
1722 int excess_cost_change
;
1725 struct reg_pressure_data
*pressure_info
;
1726 int *max_reg_pressure
;
1727 static int death
[N_REG_CLASSES
];
1729 gcc_checking_assert (!DEBUG_INSN_P (insn
));
1731 excess_cost_change
= 0;
1732 calculate_reg_deaths (insn
, death
);
1733 pressure_info
= INSN_REG_PRESSURE (insn
);
1734 max_reg_pressure
= INSN_MAX_REG_PRESSURE (insn
);
1735 gcc_assert (pressure_info
!= NULL
&& max_reg_pressure
!= NULL
);
1736 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1738 cl
= ira_pressure_classes
[i
];
1739 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1740 change
= (int) pressure_info
[i
].set_increase
- death
[cl
];
1741 before
= MAX (0, max_reg_pressure
[i
] - sched_class_regs_num
[cl
]);
1742 after
= MAX (0, max_reg_pressure
[i
] + change
1743 - sched_class_regs_num
[cl
]);
1744 hard_regno
= ira_class_hard_regs
[cl
][0];
1745 gcc_assert (hard_regno
>= 0);
1746 mode
= reg_raw_mode
[hard_regno
];
1747 excess_cost_change
+= ((after
- before
)
1748 * (ira_memory_move_cost
[mode
][cl
][0]
1749 + ira_memory_move_cost
[mode
][cl
][1]));
1751 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn
) = excess_cost_change
;
1754 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1755 It tries to make the scheduler take register pressure into account
1756 without introducing too many unnecessary stalls. It hooks into the
1757 main scheduling algorithm at several points:
1759 - Before scheduling starts, model_start_schedule constructs a
1760 "model schedule" for the current block. This model schedule is
1761 chosen solely to keep register pressure down. It does not take the
1762 target's pipeline or the original instruction order into account,
1763 except as a tie-breaker. It also doesn't work to a particular
1766 This model schedule gives us an idea of what pressure can be
1767 achieved for the block and gives us an example of a schedule that
1768 keeps to that pressure. It also makes the final schedule less
1769 dependent on the original instruction order. This is important
1770 because the original order can either be "wide" (many values live
1771 at once, such as in user-scheduled code) or "narrow" (few values
1772 live at once, such as after loop unrolling, where several
1773 iterations are executed sequentially).
1775 We do not apply this model schedule to the rtx stream. We simply
1776 record it in model_schedule. We also compute the maximum pressure,
1777 MP, that was seen during this schedule.
1779 - Instructions are added to the ready queue even if they require
1780 a stall. The length of the stall is instead computed as:
1782 MAX (INSN_TICK (INSN) - clock_var, 0)
1784 (= insn_delay). This allows rank_for_schedule to choose between
1785 introducing a deliberate stall or increasing pressure.
1787 - Before sorting the ready queue, model_set_excess_costs assigns
1788 a pressure-based cost to each ready instruction in the queue.
1789 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1790 (ECC for short) and is effectively measured in cycles.
1792 - rank_for_schedule ranks instructions based on:
1794 ECC (insn) + insn_delay (insn)
1800 So, for example, an instruction X1 with an ECC of 1 that can issue
1801 now will win over an instruction X0 with an ECC of zero that would
1802 introduce a stall of one cycle. However, an instruction X2 with an
1803 ECC of 2 that can issue now will lose to both X0 and X1.
1805 - When an instruction is scheduled, model_recompute updates the model
1806 schedule with the new pressures (some of which might now exceed the
1807 original maximum pressure MP). model_update_limit_points then searches
1808 for the new point of maximum pressure, if not already known. */
1810 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1811 from surrounding debug information. */
1813 ";;\t\t+------------------------------------------------------\n"
1815 /* Information about the pressure on a particular register class at a
1816 particular point of the model schedule. */
1817 struct model_pressure_data
{
1818 /* The pressure at this point of the model schedule, or -1 if the
1819 point is associated with an instruction that has already been
1823 /* The maximum pressure during or after this point of the model schedule. */
1827 /* Per-instruction information that is used while building the model
1828 schedule. Here, "schedule" refers to the model schedule rather
1829 than the main schedule. */
1830 struct model_insn_info
{
1831 /* The instruction itself. */
1834 /* If this instruction is in model_worklist, these fields link to the
1835 previous (higher-priority) and next (lower-priority) instructions
1837 struct model_insn_info
*prev
;
1838 struct model_insn_info
*next
;
1840 /* While constructing the schedule, QUEUE_INDEX describes whether an
1841 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1842 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1843 old_queue records the value that QUEUE_INDEX had before scheduling
1844 started, so that we can restore it once the schedule is complete. */
1847 /* The relative importance of an unscheduled instruction. Higher
1848 values indicate greater importance. */
1849 unsigned int model_priority
;
1851 /* The length of the longest path of satisfied true dependencies
1852 that leads to this instruction. */
1855 /* The length of the longest path of dependencies of any kind
1856 that leads from this instruction. */
1859 /* The number of predecessor nodes that must still be scheduled. */
1860 int unscheduled_preds
;
1863 /* Information about the pressure limit for a particular register class.
1864 This structure is used when applying a model schedule to the main
1866 struct model_pressure_limit
{
1867 /* The maximum register pressure seen in the original model schedule. */
1870 /* The maximum register pressure seen in the current model schedule
1871 (which excludes instructions that have already been scheduled). */
1874 /* The point of the current model schedule at which PRESSURE is first
1875 reached. It is set to -1 if the value needs to be recomputed. */
1879 /* Describes a particular way of measuring register pressure. */
1880 struct model_pressure_group
{
1881 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1882 struct model_pressure_limit limits
[N_REG_CLASSES
];
1884 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1885 on register class ira_pressure_classes[PCI] at point POINT of the
1886 current model schedule. A POINT of model_num_insns describes the
1887 pressure at the end of the schedule. */
1888 struct model_pressure_data
*model
;
1891 /* Index POINT gives the instruction at point POINT of the model schedule.
1892 This array doesn't change during main scheduling. */
1893 static vec
<rtx_insn
*> model_schedule
;
1895 /* The list of instructions in the model worklist, sorted in order of
1896 decreasing priority. */
1897 static struct model_insn_info
*model_worklist
;
1899 /* Index I describes the instruction with INSN_LUID I. */
1900 static struct model_insn_info
*model_insns
;
1902 /* The number of instructions in the model schedule. */
1903 static int model_num_insns
;
1905 /* The index of the first instruction in model_schedule that hasn't yet been
1906 added to the main schedule, or model_num_insns if all of them have. */
1907 static int model_curr_point
;
1909 /* Describes the pressure before each instruction in the model schedule. */
1910 static struct model_pressure_group model_before_pressure
;
1912 /* The first unused model_priority value (as used in model_insn_info). */
1913 static unsigned int model_next_priority
;
1916 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1917 at point POINT of the model schedule. */
1918 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1919 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1921 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1922 after point POINT of the model schedule. */
1923 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1924 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1926 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1927 of the model schedule. */
1928 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1929 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1931 /* Information about INSN that is used when creating the model schedule. */
1932 #define MODEL_INSN_INFO(INSN) \
1933 (&model_insns[INSN_LUID (INSN)])
1935 /* The instruction at point POINT of the model schedule. */
1936 #define MODEL_INSN(POINT) \
1937 (model_schedule[POINT])
1940 /* Return INSN's index in the model schedule, or model_num_insns if it
1941 doesn't belong to that schedule. */
1944 model_index (rtx_insn
*insn
)
1946 if (INSN_MODEL_INDEX (insn
) == 0)
1947 return model_num_insns
;
1948 return INSN_MODEL_INDEX (insn
) - 1;
1951 /* Make sure that GROUP->limits is up-to-date for the current point
1952 of the model schedule. */
1955 model_update_limit_points_in_group (struct model_pressure_group
*group
)
1957 int pci
, max_pressure
, point
;
1959 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
1961 /* We may have passed the final point at which the pressure in
1962 group->limits[pci].pressure was reached. Update the limit if so. */
1963 max_pressure
= MODEL_MAX_PRESSURE (group
, model_curr_point
, pci
);
1964 group
->limits
[pci
].pressure
= max_pressure
;
1966 /* Find the point at which MAX_PRESSURE is first reached. We need
1967 to search in three cases:
1969 - We've already moved past the previous pressure point.
1970 In this case we search forward from model_curr_point.
1972 - We scheduled the previous point of maximum pressure ahead of
1973 its position in the model schedule, but doing so didn't bring
1974 the pressure point earlier. In this case we search forward
1975 from that previous pressure point.
1977 - Scheduling an instruction early caused the maximum pressure
1978 to decrease. In this case we will have set the pressure
1979 point to -1, and we search forward from model_curr_point. */
1980 point
= MAX (group
->limits
[pci
].point
, model_curr_point
);
1981 while (point
< model_num_insns
1982 && MODEL_REF_PRESSURE (group
, point
, pci
) < max_pressure
)
1984 group
->limits
[pci
].point
= point
;
1986 gcc_assert (MODEL_REF_PRESSURE (group
, point
, pci
) == max_pressure
);
1987 gcc_assert (MODEL_MAX_PRESSURE (group
, point
, pci
) == max_pressure
);
1991 /* Make sure that all register-pressure limits are up-to-date for the
1992 current position in the model schedule. */
1995 model_update_limit_points (void)
1997 model_update_limit_points_in_group (&model_before_pressure
);
2000 /* Return the model_index of the last unscheduled use in chain USE
2001 outside of USE's instruction. Return -1 if there are no other uses,
2002 or model_num_insns if the register is live at the end of the block. */
2005 model_last_use_except (struct reg_use_data
*use
)
2007 struct reg_use_data
*next
;
2011 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
2012 if (NONDEBUG_INSN_P (next
->insn
)
2013 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
2015 index
= model_index (next
->insn
);
2016 if (index
== model_num_insns
)
2017 return model_num_insns
;
2024 /* An instruction with model_index POINT has just been scheduled, and it
2025 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2026 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2027 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2030 model_start_update_pressure (struct model_pressure_group
*group
,
2031 int point
, int pci
, int delta
)
2033 int next_max_pressure
;
2035 if (point
== model_num_insns
)
2037 /* The instruction wasn't part of the model schedule; it was moved
2038 from a different block. Update the pressure for the end of
2039 the model schedule. */
2040 MODEL_REF_PRESSURE (group
, point
, pci
) += delta
;
2041 MODEL_MAX_PRESSURE (group
, point
, pci
) += delta
;
2045 /* Record that this instruction has been scheduled. Nothing now
2046 changes between POINT and POINT + 1, so get the maximum pressure
2047 from the latter. If the maximum pressure decreases, the new
2048 pressure point may be before POINT. */
2049 MODEL_REF_PRESSURE (group
, point
, pci
) = -1;
2050 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2051 if (MODEL_MAX_PRESSURE (group
, point
, pci
) > next_max_pressure
)
2053 MODEL_MAX_PRESSURE (group
, point
, pci
) = next_max_pressure
;
2054 if (group
->limits
[pci
].point
== point
)
2055 group
->limits
[pci
].point
= -1;
2060 /* Record that scheduling a later instruction has changed the pressure
2061 at point POINT of the model schedule by DELTA (which might be 0).
2062 Update GROUP accordingly. Return nonzero if these changes might
2063 trigger changes to previous points as well. */
2066 model_update_pressure (struct model_pressure_group
*group
,
2067 int point
, int pci
, int delta
)
2069 int ref_pressure
, max_pressure
, next_max_pressure
;
2071 /* If POINT hasn't yet been scheduled, update its pressure. */
2072 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
2073 if (ref_pressure
>= 0 && delta
!= 0)
2075 ref_pressure
+= delta
;
2076 MODEL_REF_PRESSURE (group
, point
, pci
) = ref_pressure
;
2078 /* Check whether the maximum pressure in the overall schedule
2079 has increased. (This means that the MODEL_MAX_PRESSURE of
2080 every point <= POINT will need to increase too; see below.) */
2081 if (group
->limits
[pci
].pressure
< ref_pressure
)
2082 group
->limits
[pci
].pressure
= ref_pressure
;
2084 /* If we are at maximum pressure, and the maximum pressure
2085 point was previously unknown or later than POINT,
2086 bring it forward. */
2087 if (group
->limits
[pci
].pressure
== ref_pressure
2088 && !IN_RANGE (group
->limits
[pci
].point
, 0, point
))
2089 group
->limits
[pci
].point
= point
;
2091 /* If POINT used to be the point of maximum pressure, but isn't
2092 any longer, we need to recalculate it using a forward walk. */
2093 if (group
->limits
[pci
].pressure
> ref_pressure
2094 && group
->limits
[pci
].point
== point
)
2095 group
->limits
[pci
].point
= -1;
2098 /* Update the maximum pressure at POINT. Changes here might also
2099 affect the maximum pressure at POINT - 1. */
2100 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2101 max_pressure
= MAX (ref_pressure
, next_max_pressure
);
2102 if (MODEL_MAX_PRESSURE (group
, point
, pci
) != max_pressure
)
2104 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
2110 /* INSN has just been scheduled. Update the model schedule accordingly. */
2113 model_recompute (rtx_insn
*insn
)
2118 } uses
[FIRST_PSEUDO_REGISTER
+ MAX_RECOG_OPERANDS
];
2119 struct reg_use_data
*use
;
2120 struct reg_pressure_data
*reg_pressure
;
2121 int delta
[N_REG_CLASSES
];
2122 int pci
, point
, mix
, new_last
, cl
, ref_pressure
, queue
;
2123 unsigned int i
, num_uses
, num_pending_births
;
2126 /* The destinations of INSN were previously live from POINT onwards, but are
2127 now live from model_curr_point onwards. Set up DELTA accordingly. */
2128 point
= model_index (insn
);
2129 reg_pressure
= INSN_REG_PRESSURE (insn
);
2130 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2132 cl
= ira_pressure_classes
[pci
];
2133 delta
[cl
] = reg_pressure
[pci
].set_increase
;
2136 /* Record which registers previously died at POINT, but which now die
2137 before POINT. Adjust DELTA so that it represents the effect of
2138 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2139 registers that will be born in the range [model_curr_point, POINT). */
2141 num_pending_births
= 0;
2142 bitmap_clear (tmp_bitmap
);
2143 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
2145 new_last
= model_last_use_except (use
);
2146 if (new_last
< point
&& bitmap_set_bit (tmp_bitmap
, use
->regno
))
2148 gcc_assert (num_uses
< ARRAY_SIZE (uses
));
2149 uses
[num_uses
].last_use
= new_last
;
2150 uses
[num_uses
].regno
= use
->regno
;
2151 /* This register is no longer live after POINT - 1. */
2152 mark_regno_birth_or_death (NULL
, delta
, use
->regno
, false);
2155 num_pending_births
++;
2159 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2160 Also set each group pressure limit for POINT. */
2161 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2163 cl
= ira_pressure_classes
[pci
];
2164 model_start_update_pressure (&model_before_pressure
,
2165 point
, pci
, delta
[cl
]);
2168 /* Walk the model schedule backwards, starting immediately before POINT. */
2170 if (point
!= model_curr_point
)
2174 insn
= MODEL_INSN (point
);
2175 queue
= QUEUE_INDEX (insn
);
2177 if (queue
!= QUEUE_SCHEDULED
)
2179 /* DELTA describes the effect of the move on the register pressure
2180 after POINT. Make it describe the effect on the pressure
2183 while (i
< num_uses
)
2185 if (uses
[i
].last_use
== point
)
2187 /* This register is now live again. */
2188 mark_regno_birth_or_death (NULL
, delta
,
2189 uses
[i
].regno
, true);
2191 /* Remove this use from the array. */
2192 uses
[i
] = uses
[num_uses
- 1];
2194 num_pending_births
--;
2200 if (sched_verbose
>= 5)
2204 fprintf (sched_dump
, MODEL_BAR
);
2205 fprintf (sched_dump
, ";;\t\t| New pressure for model"
2207 fprintf (sched_dump
, MODEL_BAR
);
2211 fprintf (sched_dump
, ";;\t\t| %3d %4d %-30s ",
2212 point
, INSN_UID (insn
),
2213 str_pattern_slim (PATTERN (insn
)));
2214 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2216 cl
= ira_pressure_classes
[pci
];
2217 ref_pressure
= MODEL_REF_PRESSURE (&model_before_pressure
,
2219 fprintf (sched_dump
, " %s:[%d->%d]",
2220 reg_class_names
[ira_pressure_classes
[pci
]],
2221 ref_pressure
, ref_pressure
+ delta
[cl
]);
2223 fprintf (sched_dump
, "\n");
2227 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2228 might have changed as well. */
2229 mix
= num_pending_births
;
2230 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2232 cl
= ira_pressure_classes
[pci
];
2234 mix
|= model_update_pressure (&model_before_pressure
,
2235 point
, pci
, delta
[cl
]);
2238 while (mix
&& point
> model_curr_point
);
2241 fprintf (sched_dump
, MODEL_BAR
);
2244 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2245 check whether the insn's pattern needs restoring. */
2247 must_restore_pattern_p (rtx_insn
*next
, dep_t dep
)
2249 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
2252 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
2254 gcc_assert (ORIG_PAT (next
) != NULL_RTX
);
2255 gcc_assert (next
== DEP_CON (dep
));
2259 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
2260 if (desc
->insn
!= next
)
2262 gcc_assert (*desc
->loc
== desc
->orig
);
2269 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2270 pressure on CL from P to P'. We use this to calculate a "base ECC",
2271 baseECC (CL, X), for each pressure class CL and each instruction X.
2272 Supposing X changes the pressure on CL from P to P', and that the
2273 maximum pressure on CL in the current model schedule is MP', then:
2275 * if X occurs before or at the next point of maximum pressure in
2276 the model schedule and P' > MP', then:
2278 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2280 The idea is that the pressure after scheduling a fixed set of
2281 instructions -- in this case, the set up to and including the
2282 next maximum pressure point -- is going to be the same regardless
2283 of the order; we simply want to keep the intermediate pressure
2284 under control. Thus X has a cost of zero unless scheduling it
2285 now would exceed MP'.
2287 If all increases in the set are by the same amount, no zero-cost
2288 instruction will ever cause the pressure to exceed MP'. However,
2289 if X is instead moved past an instruction X' with pressure in the
2290 range (MP' - (P' - P), MP'), the pressure at X' will increase
2291 beyond MP'. Since baseECC is very much a heuristic anyway,
2292 it doesn't seem worth the overhead of tracking cases like these.
2294 The cost of exceeding MP' is always based on the original maximum
2295 pressure MP. This is so that going 2 registers over the original
2296 limit has the same cost regardless of whether it comes from two
2297 separate +1 deltas or from a single +2 delta.
2299 * if X occurs after the next point of maximum pressure in the model
2300 schedule and P' > P, then:
2302 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2304 That is, if we move X forward across a point of maximum pressure,
2305 and if X increases the pressure by P' - P, then we conservatively
2306 assume that scheduling X next would increase the maximum pressure
2307 by P' - P. Again, the cost of doing this is based on the original
2308 maximum pressure MP, for the same reason as above.
2310 * if P' < P, P > MP, and X occurs at or after the next point of
2311 maximum pressure, then:
2313 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2315 That is, if we have already exceeded the original maximum pressure MP,
2316 and if X might reduce the maximum pressure again -- or at least push
2317 it further back, and thus allow more scheduling freedom -- it is given
2318 a negative cost to reflect the improvement.
2324 In this case, X is not expected to affect the maximum pressure MP',
2325 so it has zero cost.
2327 We then create a combined value baseECC (X) that is the sum of
2328 baseECC (CL, X) for each pressure class CL.
2330 baseECC (X) could itself be used as the ECC value described above.
2331 However, this is often too conservative, in the sense that it
2332 tends to make high-priority instructions that increase pressure
2333 wait too long in cases where introducing a spill would be better.
2334 For this reason the final ECC is a priority-adjusted form of
2335 baseECC (X). Specifically, we calculate:
2337 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2338 baseP = MAX { P (X) | baseECC (X) <= 0 }
2342 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2344 Thus an instruction's effect on pressure is ignored if it has a high
2345 enough priority relative to the ones that don't increase pressure.
2346 Negative values of baseECC (X) do not increase the priority of X
2347 itself, but they do make it harder for other instructions to
2348 increase the pressure further.
2350 This pressure cost is deliberately timid. The intention has been
2351 to choose a heuristic that rarely interferes with the normal list
2352 scheduler in cases where that scheduler would produce good code.
2353 We simply want to curb some of its worst excesses. */
2355 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2357 Here we use the very simplistic cost model that every register above
2358 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2359 measures instead, such as one based on MEMORY_MOVE_COST. However:
2361 (1) In order for an instruction to be scheduled, the higher cost
2362 would need to be justified in a single saving of that many stalls.
2363 This is overly pessimistic, because the benefit of spilling is
2364 often to avoid a sequence of several short stalls rather than
2367 (2) The cost is still arbitrary. Because we are not allocating
2368 registers during scheduling, we have no way of knowing for
2369 sure how many memory accesses will be required by each spill,
2370 where the spills will be placed within the block, or even
2371 which block(s) will contain the spills.
2373 So a higher cost than 1 is often too conservative in practice,
2374 forcing blocks to contain unnecessary stalls instead of spill code.
2375 The simple cost below seems to be the best compromise. It reduces
2376 the interference with the normal list scheduler, which helps make
2377 it more suitable for a default-on option. */
2380 model_spill_cost (int cl
, int from
, int to
)
2382 from
= MAX (from
, sched_class_regs_num
[cl
]);
2383 return MAX (to
, from
) - from
;
2386 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2387 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2391 model_excess_group_cost (struct model_pressure_group
*group
,
2392 int point
, int pci
, int delta
)
2396 cl
= ira_pressure_classes
[pci
];
2397 if (delta
< 0 && point
>= group
->limits
[pci
].point
)
2399 pressure
= MAX (group
->limits
[pci
].orig_pressure
,
2400 curr_reg_pressure
[cl
] + delta
);
2401 return -model_spill_cost (cl
, pressure
, curr_reg_pressure
[cl
]);
2406 if (point
> group
->limits
[pci
].point
)
2407 pressure
= group
->limits
[pci
].pressure
+ delta
;
2409 pressure
= curr_reg_pressure
[cl
] + delta
;
2411 if (pressure
> group
->limits
[pci
].pressure
)
2412 return model_spill_cost (cl
, group
->limits
[pci
].orig_pressure
,
2419 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2423 model_excess_cost (rtx_insn
*insn
, bool print_p
)
2425 int point
, pci
, cl
, cost
, this_cost
, delta
;
2426 struct reg_pressure_data
*insn_reg_pressure
;
2427 int insn_death
[N_REG_CLASSES
];
2429 calculate_reg_deaths (insn
, insn_death
);
2430 point
= model_index (insn
);
2431 insn_reg_pressure
= INSN_REG_PRESSURE (insn
);
2435 fprintf (sched_dump
, ";;\t\t| %3d %4d | %4d %+3d |", point
,
2436 INSN_UID (insn
), INSN_PRIORITY (insn
), insn_delay (insn
));
2438 /* Sum up the individual costs for each register class. */
2439 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2441 cl
= ira_pressure_classes
[pci
];
2442 delta
= insn_reg_pressure
[pci
].set_increase
- insn_death
[cl
];
2443 this_cost
= model_excess_group_cost (&model_before_pressure
,
2447 fprintf (sched_dump
, " %s:[%d base cost %d]",
2448 reg_class_names
[cl
], delta
, this_cost
);
2452 fprintf (sched_dump
, "\n");
2457 /* Dump the next points of maximum pressure for GROUP. */
2460 model_dump_pressure_points (struct model_pressure_group
*group
)
2464 fprintf (sched_dump
, ";;\t\t| pressure points");
2465 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2467 cl
= ira_pressure_classes
[pci
];
2468 fprintf (sched_dump
, " %s:[%d->%d at ", reg_class_names
[cl
],
2469 curr_reg_pressure
[cl
], group
->limits
[pci
].pressure
);
2470 if (group
->limits
[pci
].point
< model_num_insns
)
2471 fprintf (sched_dump
, "%d:%d]", group
->limits
[pci
].point
,
2472 INSN_UID (MODEL_INSN (group
->limits
[pci
].point
)));
2474 fprintf (sched_dump
, "end]");
2476 fprintf (sched_dump
, "\n");
2479 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2482 model_set_excess_costs (rtx_insn
**insns
, int count
)
2484 int i
, cost
, priority_base
, priority
;
2487 /* Record the baseECC value for each instruction in the model schedule,
2488 except that negative costs are converted to zero ones now rather than
2489 later. Do not assign a cost to debug instructions, since they must
2490 not change code-generation decisions. Experiments suggest we also
2491 get better results by not assigning a cost to instructions from
2494 Set PRIORITY_BASE to baseP in the block comment above. This is the
2495 maximum priority of the "cheap" instructions, which should always
2496 include the next model instruction. */
2499 for (i
= 0; i
< count
; i
++)
2500 if (INSN_MODEL_INDEX (insns
[i
]))
2502 if (sched_verbose
>= 6 && !print_p
)
2504 fprintf (sched_dump
, MODEL_BAR
);
2505 fprintf (sched_dump
, ";;\t\t| Pressure costs for ready queue\n");
2506 model_dump_pressure_points (&model_before_pressure
);
2507 fprintf (sched_dump
, MODEL_BAR
);
2510 cost
= model_excess_cost (insns
[i
], print_p
);
2513 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]) - cost
;
2514 priority_base
= MAX (priority_base
, priority
);
2517 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = cost
;
2520 fprintf (sched_dump
, MODEL_BAR
);
2522 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2524 for (i
= 0; i
< count
; i
++)
2526 cost
= INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]);
2527 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]);
2528 if (cost
> 0 && priority
> priority_base
)
2530 cost
+= priority_base
- priority
;
2531 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = MAX (cost
, 0);
2537 /* Enum of rank_for_schedule heuristic decisions. */
2539 RFS_LIVE_RANGE_SHRINK1
, RFS_LIVE_RANGE_SHRINK2
,
2540 RFS_SCHED_GROUP
, RFS_PRESSURE_DELAY
, RFS_PRESSURE_TICK
,
2541 RFS_FEEDS_BACKTRACK_INSN
, RFS_PRIORITY
, RFS_AUTOPREF
, RFS_SPECULATION
,
2542 RFS_SCHED_RANK
, RFS_LAST_INSN
, RFS_PRESSURE_INDEX
,
2543 RFS_DEP_COUNT
, RFS_TIE
, RFS_FUSION
, RFS_COST
, RFS_N
};
2545 /* Corresponding strings for print outs. */
2546 static const char *rfs_str
[RFS_N
] = {
2547 "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2548 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2549 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_AUTOPREF", "RFS_SPECULATION",
2550 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2551 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
2553 /* Statistical breakdown of rank_for_schedule decisions. */
2554 struct rank_for_schedule_stats_t
{ unsigned stats
[RFS_N
]; };
2555 static rank_for_schedule_stats_t rank_for_schedule_stats
;
2557 /* Return the result of comparing insns TMP and TMP2 and update
2558 Rank_For_Schedule statistics. */
2560 rfs_result (enum rfs_decision decision
, int result
, rtx tmp
, rtx tmp2
)
2562 ++rank_for_schedule_stats
.stats
[decision
];
2564 INSN_LAST_RFS_WIN (tmp
) = decision
;
2565 else if (result
> 0)
2566 INSN_LAST_RFS_WIN (tmp2
) = decision
;
2572 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2573 keeping normal insns in original order. */
2576 rank_for_schedule_debug (const void *x
, const void *y
)
2578 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2579 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2581 /* Schedule debug insns as early as possible. */
2582 if (DEBUG_INSN_P (tmp
) && !DEBUG_INSN_P (tmp2
))
2584 else if (!DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2586 else if (DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2587 return INSN_LUID (tmp
) - INSN_LUID (tmp2
);
2589 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2
) - INSN_RFS_DEBUG_ORIG_ORDER (tmp
);
2592 /* Returns a positive value if x is preferred; returns a negative value if
2593 y is preferred. Should never return 0, since that will make the sort
2597 rank_for_schedule (const void *x
, const void *y
)
2599 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2600 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2601 int tmp_class
, tmp2_class
;
2602 int val
, priority_val
, info_val
, diff
;
2604 if (live_range_shrinkage_p
)
2606 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2608 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
2609 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
) < 0
2610 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
) < 0)
2611 && (diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2612 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
))) != 0)
2613 return rfs_result (RFS_LIVE_RANGE_SHRINK1
, diff
, tmp
, tmp2
);
2614 /* Sort by INSN_LUID (original insn order), so that we make the
2615 sort stable. This minimizes instruction movement, thus
2616 minimizing sched's effect on debugging and cross-jumping. */
2617 return rfs_result (RFS_LIVE_RANGE_SHRINK2
,
2618 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2621 /* The insn in a schedule group should be issued the first. */
2622 if (flag_sched_group_heuristic
&&
2623 SCHED_GROUP_P (tmp
) != SCHED_GROUP_P (tmp2
))
2624 return rfs_result (RFS_SCHED_GROUP
, SCHED_GROUP_P (tmp2
) ? 1 : -1,
2627 /* Make sure that priority of TMP and TMP2 are initialized. */
2628 gcc_assert (INSN_PRIORITY_KNOWN (tmp
) && INSN_PRIORITY_KNOWN (tmp2
));
2632 /* The instruction that has the same fusion priority as the last
2633 instruction is the instruction we picked next. If that is not
2634 the case, we sort ready list firstly by fusion priority, then
2635 by priority, and at last by INSN_LUID. */
2636 int a
= INSN_FUSION_PRIORITY (tmp
);
2637 int b
= INSN_FUSION_PRIORITY (tmp2
);
2640 if (last_nondebug_scheduled_insn
2641 && !NOTE_P (last_nondebug_scheduled_insn
)
2642 && BLOCK_FOR_INSN (tmp
)
2643 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn
))
2644 last
= INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn
);
2646 if (a
!= last
&& b
!= last
)
2650 a
= INSN_PRIORITY (tmp
);
2651 b
= INSN_PRIORITY (tmp2
);
2654 return rfs_result (RFS_FUSION
, b
- a
, tmp
, tmp2
);
2656 return rfs_result (RFS_FUSION
,
2657 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2661 gcc_assert (last_nondebug_scheduled_insn
2662 && !NOTE_P (last_nondebug_scheduled_insn
));
2663 last
= INSN_PRIORITY (last_nondebug_scheduled_insn
);
2665 a
= abs (INSN_PRIORITY (tmp
) - last
);
2666 b
= abs (INSN_PRIORITY (tmp2
) - last
);
2668 return rfs_result (RFS_FUSION
, a
- b
, tmp
, tmp2
);
2670 return rfs_result (RFS_FUSION
,
2671 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2674 return rfs_result (RFS_FUSION
, -1, tmp
, tmp2
);
2676 return rfs_result (RFS_FUSION
, 1, tmp
, tmp2
);
2679 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
2681 /* Prefer insn whose scheduling results in the smallest register
2683 if ((diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2685 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
)
2686 - insn_delay (tmp2
))))
2687 return rfs_result (RFS_PRESSURE_DELAY
, diff
, tmp
, tmp2
);
2690 if (sched_pressure
!= SCHED_PRESSURE_NONE
2691 && (INSN_TICK (tmp2
) > clock_var
|| INSN_TICK (tmp
) > clock_var
)
2692 && INSN_TICK (tmp2
) != INSN_TICK (tmp
))
2694 diff
= INSN_TICK (tmp
) - INSN_TICK (tmp2
);
2695 return rfs_result (RFS_PRESSURE_TICK
, diff
, tmp
, tmp2
);
2698 /* If we are doing backtracking in this schedule, prefer insns that
2699 have forward dependencies with negative cost against an insn that
2700 was already scheduled. */
2701 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2703 priority_val
= FEEDS_BACKTRACK_INSN (tmp2
) - FEEDS_BACKTRACK_INSN (tmp
);
2705 return rfs_result (RFS_FEEDS_BACKTRACK_INSN
, priority_val
, tmp
, tmp2
);
2708 /* Prefer insn with higher priority. */
2709 priority_val
= INSN_PRIORITY (tmp2
) - INSN_PRIORITY (tmp
);
2711 if (flag_sched_critical_path_heuristic
&& priority_val
)
2712 return rfs_result (RFS_PRIORITY
, priority_val
, tmp
, tmp2
);
2714 if (param_sched_autopref_queue_depth
>= 0)
2716 int autopref
= autopref_rank_for_schedule (tmp
, tmp2
);
2718 return rfs_result (RFS_AUTOPREF
, autopref
, tmp
, tmp2
);
2721 /* Prefer speculative insn with greater dependencies weakness. */
2722 if (flag_sched_spec_insn_heuristic
&& spec_info
)
2728 ds1
= TODO_SPEC (tmp
) & SPECULATIVE
;
2730 dw1
= ds_weak (ds1
);
2734 ds2
= TODO_SPEC (tmp2
) & SPECULATIVE
;
2736 dw2
= ds_weak (ds2
);
2741 if (dw
> (NO_DEP_WEAK
/ 8) || dw
< -(NO_DEP_WEAK
/ 8))
2742 return rfs_result (RFS_SPECULATION
, dw
, tmp
, tmp2
);
2745 info_val
= (*current_sched_info
->rank
) (tmp
, tmp2
);
2746 if (flag_sched_rank_heuristic
&& info_val
)
2747 return rfs_result (RFS_SCHED_RANK
, info_val
, tmp
, tmp2
);
2749 /* Compare insns based on their relation to the last scheduled
2751 if (flag_sched_last_insn_heuristic
&& last_nondebug_scheduled_insn
)
2755 rtx_insn
*last
= last_nondebug_scheduled_insn
;
2757 /* Classify the instructions into three classes:
2758 1) Data dependent on last schedule insn.
2759 2) Anti/Output dependent on last scheduled insn.
2760 3) Independent of last scheduled insn, or has latency of one.
2761 Choose the insn from the highest numbered class if different. */
2762 dep1
= sd_find_dep_between (last
, tmp
, true);
2764 if (dep1
== NULL
|| dep_cost (dep1
) == 1)
2766 else if (/* Data dependence. */
2767 DEP_TYPE (dep1
) == REG_DEP_TRUE
)
2772 dep2
= sd_find_dep_between (last
, tmp2
, true);
2774 if (dep2
== NULL
|| dep_cost (dep2
) == 1)
2776 else if (/* Data dependence. */
2777 DEP_TYPE (dep2
) == REG_DEP_TRUE
)
2782 if ((val
= tmp2_class
- tmp_class
))
2783 return rfs_result (RFS_LAST_INSN
, val
, tmp
, tmp2
);
2786 /* Prefer instructions that occur earlier in the model schedule. */
2787 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
2789 diff
= model_index (tmp
) - model_index (tmp2
);
2791 return rfs_result (RFS_PRESSURE_INDEX
, diff
, tmp
, tmp2
);
2794 /* Prefer the insn which has more later insns that depend on it.
2795 This gives the scheduler more freedom when scheduling later
2796 instructions at the expense of added register pressure. */
2798 val
= (dep_list_size (tmp2
, SD_LIST_FORW
)
2799 - dep_list_size (tmp
, SD_LIST_FORW
));
2801 if (flag_sched_dep_count_heuristic
&& val
!= 0)
2802 return rfs_result (RFS_DEP_COUNT
, val
, tmp
, tmp2
);
2804 /* Sort by INSN_COST rather than INSN_LUID. This means that instructions
2805 which take longer to execute are prioritised and it leads to more
2806 dual-issue opportunities on in-order cores which have this feature. */
2808 if (INSN_COST (tmp
) != INSN_COST (tmp2
))
2809 return rfs_result (RFS_COST
, INSN_COST (tmp2
) - INSN_COST (tmp
),
2812 /* If insns are equally good, sort by INSN_LUID (original insn order),
2813 so that we make the sort stable. This minimizes instruction movement,
2814 thus minimizing sched's effect on debugging and cross-jumping. */
2815 return rfs_result (RFS_TIE
, INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2818 /* Resort the array A in which only element at index N may be out of order. */
2820 HAIFA_INLINE
static void
2821 swap_sort (rtx_insn
**a
, int n
)
2823 rtx_insn
*insn
= a
[n
- 1];
2826 while (i
>= 0 && rank_for_schedule (a
+ i
, &insn
) >= 0)
2834 /* Add INSN to the insn queue so that it can be executed at least
2835 N_CYCLES after the currently executing insn. Preserve insns
2836 chain for debugging purposes. REASON will be printed in debugging
2839 HAIFA_INLINE
static void
2840 queue_insn (rtx_insn
*insn
, int n_cycles
, const char *reason
)
2842 int next_q
= NEXT_Q_AFTER (q_ptr
, n_cycles
);
2843 rtx_insn_list
*link
= alloc_INSN_LIST (insn
, insn_queue
[next_q
]);
2846 gcc_assert (n_cycles
<= max_insn_queue_index
);
2847 gcc_assert (!DEBUG_INSN_P (insn
));
2849 insn_queue
[next_q
] = link
;
2852 if (sched_verbose
>= 2)
2854 fprintf (sched_dump
, ";;\t\tReady-->Q: insn %s: ",
2855 (*current_sched_info
->print_insn
) (insn
, 0));
2857 fprintf (sched_dump
, "queued for %d cycles (%s).\n", n_cycles
, reason
);
2860 QUEUE_INDEX (insn
) = next_q
;
2862 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2864 new_tick
= clock_var
+ n_cycles
;
2865 if (INSN_TICK (insn
) == INVALID_TICK
|| INSN_TICK (insn
) < new_tick
)
2866 INSN_TICK (insn
) = new_tick
;
2868 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2869 && INSN_EXACT_TICK (insn
) < clock_var
+ n_cycles
)
2871 must_backtrack
= true;
2872 if (sched_verbose
>= 2)
2873 fprintf (sched_dump
, ";;\t\tcausing a backtrack.\n");
2878 /* Remove INSN from queue. */
2880 queue_remove (rtx_insn
*insn
)
2882 gcc_assert (QUEUE_INDEX (insn
) >= 0);
2883 remove_free_INSN_LIST_elem (insn
, &insn_queue
[QUEUE_INDEX (insn
)]);
2885 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
2888 /* Return a pointer to the bottom of the ready list, i.e. the insn
2889 with the lowest priority. */
2892 ready_lastpos (struct ready_list
*ready
)
2894 gcc_assert (ready
->n_ready
>= 1);
2895 return ready
->vec
+ ready
->first
- ready
->n_ready
+ 1;
2898 /* Add an element INSN to the ready list so that it ends up with the
2899 lowest/highest priority depending on FIRST_P. */
2901 HAIFA_INLINE
static void
2902 ready_add (struct ready_list
*ready
, rtx_insn
*insn
, bool first_p
)
2906 if (ready
->first
== ready
->n_ready
)
2908 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
,
2909 ready_lastpos (ready
),
2910 ready
->n_ready
* sizeof (rtx
));
2911 ready
->first
= ready
->veclen
- 1;
2913 ready
->vec
[ready
->first
- ready
->n_ready
] = insn
;
2917 if (ready
->first
== ready
->veclen
- 1)
2920 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2921 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
- 1,
2922 ready_lastpos (ready
),
2923 ready
->n_ready
* sizeof (rtx
));
2924 ready
->first
= ready
->veclen
- 2;
2926 ready
->vec
[++(ready
->first
)] = insn
;
2930 if (DEBUG_INSN_P (insn
))
2933 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_READY
);
2934 QUEUE_INDEX (insn
) = QUEUE_READY
;
2936 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2937 && INSN_EXACT_TICK (insn
) < clock_var
)
2939 must_backtrack
= true;
2943 /* Remove the element with the highest priority from the ready list and
2946 HAIFA_INLINE
static rtx_insn
*
2947 ready_remove_first (struct ready_list
*ready
)
2951 gcc_assert (ready
->n_ready
);
2952 t
= ready
->vec
[ready
->first
--];
2954 if (DEBUG_INSN_P (t
))
2956 /* If the queue becomes empty, reset it. */
2957 if (ready
->n_ready
== 0)
2958 ready
->first
= ready
->veclen
- 1;
2960 gcc_assert (QUEUE_INDEX (t
) == QUEUE_READY
);
2961 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
2966 /* The following code implements multi-pass scheduling for the first
2967 cycle. In other words, we will try to choose ready insn which
2968 permits to start maximum number of insns on the same cycle. */
2970 /* Return a pointer to the element INDEX from the ready. INDEX for
2971 insn with the highest priority is 0, and the lowest priority has
2975 ready_element (struct ready_list
*ready
, int index
)
2977 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
2979 return ready
->vec
[ready
->first
- index
];
2982 /* Remove the element INDEX from the ready list and return it. INDEX
2983 for insn with the highest priority is 0, and the lowest priority
2986 HAIFA_INLINE
static rtx_insn
*
2987 ready_remove (struct ready_list
*ready
, int index
)
2993 return ready_remove_first (ready
);
2994 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
2995 t
= ready
->vec
[ready
->first
- index
];
2997 if (DEBUG_INSN_P (t
))
2999 for (i
= index
; i
< ready
->n_ready
; i
++)
3000 ready
->vec
[ready
->first
- i
] = ready
->vec
[ready
->first
- i
- 1];
3001 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
3005 /* Remove INSN from the ready list. */
3007 ready_remove_insn (rtx_insn
*insn
)
3011 for (i
= 0; i
< readyp
->n_ready
; i
++)
3012 if (ready_element (readyp
, i
) == insn
)
3014 ready_remove (readyp
, i
);
3020 /* Calculate difference of two statistics set WAS and NOW.
3021 Result returned in WAS. */
3023 rank_for_schedule_stats_diff (rank_for_schedule_stats_t
*was
,
3024 const rank_for_schedule_stats_t
*now
)
3026 for (int i
= 0; i
< RFS_N
; ++i
)
3027 was
->stats
[i
] = now
->stats
[i
] - was
->stats
[i
];
3030 /* Print rank_for_schedule statistics. */
3032 print_rank_for_schedule_stats (const char *prefix
,
3033 const rank_for_schedule_stats_t
*stats
,
3034 struct ready_list
*ready
)
3036 for (int i
= 0; i
< RFS_N
; ++i
)
3037 if (stats
->stats
[i
])
3039 fprintf (sched_dump
, "%s%20s: %u", prefix
, rfs_str
[i
], stats
->stats
[i
]);
3042 /* Print out insns that won due to RFS_<I>. */
3044 rtx_insn
**p
= ready_lastpos (ready
);
3046 fprintf (sched_dump
, ":");
3047 /* Start with 1 since least-priority insn didn't have any wins. */
3048 for (int j
= 1; j
< ready
->n_ready
; ++j
)
3049 if (INSN_LAST_RFS_WIN (p
[j
]) == i
)
3050 fprintf (sched_dump
, " %s",
3051 (*current_sched_info
->print_insn
) (p
[j
], 0));
3053 fprintf (sched_dump
, "\n");
3057 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3060 ready_sort_debug (struct ready_list
*ready
)
3063 rtx_insn
**first
= ready_lastpos (ready
);
3065 for (i
= 0; i
< ready
->n_ready
; ++i
)
3066 if (!DEBUG_INSN_P (first
[i
]))
3067 INSN_RFS_DEBUG_ORIG_ORDER (first
[i
]) = i
;
3069 qsort (first
, ready
->n_ready
, sizeof (rtx
), rank_for_schedule_debug
);
3072 /* Sort non-debug insns in the ready list READY by ascending priority.
3073 Assumes that all debug insns are separated from the real insns. */
3075 ready_sort_real (struct ready_list
*ready
)
3078 rtx_insn
**first
= ready_lastpos (ready
);
3079 int n_ready_real
= ready
->n_ready
- ready
->n_debug
;
3081 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
3082 for (i
= 0; i
< n_ready_real
; ++i
)
3083 setup_insn_reg_pressure_info (first
[i
]);
3084 else if (sched_pressure
== SCHED_PRESSURE_MODEL
3085 && model_curr_point
< model_num_insns
)
3086 model_set_excess_costs (first
, n_ready_real
);
3088 rank_for_schedule_stats_t stats1
;
3089 if (sched_verbose
>= 4)
3090 stats1
= rank_for_schedule_stats
;
3092 if (n_ready_real
== 2)
3093 swap_sort (first
, n_ready_real
);
3094 else if (n_ready_real
> 2)
3095 qsort (first
, n_ready_real
, sizeof (rtx
), rank_for_schedule
);
3097 if (sched_verbose
>= 4)
3099 rank_for_schedule_stats_diff (&stats1
, &rank_for_schedule_stats
);
3100 print_rank_for_schedule_stats (";;\t\t", &stats1
, ready
);
3104 /* Sort the ready list READY by ascending priority. */
3106 ready_sort (struct ready_list
*ready
)
3108 if (ready
->n_debug
> 0)
3109 ready_sort_debug (ready
);
3111 ready_sort_real (ready
);
3114 /* PREV is an insn that is ready to execute. Adjust its priority if that
3115 will help shorten or lengthen register lifetimes as appropriate. Also
3116 provide a hook for the target to tweak itself. */
3118 HAIFA_INLINE
static void
3119 adjust_priority (rtx_insn
*prev
)
3121 /* ??? There used to be code here to try and estimate how an insn
3122 affected register lifetimes, but it did it by looking at REG_DEAD
3123 notes, which we removed in schedule_region. Nor did it try to
3124 take into account register pressure or anything useful like that.
3126 Revisit when we have a machine model to work with and not before. */
3128 if (targetm
.sched
.adjust_priority
)
3129 INSN_PRIORITY (prev
) =
3130 targetm
.sched
.adjust_priority (prev
, INSN_PRIORITY (prev
));
3133 /* Advance DFA state STATE on one cycle. */
3135 advance_state (state_t state
)
3137 if (targetm
.sched
.dfa_pre_advance_cycle
)
3138 targetm
.sched
.dfa_pre_advance_cycle ();
3140 if (targetm
.sched
.dfa_pre_cycle_insn
)
3141 state_transition (state
,
3142 targetm
.sched
.dfa_pre_cycle_insn ());
3144 state_transition (state
, NULL
);
3146 if (targetm
.sched
.dfa_post_cycle_insn
)
3147 state_transition (state
,
3148 targetm
.sched
.dfa_post_cycle_insn ());
3150 if (targetm
.sched
.dfa_post_advance_cycle
)
3151 targetm
.sched
.dfa_post_advance_cycle ();
3154 /* Advance time on one cycle. */
3155 HAIFA_INLINE
static void
3156 advance_one_cycle (void)
3160 advance_state (curr_state
);
3161 for (i
= 4; i
<= sched_verbose
; ++i
)
3162 fprintf (sched_dump
, ";;\tAdvance the current state: %d.\n", clock_var
);
3165 /* Update register pressure after scheduling INSN. */
3167 update_register_pressure (rtx_insn
*insn
)
3169 struct reg_use_data
*use
;
3170 struct reg_set_data
*set
;
3172 gcc_checking_assert (!DEBUG_INSN_P (insn
));
3174 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
3175 if (dying_use_p (use
))
3176 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3178 for (set
= INSN_REG_SET_LIST (insn
); set
!= NULL
; set
= set
->next_insn_set
)
3179 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3183 /* Set up or update (if UPDATE_P) max register pressure (see its
3184 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3185 after insn AFTER. */
3187 setup_insn_max_reg_pressure (rtx_insn
*after
, bool update_p
)
3192 static int max_reg_pressure
[N_REG_CLASSES
];
3194 save_reg_pressure ();
3195 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3196 max_reg_pressure
[ira_pressure_classes
[i
]]
3197 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3198 for (insn
= NEXT_INSN (after
);
3199 insn
!= NULL_RTX
&& ! BARRIER_P (insn
)
3200 && BLOCK_FOR_INSN (insn
) == BLOCK_FOR_INSN (after
);
3201 insn
= NEXT_INSN (insn
))
3202 if (NONDEBUG_INSN_P (insn
))
3205 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3207 p
= max_reg_pressure
[ira_pressure_classes
[i
]];
3208 if (INSN_MAX_REG_PRESSURE (insn
)[i
] != p
)
3211 INSN_MAX_REG_PRESSURE (insn
)[i
]
3212 = max_reg_pressure
[ira_pressure_classes
[i
]];
3215 if (update_p
&& eq_p
)
3217 update_register_pressure (insn
);
3218 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3219 if (max_reg_pressure
[ira_pressure_classes
[i
]]
3220 < curr_reg_pressure
[ira_pressure_classes
[i
]])
3221 max_reg_pressure
[ira_pressure_classes
[i
]]
3222 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3224 restore_reg_pressure ();
3227 /* Update the current register pressure after scheduling INSN. Update
3228 also max register pressure for unscheduled insns of the current
3231 update_reg_and_insn_max_reg_pressure (rtx_insn
*insn
)
3234 int before
[N_REG_CLASSES
];
3236 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3237 before
[i
] = curr_reg_pressure
[ira_pressure_classes
[i
]];
3238 update_register_pressure (insn
);
3239 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3240 if (curr_reg_pressure
[ira_pressure_classes
[i
]] != before
[i
])
3242 if (i
< ira_pressure_classes_num
)
3243 setup_insn_max_reg_pressure (insn
, true);
3246 /* Set up register pressure at the beginning of basic block BB whose
3247 insns starting after insn AFTER. Set up also max register pressure
3248 for all insns of the basic block. */
3250 sched_setup_bb_reg_pressure_info (basic_block bb
, rtx_insn
*after
)
3252 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
3253 initiate_bb_reg_pressure_info (bb
);
3254 setup_insn_max_reg_pressure (after
, false);
3257 /* If doing predication while scheduling, verify whether INSN, which
3258 has just been scheduled, clobbers the conditions of any
3259 instructions that must be predicated in order to break their
3260 dependencies. If so, remove them from the queues so that they will
3261 only be scheduled once their control dependency is resolved. */
3264 check_clobbered_conditions (rtx_insn
*insn
)
3269 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0)
3272 find_all_hard_reg_sets (insn
, &t
, true);
3275 for (i
= 0; i
< ready
.n_ready
; i
++)
3277 rtx_insn
*x
= ready_element (&ready
, i
);
3278 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3280 ready_remove_insn (x
);
3284 for (i
= 0; i
<= max_insn_queue_index
; i
++)
3286 rtx_insn_list
*link
;
3287 int q
= NEXT_Q_AFTER (q_ptr
, i
);
3290 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
3292 rtx_insn
*x
= link
->insn ();
3293 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3302 /* Return (in order):
3304 - positive if INSN adversely affects the pressure on one
3307 - negative if INSN reduces the pressure on one register class
3309 - 0 if INSN doesn't affect the pressure on any register class. */
3312 model_classify_pressure (struct model_insn_info
*insn
)
3314 struct reg_pressure_data
*reg_pressure
;
3315 int death
[N_REG_CLASSES
];
3318 calculate_reg_deaths (insn
->insn
, death
);
3319 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3321 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3323 cl
= ira_pressure_classes
[pci
];
3324 if (death
[cl
] < reg_pressure
[pci
].set_increase
)
3326 sum
+= reg_pressure
[pci
].set_increase
- death
[cl
];
3331 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3334 model_order_p (struct model_insn_info
*insn1
, struct model_insn_info
*insn2
)
3336 unsigned int height1
, height2
;
3337 unsigned int priority1
, priority2
;
3339 /* Prefer instructions with a higher model priority. */
3340 if (insn1
->model_priority
!= insn2
->model_priority
)
3341 return insn1
->model_priority
> insn2
->model_priority
;
3343 /* Combine the length of the longest path of satisfied true dependencies
3344 that leads to each instruction (depth) with the length of the longest
3345 path of any dependencies that leads from the instruction (alap).
3346 Prefer instructions with the greatest combined length. If the combined
3347 lengths are equal, prefer instructions with the greatest depth.
3349 The idea is that, if we have a set S of "equal" instructions that each
3350 have ALAP value X, and we pick one such instruction I, any true-dependent
3351 successors of I that have ALAP value X - 1 should be preferred over S.
3352 This encourages the schedule to be "narrow" rather than "wide".
3353 However, if I is a low-priority instruction that we decided to
3354 schedule because of its model_classify_pressure, and if there
3355 is a set of higher-priority instructions T, the aforementioned
3356 successors of I should not have the edge over T. */
3357 height1
= insn1
->depth
+ insn1
->alap
;
3358 height2
= insn2
->depth
+ insn2
->alap
;
3359 if (height1
!= height2
)
3360 return height1
> height2
;
3361 if (insn1
->depth
!= insn2
->depth
)
3362 return insn1
->depth
> insn2
->depth
;
3364 /* We have no real preference between INSN1 an INSN2 as far as attempts
3365 to reduce pressure go. Prefer instructions with higher priorities. */
3366 priority1
= INSN_PRIORITY (insn1
->insn
);
3367 priority2
= INSN_PRIORITY (insn2
->insn
);
3368 if (priority1
!= priority2
)
3369 return priority1
> priority2
;
3371 /* Use the original rtl sequence as a tie-breaker. */
3372 return insn1
< insn2
;
3375 /* Add INSN to the model worklist immediately after PREV. Add it to the
3376 beginning of the list if PREV is null. */
3379 model_add_to_worklist_at (struct model_insn_info
*insn
,
3380 struct model_insn_info
*prev
)
3382 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_NOWHERE
);
3383 QUEUE_INDEX (insn
->insn
) = QUEUE_READY
;
3388 insn
->next
= prev
->next
;
3393 insn
->next
= model_worklist
;
3394 model_worklist
= insn
;
3397 insn
->next
->prev
= insn
;
3400 /* Remove INSN from the model worklist. */
3403 model_remove_from_worklist (struct model_insn_info
*insn
)
3405 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_READY
);
3406 QUEUE_INDEX (insn
->insn
) = QUEUE_NOWHERE
;
3409 insn
->prev
->next
= insn
->next
;
3411 model_worklist
= insn
->next
;
3413 insn
->next
->prev
= insn
->prev
;
3416 /* Add INSN to the model worklist. Start looking for a suitable position
3417 between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
3418 insns either side. A null PREV indicates the beginning of the list and
3419 a null NEXT indicates the end. */
3422 model_add_to_worklist (struct model_insn_info
*insn
,
3423 struct model_insn_info
*prev
,
3424 struct model_insn_info
*next
)
3428 count
= param_max_sched_ready_insns
;
3429 if (count
> 0 && prev
&& model_order_p (insn
, prev
))
3435 while (count
> 0 && prev
&& model_order_p (insn
, prev
));
3437 while (count
> 0 && next
&& model_order_p (next
, insn
))
3443 model_add_to_worklist_at (insn
, prev
);
3446 /* INSN may now have a higher priority (in the model_order_p sense)
3447 than before. Move it up the worklist if necessary. */
3450 model_promote_insn (struct model_insn_info
*insn
)
3452 struct model_insn_info
*prev
;
3456 count
= param_max_sched_ready_insns
;
3457 while (count
> 0 && prev
&& model_order_p (insn
, prev
))
3462 if (prev
!= insn
->prev
)
3464 model_remove_from_worklist (insn
);
3465 model_add_to_worklist_at (insn
, prev
);
3469 /* Add INSN to the end of the model schedule. */
3472 model_add_to_schedule (rtx_insn
*insn
)
3476 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
3477 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
3479 point
= model_schedule
.length ();
3480 model_schedule
.quick_push (insn
);
3481 INSN_MODEL_INDEX (insn
) = point
+ 1;
3484 /* Analyze the instructions that are to be scheduled, setting up
3485 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3486 instructions to model_worklist. */
3489 model_analyze_insns (void)
3491 rtx_insn
*start
, *end
, *iter
;
3492 sd_iterator_def sd_it
;
3494 struct model_insn_info
*insn
, *con
;
3496 model_num_insns
= 0;
3497 start
= PREV_INSN (current_sched_info
->next_tail
);
3498 end
= current_sched_info
->prev_head
;
3499 for (iter
= start
; iter
!= end
; iter
= PREV_INSN (iter
))
3500 if (NONDEBUG_INSN_P (iter
))
3502 insn
= MODEL_INSN_INFO (iter
);
3504 FOR_EACH_DEP (iter
, SD_LIST_FORW
, sd_it
, dep
)
3506 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3507 if (con
->insn
&& insn
->alap
< con
->alap
+ 1)
3508 insn
->alap
= con
->alap
+ 1;
3511 insn
->old_queue
= QUEUE_INDEX (iter
);
3512 QUEUE_INDEX (iter
) = QUEUE_NOWHERE
;
3514 insn
->unscheduled_preds
= dep_list_size (iter
, SD_LIST_HARD_BACK
);
3515 if (insn
->unscheduled_preds
== 0)
3516 model_add_to_worklist (insn
, NULL
, model_worklist
);
3522 /* The global state describes the register pressure at the start of the
3523 model schedule. Initialize GROUP accordingly. */
3526 model_init_pressure_group (struct model_pressure_group
*group
)
3530 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3532 cl
= ira_pressure_classes
[pci
];
3533 group
->limits
[pci
].pressure
= curr_reg_pressure
[cl
];
3534 group
->limits
[pci
].point
= 0;
3536 /* Use index model_num_insns to record the state after the last
3537 instruction in the model schedule. */
3538 group
->model
= XNEWVEC (struct model_pressure_data
,
3539 (model_num_insns
+ 1) * ira_pressure_classes_num
);
3542 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3543 Update the maximum pressure for the whole schedule. */
3546 model_record_pressure (struct model_pressure_group
*group
,
3547 int point
, int pci
, int pressure
)
3549 MODEL_REF_PRESSURE (group
, point
, pci
) = pressure
;
3550 if (group
->limits
[pci
].pressure
< pressure
)
3552 group
->limits
[pci
].pressure
= pressure
;
3553 group
->limits
[pci
].point
= point
;
3557 /* INSN has just been added to the end of the model schedule. Record its
3558 register-pressure information. */
3561 model_record_pressures (struct model_insn_info
*insn
)
3563 struct reg_pressure_data
*reg_pressure
;
3564 int point
, pci
, cl
, delta
;
3565 int death
[N_REG_CLASSES
];
3567 point
= model_index (insn
->insn
);
3568 if (sched_verbose
>= 2)
3572 fprintf (sched_dump
, "\n;;\tModel schedule:\n;;\n");
3573 fprintf (sched_dump
, ";;\t| idx insn | mpri hght dpth prio |\n");
3575 fprintf (sched_dump
, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3576 point
, INSN_UID (insn
->insn
), insn
->model_priority
,
3577 insn
->depth
+ insn
->alap
, insn
->depth
,
3578 INSN_PRIORITY (insn
->insn
),
3579 str_pattern_slim (PATTERN (insn
->insn
)));
3581 calculate_reg_deaths (insn
->insn
, death
);
3582 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3583 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3585 cl
= ira_pressure_classes
[pci
];
3586 delta
= reg_pressure
[pci
].set_increase
- death
[cl
];
3587 if (sched_verbose
>= 2)
3588 fprintf (sched_dump
, " %s:[%d,%+d]", reg_class_names
[cl
],
3589 curr_reg_pressure
[cl
], delta
);
3590 model_record_pressure (&model_before_pressure
, point
, pci
,
3591 curr_reg_pressure
[cl
]);
3593 if (sched_verbose
>= 2)
3594 fprintf (sched_dump
, "\n");
3597 /* All instructions have been added to the model schedule. Record the
3598 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3601 model_record_final_pressures (struct model_pressure_group
*group
)
3603 int point
, pci
, max_pressure
, ref_pressure
, cl
;
3605 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3607 /* Record the final pressure for this class. */
3608 cl
= ira_pressure_classes
[pci
];
3609 point
= model_num_insns
;
3610 ref_pressure
= curr_reg_pressure
[cl
];
3611 model_record_pressure (group
, point
, pci
, ref_pressure
);
3613 /* Record the original maximum pressure. */
3614 group
->limits
[pci
].orig_pressure
= group
->limits
[pci
].pressure
;
3616 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3617 max_pressure
= ref_pressure
;
3618 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3622 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
3623 max_pressure
= MAX (max_pressure
, ref_pressure
);
3624 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3629 /* Update all successors of INSN, given that INSN has just been scheduled. */
3632 model_add_successors_to_worklist (struct model_insn_info
*insn
)
3634 sd_iterator_def sd_it
;
3635 struct model_insn_info
*con
;
3638 FOR_EACH_DEP (insn
->insn
, SD_LIST_FORW
, sd_it
, dep
)
3640 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3641 /* Ignore debug instructions, and instructions from other blocks. */
3644 con
->unscheduled_preds
--;
3646 /* Update the depth field of each true-dependent successor.
3647 Increasing the depth gives them a higher priority than
3649 if (DEP_TYPE (dep
) == REG_DEP_TRUE
&& con
->depth
< insn
->depth
+ 1)
3651 con
->depth
= insn
->depth
+ 1;
3652 if (QUEUE_INDEX (con
->insn
) == QUEUE_READY
)
3653 model_promote_insn (con
);
3656 /* If this is a true dependency, or if there are no remaining
3657 dependencies for CON (meaning that CON only had non-true
3658 dependencies), make sure that CON is on the worklist.
3659 We don't bother otherwise because it would tend to fill the
3660 worklist with a lot of low-priority instructions that are not
3661 yet ready to issue. */
3662 if ((con
->depth
> 0 || con
->unscheduled_preds
== 0)
3663 && QUEUE_INDEX (con
->insn
) == QUEUE_NOWHERE
)
3664 model_add_to_worklist (con
, insn
, insn
->next
);
3669 /* Give INSN a higher priority than any current instruction, then give
3670 unscheduled predecessors of INSN a higher priority still. If any of
3671 those predecessors are not on the model worklist, do the same for its
3672 predecessors, and so on. */
3675 model_promote_predecessors (struct model_insn_info
*insn
)
3677 struct model_insn_info
*pro
, *first
;
3678 sd_iterator_def sd_it
;
3681 if (sched_verbose
>= 7)
3682 fprintf (sched_dump
, ";;\t+--- priority of %d = %d, priority of",
3683 INSN_UID (insn
->insn
), model_next_priority
);
3684 insn
->model_priority
= model_next_priority
++;
3685 model_remove_from_worklist (insn
);
3686 model_add_to_worklist_at (insn
, NULL
);
3691 FOR_EACH_DEP (insn
->insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
3693 pro
= MODEL_INSN_INFO (DEP_PRO (dep
));
3694 /* The first test is to ignore debug instructions, and instructions
3695 from other blocks. */
3697 && pro
->model_priority
!= model_next_priority
3698 && QUEUE_INDEX (pro
->insn
) != QUEUE_SCHEDULED
)
3700 pro
->model_priority
= model_next_priority
;
3701 if (sched_verbose
>= 7)
3702 fprintf (sched_dump
, " %d", INSN_UID (pro
->insn
));
3703 if (QUEUE_INDEX (pro
->insn
) == QUEUE_READY
)
3705 /* PRO is already in the worklist, but it now has
3706 a higher priority than before. Move it at the
3707 appropriate place. */
3708 model_remove_from_worklist (pro
);
3709 model_add_to_worklist (pro
, NULL
, model_worklist
);
3713 /* PRO isn't in the worklist. Recursively process
3714 its predecessors until we find one that is. */
3725 if (sched_verbose
>= 7)
3726 fprintf (sched_dump
, " = %d\n", model_next_priority
);
3727 model_next_priority
++;
3730 /* Pick one instruction from model_worklist and process it. */
3733 model_choose_insn (void)
3735 struct model_insn_info
*insn
, *fallback
;
3738 if (sched_verbose
>= 7)
3740 fprintf (sched_dump
, ";;\t+--- worklist:\n");
3741 insn
= model_worklist
;
3742 count
= param_max_sched_ready_insns
;
3743 while (count
> 0 && insn
)
3745 fprintf (sched_dump
, ";;\t+--- %d [%d, %d, %d, %d]\n",
3746 INSN_UID (insn
->insn
), insn
->model_priority
,
3747 insn
->depth
+ insn
->alap
, insn
->depth
,
3748 INSN_PRIORITY (insn
->insn
));
3754 /* Look for a ready instruction whose model_classify_priority is zero
3755 or negative, picking the highest-priority one. Adding such an
3756 instruction to the schedule now should do no harm, and may actually
3759 Failing that, see whether there is an instruction with the highest
3760 extant model_priority that is not yet ready, but which would reduce
3761 pressure if it became ready. This is designed to catch cases like:
3763 (set (mem (reg R1)) (reg R2))
3765 where the instruction is the last remaining use of R1 and where the
3766 value of R2 is not yet available (or vice versa). The death of R1
3767 means that this instruction already reduces pressure. It is of
3768 course possible that the computation of R2 involves other registers
3769 that are hard to kill, but such cases are rare enough for this
3770 heuristic to be a win in general.
3772 Failing that, just pick the highest-priority instruction in the
3774 count
= param_max_sched_ready_insns
;
3775 insn
= model_worklist
;
3779 if (count
== 0 || !insn
)
3781 insn
= fallback
? fallback
: model_worklist
;
3784 if (insn
->unscheduled_preds
)
3786 if (model_worklist
->model_priority
== insn
->model_priority
3788 && model_classify_pressure (insn
) < 0)
3793 if (model_classify_pressure (insn
) <= 0)
3800 if (sched_verbose
>= 7 && insn
!= model_worklist
)
3802 if (insn
->unscheduled_preds
)
3803 fprintf (sched_dump
, ";;\t+--- promoting insn %d, with dependencies\n",
3804 INSN_UID (insn
->insn
));
3806 fprintf (sched_dump
, ";;\t+--- promoting insn %d, which is ready\n",
3807 INSN_UID (insn
->insn
));
3809 if (insn
->unscheduled_preds
)
3810 /* INSN isn't yet ready to issue. Give all its predecessors the
3811 highest priority. */
3812 model_promote_predecessors (insn
);
3815 /* INSN is ready. Add it to the end of model_schedule and
3816 process its successors. */
3817 model_add_successors_to_worklist (insn
);
3818 model_remove_from_worklist (insn
);
3819 model_add_to_schedule (insn
->insn
);
3820 model_record_pressures (insn
);
3821 update_register_pressure (insn
->insn
);
3825 /* Restore all QUEUE_INDEXs to the values that they had before
3826 model_start_schedule was called. */
3829 model_reset_queue_indices (void)
3834 FOR_EACH_VEC_ELT (model_schedule
, i
, insn
)
3835 QUEUE_INDEX (insn
) = MODEL_INSN_INFO (insn
)->old_queue
;
3838 /* We have calculated the model schedule and spill costs. Print a summary
3842 model_dump_pressure_summary (void)
3846 fprintf (sched_dump
, ";; Pressure summary:");
3847 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3849 cl
= ira_pressure_classes
[pci
];
3850 fprintf (sched_dump
, " %s:%d", reg_class_names
[cl
],
3851 model_before_pressure
.limits
[pci
].pressure
);
3853 fprintf (sched_dump
, "\n\n");
3856 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3857 scheduling region. */
3860 model_start_schedule (basic_block bb
)
3862 model_next_priority
= 1;
3863 model_schedule
.create (sched_max_luid
);
3864 model_insns
= XCNEWVEC (struct model_insn_info
, sched_max_luid
);
3866 gcc_assert (bb
== BLOCK_FOR_INSN (NEXT_INSN (current_sched_info
->prev_head
)));
3867 initiate_reg_pressure_info (df_get_live_in (bb
));
3869 model_analyze_insns ();
3870 model_init_pressure_group (&model_before_pressure
);
3871 while (model_worklist
)
3872 model_choose_insn ();
3873 gcc_assert (model_num_insns
== (int) model_schedule
.length ());
3874 if (sched_verbose
>= 2)
3875 fprintf (sched_dump
, "\n");
3877 model_record_final_pressures (&model_before_pressure
);
3878 model_reset_queue_indices ();
3880 XDELETEVEC (model_insns
);
3882 model_curr_point
= 0;
3883 initiate_reg_pressure_info (df_get_live_in (bb
));
3884 if (sched_verbose
>= 1)
3885 model_dump_pressure_summary ();
3888 /* Free the information associated with GROUP. */
3891 model_finalize_pressure_group (struct model_pressure_group
*group
)
3893 XDELETEVEC (group
->model
);
3896 /* Free the information created by model_start_schedule. */
3899 model_end_schedule (void)
3901 model_finalize_pressure_group (&model_before_pressure
);
3902 model_schedule
.release ();
3905 /* Prepare reg pressure scheduling for basic block BB. */
3907 sched_pressure_start_bb (basic_block bb
)
3909 /* Set the number of available registers for each class taking into account
3910 relative probability of current basic block versus function prologue and
3912 * If the basic block executes much more often than the prologue/epilogue
3913 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3914 nil, so the effective number of available registers is
3915 (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
3916 * If the basic block executes as often as the prologue/epilogue,
3917 then spill in the block is as costly as in the prologue, so the effective
3918 number of available registers is
3919 (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3920 - call_saved_regs_num[cl]).
3921 Note that all-else-equal, we prefer to spill in the prologue, since that
3922 allows "extra" registers for other basic blocks of the function.
3923 * If the basic block is on the cold path of the function and executes
3924 rarely, then we should always prefer to spill in the block, rather than
3925 in the prologue/epilogue. The effective number of available register is
3926 (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3927 - call_saved_regs_num[cl]). */
3930 int entry_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
.to_frequency (cfun
);
3931 int bb_freq
= bb
->count
.to_frequency (cfun
);
3935 if (entry_freq
== 0)
3936 entry_freq
= bb_freq
= 1;
3938 if (bb_freq
< entry_freq
)
3939 bb_freq
= entry_freq
;
3941 for (i
= 0; i
< ira_pressure_classes_num
; ++i
)
3943 enum reg_class cl
= ira_pressure_classes
[i
];
3944 sched_class_regs_num
[cl
] = ira_class_hard_regs_num
[cl
]
3945 - fixed_regs_num
[cl
];
3946 sched_class_regs_num
[cl
]
3947 -= (call_saved_regs_num
[cl
] * entry_freq
) / bb_freq
;
3951 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
3952 model_start_schedule (bb
);
3955 /* A structure that holds local state for the loop in schedule_block. */
3956 struct sched_block_state
3958 /* True if no real insns have been scheduled in the current cycle. */
3959 bool first_cycle_insn_p
;
3960 /* True if a shadow insn has been scheduled in the current cycle, which
3961 means that no more normal insns can be issued. */
3962 bool shadows_only_p
;
3963 /* True if we're winding down a modulo schedule, which means that we only
3964 issue insns with INSN_EXACT_TICK set. */
3965 bool modulo_epilogue
;
3966 /* Initialized with the machine's issue rate every cycle, and updated
3967 by calls to the variable_issue hook. */
3971 /* INSN is the "currently executing insn". Launch each insn which was
3972 waiting on INSN. READY is the ready list which contains the insns
3973 that are ready to fire. CLOCK is the current cycle. The function
3974 returns necessary cycle advance after issuing the insn (it is not
3975 zero for insns in a schedule group). */
3978 schedule_insn (rtx_insn
*insn
)
3980 sd_iterator_def sd_it
;
3985 if (sched_verbose
>= 1)
3987 struct reg_pressure_data
*pressure_info
;
3988 fprintf (sched_dump
, ";;\t%3i--> %s %-40s:",
3989 clock_var
, (*current_sched_info
->print_insn
) (insn
, 1),
3990 str_pattern_slim (PATTERN (insn
)));
3992 if (recog_memoized (insn
) < 0)
3993 fprintf (sched_dump
, "nothing");
3995 print_reservation (sched_dump
, insn
);
3996 pressure_info
= INSN_REG_PRESSURE (insn
);
3997 if (pressure_info
!= NULL
)
3999 fputc (':', sched_dump
);
4000 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
4001 fprintf (sched_dump
, "%s%s%+d(%d)",
4002 scheduled_insns
.length () > 1
4004 < INSN_LUID (scheduled_insns
[scheduled_insns
.length () - 2]) ? "@" : "",
4005 reg_class_names
[ira_pressure_classes
[i
]],
4006 pressure_info
[i
].set_increase
, pressure_info
[i
].change
);
4008 if (sched_pressure
== SCHED_PRESSURE_MODEL
4009 && model_curr_point
< model_num_insns
4010 && model_index (insn
) == model_curr_point
)
4011 fprintf (sched_dump
, ":model %d", model_curr_point
);
4012 fputc ('\n', sched_dump
);
4015 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
&& !DEBUG_INSN_P (insn
))
4016 update_reg_and_insn_max_reg_pressure (insn
);
4018 /* Scheduling instruction should have all its dependencies resolved and
4019 should have been removed from the ready list. */
4020 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_HARD_BACK
));
4022 /* Reset debug insns invalidated by moving this insn. */
4023 if (MAY_HAVE_DEBUG_BIND_INSNS
&& !DEBUG_INSN_P (insn
))
4024 for (sd_it
= sd_iterator_start (insn
, SD_LIST_BACK
);
4025 sd_iterator_cond (&sd_it
, &dep
);)
4027 rtx_insn
*dbg
= DEP_PRO (dep
);
4028 struct reg_use_data
*use
, *next
;
4030 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4032 sd_iterator_next (&sd_it
);
4036 gcc_assert (DEBUG_BIND_INSN_P (dbg
));
4038 if (sched_verbose
>= 6)
4039 fprintf (sched_dump
, ";;\t\tresetting: debug insn %d\n",
4042 /* ??? Rather than resetting the debug insn, we might be able
4043 to emit a debug temp before the just-scheduled insn, but
4044 this would involve checking that the expression at the
4045 point of the debug insn is equivalent to the expression
4046 before the just-scheduled insn. They might not be: the
4047 expression in the debug insn may depend on other insns not
4048 yet scheduled that set MEMs, REGs or even other debug
4049 insns. It's not clear that attempting to preserve debug
4050 information in these cases is worth the effort, given how
4051 uncommon these resets are and the likelihood that the debug
4052 temps introduced won't survive the schedule change. */
4053 INSN_VAR_LOCATION_LOC (dbg
) = gen_rtx_UNKNOWN_VAR_LOC ();
4054 df_insn_rescan (dbg
);
4056 /* Unknown location doesn't use any registers. */
4057 for (use
= INSN_REG_USE_LIST (dbg
); use
!= NULL
; use
= next
)
4059 struct reg_use_data
*prev
= use
;
4061 /* Remove use from the cyclic next_regno_use chain first. */
4062 while (prev
->next_regno_use
!= use
)
4063 prev
= prev
->next_regno_use
;
4064 prev
->next_regno_use
= use
->next_regno_use
;
4065 next
= use
->next_insn_use
;
4068 INSN_REG_USE_LIST (dbg
) = NULL
;
4070 /* We delete rather than resolve these deps, otherwise we
4071 crash in sched_free_deps(), because forward deps are
4072 expected to be released before backward deps. */
4073 sd_delete_dep (sd_it
);
4076 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
4077 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
4079 if (sched_pressure
== SCHED_PRESSURE_MODEL
4080 && model_curr_point
< model_num_insns
4081 && NONDEBUG_INSN_P (insn
))
4083 if (model_index (insn
) == model_curr_point
)
4086 while (model_curr_point
< model_num_insns
4087 && (QUEUE_INDEX (MODEL_INSN (model_curr_point
))
4088 == QUEUE_SCHEDULED
));
4090 model_recompute (insn
);
4091 model_update_limit_points ();
4092 update_register_pressure (insn
);
4093 if (sched_verbose
>= 2)
4094 print_curr_reg_pressure ();
4097 gcc_assert (INSN_TICK (insn
) >= MIN_TICK
);
4098 if (INSN_TICK (insn
) > clock_var
)
4099 /* INSN has been prematurely moved from the queue to the ready list.
4100 This is possible only if following flags are set. */
4101 gcc_assert (flag_sched_stalled_insns
|| sched_fusion
);
4103 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4104 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4105 INSN_TICK (insn
) = clock_var
;
4107 check_clobbered_conditions (insn
);
4109 /* Update dependent instructions. First, see if by scheduling this insn
4110 now we broke a dependence in a way that requires us to change another
4112 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
4113 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
4115 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4116 rtx_insn
*pro
= DEP_PRO (dep
);
4117 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
4118 && desc
!= NULL
&& desc
->insn
== pro
)
4119 apply_replacement (dep
, false);
4122 /* Go through and resolve forward dependencies. */
4123 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4124 sd_iterator_cond (&sd_it
, &dep
);)
4126 rtx_insn
*next
= DEP_CON (dep
);
4127 bool cancelled
= (DEP_STATUS (dep
) & DEP_CANCELLED
) != 0;
4129 /* Resolve the dependence between INSN and NEXT.
4130 sd_resolve_dep () moves current dep to another list thus
4131 advancing the iterator. */
4132 sd_resolve_dep (sd_it
);
4136 if (must_restore_pattern_p (next
, dep
))
4137 restore_pattern (dep
, false);
4141 /* Don't bother trying to mark next as ready if insn is a debug
4142 insn. If insn is the last hard dependency, it will have
4143 already been discounted. */
4144 if (DEBUG_INSN_P (insn
) && !DEBUG_INSN_P (next
))
4147 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4151 effective_cost
= try_ready (next
);
4153 if (effective_cost
>= 0
4154 && SCHED_GROUP_P (next
)
4155 && advance
< effective_cost
)
4156 advance
= effective_cost
;
4159 /* Check always has only one forward dependence (to the first insn in
4160 the recovery block), therefore, this will be executed only once. */
4162 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4163 fix_recovery_deps (RECOVERY_BLOCK (insn
));
4167 /* Annotate the instruction with issue information -- TImode
4168 indicates that the instruction is expected not to be able
4169 to issue on the same cycle as the previous insn. A machine
4170 may use this information to decide how the instruction should
4173 && GET_CODE (PATTERN (insn
)) != USE
4174 && GET_CODE (PATTERN (insn
)) != CLOBBER
4175 && !DEBUG_INSN_P (insn
))
4177 if (reload_completed
)
4178 PUT_MODE (insn
, clock_var
> last_clock_var
? TImode
: VOIDmode
);
4179 last_clock_var
= clock_var
;
4182 if (nonscheduled_insns_begin
!= NULL_RTX
)
4183 /* Indicate to debug counters that INSN is scheduled. */
4184 nonscheduled_insns_begin
= insn
;
4189 /* Functions for handling of notes. */
4191 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4193 concat_note_lists (rtx_insn
*from_end
, rtx_insn
**to_endp
)
4195 rtx_insn
*from_start
;
4197 /* It's easy when have nothing to concat. */
4198 if (from_end
== NULL
)
4201 /* It's also easy when destination is empty. */
4202 if (*to_endp
== NULL
)
4204 *to_endp
= from_end
;
4208 from_start
= from_end
;
4209 while (PREV_INSN (from_start
) != NULL
)
4210 from_start
= PREV_INSN (from_start
);
4212 SET_PREV_INSN (from_start
) = *to_endp
;
4213 SET_NEXT_INSN (*to_endp
) = from_start
;
4214 *to_endp
= from_end
;
4217 /* Delete notes between HEAD and TAIL and put them in the chain
4218 of notes ended by NOTE_LIST. */
4220 remove_notes (rtx_insn
*head
, rtx_insn
*tail
)
4222 rtx_insn
*next_tail
, *insn
, *next
;
4225 if (head
== tail
&& !INSN_P (head
))
4228 next_tail
= NEXT_INSN (tail
);
4229 for (insn
= head
; insn
!= next_tail
; insn
= next
)
4231 next
= NEXT_INSN (insn
);
4235 switch (NOTE_KIND (insn
))
4237 case NOTE_INSN_BASIC_BLOCK
:
4240 case NOTE_INSN_EPILOGUE_BEG
:
4244 /* If an insn was split just before the EPILOGUE_BEG note and
4245 that split created new basic blocks, we could have a
4246 BASIC_BLOCK note here. Safely advance over it in that case
4247 and assert that we land on a real insn. */
4249 && NOTE_KIND (next
) == NOTE_INSN_BASIC_BLOCK
4250 && next
!= next_tail
)
4251 next
= NEXT_INSN (next
);
4252 gcc_assert (INSN_P (next
));
4253 add_reg_note (next
, REG_SAVE_NOTE
,
4254 GEN_INT (NOTE_INSN_EPILOGUE_BEG
));
4262 /* Add the note to list that ends at NOTE_LIST. */
4263 SET_PREV_INSN (insn
) = note_list
;
4264 SET_NEXT_INSN (insn
) = NULL_RTX
;
4266 SET_NEXT_INSN (note_list
) = insn
;
4271 gcc_assert ((sel_sched_p () || insn
!= tail
) && insn
!= head
);
4275 /* A structure to record enough data to allow us to backtrack the scheduler to
4276 a previous state. */
4277 struct haifa_saved_data
4279 /* Next entry on the list. */
4280 struct haifa_saved_data
*next
;
4282 /* Backtracking is associated with scheduling insns that have delay slots.
4283 DELAY_PAIR points to the structure that contains the insns involved, and
4284 the number of cycles between them. */
4285 struct delay_pair
*delay_pair
;
4287 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4288 void *fe_saved_data
;
4289 /* Data used by the backend. */
4290 void *be_saved_data
;
4292 /* Copies of global state. */
4293 int clock_var
, last_clock_var
;
4294 struct ready_list ready
;
4297 rtx_insn
*last_scheduled_insn
;
4298 rtx_insn
*last_nondebug_scheduled_insn
;
4299 rtx_insn
*nonscheduled_insns_begin
;
4300 int cycle_issued_insns
;
4302 /* Copies of state used in the inner loop of schedule_block. */
4303 struct sched_block_state sched_block
;
4305 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4306 to 0 when restoring. */
4308 rtx_insn_list
**insn_queue
;
4310 /* Describe pattern replacements that occurred since this backtrack point
4312 vec
<dep_t
> replacement_deps
;
4313 vec
<int> replace_apply
;
4315 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4317 vec
<dep_t
> next_cycle_deps
;
4318 vec
<int> next_cycle_apply
;
4321 /* A record, in reverse order, of all scheduled insns which have delay slots
4322 and may require backtracking. */
4323 static struct haifa_saved_data
*backtrack_queue
;
4325 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4328 mark_backtrack_feeds (rtx_insn
*insn
, int set_p
)
4330 sd_iterator_def sd_it
;
4332 FOR_EACH_DEP (insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
4334 FEEDS_BACKTRACK_INSN (DEP_PRO (dep
)) = set_p
;
4338 /* Save the current scheduler state so that we can backtrack to it
4339 later if necessary. PAIR gives the insns that make it necessary to
4340 save this point. SCHED_BLOCK is the local state of schedule_block
4341 that need to be saved. */
4343 save_backtrack_point (struct delay_pair
*pair
,
4344 struct sched_block_state sched_block
)
4347 struct haifa_saved_data
*save
= XNEW (struct haifa_saved_data
);
4349 save
->curr_state
= xmalloc (dfa_state_size
);
4350 memcpy (save
->curr_state
, curr_state
, dfa_state_size
);
4352 save
->ready
.first
= ready
.first
;
4353 save
->ready
.n_ready
= ready
.n_ready
;
4354 save
->ready
.n_debug
= ready
.n_debug
;
4355 save
->ready
.veclen
= ready
.veclen
;
4356 save
->ready
.vec
= XNEWVEC (rtx_insn
*, ready
.veclen
);
4357 memcpy (save
->ready
.vec
, ready
.vec
, ready
.veclen
* sizeof (rtx
));
4359 save
->insn_queue
= XNEWVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
4360 save
->q_size
= q_size
;
4361 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4363 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4364 save
->insn_queue
[i
] = copy_INSN_LIST (insn_queue
[q
]);
4367 save
->clock_var
= clock_var
;
4368 save
->last_clock_var
= last_clock_var
;
4369 save
->cycle_issued_insns
= cycle_issued_insns
;
4370 save
->last_scheduled_insn
= last_scheduled_insn
;
4371 save
->last_nondebug_scheduled_insn
= last_nondebug_scheduled_insn
;
4372 save
->nonscheduled_insns_begin
= nonscheduled_insns_begin
;
4374 save
->sched_block
= sched_block
;
4376 save
->replacement_deps
.create (0);
4377 save
->replace_apply
.create (0);
4378 save
->next_cycle_deps
= next_cycle_replace_deps
.copy ();
4379 save
->next_cycle_apply
= next_cycle_apply
.copy ();
4381 if (current_sched_info
->save_state
)
4382 save
->fe_saved_data
= (*current_sched_info
->save_state
) ();
4384 if (targetm
.sched
.alloc_sched_context
)
4386 save
->be_saved_data
= targetm
.sched
.alloc_sched_context ();
4387 targetm
.sched
.init_sched_context (save
->be_saved_data
, false);
4390 save
->be_saved_data
= NULL
;
4392 save
->delay_pair
= pair
;
4394 save
->next
= backtrack_queue
;
4395 backtrack_queue
= save
;
4399 mark_backtrack_feeds (pair
->i2
, 1);
4400 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4401 INSN_EXACT_TICK (pair
->i2
) = clock_var
+ pair_delay (pair
);
4402 SHADOW_P (pair
->i2
) = pair
->stages
== 0;
4403 pair
= pair
->next_same_i1
;
4407 /* Walk the ready list and all queues. If any insns have unresolved backwards
4408 dependencies, these must be cancelled deps, broken by predication. Set or
4409 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4412 toggle_cancelled_flags (bool set
)
4415 sd_iterator_def sd_it
;
4418 if (ready
.n_ready
> 0)
4420 rtx_insn
**first
= ready_lastpos (&ready
);
4421 for (i
= 0; i
< ready
.n_ready
; i
++)
4422 FOR_EACH_DEP (first
[i
], SD_LIST_BACK
, sd_it
, dep
)
4423 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4426 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4428 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4431 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4433 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4434 rtx_insn_list
*link
;
4435 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
4437 rtx_insn
*insn
= link
->insn ();
4438 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4439 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4442 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4444 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4450 /* Undo the replacements that have occurred after backtrack point SAVE
4453 undo_replacements_for_backtrack (struct haifa_saved_data
*save
)
4455 while (!save
->replacement_deps
.is_empty ())
4457 dep_t dep
= save
->replacement_deps
.pop ();
4458 int apply_p
= save
->replace_apply
.pop ();
4461 restore_pattern (dep
, true);
4463 apply_replacement (dep
, true);
4465 save
->replacement_deps
.release ();
4466 save
->replace_apply
.release ();
4469 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4470 Restore their dependencies to an unresolved state, and mark them as
4474 unschedule_insns_until (rtx_insn
*insn
)
4476 auto_vec
<rtx_insn
*> recompute_vec
;
4478 /* Make two passes over the insns to be unscheduled. First, we clear out
4479 dependencies and other trivial bookkeeping. */
4483 sd_iterator_def sd_it
;
4486 last
= scheduled_insns
.pop ();
4488 /* This will be changed by restore_backtrack_point if the insn is in
4490 QUEUE_INDEX (last
) = QUEUE_NOWHERE
;
4492 INSN_TICK (last
) = INVALID_TICK
;
4494 if (modulo_ii
> 0 && INSN_UID (last
) < modulo_iter0_max_uid
)
4495 modulo_insns_scheduled
--;
4497 for (sd_it
= sd_iterator_start (last
, SD_LIST_RES_FORW
);
4498 sd_iterator_cond (&sd_it
, &dep
);)
4500 rtx_insn
*con
= DEP_CON (dep
);
4501 sd_unresolve_dep (sd_it
);
4502 if (!MUST_RECOMPUTE_SPEC_P (con
))
4504 MUST_RECOMPUTE_SPEC_P (con
) = 1;
4505 recompute_vec
.safe_push (con
);
4513 /* A second pass, to update ready and speculation status for insns
4514 depending on the unscheduled ones. The first pass must have
4515 popped the scheduled_insns vector up to the point where we
4516 restart scheduling, as recompute_todo_spec requires it to be
4518 while (!recompute_vec
.is_empty ())
4522 con
= recompute_vec
.pop ();
4523 MUST_RECOMPUTE_SPEC_P (con
) = 0;
4524 if (!sd_lists_empty_p (con
, SD_LIST_HARD_BACK
))
4526 TODO_SPEC (con
) = HARD_DEP
;
4527 INSN_TICK (con
) = INVALID_TICK
;
4528 if (PREDICATED_PAT (con
) != NULL_RTX
)
4529 haifa_change_pattern (con
, ORIG_PAT (con
));
4531 else if (QUEUE_INDEX (con
) != QUEUE_SCHEDULED
)
4532 TODO_SPEC (con
) = recompute_todo_spec (con
, true);
4536 /* Restore scheduler state from the topmost entry on the backtracking queue.
4537 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4538 overwrite with the saved data.
4539 The caller must already have called unschedule_insns_until. */
4542 restore_last_backtrack_point (struct sched_block_state
*psched_block
)
4545 struct haifa_saved_data
*save
= backtrack_queue
;
4547 backtrack_queue
= save
->next
;
4549 if (current_sched_info
->restore_state
)
4550 (*current_sched_info
->restore_state
) (save
->fe_saved_data
);
4552 if (targetm
.sched
.alloc_sched_context
)
4554 targetm
.sched
.set_sched_context (save
->be_saved_data
);
4555 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4558 /* Do this first since it clobbers INSN_TICK of the involved
4560 undo_replacements_for_backtrack (save
);
4562 /* Clear the QUEUE_INDEX of everything in the ready list or one
4564 if (ready
.n_ready
> 0)
4566 rtx_insn
**first
= ready_lastpos (&ready
);
4567 for (i
= 0; i
< ready
.n_ready
; i
++)
4569 rtx_insn
*insn
= first
[i
];
4570 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
4571 INSN_TICK (insn
) = INVALID_TICK
;
4574 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4576 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4578 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4580 rtx_insn
*x
= link
->insn ();
4581 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
4582 INSN_TICK (x
) = INVALID_TICK
;
4584 free_INSN_LIST_list (&insn_queue
[q
]);
4588 ready
= save
->ready
;
4590 if (ready
.n_ready
> 0)
4592 rtx_insn
**first
= ready_lastpos (&ready
);
4593 for (i
= 0; i
< ready
.n_ready
; i
++)
4595 rtx_insn
*insn
= first
[i
];
4596 QUEUE_INDEX (insn
) = QUEUE_READY
;
4597 TODO_SPEC (insn
) = recompute_todo_spec (insn
, true);
4598 INSN_TICK (insn
) = save
->clock_var
;
4603 q_size
= save
->q_size
;
4604 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4606 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4608 insn_queue
[q
] = save
->insn_queue
[q
];
4610 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4612 rtx_insn
*x
= link
->insn ();
4613 QUEUE_INDEX (x
) = i
;
4614 TODO_SPEC (x
) = recompute_todo_spec (x
, true);
4615 INSN_TICK (x
) = save
->clock_var
+ i
;
4618 free (save
->insn_queue
);
4620 toggle_cancelled_flags (true);
4622 clock_var
= save
->clock_var
;
4623 last_clock_var
= save
->last_clock_var
;
4624 cycle_issued_insns
= save
->cycle_issued_insns
;
4625 last_scheduled_insn
= save
->last_scheduled_insn
;
4626 last_nondebug_scheduled_insn
= save
->last_nondebug_scheduled_insn
;
4627 nonscheduled_insns_begin
= save
->nonscheduled_insns_begin
;
4629 *psched_block
= save
->sched_block
;
4631 memcpy (curr_state
, save
->curr_state
, dfa_state_size
);
4632 free (save
->curr_state
);
4634 mark_backtrack_feeds (save
->delay_pair
->i2
, 0);
4636 gcc_assert (next_cycle_replace_deps
.is_empty ());
4637 next_cycle_replace_deps
= save
->next_cycle_deps
.copy ();
4638 next_cycle_apply
= save
->next_cycle_apply
.copy ();
4642 for (save
= backtrack_queue
; save
; save
= save
->next
)
4644 mark_backtrack_feeds (save
->delay_pair
->i2
, 1);
4648 /* Discard all data associated with the topmost entry in the backtrack
4649 queue. If RESET_TICK is false, we just want to free the data. If true,
4650 we are doing this because we discovered a reason to backtrack. In the
4651 latter case, also reset the INSN_TICK for the shadow insn. */
4653 free_topmost_backtrack_point (bool reset_tick
)
4655 struct haifa_saved_data
*save
= backtrack_queue
;
4658 backtrack_queue
= save
->next
;
4662 struct delay_pair
*pair
= save
->delay_pair
;
4665 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4666 INSN_EXACT_TICK (pair
->i2
) = INVALID_TICK
;
4667 pair
= pair
->next_same_i1
;
4669 undo_replacements_for_backtrack (save
);
4673 save
->replacement_deps
.release ();
4674 save
->replace_apply
.release ();
4677 if (targetm
.sched
.free_sched_context
)
4678 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4679 if (current_sched_info
->restore_state
)
4680 free (save
->fe_saved_data
);
4681 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4682 free_INSN_LIST_list (&save
->insn_queue
[i
]);
4683 free (save
->insn_queue
);
4684 free (save
->curr_state
);
4685 free (save
->ready
.vec
);
4689 /* Free the entire backtrack queue. */
4691 free_backtrack_queue (void)
4693 while (backtrack_queue
)
4694 free_topmost_backtrack_point (false);
4697 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4698 may have to postpone the replacement until the start of the next cycle,
4699 at which point we will be called again with IMMEDIATELY true. This is
4700 only done for machines which have instruction packets with explicit
4701 parallelism however. */
4703 apply_replacement (dep_t dep
, bool immediately
)
4705 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4706 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4708 next_cycle_replace_deps
.safe_push (dep
);
4709 next_cycle_apply
.safe_push (1);
4715 if (QUEUE_INDEX (desc
->insn
) == QUEUE_SCHEDULED
)
4718 if (sched_verbose
>= 5)
4719 fprintf (sched_dump
, "applying replacement for insn %d\n",
4720 INSN_UID (desc
->insn
));
4722 success
= validate_change (desc
->insn
, desc
->loc
, desc
->newval
, 0);
4723 gcc_assert (success
);
4725 rtx_insn
*insn
= DEP_PRO (dep
);
4727 /* Recompute priority since dependent priorities may have changed. */
4728 priority (insn
, true);
4729 update_insn_after_change (desc
->insn
);
4731 if ((TODO_SPEC (desc
->insn
) & (HARD_DEP
| DEP_POSTPONED
)) == 0)
4732 fix_tick_ready (desc
->insn
);
4734 if (backtrack_queue
!= NULL
)
4736 backtrack_queue
->replacement_deps
.safe_push (dep
);
4737 backtrack_queue
->replace_apply
.safe_push (1);
4742 /* We have determined that a pattern involved in DEP must be restored.
4743 If IMMEDIATELY is false, we may have to postpone the replacement
4744 until the start of the next cycle, at which point we will be called
4745 again with IMMEDIATELY true. */
4747 restore_pattern (dep_t dep
, bool immediately
)
4749 rtx_insn
*next
= DEP_CON (dep
);
4750 int tick
= INSN_TICK (next
);
4752 /* If we already scheduled the insn, the modified version is
4754 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
4757 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4759 next_cycle_replace_deps
.safe_push (dep
);
4760 next_cycle_apply
.safe_push (0);
4765 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
4767 if (sched_verbose
>= 5)
4768 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4770 haifa_change_pattern (next
, ORIG_PAT (next
));
4774 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4777 if (sched_verbose
>= 5)
4778 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4779 INSN_UID (desc
->insn
));
4780 tick
= INSN_TICK (desc
->insn
);
4782 success
= validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
4783 gcc_assert (success
);
4785 rtx_insn
*insn
= DEP_PRO (dep
);
4787 if (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
4789 /* Recompute priority since dependent priorities may have changed. */
4790 priority (insn
, true);
4793 update_insn_after_change (desc
->insn
);
4795 if (backtrack_queue
!= NULL
)
4797 backtrack_queue
->replacement_deps
.safe_push (dep
);
4798 backtrack_queue
->replace_apply
.safe_push (0);
4801 INSN_TICK (next
) = tick
;
4802 if (TODO_SPEC (next
) == DEP_POSTPONED
)
4805 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
4806 TODO_SPEC (next
) = 0;
4807 else if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
4808 TODO_SPEC (next
) = HARD_DEP
;
4811 /* Perform pattern replacements that were queued up until the next
4814 perform_replacements_new_cycle (void)
4818 FOR_EACH_VEC_ELT (next_cycle_replace_deps
, i
, dep
)
4820 int apply_p
= next_cycle_apply
[i
];
4822 apply_replacement (dep
, true);
4824 restore_pattern (dep
, true);
4826 next_cycle_replace_deps
.truncate (0);
4827 next_cycle_apply
.truncate (0);
4830 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4831 instructions we've previously encountered, a set bit prevents
4832 recursion. BUDGET is a limit on how far ahead we look, it is
4833 reduced on recursive calls. Return true if we produced a good
4834 estimate, or false if we exceeded the budget. */
4836 estimate_insn_tick (bitmap processed
, rtx_insn
*insn
, int budget
)
4838 sd_iterator_def sd_it
;
4840 int earliest
= INSN_TICK (insn
);
4842 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4844 rtx_insn
*pro
= DEP_PRO (dep
);
4847 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4850 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
4851 gcc_assert (INSN_TICK (pro
) + dep_cost (dep
) <= INSN_TICK (insn
));
4854 int cost
= dep_cost (dep
);
4857 if (!bitmap_bit_p (processed
, INSN_LUID (pro
)))
4859 if (!estimate_insn_tick (processed
, pro
, budget
- cost
))
4862 gcc_assert (INSN_TICK_ESTIMATE (pro
) != INVALID_TICK
);
4863 t
= INSN_TICK_ESTIMATE (pro
) + cost
;
4864 if (earliest
== INVALID_TICK
|| t
> earliest
)
4868 bitmap_set_bit (processed
, INSN_LUID (insn
));
4869 INSN_TICK_ESTIMATE (insn
) = earliest
;
4873 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4874 infinite resources) the cycle in which the delayed shadow can be issued.
4875 Return the number of cycles that must pass before the real insn can be
4876 issued in order to meet this constraint. */
4878 estimate_shadow_tick (struct delay_pair
*p
)
4880 auto_bitmap processed
;
4884 cutoff
= !estimate_insn_tick (processed
, p
->i2
,
4885 max_insn_queue_index
+ pair_delay (p
));
4887 return max_insn_queue_index
;
4888 t
= INSN_TICK_ESTIMATE (p
->i2
) - (clock_var
+ pair_delay (p
) + 1);
4894 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4895 recursively resolve all its forward dependencies. */
4897 resolve_dependencies (rtx_insn
*insn
)
4899 sd_iterator_def sd_it
;
4902 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4903 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn
)) != NULL
4904 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn
)) != NULL
)
4907 if (sched_verbose
>= 4)
4908 fprintf (sched_dump
, ";;\tquickly resolving %d\n", INSN_UID (insn
));
4910 if (QUEUE_INDEX (insn
) >= 0)
4911 queue_remove (insn
);
4913 scheduled_insns
.safe_push (insn
);
4915 /* Update dependent instructions. */
4916 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4917 sd_iterator_cond (&sd_it
, &dep
);)
4919 rtx_insn
*next
= DEP_CON (dep
);
4921 if (sched_verbose
>= 4)
4922 fprintf (sched_dump
, ";;\t\tdep %d against %d\n", INSN_UID (insn
),
4925 /* Resolve the dependence between INSN and NEXT.
4926 sd_resolve_dep () moves current dep to another list thus
4927 advancing the iterator. */
4928 sd_resolve_dep (sd_it
);
4930 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4932 resolve_dependencies (next
);
4935 /* Check always has only one forward dependence (to the first insn in
4936 the recovery block), therefore, this will be executed only once. */
4938 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4944 /* Return the head and tail pointers of ebb starting at BEG and ending
4947 get_ebb_head_tail (basic_block beg
, basic_block end
,
4948 rtx_insn
**headp
, rtx_insn
**tailp
)
4950 rtx_insn
*beg_head
= BB_HEAD (beg
);
4951 rtx_insn
* beg_tail
= BB_END (beg
);
4952 rtx_insn
* end_head
= BB_HEAD (end
);
4953 rtx_insn
* end_tail
= BB_END (end
);
4955 /* Don't include any notes or labels at the beginning of the BEG
4956 basic block, or notes at the end of the END basic blocks. */
4958 if (LABEL_P (beg_head
))
4959 beg_head
= NEXT_INSN (beg_head
);
4961 while (beg_head
!= beg_tail
)
4962 if (NOTE_P (beg_head
))
4963 beg_head
= NEXT_INSN (beg_head
);
4964 else if (DEBUG_INSN_P (beg_head
))
4966 rtx_insn
* note
, *next
;
4968 for (note
= NEXT_INSN (beg_head
);
4972 next
= NEXT_INSN (note
);
4975 if (sched_verbose
>= 9)
4976 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4978 reorder_insns_nobb (note
, note
, PREV_INSN (beg_head
));
4980 if (BLOCK_FOR_INSN (note
) != beg
)
4981 df_insn_change_bb (note
, beg
);
4983 else if (!DEBUG_INSN_P (note
))
4995 end_head
= beg_head
;
4996 else if (LABEL_P (end_head
))
4997 end_head
= NEXT_INSN (end_head
);
4999 while (end_head
!= end_tail
)
5000 if (NOTE_P (end_tail
))
5001 end_tail
= PREV_INSN (end_tail
);
5002 else if (DEBUG_INSN_P (end_tail
))
5004 rtx_insn
* note
, *prev
;
5006 for (note
= PREV_INSN (end_tail
);
5010 prev
= PREV_INSN (note
);
5013 if (sched_verbose
>= 9)
5014 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
5016 reorder_insns_nobb (note
, note
, end_tail
);
5018 if (end_tail
== BB_END (end
))
5019 BB_END (end
) = note
;
5021 if (BLOCK_FOR_INSN (note
) != end
)
5022 df_insn_change_bb (note
, end
);
5024 else if (!DEBUG_INSN_P (note
))
5036 /* Return true if there are no real insns in the range [ HEAD, TAIL ]. */
5039 no_real_insns_p (const rtx_insn
*head
, const rtx_insn
*tail
)
5041 while (head
!= NEXT_INSN (tail
))
5043 if (!NOTE_P (head
) && !LABEL_P (head
))
5045 head
= NEXT_INSN (head
);
5050 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5051 previously found among the insns. Insert them just before HEAD. */
5053 restore_other_notes (rtx_insn
*head
, basic_block head_bb
)
5057 rtx_insn
*note_head
= note_list
;
5060 head_bb
= BLOCK_FOR_INSN (head
);
5062 head
= NEXT_INSN (bb_note (head_bb
));
5064 while (PREV_INSN (note_head
))
5066 set_block_for_insn (note_head
, head_bb
);
5067 note_head
= PREV_INSN (note_head
);
5069 /* In the above cycle we've missed this note. */
5070 set_block_for_insn (note_head
, head_bb
);
5072 SET_PREV_INSN (note_head
) = PREV_INSN (head
);
5073 SET_NEXT_INSN (PREV_INSN (head
)) = note_head
;
5074 SET_PREV_INSN (head
) = note_list
;
5075 SET_NEXT_INSN (note_list
) = head
;
5077 if (BLOCK_FOR_INSN (head
) != head_bb
)
5078 BB_END (head_bb
) = note_list
;
5086 /* When we know we are going to discard the schedule due to a failed attempt
5087 at modulo scheduling, undo all replacements. */
5089 undo_all_replacements (void)
5094 FOR_EACH_VEC_ELT (scheduled_insns
, i
, insn
)
5096 sd_iterator_def sd_it
;
5099 /* See if we must undo a replacement. */
5100 for (sd_it
= sd_iterator_start (insn
, SD_LIST_RES_FORW
);
5101 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
5103 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
5105 validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
5110 /* Return first non-scheduled insn in the current scheduling block.
5111 This is mostly used for debug-counter purposes. */
5113 first_nonscheduled_insn (void)
5115 rtx_insn
*insn
= (nonscheduled_insns_begin
!= NULL_RTX
5116 ? nonscheduled_insns_begin
5117 : current_sched_info
->prev_head
);
5121 insn
= next_nonnote_nondebug_insn (insn
);
5123 while (QUEUE_INDEX (insn
) == QUEUE_SCHEDULED
);
5128 /* Move insns that became ready to fire from queue to ready list. */
5131 queue_to_ready (struct ready_list
*ready
)
5134 rtx_insn_list
*link
;
5135 rtx_insn
*skip_insn
;
5137 q_ptr
= NEXT_Q (q_ptr
);
5139 if (dbg_cnt (sched_insn
) == false)
5140 /* If debug counter is activated do not requeue the first
5141 nonscheduled insn. */
5142 skip_insn
= first_nonscheduled_insn ();
5146 /* Add all pending insns that can be scheduled without stalls to the
5148 for (link
= insn_queue
[q_ptr
]; link
; link
= link
->next ())
5150 insn
= link
->insn ();
5153 if (sched_verbose
>= 2)
5154 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5155 (*current_sched_info
->print_insn
) (insn
, 0));
5157 /* If the ready list is full, delay the insn for 1 cycle.
5158 See the comment in schedule_block for the rationale. */
5159 if (!reload_completed
5160 && (ready
->n_ready
- ready
->n_debug
> param_max_sched_ready_insns
5161 || (sched_pressure
== SCHED_PRESSURE_MODEL
5162 /* Limit pressure recalculations to
5163 param_max_sched_ready_insns instructions too. */
5164 && model_index (insn
) > (model_curr_point
5165 + param_max_sched_ready_insns
)))
5166 && !(sched_pressure
== SCHED_PRESSURE_MODEL
5167 && model_curr_point
< model_num_insns
5168 /* Always allow the next model instruction to issue. */
5169 && model_index (insn
) == model_curr_point
)
5170 && !SCHED_GROUP_P (insn
)
5171 && insn
!= skip_insn
)
5173 if (sched_verbose
>= 2)
5174 fprintf (sched_dump
, "keeping in queue, ready full\n");
5175 queue_insn (insn
, 1, "ready full");
5179 ready_add (ready
, insn
, false);
5180 if (sched_verbose
>= 2)
5181 fprintf (sched_dump
, "moving to ready without stalls\n");
5184 free_INSN_LIST_list (&insn_queue
[q_ptr
]);
5186 /* If there are no ready insns, stall until one is ready and add all
5187 of the pending insns at that point to the ready list. */
5188 if (ready
->n_ready
== 0)
5192 for (stalls
= 1; stalls
<= max_insn_queue_index
; stalls
++)
5194 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5196 for (; link
; link
= link
->next ())
5198 insn
= link
->insn ();
5201 if (sched_verbose
>= 2)
5202 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5203 (*current_sched_info
->print_insn
) (insn
, 0));
5205 ready_add (ready
, insn
, false);
5206 if (sched_verbose
>= 2)
5207 fprintf (sched_dump
, "moving to ready with %d stalls\n", stalls
);
5209 free_INSN_LIST_list (&insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]);
5211 advance_one_cycle ();
5216 advance_one_cycle ();
5219 q_ptr
= NEXT_Q_AFTER (q_ptr
, stalls
);
5220 clock_var
+= stalls
;
5221 if (sched_verbose
>= 2)
5222 fprintf (sched_dump
, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5227 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5228 prematurely move INSN from the queue to the ready list. Currently,
5229 if a target defines the hook 'is_costly_dependence', this function
5230 uses the hook to check whether there exist any dependences which are
5231 considered costly by the target, between INSN and other insns that
5232 have already been scheduled. Dependences are checked up to Y cycles
5233 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5234 controlling this value.
5235 (Other considerations could be taken into account instead (or in
5236 addition) depending on user flags and target hooks. */
5239 ok_for_early_queue_removal (rtx_insn
*insn
)
5241 if (targetm
.sched
.is_costly_dependence
)
5244 int i
= scheduled_insns
.length ();
5245 for (n_cycles
= flag_sched_stalled_insns_dep
; n_cycles
; n_cycles
--)
5251 rtx_insn
*prev_insn
= scheduled_insns
[i
];
5253 if (!NOTE_P (prev_insn
))
5257 dep
= sd_find_dep_between (prev_insn
, insn
, true);
5261 cost
= dep_cost (dep
);
5263 if (targetm
.sched
.is_costly_dependence (dep
, cost
,
5264 flag_sched_stalled_insns_dep
- n_cycles
))
5269 if (GET_MODE (prev_insn
) == TImode
) /* end of dispatch group */
5282 /* Remove insns from the queue, before they become "ready" with respect
5283 to FU latency considerations. */
5286 early_queue_to_ready (state_t state
, struct ready_list
*ready
)
5289 rtx_insn_list
*link
;
5290 rtx_insn_list
*next_link
;
5291 rtx_insn_list
*prev_link
;
5294 state_t temp_state
= alloca (dfa_state_size
);
5296 int insns_removed
= 0;
5299 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5302 X == 0: There is no limit on how many queued insns can be removed
5303 prematurely. (flag_sched_stalled_insns = -1).
5305 X >= 1: Only X queued insns can be removed prematurely in each
5306 invocation. (flag_sched_stalled_insns = X).
5308 Otherwise: Early queue removal is disabled.
5309 (flag_sched_stalled_insns = 0)
5312 if (! flag_sched_stalled_insns
)
5315 for (stalls
= 0; stalls
<= max_insn_queue_index
; stalls
++)
5317 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5319 if (sched_verbose
> 6)
5320 fprintf (sched_dump
, ";; look at index %d + %d\n", q_ptr
, stalls
);
5325 next_link
= link
->next ();
5326 insn
= link
->insn ();
5327 if (insn
&& sched_verbose
> 6)
5328 print_rtl_single (sched_dump
, insn
);
5330 memcpy (temp_state
, state
, dfa_state_size
);
5331 if (recog_memoized (insn
) < 0)
5332 /* non-negative to indicate that it's not ready
5333 to avoid infinite Q->R->Q->R... */
5336 cost
= state_transition (temp_state
, insn
);
5338 if (sched_verbose
>= 6)
5339 fprintf (sched_dump
, "transition cost = %d\n", cost
);
5341 move_to_ready
= false;
5344 move_to_ready
= ok_for_early_queue_removal (insn
);
5345 if (move_to_ready
== true)
5347 /* move from Q to R */
5349 ready_add (ready
, insn
, false);
5352 XEXP (prev_link
, 1) = next_link
;
5354 insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)] = next_link
;
5356 free_INSN_LIST_node (link
);
5358 if (sched_verbose
>= 2)
5359 fprintf (sched_dump
, ";;\t\tEarly Q-->Ready: insn %s\n",
5360 (*current_sched_info
->print_insn
) (insn
, 0));
5363 if (insns_removed
== flag_sched_stalled_insns
)
5364 /* Remove no more than flag_sched_stalled_insns insns
5365 from Q at a time. */
5366 return insns_removed
;
5370 if (move_to_ready
== false)
5377 } /* for stalls.. */
5379 return insns_removed
;
5383 /* Print the ready list for debugging purposes.
5384 If READY_TRY is non-zero then only print insns that max_issue
5387 debug_ready_list_1 (struct ready_list
*ready
, signed char *ready_try
)
5392 if (ready
->n_ready
== 0)
5394 fprintf (sched_dump
, "\n");
5398 p
= ready_lastpos (ready
);
5399 for (i
= 0; i
< ready
->n_ready
; i
++)
5401 if (ready_try
!= NULL
&& ready_try
[ready
->n_ready
- i
- 1])
5404 fprintf (sched_dump
, " %s:%d",
5405 (*current_sched_info
->print_insn
) (p
[i
], 0),
5407 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5408 fprintf (sched_dump
, "(cost=%d",
5409 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p
[i
]));
5410 fprintf (sched_dump
, ":prio=%d", INSN_PRIORITY (p
[i
]));
5411 if (INSN_TICK (p
[i
]) > clock_var
)
5412 fprintf (sched_dump
, ":delay=%d", INSN_TICK (p
[i
]) - clock_var
);
5413 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
5414 fprintf (sched_dump
, ":idx=%d",
5415 model_index (p
[i
]));
5416 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5417 fprintf (sched_dump
, ")");
5419 fprintf (sched_dump
, "\n");
5422 /* Print the ready list. Callable from debugger. */
5424 debug_ready_list (struct ready_list
*ready
)
5426 debug_ready_list_1 (ready
, NULL
);
5429 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5430 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5431 replaces the epilogue note in the correct basic block. */
5433 reemit_notes (rtx_insn
*insn
)
5436 rtx_insn
*last
= insn
;
5438 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
5440 if (REG_NOTE_KIND (note
) == REG_SAVE_NOTE
)
5442 enum insn_note note_type
= (enum insn_note
) INTVAL (XEXP (note
, 0));
5444 last
= emit_note_before (note_type
, last
);
5445 remove_note (insn
, note
);
5446 df_insn_create_insn_record (last
);
5451 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5453 move_insn (rtx_insn
*insn
, rtx_insn
*last
, rtx nt
)
5455 if (PREV_INSN (insn
) != last
)
5461 bb
= BLOCK_FOR_INSN (insn
);
5463 /* BB_HEAD is either LABEL or NOTE. */
5464 gcc_assert (BB_HEAD (bb
) != insn
);
5466 if (BB_END (bb
) == insn
)
5467 /* If this is last instruction in BB, move end marker one
5470 /* Jumps are always placed at the end of basic block. */
5471 jump_p
= control_flow_insn_p (insn
);
5474 || ((common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
5475 && IS_SPECULATION_BRANCHY_CHECK_P (insn
))
5476 || (common_sched_info
->sched_pass_id
5477 == SCHED_EBB_PASS
));
5479 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn
)) == bb
);
5481 BB_END (bb
) = PREV_INSN (insn
);
5484 gcc_assert (BB_END (bb
) != last
);
5487 /* We move the block note along with jump. */
5491 note
= NEXT_INSN (insn
);
5492 while (NOTE_NOT_BB_P (note
) && note
!= nt
)
5493 note
= NEXT_INSN (note
);
5497 || BARRIER_P (note
)))
5498 note
= NEXT_INSN (note
);
5500 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
5505 SET_NEXT_INSN (PREV_INSN (insn
)) = NEXT_INSN (note
);
5506 SET_PREV_INSN (NEXT_INSN (note
)) = PREV_INSN (insn
);
5508 SET_NEXT_INSN (note
) = NEXT_INSN (last
);
5509 SET_PREV_INSN (NEXT_INSN (last
)) = note
;
5511 SET_NEXT_INSN (last
) = insn
;
5512 SET_PREV_INSN (insn
) = last
;
5514 bb
= BLOCK_FOR_INSN (last
);
5518 fix_jump_move (insn
);
5520 if (BLOCK_FOR_INSN (insn
) != bb
)
5521 move_block_after_check (insn
);
5523 gcc_assert (BB_END (bb
) == last
);
5526 df_insn_change_bb (insn
, bb
);
5528 /* Update BB_END, if needed. */
5529 if (BB_END (bb
) == last
)
5533 SCHED_GROUP_P (insn
) = 0;
5536 /* Return true if scheduling INSN will finish current clock cycle. */
5538 insn_finishes_cycle_p (rtx_insn
*insn
)
5540 if (SCHED_GROUP_P (insn
))
5541 /* After issuing INSN, rest of the sched_group will be forced to issue
5542 in order. Don't make any plans for the rest of cycle. */
5545 /* Finishing the block will, apparently, finish the cycle. */
5546 if (current_sched_info
->insn_finishes_block_p
5547 && current_sched_info
->insn_finishes_block_p (insn
))
5553 /* Helper for autopref_multipass_init. Given a SET in PAT and whether
5554 we're expecting a memory WRITE or not, check that the insn is relevant to
5555 the autoprefetcher modelling code. Return true iff that is the case.
5556 If it is relevant, record the base register of the memory op in BASE and
5557 the offset in OFFSET. */
5560 analyze_set_insn_for_autopref (rtx pat
, bool write
, rtx
*base
, int *offset
)
5562 if (GET_CODE (pat
) != SET
)
5565 rtx mem
= write
? SET_DEST (pat
) : SET_SRC (pat
);
5569 struct address_info info
;
5570 decompose_mem_address (&info
, mem
);
5572 /* TODO: Currently only (base+const) addressing is supported. */
5573 if (info
.base
== NULL
|| !REG_P (*info
.base
)
5574 || (info
.disp
!= NULL
&& !CONST_INT_P (*info
.disp
)))
5578 *offset
= info
.disp
? INTVAL (*info
.disp
) : 0;
5582 /* Functions to model cache auto-prefetcher.
5584 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5585 memory prefetches if it sees instructions with consequitive memory accesses
5586 in the instruction stream. Details of such hardware units are not published,
5587 so we can only guess what exactly is going on there.
5588 In the scheduler, we model abstract auto-prefetcher. If there are memory
5589 insns in the ready list (or the queue) that have same memory base, but
5590 different offsets, then we delay the insns with larger offsets until insns
5591 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5592 is "1", then we look at the ready list; if it is N>1, then we also look
5593 through N-1 queue entries.
5594 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5595 among its heuristics.
5596 Param value of "-1" disables modelling of the auto-prefetcher. */
5598 /* Initialize autoprefetcher model data for INSN. */
5600 autopref_multipass_init (const rtx_insn
*insn
, int write
)
5602 autopref_multipass_data_t data
= &INSN_AUTOPREF_MULTIPASS_DATA (insn
)[write
];
5604 gcc_assert (data
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
);
5605 data
->base
= NULL_RTX
;
5607 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5608 data
->status
= AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5610 rtx pat
= PATTERN (insn
);
5612 /* We have a multi-set insn like a load-multiple or store-multiple.
5613 We care about these as long as all the memory ops inside the PARALLEL
5614 have the same base register. We care about the minimum and maximum
5615 offsets from that base but don't check for the order of those offsets
5616 within the PARALLEL insn itself. */
5617 if (GET_CODE (pat
) == PARALLEL
)
5619 int n_elems
= XVECLEN (pat
, 0);
5622 rtx base
, prev_base
= NULL_RTX
;
5623 int min_offset
= INT_MAX
;
5625 for (i
= 0; i
< n_elems
; i
++)
5627 rtx set
= XVECEXP (pat
, 0, i
);
5628 if (GET_CODE (set
) != SET
)
5631 if (!analyze_set_insn_for_autopref (set
, write
, &base
, &offset
))
5634 /* Ensure that all memory operations in the PARALLEL use the same
5636 if (i
> 0 && REGNO (base
) != REGNO (prev_base
))
5639 min_offset
= MIN (min_offset
, offset
);
5642 /* If we reached here then we have a valid PARALLEL of multiple memory ops
5643 with prev_base as the base and min_offset containing the offset. */
5644 gcc_assert (prev_base
);
5645 data
->base
= prev_base
;
5646 data
->offset
= min_offset
;
5647 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5651 /* Otherwise this is a single set memory operation. */
5652 rtx set
= single_set (insn
);
5653 if (set
== NULL_RTX
)
5656 if (!analyze_set_insn_for_autopref (set
, write
, &data
->base
,
5660 /* This insn is relevant for the auto-prefetcher.
5661 The base and offset fields will have been filled in the
5662 analyze_set_insn_for_autopref call above. */
5663 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5666 /* Helper function for rank_for_schedule sorting. */
5668 autopref_rank_for_schedule (const rtx_insn
*insn1
, const rtx_insn
*insn2
)
5671 for (int write
= 0; write
< 2 && !r
; ++write
)
5673 autopref_multipass_data_t data1
5674 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5675 autopref_multipass_data_t data2
5676 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5678 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5679 autopref_multipass_init (insn1
, write
);
5681 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5682 autopref_multipass_init (insn2
, write
);
5684 int irrel1
= data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5685 int irrel2
= data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5687 if (!irrel1
&& !irrel2
)
5688 /* Sort memory references from lowest offset to the largest. */
5689 r
= (data1
->offset
> data2
->offset
) - (data1
->offset
< data2
->offset
);
5691 /* Schedule "irrelevant" insns before memory stores to resolve
5692 as many producer dependencies of stores as possible. */
5693 r
= irrel2
- irrel1
;
5695 /* Schedule "irrelevant" insns after memory reads to avoid breaking
5696 memory read sequences. */
5697 r
= irrel1
- irrel2
;
5703 /* True if header of debug dump was printed. */
5704 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p
;
5706 /* Helper for autopref_multipass_dfa_lookahead_guard.
5707 Return "1" if INSN1 should be delayed in favor of INSN2. */
5709 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn
*insn1
,
5710 const rtx_insn
*insn2
, int write
)
5712 autopref_multipass_data_t data1
5713 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5714 autopref_multipass_data_t data2
5715 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5717 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5718 autopref_multipass_init (insn2
, write
);
5719 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5722 if (rtx_equal_p (data1
->base
, data2
->base
)
5723 && data1
->offset
> data2
->offset
)
5725 if (sched_verbose
>= 2)
5727 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5729 fprintf (sched_dump
,
5730 ";;\t\tnot trying in max_issue due to autoprefetch "
5732 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5735 fprintf (sched_dump
, " %d(%d)", INSN_UID (insn1
), INSN_UID (insn2
));
5746 We could have also hooked autoprefetcher model into
5747 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5748 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5749 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5750 unblocked). We don't bother about this yet because target of interest
5751 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5753 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5754 Return "1" if INSN1 should not be considered in max_issue due to
5755 auto-prefetcher considerations. */
5757 autopref_multipass_dfa_lookahead_guard (rtx_insn
*insn1
, int ready_index
)
5761 /* Exit early if the param forbids this or if we're not entering here through
5762 normal haifa scheduling. This can happen if selective scheduling is
5763 explicitly enabled. */
5764 if (!insn_queue
|| param_sched_autopref_queue_depth
<= 0)
5767 if (sched_verbose
>= 2 && ready_index
== 0)
5768 autopref_multipass_dfa_lookahead_guard_started_dump_p
= false;
5770 for (int write
= 0; write
< 2; ++write
)
5772 autopref_multipass_data_t data1
5773 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5775 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5776 autopref_multipass_init (insn1
, write
);
5777 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5780 if (ready_index
== 0
5781 && data1
->status
== AUTOPREF_MULTIPASS_DATA_DONT_DELAY
)
5782 /* We allow only a single delay on priviledged instructions.
5783 Doing otherwise would cause infinite loop. */
5785 if (sched_verbose
>= 2)
5787 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5789 fprintf (sched_dump
,
5790 ";;\t\tnot trying in max_issue due to autoprefetch "
5792 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5795 fprintf (sched_dump
, " *%d*", INSN_UID (insn1
));
5800 for (int i2
= 0; i2
< ready
.n_ready
; ++i2
)
5802 rtx_insn
*insn2
= get_ready_element (i2
);
5805 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
, write
);
5808 if (ready_index
== 0)
5811 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5817 if (param_sched_autopref_queue_depth
== 1)
5820 /* Everything from the current queue slot should have been moved to
5822 gcc_assert (insn_queue
[NEXT_Q_AFTER (q_ptr
, 0)] == NULL_RTX
);
5824 int n_stalls
= param_sched_autopref_queue_depth
- 1;
5825 if (n_stalls
> max_insn_queue_index
)
5826 n_stalls
= max_insn_queue_index
;
5828 for (int stalls
= 1; stalls
<= n_stalls
; ++stalls
)
5830 for (rtx_insn_list
*link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)];
5832 link
= link
->next ())
5834 rtx_insn
*insn2
= link
->insn ();
5835 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
,
5839 /* Queue INSN1 until INSN2 can issue. */
5841 if (ready_index
== 0)
5842 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5850 if (sched_verbose
>= 2
5851 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5852 && (ready_index
== ready
.n_ready
- 1 || r
< 0))
5853 /* This does not /always/ trigger. We don't output EOL if the last
5854 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5855 called. We can live with this. */
5856 fprintf (sched_dump
, "\n");
5861 /* Define type for target data used in multipass scheduling. */
5862 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5863 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5865 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t
;
5867 /* The following structure describe an entry of the stack of choices. */
5870 /* Ordinal number of the issued insn in the ready queue. */
5872 /* The number of the rest insns whose issues we should try. */
5874 /* The number of issued essential insns. */
5876 /* State after issuing the insn. */
5878 /* Target-specific data. */
5879 first_cycle_multipass_data_t target_data
;
5882 /* The following array is used to implement a stack of choices used in
5883 function max_issue. */
5884 static struct choice_entry
*choice_stack
;
5886 /* This holds the value of the target dfa_lookahead hook. */
5889 /* The following variable value is maximal number of tries of issuing
5890 insns for the first cycle multipass insn scheduling. We define
5891 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5892 need this constraint if all real insns (with non-negative codes)
5893 had reservations because in this case the algorithm complexity is
5894 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5895 might be incomplete and such insn might occur. For such
5896 descriptions, the complexity of algorithm (without the constraint)
5897 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5898 static int max_lookahead_tries
;
5900 /* The following function returns maximal (or close to maximal) number
5901 of insns which can be issued on the same cycle and one of which
5902 insns is insns with the best rank (the first insn in READY). To
5903 make this function tries different samples of ready insns. READY
5904 is current queue `ready'. Global array READY_TRY reflects what
5905 insns are already issued in this try. The function stops immediately,
5906 if it reached the such a solution, that all instruction can be issued.
5907 INDEX will contain index of the best insn in READY. The following
5908 function is used only for first cycle multipass scheduling.
5912 This function expects recognized insns only. All USEs,
5913 CLOBBERs, etc must be filtered elsewhere. */
5915 max_issue (struct ready_list
*ready
, int privileged_n
, state_t state
,
5916 bool first_cycle_insn_p
, int *index
)
5918 int n
, i
, all
, n_ready
, best
, delay
, tries_num
;
5920 struct choice_entry
*top
;
5926 n_ready
= ready
->n_ready
;
5927 gcc_assert (dfa_lookahead
>= 1 && privileged_n
>= 0
5928 && privileged_n
<= n_ready
);
5930 /* Init MAX_LOOKAHEAD_TRIES. */
5931 if (max_lookahead_tries
== 0)
5933 max_lookahead_tries
= 100;
5934 for (i
= 0; i
< issue_rate
; i
++)
5935 max_lookahead_tries
*= dfa_lookahead
;
5938 /* Init max_points. */
5939 more_issue
= issue_rate
- cycle_issued_insns
;
5940 gcc_assert (more_issue
>= 0);
5942 /* The number of the issued insns in the best solution. */
5947 /* Set initial state of the search. */
5948 memcpy (top
->state
, state
, dfa_state_size
);
5949 top
->rest
= dfa_lookahead
;
5951 if (targetm
.sched
.first_cycle_multipass_begin
)
5952 targetm
.sched
.first_cycle_multipass_begin (&top
->target_data
,
5954 first_cycle_insn_p
);
5956 /* Count the number of the insns to search among. */
5957 for (all
= i
= 0; i
< n_ready
; i
++)
5961 if (sched_verbose
>= 2)
5963 fprintf (sched_dump
, ";;\t\tmax_issue among %d insns:", all
);
5964 debug_ready_list_1 (ready
, ready_try
);
5967 /* I is the index of the insn to try next. */
5972 if (/* If we've reached a dead end or searched enough of what we have
5975 /* or have nothing else to try... */
5977 /* or should not issue more. */
5978 || top
->n
>= more_issue
)
5980 /* ??? (... || i == n_ready). */
5981 gcc_assert (i
<= n_ready
);
5983 /* We should not issue more than issue_rate instructions. */
5984 gcc_assert (top
->n
<= more_issue
);
5986 if (top
== choice_stack
)
5989 if (best
< top
- choice_stack
)
5994 /* Try to find issued privileged insn. */
5995 while (n
&& !ready_try
[--n
])
5999 if (/* If all insns are equally good... */
6001 /* Or a privileged insn will be issued. */
6003 /* Then we have a solution. */
6005 best
= top
- choice_stack
;
6006 /* This is the index of the insn issued first in this
6008 *index
= choice_stack
[1].index
;
6009 if (top
->n
== more_issue
|| best
== all
)
6014 /* Set ready-list index to point to the last insn
6015 ('i++' below will advance it to the next insn). */
6021 if (targetm
.sched
.first_cycle_multipass_backtrack
)
6022 targetm
.sched
.first_cycle_multipass_backtrack (&top
->target_data
,
6023 ready_try
, n_ready
);
6026 memcpy (state
, top
->state
, dfa_state_size
);
6028 else if (!ready_try
[i
])
6031 if (tries_num
> max_lookahead_tries
)
6033 insn
= ready_element (ready
, i
);
6034 delay
= state_transition (state
, insn
);
6037 if (state_dead_lock_p (state
)
6038 || insn_finishes_cycle_p (insn
))
6039 /* We won't issue any more instructions in the next
6046 if (memcmp (top
->state
, state
, dfa_state_size
) != 0)
6049 /* Advance to the next choice_entry. */
6051 /* Initialize it. */
6052 top
->rest
= dfa_lookahead
;
6055 memcpy (top
->state
, state
, dfa_state_size
);
6058 if (targetm
.sched
.first_cycle_multipass_issue
)
6059 targetm
.sched
.first_cycle_multipass_issue (&top
->target_data
,
6069 /* Increase ready-list index. */
6073 if (targetm
.sched
.first_cycle_multipass_end
)
6074 targetm
.sched
.first_cycle_multipass_end (best
!= 0
6075 ? &choice_stack
[1].target_data
6078 /* Restore the original state of the DFA. */
6079 memcpy (state
, choice_stack
->state
, dfa_state_size
);
6084 /* The following function chooses insn from READY and modifies
6085 READY. The following function is used only for first
6086 cycle multipass scheduling.
6088 -1 if cycle should be advanced,
6089 0 if INSN_PTR is set to point to the desirable insn,
6090 1 if choose_ready () should be restarted without advancing the cycle. */
6092 choose_ready (struct ready_list
*ready
, bool first_cycle_insn_p
,
6093 rtx_insn
**insn_ptr
)
6095 if (dbg_cnt (sched_insn
) == false)
6097 if (nonscheduled_insns_begin
== NULL_RTX
)
6098 nonscheduled_insns_begin
= current_sched_info
->prev_head
;
6100 rtx_insn
*insn
= first_nonscheduled_insn ();
6102 if (QUEUE_INDEX (insn
) == QUEUE_READY
)
6103 /* INSN is in the ready_list. */
6105 ready_remove_insn (insn
);
6110 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6111 gcc_assert (QUEUE_INDEX (insn
) >= 0);
6115 if (dfa_lookahead
<= 0 || SCHED_GROUP_P (ready_element (ready
, 0))
6116 || DEBUG_INSN_P (ready_element (ready
, 0)))
6118 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6119 *insn_ptr
= ready_remove_first_dispatch (ready
);
6121 *insn_ptr
= ready_remove_first (ready
);
6127 /* Try to choose the best insn. */
6131 insn
= ready_element (ready
, 0);
6132 if (INSN_CODE (insn
) < 0)
6134 *insn_ptr
= ready_remove_first (ready
);
6138 /* Filter the search space. */
6139 for (i
= 0; i
< ready
->n_ready
; i
++)
6143 insn
= ready_element (ready
, i
);
6145 /* If this insn is recognizable we should have already
6146 recognized it earlier.
6147 ??? Not very clear where this is supposed to be done.
6149 gcc_checking_assert (INSN_CODE (insn
) >= 0
6150 || recog_memoized (insn
) < 0);
6151 if (INSN_CODE (insn
) < 0)
6153 /* Non-recognized insns at position 0 are handled above. */
6159 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
)
6162 = (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
6165 if (ready_try
[i
] < 0)
6166 /* Queue instruction for several cycles.
6167 We need to restart choose_ready as we have changed
6170 change_queue_index (insn
, -ready_try
[i
]);
6174 /* Make sure that we didn't end up with 0'th insn filtered out.
6175 Don't be tempted to make life easier for backends and just
6176 requeue 0'th insn if (ready_try[0] == 0) and restart
6177 choose_ready. Backends should be very considerate about
6178 requeueing instructions -- especially the highest priority
6179 one at position 0. */
6180 gcc_assert (ready_try
[i
] == 0 || i
> 0);
6185 gcc_assert (ready_try
[i
] == 0);
6186 /* INSN made it through the scrutiny of filters! */
6189 if (max_issue (ready
, 1, curr_state
, first_cycle_insn_p
, &index
) == 0)
6191 *insn_ptr
= ready_remove_first (ready
);
6192 if (sched_verbose
>= 4)
6193 fprintf (sched_dump
, ";;\t\tChosen insn (but can't issue) : %s \n",
6194 (*current_sched_info
->print_insn
) (*insn_ptr
, 0));
6199 if (sched_verbose
>= 4)
6200 fprintf (sched_dump
, ";;\t\tChosen insn : %s\n",
6201 (*current_sched_info
->print_insn
)
6202 (ready_element (ready
, index
), 0));
6204 *insn_ptr
= ready_remove (ready
, index
);
6210 /* This function is called when we have successfully scheduled a
6211 block. It uses the schedule stored in the scheduled_insns vector
6212 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6213 append the scheduled insns; TAIL is the insn after the scheduled
6214 block. TARGET_BB is the argument passed to schedule_block. */
6217 commit_schedule (rtx_insn
*prev_head
, rtx_insn
*tail
, basic_block
*target_bb
)
6222 last_scheduled_insn
= prev_head
;
6224 scheduled_insns
.iterate (i
, &insn
);
6227 if (control_flow_insn_p (last_scheduled_insn
)
6228 || current_sched_info
->advance_target_bb (*target_bb
, insn
))
6230 *target_bb
= current_sched_info
->advance_target_bb (*target_bb
, 0);
6236 x
= next_real_insn (last_scheduled_insn
);
6238 dump_new_block_header (1, *target_bb
, x
, tail
);
6241 last_scheduled_insn
= bb_note (*target_bb
);
6244 if (current_sched_info
->begin_move_insn
)
6245 (*current_sched_info
->begin_move_insn
) (insn
, last_scheduled_insn
);
6246 move_insn (insn
, last_scheduled_insn
,
6247 current_sched_info
->next_tail
);
6248 if (!DEBUG_INSN_P (insn
))
6249 reemit_notes (insn
);
6250 last_scheduled_insn
= insn
;
6253 scheduled_insns
.truncate (0);
6256 /* Examine all insns on the ready list and queue those which can't be
6257 issued in this cycle. TEMP_STATE is temporary scheduler state we
6258 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6259 have been issued for the current cycle, which means it is valid to
6260 issue an asm statement.
6262 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6263 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6264 we only leave insns which have an INSN_EXACT_TICK. */
6267 prune_ready_list (state_t temp_state
, bool first_cycle_insn_p
,
6268 bool shadows_only_p
, bool modulo_epilogue_p
)
6271 bool sched_group_found
= false;
6272 int min_cost_group
= 0;
6277 for (i
= 0; i
< ready
.n_ready
; i
++)
6279 rtx_insn
*insn
= ready_element (&ready
, i
);
6280 if (SCHED_GROUP_P (insn
))
6282 sched_group_found
= true;
6287 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6288 such an insn first and note its cost. If at least one SCHED_GROUP_P insn
6289 gets queued, then all other insns get queued for one cycle later. */
6290 for (pass
= sched_group_found
? 0 : 1; pass
< 2; )
6292 int n
= ready
.n_ready
;
6293 for (i
= 0; i
< n
; i
++)
6295 rtx_insn
*insn
= ready_element (&ready
, i
);
6297 const char *reason
= "resource conflict";
6299 if (DEBUG_INSN_P (insn
))
6302 if (sched_group_found
&& !SCHED_GROUP_P (insn
)
6303 && ((pass
== 0) || (min_cost_group
>= 1)))
6307 cost
= min_cost_group
;
6308 reason
= "not in sched group";
6310 else if (modulo_epilogue_p
6311 && INSN_EXACT_TICK (insn
) == INVALID_TICK
)
6313 cost
= max_insn_queue_index
;
6314 reason
= "not an epilogue insn";
6316 else if (shadows_only_p
&& !SHADOW_P (insn
))
6319 reason
= "not a shadow";
6321 else if (recog_memoized (insn
) < 0)
6323 if (!first_cycle_insn_p
6324 && (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6325 || asm_noperands (PATTERN (insn
)) >= 0))
6329 else if (sched_pressure
!= SCHED_PRESSURE_NONE
)
6331 if (sched_pressure
== SCHED_PRESSURE_MODEL
6332 && INSN_TICK (insn
) <= clock_var
)
6334 memcpy (temp_state
, curr_state
, dfa_state_size
);
6335 if (state_transition (temp_state
, insn
) >= 0)
6336 INSN_TICK (insn
) = clock_var
+ 1;
6346 struct delay_pair
*delay_entry
;
6348 = delay_htab
->find_with_hash (insn
,
6349 htab_hash_pointer (insn
));
6350 while (delay_entry
&& delay_cost
== 0)
6352 delay_cost
= estimate_shadow_tick (delay_entry
);
6353 if (delay_cost
> max_insn_queue_index
)
6354 delay_cost
= max_insn_queue_index
;
6355 delay_entry
= delay_entry
->next_same_i1
;
6359 memcpy (temp_state
, curr_state
, dfa_state_size
);
6360 cost
= state_transition (temp_state
, insn
);
6365 if (cost
< delay_cost
)
6368 reason
= "shadow tick";
6373 if (SCHED_GROUP_P (insn
) && cost
> min_cost_group
)
6374 min_cost_group
= cost
;
6375 ready_remove (&ready
, i
);
6376 /* Normally we'd want to queue INSN for COST cycles. However,
6377 if SCHED_GROUP_P is set, then we must ensure that nothing
6378 else comes between INSN and its predecessor. If there is
6379 some other insn ready to fire on the next cycle, then that
6380 invariant would be broken.
6382 So when SCHED_GROUP_P is set, just queue this insn for a
6384 queue_insn (insn
, SCHED_GROUP_P (insn
) ? 1 : cost
, reason
);
6394 /* Called when we detect that the schedule is impossible. We examine the
6395 backtrack queue to find the earliest insn that caused this condition. */
6397 static struct haifa_saved_data
*
6398 verify_shadows (void)
6400 struct haifa_saved_data
*save
, *earliest_fail
= NULL
;
6401 for (save
= backtrack_queue
; save
; save
= save
->next
)
6404 struct delay_pair
*pair
= save
->delay_pair
;
6405 rtx_insn
*i1
= pair
->i1
;
6407 for (; pair
; pair
= pair
->next_same_i1
)
6409 rtx_insn
*i2
= pair
->i2
;
6411 if (QUEUE_INDEX (i2
) == QUEUE_SCHEDULED
)
6414 t
= INSN_TICK (i1
) + pair_delay (pair
);
6417 if (sched_verbose
>= 2)
6418 fprintf (sched_dump
,
6419 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6421 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6422 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6423 earliest_fail
= save
;
6426 if (QUEUE_INDEX (i2
) >= 0)
6428 int queued_for
= INSN_TICK (i2
);
6432 if (sched_verbose
>= 2)
6433 fprintf (sched_dump
,
6434 ";;\t\tfailed delay requirements for %d/%d"
6435 " (%d->%d), queued too late\n",
6436 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6437 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6438 earliest_fail
= save
;
6445 return earliest_fail
;
6448 /* Print instructions together with useful scheduling information between
6449 HEAD and TAIL (inclusive). */
6451 dump_insn_stream (rtx_insn
*head
, rtx_insn
*tail
)
6453 fprintf (sched_dump
, ";;\t| insn | prio |\n");
6455 rtx_insn
*next_tail
= NEXT_INSN (tail
);
6456 for (rtx_insn
*insn
= head
; insn
!= next_tail
; insn
= NEXT_INSN (insn
))
6458 int priority
= NOTE_P (insn
) ? 0 : INSN_PRIORITY (insn
);
6459 const char *pattern
= (NOTE_P (insn
)
6461 : str_pattern_slim (PATTERN (insn
)));
6463 fprintf (sched_dump
, ";;\t| %4d | %4d | %-30s ",
6464 INSN_UID (insn
), priority
, pattern
);
6466 if (sched_verbose
>= 4)
6468 if (NOTE_P (insn
) || LABEL_P (insn
) || recog_memoized (insn
) < 0)
6469 fprintf (sched_dump
, "nothing");
6471 print_reservation (sched_dump
, insn
);
6473 fprintf (sched_dump
, "\n");
6477 /* Use forward list scheduling to rearrange insns of block pointed to by
6478 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6482 schedule_block (basic_block
*target_bb
, state_t init_state
)
6485 bool success
= modulo_ii
== 0;
6486 struct sched_block_state ls
;
6487 state_t temp_state
= NULL
; /* It is used for multipass scheduling. */
6488 int sort_p
, advance
, start_clock_var
;
6490 /* Head/tail info for this block. */
6491 rtx_insn
*prev_head
= current_sched_info
->prev_head
;
6492 rtx_insn
*next_tail
= current_sched_info
->next_tail
;
6493 rtx_insn
*head
= NEXT_INSN (prev_head
);
6494 rtx_insn
*tail
= PREV_INSN (next_tail
);
6496 if ((current_sched_info
->flags
& DONT_BREAK_DEPENDENCIES
) == 0
6497 && sched_pressure
!= SCHED_PRESSURE_MODEL
&& !sched_fusion
)
6498 find_modifiable_mems (head
, tail
);
6500 /* We used to have code to avoid getting parameters moved from hard
6501 argument registers into pseudos.
6503 However, it was removed when it proved to be of marginal benefit
6504 and caused problems because schedule_block and compute_forward_dependences
6505 had different notions of what the "head" insn was. */
6507 gcc_assert (head
!= tail
|| INSN_P (head
));
6509 haifa_recovery_bb_recently_added_p
= false;
6511 backtrack_queue
= NULL
;
6516 dump_new_block_header (0, *target_bb
, head
, tail
);
6518 if (sched_verbose
>= 2)
6520 dump_insn_stream (head
, tail
);
6521 memset (&rank_for_schedule_stats
, 0,
6522 sizeof (rank_for_schedule_stats
));
6526 if (init_state
== NULL
)
6527 state_reset (curr_state
);
6529 memcpy (curr_state
, init_state
, dfa_state_size
);
6531 /* Clear the ready list. */
6532 ready
.first
= ready
.veclen
- 1;
6536 /* It is used for first cycle multipass scheduling. */
6537 temp_state
= alloca (dfa_state_size
);
6539 if (targetm
.sched
.init
)
6540 targetm
.sched
.init (sched_dump
, sched_verbose
, ready
.veclen
);
6542 /* We start inserting insns after PREV_HEAD. */
6543 last_scheduled_insn
= prev_head
;
6544 last_nondebug_scheduled_insn
= NULL
;
6545 nonscheduled_insns_begin
= NULL
;
6547 gcc_assert ((NOTE_P (last_scheduled_insn
)
6548 || DEBUG_INSN_P (last_scheduled_insn
))
6549 && BLOCK_FOR_INSN (last_scheduled_insn
) == *target_bb
);
6551 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6556 insn_queue
= XALLOCAVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
6557 memset (insn_queue
, 0, (max_insn_queue_index
+ 1) * sizeof (rtx
));
6559 /* Start just before the beginning of time. */
6562 /* We need queue and ready lists and clock_var be initialized
6563 in try_ready () (which is called through init_ready_list ()). */
6564 (*current_sched_info
->init_ready_list
) ();
6567 sched_pressure_start_bb (*target_bb
);
6569 /* The algorithm is O(n^2) in the number of ready insns at any given
6570 time in the worst case. Before reload we are more likely to have
6571 big lists so truncate them to a reasonable size. */
6572 if (!reload_completed
6573 && ready
.n_ready
- ready
.n_debug
> param_max_sched_ready_insns
)
6575 ready_sort_debug (&ready
);
6576 ready_sort_real (&ready
);
6578 /* Find first free-standing insn past param_max_sched_ready_insns.
6579 If there are debug insns, we know they're first. */
6580 for (i
= param_max_sched_ready_insns
+ ready
.n_debug
; i
< ready
.n_ready
;
6582 if (!SCHED_GROUP_P (ready_element (&ready
, i
)))
6585 if (sched_verbose
>= 2)
6587 fprintf (sched_dump
,
6588 ";;\t\tReady list on entry: %d insns: ", ready
.n_ready
);
6589 debug_ready_list (&ready
);
6590 fprintf (sched_dump
,
6591 ";;\t\t before reload => truncated to %d insns\n", i
);
6594 /* Delay all insns past it for 1 cycle. If debug counter is
6595 activated make an exception for the insn right after
6596 nonscheduled_insns_begin. */
6598 rtx_insn
*skip_insn
;
6600 if (dbg_cnt (sched_insn
) == false)
6601 skip_insn
= first_nonscheduled_insn ();
6605 while (i
< ready
.n_ready
)
6609 insn
= ready_remove (&ready
, i
);
6611 if (insn
!= skip_insn
)
6612 queue_insn (insn
, 1, "list truncated");
6615 ready_add (&ready
, skip_insn
, true);
6619 /* Now we can restore basic block notes and maintain precise cfg. */
6620 restore_bb_notes (*target_bb
);
6622 last_clock_var
= -1;
6626 gcc_assert (scheduled_insns
.length () == 0);
6628 must_backtrack
= false;
6629 modulo_insns_scheduled
= 0;
6631 ls
.modulo_epilogue
= false;
6632 ls
.first_cycle_insn_p
= true;
6634 /* Loop until all the insns in BB are scheduled. */
6635 while ((*current_sched_info
->schedule_more_p
) ())
6637 perform_replacements_new_cycle ();
6640 start_clock_var
= clock_var
;
6644 advance_one_cycle ();
6646 /* Add to the ready list all pending insns that can be issued now.
6647 If there are no ready insns, increment clock until one
6648 is ready and add all pending insns at that point to the ready
6650 queue_to_ready (&ready
);
6652 gcc_assert (ready
.n_ready
);
6654 if (sched_verbose
>= 2)
6656 fprintf (sched_dump
, ";;\t\tReady list after queue_to_ready:");
6657 debug_ready_list (&ready
);
6659 advance
-= clock_var
- start_clock_var
;
6661 while (advance
> 0);
6663 if (ls
.modulo_epilogue
)
6665 int stage
= clock_var
/ modulo_ii
;
6666 if (stage
> modulo_last_stage
* 2 + 2)
6668 if (sched_verbose
>= 2)
6669 fprintf (sched_dump
,
6670 ";;\t\tmodulo scheduled succeeded at II %d\n",
6676 else if (modulo_ii
> 0)
6678 int stage
= clock_var
/ modulo_ii
;
6679 if (stage
> modulo_max_stages
)
6681 if (sched_verbose
>= 2)
6682 fprintf (sched_dump
,
6683 ";;\t\tfailing schedule due to excessive stages\n");
6686 if (modulo_n_insns
== modulo_insns_scheduled
6687 && stage
> modulo_last_stage
)
6689 if (sched_verbose
>= 2)
6690 fprintf (sched_dump
,
6691 ";;\t\tfound kernel after %d stages, II %d\n",
6693 ls
.modulo_epilogue
= true;
6697 prune_ready_list (temp_state
, true, false, ls
.modulo_epilogue
);
6698 if (ready
.n_ready
== 0)
6703 ls
.shadows_only_p
= false;
6704 cycle_issued_insns
= 0;
6705 ls
.can_issue_more
= issue_rate
;
6712 if (sort_p
&& ready
.n_ready
> 0)
6714 /* Sort the ready list based on priority. This must be
6715 done every iteration through the loop, as schedule_insn
6716 may have readied additional insns that will not be
6717 sorted correctly. */
6718 ready_sort (&ready
);
6720 if (sched_verbose
>= 2)
6722 fprintf (sched_dump
,
6723 ";;\t\tReady list after ready_sort: ");
6724 debug_ready_list (&ready
);
6728 /* We don't want md sched reorder to even see debug isns, so put
6729 them out right away. */
6730 if (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0))
6731 && (*current_sched_info
->schedule_more_p
) ())
6733 while (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0)))
6735 rtx_insn
*insn
= ready_remove_first (&ready
);
6736 gcc_assert (DEBUG_INSN_P (insn
));
6737 (*current_sched_info
->begin_schedule_ready
) (insn
);
6738 scheduled_insns
.safe_push (insn
);
6739 last_scheduled_insn
= insn
;
6740 advance
= schedule_insn (insn
);
6741 gcc_assert (advance
== 0);
6742 if (ready
.n_ready
> 0)
6743 ready_sort (&ready
);
6747 if (ls
.first_cycle_insn_p
&& !ready
.n_ready
)
6750 resume_after_backtrack
:
6751 /* Allow the target to reorder the list, typically for
6752 better instruction bundling. */
6754 && (ready
.n_ready
== 0
6755 || !SCHED_GROUP_P (ready_element (&ready
, 0))))
6757 if (ls
.first_cycle_insn_p
&& targetm
.sched
.reorder
)
6759 = targetm
.sched
.reorder (sched_dump
, sched_verbose
,
6760 ready_lastpos (&ready
),
6761 &ready
.n_ready
, clock_var
);
6762 else if (!ls
.first_cycle_insn_p
&& targetm
.sched
.reorder2
)
6764 = targetm
.sched
.reorder2 (sched_dump
, sched_verbose
,
6766 ? ready_lastpos (&ready
) : NULL
,
6767 &ready
.n_ready
, clock_var
);
6770 restart_choose_ready
:
6771 if (sched_verbose
>= 2)
6773 fprintf (sched_dump
, ";;\tReady list (t = %3d): ",
6775 debug_ready_list (&ready
);
6776 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
6777 print_curr_reg_pressure ();
6780 if (ready
.n_ready
== 0
6781 && ls
.can_issue_more
6782 && reload_completed
)
6784 /* Allow scheduling insns directly from the queue in case
6785 there's nothing better to do (ready list is empty) but
6786 there are still vacant dispatch slots in the current cycle. */
6787 if (sched_verbose
>= 6)
6788 fprintf (sched_dump
,";;\t\tSecond chance\n");
6789 memcpy (temp_state
, curr_state
, dfa_state_size
);
6790 if (early_queue_to_ready (temp_state
, &ready
))
6791 ready_sort (&ready
);
6794 if (ready
.n_ready
== 0
6795 || !ls
.can_issue_more
6796 || state_dead_lock_p (curr_state
)
6797 || !(*current_sched_info
->schedule_more_p
) ())
6800 /* Select and remove the insn from the ready list. */
6806 res
= choose_ready (&ready
, ls
.first_cycle_insn_p
, &insn
);
6812 goto restart_choose_ready
;
6814 gcc_assert (insn
!= NULL_RTX
);
6817 insn
= ready_remove_first (&ready
);
6819 if (sched_pressure
!= SCHED_PRESSURE_NONE
6820 && INSN_TICK (insn
) > clock_var
)
6822 ready_add (&ready
, insn
, true);
6827 if (targetm
.sched
.dfa_new_cycle
6828 && targetm
.sched
.dfa_new_cycle (sched_dump
, sched_verbose
,
6829 insn
, last_clock_var
,
6830 clock_var
, &sort_p
))
6831 /* SORT_P is used by the target to override sorting
6832 of the ready list. This is needed when the target
6833 has modified its internal structures expecting that
6834 the insn will be issued next. As we need the insn
6835 to have the highest priority (so it will be returned by
6836 the ready_remove_first call above), we invoke
6837 ready_add (&ready, insn, true).
6838 But, still, there is one issue: INSN can be later
6839 discarded by scheduler's front end through
6840 current_sched_info->can_schedule_ready_p, hence, won't
6843 ready_add (&ready
, insn
, true);
6849 if (current_sched_info
->can_schedule_ready_p
6850 && ! (*current_sched_info
->can_schedule_ready_p
) (insn
))
6851 /* We normally get here only if we don't want to move
6852 insn from the split block. */
6854 TODO_SPEC (insn
) = DEP_POSTPONED
;
6855 goto restart_choose_ready
;
6860 /* If this insn is the first part of a delay-slot pair, record a
6862 struct delay_pair
*delay_entry
;
6864 = delay_htab
->find_with_hash (insn
, htab_hash_pointer (insn
));
6867 save_backtrack_point (delay_entry
, ls
);
6868 if (sched_verbose
>= 2)
6869 fprintf (sched_dump
, ";;\t\tsaving backtrack point\n");
6873 /* DECISION is made. */
6875 if (modulo_ii
> 0 && INSN_UID (insn
) < modulo_iter0_max_uid
)
6877 modulo_insns_scheduled
++;
6878 modulo_last_stage
= clock_var
/ modulo_ii
;
6880 if (TODO_SPEC (insn
) & SPECULATIVE
)
6881 generate_recovery_code (insn
);
6883 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6884 targetm
.sched
.dispatch_do (insn
, ADD_TO_DISPATCH_WINDOW
);
6886 /* Update counters, etc in the scheduler's front end. */
6887 (*current_sched_info
->begin_schedule_ready
) (insn
);
6888 scheduled_insns
.safe_push (insn
);
6889 gcc_assert (NONDEBUG_INSN_P (insn
));
6890 last_nondebug_scheduled_insn
= last_scheduled_insn
= insn
;
6892 if (recog_memoized (insn
) >= 0)
6894 memcpy (temp_state
, curr_state
, dfa_state_size
);
6895 cost
= state_transition (curr_state
, insn
);
6896 if (sched_pressure
!= SCHED_PRESSURE_WEIGHTED
&& !sched_fusion
)
6897 gcc_assert (cost
< 0);
6898 if (memcmp (temp_state
, curr_state
, dfa_state_size
) != 0)
6899 cycle_issued_insns
++;
6903 asm_p
= (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6904 || asm_noperands (PATTERN (insn
)) >= 0);
6906 if (targetm
.sched
.variable_issue
)
6908 targetm
.sched
.variable_issue (sched_dump
, sched_verbose
,
6909 insn
, ls
.can_issue_more
);
6910 /* A naked CLOBBER or USE generates no instruction, so do
6911 not count them against the issue rate. */
6912 else if (GET_CODE (PATTERN (insn
)) != USE
6913 && GET_CODE (PATTERN (insn
)) != CLOBBER
)
6914 ls
.can_issue_more
--;
6915 advance
= schedule_insn (insn
);
6917 if (SHADOW_P (insn
))
6918 ls
.shadows_only_p
= true;
6920 /* After issuing an asm insn we should start a new cycle. */
6921 if (advance
== 0 && asm_p
)
6930 ls
.first_cycle_insn_p
= false;
6931 if (ready
.n_ready
> 0)
6932 prune_ready_list (temp_state
, false, ls
.shadows_only_p
,
6933 ls
.modulo_epilogue
);
6937 if (!must_backtrack
)
6938 for (i
= 0; i
< ready
.n_ready
; i
++)
6940 rtx_insn
*insn
= ready_element (&ready
, i
);
6941 if (INSN_EXACT_TICK (insn
) == clock_var
)
6943 must_backtrack
= true;
6948 if (must_backtrack
&& modulo_ii
> 0)
6950 if (modulo_backtracks_left
== 0)
6952 modulo_backtracks_left
--;
6954 while (must_backtrack
)
6956 struct haifa_saved_data
*failed
;
6957 rtx_insn
*failed_insn
;
6959 must_backtrack
= false;
6960 failed
= verify_shadows ();
6961 gcc_assert (failed
);
6963 failed_insn
= failed
->delay_pair
->i1
;
6964 /* Clear these queues. */
6965 perform_replacements_new_cycle ();
6966 toggle_cancelled_flags (false);
6967 unschedule_insns_until (failed_insn
);
6968 while (failed
!= backtrack_queue
)
6969 free_topmost_backtrack_point (true);
6970 restore_last_backtrack_point (&ls
);
6971 if (sched_verbose
>= 2)
6972 fprintf (sched_dump
, ";;\t\trewind to cycle %d\n", clock_var
);
6973 /* Delay by at least a cycle. This could cause additional
6975 queue_insn (failed_insn
, 1, "backtracked");
6979 if (ready
.n_ready
> 0)
6980 goto resume_after_backtrack
;
6983 if (clock_var
== 0 && ls
.first_cycle_insn_p
)
6989 ls
.first_cycle_insn_p
= true;
6991 if (ls
.modulo_epilogue
)
6994 if (!ls
.first_cycle_insn_p
|| advance
)
6995 advance_one_cycle ();
6996 perform_replacements_new_cycle ();
6999 /* Once again, debug insn suckiness: they can be on the ready list
7000 even if they have unresolved dependencies. To make our view
7001 of the world consistent, remove such "ready" insns. */
7002 restart_debug_insn_loop
:
7003 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7007 x
= ready_element (&ready
, i
);
7008 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x
)) != NULL
7009 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x
)) != NULL
)
7011 ready_remove (&ready
, i
);
7012 goto restart_debug_insn_loop
;
7015 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7019 x
= ready_element (&ready
, i
);
7020 resolve_dependencies (x
);
7022 for (i
= 0; i
<= max_insn_queue_index
; i
++)
7024 rtx_insn_list
*link
;
7025 while ((link
= insn_queue
[i
]) != NULL
)
7027 rtx_insn
*x
= link
->insn ();
7028 insn_queue
[i
] = link
->next ();
7029 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7030 free_INSN_LIST_node (link
);
7031 resolve_dependencies (x
);
7037 undo_all_replacements ();
7042 fprintf (sched_dump
, ";;\tReady list (final): ");
7043 debug_ready_list (&ready
);
7046 if (modulo_ii
== 0 && current_sched_info
->queue_must_finish_empty
)
7047 /* Sanity check -- queue must be empty now. Meaningless if region has
7049 gcc_assert (!q_size
&& !ready
.n_ready
&& !ready
.n_debug
);
7050 else if (modulo_ii
== 0)
7052 /* We must maintain QUEUE_INDEX between blocks in region. */
7053 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7057 x
= ready_element (&ready
, i
);
7058 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7059 TODO_SPEC (x
) = HARD_DEP
;
7063 for (i
= 0; i
<= max_insn_queue_index
; i
++)
7065 rtx_insn_list
*link
;
7066 for (link
= insn_queue
[i
]; link
; link
= link
->next ())
7071 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7072 TODO_SPEC (x
) = HARD_DEP
;
7074 free_INSN_LIST_list (&insn_queue
[i
]);
7078 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
7079 model_end_schedule ();
7083 commit_schedule (prev_head
, tail
, target_bb
);
7085 fprintf (sched_dump
, ";; total time = %d\n", clock_var
);
7088 last_scheduled_insn
= tail
;
7090 scheduled_insns
.truncate (0);
7092 if (!current_sched_info
->queue_must_finish_empty
7093 || haifa_recovery_bb_recently_added_p
)
7095 /* INSN_TICK (minimum clock tick at which the insn becomes
7096 ready) may be not correct for the insn in the subsequent
7097 blocks of the region. We should use a correct value of
7098 `clock_var' or modify INSN_TICK. It is better to keep
7099 clock_var value equal to 0 at the start of a basic block.
7100 Therefore we modify INSN_TICK here. */
7101 fix_inter_tick (NEXT_INSN (prev_head
), last_scheduled_insn
);
7104 if (targetm
.sched
.finish
)
7106 targetm
.sched
.finish (sched_dump
, sched_verbose
);
7107 /* Target might have added some instructions to the scheduled block
7108 in its md_finish () hook. These new insns don't have any data
7109 initialized and to identify them we extend h_i_d so that they'll
7111 sched_extend_luids ();
7114 /* Update head/tail boundaries. */
7115 head
= NEXT_INSN (prev_head
);
7116 tail
= last_scheduled_insn
;
7120 fprintf (sched_dump
, ";; new head = %d\n;; new tail = %d\n",
7121 INSN_UID (head
), INSN_UID (tail
));
7123 if (sched_verbose
>= 2)
7125 dump_insn_stream (head
, tail
);
7126 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats
,
7130 fprintf (sched_dump
, "\n");
7133 head
= restore_other_notes (head
, NULL
);
7135 current_sched_info
->head
= head
;
7136 current_sched_info
->tail
= tail
;
7138 free_backtrack_queue ();
7143 /* Set_priorities: compute priority of each insn in the block. */
7146 set_priorities (rtx_insn
*head
, rtx_insn
*tail
)
7150 int sched_max_insns_priority
=
7151 current_sched_info
->sched_max_insns_priority
;
7152 rtx_insn
*prev_head
;
7154 if (head
== tail
&& ! INSN_P (head
))
7159 prev_head
= PREV_INSN (head
);
7160 for (insn
= tail
; insn
!= prev_head
; insn
= PREV_INSN (insn
))
7166 (void) priority (insn
);
7168 gcc_assert (INSN_PRIORITY_KNOWN (insn
));
7170 sched_max_insns_priority
= MAX (sched_max_insns_priority
,
7171 INSN_PRIORITY (insn
));
7174 current_sched_info
->sched_max_insns_priority
= sched_max_insns_priority
;
7179 /* Set sched_dump and sched_verbose for the desired debugging output. */
7181 setup_sched_dump (void)
7183 sched_verbose
= sched_verbose_param
;
7184 sched_dump
= dump_file
;
7189 /* Allocate data for register pressure sensitive scheduling. */
7191 alloc_global_sched_pressure_data (void)
7193 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7195 int i
, max_regno
= max_reg_num ();
7197 if (sched_dump
!= NULL
)
7198 /* We need info about pseudos for rtl dumps about pseudo
7199 classes and costs. */
7200 regstat_init_n_sets_and_refs ();
7201 ira_set_pseudo_classes (true, sched_verbose
? sched_dump
: NULL
);
7202 sched_regno_pressure_class
7203 = (enum reg_class
*) xmalloc (max_regno
* sizeof (enum reg_class
));
7204 for (i
= 0; i
< max_regno
; i
++)
7205 sched_regno_pressure_class
[i
]
7206 = (i
< FIRST_PSEUDO_REGISTER
7207 ? ira_pressure_class_translate
[REGNO_REG_CLASS (i
)]
7208 : ira_pressure_class_translate
[reg_allocno_class (i
)]);
7209 curr_reg_live
= BITMAP_ALLOC (NULL
);
7210 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7212 saved_reg_live
= BITMAP_ALLOC (NULL
);
7213 region_ref_regs
= BITMAP_ALLOC (NULL
);
7215 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
7216 tmp_bitmap
= BITMAP_ALLOC (NULL
);
7218 /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes
7219 that we calculate register pressure for. */
7220 for (int c
= 0; c
< ira_pressure_classes_num
; ++c
)
7222 enum reg_class cl
= ira_pressure_classes
[c
];
7224 call_saved_regs_num
[cl
] = 0;
7225 fixed_regs_num
[cl
] = 0;
7227 for (int i
= 0; i
< ira_class_hard_regs_num
[cl
]; ++i
)
7229 unsigned int regno
= ira_class_hard_regs
[cl
][i
];
7230 if (fixed_regs
[regno
])
7231 ++fixed_regs_num
[cl
];
7232 else if (!crtl
->abi
->clobbers_full_reg_p (regno
))
7233 ++call_saved_regs_num
[cl
];
7239 /* Free data for register pressure sensitive scheduling. Also called
7240 from schedule_region when stopping sched-pressure early. */
7242 free_global_sched_pressure_data (void)
7244 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7246 if (regstat_n_sets_and_refs
!= NULL
)
7247 regstat_free_n_sets_and_refs ();
7248 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7250 BITMAP_FREE (region_ref_regs
);
7251 BITMAP_FREE (saved_reg_live
);
7253 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
7254 BITMAP_FREE (tmp_bitmap
);
7255 BITMAP_FREE (curr_reg_live
);
7256 free (sched_regno_pressure_class
);
7260 /* Initialize some global state for the scheduler. This function works
7261 with the common data shared between all the schedulers. It is called
7262 from the scheduler specific initialization routine. */
7267 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
7268 targetm
.sched
.dispatch_do (NULL
, DISPATCH_INIT
);
7270 if (live_range_shrinkage_p
)
7271 sched_pressure
= SCHED_PRESSURE_WEIGHTED
;
7272 else if (flag_sched_pressure
7273 && !reload_completed
7274 && common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
7275 sched_pressure
= ((enum sched_pressure_algorithm
)
7276 param_sched_pressure_algorithm
);
7278 sched_pressure
= SCHED_PRESSURE_NONE
;
7280 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7281 ira_setup_eliminable_regset ();
7283 /* Initialize SPEC_INFO. */
7284 if (targetm
.sched
.set_sched_flags
)
7286 spec_info
= &spec_info_var
;
7287 targetm
.sched
.set_sched_flags (spec_info
);
7289 if (spec_info
->mask
!= 0)
7291 spec_info
->data_weakness_cutoff
7292 = (param_sched_spec_prob_cutoff
* MAX_DEP_WEAK
) / 100;
7293 spec_info
->control_weakness_cutoff
7294 = (param_sched_spec_prob_cutoff
* REG_BR_PROB_BASE
) / 100;
7297 /* So we won't read anything accidentally. */
7302 /* So we won't read anything accidentally. */
7305 /* Initialize issue_rate. */
7306 if (targetm
.sched
.issue_rate
)
7307 issue_rate
= targetm
.sched
.issue_rate ();
7311 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead
7312 /* Don't use max_issue with reg_pressure scheduling. Multipass
7313 scheduling and reg_pressure scheduling undo each other's decisions. */
7314 && sched_pressure
== SCHED_PRESSURE_NONE
)
7315 dfa_lookahead
= targetm
.sched
.first_cycle_multipass_dfa_lookahead ();
7319 /* Set to "0" so that we recalculate. */
7320 max_lookahead_tries
= 0;
7322 if (targetm
.sched
.init_dfa_pre_cycle_insn
)
7323 targetm
.sched
.init_dfa_pre_cycle_insn ();
7325 if (targetm
.sched
.init_dfa_post_cycle_insn
)
7326 targetm
.sched
.init_dfa_post_cycle_insn ();
7329 dfa_state_size
= state_size ();
7331 init_alias_analysis ();
7334 df_set_flags (DF_LR_RUN_DCE
);
7335 df_note_add_problem ();
7337 /* More problems needed for interloop dep calculation in SMS. */
7338 if (common_sched_info
->sched_pass_id
== SCHED_SMS_PASS
)
7340 df_rd_add_problem ();
7341 df_chain_add_problem (DF_DU_CHAIN
+ DF_UD_CHAIN
);
7346 /* Do not run DCE after reload, as this can kill nops inserted
7348 if (reload_completed
)
7349 df_clear_flags (DF_LR_RUN_DCE
);
7351 regstat_compute_calls_crossed ();
7353 if (targetm
.sched
.init_global
)
7354 targetm
.sched
.init_global (sched_dump
, sched_verbose
, get_max_uid () + 1);
7356 alloc_global_sched_pressure_data ();
7358 curr_state
= xmalloc (dfa_state_size
);
7361 static void haifa_init_only_bb (basic_block
, basic_block
);
7363 /* Initialize data structures specific to the Haifa scheduler. */
7365 haifa_sched_init (void)
7367 setup_sched_dump ();
7370 scheduled_insns
.create (0);
7372 if (spec_info
!= NULL
)
7374 sched_deps_info
->use_deps_list
= 1;
7375 sched_deps_info
->generate_spec_deps
= 1;
7378 /* Initialize luids, dependency caches, target and h_i_d for the
7383 auto_vec
<basic_block
> bbs (n_basic_blocks_for_fn (cfun
));
7385 FOR_EACH_BB_FN (bb
, cfun
)
7386 bbs
.quick_push (bb
);
7387 sched_init_luids (bbs
);
7388 sched_deps_init (true);
7389 sched_extend_target ();
7390 haifa_init_h_i_d (bbs
);
7393 sched_init_only_bb
= haifa_init_only_bb
;
7394 sched_split_block
= sched_split_block_1
;
7395 sched_create_empty_bb
= sched_create_empty_bb_1
;
7396 haifa_recovery_bb_ever_added_p
= false;
7398 nr_begin_data
= nr_begin_control
= nr_be_in_data
= nr_be_in_control
= 0;
7399 before_recovery
= 0;
7405 /* Finish work with the data specific to the Haifa scheduler. */
7407 haifa_sched_finish (void)
7409 sched_create_empty_bb
= NULL
;
7410 sched_split_block
= NULL
;
7411 sched_init_only_bb
= NULL
;
7413 if (spec_info
&& spec_info
->dump
)
7415 char c
= reload_completed
? 'a' : 'b';
7417 fprintf (spec_info
->dump
,
7418 ";; %s:\n", current_function_name ());
7420 fprintf (spec_info
->dump
,
7421 ";; Procedure %cr-begin-data-spec motions == %d\n",
7423 fprintf (spec_info
->dump
,
7424 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7426 fprintf (spec_info
->dump
,
7427 ";; Procedure %cr-begin-control-spec motions == %d\n",
7428 c
, nr_begin_control
);
7429 fprintf (spec_info
->dump
,
7430 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7431 c
, nr_be_in_control
);
7434 scheduled_insns
.release ();
7436 /* Finalize h_i_d, dependency caches, and luids for the whole
7437 function. Target will be finalized in md_global_finish (). */
7438 sched_deps_finish ();
7439 sched_finish_luids ();
7440 current_sched_info
= NULL
;
7445 /* Free global data used during insn scheduling. This function works with
7446 the common data shared between the schedulers. */
7451 haifa_finish_h_i_d ();
7452 free_global_sched_pressure_data ();
7455 if (targetm
.sched
.finish_global
)
7456 targetm
.sched
.finish_global (sched_dump
, sched_verbose
);
7458 end_alias_analysis ();
7460 regstat_free_calls_crossed ();
7465 /* Free all delay_pair structures that were recorded. */
7467 free_delay_pairs (void)
7471 delay_htab
->empty ();
7472 delay_htab_i2
->empty ();
7476 /* Fix INSN_TICKs of the instructions in the current block as well as
7477 INSN_TICKs of their dependents.
7478 HEAD and TAIL are the begin and the end of the current scheduled block. */
7480 fix_inter_tick (rtx_insn
*head
, rtx_insn
*tail
)
7482 /* Set of instructions with corrected INSN_TICK. */
7483 auto_bitmap processed
;
7484 /* ??? It is doubtful if we should assume that cycle advance happens on
7485 basic block boundaries. Basically insns that are unconditionally ready
7486 on the start of the block are more preferable then those which have
7487 a one cycle dependency over insn from the previous block. */
7488 int next_clock
= clock_var
+ 1;
7490 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7491 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7492 across different blocks. */
7493 for (tail
= NEXT_INSN (tail
); head
!= tail
; head
= NEXT_INSN (head
))
7498 sd_iterator_def sd_it
;
7501 tick
= INSN_TICK (head
);
7502 gcc_assert (tick
>= MIN_TICK
);
7504 /* Fix INSN_TICK of instruction from just scheduled block. */
7505 if (bitmap_set_bit (processed
, INSN_LUID (head
)))
7509 if (tick
< MIN_TICK
)
7512 INSN_TICK (head
) = tick
;
7515 if (DEBUG_INSN_P (head
))
7518 FOR_EACH_DEP (head
, SD_LIST_RES_FORW
, sd_it
, dep
)
7522 next
= DEP_CON (dep
);
7523 tick
= INSN_TICK (next
);
7525 if (tick
!= INVALID_TICK
7526 /* If NEXT has its INSN_TICK calculated, fix it.
7527 If not - it will be properly calculated from
7528 scratch later in fix_tick_ready. */
7529 && bitmap_set_bit (processed
, INSN_LUID (next
)))
7533 if (tick
< MIN_TICK
)
7536 if (tick
> INTER_TICK (next
))
7537 INTER_TICK (next
) = tick
;
7539 tick
= INTER_TICK (next
);
7541 INSN_TICK (next
) = tick
;
7548 /* Check if NEXT is ready to be added to the ready or queue list.
7549 If "yes", add it to the proper list.
7551 -1 - is not ready yet,
7552 0 - added to the ready list,
7553 0 < N - queued for N cycles. */
7555 try_ready (rtx_insn
*next
)
7557 ds_t old_ts
, new_ts
;
7559 old_ts
= TODO_SPEC (next
);
7561 gcc_assert (!(old_ts
& ~(SPECULATIVE
| HARD_DEP
| DEP_CONTROL
| DEP_POSTPONED
))
7562 && (old_ts
== HARD_DEP
7563 || old_ts
== DEP_POSTPONED
7564 || (old_ts
& SPECULATIVE
)
7565 || old_ts
== DEP_CONTROL
));
7567 new_ts
= recompute_todo_spec (next
, false);
7569 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7570 gcc_assert (new_ts
== old_ts
7571 && QUEUE_INDEX (next
) == QUEUE_NOWHERE
);
7572 else if (current_sched_info
->new_ready
)
7573 new_ts
= current_sched_info
->new_ready (next
, new_ts
);
7575 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7576 have its original pattern or changed (speculative) one. This is due
7577 to changing ebb in region scheduling.
7578 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7579 has speculative pattern.
7581 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7582 control-speculative NEXT could have been discarded by sched-rgn.cc
7583 (the same case as when discarded by can_schedule_ready_p ()). */
7585 if ((new_ts
& SPECULATIVE
)
7586 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7587 need to change anything. */
7588 && new_ts
!= old_ts
)
7593 gcc_assert ((new_ts
& SPECULATIVE
) && !(new_ts
& ~SPECULATIVE
));
7595 res
= haifa_speculate_insn (next
, new_ts
, &new_pat
);
7600 /* It would be nice to change DEP_STATUS of all dependences,
7601 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7602 so we won't reanalyze anything. */
7607 /* We follow the rule, that every speculative insn
7608 has non-null ORIG_PAT. */
7609 if (!ORIG_PAT (next
))
7610 ORIG_PAT (next
) = PATTERN (next
);
7614 if (!ORIG_PAT (next
))
7615 /* If we gonna to overwrite the original pattern of insn,
7617 ORIG_PAT (next
) = PATTERN (next
);
7619 res
= haifa_change_pattern (next
, new_pat
);
7628 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7629 either correct (new_ts & SPECULATIVE),
7630 or we simply don't care (new_ts & HARD_DEP). */
7632 gcc_assert (!ORIG_PAT (next
)
7633 || !IS_SPECULATION_BRANCHY_CHECK_P (next
));
7635 TODO_SPEC (next
) = new_ts
;
7637 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7639 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7640 control-speculative NEXT could have been discarded by sched-rgn.cc
7641 (the same case as when discarded by can_schedule_ready_p ()). */
7642 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7644 change_queue_index (next
, QUEUE_NOWHERE
);
7648 else if (!(new_ts
& BEGIN_SPEC
)
7649 && ORIG_PAT (next
) && PREDICATED_PAT (next
) == NULL_RTX
7650 && !IS_SPECULATION_CHECK_P (next
))
7651 /* We should change pattern of every previously speculative
7652 instruction - and we determine if NEXT was speculative by using
7653 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7654 pat too, so skip them. */
7656 bool success
= haifa_change_pattern (next
, ORIG_PAT (next
));
7657 gcc_assert (success
);
7658 ORIG_PAT (next
) = 0;
7661 if (sched_verbose
>= 2)
7663 fprintf (sched_dump
, ";;\t\tdependencies resolved: insn %s",
7664 (*current_sched_info
->print_insn
) (next
, 0));
7666 if (spec_info
&& spec_info
->dump
)
7668 if (new_ts
& BEGIN_DATA
)
7669 fprintf (spec_info
->dump
, "; data-spec;");
7670 if (new_ts
& BEGIN_CONTROL
)
7671 fprintf (spec_info
->dump
, "; control-spec;");
7672 if (new_ts
& BE_IN_CONTROL
)
7673 fprintf (spec_info
->dump
, "; in-control-spec;");
7675 if (TODO_SPEC (next
) & DEP_CONTROL
)
7676 fprintf (sched_dump
, " predicated");
7677 fprintf (sched_dump
, "\n");
7680 adjust_priority (next
);
7682 return fix_tick_ready (next
);
7685 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7687 fix_tick_ready (rtx_insn
*next
)
7691 if (!DEBUG_INSN_P (next
) && !sd_lists_empty_p (next
, SD_LIST_RES_BACK
))
7694 sd_iterator_def sd_it
;
7697 tick
= INSN_TICK (next
);
7698 /* if tick is not equal to INVALID_TICK, then update
7699 INSN_TICK of NEXT with the most recent resolved dependence
7700 cost. Otherwise, recalculate from scratch. */
7701 full_p
= (tick
== INVALID_TICK
);
7703 FOR_EACH_DEP (next
, SD_LIST_RES_BACK
, sd_it
, dep
)
7705 rtx_insn
*pro
= DEP_PRO (dep
);
7708 gcc_assert (INSN_TICK (pro
) >= MIN_TICK
);
7710 tick1
= INSN_TICK (pro
) + dep_cost (dep
);
7721 INSN_TICK (next
) = tick
;
7723 delay
= tick
- clock_var
;
7724 if (delay
<= 0 || sched_pressure
!= SCHED_PRESSURE_NONE
|| sched_fusion
)
7725 delay
= QUEUE_READY
;
7727 change_queue_index (next
, delay
);
7732 /* Move NEXT to the proper queue list with (DELAY >= 1),
7733 or add it to the ready list (DELAY == QUEUE_READY),
7734 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7736 change_queue_index (rtx_insn
*next
, int delay
)
7738 int i
= QUEUE_INDEX (next
);
7740 gcc_assert (QUEUE_NOWHERE
<= delay
&& delay
<= max_insn_queue_index
7742 gcc_assert (i
!= QUEUE_SCHEDULED
);
7744 if ((delay
> 0 && NEXT_Q_AFTER (q_ptr
, delay
) == i
)
7745 || (delay
< 0 && delay
== i
))
7746 /* We have nothing to do. */
7749 /* Remove NEXT from wherever it is now. */
7750 if (i
== QUEUE_READY
)
7751 ready_remove_insn (next
);
7753 queue_remove (next
);
7755 /* Add it to the proper place. */
7756 if (delay
== QUEUE_READY
)
7757 ready_add (readyp
, next
, false);
7758 else if (delay
>= 1)
7759 queue_insn (next
, delay
, "change queue index");
7761 if (sched_verbose
>= 2)
7763 fprintf (sched_dump
, ";;\t\ttick updated: insn %s",
7764 (*current_sched_info
->print_insn
) (next
, 0));
7766 if (delay
== QUEUE_READY
)
7767 fprintf (sched_dump
, " into ready\n");
7768 else if (delay
>= 1)
7769 fprintf (sched_dump
, " into queue with cost=%d\n", delay
);
7771 fprintf (sched_dump
, " removed from ready or queue lists\n");
7775 static int sched_ready_n_insns
= -1;
7777 /* Initialize per region data structures. */
7779 sched_extend_ready_list (int new_sched_ready_n_insns
)
7783 if (sched_ready_n_insns
== -1)
7784 /* At the first call we need to initialize one more choice_stack
7788 sched_ready_n_insns
= 0;
7789 scheduled_insns
.reserve (new_sched_ready_n_insns
);
7792 i
= sched_ready_n_insns
+ 1;
7794 ready
.veclen
= new_sched_ready_n_insns
+ issue_rate
;
7795 ready
.vec
= XRESIZEVEC (rtx_insn
*, ready
.vec
, ready
.veclen
);
7797 gcc_assert (new_sched_ready_n_insns
>= sched_ready_n_insns
);
7799 ready_try
= (signed char *) xrecalloc (ready_try
, new_sched_ready_n_insns
,
7800 sched_ready_n_insns
,
7801 sizeof (*ready_try
));
7803 /* We allocate +1 element to save initial state in the choice_stack[0]
7805 choice_stack
= XRESIZEVEC (struct choice_entry
, choice_stack
,
7806 new_sched_ready_n_insns
+ 1);
7808 for (; i
<= new_sched_ready_n_insns
; i
++)
7810 choice_stack
[i
].state
= xmalloc (dfa_state_size
);
7812 if (targetm
.sched
.first_cycle_multipass_init
)
7813 targetm
.sched
.first_cycle_multipass_init (&(choice_stack
[i
]
7817 sched_ready_n_insns
= new_sched_ready_n_insns
;
7820 /* Free per region data structures. */
7822 sched_finish_ready_list (void)
7833 for (i
= 0; i
<= sched_ready_n_insns
; i
++)
7835 if (targetm
.sched
.first_cycle_multipass_fini
)
7836 targetm
.sched
.first_cycle_multipass_fini (&(choice_stack
[i
]
7839 free (choice_stack
[i
].state
);
7841 free (choice_stack
);
7842 choice_stack
= NULL
;
7844 sched_ready_n_insns
= -1;
7848 haifa_luid_for_non_insn (rtx x
)
7850 gcc_assert (NOTE_P (x
) || LABEL_P (x
));
7855 /* Generates recovery code for INSN. */
7857 generate_recovery_code (rtx_insn
*insn
)
7859 if (TODO_SPEC (insn
) & BEGIN_SPEC
)
7860 begin_speculative_block (insn
);
7862 /* Here we have insn with no dependencies to
7863 instructions other then CHECK_SPEC ones. */
7865 if (TODO_SPEC (insn
) & BE_IN_SPEC
)
7866 add_to_speculative_block (insn
);
7870 Tries to add speculative dependencies of type FS between instructions
7871 in deps_list L and TWIN. */
7873 process_insn_forw_deps_be_in_spec (rtx_insn
*insn
, rtx_insn
*twin
, ds_t fs
)
7875 sd_iterator_def sd_it
;
7878 FOR_EACH_DEP (insn
, SD_LIST_FORW
, sd_it
, dep
)
7883 consumer
= DEP_CON (dep
);
7885 ds
= DEP_STATUS (dep
);
7887 if (/* If we want to create speculative dep. */
7889 /* And we can do that because this is a true dep. */
7890 && (ds
& DEP_TYPES
) == DEP_TRUE
)
7892 gcc_assert (!(ds
& BE_IN_SPEC
));
7894 if (/* If this dep can be overcome with 'begin speculation'. */
7896 /* Then we have a choice: keep the dep 'begin speculative'
7897 or transform it into 'be in speculative'. */
7899 if (/* In try_ready we assert that if insn once became ready
7900 it can be removed from the ready (or queue) list only
7901 due to backend decision. Hence we can't let the
7902 probability of the speculative dep to decrease. */
7903 ds_weak (ds
) <= ds_weak (fs
))
7907 new_ds
= (ds
& ~BEGIN_SPEC
) | fs
;
7909 if (/* consumer can 'be in speculative'. */
7910 sched_insn_is_legitimate_for_speculation_p (consumer
,
7912 /* Transform it to be in speculative. */
7917 /* Mark the dep as 'be in speculative'. */
7922 dep_def _new_dep
, *new_dep
= &_new_dep
;
7924 init_dep_1 (new_dep
, twin
, consumer
, DEP_TYPE (dep
), ds
);
7925 sd_add_dep (new_dep
, false);
7930 /* Generates recovery code for BEGIN speculative INSN. */
7932 begin_speculative_block (rtx_insn
*insn
)
7934 if (TODO_SPEC (insn
) & BEGIN_DATA
)
7936 if (TODO_SPEC (insn
) & BEGIN_CONTROL
)
7939 create_check_block_twin (insn
, false);
7941 TODO_SPEC (insn
) &= ~BEGIN_SPEC
;
7944 static void haifa_init_insn (rtx_insn
*);
7946 /* Generates recovery code for BE_IN speculative INSN. */
7948 add_to_speculative_block (rtx_insn
*insn
)
7951 sd_iterator_def sd_it
;
7953 auto_vec
<rtx_insn
*, 10> twins
;
7955 ts
= TODO_SPEC (insn
);
7956 gcc_assert (!(ts
& ~BE_IN_SPEC
));
7958 if (ts
& BE_IN_DATA
)
7960 if (ts
& BE_IN_CONTROL
)
7963 TODO_SPEC (insn
) &= ~BE_IN_SPEC
;
7964 gcc_assert (!TODO_SPEC (insn
));
7966 DONE_SPEC (insn
) |= ts
;
7968 /* First we convert all simple checks to branchy. */
7969 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7970 sd_iterator_cond (&sd_it
, &dep
);)
7972 rtx_insn
*check
= DEP_PRO (dep
);
7974 if (IS_SPECULATION_SIMPLE_CHECK_P (check
))
7976 create_check_block_twin (check
, true);
7978 /* Restart search. */
7979 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7982 /* Continue search. */
7983 sd_iterator_next (&sd_it
);
7986 auto_vec
<rtx_insn
*> priorities_roots
;
7987 clear_priorities (insn
, &priorities_roots
);
7991 rtx_insn
*check
, *twin
;
7994 /* Get the first backward dependency of INSN. */
7995 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7996 if (!sd_iterator_cond (&sd_it
, &dep
))
7997 /* INSN has no backward dependencies left. */
8000 gcc_assert ((DEP_STATUS (dep
) & BEGIN_SPEC
) == 0
8001 && (DEP_STATUS (dep
) & BE_IN_SPEC
) != 0
8002 && (DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8004 check
= DEP_PRO (dep
);
8006 gcc_assert (!IS_SPECULATION_CHECK_P (check
) && !ORIG_PAT (check
)
8007 && QUEUE_INDEX (check
) == QUEUE_NOWHERE
);
8009 rec
= BLOCK_FOR_INSN (check
);
8011 twin
= emit_insn_before (copy_insn (PATTERN (insn
)), BB_END (rec
));
8012 haifa_init_insn (twin
);
8014 sd_copy_back_deps (twin
, insn
, true);
8016 if (sched_verbose
&& spec_info
->dump
)
8017 /* INSN_BB (insn) isn't determined for twin insns yet.
8018 So we can't use current_sched_info->print_insn. */
8019 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8020 INSN_UID (twin
), rec
->index
);
8022 twins
.safe_push (twin
);
8024 /* Add dependences between TWIN and all appropriate
8025 instructions from REC. */
8026 FOR_EACH_DEP (insn
, SD_LIST_SPEC_BACK
, sd_it
, dep
)
8028 rtx_insn
*pro
= DEP_PRO (dep
);
8030 gcc_assert (DEP_TYPE (dep
) == REG_DEP_TRUE
);
8032 /* INSN might have dependencies from the instructions from
8033 several recovery blocks. At this iteration we process those
8034 producers that reside in REC. */
8035 if (BLOCK_FOR_INSN (pro
) == rec
)
8037 dep_def _new_dep
, *new_dep
= &_new_dep
;
8039 init_dep (new_dep
, pro
, twin
, REG_DEP_TRUE
);
8040 sd_add_dep (new_dep
, false);
8044 process_insn_forw_deps_be_in_spec (insn
, twin
, ts
);
8046 /* Remove all dependencies between INSN and insns in REC. */
8047 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8048 sd_iterator_cond (&sd_it
, &dep
);)
8050 rtx_insn
*pro
= DEP_PRO (dep
);
8052 if (BLOCK_FOR_INSN (pro
) == rec
)
8053 sd_delete_dep (sd_it
);
8055 sd_iterator_next (&sd_it
);
8059 /* We couldn't have added the dependencies between INSN and TWINS earlier
8060 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
8063 FOR_EACH_VEC_ELT_REVERSE (twins
, i
, twin
)
8065 dep_def _new_dep
, *new_dep
= &_new_dep
;
8067 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8068 sd_add_dep (new_dep
, false);
8071 calc_priorities (priorities_roots
);
8074 /* Extends and fills with zeros (only the new part) array pointed to by P. */
8076 xrecalloc (void *p
, size_t new_nmemb
, size_t old_nmemb
, size_t size
)
8078 gcc_assert (new_nmemb
>= old_nmemb
);
8079 p
= XRESIZEVAR (void, p
, new_nmemb
* size
);
8080 memset (((char *) p
) + old_nmemb
* size
, 0, (new_nmemb
- old_nmemb
) * size
);
8085 Find fallthru edge from PRED. */
8087 find_fallthru_edge_from (basic_block pred
)
8092 succ
= pred
->next_bb
;
8093 gcc_assert (succ
->prev_bb
== pred
);
8095 if (EDGE_COUNT (pred
->succs
) <= EDGE_COUNT (succ
->preds
))
8097 e
= find_fallthru_edge (pred
->succs
);
8101 gcc_assert (e
->dest
== succ
|| e
->dest
->index
== EXIT_BLOCK
);
8107 e
= find_fallthru_edge (succ
->preds
);
8111 gcc_assert (e
->src
== pred
);
8119 /* Extend per basic block data structures. */
8121 sched_extend_bb (void)
8123 /* The following is done to keep current_sched_info->next_tail non null. */
8124 rtx_insn
*end
= BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8125 rtx_insn
*insn
= DEBUG_INSN_P (end
) ? prev_nondebug_insn (end
) : end
;
8126 if (NEXT_INSN (end
) == 0
8129 /* Don't emit a NOTE if it would end up before a BARRIER. */
8130 && !BARRIER_P (next_nondebug_insn (end
))))
8132 rtx_note
*note
= emit_note_after (NOTE_INSN_DELETED
, end
);
8133 /* Make note appear outside BB. */
8134 set_block_for_insn (note
, NULL
);
8135 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
) = end
;
8139 /* Init per basic block data structures. */
8141 sched_init_bbs (void)
8146 /* Initialize BEFORE_RECOVERY variable. */
8148 init_before_recovery (basic_block
*before_recovery_ptr
)
8153 last
= EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
;
8154 e
= find_fallthru_edge_from (last
);
8158 /* We create two basic blocks:
8159 1. Single instruction block is inserted right after E->SRC
8161 2. Empty block right before EXIT_BLOCK.
8162 Between these two blocks recovery blocks will be emitted. */
8164 basic_block single
, empty
;
8166 /* If the fallthrough edge to exit we've found is from the block we've
8167 created before, don't do anything more. */
8168 if (last
== after_recovery
)
8171 adding_bb_to_current_region_p
= false;
8173 single
= sched_create_empty_bb (last
);
8174 empty
= sched_create_empty_bb (single
);
8176 /* Add new blocks to the root loop. */
8177 if (current_loops
!= NULL
)
8179 add_bb_to_loop (single
, (*current_loops
->larray
)[0]);
8180 add_bb_to_loop (empty
, (*current_loops
->larray
)[0]);
8183 single
->count
= last
->count
;
8184 empty
->count
= last
->count
;
8185 BB_COPY_PARTITION (single
, last
);
8186 BB_COPY_PARTITION (empty
, last
);
8188 redirect_edge_succ (e
, single
);
8189 make_single_succ_edge (single
, empty
, 0);
8190 make_single_succ_edge (empty
, EXIT_BLOCK_PTR_FOR_FN (cfun
),
8193 rtx_code_label
*label
= block_label (empty
);
8194 rtx_jump_insn
*x
= emit_jump_insn_after (targetm
.gen_jump (label
),
8196 JUMP_LABEL (x
) = label
;
8197 LABEL_NUSES (label
)++;
8198 haifa_init_insn (x
);
8200 emit_barrier_after (x
);
8202 sched_init_only_bb (empty
, NULL
);
8203 sched_init_only_bb (single
, NULL
);
8206 adding_bb_to_current_region_p
= true;
8207 before_recovery
= single
;
8208 after_recovery
= empty
;
8210 if (before_recovery_ptr
)
8211 *before_recovery_ptr
= before_recovery
;
8213 if (sched_verbose
>= 2 && spec_info
->dump
)
8214 fprintf (spec_info
->dump
,
8215 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8216 last
->index
, single
->index
, empty
->index
);
8219 before_recovery
= last
;
8222 /* Returns new recovery block. */
8224 sched_create_recovery_block (basic_block
*before_recovery_ptr
)
8229 haifa_recovery_bb_recently_added_p
= true;
8230 haifa_recovery_bb_ever_added_p
= true;
8232 init_before_recovery (before_recovery_ptr
);
8234 barrier
= get_last_bb_insn (before_recovery
);
8235 gcc_assert (BARRIER_P (barrier
));
8237 rtx_insn
*label
= emit_label_after (gen_label_rtx (), barrier
);
8239 rec
= create_basic_block (label
, label
, before_recovery
);
8241 /* A recovery block always ends with an unconditional jump. */
8242 emit_barrier_after (BB_END (rec
));
8244 if (BB_PARTITION (before_recovery
) != BB_UNPARTITIONED
)
8245 BB_SET_PARTITION (rec
, BB_COLD_PARTITION
);
8247 if (sched_verbose
&& spec_info
->dump
)
8248 fprintf (spec_info
->dump
, ";;\t\tGenerated recovery block rec%d\n",
8254 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8255 and emit necessary jumps. */
8257 sched_create_recovery_edges (basic_block first_bb
, basic_block rec
,
8258 basic_block second_bb
)
8262 /* This is fixing of incoming edge. */
8263 /* ??? Which other flags should be specified? */
8264 if (BB_PARTITION (first_bb
) != BB_PARTITION (rec
))
8265 /* Partition type is the same, if it is "unpartitioned". */
8266 edge_flags
= EDGE_CROSSING
;
8270 edge e2
= single_succ_edge (first_bb
);
8271 edge e
= make_edge (first_bb
, rec
, edge_flags
);
8273 /* TODO: The actual probability can be determined and is computed as
8274 'todo_spec' variable in create_check_block_twin and
8275 in sel-sched.cc `check_ds' in create_speculation_check. */
8276 e
->probability
= profile_probability::very_unlikely ();
8277 rec
->count
= e
->count ();
8278 e2
->probability
= e
->probability
.invert ();
8280 rtx_code_label
*label
= block_label (second_bb
);
8281 rtx_jump_insn
*jump
= emit_jump_insn_after (targetm
.gen_jump (label
),
8283 JUMP_LABEL (jump
) = label
;
8284 LABEL_NUSES (label
)++;
8286 if (BB_PARTITION (second_bb
) != BB_PARTITION (rec
))
8287 /* Partition type is the same, if it is "unpartitioned". */
8289 /* Rewritten from cfgrtl.cc. */
8290 if (crtl
->has_bb_partition
&& targetm_common
.have_named_sections
)
8292 /* We don't need the same note for the check because
8293 any_condjump_p (check) == true. */
8294 CROSSING_JUMP_P (jump
) = 1;
8296 edge_flags
= EDGE_CROSSING
;
8301 make_single_succ_edge (rec
, second_bb
, edge_flags
);
8302 if (dom_info_available_p (CDI_DOMINATORS
))
8303 set_immediate_dominator (CDI_DOMINATORS
, rec
, first_bb
);
8306 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8307 INSN is a simple check, that should be converted to branchy one. */
8309 create_check_block_twin (rtx_insn
*insn
, bool mutate_p
)
8312 rtx_insn
*label
, *check
, *twin
;
8315 sd_iterator_def sd_it
;
8317 dep_def _new_dep
, *new_dep
= &_new_dep
;
8320 gcc_assert (ORIG_PAT (insn
) != NULL_RTX
);
8323 todo_spec
= TODO_SPEC (insn
);
8326 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn
)
8327 && (TODO_SPEC (insn
) & SPECULATIVE
) == 0);
8329 todo_spec
= CHECK_SPEC (insn
);
8332 todo_spec
&= SPECULATIVE
;
8334 /* Create recovery block. */
8335 if (mutate_p
|| targetm
.sched
.needs_block_p (todo_spec
))
8337 rec
= sched_create_recovery_block (NULL
);
8338 label
= BB_HEAD (rec
);
8342 rec
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
8347 check_pat
= targetm
.sched
.gen_spec_check (insn
, label
, todo_spec
);
8349 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8351 /* To have mem_reg alive at the beginning of second_bb,
8352 we emit check BEFORE insn, so insn after splitting
8353 insn will be at the beginning of second_bb, which will
8354 provide us with the correct life information. */
8355 check
= emit_jump_insn_before (check_pat
, insn
);
8356 JUMP_LABEL (check
) = label
;
8357 LABEL_NUSES (label
)++;
8360 check
= emit_insn_before (check_pat
, insn
);
8362 /* Extend data structures. */
8363 haifa_init_insn (check
);
8365 /* CHECK is being added to current region. Extend ready list. */
8366 gcc_assert (sched_ready_n_insns
!= -1);
8367 sched_extend_ready_list (sched_ready_n_insns
+ 1);
8369 if (current_sched_info
->add_remove_insn
)
8370 current_sched_info
->add_remove_insn (insn
, 0);
8372 RECOVERY_BLOCK (check
) = rec
;
8374 if (sched_verbose
&& spec_info
->dump
)
8375 fprintf (spec_info
->dump
, ";;\t\tGenerated check insn : %s\n",
8376 (*current_sched_info
->print_insn
) (check
, 0));
8378 gcc_assert (ORIG_PAT (insn
));
8380 /* Initialize TWIN (twin is a duplicate of original instruction
8381 in the recovery block). */
8382 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8384 sd_iterator_def sd_it
;
8387 FOR_EACH_DEP (insn
, SD_LIST_RES_BACK
, sd_it
, dep
)
8388 if ((DEP_STATUS (dep
) & DEP_OUTPUT
) != 0)
8390 struct _dep _dep2
, *dep2
= &_dep2
;
8392 init_dep (dep2
, DEP_PRO (dep
), check
, REG_DEP_TRUE
);
8394 sd_add_dep (dep2
, true);
8397 twin
= emit_insn_after (ORIG_PAT (insn
), BB_END (rec
));
8398 haifa_init_insn (twin
);
8400 if (sched_verbose
&& spec_info
->dump
)
8401 /* INSN_BB (insn) isn't determined for twin insns yet.
8402 So we can't use current_sched_info->print_insn. */
8403 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8404 INSN_UID (twin
), rec
->index
);
8408 ORIG_PAT (check
) = ORIG_PAT (insn
);
8409 HAS_INTERNAL_DEP (check
) = 1;
8411 /* ??? We probably should change all OUTPUT dependencies to
8415 /* Copy all resolved back dependencies of INSN to TWIN. This will
8416 provide correct value for INSN_TICK (TWIN). */
8417 sd_copy_back_deps (twin
, insn
, true);
8419 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8420 /* In case of branchy check, fix CFG. */
8422 basic_block first_bb
, second_bb
;
8425 first_bb
= BLOCK_FOR_INSN (check
);
8426 second_bb
= sched_split_block (first_bb
, check
);
8428 sched_create_recovery_edges (first_bb
, rec
, second_bb
);
8430 sched_init_only_bb (second_bb
, first_bb
);
8431 sched_init_only_bb (rec
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
8433 jump
= BB_END (rec
);
8434 haifa_init_insn (jump
);
8437 /* Move backward dependences from INSN to CHECK and
8438 move forward dependences from INSN to TWIN. */
8440 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8441 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8443 rtx_insn
*pro
= DEP_PRO (dep
);
8446 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8447 check --TRUE--> producer ??? or ANTI ???
8448 twin --TRUE--> producer
8449 twin --ANTI--> check
8451 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8452 check --ANTI--> producer
8453 twin --ANTI--> producer
8454 twin --ANTI--> check
8456 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8457 check ~~TRUE~~> producer
8458 twin ~~TRUE~~> producer
8459 twin --ANTI--> check */
8461 ds
= DEP_STATUS (dep
);
8463 if (ds
& BEGIN_SPEC
)
8465 gcc_assert (!mutate_p
);
8469 init_dep_1 (new_dep
, pro
, check
, DEP_TYPE (dep
), ds
);
8470 sd_add_dep (new_dep
, false);
8472 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8474 DEP_CON (new_dep
) = twin
;
8475 sd_add_dep (new_dep
, false);
8479 /* Second, remove backward dependencies of INSN. */
8480 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8481 sd_iterator_cond (&sd_it
, &dep
);)
8483 if ((DEP_STATUS (dep
) & BEGIN_SPEC
)
8485 /* We can delete this dep because we overcome it with
8486 BEGIN_SPECULATION. */
8487 sd_delete_dep (sd_it
);
8489 sd_iterator_next (&sd_it
);
8492 /* Future Speculations. Determine what BE_IN speculations will be like. */
8495 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8498 gcc_assert (!DONE_SPEC (insn
));
8502 ds_t ts
= TODO_SPEC (insn
);
8504 DONE_SPEC (insn
) = ts
& BEGIN_SPEC
;
8505 CHECK_SPEC (check
) = ts
& BEGIN_SPEC
;
8507 /* Luckiness of future speculations solely depends upon initial
8508 BEGIN speculation. */
8509 if (ts
& BEGIN_DATA
)
8510 fs
= set_dep_weak (fs
, BE_IN_DATA
, get_dep_weak (ts
, BEGIN_DATA
));
8511 if (ts
& BEGIN_CONTROL
)
8512 fs
= set_dep_weak (fs
, BE_IN_CONTROL
,
8513 get_dep_weak (ts
, BEGIN_CONTROL
));
8516 CHECK_SPEC (check
) = CHECK_SPEC (insn
);
8518 /* Future speculations: call the helper. */
8519 process_insn_forw_deps_be_in_spec (insn
, twin
, fs
);
8521 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8523 /* Which types of dependencies should we use here is,
8524 generally, machine-dependent question... But, for now,
8529 init_dep (new_dep
, insn
, check
, REG_DEP_TRUE
);
8530 sd_add_dep (new_dep
, false);
8532 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8533 sd_add_dep (new_dep
, false);
8537 if (spec_info
->dump
)
8538 fprintf (spec_info
->dump
, ";;\t\tRemoved simple check : %s\n",
8539 (*current_sched_info
->print_insn
) (insn
, 0));
8541 /* Remove all dependencies of the INSN. */
8543 sd_it
= sd_iterator_start (insn
, (SD_LIST_FORW
8545 | SD_LIST_RES_BACK
));
8546 while (sd_iterator_cond (&sd_it
, &dep
))
8547 sd_delete_dep (sd_it
);
8550 /* If former check (INSN) already was moved to the ready (or queue)
8551 list, add new check (CHECK) there too. */
8552 if (QUEUE_INDEX (insn
) != QUEUE_NOWHERE
)
8555 /* Remove old check from instruction stream and free its
8557 sched_remove_insn (insn
);
8560 init_dep (new_dep
, check
, twin
, REG_DEP_ANTI
);
8561 sd_add_dep (new_dep
, false);
8565 init_dep_1 (new_dep
, insn
, check
, REG_DEP_TRUE
, DEP_TRUE
| DEP_OUTPUT
);
8566 sd_add_dep (new_dep
, false);
8570 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8571 because it'll be done later in add_to_speculative_block. */
8573 auto_vec
<rtx_insn
*> priorities_roots
;
8575 clear_priorities (twin
, &priorities_roots
);
8576 calc_priorities (priorities_roots
);
8580 /* Removes dependency between instructions in the recovery block REC
8581 and usual region instructions. It keeps inner dependences so it
8582 won't be necessary to recompute them. */
8584 fix_recovery_deps (basic_block rec
)
8586 rtx_insn
*note
, *insn
, *jump
;
8587 auto_vec
<rtx_insn
*, 10> ready_list
;
8588 auto_bitmap in_ready
;
8590 /* NOTE - a basic block note. */
8591 note
= NEXT_INSN (BB_HEAD (rec
));
8592 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8593 insn
= BB_END (rec
);
8594 gcc_assert (JUMP_P (insn
));
8595 insn
= PREV_INSN (insn
);
8599 sd_iterator_def sd_it
;
8602 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
8603 sd_iterator_cond (&sd_it
, &dep
);)
8605 rtx_insn
*consumer
= DEP_CON (dep
);
8607 if (BLOCK_FOR_INSN (consumer
) != rec
)
8609 sd_delete_dep (sd_it
);
8611 if (bitmap_set_bit (in_ready
, INSN_LUID (consumer
)))
8612 ready_list
.safe_push (consumer
);
8616 gcc_assert ((DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8618 sd_iterator_next (&sd_it
);
8622 insn
= PREV_INSN (insn
);
8624 while (insn
!= note
);
8626 /* Try to add instructions to the ready or queue list. */
8629 FOR_EACH_VEC_ELT_REVERSE (ready_list
, i
, temp
)
8632 /* Fixing jump's dependences. */
8633 insn
= BB_HEAD (rec
);
8634 jump
= BB_END (rec
);
8636 gcc_assert (LABEL_P (insn
));
8637 insn
= NEXT_INSN (insn
);
8639 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn
));
8640 add_jump_dependencies (insn
, jump
);
8643 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8644 instruction data. */
8646 haifa_change_pattern (rtx_insn
*insn
, rtx new_pat
)
8650 t
= validate_change (insn
, &PATTERN (insn
), new_pat
, 0);
8654 update_insn_after_change (insn
);
8658 /* -1 - can't speculate,
8659 0 - for speculation with REQUEST mode it is OK to use
8660 current instruction pattern,
8661 1 - need to change pattern for *NEW_PAT to be speculative. */
8663 sched_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8665 gcc_assert (current_sched_info
->flags
& DO_SPECULATION
8666 && (request
& SPECULATIVE
)
8667 && sched_insn_is_legitimate_for_speculation_p (insn
, request
));
8669 if ((request
& spec_info
->mask
) != request
)
8672 if (request
& BE_IN_SPEC
8673 && !(request
& BEGIN_SPEC
))
8676 return targetm
.sched
.speculate_insn (insn
, request
, new_pat
);
8680 haifa_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8682 gcc_assert (sched_deps_info
->generate_spec_deps
8683 && !IS_SPECULATION_CHECK_P (insn
));
8685 if (HAS_INTERNAL_DEP (insn
)
8686 || SCHED_GROUP_P (insn
))
8689 return sched_speculate_insn (insn
, request
, new_pat
);
8692 /* Print some information about block BB, which starts with HEAD and
8693 ends with TAIL, before scheduling it.
8694 I is zero, if scheduler is about to start with the fresh ebb. */
8696 dump_new_block_header (int i
, basic_block bb
, rtx_insn
*head
, rtx_insn
*tail
)
8699 fprintf (sched_dump
,
8700 ";; ======================================================\n");
8702 fprintf (sched_dump
,
8703 ";; =====================ADVANCING TO=====================\n");
8704 fprintf (sched_dump
,
8705 ";; -- basic block %d from %d to %d -- %s reload\n",
8706 bb
->index
, INSN_UID (head
), INSN_UID (tail
),
8707 (reload_completed
? "after" : "before"));
8708 fprintf (sched_dump
,
8709 ";; ======================================================\n");
8710 fprintf (sched_dump
, "\n");
8713 /* Unlink basic block notes and labels and saves them, so they
8714 can be easily restored. We unlink basic block notes in EBB to
8715 provide back-compatibility with the previous code, as target backends
8716 assume, that there'll be only instructions between
8717 current_sched_info->{head and tail}. We restore these notes as soon
8719 FIRST (LAST) is the first (last) basic block in the ebb.
8720 NB: In usual case (FIRST == LAST) nothing is really done. */
8722 unlink_bb_notes (basic_block first
, basic_block last
)
8724 /* We DON'T unlink basic block notes of the first block in the ebb. */
8728 bb_header
= XNEWVEC (rtx_insn
*, last_basic_block_for_fn (cfun
));
8730 /* Make a sentinel. */
8731 if (last
->next_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8732 bb_header
[last
->next_bb
->index
] = 0;
8734 first
= first
->next_bb
;
8737 rtx_insn
*prev
, *label
, *note
, *next
;
8739 label
= BB_HEAD (last
);
8740 if (LABEL_P (label
))
8741 note
= NEXT_INSN (label
);
8744 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8746 prev
= PREV_INSN (label
);
8747 next
= NEXT_INSN (note
);
8748 gcc_assert (prev
&& next
);
8750 SET_NEXT_INSN (prev
) = next
;
8751 SET_PREV_INSN (next
) = prev
;
8753 bb_header
[last
->index
] = label
;
8758 last
= last
->prev_bb
;
8763 /* Restore basic block notes.
8764 FIRST is the first basic block in the ebb. */
8766 restore_bb_notes (basic_block first
)
8771 /* We DON'T unlink basic block notes of the first block in the ebb. */
8772 first
= first
->next_bb
;
8773 /* Remember: FIRST is actually a second basic block in the ebb. */
8775 while (first
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
8776 && bb_header
[first
->index
])
8778 rtx_insn
*prev
, *label
, *note
, *next
;
8780 label
= bb_header
[first
->index
];
8781 prev
= PREV_INSN (label
);
8782 next
= NEXT_INSN (prev
);
8784 if (LABEL_P (label
))
8785 note
= NEXT_INSN (label
);
8788 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8790 bb_header
[first
->index
] = 0;
8792 SET_NEXT_INSN (prev
) = label
;
8793 SET_NEXT_INSN (note
) = next
;
8794 SET_PREV_INSN (next
) = note
;
8796 first
= first
->next_bb
;
8804 Fix CFG after both in- and inter-block movement of
8805 control_flow_insn_p JUMP. */
8807 fix_jump_move (rtx_insn
*jump
)
8809 basic_block bb
, jump_bb
, jump_bb_next
;
8811 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8812 jump_bb
= BLOCK_FOR_INSN (jump
);
8813 jump_bb_next
= jump_bb
->next_bb
;
8815 gcc_assert (common_sched_info
->sched_pass_id
== SCHED_EBB_PASS
8816 || IS_SPECULATION_BRANCHY_CHECK_P (jump
));
8818 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next
)))
8819 /* if jump_bb_next is not empty. */
8820 BB_END (jump_bb
) = BB_END (jump_bb_next
);
8822 if (BB_END (bb
) != PREV_INSN (jump
))
8823 /* Then there are instruction after jump that should be placed
8825 BB_END (jump_bb_next
) = BB_END (bb
);
8827 /* Otherwise jump_bb_next is empty. */
8828 BB_END (jump_bb_next
) = NEXT_INSN (BB_HEAD (jump_bb_next
));
8830 /* To make assertion in move_insn happy. */
8831 BB_END (bb
) = PREV_INSN (jump
);
8833 update_bb_for_insn (jump_bb_next
);
8836 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8838 move_block_after_check (rtx_insn
*jump
)
8840 basic_block bb
, jump_bb
, jump_bb_next
;
8841 vec
<edge
, va_gc
> *t
;
8843 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8844 jump_bb
= BLOCK_FOR_INSN (jump
);
8845 jump_bb_next
= jump_bb
->next_bb
;
8847 update_bb_for_insn (jump_bb
);
8849 gcc_assert (IS_SPECULATION_CHECK_P (jump
)
8850 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next
)));
8852 unlink_block (jump_bb_next
);
8853 link_block (jump_bb_next
, bb
);
8857 move_succs (&(jump_bb
->succs
), bb
);
8858 move_succs (&(jump_bb_next
->succs
), jump_bb
);
8859 move_succs (&t
, jump_bb_next
);
8861 df_mark_solutions_dirty ();
8863 common_sched_info
->fix_recovery_cfg
8864 (bb
->index
, jump_bb
->index
, jump_bb_next
->index
);
8867 /* Helper function for move_block_after_check.
8868 This functions attaches edge vector pointed to by SUCCSP to
8871 move_succs (vec
<edge
, va_gc
> **succsp
, basic_block to
)
8876 gcc_assert (to
->succs
== 0);
8878 to
->succs
= *succsp
;
8880 FOR_EACH_EDGE (e
, ei
, to
->succs
)
8886 /* Remove INSN from the instruction stream.
8887 INSN should have any dependencies. */
8889 sched_remove_insn (rtx_insn
*insn
)
8891 sd_finish_insn (insn
);
8893 change_queue_index (insn
, QUEUE_NOWHERE
);
8894 current_sched_info
->add_remove_insn (insn
, 1);
8898 /* Clear priorities of all instructions, that are forward dependent on INSN.
8899 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8900 be invoked to initialize all cleared priorities. */
8902 clear_priorities (rtx_insn
*insn
, rtx_vec_t
*roots_ptr
)
8904 sd_iterator_def sd_it
;
8906 bool insn_is_root_p
= true;
8908 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
8910 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8912 rtx_insn
*pro
= DEP_PRO (dep
);
8914 if (INSN_PRIORITY_STATUS (pro
) >= 0
8915 && QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
8917 /* If DEP doesn't contribute to priority then INSN itself should
8918 be added to priority roots. */
8919 if (contributes_to_priority_p (dep
))
8920 insn_is_root_p
= false;
8922 INSN_PRIORITY_STATUS (pro
) = -1;
8923 clear_priorities (pro
, roots_ptr
);
8928 roots_ptr
->safe_push (insn
);
8931 /* Recompute priorities of instructions, whose priorities might have been
8932 changed. ROOTS is a vector of instructions whose priority computation will
8933 trigger initialization of all cleared priorities. */
8935 calc_priorities (const rtx_vec_t
&roots
)
8940 FOR_EACH_VEC_ELT (roots
, i
, insn
)
8945 /* Add dependences between JUMP and other instructions in the recovery
8946 block. INSN is the first insn the recovery block. */
8948 add_jump_dependencies (rtx_insn
*insn
, rtx_insn
*jump
)
8952 insn
= NEXT_INSN (insn
);
8956 if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
8958 dep_def _new_dep
, *new_dep
= &_new_dep
;
8960 init_dep (new_dep
, insn
, jump
, REG_DEP_ANTI
);
8961 sd_add_dep (new_dep
, false);
8966 gcc_assert (!sd_lists_empty_p (jump
, SD_LIST_BACK
));
8969 /* Extend data structures for logical insn UID. */
8971 sched_extend_luids (void)
8973 int new_luids_max_uid
= get_max_uid () + 1;
8975 sched_luids
.safe_grow_cleared (new_luids_max_uid
, true);
8978 /* Initialize LUID for INSN. */
8980 sched_init_insn_luid (rtx_insn
*insn
)
8982 int i
= INSN_P (insn
) ? 1 : common_sched_info
->luid_for_non_insn (insn
);
8987 luid
= sched_max_luid
;
8988 sched_max_luid
+= i
;
8993 SET_INSN_LUID (insn
, luid
);
8996 /* Initialize luids for BBS.
8997 The hook common_sched_info->luid_for_non_insn () is used to determine
8998 if notes, labels, etc. need luids. */
9000 sched_init_luids (const bb_vec_t
&bbs
)
9005 sched_extend_luids ();
9006 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9010 FOR_BB_INSNS (bb
, insn
)
9011 sched_init_insn_luid (insn
);
9017 sched_finish_luids (void)
9019 sched_luids
.release ();
9023 /* Return logical uid of INSN. Helpful while debugging. */
9025 insn_luid (rtx_insn
*insn
)
9027 return INSN_LUID (insn
);
9030 /* Extend per insn data in the target. */
9032 sched_extend_target (void)
9034 if (targetm
.sched
.h_i_d_extended
)
9035 targetm
.sched
.h_i_d_extended ();
9038 /* Extend global scheduler structures (those, that live across calls to
9039 schedule_block) to include information about just emitted INSN. */
9043 int reserve
= (get_max_uid () + 1 - h_i_d
.length ());
9045 && ! h_i_d
.space (reserve
))
9047 h_i_d
.safe_grow_cleared (3U * get_max_uid () / 2, true);
9048 sched_extend_target ();
9052 /* Initialize h_i_d entry of the INSN with default values.
9053 Values, that are not explicitly initialized here, hold zero. */
9055 init_h_i_d (rtx_insn
*insn
)
9057 if (INSN_LUID (insn
) > 0)
9059 INSN_COST (insn
) = -1;
9060 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
9061 INSN_TICK (insn
) = INVALID_TICK
;
9062 INSN_EXACT_TICK (insn
) = INVALID_TICK
;
9063 INTER_TICK (insn
) = INVALID_TICK
;
9064 TODO_SPEC (insn
) = HARD_DEP
;
9065 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
9066 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9067 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
9068 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9072 /* Initialize haifa_insn_data for BBS. */
9074 haifa_init_h_i_d (const bb_vec_t
&bbs
)
9080 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9084 FOR_BB_INSNS (bb
, insn
)
9089 /* Finalize haifa_insn_data. */
9091 haifa_finish_h_i_d (void)
9094 haifa_insn_data_t data
;
9095 reg_use_data
*use
, *next_use
;
9096 reg_set_data
*set
, *next_set
;
9098 FOR_EACH_VEC_ELT (h_i_d
, i
, data
)
9100 free (data
->max_reg_pressure
);
9101 free (data
->reg_pressure
);
9102 for (use
= data
->reg_use_list
; use
!= NULL
; use
= next_use
)
9104 next_use
= use
->next_insn_use
;
9107 for (set
= data
->reg_set_list
; set
!= NULL
; set
= next_set
)
9109 next_set
= set
->next_insn_set
;
9117 /* Init data for the new insn INSN. */
9119 haifa_init_insn (rtx_insn
*insn
)
9121 gcc_assert (insn
!= NULL
);
9123 sched_extend_luids ();
9124 sched_init_insn_luid (insn
);
9125 sched_extend_target ();
9126 sched_deps_init (false);
9130 if (adding_bb_to_current_region_p
)
9132 sd_init_insn (insn
);
9134 /* Extend dependency caches by one element. */
9135 extend_dependency_caches (1, false);
9137 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
9138 init_insn_reg_pressure_info (insn
);
9141 /* Init data for the new basic block BB which comes after AFTER. */
9143 haifa_init_only_bb (basic_block bb
, basic_block after
)
9145 gcc_assert (bb
!= NULL
);
9149 if (common_sched_info
->add_block
)
9150 /* This changes only data structures of the front-end. */
9151 common_sched_info
->add_block (bb
, after
);
9154 /* A generic version of sched_split_block (). */
9156 sched_split_block_1 (basic_block first_bb
, rtx after
)
9160 e
= split_block (first_bb
, after
);
9161 gcc_assert (e
->src
== first_bb
);
9163 /* sched_split_block emits note if *check == BB_END. Probably it
9164 is better to rip that note off. */
9169 /* A generic version of sched_create_empty_bb (). */
9171 sched_create_empty_bb_1 (basic_block after
)
9173 return create_empty_bb (after
);
9176 /* Insert PAT as an INSN into the schedule and update the necessary data
9177 structures to account for it. */
9179 sched_emit_insn (rtx pat
)
9181 rtx_insn
*insn
= emit_insn_before (pat
, first_nonscheduled_insn ());
9182 haifa_init_insn (insn
);
9184 if (current_sched_info
->add_remove_insn
)
9185 current_sched_info
->add_remove_insn (insn
, 0);
9187 (*current_sched_info
->begin_schedule_ready
) (insn
);
9188 scheduled_insns
.safe_push (insn
);
9190 last_scheduled_insn
= insn
;
9194 /* This function returns a candidate satisfying dispatch constraints from
9198 ready_remove_first_dispatch (struct ready_list
*ready
)
9201 rtx_insn
*insn
= ready_element (ready
, 0);
9203 if (ready
->n_ready
== 1
9205 || INSN_CODE (insn
) < 0
9206 || !active_insn_p (insn
)
9207 || targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9208 return ready_remove_first (ready
);
9210 for (i
= 1; i
< ready
->n_ready
; i
++)
9212 insn
= ready_element (ready
, i
);
9215 || INSN_CODE (insn
) < 0
9216 || !active_insn_p (insn
))
9219 if (targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9221 /* Return ith element of ready. */
9222 insn
= ready_remove (ready
, i
);
9227 if (targetm
.sched
.dispatch (NULL
, DISPATCH_VIOLATION
))
9228 return ready_remove_first (ready
);
9230 for (i
= 1; i
< ready
->n_ready
; i
++)
9232 insn
= ready_element (ready
, i
);
9235 || INSN_CODE (insn
) < 0
9236 || !active_insn_p (insn
))
9239 /* Return i-th element of ready. */
9240 if (targetm
.sched
.dispatch (insn
, IS_CMP
))
9241 return ready_remove (ready
, i
);
9244 return ready_remove_first (ready
);
9247 /* Get number of ready insn in the ready list. */
9250 number_in_ready (void)
9252 return ready
.n_ready
;
9255 /* Get number of ready's in the ready list. */
9258 get_ready_element (int i
)
9260 return ready_element (&ready
, i
);
9263 #endif /* INSN_SCHEDULING */