jit: document union types
[official-gcc.git] / gcc / haifa-sched.c
blobe685cc8a1af86fba5f81ed523ca7dd928fb89bfb
1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2015 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.c,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.c.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
53 remaining slots.
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
59 broken by
60 2. choose insn with least contribution to register pressure,
61 ties broken by
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
65 broken by
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
94 utilization.
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
99 of this case.
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
125 #include "config.h"
126 #include "system.h"
127 #include "coretypes.h"
128 #include "tm.h"
129 #include "diagnostic-core.h"
130 #include "hard-reg-set.h"
131 #include "rtl.h"
132 #include "tm_p.h"
133 #include "regs.h"
134 #include "function.h"
135 #include "flags.h"
136 #include "insn-config.h"
137 #include "insn-attr.h"
138 #include "except.h"
139 #include "recog.h"
140 #include "dominance.h"
141 #include "cfg.h"
142 #include "cfgrtl.h"
143 #include "cfgbuild.h"
144 #include "predict.h"
145 #include "basic-block.h"
146 #include "sched-int.h"
147 #include "target.h"
148 #include "common/common-target.h"
149 #include "params.h"
150 #include "dbgcnt.h"
151 #include "cfgloop.h"
152 #include "ira.h"
153 #include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
154 #include "dumpfile.h"
156 #ifdef INSN_SCHEDULING
158 /* True if we do register pressure relief through live-range
159 shrinkage. */
160 static bool live_range_shrinkage_p;
162 /* Switch on live range shrinkage. */
163 void
164 initialize_live_range_shrinkage (void)
166 live_range_shrinkage_p = true;
169 /* Switch off live range shrinkage. */
170 void
171 finish_live_range_shrinkage (void)
173 live_range_shrinkage_p = false;
176 /* issue_rate is the number of insns that can be scheduled in the same
177 machine cycle. It can be defined in the config/mach/mach.h file,
178 otherwise we set it to 1. */
180 int issue_rate;
182 /* This can be set to true by a backend if the scheduler should not
183 enable a DCE pass. */
184 bool sched_no_dce;
186 /* The current initiation interval used when modulo scheduling. */
187 static int modulo_ii;
189 /* The maximum number of stages we are prepared to handle. */
190 static int modulo_max_stages;
192 /* The number of insns that exist in each iteration of the loop. We use this
193 to detect when we've scheduled all insns from the first iteration. */
194 static int modulo_n_insns;
196 /* The current count of insns in the first iteration of the loop that have
197 already been scheduled. */
198 static int modulo_insns_scheduled;
200 /* The maximum uid of insns from the first iteration of the loop. */
201 static int modulo_iter0_max_uid;
203 /* The number of times we should attempt to backtrack when modulo scheduling.
204 Decreased each time we have to backtrack. */
205 static int modulo_backtracks_left;
207 /* The stage in which the last insn from the original loop was
208 scheduled. */
209 static int modulo_last_stage;
211 /* sched-verbose controls the amount of debugging output the
212 scheduler prints. It is controlled by -fsched-verbose=N:
213 N>0 and no -DSR : the output is directed to stderr.
214 N>=10 will direct the printouts to stderr (regardless of -dSR).
215 N=1: same as -dSR.
216 N=2: bb's probabilities, detailed ready list info, unit/insn info.
217 N=3: rtl at abort point, control-flow, regions info.
218 N=5: dependences info. */
220 int sched_verbose = 0;
222 /* Debugging file. All printouts are sent to dump, which is always set,
223 either to stderr, or to the dump listing file (-dRS). */
224 FILE *sched_dump = 0;
226 /* This is a placeholder for the scheduler parameters common
227 to all schedulers. */
228 struct common_sched_info_def *common_sched_info;
230 #define INSN_TICK(INSN) (HID (INSN)->tick)
231 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
232 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
233 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
234 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
235 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
236 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
237 /* Cached cost of the instruction. Use insn_cost to get cost of the
238 insn. -1 here means that the field is not initialized. */
239 #define INSN_COST(INSN) (HID (INSN)->cost)
241 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
242 then it should be recalculated from scratch. */
243 #define INVALID_TICK (-(max_insn_queue_index + 1))
244 /* The minimal value of the INSN_TICK of an instruction. */
245 #define MIN_TICK (-max_insn_queue_index)
247 /* Original order of insns in the ready list.
248 Used to keep order of normal insns while separating DEBUG_INSNs. */
249 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
251 /* The deciding reason for INSN's place in the ready list. */
252 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
254 /* List of important notes we must keep around. This is a pointer to the
255 last element in the list. */
256 rtx_insn *note_list;
258 static struct spec_info_def spec_info_var;
259 /* Description of the speculative part of the scheduling.
260 If NULL - no speculation. */
261 spec_info_t spec_info = NULL;
263 /* True, if recovery block was added during scheduling of current block.
264 Used to determine, if we need to fix INSN_TICKs. */
265 static bool haifa_recovery_bb_recently_added_p;
267 /* True, if recovery block was added during this scheduling pass.
268 Used to determine if we should have empty memory pools of dependencies
269 after finishing current region. */
270 bool haifa_recovery_bb_ever_added_p;
272 /* Counters of different types of speculative instructions. */
273 static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control;
275 /* Array used in {unlink, restore}_bb_notes. */
276 static rtx_insn **bb_header = 0;
278 /* Basic block after which recovery blocks will be created. */
279 static basic_block before_recovery;
281 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
282 created it. */
283 basic_block after_recovery;
285 /* FALSE if we add bb to another region, so we don't need to initialize it. */
286 bool adding_bb_to_current_region_p = true;
288 /* Queues, etc. */
290 /* An instruction is ready to be scheduled when all insns preceding it
291 have already been scheduled. It is important to ensure that all
292 insns which use its result will not be executed until its result
293 has been computed. An insn is maintained in one of four structures:
295 (P) the "Pending" set of insns which cannot be scheduled until
296 their dependencies have been satisfied.
297 (Q) the "Queued" set of insns that can be scheduled when sufficient
298 time has passed.
299 (R) the "Ready" list of unscheduled, uncommitted insns.
300 (S) the "Scheduled" list of insns.
302 Initially, all insns are either "Pending" or "Ready" depending on
303 whether their dependencies are satisfied.
305 Insns move from the "Ready" list to the "Scheduled" list as they
306 are committed to the schedule. As this occurs, the insns in the
307 "Pending" list have their dependencies satisfied and move to either
308 the "Ready" list or the "Queued" set depending on whether
309 sufficient time has passed to make them ready. As time passes,
310 insns move from the "Queued" set to the "Ready" list.
312 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
313 unscheduled insns, i.e., those that are ready, queued, and pending.
314 The "Queued" set (Q) is implemented by the variable `insn_queue'.
315 The "Ready" list (R) is implemented by the variables `ready' and
316 `n_ready'.
317 The "Scheduled" list (S) is the new insn chain built by this pass.
319 The transition (R->S) is implemented in the scheduling loop in
320 `schedule_block' when the best insn to schedule is chosen.
321 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
322 insns move from the ready list to the scheduled list.
323 The transition (Q->R) is implemented in 'queue_to_insn' as time
324 passes or stalls are introduced. */
326 /* Implement a circular buffer to delay instructions until sufficient
327 time has passed. For the new pipeline description interface,
328 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
329 than maximal time of instruction execution computed by genattr.c on
330 the base maximal time of functional unit reservations and getting a
331 result. This is the longest time an insn may be queued. */
333 static rtx_insn_list **insn_queue;
334 static int q_ptr = 0;
335 static int q_size = 0;
336 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
337 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
339 #define QUEUE_SCHEDULED (-3)
340 #define QUEUE_NOWHERE (-2)
341 #define QUEUE_READY (-1)
342 /* QUEUE_SCHEDULED - INSN is scheduled.
343 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
344 queue or ready list.
345 QUEUE_READY - INSN is in ready list.
346 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
348 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
350 /* The following variable value refers for all current and future
351 reservations of the processor units. */
352 state_t curr_state;
354 /* The following variable value is size of memory representing all
355 current and future reservations of the processor units. */
356 size_t dfa_state_size;
358 /* The following array is used to find the best insn from ready when
359 the automaton pipeline interface is used. */
360 signed char *ready_try = NULL;
362 /* The ready list. */
363 struct ready_list ready = {NULL, 0, 0, 0, 0};
365 /* The pointer to the ready list (to be removed). */
366 static struct ready_list *readyp = &ready;
368 /* Scheduling clock. */
369 static int clock_var;
371 /* Clock at which the previous instruction was issued. */
372 static int last_clock_var;
374 /* Set to true if, when queuing a shadow insn, we discover that it would be
375 scheduled too late. */
376 static bool must_backtrack;
378 /* The following variable value is number of essential insns issued on
379 the current cycle. An insn is essential one if it changes the
380 processors state. */
381 int cycle_issued_insns;
383 /* This records the actual schedule. It is built up during the main phase
384 of schedule_block, and afterwards used to reorder the insns in the RTL. */
385 static vec<rtx_insn *> scheduled_insns;
387 static int may_trap_exp (const_rtx, int);
389 /* Nonzero iff the address is comprised from at most 1 register. */
390 #define CONST_BASED_ADDRESS_P(x) \
391 (REG_P (x) \
392 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
393 || (GET_CODE (x) == LO_SUM)) \
394 && (CONSTANT_P (XEXP (x, 0)) \
395 || CONSTANT_P (XEXP (x, 1)))))
397 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
398 as found by analyzing insn's expression. */
401 static int haifa_luid_for_non_insn (rtx x);
403 /* Haifa version of sched_info hooks common to all headers. */
404 const struct common_sched_info_def haifa_common_sched_info =
406 NULL, /* fix_recovery_cfg */
407 NULL, /* add_block */
408 NULL, /* estimate_number_of_insns */
409 haifa_luid_for_non_insn, /* luid_for_non_insn */
410 SCHED_PASS_UNKNOWN /* sched_pass_id */
413 /* Mapping from instruction UID to its Logical UID. */
414 vec<int> sched_luids = vNULL;
416 /* Next LUID to assign to an instruction. */
417 int sched_max_luid = 1;
419 /* Haifa Instruction Data. */
420 vec<haifa_insn_data_def> h_i_d = vNULL;
422 void (* sched_init_only_bb) (basic_block, basic_block);
424 /* Split block function. Different schedulers might use different functions
425 to handle their internal data consistent. */
426 basic_block (* sched_split_block) (basic_block, rtx);
428 /* Create empty basic block after the specified block. */
429 basic_block (* sched_create_empty_bb) (basic_block);
431 /* Return the number of cycles until INSN is expected to be ready.
432 Return zero if it already is. */
433 static int
434 insn_delay (rtx_insn *insn)
436 return MAX (INSN_TICK (insn) - clock_var, 0);
439 static int
440 may_trap_exp (const_rtx x, int is_store)
442 enum rtx_code code;
444 if (x == 0)
445 return TRAP_FREE;
446 code = GET_CODE (x);
447 if (is_store)
449 if (code == MEM && may_trap_p (x))
450 return TRAP_RISKY;
451 else
452 return TRAP_FREE;
454 if (code == MEM)
456 /* The insn uses memory: a volatile load. */
457 if (MEM_VOLATILE_P (x))
458 return IRISKY;
459 /* An exception-free load. */
460 if (!may_trap_p (x))
461 return IFREE;
462 /* A load with 1 base register, to be further checked. */
463 if (CONST_BASED_ADDRESS_P (XEXP (x, 0)))
464 return PFREE_CANDIDATE;
465 /* No info on the load, to be further checked. */
466 return PRISKY_CANDIDATE;
468 else
470 const char *fmt;
471 int i, insn_class = TRAP_FREE;
473 /* Neither store nor load, check if it may cause a trap. */
474 if (may_trap_p (x))
475 return TRAP_RISKY;
476 /* Recursive step: walk the insn... */
477 fmt = GET_RTX_FORMAT (code);
478 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
480 if (fmt[i] == 'e')
482 int tmp_class = may_trap_exp (XEXP (x, i), is_store);
483 insn_class = WORST_CLASS (insn_class, tmp_class);
485 else if (fmt[i] == 'E')
487 int j;
488 for (j = 0; j < XVECLEN (x, i); j++)
490 int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store);
491 insn_class = WORST_CLASS (insn_class, tmp_class);
492 if (insn_class == TRAP_RISKY || insn_class == IRISKY)
493 break;
496 if (insn_class == TRAP_RISKY || insn_class == IRISKY)
497 break;
499 return insn_class;
503 /* Classifies rtx X of an insn for the purpose of verifying that X can be
504 executed speculatively (and consequently the insn can be moved
505 speculatively), by examining X, returning:
506 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
507 TRAP_FREE: non-load insn.
508 IFREE: load from a globally safe location.
509 IRISKY: volatile load.
510 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
511 being either PFREE or PRISKY. */
513 static int
514 haifa_classify_rtx (const_rtx x)
516 int tmp_class = TRAP_FREE;
517 int insn_class = TRAP_FREE;
518 enum rtx_code code;
520 if (GET_CODE (x) == PARALLEL)
522 int i, len = XVECLEN (x, 0);
524 for (i = len - 1; i >= 0; i--)
526 tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i));
527 insn_class = WORST_CLASS (insn_class, tmp_class);
528 if (insn_class == TRAP_RISKY || insn_class == IRISKY)
529 break;
532 else
534 code = GET_CODE (x);
535 switch (code)
537 case CLOBBER:
538 /* Test if it is a 'store'. */
539 tmp_class = may_trap_exp (XEXP (x, 0), 1);
540 break;
541 case SET:
542 /* Test if it is a store. */
543 tmp_class = may_trap_exp (SET_DEST (x), 1);
544 if (tmp_class == TRAP_RISKY)
545 break;
546 /* Test if it is a load. */
547 tmp_class =
548 WORST_CLASS (tmp_class,
549 may_trap_exp (SET_SRC (x), 0));
550 break;
551 case COND_EXEC:
552 tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x));
553 if (tmp_class == TRAP_RISKY)
554 break;
555 tmp_class = WORST_CLASS (tmp_class,
556 may_trap_exp (COND_EXEC_TEST (x), 0));
557 break;
558 case TRAP_IF:
559 tmp_class = TRAP_RISKY;
560 break;
561 default:;
563 insn_class = tmp_class;
566 return insn_class;
570 haifa_classify_insn (const_rtx insn)
572 return haifa_classify_rtx (PATTERN (insn));
575 /* After the scheduler initialization function has been called, this function
576 can be called to enable modulo scheduling. II is the initiation interval
577 we should use, it affects the delays for delay_pairs that were recorded as
578 separated by a given number of stages.
580 MAX_STAGES provides us with a limit
581 after which we give up scheduling; the caller must have unrolled at least
582 as many copies of the loop body and recorded delay_pairs for them.
584 INSNS is the number of real (non-debug) insns in one iteration of
585 the loop. MAX_UID can be used to test whether an insn belongs to
586 the first iteration of the loop; all of them have a uid lower than
587 MAX_UID. */
588 void
589 set_modulo_params (int ii, int max_stages, int insns, int max_uid)
591 modulo_ii = ii;
592 modulo_max_stages = max_stages;
593 modulo_n_insns = insns;
594 modulo_iter0_max_uid = max_uid;
595 modulo_backtracks_left = PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS);
598 /* A structure to record a pair of insns where the first one is a real
599 insn that has delay slots, and the second is its delayed shadow.
600 I1 is scheduled normally and will emit an assembly instruction,
601 while I2 describes the side effect that takes place at the
602 transition between cycles CYCLES and (CYCLES + 1) after I1. */
603 struct delay_pair
605 struct delay_pair *next_same_i1;
606 rtx_insn *i1, *i2;
607 int cycles;
608 /* When doing modulo scheduling, we a delay_pair can also be used to
609 show that I1 and I2 are the same insn in a different stage. If that
610 is the case, STAGES will be nonzero. */
611 int stages;
614 /* Helpers for delay hashing. */
616 struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
618 typedef void *compare_type;
619 static inline hashval_t hash (const delay_pair *);
620 static inline bool equal (const delay_pair *, const void *);
623 /* Returns a hash value for X, based on hashing just I1. */
625 inline hashval_t
626 delay_i1_hasher::hash (const delay_pair *x)
628 return htab_hash_pointer (x->i1);
631 /* Return true if I1 of pair X is the same as that of pair Y. */
633 inline bool
634 delay_i1_hasher::equal (const delay_pair *x, const void *y)
636 return x->i1 == y;
639 struct delay_i2_hasher : free_ptr_hash <delay_pair>
641 typedef void *compare_type;
642 static inline hashval_t hash (const delay_pair *);
643 static inline bool equal (const delay_pair *, const void *);
646 /* Returns a hash value for X, based on hashing just I2. */
648 inline hashval_t
649 delay_i2_hasher::hash (const delay_pair *x)
651 return htab_hash_pointer (x->i2);
654 /* Return true if I2 of pair X is the same as that of pair Y. */
656 inline bool
657 delay_i2_hasher::equal (const delay_pair *x, const void *y)
659 return x->i2 == y;
662 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
663 indexed by I2. */
664 static hash_table<delay_i1_hasher> *delay_htab;
665 static hash_table<delay_i2_hasher> *delay_htab_i2;
667 /* Called through htab_traverse. Walk the hashtable using I2 as
668 index, and delete all elements involving an UID higher than
669 that pointed to by *DATA. */
671 haifa_htab_i2_traverse (delay_pair **slot, int *data)
673 int maxuid = *data;
674 struct delay_pair *p = *slot;
675 if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid)
677 delay_htab_i2->clear_slot (slot);
679 return 1;
682 /* Called through htab_traverse. Walk the hashtable using I2 as
683 index, and delete all elements involving an UID higher than
684 that pointed to by *DATA. */
686 haifa_htab_i1_traverse (delay_pair **pslot, int *data)
688 int maxuid = *data;
689 struct delay_pair *p, *first, **pprev;
691 if (INSN_UID ((*pslot)->i1) >= maxuid)
693 delay_htab->clear_slot (pslot);
694 return 1;
696 pprev = &first;
697 for (p = *pslot; p; p = p->next_same_i1)
699 if (INSN_UID (p->i2) < maxuid)
701 *pprev = p;
702 pprev = &p->next_same_i1;
705 *pprev = NULL;
706 if (first == NULL)
707 delay_htab->clear_slot (pslot);
708 else
709 *pslot = first;
710 return 1;
713 /* Discard all delay pairs which involve an insn with an UID higher
714 than MAX_UID. */
715 void
716 discard_delay_pairs_above (int max_uid)
718 delay_htab->traverse <int *, haifa_htab_i1_traverse> (&max_uid);
719 delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (&max_uid);
722 /* This function can be called by a port just before it starts the final
723 scheduling pass. It records the fact that an instruction with delay
724 slots has been split into two insns, I1 and I2. The first one will be
725 scheduled normally and initiates the operation. The second one is a
726 shadow which must follow a specific number of cycles after I1; its only
727 purpose is to show the side effect that occurs at that cycle in the RTL.
728 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
729 while I2 retains the original insn type.
731 There are two ways in which the number of cycles can be specified,
732 involving the CYCLES and STAGES arguments to this function. If STAGES
733 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
734 which is multiplied by MODULO_II to give the number of cycles. This is
735 only useful if the caller also calls set_modulo_params to enable modulo
736 scheduling. */
738 void
739 record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages)
741 struct delay_pair *p = XNEW (struct delay_pair);
742 struct delay_pair **slot;
744 p->i1 = i1;
745 p->i2 = i2;
746 p->cycles = cycles;
747 p->stages = stages;
749 if (!delay_htab)
751 delay_htab = new hash_table<delay_i1_hasher> (10);
752 delay_htab_i2 = new hash_table<delay_i2_hasher> (10);
754 slot = delay_htab->find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT);
755 p->next_same_i1 = *slot;
756 *slot = p;
757 slot = delay_htab_i2->find_slot (p, INSERT);
758 *slot = p;
761 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
762 and return the other insn if so. Return NULL otherwise. */
763 rtx_insn *
764 real_insn_for_shadow (rtx_insn *insn)
766 struct delay_pair *pair;
768 if (!delay_htab)
769 return NULL;
771 pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
772 if (!pair || pair->stages > 0)
773 return NULL;
774 return pair->i1;
777 /* For a pair P of insns, return the fixed distance in cycles from the first
778 insn after which the second must be scheduled. */
779 static int
780 pair_delay (struct delay_pair *p)
782 if (p->stages == 0)
783 return p->cycles;
784 else
785 return p->stages * modulo_ii;
788 /* Given an insn INSN, add a dependence on its delayed shadow if it
789 has one. Also try to find situations where shadows depend on each other
790 and add dependencies to the real insns to limit the amount of backtracking
791 needed. */
792 void
793 add_delay_dependencies (rtx_insn *insn)
795 struct delay_pair *pair;
796 sd_iterator_def sd_it;
797 dep_t dep;
799 if (!delay_htab)
800 return;
802 pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
803 if (!pair)
804 return;
805 add_dependence (insn, pair->i1, REG_DEP_ANTI);
806 if (pair->stages)
807 return;
809 FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep)
811 rtx_insn *pro = DEP_PRO (dep);
812 struct delay_pair *other_pair
813 = delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro));
814 if (!other_pair || other_pair->stages)
815 continue;
816 if (pair_delay (other_pair) >= pair_delay (pair))
818 if (sched_verbose >= 4)
820 fprintf (sched_dump, ";;\tadding dependence %d <- %d\n",
821 INSN_UID (other_pair->i1),
822 INSN_UID (pair->i1));
823 fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n",
824 INSN_UID (pair->i1),
825 INSN_UID (pair->i2),
826 pair_delay (pair));
827 fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n",
828 INSN_UID (other_pair->i1),
829 INSN_UID (other_pair->i2),
830 pair_delay (other_pair));
832 add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI);
837 /* Forward declarations. */
839 static int priority (rtx_insn *);
840 static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
841 static int rank_for_schedule (const void *, const void *);
842 static void swap_sort (rtx_insn **, int);
843 static void queue_insn (rtx_insn *, int, const char *);
844 static int schedule_insn (rtx_insn *);
845 static void adjust_priority (rtx_insn *);
846 static void advance_one_cycle (void);
847 static void extend_h_i_d (void);
850 /* Notes handling mechanism:
851 =========================
852 Generally, NOTES are saved before scheduling and restored after scheduling.
853 The scheduler distinguishes between two types of notes:
855 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
856 Before scheduling a region, a pointer to the note is added to the insn
857 that follows or precedes it. (This happens as part of the data dependence
858 computation). After scheduling an insn, the pointer contained in it is
859 used for regenerating the corresponding note (in reemit_notes).
861 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
862 these notes are put in a list (in rm_other_notes() and
863 unlink_other_notes ()). After scheduling the block, these notes are
864 inserted at the beginning of the block (in schedule_block()). */
866 static void ready_add (struct ready_list *, rtx_insn *, bool);
867 static rtx_insn *ready_remove_first (struct ready_list *);
868 static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready);
870 static void queue_to_ready (struct ready_list *);
871 static int early_queue_to_ready (state_t, struct ready_list *);
873 /* The following functions are used to implement multi-pass scheduling
874 on the first cycle. */
875 static rtx_insn *ready_remove (struct ready_list *, int);
876 static void ready_remove_insn (rtx_insn *);
878 static void fix_inter_tick (rtx_insn *, rtx_insn *);
879 static int fix_tick_ready (rtx_insn *);
880 static void change_queue_index (rtx_insn *, int);
882 /* The following functions are used to implement scheduling of data/control
883 speculative instructions. */
885 static void extend_h_i_d (void);
886 static void init_h_i_d (rtx_insn *);
887 static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *);
888 static void generate_recovery_code (rtx_insn *);
889 static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t);
890 static void begin_speculative_block (rtx_insn *);
891 static void add_to_speculative_block (rtx_insn *);
892 static void init_before_recovery (basic_block *);
893 static void create_check_block_twin (rtx_insn *, bool);
894 static void fix_recovery_deps (basic_block);
895 static bool haifa_change_pattern (rtx_insn *, rtx);
896 static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *);
897 static void restore_bb_notes (basic_block);
898 static void fix_jump_move (rtx_insn *);
899 static void move_block_after_check (rtx_insn *);
900 static void move_succs (vec<edge, va_gc> **, basic_block);
901 static void sched_remove_insn (rtx_insn *);
902 static void clear_priorities (rtx_insn *, rtx_vec_t *);
903 static void calc_priorities (rtx_vec_t);
904 static void add_jump_dependencies (rtx_insn *, rtx_insn *);
906 #endif /* INSN_SCHEDULING */
908 /* Point to state used for the current scheduling pass. */
909 struct haifa_sched_info *current_sched_info;
911 #ifndef INSN_SCHEDULING
912 void
913 schedule_insns (void)
916 #else
918 /* Do register pressure sensitive insn scheduling if the flag is set
919 up. */
920 enum sched_pressure_algorithm sched_pressure;
922 /* Map regno -> its pressure class. The map defined only when
923 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
924 enum reg_class *sched_regno_pressure_class;
926 /* The current register pressure. Only elements corresponding pressure
927 classes are defined. */
928 static int curr_reg_pressure[N_REG_CLASSES];
930 /* Saved value of the previous array. */
931 static int saved_reg_pressure[N_REG_CLASSES];
933 /* Register living at given scheduling point. */
934 static bitmap curr_reg_live;
936 /* Saved value of the previous array. */
937 static bitmap saved_reg_live;
939 /* Registers mentioned in the current region. */
940 static bitmap region_ref_regs;
942 /* Effective number of available registers of a given class (see comment
943 in sched_pressure_start_bb). */
944 static int sched_class_regs_num[N_REG_CLASSES];
945 /* Number of call_used_regs. This is a helper for calculating of
946 sched_class_regs_num. */
947 static int call_used_regs_num[N_REG_CLASSES];
949 /* Initiate register pressure relative info for scheduling the current
950 region. Currently it is only clearing register mentioned in the
951 current region. */
952 void
953 sched_init_region_reg_pressure_info (void)
955 bitmap_clear (region_ref_regs);
958 /* PRESSURE[CL] describes the pressure on register class CL. Update it
959 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
960 LIVE tracks the set of live registers; if it is null, assume that
961 every birth or death is genuine. */
962 static inline void
963 mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p)
965 enum reg_class pressure_class;
967 pressure_class = sched_regno_pressure_class[regno];
968 if (regno >= FIRST_PSEUDO_REGISTER)
970 if (pressure_class != NO_REGS)
972 if (birth_p)
974 if (!live || bitmap_set_bit (live, regno))
975 pressure[pressure_class]
976 += (ira_reg_class_max_nregs
977 [pressure_class][PSEUDO_REGNO_MODE (regno)]);
979 else
981 if (!live || bitmap_clear_bit (live, regno))
982 pressure[pressure_class]
983 -= (ira_reg_class_max_nregs
984 [pressure_class][PSEUDO_REGNO_MODE (regno)]);
988 else if (pressure_class != NO_REGS
989 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno))
991 if (birth_p)
993 if (!live || bitmap_set_bit (live, regno))
994 pressure[pressure_class]++;
996 else
998 if (!live || bitmap_clear_bit (live, regno))
999 pressure[pressure_class]--;
1004 /* Initiate current register pressure related info from living
1005 registers given by LIVE. */
1006 static void
1007 initiate_reg_pressure_info (bitmap live)
1009 int i;
1010 unsigned int j;
1011 bitmap_iterator bi;
1013 for (i = 0; i < ira_pressure_classes_num; i++)
1014 curr_reg_pressure[ira_pressure_classes[i]] = 0;
1015 bitmap_clear (curr_reg_live);
1016 EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi)
1017 if (sched_pressure == SCHED_PRESSURE_MODEL
1018 || current_nr_blocks == 1
1019 || bitmap_bit_p (region_ref_regs, j))
1020 mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true);
1023 /* Mark registers in X as mentioned in the current region. */
1024 static void
1025 setup_ref_regs (rtx x)
1027 int i, j;
1028 const RTX_CODE code = GET_CODE (x);
1029 const char *fmt;
1031 if (REG_P (x))
1033 bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
1034 return;
1036 fmt = GET_RTX_FORMAT (code);
1037 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1038 if (fmt[i] == 'e')
1039 setup_ref_regs (XEXP (x, i));
1040 else if (fmt[i] == 'E')
1042 for (j = 0; j < XVECLEN (x, i); j++)
1043 setup_ref_regs (XVECEXP (x, i, j));
1047 /* Initiate current register pressure related info at the start of
1048 basic block BB. */
1049 static void
1050 initiate_bb_reg_pressure_info (basic_block bb)
1052 unsigned int i ATTRIBUTE_UNUSED;
1053 rtx_insn *insn;
1055 if (current_nr_blocks > 1)
1056 FOR_BB_INSNS (bb, insn)
1057 if (NONDEBUG_INSN_P (insn))
1058 setup_ref_regs (PATTERN (insn));
1059 initiate_reg_pressure_info (df_get_live_in (bb));
1060 if (bb_has_eh_pred (bb))
1061 for (i = 0; ; ++i)
1063 unsigned int regno = EH_RETURN_DATA_REGNO (i);
1065 if (regno == INVALID_REGNUM)
1066 break;
1067 if (! bitmap_bit_p (df_get_live_in (bb), regno))
1068 mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
1069 regno, true);
1073 /* Save current register pressure related info. */
1074 static void
1075 save_reg_pressure (void)
1077 int i;
1079 for (i = 0; i < ira_pressure_classes_num; i++)
1080 saved_reg_pressure[ira_pressure_classes[i]]
1081 = curr_reg_pressure[ira_pressure_classes[i]];
1082 bitmap_copy (saved_reg_live, curr_reg_live);
1085 /* Restore saved register pressure related info. */
1086 static void
1087 restore_reg_pressure (void)
1089 int i;
1091 for (i = 0; i < ira_pressure_classes_num; i++)
1092 curr_reg_pressure[ira_pressure_classes[i]]
1093 = saved_reg_pressure[ira_pressure_classes[i]];
1094 bitmap_copy (curr_reg_live, saved_reg_live);
1097 /* Return TRUE if the register is dying after its USE. */
1098 static bool
1099 dying_use_p (struct reg_use_data *use)
1101 struct reg_use_data *next;
1103 for (next = use->next_regno_use; next != use; next = next->next_regno_use)
1104 if (NONDEBUG_INSN_P (next->insn)
1105 && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
1106 return false;
1107 return true;
1110 /* Print info about the current register pressure and its excess for
1111 each pressure class. */
1112 static void
1113 print_curr_reg_pressure (void)
1115 int i;
1116 enum reg_class cl;
1118 fprintf (sched_dump, ";;\t");
1119 for (i = 0; i < ira_pressure_classes_num; i++)
1121 cl = ira_pressure_classes[i];
1122 gcc_assert (curr_reg_pressure[cl] >= 0);
1123 fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl],
1124 curr_reg_pressure[cl],
1125 curr_reg_pressure[cl] - sched_class_regs_num[cl]);
1127 fprintf (sched_dump, "\n");
1130 /* Determine if INSN has a condition that is clobbered if a register
1131 in SET_REGS is modified. */
1132 static bool
1133 cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs)
1135 rtx pat = PATTERN (insn);
1136 gcc_assert (GET_CODE (pat) == COND_EXEC);
1137 if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0))))
1139 sd_iterator_def sd_it;
1140 dep_t dep;
1141 haifa_change_pattern (insn, ORIG_PAT (insn));
1142 FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
1143 DEP_STATUS (dep) &= ~DEP_CANCELLED;
1144 TODO_SPEC (insn) = HARD_DEP;
1145 if (sched_verbose >= 2)
1146 fprintf (sched_dump,
1147 ";;\t\tdequeue insn %s because of clobbered condition\n",
1148 (*current_sched_info->print_insn) (insn, 0));
1149 return true;
1152 return false;
1155 /* This function should be called after modifying the pattern of INSN,
1156 to update scheduler data structures as needed. */
1157 static void
1158 update_insn_after_change (rtx_insn *insn)
1160 sd_iterator_def sd_it;
1161 dep_t dep;
1163 dfa_clear_single_insn_cache (insn);
1165 sd_it = sd_iterator_start (insn,
1166 SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK);
1167 while (sd_iterator_cond (&sd_it, &dep))
1169 DEP_COST (dep) = UNKNOWN_DEP_COST;
1170 sd_iterator_next (&sd_it);
1173 /* Invalidate INSN_COST, so it'll be recalculated. */
1174 INSN_COST (insn) = -1;
1175 /* Invalidate INSN_TICK, so it'll be recalculated. */
1176 INSN_TICK (insn) = INVALID_TICK;
1178 /* Invalidate autoprefetch data entry. */
1179 INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
1180 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1181 INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
1182 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1186 /* Two VECs, one to hold dependencies for which pattern replacements
1187 need to be applied or restored at the start of the next cycle, and
1188 another to hold an integer that is either one, to apply the
1189 corresponding replacement, or zero to restore it. */
1190 static vec<dep_t> next_cycle_replace_deps;
1191 static vec<int> next_cycle_apply;
1193 static void apply_replacement (dep_t, bool);
1194 static void restore_pattern (dep_t, bool);
1196 /* Look at the remaining dependencies for insn NEXT, and compute and return
1197 the TODO_SPEC value we should use for it. This is called after one of
1198 NEXT's dependencies has been resolved.
1199 We also perform pattern replacements for predication, and for broken
1200 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1201 false. */
1203 static ds_t
1204 recompute_todo_spec (rtx_insn *next, bool for_backtrack)
1206 ds_t new_ds;
1207 sd_iterator_def sd_it;
1208 dep_t dep, modify_dep = NULL;
1209 int n_spec = 0;
1210 int n_control = 0;
1211 int n_replace = 0;
1212 bool first_p = true;
1214 if (sd_lists_empty_p (next, SD_LIST_BACK))
1215 /* NEXT has all its dependencies resolved. */
1216 return 0;
1218 if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
1219 return HARD_DEP;
1221 /* If NEXT is intended to sit adjacent to this instruction, we don't
1222 want to try to break any dependencies. Treat it as a HARD_DEP. */
1223 if (SCHED_GROUP_P (next))
1224 return HARD_DEP;
1226 /* Now we've got NEXT with speculative deps only.
1227 1. Look at the deps to see what we have to do.
1228 2. Check if we can do 'todo'. */
1229 new_ds = 0;
1231 FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1233 rtx_insn *pro = DEP_PRO (dep);
1234 ds_t ds = DEP_STATUS (dep) & SPECULATIVE;
1236 if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next))
1237 continue;
1239 if (ds)
1241 n_spec++;
1242 if (first_p)
1244 first_p = false;
1246 new_ds = ds;
1248 else
1249 new_ds = ds_merge (new_ds, ds);
1251 else if (DEP_TYPE (dep) == REG_DEP_CONTROL)
1253 if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1255 n_control++;
1256 modify_dep = dep;
1258 DEP_STATUS (dep) &= ~DEP_CANCELLED;
1260 else if (DEP_REPLACE (dep) != NULL)
1262 if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1264 n_replace++;
1265 modify_dep = dep;
1267 DEP_STATUS (dep) &= ~DEP_CANCELLED;
1271 if (n_replace > 0 && n_control == 0 && n_spec == 0)
1273 if (!dbg_cnt (sched_breakdep))
1274 return HARD_DEP;
1275 FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1277 struct dep_replacement *desc = DEP_REPLACE (dep);
1278 if (desc != NULL)
1280 if (desc->insn == next && !for_backtrack)
1282 gcc_assert (n_replace == 1);
1283 apply_replacement (dep, true);
1285 DEP_STATUS (dep) |= DEP_CANCELLED;
1288 return 0;
1291 else if (n_control == 1 && n_replace == 0 && n_spec == 0)
1293 rtx_insn *pro, *other;
1294 rtx new_pat;
1295 rtx cond = NULL_RTX;
1296 bool success;
1297 rtx_insn *prev = NULL;
1298 int i;
1299 unsigned regno;
1301 if ((current_sched_info->flags & DO_PREDICATION) == 0
1302 || (ORIG_PAT (next) != NULL_RTX
1303 && PREDICATED_PAT (next) == NULL_RTX))
1304 return HARD_DEP;
1306 pro = DEP_PRO (modify_dep);
1307 other = real_insn_for_shadow (pro);
1308 if (other != NULL_RTX)
1309 pro = other;
1311 cond = sched_get_reverse_condition_uncached (pro);
1312 regno = REGNO (XEXP (cond, 0));
1314 /* Find the last scheduled insn that modifies the condition register.
1315 We can stop looking once we find the insn we depend on through the
1316 REG_DEP_CONTROL; if the condition register isn't modified after it,
1317 we know that it still has the right value. */
1318 if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
1319 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev)
1321 HARD_REG_SET t;
1323 find_all_hard_reg_sets (prev, &t, true);
1324 if (TEST_HARD_REG_BIT (t, regno))
1325 return HARD_DEP;
1326 if (prev == pro)
1327 break;
1329 if (ORIG_PAT (next) == NULL_RTX)
1331 ORIG_PAT (next) = PATTERN (next);
1333 new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next));
1334 success = haifa_change_pattern (next, new_pat);
1335 if (!success)
1336 return HARD_DEP;
1337 PREDICATED_PAT (next) = new_pat;
1339 else if (PATTERN (next) != PREDICATED_PAT (next))
1341 bool success = haifa_change_pattern (next,
1342 PREDICATED_PAT (next));
1343 gcc_assert (success);
1345 DEP_STATUS (modify_dep) |= DEP_CANCELLED;
1346 return DEP_CONTROL;
1349 if (PREDICATED_PAT (next) != NULL_RTX)
1351 int tick = INSN_TICK (next);
1352 bool success = haifa_change_pattern (next,
1353 ORIG_PAT (next));
1354 INSN_TICK (next) = tick;
1355 gcc_assert (success);
1358 /* We can't handle the case where there are both speculative and control
1359 dependencies, so we return HARD_DEP in such a case. Also fail if
1360 we have speculative dependencies with not enough points, or more than
1361 one control dependency. */
1362 if ((n_spec > 0 && (n_control > 0 || n_replace > 0))
1363 || (n_spec > 0
1364 /* Too few points? */
1365 && ds_weak (new_ds) < spec_info->data_weakness_cutoff)
1366 || n_control > 0
1367 || n_replace > 0)
1368 return HARD_DEP;
1370 return new_ds;
1373 /* Pointer to the last instruction scheduled. */
1374 static rtx_insn *last_scheduled_insn;
1376 /* Pointer to the last nondebug instruction scheduled within the
1377 block, or the prev_head of the scheduling block. Used by
1378 rank_for_schedule, so that insns independent of the last scheduled
1379 insn will be preferred over dependent instructions. */
1380 static rtx_insn *last_nondebug_scheduled_insn;
1382 /* Pointer that iterates through the list of unscheduled insns if we
1383 have a dbg_cnt enabled. It always points at an insn prior to the
1384 first unscheduled one. */
1385 static rtx_insn *nonscheduled_insns_begin;
1387 /* Compute cost of executing INSN.
1388 This is the number of cycles between instruction issue and
1389 instruction results. */
1391 insn_cost (rtx_insn *insn)
1393 int cost;
1395 if (sched_fusion)
1396 return 0;
1398 if (sel_sched_p ())
1400 if (recog_memoized (insn) < 0)
1401 return 0;
1403 cost = insn_default_latency (insn);
1404 if (cost < 0)
1405 cost = 0;
1407 return cost;
1410 cost = INSN_COST (insn);
1412 if (cost < 0)
1414 /* A USE insn, or something else we don't need to
1415 understand. We can't pass these directly to
1416 result_ready_cost or insn_default_latency because it will
1417 trigger a fatal error for unrecognizable insns. */
1418 if (recog_memoized (insn) < 0)
1420 INSN_COST (insn) = 0;
1421 return 0;
1423 else
1425 cost = insn_default_latency (insn);
1426 if (cost < 0)
1427 cost = 0;
1429 INSN_COST (insn) = cost;
1433 return cost;
1436 /* Compute cost of dependence LINK.
1437 This is the number of cycles between instruction issue and
1438 instruction results.
1439 ??? We also use this function to call recog_memoized on all insns. */
1441 dep_cost_1 (dep_t link, dw_t dw)
1443 rtx_insn *insn = DEP_PRO (link);
1444 rtx_insn *used = DEP_CON (link);
1445 int cost;
1447 if (DEP_COST (link) != UNKNOWN_DEP_COST)
1448 return DEP_COST (link);
1450 if (delay_htab)
1452 struct delay_pair *delay_entry;
1453 delay_entry
1454 = delay_htab_i2->find_with_hash (used, htab_hash_pointer (used));
1455 if (delay_entry)
1457 if (delay_entry->i1 == insn)
1459 DEP_COST (link) = pair_delay (delay_entry);
1460 return DEP_COST (link);
1465 /* A USE insn should never require the value used to be computed.
1466 This allows the computation of a function's result and parameter
1467 values to overlap the return and call. We don't care about the
1468 dependence cost when only decreasing register pressure. */
1469 if (recog_memoized (used) < 0)
1471 cost = 0;
1472 recog_memoized (insn);
1474 else
1476 enum reg_note dep_type = DEP_TYPE (link);
1478 cost = insn_cost (insn);
1480 if (INSN_CODE (insn) >= 0)
1482 if (dep_type == REG_DEP_ANTI)
1483 cost = 0;
1484 else if (dep_type == REG_DEP_OUTPUT)
1486 cost = (insn_default_latency (insn)
1487 - insn_default_latency (used));
1488 if (cost <= 0)
1489 cost = 1;
1491 else if (bypass_p (insn))
1492 cost = insn_latency (insn, used);
1496 if (targetm.sched.adjust_cost_2)
1497 cost = targetm.sched.adjust_cost_2 (used, (int) dep_type, insn, cost,
1498 dw);
1499 else if (targetm.sched.adjust_cost != NULL)
1501 /* This variable is used for backward compatibility with the
1502 targets. */
1503 rtx_insn_list *dep_cost_rtx_link =
1504 alloc_INSN_LIST (NULL_RTX, NULL);
1506 /* Make it self-cycled, so that if some tries to walk over this
1507 incomplete list he/she will be caught in an endless loop. */
1508 XEXP (dep_cost_rtx_link, 1) = dep_cost_rtx_link;
1510 /* Targets use only REG_NOTE_KIND of the link. */
1511 PUT_REG_NOTE_KIND (dep_cost_rtx_link, DEP_TYPE (link));
1513 cost = targetm.sched.adjust_cost (used, dep_cost_rtx_link,
1514 insn, cost);
1516 free_INSN_LIST_node (dep_cost_rtx_link);
1519 if (cost < 0)
1520 cost = 0;
1523 DEP_COST (link) = cost;
1524 return cost;
1527 /* Compute cost of dependence LINK.
1528 This is the number of cycles between instruction issue and
1529 instruction results. */
1531 dep_cost (dep_t link)
1533 return dep_cost_1 (link, 0);
1536 /* Use this sel-sched.c friendly function in reorder2 instead of increasing
1537 INSN_PRIORITY explicitly. */
1538 void
1539 increase_insn_priority (rtx_insn *insn, int amount)
1541 if (!sel_sched_p ())
1543 /* We're dealing with haifa-sched.c INSN_PRIORITY. */
1544 if (INSN_PRIORITY_KNOWN (insn))
1545 INSN_PRIORITY (insn) += amount;
1547 else
1549 /* In sel-sched.c INSN_PRIORITY is not kept up to date.
1550 Use EXPR_PRIORITY instead. */
1551 sel_add_to_insn_priority (insn, amount);
1555 /* Return 'true' if DEP should be included in priority calculations. */
1556 static bool
1557 contributes_to_priority_p (dep_t dep)
1559 if (DEBUG_INSN_P (DEP_CON (dep))
1560 || DEBUG_INSN_P (DEP_PRO (dep)))
1561 return false;
1563 /* Critical path is meaningful in block boundaries only. */
1564 if (!current_sched_info->contributes_to_priority (DEP_CON (dep),
1565 DEP_PRO (dep)))
1566 return false;
1568 if (DEP_REPLACE (dep) != NULL)
1569 return false;
1571 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1572 then speculative instructions will less likely be
1573 scheduled. That is because the priority of
1574 their producers will increase, and, thus, the
1575 producers will more likely be scheduled, thus,
1576 resolving the dependence. */
1577 if (sched_deps_info->generate_spec_deps
1578 && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH)
1579 && (DEP_STATUS (dep) & SPECULATIVE))
1580 return false;
1582 return true;
1585 /* Compute the number of nondebug deps in list LIST for INSN. */
1587 static int
1588 dep_list_size (rtx_insn *insn, sd_list_types_def list)
1590 sd_iterator_def sd_it;
1591 dep_t dep;
1592 int dbgcount = 0, nodbgcount = 0;
1594 if (!MAY_HAVE_DEBUG_INSNS)
1595 return sd_lists_size (insn, list);
1597 FOR_EACH_DEP (insn, list, sd_it, dep)
1599 if (DEBUG_INSN_P (DEP_CON (dep)))
1600 dbgcount++;
1601 else if (!DEBUG_INSN_P (DEP_PRO (dep)))
1602 nodbgcount++;
1605 gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list));
1607 return nodbgcount;
1610 bool sched_fusion;
1612 /* Compute the priority number for INSN. */
1613 static int
1614 priority (rtx_insn *insn)
1616 if (! INSN_P (insn))
1617 return 0;
1619 /* We should not be interested in priority of an already scheduled insn. */
1620 gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
1622 if (!INSN_PRIORITY_KNOWN (insn))
1624 int this_priority = -1;
1626 if (sched_fusion)
1628 int this_fusion_priority;
1630 targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
1631 &this_fusion_priority, &this_priority);
1632 INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
1634 else if (dep_list_size (insn, SD_LIST_FORW) == 0)
1635 /* ??? We should set INSN_PRIORITY to insn_cost when and insn has
1636 some forward deps but all of them are ignored by
1637 contributes_to_priority hook. At the moment we set priority of
1638 such insn to 0. */
1639 this_priority = insn_cost (insn);
1640 else
1642 rtx_insn *prev_first, *twin;
1643 basic_block rec;
1645 /* For recovery check instructions we calculate priority slightly
1646 different than that of normal instructions. Instead of walking
1647 through INSN_FORW_DEPS (check) list, we walk through
1648 INSN_FORW_DEPS list of each instruction in the corresponding
1649 recovery block. */
1651 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1652 rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn);
1653 if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun))
1655 prev_first = PREV_INSN (insn);
1656 twin = insn;
1658 else
1660 prev_first = NEXT_INSN (BB_HEAD (rec));
1661 twin = PREV_INSN (BB_END (rec));
1666 sd_iterator_def sd_it;
1667 dep_t dep;
1669 FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep)
1671 rtx_insn *next;
1672 int next_priority;
1674 next = DEP_CON (dep);
1676 if (BLOCK_FOR_INSN (next) != rec)
1678 int cost;
1680 if (!contributes_to_priority_p (dep))
1681 continue;
1683 if (twin == insn)
1684 cost = dep_cost (dep);
1685 else
1687 struct _dep _dep1, *dep1 = &_dep1;
1689 init_dep (dep1, insn, next, REG_DEP_ANTI);
1691 cost = dep_cost (dep1);
1694 next_priority = cost + priority (next);
1696 if (next_priority > this_priority)
1697 this_priority = next_priority;
1701 twin = PREV_INSN (twin);
1703 while (twin != prev_first);
1706 if (this_priority < 0)
1708 gcc_assert (this_priority == -1);
1710 this_priority = insn_cost (insn);
1713 INSN_PRIORITY (insn) = this_priority;
1714 INSN_PRIORITY_STATUS (insn) = 1;
1717 return INSN_PRIORITY (insn);
1720 /* Macros and functions for keeping the priority queue sorted, and
1721 dealing with queuing and dequeuing of instructions. */
1723 /* For each pressure class CL, set DEATH[CL] to the number of registers
1724 in that class that die in INSN. */
1726 static void
1727 calculate_reg_deaths (rtx_insn *insn, int *death)
1729 int i;
1730 struct reg_use_data *use;
1732 for (i = 0; i < ira_pressure_classes_num; i++)
1733 death[ira_pressure_classes[i]] = 0;
1734 for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
1735 if (dying_use_p (use))
1736 mark_regno_birth_or_death (0, death, use->regno, true);
1739 /* Setup info about the current register pressure impact of scheduling
1740 INSN at the current scheduling point. */
1741 static void
1742 setup_insn_reg_pressure_info (rtx_insn *insn)
1744 int i, change, before, after, hard_regno;
1745 int excess_cost_change;
1746 machine_mode mode;
1747 enum reg_class cl;
1748 struct reg_pressure_data *pressure_info;
1749 int *max_reg_pressure;
1750 static int death[N_REG_CLASSES];
1752 gcc_checking_assert (!DEBUG_INSN_P (insn));
1754 excess_cost_change = 0;
1755 calculate_reg_deaths (insn, death);
1756 pressure_info = INSN_REG_PRESSURE (insn);
1757 max_reg_pressure = INSN_MAX_REG_PRESSURE (insn);
1758 gcc_assert (pressure_info != NULL && max_reg_pressure != NULL);
1759 for (i = 0; i < ira_pressure_classes_num; i++)
1761 cl = ira_pressure_classes[i];
1762 gcc_assert (curr_reg_pressure[cl] >= 0);
1763 change = (int) pressure_info[i].set_increase - death[cl];
1764 before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]);
1765 after = MAX (0, max_reg_pressure[i] + change
1766 - sched_class_regs_num[cl]);
1767 hard_regno = ira_class_hard_regs[cl][0];
1768 gcc_assert (hard_regno >= 0);
1769 mode = reg_raw_mode[hard_regno];
1770 excess_cost_change += ((after - before)
1771 * (ira_memory_move_cost[mode][cl][0]
1772 + ira_memory_move_cost[mode][cl][1]));
1774 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change;
1777 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1778 It tries to make the scheduler take register pressure into account
1779 without introducing too many unnecessary stalls. It hooks into the
1780 main scheduling algorithm at several points:
1782 - Before scheduling starts, model_start_schedule constructs a
1783 "model schedule" for the current block. This model schedule is
1784 chosen solely to keep register pressure down. It does not take the
1785 target's pipeline or the original instruction order into account,
1786 except as a tie-breaker. It also doesn't work to a particular
1787 pressure limit.
1789 This model schedule gives us an idea of what pressure can be
1790 achieved for the block and gives us an example of a schedule that
1791 keeps to that pressure. It also makes the final schedule less
1792 dependent on the original instruction order. This is important
1793 because the original order can either be "wide" (many values live
1794 at once, such as in user-scheduled code) or "narrow" (few values
1795 live at once, such as after loop unrolling, where several
1796 iterations are executed sequentially).
1798 We do not apply this model schedule to the rtx stream. We simply
1799 record it in model_schedule. We also compute the maximum pressure,
1800 MP, that was seen during this schedule.
1802 - Instructions are added to the ready queue even if they require
1803 a stall. The length of the stall is instead computed as:
1805 MAX (INSN_TICK (INSN) - clock_var, 0)
1807 (= insn_delay). This allows rank_for_schedule to choose between
1808 introducing a deliberate stall or increasing pressure.
1810 - Before sorting the ready queue, model_set_excess_costs assigns
1811 a pressure-based cost to each ready instruction in the queue.
1812 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1813 (ECC for short) and is effectively measured in cycles.
1815 - rank_for_schedule ranks instructions based on:
1817 ECC (insn) + insn_delay (insn)
1819 then as:
1821 insn_delay (insn)
1823 So, for example, an instruction X1 with an ECC of 1 that can issue
1824 now will win over an instruction X0 with an ECC of zero that would
1825 introduce a stall of one cycle. However, an instruction X2 with an
1826 ECC of 2 that can issue now will lose to both X0 and X1.
1828 - When an instruction is scheduled, model_recompute updates the model
1829 schedule with the new pressures (some of which might now exceed the
1830 original maximum pressure MP). model_update_limit_points then searches
1831 for the new point of maximum pressure, if not already known. */
1833 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1834 from surrounding debug information. */
1835 #define MODEL_BAR \
1836 ";;\t\t+------------------------------------------------------\n"
1838 /* Information about the pressure on a particular register class at a
1839 particular point of the model schedule. */
1840 struct model_pressure_data {
1841 /* The pressure at this point of the model schedule, or -1 if the
1842 point is associated with an instruction that has already been
1843 scheduled. */
1844 int ref_pressure;
1846 /* The maximum pressure during or after this point of the model schedule. */
1847 int max_pressure;
1850 /* Per-instruction information that is used while building the model
1851 schedule. Here, "schedule" refers to the model schedule rather
1852 than the main schedule. */
1853 struct model_insn_info {
1854 /* The instruction itself. */
1855 rtx_insn *insn;
1857 /* If this instruction is in model_worklist, these fields link to the
1858 previous (higher-priority) and next (lower-priority) instructions
1859 in the list. */
1860 struct model_insn_info *prev;
1861 struct model_insn_info *next;
1863 /* While constructing the schedule, QUEUE_INDEX describes whether an
1864 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1865 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1866 old_queue records the value that QUEUE_INDEX had before scheduling
1867 started, so that we can restore it once the schedule is complete. */
1868 int old_queue;
1870 /* The relative importance of an unscheduled instruction. Higher
1871 values indicate greater importance. */
1872 unsigned int model_priority;
1874 /* The length of the longest path of satisfied true dependencies
1875 that leads to this instruction. */
1876 unsigned int depth;
1878 /* The length of the longest path of dependencies of any kind
1879 that leads from this instruction. */
1880 unsigned int alap;
1882 /* The number of predecessor nodes that must still be scheduled. */
1883 int unscheduled_preds;
1886 /* Information about the pressure limit for a particular register class.
1887 This structure is used when applying a model schedule to the main
1888 schedule. */
1889 struct model_pressure_limit {
1890 /* The maximum register pressure seen in the original model schedule. */
1891 int orig_pressure;
1893 /* The maximum register pressure seen in the current model schedule
1894 (which excludes instructions that have already been scheduled). */
1895 int pressure;
1897 /* The point of the current model schedule at which PRESSURE is first
1898 reached. It is set to -1 if the value needs to be recomputed. */
1899 int point;
1902 /* Describes a particular way of measuring register pressure. */
1903 struct model_pressure_group {
1904 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1905 struct model_pressure_limit limits[N_REG_CLASSES];
1907 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1908 on register class ira_pressure_classes[PCI] at point POINT of the
1909 current model schedule. A POINT of model_num_insns describes the
1910 pressure at the end of the schedule. */
1911 struct model_pressure_data *model;
1914 /* Index POINT gives the instruction at point POINT of the model schedule.
1915 This array doesn't change during main scheduling. */
1916 static vec<rtx_insn *> model_schedule;
1918 /* The list of instructions in the model worklist, sorted in order of
1919 decreasing priority. */
1920 static struct model_insn_info *model_worklist;
1922 /* Index I describes the instruction with INSN_LUID I. */
1923 static struct model_insn_info *model_insns;
1925 /* The number of instructions in the model schedule. */
1926 static int model_num_insns;
1928 /* The index of the first instruction in model_schedule that hasn't yet been
1929 added to the main schedule, or model_num_insns if all of them have. */
1930 static int model_curr_point;
1932 /* Describes the pressure before each instruction in the model schedule. */
1933 static struct model_pressure_group model_before_pressure;
1935 /* The first unused model_priority value (as used in model_insn_info). */
1936 static unsigned int model_next_priority;
1939 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1940 at point POINT of the model schedule. */
1941 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1942 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1944 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1945 after point POINT of the model schedule. */
1946 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1947 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1949 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1950 of the model schedule. */
1951 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1952 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1954 /* Information about INSN that is used when creating the model schedule. */
1955 #define MODEL_INSN_INFO(INSN) \
1956 (&model_insns[INSN_LUID (INSN)])
1958 /* The instruction at point POINT of the model schedule. */
1959 #define MODEL_INSN(POINT) \
1960 (model_schedule[POINT])
1963 /* Return INSN's index in the model schedule, or model_num_insns if it
1964 doesn't belong to that schedule. */
1966 static int
1967 model_index (rtx_insn *insn)
1969 if (INSN_MODEL_INDEX (insn) == 0)
1970 return model_num_insns;
1971 return INSN_MODEL_INDEX (insn) - 1;
1974 /* Make sure that GROUP->limits is up-to-date for the current point
1975 of the model schedule. */
1977 static void
1978 model_update_limit_points_in_group (struct model_pressure_group *group)
1980 int pci, max_pressure, point;
1982 for (pci = 0; pci < ira_pressure_classes_num; pci++)
1984 /* We may have passed the final point at which the pressure in
1985 group->limits[pci].pressure was reached. Update the limit if so. */
1986 max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci);
1987 group->limits[pci].pressure = max_pressure;
1989 /* Find the point at which MAX_PRESSURE is first reached. We need
1990 to search in three cases:
1992 - We've already moved past the previous pressure point.
1993 In this case we search forward from model_curr_point.
1995 - We scheduled the previous point of maximum pressure ahead of
1996 its position in the model schedule, but doing so didn't bring
1997 the pressure point earlier. In this case we search forward
1998 from that previous pressure point.
2000 - Scheduling an instruction early caused the maximum pressure
2001 to decrease. In this case we will have set the pressure
2002 point to -1, and we search forward from model_curr_point. */
2003 point = MAX (group->limits[pci].point, model_curr_point);
2004 while (point < model_num_insns
2005 && MODEL_REF_PRESSURE (group, point, pci) < max_pressure)
2006 point++;
2007 group->limits[pci].point = point;
2009 gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure);
2010 gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure);
2014 /* Make sure that all register-pressure limits are up-to-date for the
2015 current position in the model schedule. */
2017 static void
2018 model_update_limit_points (void)
2020 model_update_limit_points_in_group (&model_before_pressure);
2023 /* Return the model_index of the last unscheduled use in chain USE
2024 outside of USE's instruction. Return -1 if there are no other uses,
2025 or model_num_insns if the register is live at the end of the block. */
2027 static int
2028 model_last_use_except (struct reg_use_data *use)
2030 struct reg_use_data *next;
2031 int last, index;
2033 last = -1;
2034 for (next = use->next_regno_use; next != use; next = next->next_regno_use)
2035 if (NONDEBUG_INSN_P (next->insn)
2036 && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
2038 index = model_index (next->insn);
2039 if (index == model_num_insns)
2040 return model_num_insns;
2041 if (last < index)
2042 last = index;
2044 return last;
2047 /* An instruction with model_index POINT has just been scheduled, and it
2048 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2049 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2050 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2052 static void
2053 model_start_update_pressure (struct model_pressure_group *group,
2054 int point, int pci, int delta)
2056 int next_max_pressure;
2058 if (point == model_num_insns)
2060 /* The instruction wasn't part of the model schedule; it was moved
2061 from a different block. Update the pressure for the end of
2062 the model schedule. */
2063 MODEL_REF_PRESSURE (group, point, pci) += delta;
2064 MODEL_MAX_PRESSURE (group, point, pci) += delta;
2066 else
2068 /* Record that this instruction has been scheduled. Nothing now
2069 changes between POINT and POINT + 1, so get the maximum pressure
2070 from the latter. If the maximum pressure decreases, the new
2071 pressure point may be before POINT. */
2072 MODEL_REF_PRESSURE (group, point, pci) = -1;
2073 next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2074 if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure)
2076 MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure;
2077 if (group->limits[pci].point == point)
2078 group->limits[pci].point = -1;
2083 /* Record that scheduling a later instruction has changed the pressure
2084 at point POINT of the model schedule by DELTA (which might be 0).
2085 Update GROUP accordingly. Return nonzero if these changes might
2086 trigger changes to previous points as well. */
2088 static int
2089 model_update_pressure (struct model_pressure_group *group,
2090 int point, int pci, int delta)
2092 int ref_pressure, max_pressure, next_max_pressure;
2094 /* If POINT hasn't yet been scheduled, update its pressure. */
2095 ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
2096 if (ref_pressure >= 0 && delta != 0)
2098 ref_pressure += delta;
2099 MODEL_REF_PRESSURE (group, point, pci) = ref_pressure;
2101 /* Check whether the maximum pressure in the overall schedule
2102 has increased. (This means that the MODEL_MAX_PRESSURE of
2103 every point <= POINT will need to increase too; see below.) */
2104 if (group->limits[pci].pressure < ref_pressure)
2105 group->limits[pci].pressure = ref_pressure;
2107 /* If we are at maximum pressure, and the maximum pressure
2108 point was previously unknown or later than POINT,
2109 bring it forward. */
2110 if (group->limits[pci].pressure == ref_pressure
2111 && !IN_RANGE (group->limits[pci].point, 0, point))
2112 group->limits[pci].point = point;
2114 /* If POINT used to be the point of maximum pressure, but isn't
2115 any longer, we need to recalculate it using a forward walk. */
2116 if (group->limits[pci].pressure > ref_pressure
2117 && group->limits[pci].point == point)
2118 group->limits[pci].point = -1;
2121 /* Update the maximum pressure at POINT. Changes here might also
2122 affect the maximum pressure at POINT - 1. */
2123 next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2124 max_pressure = MAX (ref_pressure, next_max_pressure);
2125 if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure)
2127 MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
2128 return 1;
2130 return 0;
2133 /* INSN has just been scheduled. Update the model schedule accordingly. */
2135 static void
2136 model_recompute (rtx_insn *insn)
2138 struct {
2139 int last_use;
2140 int regno;
2141 } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS];
2142 struct reg_use_data *use;
2143 struct reg_pressure_data *reg_pressure;
2144 int delta[N_REG_CLASSES];
2145 int pci, point, mix, new_last, cl, ref_pressure, queue;
2146 unsigned int i, num_uses, num_pending_births;
2147 bool print_p;
2149 /* The destinations of INSN were previously live from POINT onwards, but are
2150 now live from model_curr_point onwards. Set up DELTA accordingly. */
2151 point = model_index (insn);
2152 reg_pressure = INSN_REG_PRESSURE (insn);
2153 for (pci = 0; pci < ira_pressure_classes_num; pci++)
2155 cl = ira_pressure_classes[pci];
2156 delta[cl] = reg_pressure[pci].set_increase;
2159 /* Record which registers previously died at POINT, but which now die
2160 before POINT. Adjust DELTA so that it represents the effect of
2161 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2162 registers that will be born in the range [model_curr_point, POINT). */
2163 num_uses = 0;
2164 num_pending_births = 0;
2165 for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
2167 new_last = model_last_use_except (use);
2168 if (new_last < point)
2170 gcc_assert (num_uses < ARRAY_SIZE (uses));
2171 uses[num_uses].last_use = new_last;
2172 uses[num_uses].regno = use->regno;
2173 /* This register is no longer live after POINT - 1. */
2174 mark_regno_birth_or_death (NULL, delta, use->regno, false);
2175 num_uses++;
2176 if (new_last >= 0)
2177 num_pending_births++;
2181 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2182 Also set each group pressure limit for POINT. */
2183 for (pci = 0; pci < ira_pressure_classes_num; pci++)
2185 cl = ira_pressure_classes[pci];
2186 model_start_update_pressure (&model_before_pressure,
2187 point, pci, delta[cl]);
2190 /* Walk the model schedule backwards, starting immediately before POINT. */
2191 print_p = false;
2192 if (point != model_curr_point)
2195 point--;
2196 insn = MODEL_INSN (point);
2197 queue = QUEUE_INDEX (insn);
2199 if (queue != QUEUE_SCHEDULED)
2201 /* DELTA describes the effect of the move on the register pressure
2202 after POINT. Make it describe the effect on the pressure
2203 before POINT. */
2204 i = 0;
2205 while (i < num_uses)
2207 if (uses[i].last_use == point)
2209 /* This register is now live again. */
2210 mark_regno_birth_or_death (NULL, delta,
2211 uses[i].regno, true);
2213 /* Remove this use from the array. */
2214 uses[i] = uses[num_uses - 1];
2215 num_uses--;
2216 num_pending_births--;
2218 else
2219 i++;
2222 if (sched_verbose >= 5)
2224 if (!print_p)
2226 fprintf (sched_dump, MODEL_BAR);
2227 fprintf (sched_dump, ";;\t\t| New pressure for model"
2228 " schedule\n");
2229 fprintf (sched_dump, MODEL_BAR);
2230 print_p = true;
2233 fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ",
2234 point, INSN_UID (insn),
2235 str_pattern_slim (PATTERN (insn)));
2236 for (pci = 0; pci < ira_pressure_classes_num; pci++)
2238 cl = ira_pressure_classes[pci];
2239 ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure,
2240 point, pci);
2241 fprintf (sched_dump, " %s:[%d->%d]",
2242 reg_class_names[ira_pressure_classes[pci]],
2243 ref_pressure, ref_pressure + delta[cl]);
2245 fprintf (sched_dump, "\n");
2249 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2250 might have changed as well. */
2251 mix = num_pending_births;
2252 for (pci = 0; pci < ira_pressure_classes_num; pci++)
2254 cl = ira_pressure_classes[pci];
2255 mix |= delta[cl];
2256 mix |= model_update_pressure (&model_before_pressure,
2257 point, pci, delta[cl]);
2260 while (mix && point > model_curr_point);
2262 if (print_p)
2263 fprintf (sched_dump, MODEL_BAR);
2266 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2267 check whether the insn's pattern needs restoring. */
2268 static bool
2269 must_restore_pattern_p (rtx_insn *next, dep_t dep)
2271 if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
2272 return false;
2274 if (DEP_TYPE (dep) == REG_DEP_CONTROL)
2276 gcc_assert (ORIG_PAT (next) != NULL_RTX);
2277 gcc_assert (next == DEP_CON (dep));
2279 else
2281 struct dep_replacement *desc = DEP_REPLACE (dep);
2282 if (desc->insn != next)
2284 gcc_assert (*desc->loc == desc->orig);
2285 return false;
2288 return true;
2291 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2292 pressure on CL from P to P'. We use this to calculate a "base ECC",
2293 baseECC (CL, X), for each pressure class CL and each instruction X.
2294 Supposing X changes the pressure on CL from P to P', and that the
2295 maximum pressure on CL in the current model schedule is MP', then:
2297 * if X occurs before or at the next point of maximum pressure in
2298 the model schedule and P' > MP', then:
2300 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2302 The idea is that the pressure after scheduling a fixed set of
2303 instructions -- in this case, the set up to and including the
2304 next maximum pressure point -- is going to be the same regardless
2305 of the order; we simply want to keep the intermediate pressure
2306 under control. Thus X has a cost of zero unless scheduling it
2307 now would exceed MP'.
2309 If all increases in the set are by the same amount, no zero-cost
2310 instruction will ever cause the pressure to exceed MP'. However,
2311 if X is instead moved past an instruction X' with pressure in the
2312 range (MP' - (P' - P), MP'), the pressure at X' will increase
2313 beyond MP'. Since baseECC is very much a heuristic anyway,
2314 it doesn't seem worth the overhead of tracking cases like these.
2316 The cost of exceeding MP' is always based on the original maximum
2317 pressure MP. This is so that going 2 registers over the original
2318 limit has the same cost regardless of whether it comes from two
2319 separate +1 deltas or from a single +2 delta.
2321 * if X occurs after the next point of maximum pressure in the model
2322 schedule and P' > P, then:
2324 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2326 That is, if we move X forward across a point of maximum pressure,
2327 and if X increases the pressure by P' - P, then we conservatively
2328 assume that scheduling X next would increase the maximum pressure
2329 by P' - P. Again, the cost of doing this is based on the original
2330 maximum pressure MP, for the same reason as above.
2332 * if P' < P, P > MP, and X occurs at or after the next point of
2333 maximum pressure, then:
2335 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2337 That is, if we have already exceeded the original maximum pressure MP,
2338 and if X might reduce the maximum pressure again -- or at least push
2339 it further back, and thus allow more scheduling freedom -- it is given
2340 a negative cost to reflect the improvement.
2342 * otherwise,
2344 baseECC (CL, X) = 0
2346 In this case, X is not expected to affect the maximum pressure MP',
2347 so it has zero cost.
2349 We then create a combined value baseECC (X) that is the sum of
2350 baseECC (CL, X) for each pressure class CL.
2352 baseECC (X) could itself be used as the ECC value described above.
2353 However, this is often too conservative, in the sense that it
2354 tends to make high-priority instructions that increase pressure
2355 wait too long in cases where introducing a spill would be better.
2356 For this reason the final ECC is a priority-adjusted form of
2357 baseECC (X). Specifically, we calculate:
2359 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2360 baseP = MAX { P (X) | baseECC (X) <= 0 }
2362 Then:
2364 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2366 Thus an instruction's effect on pressure is ignored if it has a high
2367 enough priority relative to the ones that don't increase pressure.
2368 Negative values of baseECC (X) do not increase the priority of X
2369 itself, but they do make it harder for other instructions to
2370 increase the pressure further.
2372 This pressure cost is deliberately timid. The intention has been
2373 to choose a heuristic that rarely interferes with the normal list
2374 scheduler in cases where that scheduler would produce good code.
2375 We simply want to curb some of its worst excesses. */
2377 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2379 Here we use the very simplistic cost model that every register above
2380 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2381 measures instead, such as one based on MEMORY_MOVE_COST. However:
2383 (1) In order for an instruction to be scheduled, the higher cost
2384 would need to be justified in a single saving of that many stalls.
2385 This is overly pessimistic, because the benefit of spilling is
2386 often to avoid a sequence of several short stalls rather than
2387 a single long one.
2389 (2) The cost is still arbitrary. Because we are not allocating
2390 registers during scheduling, we have no way of knowing for
2391 sure how many memory accesses will be required by each spill,
2392 where the spills will be placed within the block, or even
2393 which block(s) will contain the spills.
2395 So a higher cost than 1 is often too conservative in practice,
2396 forcing blocks to contain unnecessary stalls instead of spill code.
2397 The simple cost below seems to be the best compromise. It reduces
2398 the interference with the normal list scheduler, which helps make
2399 it more suitable for a default-on option. */
2401 static int
2402 model_spill_cost (int cl, int from, int to)
2404 from = MAX (from, sched_class_regs_num[cl]);
2405 return MAX (to, from) - from;
2408 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2409 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2410 P' = P + DELTA. */
2412 static int
2413 model_excess_group_cost (struct model_pressure_group *group,
2414 int point, int pci, int delta)
2416 int pressure, cl;
2418 cl = ira_pressure_classes[pci];
2419 if (delta < 0 && point >= group->limits[pci].point)
2421 pressure = MAX (group->limits[pci].orig_pressure,
2422 curr_reg_pressure[cl] + delta);
2423 return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]);
2426 if (delta > 0)
2428 if (point > group->limits[pci].point)
2429 pressure = group->limits[pci].pressure + delta;
2430 else
2431 pressure = curr_reg_pressure[cl] + delta;
2433 if (pressure > group->limits[pci].pressure)
2434 return model_spill_cost (cl, group->limits[pci].orig_pressure,
2435 pressure);
2438 return 0;
2441 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2442 if PRINT_P. */
2444 static int
2445 model_excess_cost (rtx_insn *insn, bool print_p)
2447 int point, pci, cl, cost, this_cost, delta;
2448 struct reg_pressure_data *insn_reg_pressure;
2449 int insn_death[N_REG_CLASSES];
2451 calculate_reg_deaths (insn, insn_death);
2452 point = model_index (insn);
2453 insn_reg_pressure = INSN_REG_PRESSURE (insn);
2454 cost = 0;
2456 if (print_p)
2457 fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point,
2458 INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn));
2460 /* Sum up the individual costs for each register class. */
2461 for (pci = 0; pci < ira_pressure_classes_num; pci++)
2463 cl = ira_pressure_classes[pci];
2464 delta = insn_reg_pressure[pci].set_increase - insn_death[cl];
2465 this_cost = model_excess_group_cost (&model_before_pressure,
2466 point, pci, delta);
2467 cost += this_cost;
2468 if (print_p)
2469 fprintf (sched_dump, " %s:[%d base cost %d]",
2470 reg_class_names[cl], delta, this_cost);
2473 if (print_p)
2474 fprintf (sched_dump, "\n");
2476 return cost;
2479 /* Dump the next points of maximum pressure for GROUP. */
2481 static void
2482 model_dump_pressure_points (struct model_pressure_group *group)
2484 int pci, cl;
2486 fprintf (sched_dump, ";;\t\t| pressure points");
2487 for (pci = 0; pci < ira_pressure_classes_num; pci++)
2489 cl = ira_pressure_classes[pci];
2490 fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl],
2491 curr_reg_pressure[cl], group->limits[pci].pressure);
2492 if (group->limits[pci].point < model_num_insns)
2493 fprintf (sched_dump, "%d:%d]", group->limits[pci].point,
2494 INSN_UID (MODEL_INSN (group->limits[pci].point)));
2495 else
2496 fprintf (sched_dump, "end]");
2498 fprintf (sched_dump, "\n");
2501 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2503 static void
2504 model_set_excess_costs (rtx_insn **insns, int count)
2506 int i, cost, priority_base, priority;
2507 bool print_p;
2509 /* Record the baseECC value for each instruction in the model schedule,
2510 except that negative costs are converted to zero ones now rather than
2511 later. Do not assign a cost to debug instructions, since they must
2512 not change code-generation decisions. Experiments suggest we also
2513 get better results by not assigning a cost to instructions from
2514 a different block.
2516 Set PRIORITY_BASE to baseP in the block comment above. This is the
2517 maximum priority of the "cheap" instructions, which should always
2518 include the next model instruction. */
2519 priority_base = 0;
2520 print_p = false;
2521 for (i = 0; i < count; i++)
2522 if (INSN_MODEL_INDEX (insns[i]))
2524 if (sched_verbose >= 6 && !print_p)
2526 fprintf (sched_dump, MODEL_BAR);
2527 fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n");
2528 model_dump_pressure_points (&model_before_pressure);
2529 fprintf (sched_dump, MODEL_BAR);
2530 print_p = true;
2532 cost = model_excess_cost (insns[i], print_p);
2533 if (cost <= 0)
2535 priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost;
2536 priority_base = MAX (priority_base, priority);
2537 cost = 0;
2539 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost;
2541 if (print_p)
2542 fprintf (sched_dump, MODEL_BAR);
2544 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2545 instruction. */
2546 for (i = 0; i < count; i++)
2548 cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]);
2549 priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]);
2550 if (cost > 0 && priority > priority_base)
2552 cost += priority_base - priority;
2553 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0);
2559 /* Enum of rank_for_schedule heuristic decisions. */
2560 enum rfs_decision {
2561 RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
2562 RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
2563 RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_SPECULATION,
2564 RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
2565 RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_N };
2567 /* Corresponding strings for print outs. */
2568 static const char *rfs_str[RFS_N] = {
2569 "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2570 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2571 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
2572 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2573 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
2575 /* Statistical breakdown of rank_for_schedule decisions. */
2576 typedef struct { unsigned stats[RFS_N]; } rank_for_schedule_stats_t;
2577 static rank_for_schedule_stats_t rank_for_schedule_stats;
2579 /* Return the result of comparing insns TMP and TMP2 and update
2580 Rank_For_Schedule statistics. */
2581 static int
2582 rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2)
2584 ++rank_for_schedule_stats.stats[decision];
2585 if (result < 0)
2586 INSN_LAST_RFS_WIN (tmp) = decision;
2587 else if (result > 0)
2588 INSN_LAST_RFS_WIN (tmp2) = decision;
2589 else
2590 gcc_unreachable ();
2591 return result;
2594 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2595 keeping normal insns in original order. */
2597 static int
2598 rank_for_schedule_debug (const void *x, const void *y)
2600 rtx_insn *tmp = *(rtx_insn * const *) y;
2601 rtx_insn *tmp2 = *(rtx_insn * const *) x;
2603 /* Schedule debug insns as early as possible. */
2604 if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
2605 return -1;
2606 else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2607 return 1;
2608 else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2609 return INSN_LUID (tmp) - INSN_LUID (tmp2);
2610 else
2611 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
2614 /* Returns a positive value if x is preferred; returns a negative value if
2615 y is preferred. Should never return 0, since that will make the sort
2616 unstable. */
2618 static int
2619 rank_for_schedule (const void *x, const void *y)
2621 rtx_insn *tmp = *(rtx_insn * const *) y;
2622 rtx_insn *tmp2 = *(rtx_insn * const *) x;
2623 int tmp_class, tmp2_class;
2624 int val, priority_val, info_val, diff;
2626 if (live_range_shrinkage_p)
2628 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2629 code. */
2630 gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
2631 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0
2632 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0)
2633 && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2634 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0)
2635 return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2);
2636 /* Sort by INSN_LUID (original insn order), so that we make the
2637 sort stable. This minimizes instruction movement, thus
2638 minimizing sched's effect on debugging and cross-jumping. */
2639 return rfs_result (RFS_LIVE_RANGE_SHRINK2,
2640 INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2643 /* The insn in a schedule group should be issued the first. */
2644 if (flag_sched_group_heuristic &&
2645 SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2))
2646 return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1,
2647 tmp, tmp2);
2649 /* Make sure that priority of TMP and TMP2 are initialized. */
2650 gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
2652 if (sched_fusion)
2654 /* The instruction that has the same fusion priority as the last
2655 instruction is the instruction we picked next. If that is not
2656 the case, we sort ready list firstly by fusion priority, then
2657 by priority, and at last by INSN_LUID. */
2658 int a = INSN_FUSION_PRIORITY (tmp);
2659 int b = INSN_FUSION_PRIORITY (tmp2);
2660 int last = -1;
2662 if (last_nondebug_scheduled_insn
2663 && !NOTE_P (last_nondebug_scheduled_insn)
2664 && BLOCK_FOR_INSN (tmp)
2665 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn))
2666 last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
2668 if (a != last && b != last)
2670 if (a == b)
2672 a = INSN_PRIORITY (tmp);
2673 b = INSN_PRIORITY (tmp2);
2675 if (a != b)
2676 return rfs_result (RFS_FUSION, b - a, tmp, tmp2);
2677 else
2678 return rfs_result (RFS_FUSION,
2679 INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2681 else if (a == b)
2683 gcc_assert (last_nondebug_scheduled_insn
2684 && !NOTE_P (last_nondebug_scheduled_insn));
2685 last = INSN_PRIORITY (last_nondebug_scheduled_insn);
2687 a = abs (INSN_PRIORITY (tmp) - last);
2688 b = abs (INSN_PRIORITY (tmp2) - last);
2689 if (a != b)
2690 return rfs_result (RFS_FUSION, a - b, tmp, tmp2);
2691 else
2692 return rfs_result (RFS_FUSION,
2693 INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2695 else if (a == last)
2696 return rfs_result (RFS_FUSION, -1, tmp, tmp2);
2697 else
2698 return rfs_result (RFS_FUSION, 1, tmp, tmp2);
2701 if (sched_pressure != SCHED_PRESSURE_NONE)
2703 /* Prefer insn whose scheduling results in the smallest register
2704 pressure excess. */
2705 if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2706 + insn_delay (tmp)
2707 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2)
2708 - insn_delay (tmp2))))
2709 return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2);
2712 if (sched_pressure != SCHED_PRESSURE_NONE
2713 && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var)
2714 && INSN_TICK (tmp2) != INSN_TICK (tmp))
2716 diff = INSN_TICK (tmp) - INSN_TICK (tmp2);
2717 return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2);
2720 /* If we are doing backtracking in this schedule, prefer insns that
2721 have forward dependencies with negative cost against an insn that
2722 was already scheduled. */
2723 if (current_sched_info->flags & DO_BACKTRACKING)
2725 priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp);
2726 if (priority_val)
2727 return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2);
2730 /* Prefer insn with higher priority. */
2731 priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp);
2733 if (flag_sched_critical_path_heuristic && priority_val)
2734 return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
2736 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) >= 0)
2738 int autopref = autopref_rank_for_schedule (tmp, tmp2);
2739 if (autopref != 0)
2740 return autopref;
2743 /* Prefer speculative insn with greater dependencies weakness. */
2744 if (flag_sched_spec_insn_heuristic && spec_info)
2746 ds_t ds1, ds2;
2747 dw_t dw1, dw2;
2748 int dw;
2750 ds1 = TODO_SPEC (tmp) & SPECULATIVE;
2751 if (ds1)
2752 dw1 = ds_weak (ds1);
2753 else
2754 dw1 = NO_DEP_WEAK;
2756 ds2 = TODO_SPEC (tmp2) & SPECULATIVE;
2757 if (ds2)
2758 dw2 = ds_weak (ds2);
2759 else
2760 dw2 = NO_DEP_WEAK;
2762 dw = dw2 - dw1;
2763 if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
2764 return rfs_result (RFS_SPECULATION, dw, tmp, tmp2);
2767 info_val = (*current_sched_info->rank) (tmp, tmp2);
2768 if (flag_sched_rank_heuristic && info_val)
2769 return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2);
2771 /* Compare insns based on their relation to the last scheduled
2772 non-debug insn. */
2773 if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn)
2775 dep_t dep1;
2776 dep_t dep2;
2777 rtx_insn *last = last_nondebug_scheduled_insn;
2779 /* Classify the instructions into three classes:
2780 1) Data dependent on last schedule insn.
2781 2) Anti/Output dependent on last scheduled insn.
2782 3) Independent of last scheduled insn, or has latency of one.
2783 Choose the insn from the highest numbered class if different. */
2784 dep1 = sd_find_dep_between (last, tmp, true);
2786 if (dep1 == NULL || dep_cost (dep1) == 1)
2787 tmp_class = 3;
2788 else if (/* Data dependence. */
2789 DEP_TYPE (dep1) == REG_DEP_TRUE)
2790 tmp_class = 1;
2791 else
2792 tmp_class = 2;
2794 dep2 = sd_find_dep_between (last, tmp2, true);
2796 if (dep2 == NULL || dep_cost (dep2) == 1)
2797 tmp2_class = 3;
2798 else if (/* Data dependence. */
2799 DEP_TYPE (dep2) == REG_DEP_TRUE)
2800 tmp2_class = 1;
2801 else
2802 tmp2_class = 2;
2804 if ((val = tmp2_class - tmp_class))
2805 return rfs_result (RFS_LAST_INSN, val, tmp, tmp2);
2808 /* Prefer instructions that occur earlier in the model schedule. */
2809 if (sched_pressure == SCHED_PRESSURE_MODEL
2810 && INSN_BB (tmp) == target_bb && INSN_BB (tmp2) == target_bb)
2812 diff = model_index (tmp) - model_index (tmp2);
2813 gcc_assert (diff != 0);
2814 return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
2817 /* Prefer the insn which has more later insns that depend on it.
2818 This gives the scheduler more freedom when scheduling later
2819 instructions at the expense of added register pressure. */
2821 val = (dep_list_size (tmp2, SD_LIST_FORW)
2822 - dep_list_size (tmp, SD_LIST_FORW));
2824 if (flag_sched_dep_count_heuristic && val != 0)
2825 return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
2827 /* If insns are equally good, sort by INSN_LUID (original insn order),
2828 so that we make the sort stable. This minimizes instruction movement,
2829 thus minimizing sched's effect on debugging and cross-jumping. */
2830 return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2833 /* Resort the array A in which only element at index N may be out of order. */
2835 HAIFA_INLINE static void
2836 swap_sort (rtx_insn **a, int n)
2838 rtx_insn *insn = a[n - 1];
2839 int i = n - 2;
2841 while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0)
2843 a[i + 1] = a[i];
2844 i -= 1;
2846 a[i + 1] = insn;
2849 /* Add INSN to the insn queue so that it can be executed at least
2850 N_CYCLES after the currently executing insn. Preserve insns
2851 chain for debugging purposes. REASON will be printed in debugging
2852 output. */
2854 HAIFA_INLINE static void
2855 queue_insn (rtx_insn *insn, int n_cycles, const char *reason)
2857 int next_q = NEXT_Q_AFTER (q_ptr, n_cycles);
2858 rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]);
2859 int new_tick;
2861 gcc_assert (n_cycles <= max_insn_queue_index);
2862 gcc_assert (!DEBUG_INSN_P (insn));
2864 insn_queue[next_q] = link;
2865 q_size += 1;
2867 if (sched_verbose >= 2)
2869 fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ",
2870 (*current_sched_info->print_insn) (insn, 0));
2872 fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason);
2875 QUEUE_INDEX (insn) = next_q;
2877 if (current_sched_info->flags & DO_BACKTRACKING)
2879 new_tick = clock_var + n_cycles;
2880 if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick)
2881 INSN_TICK (insn) = new_tick;
2883 if (INSN_EXACT_TICK (insn) != INVALID_TICK
2884 && INSN_EXACT_TICK (insn) < clock_var + n_cycles)
2886 must_backtrack = true;
2887 if (sched_verbose >= 2)
2888 fprintf (sched_dump, ";;\t\tcausing a backtrack.\n");
2893 /* Remove INSN from queue. */
2894 static void
2895 queue_remove (rtx_insn *insn)
2897 gcc_assert (QUEUE_INDEX (insn) >= 0);
2898 remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]);
2899 q_size--;
2900 QUEUE_INDEX (insn) = QUEUE_NOWHERE;
2903 /* Return a pointer to the bottom of the ready list, i.e. the insn
2904 with the lowest priority. */
2906 rtx_insn **
2907 ready_lastpos (struct ready_list *ready)
2909 gcc_assert (ready->n_ready >= 1);
2910 return ready->vec + ready->first - ready->n_ready + 1;
2913 /* Add an element INSN to the ready list so that it ends up with the
2914 lowest/highest priority depending on FIRST_P. */
2916 HAIFA_INLINE static void
2917 ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p)
2919 if (!first_p)
2921 if (ready->first == ready->n_ready)
2923 memmove (ready->vec + ready->veclen - ready->n_ready,
2924 ready_lastpos (ready),
2925 ready->n_ready * sizeof (rtx));
2926 ready->first = ready->veclen - 1;
2928 ready->vec[ready->first - ready->n_ready] = insn;
2930 else
2932 if (ready->first == ready->veclen - 1)
2934 if (ready->n_ready)
2935 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2936 memmove (ready->vec + ready->veclen - ready->n_ready - 1,
2937 ready_lastpos (ready),
2938 ready->n_ready * sizeof (rtx));
2939 ready->first = ready->veclen - 2;
2941 ready->vec[++(ready->first)] = insn;
2944 ready->n_ready++;
2945 if (DEBUG_INSN_P (insn))
2946 ready->n_debug++;
2948 gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY);
2949 QUEUE_INDEX (insn) = QUEUE_READY;
2951 if (INSN_EXACT_TICK (insn) != INVALID_TICK
2952 && INSN_EXACT_TICK (insn) < clock_var)
2954 must_backtrack = true;
2958 /* Remove the element with the highest priority from the ready list and
2959 return it. */
2961 HAIFA_INLINE static rtx_insn *
2962 ready_remove_first (struct ready_list *ready)
2964 rtx_insn *t;
2966 gcc_assert (ready->n_ready);
2967 t = ready->vec[ready->first--];
2968 ready->n_ready--;
2969 if (DEBUG_INSN_P (t))
2970 ready->n_debug--;
2971 /* If the queue becomes empty, reset it. */
2972 if (ready->n_ready == 0)
2973 ready->first = ready->veclen - 1;
2975 gcc_assert (QUEUE_INDEX (t) == QUEUE_READY);
2976 QUEUE_INDEX (t) = QUEUE_NOWHERE;
2978 return t;
2981 /* The following code implements multi-pass scheduling for the first
2982 cycle. In other words, we will try to choose ready insn which
2983 permits to start maximum number of insns on the same cycle. */
2985 /* Return a pointer to the element INDEX from the ready. INDEX for
2986 insn with the highest priority is 0, and the lowest priority has
2987 N_READY - 1. */
2989 rtx_insn *
2990 ready_element (struct ready_list *ready, int index)
2992 gcc_assert (ready->n_ready && index < ready->n_ready);
2994 return ready->vec[ready->first - index];
2997 /* Remove the element INDEX from the ready list and return it. INDEX
2998 for insn with the highest priority is 0, and the lowest priority
2999 has N_READY - 1. */
3001 HAIFA_INLINE static rtx_insn *
3002 ready_remove (struct ready_list *ready, int index)
3004 rtx_insn *t;
3005 int i;
3007 if (index == 0)
3008 return ready_remove_first (ready);
3009 gcc_assert (ready->n_ready && index < ready->n_ready);
3010 t = ready->vec[ready->first - index];
3011 ready->n_ready--;
3012 if (DEBUG_INSN_P (t))
3013 ready->n_debug--;
3014 for (i = index; i < ready->n_ready; i++)
3015 ready->vec[ready->first - i] = ready->vec[ready->first - i - 1];
3016 QUEUE_INDEX (t) = QUEUE_NOWHERE;
3017 return t;
3020 /* Remove INSN from the ready list. */
3021 static void
3022 ready_remove_insn (rtx_insn *insn)
3024 int i;
3026 for (i = 0; i < readyp->n_ready; i++)
3027 if (ready_element (readyp, i) == insn)
3029 ready_remove (readyp, i);
3030 return;
3032 gcc_unreachable ();
3035 /* Calculate difference of two statistics set WAS and NOW.
3036 Result returned in WAS. */
3037 static void
3038 rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was,
3039 const rank_for_schedule_stats_t *now)
3041 for (int i = 0; i < RFS_N; ++i)
3042 was->stats[i] = now->stats[i] - was->stats[i];
3045 /* Print rank_for_schedule statistics. */
3046 static void
3047 print_rank_for_schedule_stats (const char *prefix,
3048 const rank_for_schedule_stats_t *stats,
3049 struct ready_list *ready)
3051 for (int i = 0; i < RFS_N; ++i)
3052 if (stats->stats[i])
3054 fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]);
3056 if (ready != NULL)
3057 /* Print out insns that won due to RFS_<I>. */
3059 rtx_insn **p = ready_lastpos (ready);
3061 fprintf (sched_dump, ":");
3062 /* Start with 1 since least-priority insn didn't have any wins. */
3063 for (int j = 1; j < ready->n_ready; ++j)
3064 if (INSN_LAST_RFS_WIN (p[j]) == i)
3065 fprintf (sched_dump, " %s",
3066 (*current_sched_info->print_insn) (p[j], 0));
3068 fprintf (sched_dump, "\n");
3072 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3073 of array. */
3074 static void
3075 ready_sort_debug (struct ready_list *ready)
3077 int i;
3078 rtx_insn **first = ready_lastpos (ready);
3080 for (i = 0; i < ready->n_ready; ++i)
3081 if (!DEBUG_INSN_P (first[i]))
3082 INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
3084 qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
3087 /* Sort non-debug insns in the ready list READY by ascending priority.
3088 Assumes that all debug insns are separated from the real insns. */
3089 static void
3090 ready_sort_real (struct ready_list *ready)
3092 int i;
3093 rtx_insn **first = ready_lastpos (ready);
3094 int n_ready_real = ready->n_ready - ready->n_debug;
3096 if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
3097 for (i = 0; i < n_ready_real; ++i)
3098 setup_insn_reg_pressure_info (first[i]);
3099 else if (sched_pressure == SCHED_PRESSURE_MODEL
3100 && model_curr_point < model_num_insns)
3101 model_set_excess_costs (first, n_ready_real);
3103 rank_for_schedule_stats_t stats1;
3104 if (sched_verbose >= 4)
3105 stats1 = rank_for_schedule_stats;
3107 if (n_ready_real == 2)
3108 swap_sort (first, n_ready_real);
3109 else if (n_ready_real > 2)
3110 qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
3112 if (sched_verbose >= 4)
3114 rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats);
3115 print_rank_for_schedule_stats (";;\t\t", &stats1, ready);
3119 /* Sort the ready list READY by ascending priority. */
3120 static void
3121 ready_sort (struct ready_list *ready)
3123 if (ready->n_debug > 0)
3124 ready_sort_debug (ready);
3125 else
3126 ready_sort_real (ready);
3129 /* PREV is an insn that is ready to execute. Adjust its priority if that
3130 will help shorten or lengthen register lifetimes as appropriate. Also
3131 provide a hook for the target to tweak itself. */
3133 HAIFA_INLINE static void
3134 adjust_priority (rtx_insn *prev)
3136 /* ??? There used to be code here to try and estimate how an insn
3137 affected register lifetimes, but it did it by looking at REG_DEAD
3138 notes, which we removed in schedule_region. Nor did it try to
3139 take into account register pressure or anything useful like that.
3141 Revisit when we have a machine model to work with and not before. */
3143 if (targetm.sched.adjust_priority)
3144 INSN_PRIORITY (prev) =
3145 targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev));
3148 /* Advance DFA state STATE on one cycle. */
3149 void
3150 advance_state (state_t state)
3152 if (targetm.sched.dfa_pre_advance_cycle)
3153 targetm.sched.dfa_pre_advance_cycle ();
3155 if (targetm.sched.dfa_pre_cycle_insn)
3156 state_transition (state,
3157 targetm.sched.dfa_pre_cycle_insn ());
3159 state_transition (state, NULL);
3161 if (targetm.sched.dfa_post_cycle_insn)
3162 state_transition (state,
3163 targetm.sched.dfa_post_cycle_insn ());
3165 if (targetm.sched.dfa_post_advance_cycle)
3166 targetm.sched.dfa_post_advance_cycle ();
3169 /* Advance time on one cycle. */
3170 HAIFA_INLINE static void
3171 advance_one_cycle (void)
3173 advance_state (curr_state);
3174 if (sched_verbose >= 4)
3175 fprintf (sched_dump, ";;\tAdvance the current state.\n");
3178 /* Update register pressure after scheduling INSN. */
3179 static void
3180 update_register_pressure (rtx_insn *insn)
3182 struct reg_use_data *use;
3183 struct reg_set_data *set;
3185 gcc_checking_assert (!DEBUG_INSN_P (insn));
3187 for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
3188 if (dying_use_p (use))
3189 mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3190 use->regno, false);
3191 for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set)
3192 mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3193 set->regno, true);
3196 /* Set up or update (if UPDATE_P) max register pressure (see its
3197 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3198 after insn AFTER. */
3199 static void
3200 setup_insn_max_reg_pressure (rtx_insn *after, bool update_p)
3202 int i, p;
3203 bool eq_p;
3204 rtx_insn *insn;
3205 static int max_reg_pressure[N_REG_CLASSES];
3207 save_reg_pressure ();
3208 for (i = 0; i < ira_pressure_classes_num; i++)
3209 max_reg_pressure[ira_pressure_classes[i]]
3210 = curr_reg_pressure[ira_pressure_classes[i]];
3211 for (insn = NEXT_INSN (after);
3212 insn != NULL_RTX && ! BARRIER_P (insn)
3213 && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after);
3214 insn = NEXT_INSN (insn))
3215 if (NONDEBUG_INSN_P (insn))
3217 eq_p = true;
3218 for (i = 0; i < ira_pressure_classes_num; i++)
3220 p = max_reg_pressure[ira_pressure_classes[i]];
3221 if (INSN_MAX_REG_PRESSURE (insn)[i] != p)
3223 eq_p = false;
3224 INSN_MAX_REG_PRESSURE (insn)[i]
3225 = max_reg_pressure[ira_pressure_classes[i]];
3228 if (update_p && eq_p)
3229 break;
3230 update_register_pressure (insn);
3231 for (i = 0; i < ira_pressure_classes_num; i++)
3232 if (max_reg_pressure[ira_pressure_classes[i]]
3233 < curr_reg_pressure[ira_pressure_classes[i]])
3234 max_reg_pressure[ira_pressure_classes[i]]
3235 = curr_reg_pressure[ira_pressure_classes[i]];
3237 restore_reg_pressure ();
3240 /* Update the current register pressure after scheduling INSN. Update
3241 also max register pressure for unscheduled insns of the current
3242 BB. */
3243 static void
3244 update_reg_and_insn_max_reg_pressure (rtx_insn *insn)
3246 int i;
3247 int before[N_REG_CLASSES];
3249 for (i = 0; i < ira_pressure_classes_num; i++)
3250 before[i] = curr_reg_pressure[ira_pressure_classes[i]];
3251 update_register_pressure (insn);
3252 for (i = 0; i < ira_pressure_classes_num; i++)
3253 if (curr_reg_pressure[ira_pressure_classes[i]] != before[i])
3254 break;
3255 if (i < ira_pressure_classes_num)
3256 setup_insn_max_reg_pressure (insn, true);
3259 /* Set up register pressure at the beginning of basic block BB whose
3260 insns starting after insn AFTER. Set up also max register pressure
3261 for all insns of the basic block. */
3262 void
3263 sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after)
3265 gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
3266 initiate_bb_reg_pressure_info (bb);
3267 setup_insn_max_reg_pressure (after, false);
3270 /* If doing predication while scheduling, verify whether INSN, which
3271 has just been scheduled, clobbers the conditions of any
3272 instructions that must be predicated in order to break their
3273 dependencies. If so, remove them from the queues so that they will
3274 only be scheduled once their control dependency is resolved. */
3276 static void
3277 check_clobbered_conditions (rtx_insn *insn)
3279 HARD_REG_SET t;
3280 int i;
3282 if ((current_sched_info->flags & DO_PREDICATION) == 0)
3283 return;
3285 find_all_hard_reg_sets (insn, &t, true);
3287 restart:
3288 for (i = 0; i < ready.n_ready; i++)
3290 rtx_insn *x = ready_element (&ready, i);
3291 if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3293 ready_remove_insn (x);
3294 goto restart;
3297 for (i = 0; i <= max_insn_queue_index; i++)
3299 rtx_insn_list *link;
3300 int q = NEXT_Q_AFTER (q_ptr, i);
3302 restart_queue:
3303 for (link = insn_queue[q]; link; link = link->next ())
3305 rtx_insn *x = link->insn ();
3306 if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3308 queue_remove (x);
3309 goto restart_queue;
3315 /* Return (in order):
3317 - positive if INSN adversely affects the pressure on one
3318 register class
3320 - negative if INSN reduces the pressure on one register class
3322 - 0 if INSN doesn't affect the pressure on any register class. */
3324 static int
3325 model_classify_pressure (struct model_insn_info *insn)
3327 struct reg_pressure_data *reg_pressure;
3328 int death[N_REG_CLASSES];
3329 int pci, cl, sum;
3331 calculate_reg_deaths (insn->insn, death);
3332 reg_pressure = INSN_REG_PRESSURE (insn->insn);
3333 sum = 0;
3334 for (pci = 0; pci < ira_pressure_classes_num; pci++)
3336 cl = ira_pressure_classes[pci];
3337 if (death[cl] < reg_pressure[pci].set_increase)
3338 return 1;
3339 sum += reg_pressure[pci].set_increase - death[cl];
3341 return sum;
3344 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3346 static int
3347 model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2)
3349 unsigned int height1, height2;
3350 unsigned int priority1, priority2;
3352 /* Prefer instructions with a higher model priority. */
3353 if (insn1->model_priority != insn2->model_priority)
3354 return insn1->model_priority > insn2->model_priority;
3356 /* Combine the length of the longest path of satisfied true dependencies
3357 that leads to each instruction (depth) with the length of the longest
3358 path of any dependencies that leads from the instruction (alap).
3359 Prefer instructions with the greatest combined length. If the combined
3360 lengths are equal, prefer instructions with the greatest depth.
3362 The idea is that, if we have a set S of "equal" instructions that each
3363 have ALAP value X, and we pick one such instruction I, any true-dependent
3364 successors of I that have ALAP value X - 1 should be preferred over S.
3365 This encourages the schedule to be "narrow" rather than "wide".
3366 However, if I is a low-priority instruction that we decided to
3367 schedule because of its model_classify_pressure, and if there
3368 is a set of higher-priority instructions T, the aforementioned
3369 successors of I should not have the edge over T. */
3370 height1 = insn1->depth + insn1->alap;
3371 height2 = insn2->depth + insn2->alap;
3372 if (height1 != height2)
3373 return height1 > height2;
3374 if (insn1->depth != insn2->depth)
3375 return insn1->depth > insn2->depth;
3377 /* We have no real preference between INSN1 an INSN2 as far as attempts
3378 to reduce pressure go. Prefer instructions with higher priorities. */
3379 priority1 = INSN_PRIORITY (insn1->insn);
3380 priority2 = INSN_PRIORITY (insn2->insn);
3381 if (priority1 != priority2)
3382 return priority1 > priority2;
3384 /* Use the original rtl sequence as a tie-breaker. */
3385 return insn1 < insn2;
3388 /* Add INSN to the model worklist immediately after PREV. Add it to the
3389 beginning of the list if PREV is null. */
3391 static void
3392 model_add_to_worklist_at (struct model_insn_info *insn,
3393 struct model_insn_info *prev)
3395 gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE);
3396 QUEUE_INDEX (insn->insn) = QUEUE_READY;
3398 insn->prev = prev;
3399 if (prev)
3401 insn->next = prev->next;
3402 prev->next = insn;
3404 else
3406 insn->next = model_worklist;
3407 model_worklist = insn;
3409 if (insn->next)
3410 insn->next->prev = insn;
3413 /* Remove INSN from the model worklist. */
3415 static void
3416 model_remove_from_worklist (struct model_insn_info *insn)
3418 gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY);
3419 QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE;
3421 if (insn->prev)
3422 insn->prev->next = insn->next;
3423 else
3424 model_worklist = insn->next;
3425 if (insn->next)
3426 insn->next->prev = insn->prev;
3429 /* Add INSN to the model worklist. Start looking for a suitable position
3430 between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
3431 insns either side. A null PREV indicates the beginning of the list and
3432 a null NEXT indicates the end. */
3434 static void
3435 model_add_to_worklist (struct model_insn_info *insn,
3436 struct model_insn_info *prev,
3437 struct model_insn_info *next)
3439 int count;
3441 count = MAX_SCHED_READY_INSNS;
3442 if (count > 0 && prev && model_order_p (insn, prev))
3445 count--;
3446 prev = prev->prev;
3448 while (count > 0 && prev && model_order_p (insn, prev));
3449 else
3450 while (count > 0 && next && model_order_p (next, insn))
3452 count--;
3453 prev = next;
3454 next = next->next;
3456 model_add_to_worklist_at (insn, prev);
3459 /* INSN may now have a higher priority (in the model_order_p sense)
3460 than before. Move it up the worklist if necessary. */
3462 static void
3463 model_promote_insn (struct model_insn_info *insn)
3465 struct model_insn_info *prev;
3466 int count;
3468 prev = insn->prev;
3469 count = MAX_SCHED_READY_INSNS;
3470 while (count > 0 && prev && model_order_p (insn, prev))
3472 count--;
3473 prev = prev->prev;
3475 if (prev != insn->prev)
3477 model_remove_from_worklist (insn);
3478 model_add_to_worklist_at (insn, prev);
3482 /* Add INSN to the end of the model schedule. */
3484 static void
3485 model_add_to_schedule (rtx_insn *insn)
3487 unsigned int point;
3489 gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
3490 QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
3492 point = model_schedule.length ();
3493 model_schedule.quick_push (insn);
3494 INSN_MODEL_INDEX (insn) = point + 1;
3497 /* Analyze the instructions that are to be scheduled, setting up
3498 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3499 instructions to model_worklist. */
3501 static void
3502 model_analyze_insns (void)
3504 rtx_insn *start, *end, *iter;
3505 sd_iterator_def sd_it;
3506 dep_t dep;
3507 struct model_insn_info *insn, *con;
3509 model_num_insns = 0;
3510 start = PREV_INSN (current_sched_info->next_tail);
3511 end = current_sched_info->prev_head;
3512 for (iter = start; iter != end; iter = PREV_INSN (iter))
3513 if (NONDEBUG_INSN_P (iter))
3515 insn = MODEL_INSN_INFO (iter);
3516 insn->insn = iter;
3517 FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep)
3519 con = MODEL_INSN_INFO (DEP_CON (dep));
3520 if (con->insn && insn->alap < con->alap + 1)
3521 insn->alap = con->alap + 1;
3524 insn->old_queue = QUEUE_INDEX (iter);
3525 QUEUE_INDEX (iter) = QUEUE_NOWHERE;
3527 insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK);
3528 if (insn->unscheduled_preds == 0)
3529 model_add_to_worklist (insn, NULL, model_worklist);
3531 model_num_insns++;
3535 /* The global state describes the register pressure at the start of the
3536 model schedule. Initialize GROUP accordingly. */
3538 static void
3539 model_init_pressure_group (struct model_pressure_group *group)
3541 int pci, cl;
3543 for (pci = 0; pci < ira_pressure_classes_num; pci++)
3545 cl = ira_pressure_classes[pci];
3546 group->limits[pci].pressure = curr_reg_pressure[cl];
3547 group->limits[pci].point = 0;
3549 /* Use index model_num_insns to record the state after the last
3550 instruction in the model schedule. */
3551 group->model = XNEWVEC (struct model_pressure_data,
3552 (model_num_insns + 1) * ira_pressure_classes_num);
3555 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3556 Update the maximum pressure for the whole schedule. */
3558 static void
3559 model_record_pressure (struct model_pressure_group *group,
3560 int point, int pci, int pressure)
3562 MODEL_REF_PRESSURE (group, point, pci) = pressure;
3563 if (group->limits[pci].pressure < pressure)
3565 group->limits[pci].pressure = pressure;
3566 group->limits[pci].point = point;
3570 /* INSN has just been added to the end of the model schedule. Record its
3571 register-pressure information. */
3573 static void
3574 model_record_pressures (struct model_insn_info *insn)
3576 struct reg_pressure_data *reg_pressure;
3577 int point, pci, cl, delta;
3578 int death[N_REG_CLASSES];
3580 point = model_index (insn->insn);
3581 if (sched_verbose >= 2)
3583 if (point == 0)
3585 fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n");
3586 fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n");
3588 fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3589 point, INSN_UID (insn->insn), insn->model_priority,
3590 insn->depth + insn->alap, insn->depth,
3591 INSN_PRIORITY (insn->insn),
3592 str_pattern_slim (PATTERN (insn->insn)));
3594 calculate_reg_deaths (insn->insn, death);
3595 reg_pressure = INSN_REG_PRESSURE (insn->insn);
3596 for (pci = 0; pci < ira_pressure_classes_num; pci++)
3598 cl = ira_pressure_classes[pci];
3599 delta = reg_pressure[pci].set_increase - death[cl];
3600 if (sched_verbose >= 2)
3601 fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl],
3602 curr_reg_pressure[cl], delta);
3603 model_record_pressure (&model_before_pressure, point, pci,
3604 curr_reg_pressure[cl]);
3606 if (sched_verbose >= 2)
3607 fprintf (sched_dump, "\n");
3610 /* All instructions have been added to the model schedule. Record the
3611 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3613 static void
3614 model_record_final_pressures (struct model_pressure_group *group)
3616 int point, pci, max_pressure, ref_pressure, cl;
3618 for (pci = 0; pci < ira_pressure_classes_num; pci++)
3620 /* Record the final pressure for this class. */
3621 cl = ira_pressure_classes[pci];
3622 point = model_num_insns;
3623 ref_pressure = curr_reg_pressure[cl];
3624 model_record_pressure (group, point, pci, ref_pressure);
3626 /* Record the original maximum pressure. */
3627 group->limits[pci].orig_pressure = group->limits[pci].pressure;
3629 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3630 max_pressure = ref_pressure;
3631 MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3632 while (point > 0)
3634 point--;
3635 ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
3636 max_pressure = MAX (max_pressure, ref_pressure);
3637 MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3642 /* Update all successors of INSN, given that INSN has just been scheduled. */
3644 static void
3645 model_add_successors_to_worklist (struct model_insn_info *insn)
3647 sd_iterator_def sd_it;
3648 struct model_insn_info *con;
3649 dep_t dep;
3651 FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep)
3653 con = MODEL_INSN_INFO (DEP_CON (dep));
3654 /* Ignore debug instructions, and instructions from other blocks. */
3655 if (con->insn)
3657 con->unscheduled_preds--;
3659 /* Update the depth field of each true-dependent successor.
3660 Increasing the depth gives them a higher priority than
3661 before. */
3662 if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1)
3664 con->depth = insn->depth + 1;
3665 if (QUEUE_INDEX (con->insn) == QUEUE_READY)
3666 model_promote_insn (con);
3669 /* If this is a true dependency, or if there are no remaining
3670 dependencies for CON (meaning that CON only had non-true
3671 dependencies), make sure that CON is on the worklist.
3672 We don't bother otherwise because it would tend to fill the
3673 worklist with a lot of low-priority instructions that are not
3674 yet ready to issue. */
3675 if ((con->depth > 0 || con->unscheduled_preds == 0)
3676 && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE)
3677 model_add_to_worklist (con, insn, insn->next);
3682 /* Give INSN a higher priority than any current instruction, then give
3683 unscheduled predecessors of INSN a higher priority still. If any of
3684 those predecessors are not on the model worklist, do the same for its
3685 predecessors, and so on. */
3687 static void
3688 model_promote_predecessors (struct model_insn_info *insn)
3690 struct model_insn_info *pro, *first;
3691 sd_iterator_def sd_it;
3692 dep_t dep;
3694 if (sched_verbose >= 7)
3695 fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of",
3696 INSN_UID (insn->insn), model_next_priority);
3697 insn->model_priority = model_next_priority++;
3698 model_remove_from_worklist (insn);
3699 model_add_to_worklist_at (insn, NULL);
3701 first = NULL;
3702 for (;;)
3704 FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep)
3706 pro = MODEL_INSN_INFO (DEP_PRO (dep));
3707 /* The first test is to ignore debug instructions, and instructions
3708 from other blocks. */
3709 if (pro->insn
3710 && pro->model_priority != model_next_priority
3711 && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED)
3713 pro->model_priority = model_next_priority;
3714 if (sched_verbose >= 7)
3715 fprintf (sched_dump, " %d", INSN_UID (pro->insn));
3716 if (QUEUE_INDEX (pro->insn) == QUEUE_READY)
3718 /* PRO is already in the worklist, but it now has
3719 a higher priority than before. Move it at the
3720 appropriate place. */
3721 model_remove_from_worklist (pro);
3722 model_add_to_worklist (pro, NULL, model_worklist);
3724 else
3726 /* PRO isn't in the worklist. Recursively process
3727 its predecessors until we find one that is. */
3728 pro->next = first;
3729 first = pro;
3733 if (!first)
3734 break;
3735 insn = first;
3736 first = insn->next;
3738 if (sched_verbose >= 7)
3739 fprintf (sched_dump, " = %d\n", model_next_priority);
3740 model_next_priority++;
3743 /* Pick one instruction from model_worklist and process it. */
3745 static void
3746 model_choose_insn (void)
3748 struct model_insn_info *insn, *fallback;
3749 int count;
3751 if (sched_verbose >= 7)
3753 fprintf (sched_dump, ";;\t+--- worklist:\n");
3754 insn = model_worklist;
3755 count = MAX_SCHED_READY_INSNS;
3756 while (count > 0 && insn)
3758 fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d]\n",
3759 INSN_UID (insn->insn), insn->model_priority,
3760 insn->depth + insn->alap, insn->depth,
3761 INSN_PRIORITY (insn->insn));
3762 count--;
3763 insn = insn->next;
3767 /* Look for a ready instruction whose model_classify_priority is zero
3768 or negative, picking the highest-priority one. Adding such an
3769 instruction to the schedule now should do no harm, and may actually
3770 do some good.
3772 Failing that, see whether there is an instruction with the highest
3773 extant model_priority that is not yet ready, but which would reduce
3774 pressure if it became ready. This is designed to catch cases like:
3776 (set (mem (reg R1)) (reg R2))
3778 where the instruction is the last remaining use of R1 and where the
3779 value of R2 is not yet available (or vice versa). The death of R1
3780 means that this instruction already reduces pressure. It is of
3781 course possible that the computation of R2 involves other registers
3782 that are hard to kill, but such cases are rare enough for this
3783 heuristic to be a win in general.
3785 Failing that, just pick the highest-priority instruction in the
3786 worklist. */
3787 count = MAX_SCHED_READY_INSNS;
3788 insn = model_worklist;
3789 fallback = 0;
3790 for (;;)
3792 if (count == 0 || !insn)
3794 insn = fallback ? fallback : model_worklist;
3795 break;
3797 if (insn->unscheduled_preds)
3799 if (model_worklist->model_priority == insn->model_priority
3800 && !fallback
3801 && model_classify_pressure (insn) < 0)
3802 fallback = insn;
3804 else
3806 if (model_classify_pressure (insn) <= 0)
3807 break;
3809 count--;
3810 insn = insn->next;
3813 if (sched_verbose >= 7 && insn != model_worklist)
3815 if (insn->unscheduled_preds)
3816 fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n",
3817 INSN_UID (insn->insn));
3818 else
3819 fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n",
3820 INSN_UID (insn->insn));
3822 if (insn->unscheduled_preds)
3823 /* INSN isn't yet ready to issue. Give all its predecessors the
3824 highest priority. */
3825 model_promote_predecessors (insn);
3826 else
3828 /* INSN is ready. Add it to the end of model_schedule and
3829 process its successors. */
3830 model_add_successors_to_worklist (insn);
3831 model_remove_from_worklist (insn);
3832 model_add_to_schedule (insn->insn);
3833 model_record_pressures (insn);
3834 update_register_pressure (insn->insn);
3838 /* Restore all QUEUE_INDEXs to the values that they had before
3839 model_start_schedule was called. */
3841 static void
3842 model_reset_queue_indices (void)
3844 unsigned int i;
3845 rtx_insn *insn;
3847 FOR_EACH_VEC_ELT (model_schedule, i, insn)
3848 QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue;
3851 /* We have calculated the model schedule and spill costs. Print a summary
3852 to sched_dump. */
3854 static void
3855 model_dump_pressure_summary (void)
3857 int pci, cl;
3859 fprintf (sched_dump, ";; Pressure summary:");
3860 for (pci = 0; pci < ira_pressure_classes_num; pci++)
3862 cl = ira_pressure_classes[pci];
3863 fprintf (sched_dump, " %s:%d", reg_class_names[cl],
3864 model_before_pressure.limits[pci].pressure);
3866 fprintf (sched_dump, "\n\n");
3869 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3870 scheduling region. */
3872 static void
3873 model_start_schedule (basic_block bb)
3875 model_next_priority = 1;
3876 model_schedule.create (sched_max_luid);
3877 model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid);
3879 gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)));
3880 initiate_reg_pressure_info (df_get_live_in (bb));
3882 model_analyze_insns ();
3883 model_init_pressure_group (&model_before_pressure);
3884 while (model_worklist)
3885 model_choose_insn ();
3886 gcc_assert (model_num_insns == (int) model_schedule.length ());
3887 if (sched_verbose >= 2)
3888 fprintf (sched_dump, "\n");
3890 model_record_final_pressures (&model_before_pressure);
3891 model_reset_queue_indices ();
3893 XDELETEVEC (model_insns);
3895 model_curr_point = 0;
3896 initiate_reg_pressure_info (df_get_live_in (bb));
3897 if (sched_verbose >= 1)
3898 model_dump_pressure_summary ();
3901 /* Free the information associated with GROUP. */
3903 static void
3904 model_finalize_pressure_group (struct model_pressure_group *group)
3906 XDELETEVEC (group->model);
3909 /* Free the information created by model_start_schedule. */
3911 static void
3912 model_end_schedule (void)
3914 model_finalize_pressure_group (&model_before_pressure);
3915 model_schedule.release ();
3918 /* Prepare reg pressure scheduling for basic block BB. */
3919 static void
3920 sched_pressure_start_bb (basic_block bb)
3922 /* Set the number of available registers for each class taking into account
3923 relative probability of current basic block versus function prologue and
3924 epilogue.
3925 * If the basic block executes much more often than the prologue/epilogue
3926 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3927 nil, so the effective number of available registers is
3928 (ira_class_hard_regs_num[cl] - 0).
3929 * If the basic block executes as often as the prologue/epilogue,
3930 then spill in the block is as costly as in the prologue, so the effective
3931 number of available registers is
3932 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
3933 Note that all-else-equal, we prefer to spill in the prologue, since that
3934 allows "extra" registers for other basic blocks of the function.
3935 * If the basic block is on the cold path of the function and executes
3936 rarely, then we should always prefer to spill in the block, rather than
3937 in the prologue/epilogue. The effective number of available register is
3938 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
3940 int i;
3941 int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
3942 int bb_freq = bb->frequency;
3944 if (bb_freq == 0)
3946 if (entry_freq == 0)
3947 entry_freq = bb_freq = 1;
3949 if (bb_freq < entry_freq)
3950 bb_freq = entry_freq;
3952 for (i = 0; i < ira_pressure_classes_num; ++i)
3954 enum reg_class cl = ira_pressure_classes[i];
3955 sched_class_regs_num[cl] = ira_class_hard_regs_num[cl];
3956 sched_class_regs_num[cl]
3957 -= (call_used_regs_num[cl] * entry_freq) / bb_freq;
3961 if (sched_pressure == SCHED_PRESSURE_MODEL)
3962 model_start_schedule (bb);
3965 /* A structure that holds local state for the loop in schedule_block. */
3966 struct sched_block_state
3968 /* True if no real insns have been scheduled in the current cycle. */
3969 bool first_cycle_insn_p;
3970 /* True if a shadow insn has been scheduled in the current cycle, which
3971 means that no more normal insns can be issued. */
3972 bool shadows_only_p;
3973 /* True if we're winding down a modulo schedule, which means that we only
3974 issue insns with INSN_EXACT_TICK set. */
3975 bool modulo_epilogue;
3976 /* Initialized with the machine's issue rate every cycle, and updated
3977 by calls to the variable_issue hook. */
3978 int can_issue_more;
3981 /* INSN is the "currently executing insn". Launch each insn which was
3982 waiting on INSN. READY is the ready list which contains the insns
3983 that are ready to fire. CLOCK is the current cycle. The function
3984 returns necessary cycle advance after issuing the insn (it is not
3985 zero for insns in a schedule group). */
3987 static int
3988 schedule_insn (rtx_insn *insn)
3990 sd_iterator_def sd_it;
3991 dep_t dep;
3992 int i;
3993 int advance = 0;
3995 if (sched_verbose >= 1)
3997 struct reg_pressure_data *pressure_info;
3998 fprintf (sched_dump, ";;\t%3i--> %s %-40s:",
3999 clock_var, (*current_sched_info->print_insn) (insn, 1),
4000 str_pattern_slim (PATTERN (insn)));
4002 if (recog_memoized (insn) < 0)
4003 fprintf (sched_dump, "nothing");
4004 else
4005 print_reservation (sched_dump, insn);
4006 pressure_info = INSN_REG_PRESSURE (insn);
4007 if (pressure_info != NULL)
4009 fputc (':', sched_dump);
4010 for (i = 0; i < ira_pressure_classes_num; i++)
4011 fprintf (sched_dump, "%s%s%+d(%d)",
4012 scheduled_insns.length () > 1
4013 && INSN_LUID (insn)
4014 < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "",
4015 reg_class_names[ira_pressure_classes[i]],
4016 pressure_info[i].set_increase, pressure_info[i].change);
4018 if (sched_pressure == SCHED_PRESSURE_MODEL
4019 && model_curr_point < model_num_insns
4020 && model_index (insn) == model_curr_point)
4021 fprintf (sched_dump, ":model %d", model_curr_point);
4022 fputc ('\n', sched_dump);
4025 if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn))
4026 update_reg_and_insn_max_reg_pressure (insn);
4028 /* Scheduling instruction should have all its dependencies resolved and
4029 should have been removed from the ready list. */
4030 gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
4032 /* Reset debug insns invalidated by moving this insn. */
4033 if (MAY_HAVE_DEBUG_INSNS && !DEBUG_INSN_P (insn))
4034 for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
4035 sd_iterator_cond (&sd_it, &dep);)
4037 rtx_insn *dbg = DEP_PRO (dep);
4038 struct reg_use_data *use, *next;
4040 if (DEP_STATUS (dep) & DEP_CANCELLED)
4042 sd_iterator_next (&sd_it);
4043 continue;
4046 gcc_assert (DEBUG_INSN_P (dbg));
4048 if (sched_verbose >= 6)
4049 fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n",
4050 INSN_UID (dbg));
4052 /* ??? Rather than resetting the debug insn, we might be able
4053 to emit a debug temp before the just-scheduled insn, but
4054 this would involve checking that the expression at the
4055 point of the debug insn is equivalent to the expression
4056 before the just-scheduled insn. They might not be: the
4057 expression in the debug insn may depend on other insns not
4058 yet scheduled that set MEMs, REGs or even other debug
4059 insns. It's not clear that attempting to preserve debug
4060 information in these cases is worth the effort, given how
4061 uncommon these resets are and the likelihood that the debug
4062 temps introduced won't survive the schedule change. */
4063 INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC ();
4064 df_insn_rescan (dbg);
4066 /* Unknown location doesn't use any registers. */
4067 for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next)
4069 struct reg_use_data *prev = use;
4071 /* Remove use from the cyclic next_regno_use chain first. */
4072 while (prev->next_regno_use != use)
4073 prev = prev->next_regno_use;
4074 prev->next_regno_use = use->next_regno_use;
4075 next = use->next_insn_use;
4076 free (use);
4078 INSN_REG_USE_LIST (dbg) = NULL;
4080 /* We delete rather than resolve these deps, otherwise we
4081 crash in sched_free_deps(), because forward deps are
4082 expected to be released before backward deps. */
4083 sd_delete_dep (sd_it);
4086 gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
4087 QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
4089 if (sched_pressure == SCHED_PRESSURE_MODEL
4090 && model_curr_point < model_num_insns
4091 && NONDEBUG_INSN_P (insn))
4093 if (model_index (insn) == model_curr_point)
4095 model_curr_point++;
4096 while (model_curr_point < model_num_insns
4097 && (QUEUE_INDEX (MODEL_INSN (model_curr_point))
4098 == QUEUE_SCHEDULED));
4099 else
4100 model_recompute (insn);
4101 model_update_limit_points ();
4102 update_register_pressure (insn);
4103 if (sched_verbose >= 2)
4104 print_curr_reg_pressure ();
4107 gcc_assert (INSN_TICK (insn) >= MIN_TICK);
4108 if (INSN_TICK (insn) > clock_var)
4109 /* INSN has been prematurely moved from the queue to the ready list.
4110 This is possible only if following flags are set. */
4111 gcc_assert (flag_sched_stalled_insns || sched_fusion);
4113 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4114 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4115 INSN_TICK (insn) = clock_var;
4117 check_clobbered_conditions (insn);
4119 /* Update dependent instructions. First, see if by scheduling this insn
4120 now we broke a dependence in a way that requires us to change another
4121 insn. */
4122 for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
4123 sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
4125 struct dep_replacement *desc = DEP_REPLACE (dep);
4126 rtx_insn *pro = DEP_PRO (dep);
4127 if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED
4128 && desc != NULL && desc->insn == pro)
4129 apply_replacement (dep, false);
4132 /* Go through and resolve forward dependencies. */
4133 for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4134 sd_iterator_cond (&sd_it, &dep);)
4136 rtx_insn *next = DEP_CON (dep);
4137 bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0;
4139 /* Resolve the dependence between INSN and NEXT.
4140 sd_resolve_dep () moves current dep to another list thus
4141 advancing the iterator. */
4142 sd_resolve_dep (sd_it);
4144 if (cancelled)
4146 if (must_restore_pattern_p (next, dep))
4147 restore_pattern (dep, false);
4148 continue;
4151 /* Don't bother trying to mark next as ready if insn is a debug
4152 insn. If insn is the last hard dependency, it will have
4153 already been discounted. */
4154 if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next))
4155 continue;
4157 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4159 int effective_cost;
4161 effective_cost = try_ready (next);
4163 if (effective_cost >= 0
4164 && SCHED_GROUP_P (next)
4165 && advance < effective_cost)
4166 advance = effective_cost;
4168 else
4169 /* Check always has only one forward dependence (to the first insn in
4170 the recovery block), therefore, this will be executed only once. */
4172 gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4173 fix_recovery_deps (RECOVERY_BLOCK (insn));
4177 /* Annotate the instruction with issue information -- TImode
4178 indicates that the instruction is expected not to be able
4179 to issue on the same cycle as the previous insn. A machine
4180 may use this information to decide how the instruction should
4181 be aligned. */
4182 if (issue_rate > 1
4183 && GET_CODE (PATTERN (insn)) != USE
4184 && GET_CODE (PATTERN (insn)) != CLOBBER
4185 && !DEBUG_INSN_P (insn))
4187 if (reload_completed)
4188 PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode);
4189 last_clock_var = clock_var;
4192 if (nonscheduled_insns_begin != NULL_RTX)
4193 /* Indicate to debug counters that INSN is scheduled. */
4194 nonscheduled_insns_begin = insn;
4196 return advance;
4199 /* Functions for handling of notes. */
4201 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4202 void
4203 concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp)
4205 rtx_insn *from_start;
4207 /* It's easy when have nothing to concat. */
4208 if (from_end == NULL)
4209 return;
4211 /* It's also easy when destination is empty. */
4212 if (*to_endp == NULL)
4214 *to_endp = from_end;
4215 return;
4218 from_start = from_end;
4219 while (PREV_INSN (from_start) != NULL)
4220 from_start = PREV_INSN (from_start);
4222 SET_PREV_INSN (from_start) = *to_endp;
4223 SET_NEXT_INSN (*to_endp) = from_start;
4224 *to_endp = from_end;
4227 /* Delete notes between HEAD and TAIL and put them in the chain
4228 of notes ended by NOTE_LIST. */
4229 void
4230 remove_notes (rtx_insn *head, rtx_insn *tail)
4232 rtx_insn *next_tail, *insn, *next;
4234 note_list = 0;
4235 if (head == tail && !INSN_P (head))
4236 return;
4238 next_tail = NEXT_INSN (tail);
4239 for (insn = head; insn != next_tail; insn = next)
4241 next = NEXT_INSN (insn);
4242 if (!NOTE_P (insn))
4243 continue;
4245 switch (NOTE_KIND (insn))
4247 case NOTE_INSN_BASIC_BLOCK:
4248 continue;
4250 case NOTE_INSN_EPILOGUE_BEG:
4251 if (insn != tail)
4253 remove_insn (insn);
4254 add_reg_note (next, REG_SAVE_NOTE,
4255 GEN_INT (NOTE_INSN_EPILOGUE_BEG));
4256 break;
4258 /* FALLTHRU */
4260 default:
4261 remove_insn (insn);
4263 /* Add the note to list that ends at NOTE_LIST. */
4264 SET_PREV_INSN (insn) = note_list;
4265 SET_NEXT_INSN (insn) = NULL_RTX;
4266 if (note_list)
4267 SET_NEXT_INSN (note_list) = insn;
4268 note_list = insn;
4269 break;
4272 gcc_assert ((sel_sched_p () || insn != tail) && insn != head);
4276 /* A structure to record enough data to allow us to backtrack the scheduler to
4277 a previous state. */
4278 struct haifa_saved_data
4280 /* Next entry on the list. */
4281 struct haifa_saved_data *next;
4283 /* Backtracking is associated with scheduling insns that have delay slots.
4284 DELAY_PAIR points to the structure that contains the insns involved, and
4285 the number of cycles between them. */
4286 struct delay_pair *delay_pair;
4288 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4289 void *fe_saved_data;
4290 /* Data used by the backend. */
4291 void *be_saved_data;
4293 /* Copies of global state. */
4294 int clock_var, last_clock_var;
4295 struct ready_list ready;
4296 state_t curr_state;
4298 rtx_insn *last_scheduled_insn;
4299 rtx_insn *last_nondebug_scheduled_insn;
4300 rtx_insn *nonscheduled_insns_begin;
4301 int cycle_issued_insns;
4303 /* Copies of state used in the inner loop of schedule_block. */
4304 struct sched_block_state sched_block;
4306 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4307 to 0 when restoring. */
4308 int q_size;
4309 rtx_insn_list **insn_queue;
4311 /* Describe pattern replacements that occurred since this backtrack point
4312 was queued. */
4313 vec<dep_t> replacement_deps;
4314 vec<int> replace_apply;
4316 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4317 point. */
4318 vec<dep_t> next_cycle_deps;
4319 vec<int> next_cycle_apply;
4322 /* A record, in reverse order, of all scheduled insns which have delay slots
4323 and may require backtracking. */
4324 static struct haifa_saved_data *backtrack_queue;
4326 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4327 to SET_P. */
4328 static void
4329 mark_backtrack_feeds (rtx_insn *insn, int set_p)
4331 sd_iterator_def sd_it;
4332 dep_t dep;
4333 FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep)
4335 FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p;
4339 /* Save the current scheduler state so that we can backtrack to it
4340 later if necessary. PAIR gives the insns that make it necessary to
4341 save this point. SCHED_BLOCK is the local state of schedule_block
4342 that need to be saved. */
4343 static void
4344 save_backtrack_point (struct delay_pair *pair,
4345 struct sched_block_state sched_block)
4347 int i;
4348 struct haifa_saved_data *save = XNEW (struct haifa_saved_data);
4350 save->curr_state = xmalloc (dfa_state_size);
4351 memcpy (save->curr_state, curr_state, dfa_state_size);
4353 save->ready.first = ready.first;
4354 save->ready.n_ready = ready.n_ready;
4355 save->ready.n_debug = ready.n_debug;
4356 save->ready.veclen = ready.veclen;
4357 save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen);
4358 memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx));
4360 save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1);
4361 save->q_size = q_size;
4362 for (i = 0; i <= max_insn_queue_index; i++)
4364 int q = NEXT_Q_AFTER (q_ptr, i);
4365 save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]);
4368 save->clock_var = clock_var;
4369 save->last_clock_var = last_clock_var;
4370 save->cycle_issued_insns = cycle_issued_insns;
4371 save->last_scheduled_insn = last_scheduled_insn;
4372 save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn;
4373 save->nonscheduled_insns_begin = nonscheduled_insns_begin;
4375 save->sched_block = sched_block;
4377 save->replacement_deps.create (0);
4378 save->replace_apply.create (0);
4379 save->next_cycle_deps = next_cycle_replace_deps.copy ();
4380 save->next_cycle_apply = next_cycle_apply.copy ();
4382 if (current_sched_info->save_state)
4383 save->fe_saved_data = (*current_sched_info->save_state) ();
4385 if (targetm.sched.alloc_sched_context)
4387 save->be_saved_data = targetm.sched.alloc_sched_context ();
4388 targetm.sched.init_sched_context (save->be_saved_data, false);
4390 else
4391 save->be_saved_data = NULL;
4393 save->delay_pair = pair;
4395 save->next = backtrack_queue;
4396 backtrack_queue = save;
4398 while (pair)
4400 mark_backtrack_feeds (pair->i2, 1);
4401 INSN_TICK (pair->i2) = INVALID_TICK;
4402 INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair);
4403 SHADOW_P (pair->i2) = pair->stages == 0;
4404 pair = pair->next_same_i1;
4408 /* Walk the ready list and all queues. If any insns have unresolved backwards
4409 dependencies, these must be cancelled deps, broken by predication. Set or
4410 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4412 static void
4413 toggle_cancelled_flags (bool set)
4415 int i;
4416 sd_iterator_def sd_it;
4417 dep_t dep;
4419 if (ready.n_ready > 0)
4421 rtx_insn **first = ready_lastpos (&ready);
4422 for (i = 0; i < ready.n_ready; i++)
4423 FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep)
4424 if (!DEBUG_INSN_P (DEP_PRO (dep)))
4426 if (set)
4427 DEP_STATUS (dep) |= DEP_CANCELLED;
4428 else
4429 DEP_STATUS (dep) &= ~DEP_CANCELLED;
4432 for (i = 0; i <= max_insn_queue_index; i++)
4434 int q = NEXT_Q_AFTER (q_ptr, i);
4435 rtx_insn_list *link;
4436 for (link = insn_queue[q]; link; link = link->next ())
4438 rtx_insn *insn = link->insn ();
4439 FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4440 if (!DEBUG_INSN_P (DEP_PRO (dep)))
4442 if (set)
4443 DEP_STATUS (dep) |= DEP_CANCELLED;
4444 else
4445 DEP_STATUS (dep) &= ~DEP_CANCELLED;
4451 /* Undo the replacements that have occurred after backtrack point SAVE
4452 was placed. */
4453 static void
4454 undo_replacements_for_backtrack (struct haifa_saved_data *save)
4456 while (!save->replacement_deps.is_empty ())
4458 dep_t dep = save->replacement_deps.pop ();
4459 int apply_p = save->replace_apply.pop ();
4461 if (apply_p)
4462 restore_pattern (dep, true);
4463 else
4464 apply_replacement (dep, true);
4466 save->replacement_deps.release ();
4467 save->replace_apply.release ();
4470 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4471 Restore their dependencies to an unresolved state, and mark them as
4472 queued nowhere. */
4474 static void
4475 unschedule_insns_until (rtx_insn *insn)
4477 auto_vec<rtx_insn *> recompute_vec;
4479 /* Make two passes over the insns to be unscheduled. First, we clear out
4480 dependencies and other trivial bookkeeping. */
4481 for (;;)
4483 rtx_insn *last;
4484 sd_iterator_def sd_it;
4485 dep_t dep;
4487 last = scheduled_insns.pop ();
4489 /* This will be changed by restore_backtrack_point if the insn is in
4490 any queue. */
4491 QUEUE_INDEX (last) = QUEUE_NOWHERE;
4492 if (last != insn)
4493 INSN_TICK (last) = INVALID_TICK;
4495 if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid)
4496 modulo_insns_scheduled--;
4498 for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW);
4499 sd_iterator_cond (&sd_it, &dep);)
4501 rtx_insn *con = DEP_CON (dep);
4502 sd_unresolve_dep (sd_it);
4503 if (!MUST_RECOMPUTE_SPEC_P (con))
4505 MUST_RECOMPUTE_SPEC_P (con) = 1;
4506 recompute_vec.safe_push (con);
4510 if (last == insn)
4511 break;
4514 /* A second pass, to update ready and speculation status for insns
4515 depending on the unscheduled ones. The first pass must have
4516 popped the scheduled_insns vector up to the point where we
4517 restart scheduling, as recompute_todo_spec requires it to be
4518 up-to-date. */
4519 while (!recompute_vec.is_empty ())
4521 rtx_insn *con;
4523 con = recompute_vec.pop ();
4524 MUST_RECOMPUTE_SPEC_P (con) = 0;
4525 if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK))
4527 TODO_SPEC (con) = HARD_DEP;
4528 INSN_TICK (con) = INVALID_TICK;
4529 if (PREDICATED_PAT (con) != NULL_RTX)
4530 haifa_change_pattern (con, ORIG_PAT (con));
4532 else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED)
4533 TODO_SPEC (con) = recompute_todo_spec (con, true);
4537 /* Restore scheduler state from the topmost entry on the backtracking queue.
4538 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4539 overwrite with the saved data.
4540 The caller must already have called unschedule_insns_until. */
4542 static void
4543 restore_last_backtrack_point (struct sched_block_state *psched_block)
4545 int i;
4546 struct haifa_saved_data *save = backtrack_queue;
4548 backtrack_queue = save->next;
4550 if (current_sched_info->restore_state)
4551 (*current_sched_info->restore_state) (save->fe_saved_data);
4553 if (targetm.sched.alloc_sched_context)
4555 targetm.sched.set_sched_context (save->be_saved_data);
4556 targetm.sched.free_sched_context (save->be_saved_data);
4559 /* Do this first since it clobbers INSN_TICK of the involved
4560 instructions. */
4561 undo_replacements_for_backtrack (save);
4563 /* Clear the QUEUE_INDEX of everything in the ready list or one
4564 of the queues. */
4565 if (ready.n_ready > 0)
4567 rtx_insn **first = ready_lastpos (&ready);
4568 for (i = 0; i < ready.n_ready; i++)
4570 rtx_insn *insn = first[i];
4571 QUEUE_INDEX (insn) = QUEUE_NOWHERE;
4572 INSN_TICK (insn) = INVALID_TICK;
4575 for (i = 0; i <= max_insn_queue_index; i++)
4577 int q = NEXT_Q_AFTER (q_ptr, i);
4579 for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4581 rtx_insn *x = link->insn ();
4582 QUEUE_INDEX (x) = QUEUE_NOWHERE;
4583 INSN_TICK (x) = INVALID_TICK;
4585 free_INSN_LIST_list (&insn_queue[q]);
4588 free (ready.vec);
4589 ready = save->ready;
4591 if (ready.n_ready > 0)
4593 rtx_insn **first = ready_lastpos (&ready);
4594 for (i = 0; i < ready.n_ready; i++)
4596 rtx_insn *insn = first[i];
4597 QUEUE_INDEX (insn) = QUEUE_READY;
4598 TODO_SPEC (insn) = recompute_todo_spec (insn, true);
4599 INSN_TICK (insn) = save->clock_var;
4603 q_ptr = 0;
4604 q_size = save->q_size;
4605 for (i = 0; i <= max_insn_queue_index; i++)
4607 int q = NEXT_Q_AFTER (q_ptr, i);
4609 insn_queue[q] = save->insn_queue[q];
4611 for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4613 rtx_insn *x = link->insn ();
4614 QUEUE_INDEX (x) = i;
4615 TODO_SPEC (x) = recompute_todo_spec (x, true);
4616 INSN_TICK (x) = save->clock_var + i;
4619 free (save->insn_queue);
4621 toggle_cancelled_flags (true);
4623 clock_var = save->clock_var;
4624 last_clock_var = save->last_clock_var;
4625 cycle_issued_insns = save->cycle_issued_insns;
4626 last_scheduled_insn = save->last_scheduled_insn;
4627 last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn;
4628 nonscheduled_insns_begin = save->nonscheduled_insns_begin;
4630 *psched_block = save->sched_block;
4632 memcpy (curr_state, save->curr_state, dfa_state_size);
4633 free (save->curr_state);
4635 mark_backtrack_feeds (save->delay_pair->i2, 0);
4637 gcc_assert (next_cycle_replace_deps.is_empty ());
4638 next_cycle_replace_deps = save->next_cycle_deps.copy ();
4639 next_cycle_apply = save->next_cycle_apply.copy ();
4641 free (save);
4643 for (save = backtrack_queue; save; save = save->next)
4645 mark_backtrack_feeds (save->delay_pair->i2, 1);
4649 /* Discard all data associated with the topmost entry in the backtrack
4650 queue. If RESET_TICK is false, we just want to free the data. If true,
4651 we are doing this because we discovered a reason to backtrack. In the
4652 latter case, also reset the INSN_TICK for the shadow insn. */
4653 static void
4654 free_topmost_backtrack_point (bool reset_tick)
4656 struct haifa_saved_data *save = backtrack_queue;
4657 int i;
4659 backtrack_queue = save->next;
4661 if (reset_tick)
4663 struct delay_pair *pair = save->delay_pair;
4664 while (pair)
4666 INSN_TICK (pair->i2) = INVALID_TICK;
4667 INSN_EXACT_TICK (pair->i2) = INVALID_TICK;
4668 pair = pair->next_same_i1;
4670 undo_replacements_for_backtrack (save);
4672 else
4674 save->replacement_deps.release ();
4675 save->replace_apply.release ();
4678 if (targetm.sched.free_sched_context)
4679 targetm.sched.free_sched_context (save->be_saved_data);
4680 if (current_sched_info->restore_state)
4681 free (save->fe_saved_data);
4682 for (i = 0; i <= max_insn_queue_index; i++)
4683 free_INSN_LIST_list (&save->insn_queue[i]);
4684 free (save->insn_queue);
4685 free (save->curr_state);
4686 free (save->ready.vec);
4687 free (save);
4690 /* Free the entire backtrack queue. */
4691 static void
4692 free_backtrack_queue (void)
4694 while (backtrack_queue)
4695 free_topmost_backtrack_point (false);
4698 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4699 may have to postpone the replacement until the start of the next cycle,
4700 at which point we will be called again with IMMEDIATELY true. This is
4701 only done for machines which have instruction packets with explicit
4702 parallelism however. */
4703 static void
4704 apply_replacement (dep_t dep, bool immediately)
4706 struct dep_replacement *desc = DEP_REPLACE (dep);
4707 if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4709 next_cycle_replace_deps.safe_push (dep);
4710 next_cycle_apply.safe_push (1);
4712 else
4714 bool success;
4716 if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED)
4717 return;
4719 if (sched_verbose >= 5)
4720 fprintf (sched_dump, "applying replacement for insn %d\n",
4721 INSN_UID (desc->insn));
4723 success = validate_change (desc->insn, desc->loc, desc->newval, 0);
4724 gcc_assert (success);
4726 update_insn_after_change (desc->insn);
4727 if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0)
4728 fix_tick_ready (desc->insn);
4730 if (backtrack_queue != NULL)
4732 backtrack_queue->replacement_deps.safe_push (dep);
4733 backtrack_queue->replace_apply.safe_push (1);
4738 /* We have determined that a pattern involved in DEP must be restored.
4739 If IMMEDIATELY is false, we may have to postpone the replacement
4740 until the start of the next cycle, at which point we will be called
4741 again with IMMEDIATELY true. */
4742 static void
4743 restore_pattern (dep_t dep, bool immediately)
4745 rtx_insn *next = DEP_CON (dep);
4746 int tick = INSN_TICK (next);
4748 /* If we already scheduled the insn, the modified version is
4749 correct. */
4750 if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
4751 return;
4753 if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4755 next_cycle_replace_deps.safe_push (dep);
4756 next_cycle_apply.safe_push (0);
4757 return;
4761 if (DEP_TYPE (dep) == REG_DEP_CONTROL)
4763 if (sched_verbose >= 5)
4764 fprintf (sched_dump, "restoring pattern for insn %d\n",
4765 INSN_UID (next));
4766 haifa_change_pattern (next, ORIG_PAT (next));
4768 else
4770 struct dep_replacement *desc = DEP_REPLACE (dep);
4771 bool success;
4773 if (sched_verbose >= 5)
4774 fprintf (sched_dump, "restoring pattern for insn %d\n",
4775 INSN_UID (desc->insn));
4776 tick = INSN_TICK (desc->insn);
4778 success = validate_change (desc->insn, desc->loc, desc->orig, 0);
4779 gcc_assert (success);
4780 update_insn_after_change (desc->insn);
4781 if (backtrack_queue != NULL)
4783 backtrack_queue->replacement_deps.safe_push (dep);
4784 backtrack_queue->replace_apply.safe_push (0);
4787 INSN_TICK (next) = tick;
4788 if (TODO_SPEC (next) == DEP_POSTPONED)
4789 return;
4791 if (sd_lists_empty_p (next, SD_LIST_BACK))
4792 TODO_SPEC (next) = 0;
4793 else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
4794 TODO_SPEC (next) = HARD_DEP;
4797 /* Perform pattern replacements that were queued up until the next
4798 cycle. */
4799 static void
4800 perform_replacements_new_cycle (void)
4802 int i;
4803 dep_t dep;
4804 FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep)
4806 int apply_p = next_cycle_apply[i];
4807 if (apply_p)
4808 apply_replacement (dep, true);
4809 else
4810 restore_pattern (dep, true);
4812 next_cycle_replace_deps.truncate (0);
4813 next_cycle_apply.truncate (0);
4816 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4817 instructions we've previously encountered, a set bit prevents
4818 recursion. BUDGET is a limit on how far ahead we look, it is
4819 reduced on recursive calls. Return true if we produced a good
4820 estimate, or false if we exceeded the budget. */
4821 static bool
4822 estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget)
4824 sd_iterator_def sd_it;
4825 dep_t dep;
4826 int earliest = INSN_TICK (insn);
4828 FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4830 rtx_insn *pro = DEP_PRO (dep);
4831 int t;
4833 if (DEP_STATUS (dep) & DEP_CANCELLED)
4834 continue;
4836 if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
4837 gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn));
4838 else
4840 int cost = dep_cost (dep);
4841 if (cost >= budget)
4842 return false;
4843 if (!bitmap_bit_p (processed, INSN_LUID (pro)))
4845 if (!estimate_insn_tick (processed, pro, budget - cost))
4846 return false;
4848 gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK);
4849 t = INSN_TICK_ESTIMATE (pro) + cost;
4850 if (earliest == INVALID_TICK || t > earliest)
4851 earliest = t;
4854 bitmap_set_bit (processed, INSN_LUID (insn));
4855 INSN_TICK_ESTIMATE (insn) = earliest;
4856 return true;
4859 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4860 infinite resources) the cycle in which the delayed shadow can be issued.
4861 Return the number of cycles that must pass before the real insn can be
4862 issued in order to meet this constraint. */
4863 static int
4864 estimate_shadow_tick (struct delay_pair *p)
4866 bitmap_head processed;
4867 int t;
4868 bool cutoff;
4869 bitmap_initialize (&processed, 0);
4871 cutoff = !estimate_insn_tick (&processed, p->i2,
4872 max_insn_queue_index + pair_delay (p));
4873 bitmap_clear (&processed);
4874 if (cutoff)
4875 return max_insn_queue_index;
4876 t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1);
4877 if (t > 0)
4878 return t;
4879 return 0;
4882 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4883 recursively resolve all its forward dependencies. */
4884 static void
4885 resolve_dependencies (rtx_insn *insn)
4887 sd_iterator_def sd_it;
4888 dep_t dep;
4890 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4891 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL
4892 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL)
4893 return;
4895 if (sched_verbose >= 4)
4896 fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn));
4898 if (QUEUE_INDEX (insn) >= 0)
4899 queue_remove (insn);
4901 scheduled_insns.safe_push (insn);
4903 /* Update dependent instructions. */
4904 for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4905 sd_iterator_cond (&sd_it, &dep);)
4907 rtx_insn *next = DEP_CON (dep);
4909 if (sched_verbose >= 4)
4910 fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn),
4911 INSN_UID (next));
4913 /* Resolve the dependence between INSN and NEXT.
4914 sd_resolve_dep () moves current dep to another list thus
4915 advancing the iterator. */
4916 sd_resolve_dep (sd_it);
4918 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4920 resolve_dependencies (next);
4922 else
4923 /* Check always has only one forward dependence (to the first insn in
4924 the recovery block), therefore, this will be executed only once. */
4926 gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4932 /* Return the head and tail pointers of ebb starting at BEG and ending
4933 at END. */
4934 void
4935 get_ebb_head_tail (basic_block beg, basic_block end,
4936 rtx_insn **headp, rtx_insn **tailp)
4938 rtx_insn *beg_head = BB_HEAD (beg);
4939 rtx_insn * beg_tail = BB_END (beg);
4940 rtx_insn * end_head = BB_HEAD (end);
4941 rtx_insn * end_tail = BB_END (end);
4943 /* Don't include any notes or labels at the beginning of the BEG
4944 basic block, or notes at the end of the END basic blocks. */
4946 if (LABEL_P (beg_head))
4947 beg_head = NEXT_INSN (beg_head);
4949 while (beg_head != beg_tail)
4950 if (NOTE_P (beg_head))
4951 beg_head = NEXT_INSN (beg_head);
4952 else if (DEBUG_INSN_P (beg_head))
4954 rtx_insn * note, *next;
4956 for (note = NEXT_INSN (beg_head);
4957 note != beg_tail;
4958 note = next)
4960 next = NEXT_INSN (note);
4961 if (NOTE_P (note))
4963 if (sched_verbose >= 9)
4964 fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
4966 reorder_insns_nobb (note, note, PREV_INSN (beg_head));
4968 if (BLOCK_FOR_INSN (note) != beg)
4969 df_insn_change_bb (note, beg);
4971 else if (!DEBUG_INSN_P (note))
4972 break;
4975 break;
4977 else
4978 break;
4980 *headp = beg_head;
4982 if (beg == end)
4983 end_head = beg_head;
4984 else if (LABEL_P (end_head))
4985 end_head = NEXT_INSN (end_head);
4987 while (end_head != end_tail)
4988 if (NOTE_P (end_tail))
4989 end_tail = PREV_INSN (end_tail);
4990 else if (DEBUG_INSN_P (end_tail))
4992 rtx_insn * note, *prev;
4994 for (note = PREV_INSN (end_tail);
4995 note != end_head;
4996 note = prev)
4998 prev = PREV_INSN (note);
4999 if (NOTE_P (note))
5001 if (sched_verbose >= 9)
5002 fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
5004 reorder_insns_nobb (note, note, end_tail);
5006 if (end_tail == BB_END (end))
5007 BB_END (end) = note;
5009 if (BLOCK_FOR_INSN (note) != end)
5010 df_insn_change_bb (note, end);
5012 else if (!DEBUG_INSN_P (note))
5013 break;
5016 break;
5018 else
5019 break;
5021 *tailp = end_tail;
5024 /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
5027 no_real_insns_p (const rtx_insn *head, const rtx_insn *tail)
5029 while (head != NEXT_INSN (tail))
5031 if (!NOTE_P (head) && !LABEL_P (head))
5032 return 0;
5033 head = NEXT_INSN (head);
5035 return 1;
5038 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5039 previously found among the insns. Insert them just before HEAD. */
5040 rtx_insn *
5041 restore_other_notes (rtx_insn *head, basic_block head_bb)
5043 if (note_list != 0)
5045 rtx_insn *note_head = note_list;
5047 if (head)
5048 head_bb = BLOCK_FOR_INSN (head);
5049 else
5050 head = NEXT_INSN (bb_note (head_bb));
5052 while (PREV_INSN (note_head))
5054 set_block_for_insn (note_head, head_bb);
5055 note_head = PREV_INSN (note_head);
5057 /* In the above cycle we've missed this note. */
5058 set_block_for_insn (note_head, head_bb);
5060 SET_PREV_INSN (note_head) = PREV_INSN (head);
5061 SET_NEXT_INSN (PREV_INSN (head)) = note_head;
5062 SET_PREV_INSN (head) = note_list;
5063 SET_NEXT_INSN (note_list) = head;
5065 if (BLOCK_FOR_INSN (head) != head_bb)
5066 BB_END (head_bb) = note_list;
5068 head = note_head;
5071 return head;
5074 /* When we know we are going to discard the schedule due to a failed attempt
5075 at modulo scheduling, undo all replacements. */
5076 static void
5077 undo_all_replacements (void)
5079 rtx_insn *insn;
5080 int i;
5082 FOR_EACH_VEC_ELT (scheduled_insns, i, insn)
5084 sd_iterator_def sd_it;
5085 dep_t dep;
5087 /* See if we must undo a replacement. */
5088 for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW);
5089 sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
5091 struct dep_replacement *desc = DEP_REPLACE (dep);
5092 if (desc != NULL)
5093 validate_change (desc->insn, desc->loc, desc->orig, 0);
5098 /* Return first non-scheduled insn in the current scheduling block.
5099 This is mostly used for debug-counter purposes. */
5100 static rtx_insn *
5101 first_nonscheduled_insn (void)
5103 rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX
5104 ? nonscheduled_insns_begin
5105 : current_sched_info->prev_head);
5109 insn = next_nonnote_nondebug_insn (insn);
5111 while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED);
5113 return insn;
5116 /* Move insns that became ready to fire from queue to ready list. */
5118 static void
5119 queue_to_ready (struct ready_list *ready)
5121 rtx_insn *insn;
5122 rtx_insn_list *link;
5123 rtx_insn *skip_insn;
5125 q_ptr = NEXT_Q (q_ptr);
5127 if (dbg_cnt (sched_insn) == false)
5128 /* If debug counter is activated do not requeue the first
5129 nonscheduled insn. */
5130 skip_insn = first_nonscheduled_insn ();
5131 else
5132 skip_insn = NULL;
5134 /* Add all pending insns that can be scheduled without stalls to the
5135 ready list. */
5136 for (link = insn_queue[q_ptr]; link; link = link->next ())
5138 insn = link->insn ();
5139 q_size -= 1;
5141 if (sched_verbose >= 2)
5142 fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5143 (*current_sched_info->print_insn) (insn, 0));
5145 /* If the ready list is full, delay the insn for 1 cycle.
5146 See the comment in schedule_block for the rationale. */
5147 if (!reload_completed
5148 && (ready->n_ready - ready->n_debug > MAX_SCHED_READY_INSNS
5149 || (sched_pressure == SCHED_PRESSURE_MODEL
5150 /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
5151 instructions too. */
5152 && model_index (insn) > (model_curr_point
5153 + MAX_SCHED_READY_INSNS)))
5154 && !(sched_pressure == SCHED_PRESSURE_MODEL
5155 && model_curr_point < model_num_insns
5156 /* Always allow the next model instruction to issue. */
5157 && model_index (insn) == model_curr_point)
5158 && !SCHED_GROUP_P (insn)
5159 && insn != skip_insn)
5161 if (sched_verbose >= 2)
5162 fprintf (sched_dump, "keeping in queue, ready full\n");
5163 queue_insn (insn, 1, "ready full");
5165 else
5167 ready_add (ready, insn, false);
5168 if (sched_verbose >= 2)
5169 fprintf (sched_dump, "moving to ready without stalls\n");
5172 free_INSN_LIST_list (&insn_queue[q_ptr]);
5174 /* If there are no ready insns, stall until one is ready and add all
5175 of the pending insns at that point to the ready list. */
5176 if (ready->n_ready == 0)
5178 int stalls;
5180 for (stalls = 1; stalls <= max_insn_queue_index; stalls++)
5182 if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5184 for (; link; link = link->next ())
5186 insn = link->insn ();
5187 q_size -= 1;
5189 if (sched_verbose >= 2)
5190 fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5191 (*current_sched_info->print_insn) (insn, 0));
5193 ready_add (ready, insn, false);
5194 if (sched_verbose >= 2)
5195 fprintf (sched_dump, "moving to ready with %d stalls\n", stalls);
5197 free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]);
5199 advance_one_cycle ();
5201 break;
5204 advance_one_cycle ();
5207 q_ptr = NEXT_Q_AFTER (q_ptr, stalls);
5208 clock_var += stalls;
5209 if (sched_verbose >= 2)
5210 fprintf (sched_dump, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5211 stalls, clock_var);
5215 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5216 prematurely move INSN from the queue to the ready list. Currently,
5217 if a target defines the hook 'is_costly_dependence', this function
5218 uses the hook to check whether there exist any dependences which are
5219 considered costly by the target, between INSN and other insns that
5220 have already been scheduled. Dependences are checked up to Y cycles
5221 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5222 controlling this value.
5223 (Other considerations could be taken into account instead (or in
5224 addition) depending on user flags and target hooks. */
5226 static bool
5227 ok_for_early_queue_removal (rtx_insn *insn)
5229 if (targetm.sched.is_costly_dependence)
5231 int n_cycles;
5232 int i = scheduled_insns.length ();
5233 for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
5235 while (i-- > 0)
5237 int cost;
5239 rtx_insn *prev_insn = scheduled_insns[i];
5241 if (!NOTE_P (prev_insn))
5243 dep_t dep;
5245 dep = sd_find_dep_between (prev_insn, insn, true);
5247 if (dep != NULL)
5249 cost = dep_cost (dep);
5251 if (targetm.sched.is_costly_dependence (dep, cost,
5252 flag_sched_stalled_insns_dep - n_cycles))
5253 return false;
5257 if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */
5258 break;
5261 if (i == 0)
5262 break;
5266 return true;
5270 /* Remove insns from the queue, before they become "ready" with respect
5271 to FU latency considerations. */
5273 static int
5274 early_queue_to_ready (state_t state, struct ready_list *ready)
5276 rtx_insn *insn;
5277 rtx_insn_list *link;
5278 rtx_insn_list *next_link;
5279 rtx_insn_list *prev_link;
5280 bool move_to_ready;
5281 int cost;
5282 state_t temp_state = alloca (dfa_state_size);
5283 int stalls;
5284 int insns_removed = 0;
5287 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5288 function:
5290 X == 0: There is no limit on how many queued insns can be removed
5291 prematurely. (flag_sched_stalled_insns = -1).
5293 X >= 1: Only X queued insns can be removed prematurely in each
5294 invocation. (flag_sched_stalled_insns = X).
5296 Otherwise: Early queue removal is disabled.
5297 (flag_sched_stalled_insns = 0)
5300 if (! flag_sched_stalled_insns)
5301 return 0;
5303 for (stalls = 0; stalls <= max_insn_queue_index; stalls++)
5305 if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5307 if (sched_verbose > 6)
5308 fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls);
5310 prev_link = 0;
5311 while (link)
5313 next_link = link->next ();
5314 insn = link->insn ();
5315 if (insn && sched_verbose > 6)
5316 print_rtl_single (sched_dump, insn);
5318 memcpy (temp_state, state, dfa_state_size);
5319 if (recog_memoized (insn) < 0)
5320 /* non-negative to indicate that it's not ready
5321 to avoid infinite Q->R->Q->R... */
5322 cost = 0;
5323 else
5324 cost = state_transition (temp_state, insn);
5326 if (sched_verbose >= 6)
5327 fprintf (sched_dump, "transition cost = %d\n", cost);
5329 move_to_ready = false;
5330 if (cost < 0)
5332 move_to_ready = ok_for_early_queue_removal (insn);
5333 if (move_to_ready == true)
5335 /* move from Q to R */
5336 q_size -= 1;
5337 ready_add (ready, insn, false);
5339 if (prev_link)
5340 XEXP (prev_link, 1) = next_link;
5341 else
5342 insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link;
5344 free_INSN_LIST_node (link);
5346 if (sched_verbose >= 2)
5347 fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n",
5348 (*current_sched_info->print_insn) (insn, 0));
5350 insns_removed++;
5351 if (insns_removed == flag_sched_stalled_insns)
5352 /* Remove no more than flag_sched_stalled_insns insns
5353 from Q at a time. */
5354 return insns_removed;
5358 if (move_to_ready == false)
5359 prev_link = link;
5361 link = next_link;
5362 } /* while link */
5363 } /* if link */
5365 } /* for stalls.. */
5367 return insns_removed;
5371 /* Print the ready list for debugging purposes.
5372 If READY_TRY is non-zero then only print insns that max_issue
5373 will consider. */
5374 static void
5375 debug_ready_list_1 (struct ready_list *ready, signed char *ready_try)
5377 rtx_insn **p;
5378 int i;
5380 if (ready->n_ready == 0)
5382 fprintf (sched_dump, "\n");
5383 return;
5386 p = ready_lastpos (ready);
5387 for (i = 0; i < ready->n_ready; i++)
5389 if (ready_try != NULL && ready_try[ready->n_ready - i - 1])
5390 continue;
5392 fprintf (sched_dump, " %s:%d",
5393 (*current_sched_info->print_insn) (p[i], 0),
5394 INSN_LUID (p[i]));
5395 if (sched_pressure != SCHED_PRESSURE_NONE)
5396 fprintf (sched_dump, "(cost=%d",
5397 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i]));
5398 fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i]));
5399 if (INSN_TICK (p[i]) > clock_var)
5400 fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var);
5401 if (sched_pressure == SCHED_PRESSURE_MODEL)
5402 fprintf (sched_dump, ":idx=%d",
5403 model_index (p[i]));
5404 if (sched_pressure != SCHED_PRESSURE_NONE)
5405 fprintf (sched_dump, ")");
5407 fprintf (sched_dump, "\n");
5410 /* Print the ready list. Callable from debugger. */
5411 static void
5412 debug_ready_list (struct ready_list *ready)
5414 debug_ready_list_1 (ready, NULL);
5417 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5418 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5419 replaces the epilogue note in the correct basic block. */
5420 void
5421 reemit_notes (rtx_insn *insn)
5423 rtx note;
5424 rtx_insn *last = insn;
5426 for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
5428 if (REG_NOTE_KIND (note) == REG_SAVE_NOTE)
5430 enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0));
5432 last = emit_note_before (note_type, last);
5433 remove_note (insn, note);
5438 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5439 static void
5440 move_insn (rtx_insn *insn, rtx_insn *last, rtx nt)
5442 if (PREV_INSN (insn) != last)
5444 basic_block bb;
5445 rtx_insn *note;
5446 int jump_p = 0;
5448 bb = BLOCK_FOR_INSN (insn);
5450 /* BB_HEAD is either LABEL or NOTE. */
5451 gcc_assert (BB_HEAD (bb) != insn);
5453 if (BB_END (bb) == insn)
5454 /* If this is last instruction in BB, move end marker one
5455 instruction up. */
5457 /* Jumps are always placed at the end of basic block. */
5458 jump_p = control_flow_insn_p (insn);
5460 gcc_assert (!jump_p
5461 || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS)
5462 && IS_SPECULATION_BRANCHY_CHECK_P (insn))
5463 || (common_sched_info->sched_pass_id
5464 == SCHED_EBB_PASS));
5466 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb);
5468 BB_END (bb) = PREV_INSN (insn);
5471 gcc_assert (BB_END (bb) != last);
5473 if (jump_p)
5474 /* We move the block note along with jump. */
5476 gcc_assert (nt);
5478 note = NEXT_INSN (insn);
5479 while (NOTE_NOT_BB_P (note) && note != nt)
5480 note = NEXT_INSN (note);
5482 if (note != nt
5483 && (LABEL_P (note)
5484 || BARRIER_P (note)))
5485 note = NEXT_INSN (note);
5487 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
5489 else
5490 note = insn;
5492 SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note);
5493 SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn);
5495 SET_NEXT_INSN (note) = NEXT_INSN (last);
5496 SET_PREV_INSN (NEXT_INSN (last)) = note;
5498 SET_NEXT_INSN (last) = insn;
5499 SET_PREV_INSN (insn) = last;
5501 bb = BLOCK_FOR_INSN (last);
5503 if (jump_p)
5505 fix_jump_move (insn);
5507 if (BLOCK_FOR_INSN (insn) != bb)
5508 move_block_after_check (insn);
5510 gcc_assert (BB_END (bb) == last);
5513 df_insn_change_bb (insn, bb);
5515 /* Update BB_END, if needed. */
5516 if (BB_END (bb) == last)
5517 BB_END (bb) = insn;
5520 SCHED_GROUP_P (insn) = 0;
5523 /* Return true if scheduling INSN will finish current clock cycle. */
5524 static bool
5525 insn_finishes_cycle_p (rtx_insn *insn)
5527 if (SCHED_GROUP_P (insn))
5528 /* After issuing INSN, rest of the sched_group will be forced to issue
5529 in order. Don't make any plans for the rest of cycle. */
5530 return true;
5532 /* Finishing the block will, apparently, finish the cycle. */
5533 if (current_sched_info->insn_finishes_block_p
5534 && current_sched_info->insn_finishes_block_p (insn))
5535 return true;
5537 return false;
5540 /* Functions to model cache auto-prefetcher.
5542 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5543 memory prefetches if it sees instructions with consequitive memory accesses
5544 in the instruction stream. Details of such hardware units are not published,
5545 so we can only guess what exactly is going on there.
5546 In the scheduler, we model abstract auto-prefetcher. If there are memory
5547 insns in the ready list (or the queue) that have same memory base, but
5548 different offsets, then we delay the insns with larger offsets until insns
5549 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5550 is "1", then we look at the ready list; if it is N>1, then we also look
5551 through N-1 queue entries.
5552 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5553 among its heuristics.
5554 Param value of "-1" disables modelling of the auto-prefetcher. */
5556 /* Initialize autoprefetcher model data for INSN. */
5557 static void
5558 autopref_multipass_init (const rtx_insn *insn, int write)
5560 autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
5562 gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
5563 data->base = NULL_RTX;
5564 data->offset = 0;
5565 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5566 data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5568 rtx set = single_set (insn);
5569 if (set == NULL_RTX)
5570 return;
5572 rtx mem = write ? SET_DEST (set) : SET_SRC (set);
5573 if (!MEM_P (mem))
5574 return;
5576 struct address_info info;
5577 decompose_mem_address (&info, mem);
5579 /* TODO: Currently only (base+const) addressing is supported. */
5580 if (info.base == NULL || !REG_P (*info.base)
5581 || (info.disp != NULL && !CONST_INT_P (*info.disp)))
5582 return;
5584 /* This insn is relevant for auto-prefetcher. */
5585 data->base = *info.base;
5586 data->offset = info.disp ? INTVAL (*info.disp) : 0;
5587 data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5590 /* Helper function for rank_for_schedule sorting. */
5591 static int
5592 autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
5594 for (int write = 0; write < 2; ++write)
5596 autopref_multipass_data_t data1
5597 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5598 autopref_multipass_data_t data2
5599 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5601 if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5602 autopref_multipass_init (insn1, write);
5603 if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5604 continue;
5606 if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5607 autopref_multipass_init (insn2, write);
5608 if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5609 continue;
5611 if (!rtx_equal_p (data1->base, data2->base))
5612 continue;
5614 return data1->offset - data2->offset;
5617 return 0;
5620 /* True if header of debug dump was printed. */
5621 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
5623 /* Helper for autopref_multipass_dfa_lookahead_guard.
5624 Return "1" if INSN1 should be delayed in favor of INSN2. */
5625 static int
5626 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
5627 const rtx_insn *insn2, int write)
5629 autopref_multipass_data_t data1
5630 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5631 autopref_multipass_data_t data2
5632 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5634 if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5635 autopref_multipass_init (insn2, write);
5636 if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5637 return 0;
5639 if (rtx_equal_p (data1->base, data2->base)
5640 && data1->offset > data2->offset)
5642 if (sched_verbose >= 2)
5644 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5646 fprintf (sched_dump,
5647 ";;\t\tnot trying in max_issue due to autoprefetch "
5648 "model: ");
5649 autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5652 fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2));
5655 return 1;
5658 return 0;
5661 /* General note:
5663 We could have also hooked autoprefetcher model into
5664 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5665 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5666 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5667 unblocked). We don't bother about this yet because target of interest
5668 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5670 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5671 Return "1" if INSN1 should not be considered in max_issue due to
5672 auto-prefetcher considerations. */
5674 autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index)
5676 int r = 0;
5678 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) <= 0)
5679 return 0;
5681 if (sched_verbose >= 2 && ready_index == 0)
5682 autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
5684 for (int write = 0; write < 2; ++write)
5686 autopref_multipass_data_t data1
5687 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5689 if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5690 autopref_multipass_init (insn1, write);
5691 if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5692 continue;
5694 if (ready_index == 0
5695 && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
5696 /* We allow only a single delay on priviledged instructions.
5697 Doing otherwise would cause infinite loop. */
5699 if (sched_verbose >= 2)
5701 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5703 fprintf (sched_dump,
5704 ";;\t\tnot trying in max_issue due to autoprefetch "
5705 "model: ");
5706 autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5709 fprintf (sched_dump, " *%d*", INSN_UID (insn1));
5711 continue;
5714 for (int i2 = 0; i2 < ready.n_ready; ++i2)
5716 rtx_insn *insn2 = get_ready_element (i2);
5717 if (insn1 == insn2)
5718 continue;
5719 r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
5720 if (r)
5722 if (ready_index == 0)
5724 r = -1;
5725 data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5727 goto finish;
5731 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) == 1)
5732 continue;
5734 /* Everything from the current queue slot should have been moved to
5735 the ready list. */
5736 gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
5738 int n_stalls = PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) - 1;
5739 if (n_stalls > max_insn_queue_index)
5740 n_stalls = max_insn_queue_index;
5742 for (int stalls = 1; stalls <= n_stalls; ++stalls)
5744 for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
5745 link != NULL_RTX;
5746 link = link->next ())
5748 rtx_insn *insn2 = link->insn ();
5749 r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
5750 write);
5751 if (r)
5753 /* Queue INSN1 until INSN2 can issue. */
5754 r = -stalls;
5755 if (ready_index == 0)
5756 data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5757 goto finish;
5763 finish:
5764 if (sched_verbose >= 2
5765 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5766 && (ready_index == ready.n_ready - 1 || r < 0))
5767 /* This does not /always/ trigger. We don't output EOL if the last
5768 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5769 called. We can live with this. */
5770 fprintf (sched_dump, "\n");
5772 return r;
5775 /* Define type for target data used in multipass scheduling. */
5776 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5777 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5778 #endif
5779 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t;
5781 /* The following structure describe an entry of the stack of choices. */
5782 struct choice_entry
5784 /* Ordinal number of the issued insn in the ready queue. */
5785 int index;
5786 /* The number of the rest insns whose issues we should try. */
5787 int rest;
5788 /* The number of issued essential insns. */
5789 int n;
5790 /* State after issuing the insn. */
5791 state_t state;
5792 /* Target-specific data. */
5793 first_cycle_multipass_data_t target_data;
5796 /* The following array is used to implement a stack of choices used in
5797 function max_issue. */
5798 static struct choice_entry *choice_stack;
5800 /* This holds the value of the target dfa_lookahead hook. */
5801 int dfa_lookahead;
5803 /* The following variable value is maximal number of tries of issuing
5804 insns for the first cycle multipass insn scheduling. We define
5805 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5806 need this constraint if all real insns (with non-negative codes)
5807 had reservations because in this case the algorithm complexity is
5808 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5809 might be incomplete and such insn might occur. For such
5810 descriptions, the complexity of algorithm (without the constraint)
5811 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5812 static int max_lookahead_tries;
5814 /* The following function returns maximal (or close to maximal) number
5815 of insns which can be issued on the same cycle and one of which
5816 insns is insns with the best rank (the first insn in READY). To
5817 make this function tries different samples of ready insns. READY
5818 is current queue `ready'. Global array READY_TRY reflects what
5819 insns are already issued in this try. The function stops immediately,
5820 if it reached the such a solution, that all instruction can be issued.
5821 INDEX will contain index of the best insn in READY. The following
5822 function is used only for first cycle multipass scheduling.
5824 PRIVILEGED_N >= 0
5826 This function expects recognized insns only. All USEs,
5827 CLOBBERs, etc must be filtered elsewhere. */
5829 max_issue (struct ready_list *ready, int privileged_n, state_t state,
5830 bool first_cycle_insn_p, int *index)
5832 int n, i, all, n_ready, best, delay, tries_num;
5833 int more_issue;
5834 struct choice_entry *top;
5835 rtx_insn *insn;
5837 if (sched_fusion)
5838 return 0;
5840 n_ready = ready->n_ready;
5841 gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0
5842 && privileged_n <= n_ready);
5844 /* Init MAX_LOOKAHEAD_TRIES. */
5845 if (max_lookahead_tries == 0)
5847 max_lookahead_tries = 100;
5848 for (i = 0; i < issue_rate; i++)
5849 max_lookahead_tries *= dfa_lookahead;
5852 /* Init max_points. */
5853 more_issue = issue_rate - cycle_issued_insns;
5854 gcc_assert (more_issue >= 0);
5856 /* The number of the issued insns in the best solution. */
5857 best = 0;
5859 top = choice_stack;
5861 /* Set initial state of the search. */
5862 memcpy (top->state, state, dfa_state_size);
5863 top->rest = dfa_lookahead;
5864 top->n = 0;
5865 if (targetm.sched.first_cycle_multipass_begin)
5866 targetm.sched.first_cycle_multipass_begin (&top->target_data,
5867 ready_try, n_ready,
5868 first_cycle_insn_p);
5870 /* Count the number of the insns to search among. */
5871 for (all = i = 0; i < n_ready; i++)
5872 if (!ready_try [i])
5873 all++;
5875 if (sched_verbose >= 2)
5877 fprintf (sched_dump, ";;\t\tmax_issue among %d insns:", all);
5878 debug_ready_list_1 (ready, ready_try);
5881 /* I is the index of the insn to try next. */
5882 i = 0;
5883 tries_num = 0;
5884 for (;;)
5886 if (/* If we've reached a dead end or searched enough of what we have
5887 been asked... */
5888 top->rest == 0
5889 /* or have nothing else to try... */
5890 || i >= n_ready
5891 /* or should not issue more. */
5892 || top->n >= more_issue)
5894 /* ??? (... || i == n_ready). */
5895 gcc_assert (i <= n_ready);
5897 /* We should not issue more than issue_rate instructions. */
5898 gcc_assert (top->n <= more_issue);
5900 if (top == choice_stack)
5901 break;
5903 if (best < top - choice_stack)
5905 if (privileged_n)
5907 n = privileged_n;
5908 /* Try to find issued privileged insn. */
5909 while (n && !ready_try[--n])
5913 if (/* If all insns are equally good... */
5914 privileged_n == 0
5915 /* Or a privileged insn will be issued. */
5916 || ready_try[n])
5917 /* Then we have a solution. */
5919 best = top - choice_stack;
5920 /* This is the index of the insn issued first in this
5921 solution. */
5922 *index = choice_stack [1].index;
5923 if (top->n == more_issue || best == all)
5924 break;
5928 /* Set ready-list index to point to the last insn
5929 ('i++' below will advance it to the next insn). */
5930 i = top->index;
5932 /* Backtrack. */
5933 ready_try [i] = 0;
5935 if (targetm.sched.first_cycle_multipass_backtrack)
5936 targetm.sched.first_cycle_multipass_backtrack (&top->target_data,
5937 ready_try, n_ready);
5939 top--;
5940 memcpy (state, top->state, dfa_state_size);
5942 else if (!ready_try [i])
5944 tries_num++;
5945 if (tries_num > max_lookahead_tries)
5946 break;
5947 insn = ready_element (ready, i);
5948 delay = state_transition (state, insn);
5949 if (delay < 0)
5951 if (state_dead_lock_p (state)
5952 || insn_finishes_cycle_p (insn))
5953 /* We won't issue any more instructions in the next
5954 choice_state. */
5955 top->rest = 0;
5956 else
5957 top->rest--;
5959 n = top->n;
5960 if (memcmp (top->state, state, dfa_state_size) != 0)
5961 n++;
5963 /* Advance to the next choice_entry. */
5964 top++;
5965 /* Initialize it. */
5966 top->rest = dfa_lookahead;
5967 top->index = i;
5968 top->n = n;
5969 memcpy (top->state, state, dfa_state_size);
5970 ready_try [i] = 1;
5972 if (targetm.sched.first_cycle_multipass_issue)
5973 targetm.sched.first_cycle_multipass_issue (&top->target_data,
5974 ready_try, n_ready,
5975 insn,
5976 &((top - 1)
5977 ->target_data));
5979 i = -1;
5983 /* Increase ready-list index. */
5984 i++;
5987 if (targetm.sched.first_cycle_multipass_end)
5988 targetm.sched.first_cycle_multipass_end (best != 0
5989 ? &choice_stack[1].target_data
5990 : NULL);
5992 /* Restore the original state of the DFA. */
5993 memcpy (state, choice_stack->state, dfa_state_size);
5995 return best;
5998 /* The following function chooses insn from READY and modifies
5999 READY. The following function is used only for first
6000 cycle multipass scheduling.
6001 Return:
6002 -1 if cycle should be advanced,
6003 0 if INSN_PTR is set to point to the desirable insn,
6004 1 if choose_ready () should be restarted without advancing the cycle. */
6005 static int
6006 choose_ready (struct ready_list *ready, bool first_cycle_insn_p,
6007 rtx_insn **insn_ptr)
6009 if (dbg_cnt (sched_insn) == false)
6011 if (nonscheduled_insns_begin == NULL_RTX)
6012 nonscheduled_insns_begin = current_sched_info->prev_head;
6014 rtx_insn *insn = first_nonscheduled_insn ();
6016 if (QUEUE_INDEX (insn) == QUEUE_READY)
6017 /* INSN is in the ready_list. */
6019 ready_remove_insn (insn);
6020 *insn_ptr = insn;
6021 return 0;
6024 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6025 gcc_assert (QUEUE_INDEX (insn) >= 0);
6026 return -1;
6029 if (dfa_lookahead <= 0 || SCHED_GROUP_P (ready_element (ready, 0))
6030 || DEBUG_INSN_P (ready_element (ready, 0)))
6032 if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6033 *insn_ptr = ready_remove_first_dispatch (ready);
6034 else
6035 *insn_ptr = ready_remove_first (ready);
6037 return 0;
6039 else
6041 /* Try to choose the best insn. */
6042 int index = 0, i;
6043 rtx_insn *insn;
6045 insn = ready_element (ready, 0);
6046 if (INSN_CODE (insn) < 0)
6048 *insn_ptr = ready_remove_first (ready);
6049 return 0;
6052 /* Filter the search space. */
6053 for (i = 0; i < ready->n_ready; i++)
6055 ready_try[i] = 0;
6057 insn = ready_element (ready, i);
6059 /* If this insn is recognizable we should have already
6060 recognized it earlier.
6061 ??? Not very clear where this is supposed to be done.
6062 See dep_cost_1. */
6063 gcc_checking_assert (INSN_CODE (insn) >= 0
6064 || recog_memoized (insn) < 0);
6065 if (INSN_CODE (insn) < 0)
6067 /* Non-recognized insns at position 0 are handled above. */
6068 gcc_assert (i > 0);
6069 ready_try[i] = 1;
6070 continue;
6073 if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard)
6075 ready_try[i]
6076 = (targetm.sched.first_cycle_multipass_dfa_lookahead_guard
6077 (insn, i));
6079 if (ready_try[i] < 0)
6080 /* Queue instruction for several cycles.
6081 We need to restart choose_ready as we have changed
6082 the ready list. */
6084 change_queue_index (insn, -ready_try[i]);
6085 return 1;
6088 /* Make sure that we didn't end up with 0'th insn filtered out.
6089 Don't be tempted to make life easier for backends and just
6090 requeue 0'th insn if (ready_try[0] == 0) and restart
6091 choose_ready. Backends should be very considerate about
6092 requeueing instructions -- especially the highest priority
6093 one at position 0. */
6094 gcc_assert (ready_try[i] == 0 || i > 0);
6095 if (ready_try[i])
6096 continue;
6099 gcc_assert (ready_try[i] == 0);
6100 /* INSN made it through the scrutiny of filters! */
6103 if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0)
6105 *insn_ptr = ready_remove_first (ready);
6106 if (sched_verbose >= 4)
6107 fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n",
6108 (*current_sched_info->print_insn) (*insn_ptr, 0));
6109 return 0;
6111 else
6113 if (sched_verbose >= 4)
6114 fprintf (sched_dump, ";;\t\tChosen insn : %s\n",
6115 (*current_sched_info->print_insn)
6116 (ready_element (ready, index), 0));
6118 *insn_ptr = ready_remove (ready, index);
6119 return 0;
6124 /* This function is called when we have successfully scheduled a
6125 block. It uses the schedule stored in the scheduled_insns vector
6126 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6127 append the scheduled insns; TAIL is the insn after the scheduled
6128 block. TARGET_BB is the argument passed to schedule_block. */
6130 static void
6131 commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb)
6133 unsigned int i;
6134 rtx_insn *insn;
6136 last_scheduled_insn = prev_head;
6137 for (i = 0;
6138 scheduled_insns.iterate (i, &insn);
6139 i++)
6141 if (control_flow_insn_p (last_scheduled_insn)
6142 || current_sched_info->advance_target_bb (*target_bb, insn))
6144 *target_bb = current_sched_info->advance_target_bb (*target_bb, 0);
6146 if (sched_verbose)
6148 rtx_insn *x;
6150 x = next_real_insn (last_scheduled_insn);
6151 gcc_assert (x);
6152 dump_new_block_header (1, *target_bb, x, tail);
6155 last_scheduled_insn = bb_note (*target_bb);
6158 if (current_sched_info->begin_move_insn)
6159 (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn);
6160 move_insn (insn, last_scheduled_insn,
6161 current_sched_info->next_tail);
6162 if (!DEBUG_INSN_P (insn))
6163 reemit_notes (insn);
6164 last_scheduled_insn = insn;
6167 scheduled_insns.truncate (0);
6170 /* Examine all insns on the ready list and queue those which can't be
6171 issued in this cycle. TEMP_STATE is temporary scheduler state we
6172 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6173 have been issued for the current cycle, which means it is valid to
6174 issue an asm statement.
6176 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6177 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6178 we only leave insns which have an INSN_EXACT_TICK. */
6180 static void
6181 prune_ready_list (state_t temp_state, bool first_cycle_insn_p,
6182 bool shadows_only_p, bool modulo_epilogue_p)
6184 int i, pass;
6185 bool sched_group_found = false;
6186 int min_cost_group = 1;
6188 if (sched_fusion)
6189 return;
6191 for (i = 0; i < ready.n_ready; i++)
6193 rtx_insn *insn = ready_element (&ready, i);
6194 if (SCHED_GROUP_P (insn))
6196 sched_group_found = true;
6197 break;
6201 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6202 such an insn first and note its cost, then schedule all other insns
6203 for one cycle later. */
6204 for (pass = sched_group_found ? 0 : 1; pass < 2; )
6206 int n = ready.n_ready;
6207 for (i = 0; i < n; i++)
6209 rtx_insn *insn = ready_element (&ready, i);
6210 int cost = 0;
6211 const char *reason = "resource conflict";
6213 if (DEBUG_INSN_P (insn))
6214 continue;
6216 if (sched_group_found && !SCHED_GROUP_P (insn))
6218 if (pass == 0)
6219 continue;
6220 cost = min_cost_group;
6221 reason = "not in sched group";
6223 else if (modulo_epilogue_p
6224 && INSN_EXACT_TICK (insn) == INVALID_TICK)
6226 cost = max_insn_queue_index;
6227 reason = "not an epilogue insn";
6229 else if (shadows_only_p && !SHADOW_P (insn))
6231 cost = 1;
6232 reason = "not a shadow";
6234 else if (recog_memoized (insn) < 0)
6236 if (!first_cycle_insn_p
6237 && (GET_CODE (PATTERN (insn)) == ASM_INPUT
6238 || asm_noperands (PATTERN (insn)) >= 0))
6239 cost = 1;
6240 reason = "asm";
6242 else if (sched_pressure != SCHED_PRESSURE_NONE)
6244 if (sched_pressure == SCHED_PRESSURE_MODEL
6245 && INSN_TICK (insn) <= clock_var)
6247 memcpy (temp_state, curr_state, dfa_state_size);
6248 if (state_transition (temp_state, insn) >= 0)
6249 INSN_TICK (insn) = clock_var + 1;
6251 cost = 0;
6253 else
6255 int delay_cost = 0;
6257 if (delay_htab)
6259 struct delay_pair *delay_entry;
6260 delay_entry
6261 = delay_htab->find_with_hash (insn,
6262 htab_hash_pointer (insn));
6263 while (delay_entry && delay_cost == 0)
6265 delay_cost = estimate_shadow_tick (delay_entry);
6266 if (delay_cost > max_insn_queue_index)
6267 delay_cost = max_insn_queue_index;
6268 delay_entry = delay_entry->next_same_i1;
6272 memcpy (temp_state, curr_state, dfa_state_size);
6273 cost = state_transition (temp_state, insn);
6274 if (cost < 0)
6275 cost = 0;
6276 else if (cost == 0)
6277 cost = 1;
6278 if (cost < delay_cost)
6280 cost = delay_cost;
6281 reason = "shadow tick";
6284 if (cost >= 1)
6286 if (SCHED_GROUP_P (insn) && cost > min_cost_group)
6287 min_cost_group = cost;
6288 ready_remove (&ready, i);
6289 /* Normally we'd want to queue INSN for COST cycles. However,
6290 if SCHED_GROUP_P is set, then we must ensure that nothing
6291 else comes between INSN and its predecessor. If there is
6292 some other insn ready to fire on the next cycle, then that
6293 invariant would be broken.
6295 So when SCHED_GROUP_P is set, just queue this insn for a
6296 single cycle. */
6297 queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason);
6298 if (i + 1 < n)
6299 break;
6302 if (i == n)
6303 pass++;
6307 /* Called when we detect that the schedule is impossible. We examine the
6308 backtrack queue to find the earliest insn that caused this condition. */
6310 static struct haifa_saved_data *
6311 verify_shadows (void)
6313 struct haifa_saved_data *save, *earliest_fail = NULL;
6314 for (save = backtrack_queue; save; save = save->next)
6316 int t;
6317 struct delay_pair *pair = save->delay_pair;
6318 rtx_insn *i1 = pair->i1;
6320 for (; pair; pair = pair->next_same_i1)
6322 rtx_insn *i2 = pair->i2;
6324 if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED)
6325 continue;
6327 t = INSN_TICK (i1) + pair_delay (pair);
6328 if (t < clock_var)
6330 if (sched_verbose >= 2)
6331 fprintf (sched_dump,
6332 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6333 ", not ready\n",
6334 INSN_UID (pair->i1), INSN_UID (pair->i2),
6335 INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6336 earliest_fail = save;
6337 break;
6339 if (QUEUE_INDEX (i2) >= 0)
6341 int queued_for = INSN_TICK (i2);
6343 if (t < queued_for)
6345 if (sched_verbose >= 2)
6346 fprintf (sched_dump,
6347 ";;\t\tfailed delay requirements for %d/%d"
6348 " (%d->%d), queued too late\n",
6349 INSN_UID (pair->i1), INSN_UID (pair->i2),
6350 INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6351 earliest_fail = save;
6352 break;
6358 return earliest_fail;
6361 /* Print instructions together with useful scheduling information between
6362 HEAD and TAIL (inclusive). */
6363 static void
6364 dump_insn_stream (rtx_insn *head, rtx_insn *tail)
6366 fprintf (sched_dump, ";;\t| insn | prio |\n");
6368 rtx_insn *next_tail = NEXT_INSN (tail);
6369 for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn))
6371 int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn);
6372 const char *pattern = (NOTE_P (insn)
6373 ? "note"
6374 : str_pattern_slim (PATTERN (insn)));
6376 fprintf (sched_dump, ";;\t| %4d | %4d | %-30s ",
6377 INSN_UID (insn), priority, pattern);
6379 if (sched_verbose >= 4)
6381 if (NOTE_P (insn) || recog_memoized (insn) < 0)
6382 fprintf (sched_dump, "nothing");
6383 else
6384 print_reservation (sched_dump, insn);
6386 fprintf (sched_dump, "\n");
6390 /* Use forward list scheduling to rearrange insns of block pointed to by
6391 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6392 region. */
6394 bool
6395 schedule_block (basic_block *target_bb, state_t init_state)
6397 int i;
6398 bool success = modulo_ii == 0;
6399 struct sched_block_state ls;
6400 state_t temp_state = NULL; /* It is used for multipass scheduling. */
6401 int sort_p, advance, start_clock_var;
6403 /* Head/tail info for this block. */
6404 rtx_insn *prev_head = current_sched_info->prev_head;
6405 rtx_insn *next_tail = current_sched_info->next_tail;
6406 rtx_insn *head = NEXT_INSN (prev_head);
6407 rtx_insn *tail = PREV_INSN (next_tail);
6409 if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0
6410 && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
6411 find_modifiable_mems (head, tail);
6413 /* We used to have code to avoid getting parameters moved from hard
6414 argument registers into pseudos.
6416 However, it was removed when it proved to be of marginal benefit
6417 and caused problems because schedule_block and compute_forward_dependences
6418 had different notions of what the "head" insn was. */
6420 gcc_assert (head != tail || INSN_P (head));
6422 haifa_recovery_bb_recently_added_p = false;
6424 backtrack_queue = NULL;
6426 /* Debug info. */
6427 if (sched_verbose)
6429 dump_new_block_header (0, *target_bb, head, tail);
6431 if (sched_verbose >= 2)
6433 dump_insn_stream (head, tail);
6434 memset (&rank_for_schedule_stats, 0,
6435 sizeof (rank_for_schedule_stats));
6439 if (init_state == NULL)
6440 state_reset (curr_state);
6441 else
6442 memcpy (curr_state, init_state, dfa_state_size);
6444 /* Clear the ready list. */
6445 ready.first = ready.veclen - 1;
6446 ready.n_ready = 0;
6447 ready.n_debug = 0;
6449 /* It is used for first cycle multipass scheduling. */
6450 temp_state = alloca (dfa_state_size);
6452 if (targetm.sched.init)
6453 targetm.sched.init (sched_dump, sched_verbose, ready.veclen);
6455 /* We start inserting insns after PREV_HEAD. */
6456 last_scheduled_insn = prev_head;
6457 last_nondebug_scheduled_insn = NULL;
6458 nonscheduled_insns_begin = NULL;
6460 gcc_assert ((NOTE_P (last_scheduled_insn)
6461 || DEBUG_INSN_P (last_scheduled_insn))
6462 && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb);
6464 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6465 queue. */
6466 q_ptr = 0;
6467 q_size = 0;
6469 insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1);
6470 memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx));
6472 /* Start just before the beginning of time. */
6473 clock_var = -1;
6475 /* We need queue and ready lists and clock_var be initialized
6476 in try_ready () (which is called through init_ready_list ()). */
6477 (*current_sched_info->init_ready_list) ();
6479 if (sched_pressure)
6480 sched_pressure_start_bb (*target_bb);
6482 /* The algorithm is O(n^2) in the number of ready insns at any given
6483 time in the worst case. Before reload we are more likely to have
6484 big lists so truncate them to a reasonable size. */
6485 if (!reload_completed
6486 && ready.n_ready - ready.n_debug > MAX_SCHED_READY_INSNS)
6488 ready_sort_debug (&ready);
6489 ready_sort_real (&ready);
6491 /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
6492 If there are debug insns, we know they're first. */
6493 for (i = MAX_SCHED_READY_INSNS + ready.n_debug; i < ready.n_ready; i++)
6494 if (!SCHED_GROUP_P (ready_element (&ready, i)))
6495 break;
6497 if (sched_verbose >= 2)
6499 fprintf (sched_dump,
6500 ";;\t\tReady list on entry: %d insns: ", ready.n_ready);
6501 debug_ready_list (&ready);
6502 fprintf (sched_dump,
6503 ";;\t\t before reload => truncated to %d insns\n", i);
6506 /* Delay all insns past it for 1 cycle. If debug counter is
6507 activated make an exception for the insn right after
6508 nonscheduled_insns_begin. */
6510 rtx_insn *skip_insn;
6512 if (dbg_cnt (sched_insn) == false)
6513 skip_insn = first_nonscheduled_insn ();
6514 else
6515 skip_insn = NULL;
6517 while (i < ready.n_ready)
6519 rtx_insn *insn;
6521 insn = ready_remove (&ready, i);
6523 if (insn != skip_insn)
6524 queue_insn (insn, 1, "list truncated");
6526 if (skip_insn)
6527 ready_add (&ready, skip_insn, true);
6531 /* Now we can restore basic block notes and maintain precise cfg. */
6532 restore_bb_notes (*target_bb);
6534 last_clock_var = -1;
6536 advance = 0;
6538 gcc_assert (scheduled_insns.length () == 0);
6539 sort_p = TRUE;
6540 must_backtrack = false;
6541 modulo_insns_scheduled = 0;
6543 ls.modulo_epilogue = false;
6544 ls.first_cycle_insn_p = true;
6546 /* Loop until all the insns in BB are scheduled. */
6547 while ((*current_sched_info->schedule_more_p) ())
6549 perform_replacements_new_cycle ();
6552 start_clock_var = clock_var;
6554 clock_var++;
6556 advance_one_cycle ();
6558 /* Add to the ready list all pending insns that can be issued now.
6559 If there are no ready insns, increment clock until one
6560 is ready and add all pending insns at that point to the ready
6561 list. */
6562 queue_to_ready (&ready);
6564 gcc_assert (ready.n_ready);
6566 if (sched_verbose >= 2)
6568 fprintf (sched_dump, ";;\t\tReady list after queue_to_ready:");
6569 debug_ready_list (&ready);
6571 advance -= clock_var - start_clock_var;
6573 while (advance > 0);
6575 if (ls.modulo_epilogue)
6577 int stage = clock_var / modulo_ii;
6578 if (stage > modulo_last_stage * 2 + 2)
6580 if (sched_verbose >= 2)
6581 fprintf (sched_dump,
6582 ";;\t\tmodulo scheduled succeeded at II %d\n",
6583 modulo_ii);
6584 success = true;
6585 goto end_schedule;
6588 else if (modulo_ii > 0)
6590 int stage = clock_var / modulo_ii;
6591 if (stage > modulo_max_stages)
6593 if (sched_verbose >= 2)
6594 fprintf (sched_dump,
6595 ";;\t\tfailing schedule due to excessive stages\n");
6596 goto end_schedule;
6598 if (modulo_n_insns == modulo_insns_scheduled
6599 && stage > modulo_last_stage)
6601 if (sched_verbose >= 2)
6602 fprintf (sched_dump,
6603 ";;\t\tfound kernel after %d stages, II %d\n",
6604 stage, modulo_ii);
6605 ls.modulo_epilogue = true;
6609 prune_ready_list (temp_state, true, false, ls.modulo_epilogue);
6610 if (ready.n_ready == 0)
6611 continue;
6612 if (must_backtrack)
6613 goto do_backtrack;
6615 ls.shadows_only_p = false;
6616 cycle_issued_insns = 0;
6617 ls.can_issue_more = issue_rate;
6618 for (;;)
6620 rtx_insn *insn;
6621 int cost;
6622 bool asm_p;
6624 if (sort_p && ready.n_ready > 0)
6626 /* Sort the ready list based on priority. This must be
6627 done every iteration through the loop, as schedule_insn
6628 may have readied additional insns that will not be
6629 sorted correctly. */
6630 ready_sort (&ready);
6632 if (sched_verbose >= 2)
6634 fprintf (sched_dump,
6635 ";;\t\tReady list after ready_sort: ");
6636 debug_ready_list (&ready);
6640 /* We don't want md sched reorder to even see debug isns, so put
6641 them out right away. */
6642 if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))
6643 && (*current_sched_info->schedule_more_p) ())
6645 while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)))
6647 rtx_insn *insn = ready_remove_first (&ready);
6648 gcc_assert (DEBUG_INSN_P (insn));
6649 (*current_sched_info->begin_schedule_ready) (insn);
6650 scheduled_insns.safe_push (insn);
6651 last_scheduled_insn = insn;
6652 advance = schedule_insn (insn);
6653 gcc_assert (advance == 0);
6654 if (ready.n_ready > 0)
6655 ready_sort (&ready);
6659 if (ls.first_cycle_insn_p && !ready.n_ready)
6660 break;
6662 resume_after_backtrack:
6663 /* Allow the target to reorder the list, typically for
6664 better instruction bundling. */
6665 if (sort_p
6666 && (ready.n_ready == 0
6667 || !SCHED_GROUP_P (ready_element (&ready, 0))))
6669 if (ls.first_cycle_insn_p && targetm.sched.reorder)
6670 ls.can_issue_more
6671 = targetm.sched.reorder (sched_dump, sched_verbose,
6672 ready_lastpos (&ready),
6673 &ready.n_ready, clock_var);
6674 else if (!ls.first_cycle_insn_p && targetm.sched.reorder2)
6675 ls.can_issue_more
6676 = targetm.sched.reorder2 (sched_dump, sched_verbose,
6677 ready.n_ready
6678 ? ready_lastpos (&ready) : NULL,
6679 &ready.n_ready, clock_var);
6682 restart_choose_ready:
6683 if (sched_verbose >= 2)
6685 fprintf (sched_dump, ";;\tReady list (t = %3d): ",
6686 clock_var);
6687 debug_ready_list (&ready);
6688 if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
6689 print_curr_reg_pressure ();
6692 if (ready.n_ready == 0
6693 && ls.can_issue_more
6694 && reload_completed)
6696 /* Allow scheduling insns directly from the queue in case
6697 there's nothing better to do (ready list is empty) but
6698 there are still vacant dispatch slots in the current cycle. */
6699 if (sched_verbose >= 6)
6700 fprintf (sched_dump,";;\t\tSecond chance\n");
6701 memcpy (temp_state, curr_state, dfa_state_size);
6702 if (early_queue_to_ready (temp_state, &ready))
6703 ready_sort (&ready);
6706 if (ready.n_ready == 0
6707 || !ls.can_issue_more
6708 || state_dead_lock_p (curr_state)
6709 || !(*current_sched_info->schedule_more_p) ())
6710 break;
6712 /* Select and remove the insn from the ready list. */
6713 if (sort_p)
6715 int res;
6717 insn = NULL;
6718 res = choose_ready (&ready, ls.first_cycle_insn_p, &insn);
6720 if (res < 0)
6721 /* Finish cycle. */
6722 break;
6723 if (res > 0)
6724 goto restart_choose_ready;
6726 gcc_assert (insn != NULL_RTX);
6728 else
6729 insn = ready_remove_first (&ready);
6731 if (sched_pressure != SCHED_PRESSURE_NONE
6732 && INSN_TICK (insn) > clock_var)
6734 ready_add (&ready, insn, true);
6735 advance = 1;
6736 break;
6739 if (targetm.sched.dfa_new_cycle
6740 && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
6741 insn, last_clock_var,
6742 clock_var, &sort_p))
6743 /* SORT_P is used by the target to override sorting
6744 of the ready list. This is needed when the target
6745 has modified its internal structures expecting that
6746 the insn will be issued next. As we need the insn
6747 to have the highest priority (so it will be returned by
6748 the ready_remove_first call above), we invoke
6749 ready_add (&ready, insn, true).
6750 But, still, there is one issue: INSN can be later
6751 discarded by scheduler's front end through
6752 current_sched_info->can_schedule_ready_p, hence, won't
6753 be issued next. */
6755 ready_add (&ready, insn, true);
6756 break;
6759 sort_p = TRUE;
6761 if (current_sched_info->can_schedule_ready_p
6762 && ! (*current_sched_info->can_schedule_ready_p) (insn))
6763 /* We normally get here only if we don't want to move
6764 insn from the split block. */
6766 TODO_SPEC (insn) = DEP_POSTPONED;
6767 goto restart_choose_ready;
6770 if (delay_htab)
6772 /* If this insn is the first part of a delay-slot pair, record a
6773 backtrack point. */
6774 struct delay_pair *delay_entry;
6775 delay_entry
6776 = delay_htab->find_with_hash (insn, htab_hash_pointer (insn));
6777 if (delay_entry)
6779 save_backtrack_point (delay_entry, ls);
6780 if (sched_verbose >= 2)
6781 fprintf (sched_dump, ";;\t\tsaving backtrack point\n");
6785 /* DECISION is made. */
6787 if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid)
6789 modulo_insns_scheduled++;
6790 modulo_last_stage = clock_var / modulo_ii;
6792 if (TODO_SPEC (insn) & SPECULATIVE)
6793 generate_recovery_code (insn);
6795 if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6796 targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW);
6798 /* Update counters, etc in the scheduler's front end. */
6799 (*current_sched_info->begin_schedule_ready) (insn);
6800 scheduled_insns.safe_push (insn);
6801 gcc_assert (NONDEBUG_INSN_P (insn));
6802 last_nondebug_scheduled_insn = last_scheduled_insn = insn;
6804 if (recog_memoized (insn) >= 0)
6806 memcpy (temp_state, curr_state, dfa_state_size);
6807 cost = state_transition (curr_state, insn);
6808 if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
6809 gcc_assert (cost < 0);
6810 if (memcmp (temp_state, curr_state, dfa_state_size) != 0)
6811 cycle_issued_insns++;
6812 asm_p = false;
6814 else
6815 asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT
6816 || asm_noperands (PATTERN (insn)) >= 0);
6818 if (targetm.sched.variable_issue)
6819 ls.can_issue_more =
6820 targetm.sched.variable_issue (sched_dump, sched_verbose,
6821 insn, ls.can_issue_more);
6822 /* A naked CLOBBER or USE generates no instruction, so do
6823 not count them against the issue rate. */
6824 else if (GET_CODE (PATTERN (insn)) != USE
6825 && GET_CODE (PATTERN (insn)) != CLOBBER)
6826 ls.can_issue_more--;
6827 advance = schedule_insn (insn);
6829 if (SHADOW_P (insn))
6830 ls.shadows_only_p = true;
6832 /* After issuing an asm insn we should start a new cycle. */
6833 if (advance == 0 && asm_p)
6834 advance = 1;
6836 if (must_backtrack)
6837 break;
6839 if (advance != 0)
6840 break;
6842 ls.first_cycle_insn_p = false;
6843 if (ready.n_ready > 0)
6844 prune_ready_list (temp_state, false, ls.shadows_only_p,
6845 ls.modulo_epilogue);
6848 do_backtrack:
6849 if (!must_backtrack)
6850 for (i = 0; i < ready.n_ready; i++)
6852 rtx_insn *insn = ready_element (&ready, i);
6853 if (INSN_EXACT_TICK (insn) == clock_var)
6855 must_backtrack = true;
6856 clock_var++;
6857 break;
6860 if (must_backtrack && modulo_ii > 0)
6862 if (modulo_backtracks_left == 0)
6863 goto end_schedule;
6864 modulo_backtracks_left--;
6866 while (must_backtrack)
6868 struct haifa_saved_data *failed;
6869 rtx_insn *failed_insn;
6871 must_backtrack = false;
6872 failed = verify_shadows ();
6873 gcc_assert (failed);
6875 failed_insn = failed->delay_pair->i1;
6876 /* Clear these queues. */
6877 perform_replacements_new_cycle ();
6878 toggle_cancelled_flags (false);
6879 unschedule_insns_until (failed_insn);
6880 while (failed != backtrack_queue)
6881 free_topmost_backtrack_point (true);
6882 restore_last_backtrack_point (&ls);
6883 if (sched_verbose >= 2)
6884 fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var);
6885 /* Delay by at least a cycle. This could cause additional
6886 backtracking. */
6887 queue_insn (failed_insn, 1, "backtracked");
6888 advance = 0;
6889 if (must_backtrack)
6890 continue;
6891 if (ready.n_ready > 0)
6892 goto resume_after_backtrack;
6893 else
6895 if (clock_var == 0 && ls.first_cycle_insn_p)
6896 goto end_schedule;
6897 advance = 1;
6898 break;
6901 ls.first_cycle_insn_p = true;
6903 if (ls.modulo_epilogue)
6904 success = true;
6905 end_schedule:
6906 if (!ls.first_cycle_insn_p || advance)
6907 advance_one_cycle ();
6908 perform_replacements_new_cycle ();
6909 if (modulo_ii > 0)
6911 /* Once again, debug insn suckiness: they can be on the ready list
6912 even if they have unresolved dependencies. To make our view
6913 of the world consistent, remove such "ready" insns. */
6914 restart_debug_insn_loop:
6915 for (i = ready.n_ready - 1; i >= 0; i--)
6917 rtx_insn *x;
6919 x = ready_element (&ready, i);
6920 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL
6921 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL)
6923 ready_remove (&ready, i);
6924 goto restart_debug_insn_loop;
6927 for (i = ready.n_ready - 1; i >= 0; i--)
6929 rtx_insn *x;
6931 x = ready_element (&ready, i);
6932 resolve_dependencies (x);
6934 for (i = 0; i <= max_insn_queue_index; i++)
6936 rtx_insn_list *link;
6937 while ((link = insn_queue[i]) != NULL)
6939 rtx_insn *x = link->insn ();
6940 insn_queue[i] = link->next ();
6941 QUEUE_INDEX (x) = QUEUE_NOWHERE;
6942 free_INSN_LIST_node (link);
6943 resolve_dependencies (x);
6948 if (!success)
6949 undo_all_replacements ();
6951 /* Debug info. */
6952 if (sched_verbose)
6954 fprintf (sched_dump, ";;\tReady list (final): ");
6955 debug_ready_list (&ready);
6958 if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty)
6959 /* Sanity check -- queue must be empty now. Meaningless if region has
6960 multiple bbs. */
6961 gcc_assert (!q_size && !ready.n_ready && !ready.n_debug);
6962 else if (modulo_ii == 0)
6964 /* We must maintain QUEUE_INDEX between blocks in region. */
6965 for (i = ready.n_ready - 1; i >= 0; i--)
6967 rtx_insn *x;
6969 x = ready_element (&ready, i);
6970 QUEUE_INDEX (x) = QUEUE_NOWHERE;
6971 TODO_SPEC (x) = HARD_DEP;
6974 if (q_size)
6975 for (i = 0; i <= max_insn_queue_index; i++)
6977 rtx_insn_list *link;
6978 for (link = insn_queue[i]; link; link = link->next ())
6980 rtx_insn *x;
6982 x = link->insn ();
6983 QUEUE_INDEX (x) = QUEUE_NOWHERE;
6984 TODO_SPEC (x) = HARD_DEP;
6986 free_INSN_LIST_list (&insn_queue[i]);
6990 if (sched_pressure == SCHED_PRESSURE_MODEL)
6991 model_end_schedule ();
6993 if (success)
6995 commit_schedule (prev_head, tail, target_bb);
6996 if (sched_verbose)
6997 fprintf (sched_dump, ";; total time = %d\n", clock_var);
6999 else
7000 last_scheduled_insn = tail;
7002 scheduled_insns.truncate (0);
7004 if (!current_sched_info->queue_must_finish_empty
7005 || haifa_recovery_bb_recently_added_p)
7007 /* INSN_TICK (minimum clock tick at which the insn becomes
7008 ready) may be not correct for the insn in the subsequent
7009 blocks of the region. We should use a correct value of
7010 `clock_var' or modify INSN_TICK. It is better to keep
7011 clock_var value equal to 0 at the start of a basic block.
7012 Therefore we modify INSN_TICK here. */
7013 fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn);
7016 if (targetm.sched.finish)
7018 targetm.sched.finish (sched_dump, sched_verbose);
7019 /* Target might have added some instructions to the scheduled block
7020 in its md_finish () hook. These new insns don't have any data
7021 initialized and to identify them we extend h_i_d so that they'll
7022 get zero luids. */
7023 sched_extend_luids ();
7026 /* Update head/tail boundaries. */
7027 head = NEXT_INSN (prev_head);
7028 tail = last_scheduled_insn;
7030 if (sched_verbose)
7032 fprintf (sched_dump, ";; new head = %d\n;; new tail = %d\n",
7033 INSN_UID (head), INSN_UID (tail));
7035 if (sched_verbose >= 2)
7037 dump_insn_stream (head, tail);
7038 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats,
7039 NULL);
7042 fprintf (sched_dump, "\n");
7045 head = restore_other_notes (head, NULL);
7047 current_sched_info->head = head;
7048 current_sched_info->tail = tail;
7050 free_backtrack_queue ();
7052 return success;
7055 /* Set_priorities: compute priority of each insn in the block. */
7058 set_priorities (rtx_insn *head, rtx_insn *tail)
7060 rtx_insn *insn;
7061 int n_insn;
7062 int sched_max_insns_priority =
7063 current_sched_info->sched_max_insns_priority;
7064 rtx_insn *prev_head;
7066 if (head == tail && ! INSN_P (head))
7067 gcc_unreachable ();
7069 n_insn = 0;
7071 prev_head = PREV_INSN (head);
7072 for (insn = tail; insn != prev_head; insn = PREV_INSN (insn))
7074 if (!INSN_P (insn))
7075 continue;
7077 n_insn++;
7078 (void) priority (insn);
7080 gcc_assert (INSN_PRIORITY_KNOWN (insn));
7082 sched_max_insns_priority = MAX (sched_max_insns_priority,
7083 INSN_PRIORITY (insn));
7086 current_sched_info->sched_max_insns_priority = sched_max_insns_priority;
7088 return n_insn;
7091 /* Set dump and sched_verbose for the desired debugging output. If no
7092 dump-file was specified, but -fsched-verbose=N (any N), print to stderr.
7093 For -fsched-verbose=N, N>=10, print everything to stderr. */
7094 void
7095 setup_sched_dump (void)
7097 sched_verbose = sched_verbose_param;
7098 if (sched_verbose_param == 0 && dump_file)
7099 sched_verbose = 1;
7100 sched_dump = ((sched_verbose_param >= 10 || !dump_file)
7101 ? stderr : dump_file);
7104 /* Allocate data for register pressure sensitive scheduling. */
7105 static void
7106 alloc_global_sched_pressure_data (void)
7108 if (sched_pressure != SCHED_PRESSURE_NONE)
7110 int i, max_regno = max_reg_num ();
7112 if (sched_dump != NULL)
7113 /* We need info about pseudos for rtl dumps about pseudo
7114 classes and costs. */
7115 regstat_init_n_sets_and_refs ();
7116 ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL);
7117 sched_regno_pressure_class
7118 = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class));
7119 for (i = 0; i < max_regno; i++)
7120 sched_regno_pressure_class[i]
7121 = (i < FIRST_PSEUDO_REGISTER
7122 ? ira_pressure_class_translate[REGNO_REG_CLASS (i)]
7123 : ira_pressure_class_translate[reg_allocno_class (i)]);
7124 curr_reg_live = BITMAP_ALLOC (NULL);
7125 if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7127 saved_reg_live = BITMAP_ALLOC (NULL);
7128 region_ref_regs = BITMAP_ALLOC (NULL);
7131 /* Calculate number of CALL_USED_REGS in register classes that
7132 we calculate register pressure for. */
7133 for (int c = 0; c < ira_pressure_classes_num; ++c)
7135 enum reg_class cl = ira_pressure_classes[c];
7137 call_used_regs_num[cl] = 0;
7139 for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
7140 if (call_used_regs[ira_class_hard_regs[cl][i]])
7141 ++call_used_regs_num[cl];
7146 /* Free data for register pressure sensitive scheduling. Also called
7147 from schedule_region when stopping sched-pressure early. */
7148 void
7149 free_global_sched_pressure_data (void)
7151 if (sched_pressure != SCHED_PRESSURE_NONE)
7153 if (regstat_n_sets_and_refs != NULL)
7154 regstat_free_n_sets_and_refs ();
7155 if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7157 BITMAP_FREE (region_ref_regs);
7158 BITMAP_FREE (saved_reg_live);
7160 BITMAP_FREE (curr_reg_live);
7161 free (sched_regno_pressure_class);
7165 /* Initialize some global state for the scheduler. This function works
7166 with the common data shared between all the schedulers. It is called
7167 from the scheduler specific initialization routine. */
7169 void
7170 sched_init (void)
7172 /* Disable speculative loads in their presence if cc0 defined. */
7173 if (HAVE_cc0)
7174 flag_schedule_speculative_load = 0;
7176 if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
7177 targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
7179 if (live_range_shrinkage_p)
7180 sched_pressure = SCHED_PRESSURE_WEIGHTED;
7181 else if (flag_sched_pressure
7182 && !reload_completed
7183 && common_sched_info->sched_pass_id == SCHED_RGN_PASS)
7184 sched_pressure = ((enum sched_pressure_algorithm)
7185 PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM));
7186 else
7187 sched_pressure = SCHED_PRESSURE_NONE;
7189 if (sched_pressure != SCHED_PRESSURE_NONE)
7190 ira_setup_eliminable_regset ();
7192 /* Initialize SPEC_INFO. */
7193 if (targetm.sched.set_sched_flags)
7195 spec_info = &spec_info_var;
7196 targetm.sched.set_sched_flags (spec_info);
7198 if (spec_info->mask != 0)
7200 spec_info->data_weakness_cutoff =
7201 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) * MAX_DEP_WEAK) / 100;
7202 spec_info->control_weakness_cutoff =
7203 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF)
7204 * REG_BR_PROB_BASE) / 100;
7206 else
7207 /* So we won't read anything accidentally. */
7208 spec_info = NULL;
7211 else
7212 /* So we won't read anything accidentally. */
7213 spec_info = 0;
7215 /* Initialize issue_rate. */
7216 if (targetm.sched.issue_rate)
7217 issue_rate = targetm.sched.issue_rate ();
7218 else
7219 issue_rate = 1;
7221 if (targetm.sched.first_cycle_multipass_dfa_lookahead
7222 /* Don't use max_issue with reg_pressure scheduling. Multipass
7223 scheduling and reg_pressure scheduling undo each other's decisions. */
7224 && sched_pressure == SCHED_PRESSURE_NONE)
7225 dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
7226 else
7227 dfa_lookahead = 0;
7229 /* Set to "0" so that we recalculate. */
7230 max_lookahead_tries = 0;
7232 if (targetm.sched.init_dfa_pre_cycle_insn)
7233 targetm.sched.init_dfa_pre_cycle_insn ();
7235 if (targetm.sched.init_dfa_post_cycle_insn)
7236 targetm.sched.init_dfa_post_cycle_insn ();
7238 dfa_start ();
7239 dfa_state_size = state_size ();
7241 init_alias_analysis ();
7243 if (!sched_no_dce)
7244 df_set_flags (DF_LR_RUN_DCE);
7245 df_note_add_problem ();
7247 /* More problems needed for interloop dep calculation in SMS. */
7248 if (common_sched_info->sched_pass_id == SCHED_SMS_PASS)
7250 df_rd_add_problem ();
7251 df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN);
7254 df_analyze ();
7256 /* Do not run DCE after reload, as this can kill nops inserted
7257 by bundling. */
7258 if (reload_completed)
7259 df_clear_flags (DF_LR_RUN_DCE);
7261 regstat_compute_calls_crossed ();
7263 if (targetm.sched.init_global)
7264 targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1);
7266 alloc_global_sched_pressure_data ();
7268 curr_state = xmalloc (dfa_state_size);
7271 static void haifa_init_only_bb (basic_block, basic_block);
7273 /* Initialize data structures specific to the Haifa scheduler. */
7274 void
7275 haifa_sched_init (void)
7277 setup_sched_dump ();
7278 sched_init ();
7280 scheduled_insns.create (0);
7282 if (spec_info != NULL)
7284 sched_deps_info->use_deps_list = 1;
7285 sched_deps_info->generate_spec_deps = 1;
7288 /* Initialize luids, dependency caches, target and h_i_d for the
7289 whole function. */
7291 bb_vec_t bbs;
7292 bbs.create (n_basic_blocks_for_fn (cfun));
7293 basic_block bb;
7295 sched_init_bbs ();
7297 FOR_EACH_BB_FN (bb, cfun)
7298 bbs.quick_push (bb);
7299 sched_init_luids (bbs);
7300 sched_deps_init (true);
7301 sched_extend_target ();
7302 haifa_init_h_i_d (bbs);
7304 bbs.release ();
7307 sched_init_only_bb = haifa_init_only_bb;
7308 sched_split_block = sched_split_block_1;
7309 sched_create_empty_bb = sched_create_empty_bb_1;
7310 haifa_recovery_bb_ever_added_p = false;
7312 nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0;
7313 before_recovery = 0;
7314 after_recovery = 0;
7316 modulo_ii = 0;
7319 /* Finish work with the data specific to the Haifa scheduler. */
7320 void
7321 haifa_sched_finish (void)
7323 sched_create_empty_bb = NULL;
7324 sched_split_block = NULL;
7325 sched_init_only_bb = NULL;
7327 if (spec_info && spec_info->dump)
7329 char c = reload_completed ? 'a' : 'b';
7331 fprintf (spec_info->dump,
7332 ";; %s:\n", current_function_name ());
7334 fprintf (spec_info->dump,
7335 ";; Procedure %cr-begin-data-spec motions == %d\n",
7336 c, nr_begin_data);
7337 fprintf (spec_info->dump,
7338 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7339 c, nr_be_in_data);
7340 fprintf (spec_info->dump,
7341 ";; Procedure %cr-begin-control-spec motions == %d\n",
7342 c, nr_begin_control);
7343 fprintf (spec_info->dump,
7344 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7345 c, nr_be_in_control);
7348 scheduled_insns.release ();
7350 /* Finalize h_i_d, dependency caches, and luids for the whole
7351 function. Target will be finalized in md_global_finish (). */
7352 sched_deps_finish ();
7353 sched_finish_luids ();
7354 current_sched_info = NULL;
7355 sched_finish ();
7358 /* Free global data used during insn scheduling. This function works with
7359 the common data shared between the schedulers. */
7361 void
7362 sched_finish (void)
7364 haifa_finish_h_i_d ();
7365 free_global_sched_pressure_data ();
7366 free (curr_state);
7368 if (targetm.sched.finish_global)
7369 targetm.sched.finish_global (sched_dump, sched_verbose);
7371 end_alias_analysis ();
7373 regstat_free_calls_crossed ();
7375 dfa_finish ();
7378 /* Free all delay_pair structures that were recorded. */
7379 void
7380 free_delay_pairs (void)
7382 if (delay_htab)
7384 delay_htab->empty ();
7385 delay_htab_i2->empty ();
7389 /* Fix INSN_TICKs of the instructions in the current block as well as
7390 INSN_TICKs of their dependents.
7391 HEAD and TAIL are the begin and the end of the current scheduled block. */
7392 static void
7393 fix_inter_tick (rtx_insn *head, rtx_insn *tail)
7395 /* Set of instructions with corrected INSN_TICK. */
7396 bitmap_head processed;
7397 /* ??? It is doubtful if we should assume that cycle advance happens on
7398 basic block boundaries. Basically insns that are unconditionally ready
7399 on the start of the block are more preferable then those which have
7400 a one cycle dependency over insn from the previous block. */
7401 int next_clock = clock_var + 1;
7403 bitmap_initialize (&processed, 0);
7405 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7406 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7407 across different blocks. */
7408 for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head))
7410 if (INSN_P (head))
7412 int tick;
7413 sd_iterator_def sd_it;
7414 dep_t dep;
7416 tick = INSN_TICK (head);
7417 gcc_assert (tick >= MIN_TICK);
7419 /* Fix INSN_TICK of instruction from just scheduled block. */
7420 if (bitmap_set_bit (&processed, INSN_LUID (head)))
7422 tick -= next_clock;
7424 if (tick < MIN_TICK)
7425 tick = MIN_TICK;
7427 INSN_TICK (head) = tick;
7430 if (DEBUG_INSN_P (head))
7431 continue;
7433 FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep)
7435 rtx_insn *next;
7437 next = DEP_CON (dep);
7438 tick = INSN_TICK (next);
7440 if (tick != INVALID_TICK
7441 /* If NEXT has its INSN_TICK calculated, fix it.
7442 If not - it will be properly calculated from
7443 scratch later in fix_tick_ready. */
7444 && bitmap_set_bit (&processed, INSN_LUID (next)))
7446 tick -= next_clock;
7448 if (tick < MIN_TICK)
7449 tick = MIN_TICK;
7451 if (tick > INTER_TICK (next))
7452 INTER_TICK (next) = tick;
7453 else
7454 tick = INTER_TICK (next);
7456 INSN_TICK (next) = tick;
7461 bitmap_clear (&processed);
7464 /* Check if NEXT is ready to be added to the ready or queue list.
7465 If "yes", add it to the proper list.
7466 Returns:
7467 -1 - is not ready yet,
7468 0 - added to the ready list,
7469 0 < N - queued for N cycles. */
7471 try_ready (rtx_insn *next)
7473 ds_t old_ts, new_ts;
7475 old_ts = TODO_SPEC (next);
7477 gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED))
7478 && (old_ts == HARD_DEP
7479 || old_ts == DEP_POSTPONED
7480 || (old_ts & SPECULATIVE)
7481 || old_ts == DEP_CONTROL));
7483 new_ts = recompute_todo_spec (next, false);
7485 if (new_ts & (HARD_DEP | DEP_POSTPONED))
7486 gcc_assert (new_ts == old_ts
7487 && QUEUE_INDEX (next) == QUEUE_NOWHERE);
7488 else if (current_sched_info->new_ready)
7489 new_ts = current_sched_info->new_ready (next, new_ts);
7491 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7492 have its original pattern or changed (speculative) one. This is due
7493 to changing ebb in region scheduling.
7494 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7495 has speculative pattern.
7497 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7498 control-speculative NEXT could have been discarded by sched-rgn.c
7499 (the same case as when discarded by can_schedule_ready_p ()). */
7501 if ((new_ts & SPECULATIVE)
7502 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7503 need to change anything. */
7504 && new_ts != old_ts)
7506 int res;
7507 rtx new_pat;
7509 gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE));
7511 res = haifa_speculate_insn (next, new_ts, &new_pat);
7513 switch (res)
7515 case -1:
7516 /* It would be nice to change DEP_STATUS of all dependences,
7517 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7518 so we won't reanalyze anything. */
7519 new_ts = HARD_DEP;
7520 break;
7522 case 0:
7523 /* We follow the rule, that every speculative insn
7524 has non-null ORIG_PAT. */
7525 if (!ORIG_PAT (next))
7526 ORIG_PAT (next) = PATTERN (next);
7527 break;
7529 case 1:
7530 if (!ORIG_PAT (next))
7531 /* If we gonna to overwrite the original pattern of insn,
7532 save it. */
7533 ORIG_PAT (next) = PATTERN (next);
7535 res = haifa_change_pattern (next, new_pat);
7536 gcc_assert (res);
7537 break;
7539 default:
7540 gcc_unreachable ();
7544 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7545 either correct (new_ts & SPECULATIVE),
7546 or we simply don't care (new_ts & HARD_DEP). */
7548 gcc_assert (!ORIG_PAT (next)
7549 || !IS_SPECULATION_BRANCHY_CHECK_P (next));
7551 TODO_SPEC (next) = new_ts;
7553 if (new_ts & (HARD_DEP | DEP_POSTPONED))
7555 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7556 control-speculative NEXT could have been discarded by sched-rgn.c
7557 (the same case as when discarded by can_schedule_ready_p ()). */
7558 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7560 change_queue_index (next, QUEUE_NOWHERE);
7562 return -1;
7564 else if (!(new_ts & BEGIN_SPEC)
7565 && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX
7566 && !IS_SPECULATION_CHECK_P (next))
7567 /* We should change pattern of every previously speculative
7568 instruction - and we determine if NEXT was speculative by using
7569 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7570 pat too, so skip them. */
7572 bool success = haifa_change_pattern (next, ORIG_PAT (next));
7573 gcc_assert (success);
7574 ORIG_PAT (next) = 0;
7577 if (sched_verbose >= 2)
7579 fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s",
7580 (*current_sched_info->print_insn) (next, 0));
7582 if (spec_info && spec_info->dump)
7584 if (new_ts & BEGIN_DATA)
7585 fprintf (spec_info->dump, "; data-spec;");
7586 if (new_ts & BEGIN_CONTROL)
7587 fprintf (spec_info->dump, "; control-spec;");
7588 if (new_ts & BE_IN_CONTROL)
7589 fprintf (spec_info->dump, "; in-control-spec;");
7591 if (TODO_SPEC (next) & DEP_CONTROL)
7592 fprintf (sched_dump, " predicated");
7593 fprintf (sched_dump, "\n");
7596 adjust_priority (next);
7598 return fix_tick_ready (next);
7601 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7602 static int
7603 fix_tick_ready (rtx_insn *next)
7605 int tick, delay;
7607 if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK))
7609 int full_p;
7610 sd_iterator_def sd_it;
7611 dep_t dep;
7613 tick = INSN_TICK (next);
7614 /* if tick is not equal to INVALID_TICK, then update
7615 INSN_TICK of NEXT with the most recent resolved dependence
7616 cost. Otherwise, recalculate from scratch. */
7617 full_p = (tick == INVALID_TICK);
7619 FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
7621 rtx_insn *pro = DEP_PRO (dep);
7622 int tick1;
7624 gcc_assert (INSN_TICK (pro) >= MIN_TICK);
7626 tick1 = INSN_TICK (pro) + dep_cost (dep);
7627 if (tick1 > tick)
7628 tick = tick1;
7630 if (!full_p)
7631 break;
7634 else
7635 tick = -1;
7637 INSN_TICK (next) = tick;
7639 delay = tick - clock_var;
7640 if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion)
7641 delay = QUEUE_READY;
7643 change_queue_index (next, delay);
7645 return delay;
7648 /* Move NEXT to the proper queue list with (DELAY >= 1),
7649 or add it to the ready list (DELAY == QUEUE_READY),
7650 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7651 static void
7652 change_queue_index (rtx_insn *next, int delay)
7654 int i = QUEUE_INDEX (next);
7656 gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index
7657 && delay != 0);
7658 gcc_assert (i != QUEUE_SCHEDULED);
7660 if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i)
7661 || (delay < 0 && delay == i))
7662 /* We have nothing to do. */
7663 return;
7665 /* Remove NEXT from wherever it is now. */
7666 if (i == QUEUE_READY)
7667 ready_remove_insn (next);
7668 else if (i >= 0)
7669 queue_remove (next);
7671 /* Add it to the proper place. */
7672 if (delay == QUEUE_READY)
7673 ready_add (readyp, next, false);
7674 else if (delay >= 1)
7675 queue_insn (next, delay, "change queue index");
7677 if (sched_verbose >= 2)
7679 fprintf (sched_dump, ";;\t\ttick updated: insn %s",
7680 (*current_sched_info->print_insn) (next, 0));
7682 if (delay == QUEUE_READY)
7683 fprintf (sched_dump, " into ready\n");
7684 else if (delay >= 1)
7685 fprintf (sched_dump, " into queue with cost=%d\n", delay);
7686 else
7687 fprintf (sched_dump, " removed from ready or queue lists\n");
7691 static int sched_ready_n_insns = -1;
7693 /* Initialize per region data structures. */
7694 void
7695 sched_extend_ready_list (int new_sched_ready_n_insns)
7697 int i;
7699 if (sched_ready_n_insns == -1)
7700 /* At the first call we need to initialize one more choice_stack
7701 entry. */
7703 i = 0;
7704 sched_ready_n_insns = 0;
7705 scheduled_insns.reserve (new_sched_ready_n_insns);
7707 else
7708 i = sched_ready_n_insns + 1;
7710 ready.veclen = new_sched_ready_n_insns + issue_rate;
7711 ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen);
7713 gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns);
7715 ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns,
7716 sched_ready_n_insns,
7717 sizeof (*ready_try));
7719 /* We allocate +1 element to save initial state in the choice_stack[0]
7720 entry. */
7721 choice_stack = XRESIZEVEC (struct choice_entry, choice_stack,
7722 new_sched_ready_n_insns + 1);
7724 for (; i <= new_sched_ready_n_insns; i++)
7726 choice_stack[i].state = xmalloc (dfa_state_size);
7728 if (targetm.sched.first_cycle_multipass_init)
7729 targetm.sched.first_cycle_multipass_init (&(choice_stack[i]
7730 .target_data));
7733 sched_ready_n_insns = new_sched_ready_n_insns;
7736 /* Free per region data structures. */
7737 void
7738 sched_finish_ready_list (void)
7740 int i;
7742 free (ready.vec);
7743 ready.vec = NULL;
7744 ready.veclen = 0;
7746 free (ready_try);
7747 ready_try = NULL;
7749 for (i = 0; i <= sched_ready_n_insns; i++)
7751 if (targetm.sched.first_cycle_multipass_fini)
7752 targetm.sched.first_cycle_multipass_fini (&(choice_stack[i]
7753 .target_data));
7755 free (choice_stack [i].state);
7757 free (choice_stack);
7758 choice_stack = NULL;
7760 sched_ready_n_insns = -1;
7763 static int
7764 haifa_luid_for_non_insn (rtx x)
7766 gcc_assert (NOTE_P (x) || LABEL_P (x));
7768 return 0;
7771 /* Generates recovery code for INSN. */
7772 static void
7773 generate_recovery_code (rtx_insn *insn)
7775 if (TODO_SPEC (insn) & BEGIN_SPEC)
7776 begin_speculative_block (insn);
7778 /* Here we have insn with no dependencies to
7779 instructions other then CHECK_SPEC ones. */
7781 if (TODO_SPEC (insn) & BE_IN_SPEC)
7782 add_to_speculative_block (insn);
7785 /* Helper function.
7786 Tries to add speculative dependencies of type FS between instructions
7787 in deps_list L and TWIN. */
7788 static void
7789 process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs)
7791 sd_iterator_def sd_it;
7792 dep_t dep;
7794 FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
7796 ds_t ds;
7797 rtx_insn *consumer;
7799 consumer = DEP_CON (dep);
7801 ds = DEP_STATUS (dep);
7803 if (/* If we want to create speculative dep. */
7805 /* And we can do that because this is a true dep. */
7806 && (ds & DEP_TYPES) == DEP_TRUE)
7808 gcc_assert (!(ds & BE_IN_SPEC));
7810 if (/* If this dep can be overcome with 'begin speculation'. */
7811 ds & BEGIN_SPEC)
7812 /* Then we have a choice: keep the dep 'begin speculative'
7813 or transform it into 'be in speculative'. */
7815 if (/* In try_ready we assert that if insn once became ready
7816 it can be removed from the ready (or queue) list only
7817 due to backend decision. Hence we can't let the
7818 probability of the speculative dep to decrease. */
7819 ds_weak (ds) <= ds_weak (fs))
7821 ds_t new_ds;
7823 new_ds = (ds & ~BEGIN_SPEC) | fs;
7825 if (/* consumer can 'be in speculative'. */
7826 sched_insn_is_legitimate_for_speculation_p (consumer,
7827 new_ds))
7828 /* Transform it to be in speculative. */
7829 ds = new_ds;
7832 else
7833 /* Mark the dep as 'be in speculative'. */
7834 ds |= fs;
7838 dep_def _new_dep, *new_dep = &_new_dep;
7840 init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds);
7841 sd_add_dep (new_dep, false);
7846 /* Generates recovery code for BEGIN speculative INSN. */
7847 static void
7848 begin_speculative_block (rtx_insn *insn)
7850 if (TODO_SPEC (insn) & BEGIN_DATA)
7851 nr_begin_data++;
7852 if (TODO_SPEC (insn) & BEGIN_CONTROL)
7853 nr_begin_control++;
7855 create_check_block_twin (insn, false);
7857 TODO_SPEC (insn) &= ~BEGIN_SPEC;
7860 static void haifa_init_insn (rtx_insn *);
7862 /* Generates recovery code for BE_IN speculative INSN. */
7863 static void
7864 add_to_speculative_block (rtx_insn *insn)
7866 ds_t ts;
7867 sd_iterator_def sd_it;
7868 dep_t dep;
7869 rtx_insn_list *twins = NULL;
7870 rtx_vec_t priorities_roots;
7872 ts = TODO_SPEC (insn);
7873 gcc_assert (!(ts & ~BE_IN_SPEC));
7875 if (ts & BE_IN_DATA)
7876 nr_be_in_data++;
7877 if (ts & BE_IN_CONTROL)
7878 nr_be_in_control++;
7880 TODO_SPEC (insn) &= ~BE_IN_SPEC;
7881 gcc_assert (!TODO_SPEC (insn));
7883 DONE_SPEC (insn) |= ts;
7885 /* First we convert all simple checks to branchy. */
7886 for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
7887 sd_iterator_cond (&sd_it, &dep);)
7889 rtx_insn *check = DEP_PRO (dep);
7891 if (IS_SPECULATION_SIMPLE_CHECK_P (check))
7893 create_check_block_twin (check, true);
7895 /* Restart search. */
7896 sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
7898 else
7899 /* Continue search. */
7900 sd_iterator_next (&sd_it);
7903 priorities_roots.create (0);
7904 clear_priorities (insn, &priorities_roots);
7906 while (1)
7908 rtx_insn *check, *twin;
7909 basic_block rec;
7911 /* Get the first backward dependency of INSN. */
7912 sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
7913 if (!sd_iterator_cond (&sd_it, &dep))
7914 /* INSN has no backward dependencies left. */
7915 break;
7917 gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == 0
7918 && (DEP_STATUS (dep) & BE_IN_SPEC) != 0
7919 && (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
7921 check = DEP_PRO (dep);
7923 gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check)
7924 && QUEUE_INDEX (check) == QUEUE_NOWHERE);
7926 rec = BLOCK_FOR_INSN (check);
7928 twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec));
7929 haifa_init_insn (twin);
7931 sd_copy_back_deps (twin, insn, true);
7933 if (sched_verbose && spec_info->dump)
7934 /* INSN_BB (insn) isn't determined for twin insns yet.
7935 So we can't use current_sched_info->print_insn. */
7936 fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
7937 INSN_UID (twin), rec->index);
7939 twins = alloc_INSN_LIST (twin, twins);
7941 /* Add dependences between TWIN and all appropriate
7942 instructions from REC. */
7943 FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep)
7945 rtx_insn *pro = DEP_PRO (dep);
7947 gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE);
7949 /* INSN might have dependencies from the instructions from
7950 several recovery blocks. At this iteration we process those
7951 producers that reside in REC. */
7952 if (BLOCK_FOR_INSN (pro) == rec)
7954 dep_def _new_dep, *new_dep = &_new_dep;
7956 init_dep (new_dep, pro, twin, REG_DEP_TRUE);
7957 sd_add_dep (new_dep, false);
7961 process_insn_forw_deps_be_in_spec (insn, twin, ts);
7963 /* Remove all dependencies between INSN and insns in REC. */
7964 for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
7965 sd_iterator_cond (&sd_it, &dep);)
7967 rtx_insn *pro = DEP_PRO (dep);
7969 if (BLOCK_FOR_INSN (pro) == rec)
7970 sd_delete_dep (sd_it);
7971 else
7972 sd_iterator_next (&sd_it);
7976 /* We couldn't have added the dependencies between INSN and TWINS earlier
7977 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
7978 while (twins)
7980 rtx_insn *twin;
7981 rtx_insn_list *next_node;
7983 twin = twins->insn ();
7986 dep_def _new_dep, *new_dep = &_new_dep;
7988 init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
7989 sd_add_dep (new_dep, false);
7992 next_node = twins->next ();
7993 free_INSN_LIST_node (twins);
7994 twins = next_node;
7997 calc_priorities (priorities_roots);
7998 priorities_roots.release ();
8001 /* Extends and fills with zeros (only the new part) array pointed to by P. */
8002 void *
8003 xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size)
8005 gcc_assert (new_nmemb >= old_nmemb);
8006 p = XRESIZEVAR (void, p, new_nmemb * size);
8007 memset (((char *) p) + old_nmemb * size, 0, (new_nmemb - old_nmemb) * size);
8008 return p;
8011 /* Helper function.
8012 Find fallthru edge from PRED. */
8013 edge
8014 find_fallthru_edge_from (basic_block pred)
8016 edge e;
8017 basic_block succ;
8019 succ = pred->next_bb;
8020 gcc_assert (succ->prev_bb == pred);
8022 if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds))
8024 e = find_fallthru_edge (pred->succs);
8026 if (e)
8028 gcc_assert (e->dest == succ);
8029 return e;
8032 else
8034 e = find_fallthru_edge (succ->preds);
8036 if (e)
8038 gcc_assert (e->src == pred);
8039 return e;
8043 return NULL;
8046 /* Extend per basic block data structures. */
8047 static void
8048 sched_extend_bb (void)
8050 /* The following is done to keep current_sched_info->next_tail non null. */
8051 rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
8052 rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end;
8053 if (NEXT_INSN (end) == 0
8054 || (!NOTE_P (insn)
8055 && !LABEL_P (insn)
8056 /* Don't emit a NOTE if it would end up before a BARRIER. */
8057 && !BARRIER_P (NEXT_INSN (end))))
8059 rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
8060 /* Make note appear outside BB. */
8061 set_block_for_insn (note, NULL);
8062 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end;
8066 /* Init per basic block data structures. */
8067 void
8068 sched_init_bbs (void)
8070 sched_extend_bb ();
8073 /* Initialize BEFORE_RECOVERY variable. */
8074 static void
8075 init_before_recovery (basic_block *before_recovery_ptr)
8077 basic_block last;
8078 edge e;
8080 last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
8081 e = find_fallthru_edge_from (last);
8083 if (e)
8085 /* We create two basic blocks:
8086 1. Single instruction block is inserted right after E->SRC
8087 and has jump to
8088 2. Empty block right before EXIT_BLOCK.
8089 Between these two blocks recovery blocks will be emitted. */
8091 basic_block single, empty;
8093 /* If the fallthrough edge to exit we've found is from the block we've
8094 created before, don't do anything more. */
8095 if (last == after_recovery)
8096 return;
8098 adding_bb_to_current_region_p = false;
8100 single = sched_create_empty_bb (last);
8101 empty = sched_create_empty_bb (single);
8103 /* Add new blocks to the root loop. */
8104 if (current_loops != NULL)
8106 add_bb_to_loop (single, (*current_loops->larray)[0]);
8107 add_bb_to_loop (empty, (*current_loops->larray)[0]);
8110 single->count = last->count;
8111 empty->count = last->count;
8112 single->frequency = last->frequency;
8113 empty->frequency = last->frequency;
8114 BB_COPY_PARTITION (single, last);
8115 BB_COPY_PARTITION (empty, last);
8117 redirect_edge_succ (e, single);
8118 make_single_succ_edge (single, empty, 0);
8119 make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun),
8120 EDGE_FALLTHRU);
8122 rtx_code_label *label = block_label (empty);
8123 rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label),
8124 BB_END (single));
8125 JUMP_LABEL (x) = label;
8126 LABEL_NUSES (label)++;
8127 haifa_init_insn (x);
8129 emit_barrier_after (x);
8131 sched_init_only_bb (empty, NULL);
8132 sched_init_only_bb (single, NULL);
8133 sched_extend_bb ();
8135 adding_bb_to_current_region_p = true;
8136 before_recovery = single;
8137 after_recovery = empty;
8139 if (before_recovery_ptr)
8140 *before_recovery_ptr = before_recovery;
8142 if (sched_verbose >= 2 && spec_info->dump)
8143 fprintf (spec_info->dump,
8144 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8145 last->index, single->index, empty->index);
8147 else
8148 before_recovery = last;
8151 /* Returns new recovery block. */
8152 basic_block
8153 sched_create_recovery_block (basic_block *before_recovery_ptr)
8155 rtx_insn *barrier;
8156 basic_block rec;
8158 haifa_recovery_bb_recently_added_p = true;
8159 haifa_recovery_bb_ever_added_p = true;
8161 init_before_recovery (before_recovery_ptr);
8163 barrier = get_last_bb_insn (before_recovery);
8164 gcc_assert (BARRIER_P (barrier));
8166 rtx_insn *label = emit_label_after (gen_label_rtx (), barrier);
8168 rec = create_basic_block (label, label, before_recovery);
8170 /* A recovery block always ends with an unconditional jump. */
8171 emit_barrier_after (BB_END (rec));
8173 if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED)
8174 BB_SET_PARTITION (rec, BB_COLD_PARTITION);
8176 if (sched_verbose && spec_info->dump)
8177 fprintf (spec_info->dump, ";;\t\tGenerated recovery block rec%d\n",
8178 rec->index);
8180 return rec;
8183 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8184 and emit necessary jumps. */
8185 void
8186 sched_create_recovery_edges (basic_block first_bb, basic_block rec,
8187 basic_block second_bb)
8189 int edge_flags;
8191 /* This is fixing of incoming edge. */
8192 /* ??? Which other flags should be specified? */
8193 if (BB_PARTITION (first_bb) != BB_PARTITION (rec))
8194 /* Partition type is the same, if it is "unpartitioned". */
8195 edge_flags = EDGE_CROSSING;
8196 else
8197 edge_flags = 0;
8199 make_edge (first_bb, rec, edge_flags);
8200 rtx_code_label *label = block_label (second_bb);
8201 rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
8202 BB_END (rec));
8203 JUMP_LABEL (jump) = label;
8204 LABEL_NUSES (label)++;
8206 if (BB_PARTITION (second_bb) != BB_PARTITION (rec))
8207 /* Partition type is the same, if it is "unpartitioned". */
8209 /* Rewritten from cfgrtl.c. */
8210 if (flag_reorder_blocks_and_partition
8211 && targetm_common.have_named_sections)
8213 /* We don't need the same note for the check because
8214 any_condjump_p (check) == true. */
8215 CROSSING_JUMP_P (jump) = 1;
8217 edge_flags = EDGE_CROSSING;
8219 else
8220 edge_flags = 0;
8222 make_single_succ_edge (rec, second_bb, edge_flags);
8223 if (dom_info_available_p (CDI_DOMINATORS))
8224 set_immediate_dominator (CDI_DOMINATORS, rec, first_bb);
8227 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8228 INSN is a simple check, that should be converted to branchy one. */
8229 static void
8230 create_check_block_twin (rtx_insn *insn, bool mutate_p)
8232 basic_block rec;
8233 rtx_insn *label, *check, *twin;
8234 rtx check_pat;
8235 ds_t fs;
8236 sd_iterator_def sd_it;
8237 dep_t dep;
8238 dep_def _new_dep, *new_dep = &_new_dep;
8239 ds_t todo_spec;
8241 gcc_assert (ORIG_PAT (insn) != NULL_RTX);
8243 if (!mutate_p)
8244 todo_spec = TODO_SPEC (insn);
8245 else
8247 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn)
8248 && (TODO_SPEC (insn) & SPECULATIVE) == 0);
8250 todo_spec = CHECK_SPEC (insn);
8253 todo_spec &= SPECULATIVE;
8255 /* Create recovery block. */
8256 if (mutate_p || targetm.sched.needs_block_p (todo_spec))
8258 rec = sched_create_recovery_block (NULL);
8259 label = BB_HEAD (rec);
8261 else
8263 rec = EXIT_BLOCK_PTR_FOR_FN (cfun);
8264 label = NULL;
8267 /* Emit CHECK. */
8268 check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec);
8270 if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8272 /* To have mem_reg alive at the beginning of second_bb,
8273 we emit check BEFORE insn, so insn after splitting
8274 insn will be at the beginning of second_bb, which will
8275 provide us with the correct life information. */
8276 check = emit_jump_insn_before (check_pat, insn);
8277 JUMP_LABEL (check) = label;
8278 LABEL_NUSES (label)++;
8280 else
8281 check = emit_insn_before (check_pat, insn);
8283 /* Extend data structures. */
8284 haifa_init_insn (check);
8286 /* CHECK is being added to current region. Extend ready list. */
8287 gcc_assert (sched_ready_n_insns != -1);
8288 sched_extend_ready_list (sched_ready_n_insns + 1);
8290 if (current_sched_info->add_remove_insn)
8291 current_sched_info->add_remove_insn (insn, 0);
8293 RECOVERY_BLOCK (check) = rec;
8295 if (sched_verbose && spec_info->dump)
8296 fprintf (spec_info->dump, ";;\t\tGenerated check insn : %s\n",
8297 (*current_sched_info->print_insn) (check, 0));
8299 gcc_assert (ORIG_PAT (insn));
8301 /* Initialize TWIN (twin is a duplicate of original instruction
8302 in the recovery block). */
8303 if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8305 sd_iterator_def sd_it;
8306 dep_t dep;
8308 FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep)
8309 if ((DEP_STATUS (dep) & DEP_OUTPUT) != 0)
8311 struct _dep _dep2, *dep2 = &_dep2;
8313 init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE);
8315 sd_add_dep (dep2, true);
8318 twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec));
8319 haifa_init_insn (twin);
8321 if (sched_verbose && spec_info->dump)
8322 /* INSN_BB (insn) isn't determined for twin insns yet.
8323 So we can't use current_sched_info->print_insn. */
8324 fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
8325 INSN_UID (twin), rec->index);
8327 else
8329 ORIG_PAT (check) = ORIG_PAT (insn);
8330 HAS_INTERNAL_DEP (check) = 1;
8331 twin = check;
8332 /* ??? We probably should change all OUTPUT dependencies to
8333 (TRUE | OUTPUT). */
8336 /* Copy all resolved back dependencies of INSN to TWIN. This will
8337 provide correct value for INSN_TICK (TWIN). */
8338 sd_copy_back_deps (twin, insn, true);
8340 if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8341 /* In case of branchy check, fix CFG. */
8343 basic_block first_bb, second_bb;
8344 rtx_insn *jump;
8346 first_bb = BLOCK_FOR_INSN (check);
8347 second_bb = sched_split_block (first_bb, check);
8349 sched_create_recovery_edges (first_bb, rec, second_bb);
8351 sched_init_only_bb (second_bb, first_bb);
8352 sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun));
8354 jump = BB_END (rec);
8355 haifa_init_insn (jump);
8358 /* Move backward dependences from INSN to CHECK and
8359 move forward dependences from INSN to TWIN. */
8361 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8362 FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8364 rtx_insn *pro = DEP_PRO (dep);
8365 ds_t ds;
8367 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8368 check --TRUE--> producer ??? or ANTI ???
8369 twin --TRUE--> producer
8370 twin --ANTI--> check
8372 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8373 check --ANTI--> producer
8374 twin --ANTI--> producer
8375 twin --ANTI--> check
8377 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8378 check ~~TRUE~~> producer
8379 twin ~~TRUE~~> producer
8380 twin --ANTI--> check */
8382 ds = DEP_STATUS (dep);
8384 if (ds & BEGIN_SPEC)
8386 gcc_assert (!mutate_p);
8387 ds &= ~BEGIN_SPEC;
8390 init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds);
8391 sd_add_dep (new_dep, false);
8393 if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8395 DEP_CON (new_dep) = twin;
8396 sd_add_dep (new_dep, false);
8400 /* Second, remove backward dependencies of INSN. */
8401 for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8402 sd_iterator_cond (&sd_it, &dep);)
8404 if ((DEP_STATUS (dep) & BEGIN_SPEC)
8405 || mutate_p)
8406 /* We can delete this dep because we overcome it with
8407 BEGIN_SPECULATION. */
8408 sd_delete_dep (sd_it);
8409 else
8410 sd_iterator_next (&sd_it);
8413 /* Future Speculations. Determine what BE_IN speculations will be like. */
8414 fs = 0;
8416 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8417 here. */
8419 gcc_assert (!DONE_SPEC (insn));
8421 if (!mutate_p)
8423 ds_t ts = TODO_SPEC (insn);
8425 DONE_SPEC (insn) = ts & BEGIN_SPEC;
8426 CHECK_SPEC (check) = ts & BEGIN_SPEC;
8428 /* Luckiness of future speculations solely depends upon initial
8429 BEGIN speculation. */
8430 if (ts & BEGIN_DATA)
8431 fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA));
8432 if (ts & BEGIN_CONTROL)
8433 fs = set_dep_weak (fs, BE_IN_CONTROL,
8434 get_dep_weak (ts, BEGIN_CONTROL));
8436 else
8437 CHECK_SPEC (check) = CHECK_SPEC (insn);
8439 /* Future speculations: call the helper. */
8440 process_insn_forw_deps_be_in_spec (insn, twin, fs);
8442 if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8444 /* Which types of dependencies should we use here is,
8445 generally, machine-dependent question... But, for now,
8446 it is not. */
8448 if (!mutate_p)
8450 init_dep (new_dep, insn, check, REG_DEP_TRUE);
8451 sd_add_dep (new_dep, false);
8453 init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8454 sd_add_dep (new_dep, false);
8456 else
8458 if (spec_info->dump)
8459 fprintf (spec_info->dump, ";;\t\tRemoved simple check : %s\n",
8460 (*current_sched_info->print_insn) (insn, 0));
8462 /* Remove all dependencies of the INSN. */
8464 sd_it = sd_iterator_start (insn, (SD_LIST_FORW
8465 | SD_LIST_BACK
8466 | SD_LIST_RES_BACK));
8467 while (sd_iterator_cond (&sd_it, &dep))
8468 sd_delete_dep (sd_it);
8471 /* If former check (INSN) already was moved to the ready (or queue)
8472 list, add new check (CHECK) there too. */
8473 if (QUEUE_INDEX (insn) != QUEUE_NOWHERE)
8474 try_ready (check);
8476 /* Remove old check from instruction stream and free its
8477 data. */
8478 sched_remove_insn (insn);
8481 init_dep (new_dep, check, twin, REG_DEP_ANTI);
8482 sd_add_dep (new_dep, false);
8484 else
8486 init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE | DEP_OUTPUT);
8487 sd_add_dep (new_dep, false);
8490 if (!mutate_p)
8491 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8492 because it'll be done later in add_to_speculative_block. */
8494 rtx_vec_t priorities_roots = rtx_vec_t ();
8496 clear_priorities (twin, &priorities_roots);
8497 calc_priorities (priorities_roots);
8498 priorities_roots.release ();
8502 /* Removes dependency between instructions in the recovery block REC
8503 and usual region instructions. It keeps inner dependences so it
8504 won't be necessary to recompute them. */
8505 static void
8506 fix_recovery_deps (basic_block rec)
8508 rtx_insn *note, *insn, *jump;
8509 rtx_insn_list *ready_list = 0;
8510 bitmap_head in_ready;
8511 rtx_insn_list *link;
8513 bitmap_initialize (&in_ready, 0);
8515 /* NOTE - a basic block note. */
8516 note = NEXT_INSN (BB_HEAD (rec));
8517 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8518 insn = BB_END (rec);
8519 gcc_assert (JUMP_P (insn));
8520 insn = PREV_INSN (insn);
8524 sd_iterator_def sd_it;
8525 dep_t dep;
8527 for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
8528 sd_iterator_cond (&sd_it, &dep);)
8530 rtx_insn *consumer = DEP_CON (dep);
8532 if (BLOCK_FOR_INSN (consumer) != rec)
8534 sd_delete_dep (sd_it);
8536 if (bitmap_set_bit (&in_ready, INSN_LUID (consumer)))
8537 ready_list = alloc_INSN_LIST (consumer, ready_list);
8539 else
8541 gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8543 sd_iterator_next (&sd_it);
8547 insn = PREV_INSN (insn);
8549 while (insn != note);
8551 bitmap_clear (&in_ready);
8553 /* Try to add instructions to the ready or queue list. */
8554 for (link = ready_list; link; link = link->next ())
8555 try_ready (link->insn ());
8556 free_INSN_LIST_list (&ready_list);
8558 /* Fixing jump's dependences. */
8559 insn = BB_HEAD (rec);
8560 jump = BB_END (rec);
8562 gcc_assert (LABEL_P (insn));
8563 insn = NEXT_INSN (insn);
8565 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn));
8566 add_jump_dependencies (insn, jump);
8569 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8570 instruction data. */
8571 static bool
8572 haifa_change_pattern (rtx_insn *insn, rtx new_pat)
8574 int t;
8576 t = validate_change (insn, &PATTERN (insn), new_pat, 0);
8577 if (!t)
8578 return false;
8580 update_insn_after_change (insn);
8581 return true;
8584 /* -1 - can't speculate,
8585 0 - for speculation with REQUEST mode it is OK to use
8586 current instruction pattern,
8587 1 - need to change pattern for *NEW_PAT to be speculative. */
8589 sched_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8591 gcc_assert (current_sched_info->flags & DO_SPECULATION
8592 && (request & SPECULATIVE)
8593 && sched_insn_is_legitimate_for_speculation_p (insn, request));
8595 if ((request & spec_info->mask) != request)
8596 return -1;
8598 if (request & BE_IN_SPEC
8599 && !(request & BEGIN_SPEC))
8600 return 0;
8602 return targetm.sched.speculate_insn (insn, request, new_pat);
8605 static int
8606 haifa_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8608 gcc_assert (sched_deps_info->generate_spec_deps
8609 && !IS_SPECULATION_CHECK_P (insn));
8611 if (HAS_INTERNAL_DEP (insn)
8612 || SCHED_GROUP_P (insn))
8613 return -1;
8615 return sched_speculate_insn (insn, request, new_pat);
8618 /* Print some information about block BB, which starts with HEAD and
8619 ends with TAIL, before scheduling it.
8620 I is zero, if scheduler is about to start with the fresh ebb. */
8621 static void
8622 dump_new_block_header (int i, basic_block bb, rtx_insn *head, rtx_insn *tail)
8624 if (!i)
8625 fprintf (sched_dump,
8626 ";; ======================================================\n");
8627 else
8628 fprintf (sched_dump,
8629 ";; =====================ADVANCING TO=====================\n");
8630 fprintf (sched_dump,
8631 ";; -- basic block %d from %d to %d -- %s reload\n",
8632 bb->index, INSN_UID (head), INSN_UID (tail),
8633 (reload_completed ? "after" : "before"));
8634 fprintf (sched_dump,
8635 ";; ======================================================\n");
8636 fprintf (sched_dump, "\n");
8639 /* Unlink basic block notes and labels and saves them, so they
8640 can be easily restored. We unlink basic block notes in EBB to
8641 provide back-compatibility with the previous code, as target backends
8642 assume, that there'll be only instructions between
8643 current_sched_info->{head and tail}. We restore these notes as soon
8644 as we can.
8645 FIRST (LAST) is the first (last) basic block in the ebb.
8646 NB: In usual case (FIRST == LAST) nothing is really done. */
8647 void
8648 unlink_bb_notes (basic_block first, basic_block last)
8650 /* We DON'T unlink basic block notes of the first block in the ebb. */
8651 if (first == last)
8652 return;
8654 bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun));
8656 /* Make a sentinel. */
8657 if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
8658 bb_header[last->next_bb->index] = 0;
8660 first = first->next_bb;
8663 rtx_insn *prev, *label, *note, *next;
8665 label = BB_HEAD (last);
8666 if (LABEL_P (label))
8667 note = NEXT_INSN (label);
8668 else
8669 note = label;
8670 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8672 prev = PREV_INSN (label);
8673 next = NEXT_INSN (note);
8674 gcc_assert (prev && next);
8676 SET_NEXT_INSN (prev) = next;
8677 SET_PREV_INSN (next) = prev;
8679 bb_header[last->index] = label;
8681 if (last == first)
8682 break;
8684 last = last->prev_bb;
8686 while (1);
8689 /* Restore basic block notes.
8690 FIRST is the first basic block in the ebb. */
8691 static void
8692 restore_bb_notes (basic_block first)
8694 if (!bb_header)
8695 return;
8697 /* We DON'T unlink basic block notes of the first block in the ebb. */
8698 first = first->next_bb;
8699 /* Remember: FIRST is actually a second basic block in the ebb. */
8701 while (first != EXIT_BLOCK_PTR_FOR_FN (cfun)
8702 && bb_header[first->index])
8704 rtx_insn *prev, *label, *note, *next;
8706 label = bb_header[first->index];
8707 prev = PREV_INSN (label);
8708 next = NEXT_INSN (prev);
8710 if (LABEL_P (label))
8711 note = NEXT_INSN (label);
8712 else
8713 note = label;
8714 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8716 bb_header[first->index] = 0;
8718 SET_NEXT_INSN (prev) = label;
8719 SET_NEXT_INSN (note) = next;
8720 SET_PREV_INSN (next) = note;
8722 first = first->next_bb;
8725 free (bb_header);
8726 bb_header = 0;
8729 /* Helper function.
8730 Fix CFG after both in- and inter-block movement of
8731 control_flow_insn_p JUMP. */
8732 static void
8733 fix_jump_move (rtx_insn *jump)
8735 basic_block bb, jump_bb, jump_bb_next;
8737 bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8738 jump_bb = BLOCK_FOR_INSN (jump);
8739 jump_bb_next = jump_bb->next_bb;
8741 gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS
8742 || IS_SPECULATION_BRANCHY_CHECK_P (jump));
8744 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next)))
8745 /* if jump_bb_next is not empty. */
8746 BB_END (jump_bb) = BB_END (jump_bb_next);
8748 if (BB_END (bb) != PREV_INSN (jump))
8749 /* Then there are instruction after jump that should be placed
8750 to jump_bb_next. */
8751 BB_END (jump_bb_next) = BB_END (bb);
8752 else
8753 /* Otherwise jump_bb_next is empty. */
8754 BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next));
8756 /* To make assertion in move_insn happy. */
8757 BB_END (bb) = PREV_INSN (jump);
8759 update_bb_for_insn (jump_bb_next);
8762 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8763 static void
8764 move_block_after_check (rtx_insn *jump)
8766 basic_block bb, jump_bb, jump_bb_next;
8767 vec<edge, va_gc> *t;
8769 bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8770 jump_bb = BLOCK_FOR_INSN (jump);
8771 jump_bb_next = jump_bb->next_bb;
8773 update_bb_for_insn (jump_bb);
8775 gcc_assert (IS_SPECULATION_CHECK_P (jump)
8776 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next)));
8778 unlink_block (jump_bb_next);
8779 link_block (jump_bb_next, bb);
8781 t = bb->succs;
8782 bb->succs = 0;
8783 move_succs (&(jump_bb->succs), bb);
8784 move_succs (&(jump_bb_next->succs), jump_bb);
8785 move_succs (&t, jump_bb_next);
8787 df_mark_solutions_dirty ();
8789 common_sched_info->fix_recovery_cfg
8790 (bb->index, jump_bb->index, jump_bb_next->index);
8793 /* Helper function for move_block_after_check.
8794 This functions attaches edge vector pointed to by SUCCSP to
8795 block TO. */
8796 static void
8797 move_succs (vec<edge, va_gc> **succsp, basic_block to)
8799 edge e;
8800 edge_iterator ei;
8802 gcc_assert (to->succs == 0);
8804 to->succs = *succsp;
8806 FOR_EACH_EDGE (e, ei, to->succs)
8807 e->src = to;
8809 *succsp = 0;
8812 /* Remove INSN from the instruction stream.
8813 INSN should have any dependencies. */
8814 static void
8815 sched_remove_insn (rtx_insn *insn)
8817 sd_finish_insn (insn);
8819 change_queue_index (insn, QUEUE_NOWHERE);
8820 current_sched_info->add_remove_insn (insn, 1);
8821 delete_insn (insn);
8824 /* Clear priorities of all instructions, that are forward dependent on INSN.
8825 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8826 be invoked to initialize all cleared priorities. */
8827 static void
8828 clear_priorities (rtx_insn *insn, rtx_vec_t *roots_ptr)
8830 sd_iterator_def sd_it;
8831 dep_t dep;
8832 bool insn_is_root_p = true;
8834 gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
8836 FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8838 rtx_insn *pro = DEP_PRO (dep);
8840 if (INSN_PRIORITY_STATUS (pro) >= 0
8841 && QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
8843 /* If DEP doesn't contribute to priority then INSN itself should
8844 be added to priority roots. */
8845 if (contributes_to_priority_p (dep))
8846 insn_is_root_p = false;
8848 INSN_PRIORITY_STATUS (pro) = -1;
8849 clear_priorities (pro, roots_ptr);
8853 if (insn_is_root_p)
8854 roots_ptr->safe_push (insn);
8857 /* Recompute priorities of instructions, whose priorities might have been
8858 changed. ROOTS is a vector of instructions whose priority computation will
8859 trigger initialization of all cleared priorities. */
8860 static void
8861 calc_priorities (rtx_vec_t roots)
8863 int i;
8864 rtx_insn *insn;
8866 FOR_EACH_VEC_ELT (roots, i, insn)
8867 priority (insn);
8871 /* Add dependences between JUMP and other instructions in the recovery
8872 block. INSN is the first insn the recovery block. */
8873 static void
8874 add_jump_dependencies (rtx_insn *insn, rtx_insn *jump)
8878 insn = NEXT_INSN (insn);
8879 if (insn == jump)
8880 break;
8882 if (dep_list_size (insn, SD_LIST_FORW) == 0)
8884 dep_def _new_dep, *new_dep = &_new_dep;
8886 init_dep (new_dep, insn, jump, REG_DEP_ANTI);
8887 sd_add_dep (new_dep, false);
8890 while (1);
8892 gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK));
8895 /* Extend data structures for logical insn UID. */
8896 void
8897 sched_extend_luids (void)
8899 int new_luids_max_uid = get_max_uid () + 1;
8901 sched_luids.safe_grow_cleared (new_luids_max_uid);
8904 /* Initialize LUID for INSN. */
8905 void
8906 sched_init_insn_luid (rtx_insn *insn)
8908 int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn);
8909 int luid;
8911 if (i >= 0)
8913 luid = sched_max_luid;
8914 sched_max_luid += i;
8916 else
8917 luid = -1;
8919 SET_INSN_LUID (insn, luid);
8922 /* Initialize luids for BBS.
8923 The hook common_sched_info->luid_for_non_insn () is used to determine
8924 if notes, labels, etc. need luids. */
8925 void
8926 sched_init_luids (bb_vec_t bbs)
8928 int i;
8929 basic_block bb;
8931 sched_extend_luids ();
8932 FOR_EACH_VEC_ELT (bbs, i, bb)
8934 rtx_insn *insn;
8936 FOR_BB_INSNS (bb, insn)
8937 sched_init_insn_luid (insn);
8941 /* Free LUIDs. */
8942 void
8943 sched_finish_luids (void)
8945 sched_luids.release ();
8946 sched_max_luid = 1;
8949 /* Return logical uid of INSN. Helpful while debugging. */
8951 insn_luid (rtx_insn *insn)
8953 return INSN_LUID (insn);
8956 /* Extend per insn data in the target. */
8957 void
8958 sched_extend_target (void)
8960 if (targetm.sched.h_i_d_extended)
8961 targetm.sched.h_i_d_extended ();
8964 /* Extend global scheduler structures (those, that live across calls to
8965 schedule_block) to include information about just emitted INSN. */
8966 static void
8967 extend_h_i_d (void)
8969 int reserve = (get_max_uid () + 1 - h_i_d.length ());
8970 if (reserve > 0
8971 && ! h_i_d.space (reserve))
8973 h_i_d.safe_grow_cleared (3 * get_max_uid () / 2);
8974 sched_extend_target ();
8978 /* Initialize h_i_d entry of the INSN with default values.
8979 Values, that are not explicitly initialized here, hold zero. */
8980 static void
8981 init_h_i_d (rtx_insn *insn)
8983 if (INSN_LUID (insn) > 0)
8985 INSN_COST (insn) = -1;
8986 QUEUE_INDEX (insn) = QUEUE_NOWHERE;
8987 INSN_TICK (insn) = INVALID_TICK;
8988 INSN_EXACT_TICK (insn) = INVALID_TICK;
8989 INTER_TICK (insn) = INVALID_TICK;
8990 TODO_SPEC (insn) = HARD_DEP;
8991 INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
8992 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
8993 INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
8994 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
8998 /* Initialize haifa_insn_data for BBS. */
8999 void
9000 haifa_init_h_i_d (bb_vec_t bbs)
9002 int i;
9003 basic_block bb;
9005 extend_h_i_d ();
9006 FOR_EACH_VEC_ELT (bbs, i, bb)
9008 rtx_insn *insn;
9010 FOR_BB_INSNS (bb, insn)
9011 init_h_i_d (insn);
9015 /* Finalize haifa_insn_data. */
9016 void
9017 haifa_finish_h_i_d (void)
9019 int i;
9020 haifa_insn_data_t data;
9021 struct reg_use_data *use, *next;
9023 FOR_EACH_VEC_ELT (h_i_d, i, data)
9025 free (data->max_reg_pressure);
9026 free (data->reg_pressure);
9027 for (use = data->reg_use_list; use != NULL; use = next)
9029 next = use->next_insn_use;
9030 free (use);
9033 h_i_d.release ();
9036 /* Init data for the new insn INSN. */
9037 static void
9038 haifa_init_insn (rtx_insn *insn)
9040 gcc_assert (insn != NULL);
9042 sched_extend_luids ();
9043 sched_init_insn_luid (insn);
9044 sched_extend_target ();
9045 sched_deps_init (false);
9046 extend_h_i_d ();
9047 init_h_i_d (insn);
9049 if (adding_bb_to_current_region_p)
9051 sd_init_insn (insn);
9053 /* Extend dependency caches by one element. */
9054 extend_dependency_caches (1, false);
9056 if (sched_pressure != SCHED_PRESSURE_NONE)
9057 init_insn_reg_pressure_info (insn);
9060 /* Init data for the new basic block BB which comes after AFTER. */
9061 static void
9062 haifa_init_only_bb (basic_block bb, basic_block after)
9064 gcc_assert (bb != NULL);
9066 sched_init_bbs ();
9068 if (common_sched_info->add_block)
9069 /* This changes only data structures of the front-end. */
9070 common_sched_info->add_block (bb, after);
9073 /* A generic version of sched_split_block (). */
9074 basic_block
9075 sched_split_block_1 (basic_block first_bb, rtx after)
9077 edge e;
9079 e = split_block (first_bb, after);
9080 gcc_assert (e->src == first_bb);
9082 /* sched_split_block emits note if *check == BB_END. Probably it
9083 is better to rip that note off. */
9085 return e->dest;
9088 /* A generic version of sched_create_empty_bb (). */
9089 basic_block
9090 sched_create_empty_bb_1 (basic_block after)
9092 return create_empty_bb (after);
9095 /* Insert PAT as an INSN into the schedule and update the necessary data
9096 structures to account for it. */
9097 rtx_insn *
9098 sched_emit_insn (rtx pat)
9100 rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ());
9101 haifa_init_insn (insn);
9103 if (current_sched_info->add_remove_insn)
9104 current_sched_info->add_remove_insn (insn, 0);
9106 (*current_sched_info->begin_schedule_ready) (insn);
9107 scheduled_insns.safe_push (insn);
9109 last_scheduled_insn = insn;
9110 return insn;
9113 /* This function returns a candidate satisfying dispatch constraints from
9114 the ready list. */
9116 static rtx_insn *
9117 ready_remove_first_dispatch (struct ready_list *ready)
9119 int i;
9120 rtx_insn *insn = ready_element (ready, 0);
9122 if (ready->n_ready == 1
9123 || !INSN_P (insn)
9124 || INSN_CODE (insn) < 0
9125 || !active_insn_p (insn)
9126 || targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9127 return ready_remove_first (ready);
9129 for (i = 1; i < ready->n_ready; i++)
9131 insn = ready_element (ready, i);
9133 if (!INSN_P (insn)
9134 || INSN_CODE (insn) < 0
9135 || !active_insn_p (insn))
9136 continue;
9138 if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9140 /* Return ith element of ready. */
9141 insn = ready_remove (ready, i);
9142 return insn;
9146 if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION))
9147 return ready_remove_first (ready);
9149 for (i = 1; i < ready->n_ready; i++)
9151 insn = ready_element (ready, i);
9153 if (!INSN_P (insn)
9154 || INSN_CODE (insn) < 0
9155 || !active_insn_p (insn))
9156 continue;
9158 /* Return i-th element of ready. */
9159 if (targetm.sched.dispatch (insn, IS_CMP))
9160 return ready_remove (ready, i);
9163 return ready_remove_first (ready);
9166 /* Get number of ready insn in the ready list. */
9169 number_in_ready (void)
9171 return ready.n_ready;
9174 /* Get number of ready's in the ready list. */
9176 rtx_insn *
9177 get_ready_element (int i)
9179 return ready_element (&ready, i);
9182 #endif /* INSN_SCHEDULING */